Skeleton: roledesk on desk and pipeline from ../news
Package layers models, config, storage, jobs, web, cli, held by tests/test_architecture.py with desk and pipeline as external trees and plugins/ limited to desk. roles.yaml is the default config, roledesk.db its own file, port 8790. create_app mounts pipeline.web.connect over Jobs(config.database); `roledesk job pull` drains a no-op pull and `roledesk db init` creates the file. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
36dbcbb29e
20 files changed
+2320
No files matched your search
@@ -0,0 +1,14 @@
|
||||
"""roledesk: a job-search engine on the `desk` plugin contract and the `pipeline` queue.
|
||||
|
||||
Layers, bottom to top, each importing only from those below it (see
|
||||
`tests/test_architecture.py`):
|
||||
|
||||
models the shapes handed round, and the key functions for clustering
|
||||
config `roles.yaml`, validated
|
||||
storage `Store`: every SQL statement against `roledesk.db`
|
||||
jobs the work this app queues
|
||||
web the FastAPI app
|
||||
cli the entry point
|
||||
|
||||
`desk` and `pipeline` come from the news repo and are imported, never edited.
|
||||
"""
|
||||
@@ -0,0 +1,84 @@
|
||||
"""`roledesk serve | job pull | db init`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from roledesk import config as config_mod
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(prog="roledesk",
|
||||
description="A job-search engine on desk.")
|
||||
parser.add_argument("-c", "--config", default=None, help="path to roles.yaml")
|
||||
parser.add_argument("-v", "--verbose", action="store_true")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
serve = sub.add_parser("serve", help="run the web app")
|
||||
serve.add_argument("--host", default="127.0.0.1")
|
||||
serve.add_argument("--port", type=int, default=8790)
|
||||
|
||||
job = sub.add_parser("job", help="queue a job and drain the queue once")
|
||||
job.add_argument("kind", choices=["pull"])
|
||||
|
||||
db = sub.add_parser("db", help="database housekeeping")
|
||||
db.add_argument("action", choices=["init"])
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
try:
|
||||
if args.command == "serve":
|
||||
from roledesk.web import serve
|
||||
config_mod.load(args.config) # a config error here, not in uvicorn
|
||||
return serve(args.host, args.port, args.config)
|
||||
if args.command == "job":
|
||||
return run_job(args)
|
||||
return init_db(args)
|
||||
except FileNotFoundError as exc:
|
||||
print(exc, file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def run_job(args: argparse.Namespace) -> int:
|
||||
"""Queue `kind`, then run everything queued until the queue is idle."""
|
||||
from pipeline import JobQueue, Jobs
|
||||
from roledesk import jobs
|
||||
from roledesk.storage import Store
|
||||
|
||||
config = config_mod.load(args.config)
|
||||
store = Store(config.database)
|
||||
queue = JobQueue(Jobs(config.database), config, store,
|
||||
workers=config.workers.model_dump(),
|
||||
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=config.path)
|
||||
queued = queue.enqueue(args.kind)
|
||||
queue.drain()
|
||||
if queued is not None:
|
||||
done = queue.jobs.get(queued.id)
|
||||
if done is not None:
|
||||
print(json.dumps({"id": done.id, "state": done.state, "result": done.result,
|
||||
"error": done.error}))
|
||||
return 0 if done.state == "done" else 1
|
||||
return 0
|
||||
|
||||
|
||||
def init_db(args: argparse.Namespace) -> int:
|
||||
"""Create the file, or bring it to the current schema."""
|
||||
from roledesk.storage import LATEST, Store
|
||||
|
||||
config = config_mod.load(args.config)
|
||||
store = Store(config.database)
|
||||
applied = store.migrations_applied
|
||||
print(f"{store.path}: schema v{LATEST}"
|
||||
+ (f", applied {applied}" if applied else ""))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Config loading: one YAML file, `roles.yaml`, validated into a model.
|
||||
|
||||
A typo in a key is an error naming the key, because `Section` forbids extras.
|
||||
`sources` stays a dict: one block per plugin, whose keys only the plugin knows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
from pydantic import Field, ValidationError
|
||||
|
||||
from desk.config import AgentConfig, ModelTaskConfig, Section, WorkersConfig
|
||||
|
||||
DEFAULT_PATH = Path(os.environ.get("ROLEDESK_CONFIG", "roles.yaml"))
|
||||
|
||||
|
||||
class ExtractConfig(ModelTaskConfig):
|
||||
"""Turning a posting's title and text into the gate columns. Off until the
|
||||
`extract` task exists; the section is here so the file's shape is settled."""
|
||||
|
||||
enabled: bool = False
|
||||
model: str = "claude-haiku-4-5"
|
||||
|
||||
|
||||
class Config(Section):
|
||||
"""The whole file."""
|
||||
|
||||
database: str = "roledesk.db"
|
||||
plugin_paths: list[str] = Field(default_factory=lambda: ["plugins"])
|
||||
workers: WorkersConfig = Field(default_factory=WorkersConfig)
|
||||
agent: AgentConfig = Field(default_factory=AgentConfig)
|
||||
extract: ExtractConfig = Field(default_factory=ExtractConfig)
|
||||
#: One block per plugin: `enabled`, and whatever that plugin reads.
|
||||
sources: dict[str, Any] = Field(default_factory=dict)
|
||||
#: Where this was loaded from. Never written back into the file.
|
||||
path: str = Field("", exclude=True)
|
||||
|
||||
|
||||
class ConfigError(FileNotFoundError):
|
||||
"""A config that exists but does not validate. A FileNotFoundError so the
|
||||
CLI reports both kinds of unusable config the same way: as a message."""
|
||||
|
||||
|
||||
def load(path: str | Path | None = None) -> Config:
|
||||
p = Path(path or DEFAULT_PATH).expanduser()
|
||||
if not p.exists():
|
||||
raise FileNotFoundError(
|
||||
f"No config at {p}. Copy config.example.yaml to {p} and edit it.")
|
||||
with p.open(encoding="utf-8") as fh:
|
||||
raw = yaml.safe_load(fh) or {}
|
||||
try:
|
||||
config = Config(**raw, path=str(p))
|
||||
except ValidationError as exc:
|
||||
raise ConfigError(f"{p}:\n{_explain(exc)}") from None
|
||||
config.agent.credentials_database = config.database
|
||||
return config
|
||||
|
||||
|
||||
def _explain(exc: ValidationError) -> str:
|
||||
lines = []
|
||||
for err in exc.errors():
|
||||
where = ".".join(str(p) for p in err["loc"]) or "(top level)"
|
||||
detail = err["msg"]
|
||||
if err["type"] == "extra_forbidden":
|
||||
detail = "unknown setting, check the spelling against config.example.yaml"
|
||||
lines.append(f" {where}: {detail}")
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,24 @@
|
||||
"""The work roledesk queues. Importing this package registers every job.
|
||||
|
||||
pull.py read every configured plugin and store the roles it yields
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from roledesk.jobs import pull
|
||||
|
||||
#: What `JobQueue(bootstrap=...)` and `pipeline.worker --bootstrap` are given.
|
||||
BOOTSTRAP = "roledesk.jobs:bootstrap"
|
||||
|
||||
|
||||
def bootstrap(config_path: str = "") -> tuple[object, object]:
|
||||
"""roledesk's `(config, store)` from the config file alone, for a job run
|
||||
in its own process or by `python -m pipeline.worker`."""
|
||||
from roledesk import config as config_mod
|
||||
from roledesk.storage import Store
|
||||
|
||||
config = config_mod.load(config_path or None)
|
||||
return config, Store(config.database)
|
||||
|
||||
|
||||
__all__ = ["BOOTSTRAP", "bootstrap", "pull"]
|
||||
@@ -0,0 +1,18 @@
|
||||
"""`pull`: read every configured plugin. A no-op until the first plugin lands."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from pipeline import NULL, RunContext, job
|
||||
from roledesk.config import Config
|
||||
from roledesk.storage import Store
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@job("pull", "Reading every board", resource="http", solo=True)
|
||||
def run(config: Config, store: Store, *, context: RunContext = NULL) -> dict[str, Any]:
|
||||
log.info("pull: no plugin is installed yet")
|
||||
return {"sources": 0, "seen": 0, "new": 0}
|
||||
@@ -0,0 +1,3 @@
|
||||
"""The shapes roledesk hands round. Filled in with the storage layer."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -0,0 +1,16 @@
|
||||
"""Where things are kept. One door: `from roledesk.storage import Store`.
|
||||
|
||||
schema.py the tables, and the numbered migrations that bring an older
|
||||
file to the current shape
|
||||
store.py `Store`, every SQL statement in the application
|
||||
|
||||
Written for roledesk rather than imported from newsdesk: `desk` has no storage
|
||||
package on purpose, and the connection plumbing here is a deliberate copy.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from roledesk.storage.schema import LATEST
|
||||
from roledesk.storage.store import Store
|
||||
|
||||
__all__ = ["LATEST", "Store"]
|
||||
@@ -0,0 +1,131 @@
|
||||
"""The tables, and how a database file gets from an old shape to this one.
|
||||
|
||||
`SCHEMA` is the current shape. A new file is created from it and stamped with
|
||||
`LATEST`, so it never runs a migration. `MIGRATIONS` is how an existing file
|
||||
catches up: ordered, numbered, applied once each, tracked in SQLite's own
|
||||
`PRAGMA user_version`.
|
||||
|
||||
Adding a column means editing both: the DDL in `SCHEMA` for new files, and an
|
||||
appended migration for old ones. Never renumber or edit a migration that has
|
||||
shipped.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SCHEMA = """
|
||||
-- The string map a desk plugin is handed as `self.state`. One namespace shared
|
||||
-- by every plugin, so a plugin prefixes its keys.
|
||||
CREATE TABLE IF NOT EXISTS state (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
"""
|
||||
|
||||
#: Applied after `SCHEMA` or the migrations, every open. An index may name a
|
||||
#: column only a migration adds, which is why these are kept apart.
|
||||
INDEXES = """
|
||||
"""
|
||||
|
||||
#: The table whose absence marks a file that has never been a roledesk
|
||||
#: database, and is therefore created from `SCHEMA` rather than migrated.
|
||||
MARKER = "state"
|
||||
|
||||
Step = str | Callable[[sqlite3.Connection], None]
|
||||
|
||||
MIGRATIONS: tuple[tuple[int, str, Step], ...] = (
|
||||
# Never run as a migration: a file without the marker table is created
|
||||
# from `SCHEMA` and stamped. It exists so the first real migration is 2.
|
||||
(1, "the first schema", ""),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
LATEST = MIGRATIONS[-1][0]
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
def _stamp(conn: sqlite3.Connection, version: int) -> None:
|
||||
# PRAGMA takes no bound parameters; the value is an int from MIGRATIONS.
|
||||
conn.execute(f"PRAGMA user_version = {int(version)}")
|
||||
|
||||
|
||||
def _statements(script: str) -> list[str]:
|
||||
"""A script split into statements, cut where `sqlite3.complete_statement`
|
||||
says one ends. Not `executescript`, which commits before it starts and
|
||||
autocommits each statement, so a failing migration would be half applied."""
|
||||
statements, buffer = [], ""
|
||||
for line in script.splitlines(keepends=True):
|
||||
buffer += line
|
||||
if sqlite3.complete_statement(buffer):
|
||||
statements.append(buffer.strip())
|
||||
buffer = ""
|
||||
if buffer.strip():
|
||||
statements.append(buffer.strip())
|
||||
return statements
|
||||
|
||||
|
||||
def migrate(conn: sqlite3.Connection) -> list[int]:
|
||||
"""Bring an existing database up to `LATEST`. Returns what it applied."""
|
||||
current = user_version(conn)
|
||||
if current > LATEST:
|
||||
raise RuntimeError(
|
||||
f"this database is at schema v{current}, but this roledesk only knows "
|
||||
f"up to v{LATEST}. Upgrade roledesk rather than let older code write to it.")
|
||||
applied = []
|
||||
for version, description, step in MIGRATIONS:
|
||||
if version <= current:
|
||||
continue
|
||||
log.info("migrating database to v%d: %s", version, description)
|
||||
_apply(conn, version, step)
|
||||
applied.append(version)
|
||||
return applied
|
||||
|
||||
|
||||
def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
|
||||
"""One migration and its version stamp, in one explicit transaction.
|
||||
|
||||
Explicit because sqlite3 only opens a transaction implicitly before DML: an
|
||||
`ALTER` would otherwise be durable at once while the stamp stayed behind,
|
||||
and the next open would replay it into a "duplicate column" forever.
|
||||
"""
|
||||
started = not conn.in_transaction
|
||||
if started:
|
||||
conn.execute("BEGIN")
|
||||
try:
|
||||
if callable(step):
|
||||
step(conn)
|
||||
else:
|
||||
for statement in _statements(step):
|
||||
conn.execute(statement)
|
||||
_stamp(conn, version)
|
||||
except Exception:
|
||||
if started:
|
||||
conn.execute("ROLLBACK")
|
||||
raise
|
||||
if started:
|
||||
conn.execute("COMMIT")
|
||||
|
||||
|
||||
def initialise(conn: sqlite3.Connection) -> list[int]:
|
||||
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
|
||||
fresh = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (MARKER,)
|
||||
).fetchone() is None
|
||||
applied: list[int] = []
|
||||
if fresh:
|
||||
# Atomic like a migration: a file half created would otherwise carry
|
||||
# the marker table and be taken for one to migrate.
|
||||
_apply(conn, LATEST, SCHEMA)
|
||||
else:
|
||||
applied = migrate(conn)
|
||||
for statement in _statements(INDEXES):
|
||||
conn.execute(statement)
|
||||
return applied
|
||||
@@ -0,0 +1,103 @@
|
||||
"""`Store`: every SQL statement in the application.
|
||||
|
||||
The connection plumbing is a copy of newsdesk's `Store`, on purpose: `desk`
|
||||
has no storage package. A connection is opened per call and closed again, so
|
||||
one caller's commit can never land inside another's transaction; in WAL mode
|
||||
that costs a file handle.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from collections.abc import Iterator, MutableMapping
|
||||
from contextlib import closing, contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from roledesk.storage import schema
|
||||
|
||||
|
||||
class _StateMap(MutableMapping[str, str]):
|
||||
"""`Store.state`: the key/value table, read and written one key at a time."""
|
||||
|
||||
def __init__(self, store: Store) -> None:
|
||||
self._store = store
|
||||
|
||||
def __getitem__(self, key: str) -> str:
|
||||
row = self._store._row("SELECT value FROM state WHERE key=?", (key,))
|
||||
if row is None:
|
||||
raise KeyError(key)
|
||||
return str(row["value"])
|
||||
|
||||
def __setitem__(self, key: str, value: str) -> None:
|
||||
self._store._write(
|
||||
"""INSERT INTO state (key, value) VALUES (?,?)
|
||||
ON CONFLICT(key) DO UPDATE SET value=excluded.value""",
|
||||
(key, value))
|
||||
|
||||
def __delitem__(self, key: str) -> None:
|
||||
if not self._store._write("DELETE FROM state WHERE key=?", (key,)):
|
||||
raise KeyError(key)
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
return iter([r["key"] for r in self._store._rows("SELECT key FROM state ORDER BY key")])
|
||||
|
||||
def __len__(self) -> int:
|
||||
row = self._store._row("SELECT COUNT(*) AS n FROM state")
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, path: str | Path) -> None:
|
||||
self.path = Path(path).expanduser()
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self.transaction() as conn:
|
||||
# Persistent: recorded in the file, inherited by every connection.
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
#: Migration versions this open applied; empty when already current.
|
||||
self.migrations_applied: list[int] = schema.initialise(conn)
|
||||
|
||||
# ------------------------------------------------------------ connections
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
"""A fresh connection. `foreign_keys` is off by default per connection;
|
||||
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
|
||||
`synchronous=NORMAL` is the standard WAL companion."""
|
||||
conn = sqlite3.connect(self.path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
conn.execute("PRAGMA busy_timeout=30000")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
return conn
|
||||
|
||||
@contextmanager
|
||||
def transaction(self) -> Iterator[sqlite3.Connection]:
|
||||
"""One connection, one transaction, closed on the way out."""
|
||||
conn = self._connect()
|
||||
try:
|
||||
with conn:
|
||||
yield conn
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]:
|
||||
with closing(self._connect()) as conn:
|
||||
return conn.execute(sql, params).fetchall()
|
||||
|
||||
def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None:
|
||||
with closing(self._connect()) as conn:
|
||||
row: sqlite3.Row | None = conn.execute(sql, params).fetchone()
|
||||
return row
|
||||
|
||||
def _write(self, sql: str, params: Any = ()) -> int:
|
||||
"""One statement in its own transaction. Returns the rowcount."""
|
||||
with self.transaction() as conn:
|
||||
return conn.execute(sql, params).rowcount
|
||||
|
||||
# ---------------------------------------------------------------- state
|
||||
|
||||
@property
|
||||
def state(self) -> MutableMapping[str, str]:
|
||||
"""The `state` table as a mapping. What the runner hands a plugin as
|
||||
`self.state`: it can keep a counter without being able to read roles."""
|
||||
return _StateMap(self)
|
||||
@@ -0,0 +1,7 @@
|
||||
"""The web face. One door: `create_app` and `serve`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from roledesk.web.app import create_app, serve
|
||||
|
||||
__all__ = ["create_app", "serve"]
|
||||
@@ -0,0 +1,46 @@
|
||||
"""The application factory. A stub page until the reader lands: the queue's
|
||||
routes and socket from `pipeline.web.connect`, and a one-line index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import HTMLResponse
|
||||
|
||||
from pipeline import Jobs
|
||||
from pipeline.web import connect
|
||||
from roledesk import config as config_mod
|
||||
from roledesk import jobs
|
||||
from roledesk.storage import Store
|
||||
|
||||
INDEX = """<!doctype html>
|
||||
<meta charset="utf-8"><title>roledesk</title>
|
||||
<p>roledesk: no reader yet.</p>
|
||||
<script type="module" src="/_jobs/jobs.js"></script>
|
||||
<desk-jobs></desk-jobs>
|
||||
"""
|
||||
|
||||
|
||||
def create_app(config_path: str | None = None) -> FastAPI:
|
||||
config = config_mod.load(config_path)
|
||||
# `Store` first, so the file is at the current schema before the queue
|
||||
# adds its own table to it.
|
||||
store = Store(config.database)
|
||||
app = FastAPI(title="roledesk", docs_url="/api/docs")
|
||||
connect(app, Jobs(config.database), config, store,
|
||||
workers=config.workers.model_dump(),
|
||||
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=config.path or "")
|
||||
app.state.config = config
|
||||
app.state.store = store
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
def index() -> str:
|
||||
return INDEX
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def serve(host: str, port: int, config_path: str | None) -> int:
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(create_app(config_path), host=host, port=port)
|
||||
return 0
|
||||
Reference in new issue
Block a user