Skeleton: roledesk on desk and pipeline from ../news

Package layers models, config, storage, jobs, web, cli, held by
tests/test_architecture.py with desk and pipeline as external trees and
plugins/ limited to desk. roles.yaml is the default config, roledesk.db
its own file, port 8790. create_app mounts pipeline.web.connect over
Jobs(config.database); `roledesk job pull` drains a no-op pull and
`roledesk db init` creates the file.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-23 10:59:17 +02:00
commit 36dbcbb29e
20 files changed
+2320

No files matched your search

+14
View File
@@ -0,0 +1,14 @@
"""roledesk: a job-search engine on the `desk` plugin contract and the `pipeline` queue.
Layers, bottom to top, each importing only from those below it (see
`tests/test_architecture.py`):
models the shapes handed round, and the key functions for clustering
config `roles.yaml`, validated
storage `Store`: every SQL statement against `roledesk.db`
jobs the work this app queues
web the FastAPI app
cli the entry point
`desk` and `pipeline` come from the news repo and are imported, never edited.
"""
+84
View File
@@ -0,0 +1,84 @@
"""`roledesk serve | job pull | db init`."""
from __future__ import annotations
import argparse
import json
import logging
import sys
from roledesk import config as config_mod
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="roledesk",
description="A job-search engine on desk.")
parser.add_argument("-c", "--config", default=None, help="path to roles.yaml")
parser.add_argument("-v", "--verbose", action="store_true")
sub = parser.add_subparsers(dest="command", required=True)
serve = sub.add_parser("serve", help="run the web app")
serve.add_argument("--host", default="127.0.0.1")
serve.add_argument("--port", type=int, default=8790)
job = sub.add_parser("job", help="queue a job and drain the queue once")
job.add_argument("kind", choices=["pull"])
db = sub.add_parser("db", help="database housekeeping")
db.add_argument("action", choices=["init"])
return parser
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s")
try:
if args.command == "serve":
from roledesk.web import serve
config_mod.load(args.config) # a config error here, not in uvicorn
return serve(args.host, args.port, args.config)
if args.command == "job":
return run_job(args)
return init_db(args)
except FileNotFoundError as exc:
print(exc, file=sys.stderr)
return 1
def run_job(args: argparse.Namespace) -> int:
"""Queue `kind`, then run everything queued until the queue is idle."""
from pipeline import JobQueue, Jobs
from roledesk import jobs
from roledesk.storage import Store
config = config_mod.load(args.config)
store = Store(config.database)
queue = JobQueue(Jobs(config.database), config, store,
workers=config.workers.model_dump(),
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=config.path)
queued = queue.enqueue(args.kind)
queue.drain()
if queued is not None:
done = queue.jobs.get(queued.id)
if done is not None:
print(json.dumps({"id": done.id, "state": done.state, "result": done.result,
"error": done.error}))
return 0 if done.state == "done" else 1
return 0
def init_db(args: argparse.Namespace) -> int:
"""Create the file, or bring it to the current schema."""
from roledesk.storage import LATEST, Store
config = config_mod.load(args.config)
store = Store(config.database)
applied = store.migrations_applied
print(f"{store.path}: schema v{LATEST}"
+ (f", applied {applied}" if applied else ""))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+71
View File
@@ -0,0 +1,71 @@
"""Config loading: one YAML file, `roles.yaml`, validated into a model.
A typo in a key is an error naming the key, because `Section` forbids extras.
`sources` stays a dict: one block per plugin, whose keys only the plugin knows.
"""
from __future__ import annotations
import os
from pathlib import Path
from typing import Any
import yaml
from pydantic import Field, ValidationError
from desk.config import AgentConfig, ModelTaskConfig, Section, WorkersConfig
DEFAULT_PATH = Path(os.environ.get("ROLEDESK_CONFIG", "roles.yaml"))
class ExtractConfig(ModelTaskConfig):
"""Turning a posting's title and text into the gate columns. Off until the
`extract` task exists; the section is here so the file's shape is settled."""
enabled: bool = False
model: str = "claude-haiku-4-5"
class Config(Section):
"""The whole file."""
database: str = "roledesk.db"
plugin_paths: list[str] = Field(default_factory=lambda: ["plugins"])
workers: WorkersConfig = Field(default_factory=WorkersConfig)
agent: AgentConfig = Field(default_factory=AgentConfig)
extract: ExtractConfig = Field(default_factory=ExtractConfig)
#: One block per plugin: `enabled`, and whatever that plugin reads.
sources: dict[str, Any] = Field(default_factory=dict)
#: Where this was loaded from. Never written back into the file.
path: str = Field("", exclude=True)
class ConfigError(FileNotFoundError):
"""A config that exists but does not validate. A FileNotFoundError so the
CLI reports both kinds of unusable config the same way: as a message."""
def load(path: str | Path | None = None) -> Config:
p = Path(path or DEFAULT_PATH).expanduser()
if not p.exists():
raise FileNotFoundError(
f"No config at {p}. Copy config.example.yaml to {p} and edit it.")
with p.open(encoding="utf-8") as fh:
raw = yaml.safe_load(fh) or {}
try:
config = Config(**raw, path=str(p))
except ValidationError as exc:
raise ConfigError(f"{p}:\n{_explain(exc)}") from None
config.agent.credentials_database = config.database
return config
def _explain(exc: ValidationError) -> str:
lines = []
for err in exc.errors():
where = ".".join(str(p) for p in err["loc"]) or "(top level)"
detail = err["msg"]
if err["type"] == "extra_forbidden":
detail = "unknown setting, check the spelling against config.example.yaml"
lines.append(f" {where}: {detail}")
return "\n".join(lines)
+24
View File
@@ -0,0 +1,24 @@
"""The work roledesk queues. Importing this package registers every job.
pull.py read every configured plugin and store the roles it yields
"""
from __future__ import annotations
from roledesk.jobs import pull
#: What `JobQueue(bootstrap=...)` and `pipeline.worker --bootstrap` are given.
BOOTSTRAP = "roledesk.jobs:bootstrap"
def bootstrap(config_path: str = "") -> tuple[object, object]:
"""roledesk's `(config, store)` from the config file alone, for a job run
in its own process or by `python -m pipeline.worker`."""
from roledesk import config as config_mod
from roledesk.storage import Store
config = config_mod.load(config_path or None)
return config, Store(config.database)
__all__ = ["BOOTSTRAP", "bootstrap", "pull"]
+18
View File
@@ -0,0 +1,18 @@
"""`pull`: read every configured plugin. A no-op until the first plugin lands."""
from __future__ import annotations
import logging
from typing import Any
from pipeline import NULL, RunContext, job
from roledesk.config import Config
from roledesk.storage import Store
log = logging.getLogger(__name__)
@job("pull", "Reading every board", resource="http", solo=True)
def run(config: Config, store: Store, *, context: RunContext = NULL) -> dict[str, Any]:
log.info("pull: no plugin is installed yet")
return {"sources": 0, "seen": 0, "new": 0}
+3
View File
@@ -0,0 +1,3 @@
"""The shapes roledesk hands round. Filled in with the storage layer."""
from __future__ import annotations
+16
View File
@@ -0,0 +1,16 @@
"""Where things are kept. One door: `from roledesk.storage import Store`.
schema.py the tables, and the numbered migrations that bring an older
file to the current shape
store.py `Store`, every SQL statement in the application
Written for roledesk rather than imported from newsdesk: `desk` has no storage
package on purpose, and the connection plumbing here is a deliberate copy.
"""
from __future__ import annotations
from roledesk.storage.schema import LATEST
from roledesk.storage.store import Store
__all__ = ["LATEST", "Store"]
+131
View File
@@ -0,0 +1,131 @@
"""The tables, and how a database file gets from an old shape to this one.
`SCHEMA` is the current shape. A new file is created from it and stamped with
`LATEST`, so it never runs a migration. `MIGRATIONS` is how an existing file
catches up: ordered, numbered, applied once each, tracked in SQLite's own
`PRAGMA user_version`.
Adding a column means editing both: the DDL in `SCHEMA` for new files, and an
appended migration for old ones. Never renumber or edit a migration that has
shipped.
"""
from __future__ import annotations
import logging
import sqlite3
from collections.abc import Callable
log = logging.getLogger(__name__)
SCHEMA = """
-- The string map a desk plugin is handed as `self.state`. One namespace shared
-- by every plugin, so a plugin prefixes its keys.
CREATE TABLE IF NOT EXISTS state (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
"""
#: Applied after `SCHEMA` or the migrations, every open. An index may name a
#: column only a migration adds, which is why these are kept apart.
INDEXES = """
"""
#: The table whose absence marks a file that has never been a roledesk
#: database, and is therefore created from `SCHEMA` rather than migrated.
MARKER = "state"
Step = str | Callable[[sqlite3.Connection], None]
MIGRATIONS: tuple[tuple[int, str, Step], ...] = (
# Never run as a migration: a file without the marker table is created
# from `SCHEMA` and stamped. It exists so the first real migration is 2.
(1, "the first schema", ""),
)
#: What a file created from `SCHEMA` today is already at.
LATEST = MIGRATIONS[-1][0]
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])
def _stamp(conn: sqlite3.Connection, version: int) -> None:
# PRAGMA takes no bound parameters; the value is an int from MIGRATIONS.
conn.execute(f"PRAGMA user_version = {int(version)}")
def _statements(script: str) -> list[str]:
"""A script split into statements, cut where `sqlite3.complete_statement`
says one ends. Not `executescript`, which commits before it starts and
autocommits each statement, so a failing migration would be half applied."""
statements, buffer = [], ""
for line in script.splitlines(keepends=True):
buffer += line
if sqlite3.complete_statement(buffer):
statements.append(buffer.strip())
buffer = ""
if buffer.strip():
statements.append(buffer.strip())
return statements
def migrate(conn: sqlite3.Connection) -> list[int]:
"""Bring an existing database up to `LATEST`. Returns what it applied."""
current = user_version(conn)
if current > LATEST:
raise RuntimeError(
f"this database is at schema v{current}, but this roledesk only knows "
f"up to v{LATEST}. Upgrade roledesk rather than let older code write to it.")
applied = []
for version, description, step in MIGRATIONS:
if version <= current:
continue
log.info("migrating database to v%d: %s", version, description)
_apply(conn, version, step)
applied.append(version)
return applied
def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
"""One migration and its version stamp, in one explicit transaction.
Explicit because sqlite3 only opens a transaction implicitly before DML: an
`ALTER` would otherwise be durable at once while the stamp stayed behind,
and the next open would replay it into a "duplicate column" forever.
"""
started = not conn.in_transaction
if started:
conn.execute("BEGIN")
try:
if callable(step):
step(conn)
else:
for statement in _statements(step):
conn.execute(statement)
_stamp(conn, version)
except Exception:
if started:
conn.execute("ROLLBACK")
raise
if started:
conn.execute("COMMIT")
def initialise(conn: sqlite3.Connection) -> list[int]:
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
fresh = conn.execute(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (MARKER,)
).fetchone() is None
applied: list[int] = []
if fresh:
# Atomic like a migration: a file half created would otherwise carry
# the marker table and be taken for one to migrate.
_apply(conn, LATEST, SCHEMA)
else:
applied = migrate(conn)
for statement in _statements(INDEXES):
conn.execute(statement)
return applied
+103
View File
@@ -0,0 +1,103 @@
"""`Store`: every SQL statement in the application.
The connection plumbing is a copy of newsdesk's `Store`, on purpose: `desk`
has no storage package. A connection is opened per call and closed again, so
one caller's commit can never land inside another's transaction; in WAL mode
that costs a file handle.
"""
from __future__ import annotations
import sqlite3
from collections.abc import Iterator, MutableMapping
from contextlib import closing, contextmanager
from pathlib import Path
from typing import Any
from roledesk.storage import schema
class _StateMap(MutableMapping[str, str]):
"""`Store.state`: the key/value table, read and written one key at a time."""
def __init__(self, store: Store) -> None:
self._store = store
def __getitem__(self, key: str) -> str:
row = self._store._row("SELECT value FROM state WHERE key=?", (key,))
if row is None:
raise KeyError(key)
return str(row["value"])
def __setitem__(self, key: str, value: str) -> None:
self._store._write(
"""INSERT INTO state (key, value) VALUES (?,?)
ON CONFLICT(key) DO UPDATE SET value=excluded.value""",
(key, value))
def __delitem__(self, key: str) -> None:
if not self._store._write("DELETE FROM state WHERE key=?", (key,)):
raise KeyError(key)
def __iter__(self) -> Iterator[str]:
return iter([r["key"] for r in self._store._rows("SELECT key FROM state ORDER BY key")])
def __len__(self) -> int:
row = self._store._row("SELECT COUNT(*) AS n FROM state")
return int(row["n"]) if row else 0
class Store:
def __init__(self, path: str | Path) -> None:
self.path = Path(path).expanduser()
self.path.parent.mkdir(parents=True, exist_ok=True)
with self.transaction() as conn:
# Persistent: recorded in the file, inherited by every connection.
conn.execute("PRAGMA journal_mode=WAL")
#: Migration versions this open applied; empty when already current.
self.migrations_applied: list[int] = schema.initialise(conn)
# ------------------------------------------------------------ connections
def _connect(self) -> sqlite3.Connection:
"""A fresh connection. `foreign_keys` is off by default per connection;
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
`synchronous=NORMAL` is the standard WAL companion."""
conn = sqlite3.connect(self.path)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA foreign_keys=ON")
conn.execute("PRAGMA busy_timeout=30000")
conn.execute("PRAGMA synchronous=NORMAL")
return conn
@contextmanager
def transaction(self) -> Iterator[sqlite3.Connection]:
"""One connection, one transaction, closed on the way out."""
conn = self._connect()
try:
with conn:
yield conn
finally:
conn.close()
def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]:
with closing(self._connect()) as conn:
return conn.execute(sql, params).fetchall()
def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None:
with closing(self._connect()) as conn:
row: sqlite3.Row | None = conn.execute(sql, params).fetchone()
return row
def _write(self, sql: str, params: Any = ()) -> int:
"""One statement in its own transaction. Returns the rowcount."""
with self.transaction() as conn:
return conn.execute(sql, params).rowcount
# ---------------------------------------------------------------- state
@property
def state(self) -> MutableMapping[str, str]:
"""The `state` table as a mapping. What the runner hands a plugin as
`self.state`: it can keep a counter without being able to read roles."""
return _StateMap(self)
+7
View File
@@ -0,0 +1,7 @@
"""The web face. One door: `create_app` and `serve`."""
from __future__ import annotations
from roledesk.web.app import create_app, serve
__all__ = ["create_app", "serve"]
+46
View File
@@ -0,0 +1,46 @@
"""The application factory. A stub page until the reader lands: the queue's
routes and socket from `pipeline.web.connect`, and a one-line index."""
from __future__ import annotations
from fastapi import FastAPI
from fastapi.responses import HTMLResponse
from pipeline import Jobs
from pipeline.web import connect
from roledesk import config as config_mod
from roledesk import jobs
from roledesk.storage import Store
INDEX = """<!doctype html>
<meta charset="utf-8"><title>roledesk</title>
<p>roledesk: no reader yet.</p>
<script type="module" src="/_jobs/jobs.js"></script>
<desk-jobs></desk-jobs>
"""
def create_app(config_path: str | None = None) -> FastAPI:
config = config_mod.load(config_path)
# `Store` first, so the file is at the current schema before the queue
# adds its own table to it.
store = Store(config.database)
app = FastAPI(title="roledesk", docs_url="/api/docs")
connect(app, Jobs(config.database), config, store,
workers=config.workers.model_dump(),
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=config.path or "")
app.state.config = config
app.state.store = store
@app.get("/", response_class=HTMLResponse)
def index() -> str:
return INDEX
return app
def serve(host: str, port: int, config_path: str | None) -> int:
import uvicorn
uvicorn.run(create_app(config_path), host=host, port=port)
return 0