Skeleton: roledesk on desk and pipeline from ../news
Package layers models, config, storage, jobs, web, cli, held by tests/test_architecture.py with desk and pipeline as external trees and plugins/ limited to desk. roles.yaml is the default config, roledesk.db its own file, port 8790. create_app mounts pipeline.web.connect over Jobs(config.database); `roledesk job pull` drains a no-op pull and `roledesk db init` creates the file. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
36dbcbb29e
20 files changed
+2320
No files matched your search
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"version": "0.0.1",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "roledesk",
|
||||
"runtimeExecutable": "uv",
|
||||
"runtimeArgs": [
|
||||
"run",
|
||||
"roledesk",
|
||||
"serve",
|
||||
"--port",
|
||||
"8790"
|
||||
],
|
||||
"port": 8790,
|
||||
"autoPort": false
|
||||
}
|
||||
]
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.egg-info/
|
||||
build/
|
||||
dist/
|
||||
|
||||
# Environments
|
||||
.venv/
|
||||
|
||||
# Local state
|
||||
roledesk.db*
|
||||
|
||||
# Tooling caches
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
.mypy_cache/
|
||||
|
||||
# A throwaway config and database for checking the app by hand
|
||||
.scratch/
|
||||
@@ -0,0 +1,30 @@
|
||||
# Rule:
|
||||
|
||||
Committed in main, branch deleted means the work is done. Having code uncommitted or committed in a branch or worktree means you failed to deliver. Pushing is not a thing we really care about.
|
||||
|
||||
# Rule:
|
||||
|
||||
MDs dont have hard wraps. IF you come across one make it soft wrapping
|
||||
|
||||
# Rule:
|
||||
|
||||
In documentations so not write bloat like:
|
||||
|
||||
> The obvious objection, answered before it's asked. "They just open a new session." Yes. **That's friction, not a wall, and it's meant to be.**
|
||||
|
||||
Instead, write more concise, more technical English, e.g. :
|
||||
|
||||
> One objection is that they just open a new session. Yet, this friction is by design because ...
|
||||
|
||||
Bad Example:
|
||||
|
||||
> **Could we verify them first to save the human a step?** Only if the receiving system will accept our verification. If it re-authenticates anyway we've added a step to save none — worth asking in a real deployment, not worth assuming.
|
||||
|
||||
Good example:
|
||||
|
||||
> We should only verify them first, if the receiving system will accept our verification? Otherwise it double work for the customer.
|
||||
|
||||
|
||||
# Rule:
|
||||
|
||||
`desk` and `pipeline` are imported from ../news and never edited here.
|
||||
@@ -0,0 +1,29 @@
|
||||
# roledesk configuration. `roles.yaml` is the one the app reads by default
|
||||
# (or $ROLEDESK_CONFIG, or `roledesk -c path`). Every key here is optional.
|
||||
|
||||
# The SQLite file: roles, companies, subscriptions, applications, and the
|
||||
# job queue's own table. Not shared with newsdesk.
|
||||
database: roledesk.db
|
||||
|
||||
# Any .py file in these directories defining a desk `Source` subclass is a
|
||||
# plugin. A plugin imports `desk` only.
|
||||
plugin_paths:
|
||||
- plugins
|
||||
|
||||
# Workers per resource class. `http` is wide because a fetch is mostly waiting.
|
||||
workers:
|
||||
llm: 4
|
||||
http: 8
|
||||
cpu: 1
|
||||
|
||||
# Who answers when a job needs a model. See desk/tools/agent.py in ../news.
|
||||
agent:
|
||||
backend: auto # auto | api | claude_code | off
|
||||
|
||||
# Title and description in, gate columns out. Off until the extract task lands.
|
||||
extract:
|
||||
enabled: false
|
||||
model: claude-haiku-4-5
|
||||
|
||||
# One block per plugin. Empty until the ats plugin lands.
|
||||
sources: {}
|
||||
Whitespace-only changes.
@@ -0,0 +1,90 @@
|
||||
[project]
|
||||
name = "roledesk"
|
||||
version = "0.1.0"
|
||||
description = "A job-search engine on the desk plugin contract and the pipeline queue."
|
||||
requires-python = ">=3.14"
|
||||
dependencies = [
|
||||
# `desk` and `pipeline` are packaged by the news repo as part of the
|
||||
# newsdesk distribution. Imported from there and never edited here.
|
||||
"newsdesk",
|
||||
"fastapi>=0.110",
|
||||
"pydantic>=2.6",
|
||||
"pyyaml>=6.0",
|
||||
"uvicorn[standard]>=0.29",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
roledesk = "roledesk.cli:main"
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools>=68"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
include = ["roledesk*"]
|
||||
|
||||
[tool.uv.sources]
|
||||
newsdesk = { path = "../news", editable = true }
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=9.1.1",
|
||||
"ruff>=0.14",
|
||||
"mypy>=1.18",
|
||||
"types-PyYAML>=6.0",
|
||||
]
|
||||
|
||||
# ----------------------------------------------------------------- linting
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 90
|
||||
target-version = "py314"
|
||||
|
||||
[tool.ruff.lint]
|
||||
# The same selection as news: bug-prone patterns, outdated idioms,
|
||||
# comprehensions and ruff's own checks. No import sorting.
|
||||
select = ["E", "F", "W", "B", "UP", "C4", "SIM", "RUF"]
|
||||
ignore = [
|
||||
"E501", # line length: prose comments wrap by meaning
|
||||
"B008", # function call in a default argument, as FastAPI writes Depends
|
||||
"SIM108", # ternary instead of if/else
|
||||
]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
# Pydantic models whose base comes from `desk.models`; ruff only recognises a
|
||||
# pydantic subclass when it sees `BaseModel` in the same file.
|
||||
"roledesk/models.py" = ["RUF012"]
|
||||
|
||||
# ------------------------------------------------------------ type checking
|
||||
|
||||
[tool.mypy]
|
||||
python_version = "3.14"
|
||||
files = ["roledesk", "tests"]
|
||||
plugins = ["pydantic.mypy"]
|
||||
# The editable install of ../news is a setuptools import hook, which mypy
|
||||
# cannot follow. Pointed at the checkout instead, and told to type what it
|
||||
# imports from there without reporting on code that is not ours.
|
||||
mypy_path = "../news"
|
||||
warn_unused_ignores = true
|
||||
warn_redundant_casts = true
|
||||
warn_unreachable = true
|
||||
no_implicit_optional = true
|
||||
check_untyped_defs = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = ["desk.*", "pipeline.*", "newsdesk.*"]
|
||||
follow_imports = "silent"
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
# Optional extras of the news repo, only imported inside functions there.
|
||||
module = ["sklearn.*", "trafilatura.*", "feedparser.*", "anthropic.*",
|
||||
"claude_agent_sdk.*", "sentence_transformers.*", "webauthn.*",
|
||||
"cbor2.*"]
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = ["anthropic.*", "claude_agent_sdk.*", "sentence_transformers.*"]
|
||||
follow_imports = "skip"
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
@@ -0,0 +1,14 @@
|
||||
"""roledesk: a job-search engine on the `desk` plugin contract and the `pipeline` queue.
|
||||
|
||||
Layers, bottom to top, each importing only from those below it (see
|
||||
`tests/test_architecture.py`):
|
||||
|
||||
models the shapes handed round, and the key functions for clustering
|
||||
config `roles.yaml`, validated
|
||||
storage `Store`: every SQL statement against `roledesk.db`
|
||||
jobs the work this app queues
|
||||
web the FastAPI app
|
||||
cli the entry point
|
||||
|
||||
`desk` and `pipeline` come from the news repo and are imported, never edited.
|
||||
"""
|
||||
@@ -0,0 +1,84 @@
|
||||
"""`roledesk serve | job pull | db init`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from roledesk import config as config_mod
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(prog="roledesk",
|
||||
description="A job-search engine on desk.")
|
||||
parser.add_argument("-c", "--config", default=None, help="path to roles.yaml")
|
||||
parser.add_argument("-v", "--verbose", action="store_true")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
serve = sub.add_parser("serve", help="run the web app")
|
||||
serve.add_argument("--host", default="127.0.0.1")
|
||||
serve.add_argument("--port", type=int, default=8790)
|
||||
|
||||
job = sub.add_parser("job", help="queue a job and drain the queue once")
|
||||
job.add_argument("kind", choices=["pull"])
|
||||
|
||||
db = sub.add_parser("db", help="database housekeeping")
|
||||
db.add_argument("action", choices=["init"])
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
try:
|
||||
if args.command == "serve":
|
||||
from roledesk.web import serve
|
||||
config_mod.load(args.config) # a config error here, not in uvicorn
|
||||
return serve(args.host, args.port, args.config)
|
||||
if args.command == "job":
|
||||
return run_job(args)
|
||||
return init_db(args)
|
||||
except FileNotFoundError as exc:
|
||||
print(exc, file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def run_job(args: argparse.Namespace) -> int:
|
||||
"""Queue `kind`, then run everything queued until the queue is idle."""
|
||||
from pipeline import JobQueue, Jobs
|
||||
from roledesk import jobs
|
||||
from roledesk.storage import Store
|
||||
|
||||
config = config_mod.load(args.config)
|
||||
store = Store(config.database)
|
||||
queue = JobQueue(Jobs(config.database), config, store,
|
||||
workers=config.workers.model_dump(),
|
||||
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=config.path)
|
||||
queued = queue.enqueue(args.kind)
|
||||
queue.drain()
|
||||
if queued is not None:
|
||||
done = queue.jobs.get(queued.id)
|
||||
if done is not None:
|
||||
print(json.dumps({"id": done.id, "state": done.state, "result": done.result,
|
||||
"error": done.error}))
|
||||
return 0 if done.state == "done" else 1
|
||||
return 0
|
||||
|
||||
|
||||
def init_db(args: argparse.Namespace) -> int:
|
||||
"""Create the file, or bring it to the current schema."""
|
||||
from roledesk.storage import LATEST, Store
|
||||
|
||||
config = config_mod.load(args.config)
|
||||
store = Store(config.database)
|
||||
applied = store.migrations_applied
|
||||
print(f"{store.path}: schema v{LATEST}"
|
||||
+ (f", applied {applied}" if applied else ""))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Config loading: one YAML file, `roles.yaml`, validated into a model.
|
||||
|
||||
A typo in a key is an error naming the key, because `Section` forbids extras.
|
||||
`sources` stays a dict: one block per plugin, whose keys only the plugin knows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
from pydantic import Field, ValidationError
|
||||
|
||||
from desk.config import AgentConfig, ModelTaskConfig, Section, WorkersConfig
|
||||
|
||||
DEFAULT_PATH = Path(os.environ.get("ROLEDESK_CONFIG", "roles.yaml"))
|
||||
|
||||
|
||||
class ExtractConfig(ModelTaskConfig):
|
||||
"""Turning a posting's title and text into the gate columns. Off until the
|
||||
`extract` task exists; the section is here so the file's shape is settled."""
|
||||
|
||||
enabled: bool = False
|
||||
model: str = "claude-haiku-4-5"
|
||||
|
||||
|
||||
class Config(Section):
|
||||
"""The whole file."""
|
||||
|
||||
database: str = "roledesk.db"
|
||||
plugin_paths: list[str] = Field(default_factory=lambda: ["plugins"])
|
||||
workers: WorkersConfig = Field(default_factory=WorkersConfig)
|
||||
agent: AgentConfig = Field(default_factory=AgentConfig)
|
||||
extract: ExtractConfig = Field(default_factory=ExtractConfig)
|
||||
#: One block per plugin: `enabled`, and whatever that plugin reads.
|
||||
sources: dict[str, Any] = Field(default_factory=dict)
|
||||
#: Where this was loaded from. Never written back into the file.
|
||||
path: str = Field("", exclude=True)
|
||||
|
||||
|
||||
class ConfigError(FileNotFoundError):
|
||||
"""A config that exists but does not validate. A FileNotFoundError so the
|
||||
CLI reports both kinds of unusable config the same way: as a message."""
|
||||
|
||||
|
||||
def load(path: str | Path | None = None) -> Config:
|
||||
p = Path(path or DEFAULT_PATH).expanduser()
|
||||
if not p.exists():
|
||||
raise FileNotFoundError(
|
||||
f"No config at {p}. Copy config.example.yaml to {p} and edit it.")
|
||||
with p.open(encoding="utf-8") as fh:
|
||||
raw = yaml.safe_load(fh) or {}
|
||||
try:
|
||||
config = Config(**raw, path=str(p))
|
||||
except ValidationError as exc:
|
||||
raise ConfigError(f"{p}:\n{_explain(exc)}") from None
|
||||
config.agent.credentials_database = config.database
|
||||
return config
|
||||
|
||||
|
||||
def _explain(exc: ValidationError) -> str:
|
||||
lines = []
|
||||
for err in exc.errors():
|
||||
where = ".".join(str(p) for p in err["loc"]) or "(top level)"
|
||||
detail = err["msg"]
|
||||
if err["type"] == "extra_forbidden":
|
||||
detail = "unknown setting, check the spelling against config.example.yaml"
|
||||
lines.append(f" {where}: {detail}")
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,24 @@
|
||||
"""The work roledesk queues. Importing this package registers every job.
|
||||
|
||||
pull.py read every configured plugin and store the roles it yields
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from roledesk.jobs import pull
|
||||
|
||||
#: What `JobQueue(bootstrap=...)` and `pipeline.worker --bootstrap` are given.
|
||||
BOOTSTRAP = "roledesk.jobs:bootstrap"
|
||||
|
||||
|
||||
def bootstrap(config_path: str = "") -> tuple[object, object]:
|
||||
"""roledesk's `(config, store)` from the config file alone, for a job run
|
||||
in its own process or by `python -m pipeline.worker`."""
|
||||
from roledesk import config as config_mod
|
||||
from roledesk.storage import Store
|
||||
|
||||
config = config_mod.load(config_path or None)
|
||||
return config, Store(config.database)
|
||||
|
||||
|
||||
__all__ = ["BOOTSTRAP", "bootstrap", "pull"]
|
||||
@@ -0,0 +1,18 @@
|
||||
"""`pull`: read every configured plugin. A no-op until the first plugin lands."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from pipeline import NULL, RunContext, job
|
||||
from roledesk.config import Config
|
||||
from roledesk.storage import Store
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@job("pull", "Reading every board", resource="http", solo=True)
|
||||
def run(config: Config, store: Store, *, context: RunContext = NULL) -> dict[str, Any]:
|
||||
log.info("pull: no plugin is installed yet")
|
||||
return {"sources": 0, "seen": 0, "new": 0}
|
||||
@@ -0,0 +1,3 @@
|
||||
"""The shapes roledesk hands round. Filled in with the storage layer."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -0,0 +1,16 @@
|
||||
"""Where things are kept. One door: `from roledesk.storage import Store`.
|
||||
|
||||
schema.py the tables, and the numbered migrations that bring an older
|
||||
file to the current shape
|
||||
store.py `Store`, every SQL statement in the application
|
||||
|
||||
Written for roledesk rather than imported from newsdesk: `desk` has no storage
|
||||
package on purpose, and the connection plumbing here is a deliberate copy.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from roledesk.storage.schema import LATEST
|
||||
from roledesk.storage.store import Store
|
||||
|
||||
__all__ = ["LATEST", "Store"]
|
||||
@@ -0,0 +1,131 @@
|
||||
"""The tables, and how a database file gets from an old shape to this one.
|
||||
|
||||
`SCHEMA` is the current shape. A new file is created from it and stamped with
|
||||
`LATEST`, so it never runs a migration. `MIGRATIONS` is how an existing file
|
||||
catches up: ordered, numbered, applied once each, tracked in SQLite's own
|
||||
`PRAGMA user_version`.
|
||||
|
||||
Adding a column means editing both: the DDL in `SCHEMA` for new files, and an
|
||||
appended migration for old ones. Never renumber or edit a migration that has
|
||||
shipped.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SCHEMA = """
|
||||
-- The string map a desk plugin is handed as `self.state`. One namespace shared
|
||||
-- by every plugin, so a plugin prefixes its keys.
|
||||
CREATE TABLE IF NOT EXISTS state (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
"""
|
||||
|
||||
#: Applied after `SCHEMA` or the migrations, every open. An index may name a
|
||||
#: column only a migration adds, which is why these are kept apart.
|
||||
INDEXES = """
|
||||
"""
|
||||
|
||||
#: The table whose absence marks a file that has never been a roledesk
|
||||
#: database, and is therefore created from `SCHEMA` rather than migrated.
|
||||
MARKER = "state"
|
||||
|
||||
Step = str | Callable[[sqlite3.Connection], None]
|
||||
|
||||
MIGRATIONS: tuple[tuple[int, str, Step], ...] = (
|
||||
# Never run as a migration: a file without the marker table is created
|
||||
# from `SCHEMA` and stamped. It exists so the first real migration is 2.
|
||||
(1, "the first schema", ""),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
LATEST = MIGRATIONS[-1][0]
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
def _stamp(conn: sqlite3.Connection, version: int) -> None:
|
||||
# PRAGMA takes no bound parameters; the value is an int from MIGRATIONS.
|
||||
conn.execute(f"PRAGMA user_version = {int(version)}")
|
||||
|
||||
|
||||
def _statements(script: str) -> list[str]:
|
||||
"""A script split into statements, cut where `sqlite3.complete_statement`
|
||||
says one ends. Not `executescript`, which commits before it starts and
|
||||
autocommits each statement, so a failing migration would be half applied."""
|
||||
statements, buffer = [], ""
|
||||
for line in script.splitlines(keepends=True):
|
||||
buffer += line
|
||||
if sqlite3.complete_statement(buffer):
|
||||
statements.append(buffer.strip())
|
||||
buffer = ""
|
||||
if buffer.strip():
|
||||
statements.append(buffer.strip())
|
||||
return statements
|
||||
|
||||
|
||||
def migrate(conn: sqlite3.Connection) -> list[int]:
|
||||
"""Bring an existing database up to `LATEST`. Returns what it applied."""
|
||||
current = user_version(conn)
|
||||
if current > LATEST:
|
||||
raise RuntimeError(
|
||||
f"this database is at schema v{current}, but this roledesk only knows "
|
||||
f"up to v{LATEST}. Upgrade roledesk rather than let older code write to it.")
|
||||
applied = []
|
||||
for version, description, step in MIGRATIONS:
|
||||
if version <= current:
|
||||
continue
|
||||
log.info("migrating database to v%d: %s", version, description)
|
||||
_apply(conn, version, step)
|
||||
applied.append(version)
|
||||
return applied
|
||||
|
||||
|
||||
def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
|
||||
"""One migration and its version stamp, in one explicit transaction.
|
||||
|
||||
Explicit because sqlite3 only opens a transaction implicitly before DML: an
|
||||
`ALTER` would otherwise be durable at once while the stamp stayed behind,
|
||||
and the next open would replay it into a "duplicate column" forever.
|
||||
"""
|
||||
started = not conn.in_transaction
|
||||
if started:
|
||||
conn.execute("BEGIN")
|
||||
try:
|
||||
if callable(step):
|
||||
step(conn)
|
||||
else:
|
||||
for statement in _statements(step):
|
||||
conn.execute(statement)
|
||||
_stamp(conn, version)
|
||||
except Exception:
|
||||
if started:
|
||||
conn.execute("ROLLBACK")
|
||||
raise
|
||||
if started:
|
||||
conn.execute("COMMIT")
|
||||
|
||||
|
||||
def initialise(conn: sqlite3.Connection) -> list[int]:
|
||||
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
|
||||
fresh = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (MARKER,)
|
||||
).fetchone() is None
|
||||
applied: list[int] = []
|
||||
if fresh:
|
||||
# Atomic like a migration: a file half created would otherwise carry
|
||||
# the marker table and be taken for one to migrate.
|
||||
_apply(conn, LATEST, SCHEMA)
|
||||
else:
|
||||
applied = migrate(conn)
|
||||
for statement in _statements(INDEXES):
|
||||
conn.execute(statement)
|
||||
return applied
|
||||
@@ -0,0 +1,103 @@
|
||||
"""`Store`: every SQL statement in the application.
|
||||
|
||||
The connection plumbing is a copy of newsdesk's `Store`, on purpose: `desk`
|
||||
has no storage package. A connection is opened per call and closed again, so
|
||||
one caller's commit can never land inside another's transaction; in WAL mode
|
||||
that costs a file handle.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from collections.abc import Iterator, MutableMapping
|
||||
from contextlib import closing, contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from roledesk.storage import schema
|
||||
|
||||
|
||||
class _StateMap(MutableMapping[str, str]):
|
||||
"""`Store.state`: the key/value table, read and written one key at a time."""
|
||||
|
||||
def __init__(self, store: Store) -> None:
|
||||
self._store = store
|
||||
|
||||
def __getitem__(self, key: str) -> str:
|
||||
row = self._store._row("SELECT value FROM state WHERE key=?", (key,))
|
||||
if row is None:
|
||||
raise KeyError(key)
|
||||
return str(row["value"])
|
||||
|
||||
def __setitem__(self, key: str, value: str) -> None:
|
||||
self._store._write(
|
||||
"""INSERT INTO state (key, value) VALUES (?,?)
|
||||
ON CONFLICT(key) DO UPDATE SET value=excluded.value""",
|
||||
(key, value))
|
||||
|
||||
def __delitem__(self, key: str) -> None:
|
||||
if not self._store._write("DELETE FROM state WHERE key=?", (key,)):
|
||||
raise KeyError(key)
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
return iter([r["key"] for r in self._store._rows("SELECT key FROM state ORDER BY key")])
|
||||
|
||||
def __len__(self) -> int:
|
||||
row = self._store._row("SELECT COUNT(*) AS n FROM state")
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, path: str | Path) -> None:
|
||||
self.path = Path(path).expanduser()
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self.transaction() as conn:
|
||||
# Persistent: recorded in the file, inherited by every connection.
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
#: Migration versions this open applied; empty when already current.
|
||||
self.migrations_applied: list[int] = schema.initialise(conn)
|
||||
|
||||
# ------------------------------------------------------------ connections
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
"""A fresh connection. `foreign_keys` is off by default per connection;
|
||||
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
|
||||
`synchronous=NORMAL` is the standard WAL companion."""
|
||||
conn = sqlite3.connect(self.path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
conn.execute("PRAGMA busy_timeout=30000")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
return conn
|
||||
|
||||
@contextmanager
|
||||
def transaction(self) -> Iterator[sqlite3.Connection]:
|
||||
"""One connection, one transaction, closed on the way out."""
|
||||
conn = self._connect()
|
||||
try:
|
||||
with conn:
|
||||
yield conn
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]:
|
||||
with closing(self._connect()) as conn:
|
||||
return conn.execute(sql, params).fetchall()
|
||||
|
||||
def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None:
|
||||
with closing(self._connect()) as conn:
|
||||
row: sqlite3.Row | None = conn.execute(sql, params).fetchone()
|
||||
return row
|
||||
|
||||
def _write(self, sql: str, params: Any = ()) -> int:
|
||||
"""One statement in its own transaction. Returns the rowcount."""
|
||||
with self.transaction() as conn:
|
||||
return conn.execute(sql, params).rowcount
|
||||
|
||||
# ---------------------------------------------------------------- state
|
||||
|
||||
@property
|
||||
def state(self) -> MutableMapping[str, str]:
|
||||
"""The `state` table as a mapping. What the runner hands a plugin as
|
||||
`self.state`: it can keep a counter without being able to read roles."""
|
||||
return _StateMap(self)
|
||||
@@ -0,0 +1,7 @@
|
||||
"""The web face. One door: `create_app` and `serve`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from roledesk.web.app import create_app, serve
|
||||
|
||||
__all__ = ["create_app", "serve"]
|
||||
@@ -0,0 +1,46 @@
|
||||
"""The application factory. A stub page until the reader lands: the queue's
|
||||
routes and socket from `pipeline.web.connect`, and a one-line index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import HTMLResponse
|
||||
|
||||
from pipeline import Jobs
|
||||
from pipeline.web import connect
|
||||
from roledesk import config as config_mod
|
||||
from roledesk import jobs
|
||||
from roledesk.storage import Store
|
||||
|
||||
INDEX = """<!doctype html>
|
||||
<meta charset="utf-8"><title>roledesk</title>
|
||||
<p>roledesk: no reader yet.</p>
|
||||
<script type="module" src="/_jobs/jobs.js"></script>
|
||||
<desk-jobs></desk-jobs>
|
||||
"""
|
||||
|
||||
|
||||
def create_app(config_path: str | None = None) -> FastAPI:
|
||||
config = config_mod.load(config_path)
|
||||
# `Store` first, so the file is at the current schema before the queue
|
||||
# adds its own table to it.
|
||||
store = Store(config.database)
|
||||
app = FastAPI(title="roledesk", docs_url="/api/docs")
|
||||
connect(app, Jobs(config.database), config, store,
|
||||
workers=config.workers.model_dump(),
|
||||
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=config.path or "")
|
||||
app.state.config = config
|
||||
app.state.store = store
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
def index() -> str:
|
||||
return INDEX
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def serve(host: str, port: int, config_path: str | None) -> int:
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(create_app(config_path), host=host, port=port)
|
||||
return 0
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
# The default config. See config.example.yaml for what each key means.
|
||||
database: roledesk.db
|
||||
plugin_paths:
|
||||
- plugins
|
||||
workers:
|
||||
llm: 4
|
||||
http: 8
|
||||
cpu: 1
|
||||
extract:
|
||||
enabled: false
|
||||
model: claude-haiku-4-5
|
||||
sources: {}
|
||||
@@ -0,0 +1,153 @@
|
||||
"""The package boundaries, held as a test rather than as a convention.
|
||||
|
||||
Adapted from news/tests/test_architecture.py. One tree of our own, `roledesk`,
|
||||
and two external ones, `desk` and `pipeline`, imported from ../news.
|
||||
|
||||
1. **Direction.** Within `roledesk`, a package imports only from packages below
|
||||
it in `LAYERS`. `desk` and `pipeline` may be imported by any of them.
|
||||
2. **Doors.** From outside a package, only `<tree>.<package>` is imported, and
|
||||
only names it lists in `__all__`. `desk.tools` has no door of its own, so
|
||||
there the door is the tool: `desk.tools.<tool>`, one level, no deeper.
|
||||
3. **Plugins.** A file under `plugins/` imports `desk` and nothing else from
|
||||
these trees. A plugin is app-agnostic; it never sees `roledesk`.
|
||||
|
||||
Tests are held to none of this.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import importlib
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
#: Bottom to top. `cli` is the entry point and may reach anything.
|
||||
LAYERS = ["models", "config", "storage", "jobs", "web", "cli"]
|
||||
APP = "roledesk"
|
||||
EXTERNAL = {"desk", "pipeline"}
|
||||
TREES = {APP} | EXTERNAL
|
||||
|
||||
|
||||
def _is_module(name: str) -> bool:
|
||||
try:
|
||||
return importlib.util.find_spec(name) is not None
|
||||
except ModuleNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
def _imports(source: str) -> list[tuple[int, str, list[str]]]:
|
||||
"""Every import of one of `TREES`: (line, module, names). `from x import y`
|
||||
where `x.y` is a module is folded into `x.y` with no names, since that is
|
||||
what it reaches for."""
|
||||
found: list[tuple[int, str, list[str]]] = []
|
||||
for node in ast.walk(ast.parse(source)):
|
||||
if isinstance(node, ast.Import):
|
||||
found += [(node.lineno, a.name, []) for a in node.names
|
||||
if a.name.split(".")[0] in TREES]
|
||||
elif isinstance(node, ast.ImportFrom) and node.level == 0 \
|
||||
and (module := node.module or "").split(".")[0] in TREES:
|
||||
names = []
|
||||
for a in node.names:
|
||||
if _is_module(f"{module}.{a.name}"):
|
||||
found.append((node.lineno, f"{module}.{a.name}", []))
|
||||
else:
|
||||
names.append(a.name)
|
||||
if names:
|
||||
found.append((node.lineno, module, names))
|
||||
return found
|
||||
|
||||
|
||||
def _door_violation(module: str, names: list[str]) -> str | None:
|
||||
"""Why this import is not through a door, or None if it is."""
|
||||
parts = module.split(".")
|
||||
depth = 3 if parts[:2] == ["desk", "tools"] else 2
|
||||
if len(parts) > depth:
|
||||
return f"reaches past {'.'.join(parts[:depth])}"
|
||||
exported = getattr(importlib.import_module(module), "__all__", None)
|
||||
hidden = [n for n in names if exported is not None and n not in exported]
|
||||
return f"{', '.join(hidden)} is not in {module}.__all__" if hidden else None
|
||||
|
||||
|
||||
def _package(path: Path) -> str:
|
||||
"""`roledesk/storage/store.py` -> "storage"; `roledesk/cli.py` -> "cli";
|
||||
the tree's own `__init__.py` -> "" (a door, exempt from direction)."""
|
||||
rel = path.relative_to(ROOT).parts
|
||||
name = rel[1].removesuffix(".py") if len(rel) > 1 else ""
|
||||
return "" if name == "__init__" else name
|
||||
|
||||
|
||||
APP_FILES = sorted(p for p in (ROOT / APP).rglob("*.py") if "__pycache__" not in p.parts)
|
||||
PLUGIN_FILES = sorted((ROOT / "plugins").glob("*.py"))
|
||||
|
||||
|
||||
def _outside(here: str, source: str):
|
||||
"""Imports in a file of package `here` that cross a package boundary."""
|
||||
for line, module, names in _imports(source):
|
||||
parts = module.split(".")
|
||||
if parts[0] == APP and len(parts) > 1 and parts[1] == here:
|
||||
continue
|
||||
yield line, module, names
|
||||
|
||||
|
||||
def direction_violations(here: str, source: str) -> list[str]:
|
||||
wrong = []
|
||||
for line, module, _ in _outside(here, source):
|
||||
parts = module.split(".")
|
||||
if parts[0] != APP or not here or len(parts) < 2 or parts[1] not in LAYERS:
|
||||
continue
|
||||
if LAYERS.index(parts[1]) >= LAYERS.index(here):
|
||||
wrong.append(f"line {line}: {module} is not below roledesk.{here}")
|
||||
return wrong
|
||||
|
||||
|
||||
def plugin_violations(source: str) -> list[str]:
|
||||
wrong = []
|
||||
for line, module, names in _imports(source):
|
||||
if module.split(".")[0] != "desk":
|
||||
wrong.append(f"line {line}: {module} (a plugin imports desk only)")
|
||||
elif reason := _door_violation(module, names):
|
||||
wrong.append(f"line {line}: {module} ({reason})")
|
||||
return wrong
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", APP_FILES, ids=lambda p: str(p.relative_to(ROOT)))
|
||||
def test_a_package_imports_only_from_below_it(path: Path) -> None:
|
||||
wrong = direction_violations(_package(path), path.read_text(encoding="utf-8"))
|
||||
assert not wrong, "\n".join(wrong)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", APP_FILES, ids=lambda p: str(p.relative_to(ROOT)))
|
||||
def test_a_package_is_reached_only_through_its_door(path: Path) -> None:
|
||||
source = path.read_text(encoding="utf-8")
|
||||
wrong = [f"line {line}: {reason}"
|
||||
for line, module, names in _outside(_package(path), source)
|
||||
if (reason := _door_violation(module, names))]
|
||||
assert not wrong, "\n".join(wrong)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", PLUGIN_FILES, ids=lambda p: p.name)
|
||||
def test_a_plugin_knows_only_desk(path: Path) -> None:
|
||||
wrong = plugin_violations(path.read_text(encoding="utf-8"))
|
||||
assert not wrong, "\n".join(wrong)
|
||||
|
||||
|
||||
def test_the_rules_catch_what_they_are_for() -> None:
|
||||
"""The checker itself, on sources that break each rule."""
|
||||
assert plugin_violations("from roledesk.storage import Store")
|
||||
assert plugin_violations("from pipeline import job")
|
||||
assert plugin_violations("from desk.sources.base import Source")
|
||||
assert not plugin_violations("from desk.sources import Source, Item")
|
||||
|
||||
assert direction_violations("storage", "from roledesk.web import create_app")
|
||||
assert direction_violations("storage", "from roledesk import jobs")
|
||||
assert not direction_violations("jobs", "from roledesk.storage import Store")
|
||||
assert not direction_violations("models", "from pipeline import job")
|
||||
|
||||
assert _door_violation("roledesk.storage.store", ["Store"])
|
||||
assert _door_violation("desk.tools.video.marks", [])
|
||||
assert _door_violation("pipeline", ["not_exported_name"])
|
||||
assert not _door_violation("pipeline.web", ["connect"])
|
||||
Reference in new issue
Block a user