Skeleton: roledesk on desk and pipeline from ../news

Package layers models, config, storage, jobs, web, cli, held by
tests/test_architecture.py with desk and pipeline as external trees and
plugins/ limited to desk. roles.yaml is the default config, roledesk.db
its own file, port 8790. create_app mounts pipeline.web.connect over
Jobs(config.database); `roledesk job pull` drains a no-op pull and
`roledesk db init` creates the file.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-23 10:59:17 +02:00
commit 36dbcbb29e
20 files changed
+2320

No files matched your search

+18
View File
@@ -0,0 +1,18 @@
{
"version": "0.0.1",
"configurations": [
{
"name": "roledesk",
"runtimeExecutable": "uv",
"runtimeArgs": [
"run",
"roledesk",
"serve",
"--port",
"8790"
],
"port": 8790,
"autoPort": false
}
]
}
+20
View File
@@ -0,0 +1,20 @@
# Python
__pycache__/
*.py[cod]
*.egg-info/
build/
dist/
# Environments
.venv/
# Local state
roledesk.db*
# Tooling caches
.pytest_cache/
.ruff_cache/
.mypy_cache/
# A throwaway config and database for checking the app by hand
.scratch/
+30
View File
@@ -0,0 +1,30 @@
# Rule:
Committed in main, branch deleted means the work is done. Having code uncommitted or committed in a branch or worktree means you failed to deliver. Pushing is not a thing we really care about.
# Rule:
MDs dont have hard wraps. IF you come across one make it soft wrapping
# Rule:
In documentations so not write bloat like:
> The obvious objection, answered before it's asked. "They just open a new session." Yes. **That's friction, not a wall, and it's meant to be.**
Instead, write more concise, more technical English, e.g. :
> One objection is that they just open a new session. Yet, this friction is by design because ...
Bad Example:
> **Could we verify them first to save the human a step?** Only if the receiving system will accept our verification. If it re-authenticates anyway we've added a step to save none — worth asking in a real deployment, not worth assuming.
Good example:
> We should only verify them first, if the receiving system will accept our verification? Otherwise it double work for the customer.
# Rule:
`desk` and `pipeline` are imported from ../news and never edited here.
+29
View File
@@ -0,0 +1,29 @@
# roledesk configuration. `roles.yaml` is the one the app reads by default
# (or $ROLEDESK_CONFIG, or `roledesk -c path`). Every key here is optional.
# The SQLite file: roles, companies, subscriptions, applications, and the
# job queue's own table. Not shared with newsdesk.
database: roledesk.db
# Any .py file in these directories defining a desk `Source` subclass is a
# plugin. A plugin imports `desk` only.
plugin_paths:
- plugins
# Workers per resource class. `http` is wide because a fetch is mostly waiting.
workers:
llm: 4
http: 8
cpu: 1
# Who answers when a job needs a model. See desk/tools/agent.py in ../news.
agent:
backend: auto # auto | api | claude_code | off
# Title and description in, gate columns out. Off until the extract task lands.
extract:
enabled: false
model: claude-haiku-4-5
# One block per plugin. Empty until the ats plugin lands.
sources: {}
View File
Whitespace-only changes.
+90
View File
@@ -0,0 +1,90 @@
[project]
name = "roledesk"
version = "0.1.0"
description = "A job-search engine on the desk plugin contract and the pipeline queue."
requires-python = ">=3.14"
dependencies = [
# `desk` and `pipeline` are packaged by the news repo as part of the
# newsdesk distribution. Imported from there and never edited here.
"newsdesk",
"fastapi>=0.110",
"pydantic>=2.6",
"pyyaml>=6.0",
"uvicorn[standard]>=0.29",
]
[project.scripts]
roledesk = "roledesk.cli:main"
[build-system]
requires = ["setuptools>=68"]
build-backend = "setuptools.build_meta"
[tool.setuptools.packages.find]
include = ["roledesk*"]
[tool.uv.sources]
newsdesk = { path = "../news", editable = true }
[dependency-groups]
dev = [
"pytest>=9.1.1",
"ruff>=0.14",
"mypy>=1.18",
"types-PyYAML>=6.0",
]
# ----------------------------------------------------------------- linting
[tool.ruff]
line-length = 90
target-version = "py314"
[tool.ruff.lint]
# The same selection as news: bug-prone patterns, outdated idioms,
# comprehensions and ruff's own checks. No import sorting.
select = ["E", "F", "W", "B", "UP", "C4", "SIM", "RUF"]
ignore = [
"E501", # line length: prose comments wrap by meaning
"B008", # function call in a default argument, as FastAPI writes Depends
"SIM108", # ternary instead of if/else
]
[tool.ruff.lint.per-file-ignores]
# Pydantic models whose base comes from `desk.models`; ruff only recognises a
# pydantic subclass when it sees `BaseModel` in the same file.
"roledesk/models.py" = ["RUF012"]
# ------------------------------------------------------------ type checking
[tool.mypy]
python_version = "3.14"
files = ["roledesk", "tests"]
plugins = ["pydantic.mypy"]
# The editable install of ../news is a setuptools import hook, which mypy
# cannot follow. Pointed at the checkout instead, and told to type what it
# imports from there without reporting on code that is not ours.
mypy_path = "../news"
warn_unused_ignores = true
warn_redundant_casts = true
warn_unreachable = true
no_implicit_optional = true
check_untyped_defs = true
[[tool.mypy.overrides]]
module = ["desk.*", "pipeline.*", "newsdesk.*"]
follow_imports = "silent"
[[tool.mypy.overrides]]
# Optional extras of the news repo, only imported inside functions there.
module = ["sklearn.*", "trafilatura.*", "feedparser.*", "anthropic.*",
"claude_agent_sdk.*", "sentence_transformers.*", "webauthn.*",
"cbor2.*"]
ignore_missing_imports = true
[[tool.mypy.overrides]]
module = ["anthropic.*", "claude_agent_sdk.*", "sentence_transformers.*"]
follow_imports = "skip"
[tool.pytest.ini_options]
testpaths = ["tests"]
+14
View File
@@ -0,0 +1,14 @@
"""roledesk: a job-search engine on the `desk` plugin contract and the `pipeline` queue.
Layers, bottom to top, each importing only from those below it (see
`tests/test_architecture.py`):
models the shapes handed round, and the key functions for clustering
config `roles.yaml`, validated
storage `Store`: every SQL statement against `roledesk.db`
jobs the work this app queues
web the FastAPI app
cli the entry point
`desk` and `pipeline` come from the news repo and are imported, never edited.
"""
+84
View File
@@ -0,0 +1,84 @@
"""`roledesk serve | job pull | db init`."""
from __future__ import annotations
import argparse
import json
import logging
import sys
from roledesk import config as config_mod
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="roledesk",
description="A job-search engine on desk.")
parser.add_argument("-c", "--config", default=None, help="path to roles.yaml")
parser.add_argument("-v", "--verbose", action="store_true")
sub = parser.add_subparsers(dest="command", required=True)
serve = sub.add_parser("serve", help="run the web app")
serve.add_argument("--host", default="127.0.0.1")
serve.add_argument("--port", type=int, default=8790)
job = sub.add_parser("job", help="queue a job and drain the queue once")
job.add_argument("kind", choices=["pull"])
db = sub.add_parser("db", help="database housekeeping")
db.add_argument("action", choices=["init"])
return parser
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s")
try:
if args.command == "serve":
from roledesk.web import serve
config_mod.load(args.config) # a config error here, not in uvicorn
return serve(args.host, args.port, args.config)
if args.command == "job":
return run_job(args)
return init_db(args)
except FileNotFoundError as exc:
print(exc, file=sys.stderr)
return 1
def run_job(args: argparse.Namespace) -> int:
"""Queue `kind`, then run everything queued until the queue is idle."""
from pipeline import JobQueue, Jobs
from roledesk import jobs
from roledesk.storage import Store
config = config_mod.load(args.config)
store = Store(config.database)
queue = JobQueue(Jobs(config.database), config, store,
workers=config.workers.model_dump(),
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=config.path)
queued = queue.enqueue(args.kind)
queue.drain()
if queued is not None:
done = queue.jobs.get(queued.id)
if done is not None:
print(json.dumps({"id": done.id, "state": done.state, "result": done.result,
"error": done.error}))
return 0 if done.state == "done" else 1
return 0
def init_db(args: argparse.Namespace) -> int:
"""Create the file, or bring it to the current schema."""
from roledesk.storage import LATEST, Store
config = config_mod.load(args.config)
store = Store(config.database)
applied = store.migrations_applied
print(f"{store.path}: schema v{LATEST}"
+ (f", applied {applied}" if applied else ""))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+71
View File
@@ -0,0 +1,71 @@
"""Config loading: one YAML file, `roles.yaml`, validated into a model.
A typo in a key is an error naming the key, because `Section` forbids extras.
`sources` stays a dict: one block per plugin, whose keys only the plugin knows.
"""
from __future__ import annotations
import os
from pathlib import Path
from typing import Any
import yaml
from pydantic import Field, ValidationError
from desk.config import AgentConfig, ModelTaskConfig, Section, WorkersConfig
DEFAULT_PATH = Path(os.environ.get("ROLEDESK_CONFIG", "roles.yaml"))
class ExtractConfig(ModelTaskConfig):
"""Turning a posting's title and text into the gate columns. Off until the
`extract` task exists; the section is here so the file's shape is settled."""
enabled: bool = False
model: str = "claude-haiku-4-5"
class Config(Section):
"""The whole file."""
database: str = "roledesk.db"
plugin_paths: list[str] = Field(default_factory=lambda: ["plugins"])
workers: WorkersConfig = Field(default_factory=WorkersConfig)
agent: AgentConfig = Field(default_factory=AgentConfig)
extract: ExtractConfig = Field(default_factory=ExtractConfig)
#: One block per plugin: `enabled`, and whatever that plugin reads.
sources: dict[str, Any] = Field(default_factory=dict)
#: Where this was loaded from. Never written back into the file.
path: str = Field("", exclude=True)
class ConfigError(FileNotFoundError):
"""A config that exists but does not validate. A FileNotFoundError so the
CLI reports both kinds of unusable config the same way: as a message."""
def load(path: str | Path | None = None) -> Config:
p = Path(path or DEFAULT_PATH).expanduser()
if not p.exists():
raise FileNotFoundError(
f"No config at {p}. Copy config.example.yaml to {p} and edit it.")
with p.open(encoding="utf-8") as fh:
raw = yaml.safe_load(fh) or {}
try:
config = Config(**raw, path=str(p))
except ValidationError as exc:
raise ConfigError(f"{p}:\n{_explain(exc)}") from None
config.agent.credentials_database = config.database
return config
def _explain(exc: ValidationError) -> str:
lines = []
for err in exc.errors():
where = ".".join(str(p) for p in err["loc"]) or "(top level)"
detail = err["msg"]
if err["type"] == "extra_forbidden":
detail = "unknown setting, check the spelling against config.example.yaml"
lines.append(f" {where}: {detail}")
return "\n".join(lines)
+24
View File
@@ -0,0 +1,24 @@
"""The work roledesk queues. Importing this package registers every job.
pull.py read every configured plugin and store the roles it yields
"""
from __future__ import annotations
from roledesk.jobs import pull
#: What `JobQueue(bootstrap=...)` and `pipeline.worker --bootstrap` are given.
BOOTSTRAP = "roledesk.jobs:bootstrap"
def bootstrap(config_path: str = "") -> tuple[object, object]:
"""roledesk's `(config, store)` from the config file alone, for a job run
in its own process or by `python -m pipeline.worker`."""
from roledesk import config as config_mod
from roledesk.storage import Store
config = config_mod.load(config_path or None)
return config, Store(config.database)
__all__ = ["BOOTSTRAP", "bootstrap", "pull"]
+18
View File
@@ -0,0 +1,18 @@
"""`pull`: read every configured plugin. A no-op until the first plugin lands."""
from __future__ import annotations
import logging
from typing import Any
from pipeline import NULL, RunContext, job
from roledesk.config import Config
from roledesk.storage import Store
log = logging.getLogger(__name__)
@job("pull", "Reading every board", resource="http", solo=True)
def run(config: Config, store: Store, *, context: RunContext = NULL) -> dict[str, Any]:
log.info("pull: no plugin is installed yet")
return {"sources": 0, "seen": 0, "new": 0}
+3
View File
@@ -0,0 +1,3 @@
"""The shapes roledesk hands round. Filled in with the storage layer."""
from __future__ import annotations
+16
View File
@@ -0,0 +1,16 @@
"""Where things are kept. One door: `from roledesk.storage import Store`.
schema.py the tables, and the numbered migrations that bring an older
file to the current shape
store.py `Store`, every SQL statement in the application
Written for roledesk rather than imported from newsdesk: `desk` has no storage
package on purpose, and the connection plumbing here is a deliberate copy.
"""
from __future__ import annotations
from roledesk.storage.schema import LATEST
from roledesk.storage.store import Store
__all__ = ["LATEST", "Store"]
+131
View File
@@ -0,0 +1,131 @@
"""The tables, and how a database file gets from an old shape to this one.
`SCHEMA` is the current shape. A new file is created from it and stamped with
`LATEST`, so it never runs a migration. `MIGRATIONS` is how an existing file
catches up: ordered, numbered, applied once each, tracked in SQLite's own
`PRAGMA user_version`.
Adding a column means editing both: the DDL in `SCHEMA` for new files, and an
appended migration for old ones. Never renumber or edit a migration that has
shipped.
"""
from __future__ import annotations
import logging
import sqlite3
from collections.abc import Callable
log = logging.getLogger(__name__)
SCHEMA = """
-- The string map a desk plugin is handed as `self.state`. One namespace shared
-- by every plugin, so a plugin prefixes its keys.
CREATE TABLE IF NOT EXISTS state (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
"""
#: Applied after `SCHEMA` or the migrations, every open. An index may name a
#: column only a migration adds, which is why these are kept apart.
INDEXES = """
"""
#: The table whose absence marks a file that has never been a roledesk
#: database, and is therefore created from `SCHEMA` rather than migrated.
MARKER = "state"
Step = str | Callable[[sqlite3.Connection], None]
MIGRATIONS: tuple[tuple[int, str, Step], ...] = (
# Never run as a migration: a file without the marker table is created
# from `SCHEMA` and stamped. It exists so the first real migration is 2.
(1, "the first schema", ""),
)
#: What a file created from `SCHEMA` today is already at.
LATEST = MIGRATIONS[-1][0]
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])
def _stamp(conn: sqlite3.Connection, version: int) -> None:
# PRAGMA takes no bound parameters; the value is an int from MIGRATIONS.
conn.execute(f"PRAGMA user_version = {int(version)}")
def _statements(script: str) -> list[str]:
"""A script split into statements, cut where `sqlite3.complete_statement`
says one ends. Not `executescript`, which commits before it starts and
autocommits each statement, so a failing migration would be half applied."""
statements, buffer = [], ""
for line in script.splitlines(keepends=True):
buffer += line
if sqlite3.complete_statement(buffer):
statements.append(buffer.strip())
buffer = ""
if buffer.strip():
statements.append(buffer.strip())
return statements
def migrate(conn: sqlite3.Connection) -> list[int]:
"""Bring an existing database up to `LATEST`. Returns what it applied."""
current = user_version(conn)
if current > LATEST:
raise RuntimeError(
f"this database is at schema v{current}, but this roledesk only knows "
f"up to v{LATEST}. Upgrade roledesk rather than let older code write to it.")
applied = []
for version, description, step in MIGRATIONS:
if version <= current:
continue
log.info("migrating database to v%d: %s", version, description)
_apply(conn, version, step)
applied.append(version)
return applied
def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
"""One migration and its version stamp, in one explicit transaction.
Explicit because sqlite3 only opens a transaction implicitly before DML: an
`ALTER` would otherwise be durable at once while the stamp stayed behind,
and the next open would replay it into a "duplicate column" forever.
"""
started = not conn.in_transaction
if started:
conn.execute("BEGIN")
try:
if callable(step):
step(conn)
else:
for statement in _statements(step):
conn.execute(statement)
_stamp(conn, version)
except Exception:
if started:
conn.execute("ROLLBACK")
raise
if started:
conn.execute("COMMIT")
def initialise(conn: sqlite3.Connection) -> list[int]:
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
fresh = conn.execute(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (MARKER,)
).fetchone() is None
applied: list[int] = []
if fresh:
# Atomic like a migration: a file half created would otherwise carry
# the marker table and be taken for one to migrate.
_apply(conn, LATEST, SCHEMA)
else:
applied = migrate(conn)
for statement in _statements(INDEXES):
conn.execute(statement)
return applied
+103
View File
@@ -0,0 +1,103 @@
"""`Store`: every SQL statement in the application.
The connection plumbing is a copy of newsdesk's `Store`, on purpose: `desk`
has no storage package. A connection is opened per call and closed again, so
one caller's commit can never land inside another's transaction; in WAL mode
that costs a file handle.
"""
from __future__ import annotations
import sqlite3
from collections.abc import Iterator, MutableMapping
from contextlib import closing, contextmanager
from pathlib import Path
from typing import Any
from roledesk.storage import schema
class _StateMap(MutableMapping[str, str]):
"""`Store.state`: the key/value table, read and written one key at a time."""
def __init__(self, store: Store) -> None:
self._store = store
def __getitem__(self, key: str) -> str:
row = self._store._row("SELECT value FROM state WHERE key=?", (key,))
if row is None:
raise KeyError(key)
return str(row["value"])
def __setitem__(self, key: str, value: str) -> None:
self._store._write(
"""INSERT INTO state (key, value) VALUES (?,?)
ON CONFLICT(key) DO UPDATE SET value=excluded.value""",
(key, value))
def __delitem__(self, key: str) -> None:
if not self._store._write("DELETE FROM state WHERE key=?", (key,)):
raise KeyError(key)
def __iter__(self) -> Iterator[str]:
return iter([r["key"] for r in self._store._rows("SELECT key FROM state ORDER BY key")])
def __len__(self) -> int:
row = self._store._row("SELECT COUNT(*) AS n FROM state")
return int(row["n"]) if row else 0
class Store:
def __init__(self, path: str | Path) -> None:
self.path = Path(path).expanduser()
self.path.parent.mkdir(parents=True, exist_ok=True)
with self.transaction() as conn:
# Persistent: recorded in the file, inherited by every connection.
conn.execute("PRAGMA journal_mode=WAL")
#: Migration versions this open applied; empty when already current.
self.migrations_applied: list[int] = schema.initialise(conn)
# ------------------------------------------------------------ connections
def _connect(self) -> sqlite3.Connection:
"""A fresh connection. `foreign_keys` is off by default per connection;
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
`synchronous=NORMAL` is the standard WAL companion."""
conn = sqlite3.connect(self.path)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA foreign_keys=ON")
conn.execute("PRAGMA busy_timeout=30000")
conn.execute("PRAGMA synchronous=NORMAL")
return conn
@contextmanager
def transaction(self) -> Iterator[sqlite3.Connection]:
"""One connection, one transaction, closed on the way out."""
conn = self._connect()
try:
with conn:
yield conn
finally:
conn.close()
def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]:
with closing(self._connect()) as conn:
return conn.execute(sql, params).fetchall()
def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None:
with closing(self._connect()) as conn:
row: sqlite3.Row | None = conn.execute(sql, params).fetchone()
return row
def _write(self, sql: str, params: Any = ()) -> int:
"""One statement in its own transaction. Returns the rowcount."""
with self.transaction() as conn:
return conn.execute(sql, params).rowcount
# ---------------------------------------------------------------- state
@property
def state(self) -> MutableMapping[str, str]:
"""The `state` table as a mapping. What the runner hands a plugin as
`self.state`: it can keep a counter without being able to read roles."""
return _StateMap(self)
+7
View File
@@ -0,0 +1,7 @@
"""The web face. One door: `create_app` and `serve`."""
from __future__ import annotations
from roledesk.web.app import create_app, serve
__all__ = ["create_app", "serve"]
+46
View File
@@ -0,0 +1,46 @@
"""The application factory. A stub page until the reader lands: the queue's
routes and socket from `pipeline.web.connect`, and a one-line index."""
from __future__ import annotations
from fastapi import FastAPI
from fastapi.responses import HTMLResponse
from pipeline import Jobs
from pipeline.web import connect
from roledesk import config as config_mod
from roledesk import jobs
from roledesk.storage import Store
INDEX = """<!doctype html>
<meta charset="utf-8"><title>roledesk</title>
<p>roledesk: no reader yet.</p>
<script type="module" src="/_jobs/jobs.js"></script>
<desk-jobs></desk-jobs>
"""
def create_app(config_path: str | None = None) -> FastAPI:
config = config_mod.load(config_path)
# `Store` first, so the file is at the current schema before the queue
# adds its own table to it.
store = Store(config.database)
app = FastAPI(title="roledesk", docs_url="/api/docs")
connect(app, Jobs(config.database), config, store,
workers=config.workers.model_dump(),
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=config.path or "")
app.state.config = config
app.state.store = store
@app.get("/", response_class=HTMLResponse)
def index() -> str:
return INDEX
return app
def serve(host: str, port: int, config_path: str | None) -> int:
import uvicorn
uvicorn.run(create_app(config_path), host=host, port=port)
return 0
+12
View File
@@ -0,0 +1,12 @@
# The default config. See config.example.yaml for what each key means.
database: roledesk.db
plugin_paths:
- plugins
workers:
llm: 4
http: 8
cpu: 1
extract:
enabled: false
model: claude-haiku-4-5
sources: {}
+153
View File
@@ -0,0 +1,153 @@
"""The package boundaries, held as a test rather than as a convention.
Adapted from news/tests/test_architecture.py. One tree of our own, `roledesk`,
and two external ones, `desk` and `pipeline`, imported from ../news.
1. **Direction.** Within `roledesk`, a package imports only from packages below
it in `LAYERS`. `desk` and `pipeline` may be imported by any of them.
2. **Doors.** From outside a package, only `<tree>.<package>` is imported, and
only names it lists in `__all__`. `desk.tools` has no door of its own, so
there the door is the tool: `desk.tools.<tool>`, one level, no deeper.
3. **Plugins.** A file under `plugins/` imports `desk` and nothing else from
these trees. A plugin is app-agnostic; it never sees `roledesk`.
Tests are held to none of this.
"""
from __future__ import annotations
import ast
import importlib
import importlib.util
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
#: Bottom to top. `cli` is the entry point and may reach anything.
LAYERS = ["models", "config", "storage", "jobs", "web", "cli"]
APP = "roledesk"
EXTERNAL = {"desk", "pipeline"}
TREES = {APP} | EXTERNAL
def _is_module(name: str) -> bool:
try:
return importlib.util.find_spec(name) is not None
except ModuleNotFoundError:
return False
def _imports(source: str) -> list[tuple[int, str, list[str]]]:
"""Every import of one of `TREES`: (line, module, names). `from x import y`
where `x.y` is a module is folded into `x.y` with no names, since that is
what it reaches for."""
found: list[tuple[int, str, list[str]]] = []
for node in ast.walk(ast.parse(source)):
if isinstance(node, ast.Import):
found += [(node.lineno, a.name, []) for a in node.names
if a.name.split(".")[0] in TREES]
elif isinstance(node, ast.ImportFrom) and node.level == 0 \
and (module := node.module or "").split(".")[0] in TREES:
names = []
for a in node.names:
if _is_module(f"{module}.{a.name}"):
found.append((node.lineno, f"{module}.{a.name}", []))
else:
names.append(a.name)
if names:
found.append((node.lineno, module, names))
return found
def _door_violation(module: str, names: list[str]) -> str | None:
"""Why this import is not through a door, or None if it is."""
parts = module.split(".")
depth = 3 if parts[:2] == ["desk", "tools"] else 2
if len(parts) > depth:
return f"reaches past {'.'.join(parts[:depth])}"
exported = getattr(importlib.import_module(module), "__all__", None)
hidden = [n for n in names if exported is not None and n not in exported]
return f"{', '.join(hidden)} is not in {module}.__all__" if hidden else None
def _package(path: Path) -> str:
"""`roledesk/storage/store.py` -> "storage"; `roledesk/cli.py` -> "cli";
the tree's own `__init__.py` -> "" (a door, exempt from direction)."""
rel = path.relative_to(ROOT).parts
name = rel[1].removesuffix(".py") if len(rel) > 1 else ""
return "" if name == "__init__" else name
APP_FILES = sorted(p for p in (ROOT / APP).rglob("*.py") if "__pycache__" not in p.parts)
PLUGIN_FILES = sorted((ROOT / "plugins").glob("*.py"))
def _outside(here: str, source: str):
"""Imports in a file of package `here` that cross a package boundary."""
for line, module, names in _imports(source):
parts = module.split(".")
if parts[0] == APP and len(parts) > 1 and parts[1] == here:
continue
yield line, module, names
def direction_violations(here: str, source: str) -> list[str]:
wrong = []
for line, module, _ in _outside(here, source):
parts = module.split(".")
if parts[0] != APP or not here or len(parts) < 2 or parts[1] not in LAYERS:
continue
if LAYERS.index(parts[1]) >= LAYERS.index(here):
wrong.append(f"line {line}: {module} is not below roledesk.{here}")
return wrong
def plugin_violations(source: str) -> list[str]:
wrong = []
for line, module, names in _imports(source):
if module.split(".")[0] != "desk":
wrong.append(f"line {line}: {module} (a plugin imports desk only)")
elif reason := _door_violation(module, names):
wrong.append(f"line {line}: {module} ({reason})")
return wrong
@pytest.mark.parametrize("path", APP_FILES, ids=lambda p: str(p.relative_to(ROOT)))
def test_a_package_imports_only_from_below_it(path: Path) -> None:
wrong = direction_violations(_package(path), path.read_text(encoding="utf-8"))
assert not wrong, "\n".join(wrong)
@pytest.mark.parametrize("path", APP_FILES, ids=lambda p: str(p.relative_to(ROOT)))
def test_a_package_is_reached_only_through_its_door(path: Path) -> None:
source = path.read_text(encoding="utf-8")
wrong = [f"line {line}: {reason}"
for line, module, names in _outside(_package(path), source)
if (reason := _door_violation(module, names))]
assert not wrong, "\n".join(wrong)
@pytest.mark.parametrize("path", PLUGIN_FILES, ids=lambda p: p.name)
def test_a_plugin_knows_only_desk(path: Path) -> None:
wrong = plugin_violations(path.read_text(encoding="utf-8"))
assert not wrong, "\n".join(wrong)
def test_the_rules_catch_what_they_are_for() -> None:
"""The checker itself, on sources that break each rule."""
assert plugin_violations("from roledesk.storage import Store")
assert plugin_violations("from pipeline import job")
assert plugin_violations("from desk.sources.base import Source")
assert not plugin_violations("from desk.sources import Source, Item")
assert direction_violations("storage", "from roledesk.web import create_app")
assert direction_violations("storage", "from roledesk import jobs")
assert not direction_violations("jobs", "from roledesk.storage import Store")
assert not direction_violations("models", "from pipeline import job")
assert _door_violation("roledesk.storage.store", ["Store"])
assert _door_violation("desk.tools.video.marks", [])
assert _door_violation("pipeline", ["not_exported_name"])
assert not _door_violation("pipeline.web", ["connect"])
Generated
+1451
View File
File diff suppressed because it is too large. Load diff