Wire the app like the boilerplate: door, queue, schedules, eight tabs

app/main.py assembles it in the boilerplate's order, auth.connect first,
then the queue with the scheduler and the pages. TABS is Roles,
Applications, Agents, Companies, Jobs, Schedules, Messages and Account:
the alpha page's three views become templates on the boilerplate's
base.html, driven by static/roles.js (the old app.js) through api() and
toasts.js, over the unchanged JSON routes. Companies lists every board with
its open roles and the affinity control. A 422 is one sentence, so a toast
never reads [object Object].

Config takes the boilerplate's shape: APP_CONFIG or config.yaml, a missing
file is the defaults, an unknown key is an error. roles.yaml is gone;
config.yaml is the committed default and carries the starter agents.

pull is now a job that asks for the pull cascade, because a schedule
enqueues a job kind; its default schedule, every 6h, is declared beside it
in app/jobs/cascade.py. pull_batch's targets are keyword-only so the Jobs
tab offers no batch button. `app worker` is the boilerplate's; `app job`
still drains a queue of its own and fires no schedules.

test_coupling holds pipeline, auth and desk apart (desk may import
pipeline) and plugins to desk; test_architecture keeps app's layers, now
with main.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-23 12:48:12 +02:00
1 parent 24508dd6fc
commit 59a301c351
42 files changed
+2407 -998

No files matched your search

+105 -26
View File
@@ -1,13 +1,28 @@
"""`app serve | job pull|extract|embed|score | companies seed|add|check | db init`."""
"""The command line.
app serve [--host H] [--port P] [--reload]
app job pull|extract|embed|score [--payload JSON]
app worker [--workers http=8,cpu=1] [--until-idle]
app companies seed|add|check
app db init
`app job` queues the work on a queue of its own and drains it here, so a
cascade such as `pull` runs every stage it starts before the command returns,
and prints one line per stage. `app worker` is the boilerplate's: the same
queue with no web app, and the schedules fired from it.
`python -m app` is the same thing, for a checkout with no console script.
"""
from __future__ import annotations
import argparse
import json
import logging
import os
import sys
from pathlib import Path
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
from app import config as config_mod
@@ -26,18 +41,27 @@ if str(_ROOT) not in sys.path:
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="app",
description="A job-search engine on desk.")
parser.add_argument("-c", "--config", default=None, help="path to roles.yaml")
parser = argparse.ArgumentParser(
prog="app", description="roledesk: a job-search engine on desk and the pipeline queue.")
parser.add_argument("-c", "--config", default=None,
help="the YAML file to read (default: $APP_CONFIG, then config.yaml)")
parser.add_argument("-v", "--verbose", action="store_true")
sub = parser.add_subparsers(dest="command", required=True)
serve = sub.add_parser("serve", help="run the web app")
serve.add_argument("--host", default="127.0.0.1")
serve.add_argument("--port", type=int, default=8790)
serve.add_argument("--host", default="", help="override config server.host")
serve.add_argument("--port", type=int, default=0, help="override config server.port")
serve.add_argument("--reload", action="store_true", help="restart on a source change")
job = sub.add_parser("job", help="queue a job or task and drain the queue once")
job.add_argument("kind", choices=["pull", "extract", "embed", "score"])
job = sub.add_parser("job", help="queue a job, task or cascade and drain the queue")
job.add_argument("kind", help="pull, extract, embed, score, or any registered job")
job.add_argument("--payload", default="{}",
help="JSON spread over a job handler's keyword arguments")
worker = sub.add_parser("worker", help="drain the job table with no web app")
worker.add_argument("--workers", default="", help="resource=count,... e.g. cpu=2")
worker.add_argument("--until-idle", action="store_true",
help="run what is queued, then exit")
companies = sub.add_parser("companies", help="the target company list")
companies_sub = companies.add_subparsers(dest="action", required=True)
@@ -60,52 +84,89 @@ def main(argv: list[str] | None = None) -> int:
format="%(asctime)s %(levelname)s %(name)s: %(message)s")
try:
if args.command == "serve":
from app.web import serve
config_mod.load(args.config) # a config error here, not in uvicorn
return serve(args.host, args.port, args.config)
return serve(args)
if args.command == "job":
return run_job(args)
if args.command == "worker":
return run_worker(args)
if args.command == "companies":
return run_companies(args)
return init_db(args)
except FileNotFoundError as exc:
except (FileNotFoundError, config_mod.ConfigError) as exc:
print(exc, file=sys.stderr)
return 1
def serve(args: argparse.Namespace) -> int:
import uvicorn
config = config_mod.load(args.config) # a config error here, not in uvicorn
host = args.host or config.server.host
port = args.port or config.server.port
level = str(config.log_level).lower()
if args.reload:
# The reloader builds the app in a fresh process from an import
# string, and that process reads the config from $APP_CONFIG.
if args.config:
os.environ["APP_CONFIG"] = str(args.config)
uvicorn.run("app.main:app", host=host, port=port, reload=True, log_level=level)
return 0
from app.main import create_app
uvicorn.run(create_app(config), host=host, port=port, log_level=level)
return 0
def run_job(args: argparse.Namespace) -> int:
"""Queue `kind` — a job, or a task from `pipeline.TASKS` such as `pull` —
then run everything queued until the queue is idle, and print how it
ended: one line per task, from each task's final job. `pull` is a
cascade, so it prints a line for every stage it went through."""
from pipeline import FINAL, JobQueue, Jobs, TASKS
"""Queue `kind` (a cascade such as `pull`, a task from `pipeline.TASKS`,
or a job), run everything queued until the queue is idle, and print how
it ended: one line per task, from each task's final job. `pull` is a
cascade, so it prints a line for every stage it went through.
The queue fires no schedules: the command asked for one piece of work."""
from app import jobs
from app.storage import Store
from pipeline import FINAL, JOBS, TASKS, JobQueue, Jobs
cascade = jobs.cascade
try:
payload: dict[str, Any] = json.loads(args.payload or "{}")
except json.JSONDecodeError as exc:
print(f"--payload is not JSON: {exc}", file=sys.stderr)
return 2
chained = args.kind in cascade.CASCADES
if not chained and args.kind not in TASKS and args.kind not in JOBS:
known = sorted(set(JOBS) | set(TASKS))
print(f"no job kind {args.kind!r}. Have: {', '.join(known)}", file=sys.stderr)
return 2
config = config_mod.load(args.config)
store = Store(config.database)
queue = JobQueue(Jobs(config.database), config, store,
queue = JobQueue(Jobs(str(config.database)), config, store,
workers=config.workers.model_dump(),
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=config.path)
if args.kind in TASKS:
task_id = queue.start_task(TASKS[args.kind]())
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=str(config.path or ""),
tz=config.timezone, schedules=False)
if chained or args.kind not in JOBS:
task_id = queue.start_task(cascade.chain(args.kind) if chained
else TASKS[args.kind]())
queue.drain()
started = queue.jobs.of_task(task_id)
if not started:
print(json.dumps({"task": args.kind, "result": "nothing to do"}))
return 0
cascade = queue.jobs.of_cascade(started[0].cascade_id)
finals = [j for j in cascade if j.kind == FINAL]
whole = queue.jobs.of_cascade(started[0].cascade_id)
finals = [j for j in whole if j.kind == FINAL]
for final in finals:
# A task's own jobs say what the work did; the final job only
# tallies how they ended. Up to five, so a pull of twenty board
# batches stays one readable line.
work = [j.result for j in cascade if j.task_id == final.task_id
work = [j.result for j in whole if j.task_id == final.task_id
and j.kind != FINAL][:5]
print(json.dumps({"id": final.id, "state": final.state,
"result": final.result, "work": work,
"error": final.error}))
return 0 if all(f.state == "done" for f in finals) else 1
queued = queue.enqueue(args.kind)
queued = queue.enqueue(args.kind, payload)
queue.drain()
if queued is not None:
done = queue.jobs.get(queued.id)
@@ -116,6 +177,24 @@ def run_job(args: argparse.Namespace) -> int:
return 0
def run_worker(args: argparse.Namespace) -> int:
"""`python -m pipeline.worker` with this app's settings filled in: the
bootstrap, the pools from `workers:`, and the timezone, without which a
`daily at 06:00` schedule would run at 06:00 on the machine's clock."""
from app.jobs import BOOTSTRAP
from pipeline.worker import main as worker_main
config = config_mod.load(args.config)
argv = ["--db", str(config.database), "--bootstrap", BOOTSTRAP,
"--bootstrap-arg", str(args.config or config.path or ""),
"--timezone", str(config.timezone)]
argv += ["--workers", args.workers or
",".join(f"{name}={n}" for name, n in config.workers.model_dump().items())]
if args.until_idle:
argv.append("--until-idle")
return worker_main(argv)
def run_companies(args: argparse.Namespace) -> int:
from app.storage import Store