Wire the app like the boilerplate: door, queue, schedules, eight tabs
app/main.py assembles it in the boilerplate's order, auth.connect first, then the queue with the scheduler and the pages. TABS is Roles, Applications, Agents, Companies, Jobs, Schedules, Messages and Account: the alpha page's three views become templates on the boilerplate's base.html, driven by static/roles.js (the old app.js) through api() and toasts.js, over the unchanged JSON routes. Companies lists every board with its open roles and the affinity control. A 422 is one sentence, so a toast never reads [object Object]. Config takes the boilerplate's shape: APP_CONFIG or config.yaml, a missing file is the defaults, an unknown key is an error. roles.yaml is gone; config.yaml is the committed default and carries the starter agents. pull is now a job that asks for the pull cascade, because a schedule enqueues a job kind; its default schedule, every 6h, is declared beside it in app/jobs/cascade.py. pull_batch's targets are keyword-only so the Jobs tab offers no batch button. `app worker` is the boilerplate's; `app job` still drains a queue of its own and fires no schedules. test_coupling holds pipeline, auth and desk apart (desk may import pipeline) and plugins to desk; test_architecture keeps app's layers, now with main. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
24508dd6fc
commit
59a301c351
42 files changed
+2407
-998
No files matched your search
+105
-26
@@ -1,13 +1,28 @@
|
||||
"""`app serve | job pull|extract|embed|score | companies seed|add|check | db init`."""
|
||||
"""The command line.
|
||||
|
||||
app serve [--host H] [--port P] [--reload]
|
||||
app job pull|extract|embed|score [--payload JSON]
|
||||
app worker [--workers http=8,cpu=1] [--until-idle]
|
||||
app companies seed|add|check
|
||||
app db init
|
||||
|
||||
`app job` queues the work on a queue of its own and drains it here, so a
|
||||
cascade such as `pull` runs every stage it starts before the command returns,
|
||||
and prints one line per stage. `app worker` is the boilerplate's: the same
|
||||
queue with no web app, and the schedules fired from it.
|
||||
|
||||
`python -m app` is the same thing, for a checkout with no console script.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from app import config as config_mod
|
||||
|
||||
@@ -26,18 +41,27 @@ if str(_ROOT) not in sys.path:
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(prog="app",
|
||||
description="A job-search engine on desk.")
|
||||
parser.add_argument("-c", "--config", default=None, help="path to roles.yaml")
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="app", description="roledesk: a job-search engine on desk and the pipeline queue.")
|
||||
parser.add_argument("-c", "--config", default=None,
|
||||
help="the YAML file to read (default: $APP_CONFIG, then config.yaml)")
|
||||
parser.add_argument("-v", "--verbose", action="store_true")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
serve = sub.add_parser("serve", help="run the web app")
|
||||
serve.add_argument("--host", default="127.0.0.1")
|
||||
serve.add_argument("--port", type=int, default=8790)
|
||||
serve.add_argument("--host", default="", help="override config server.host")
|
||||
serve.add_argument("--port", type=int, default=0, help="override config server.port")
|
||||
serve.add_argument("--reload", action="store_true", help="restart on a source change")
|
||||
|
||||
job = sub.add_parser("job", help="queue a job or task and drain the queue once")
|
||||
job.add_argument("kind", choices=["pull", "extract", "embed", "score"])
|
||||
job = sub.add_parser("job", help="queue a job, task or cascade and drain the queue")
|
||||
job.add_argument("kind", help="pull, extract, embed, score, or any registered job")
|
||||
job.add_argument("--payload", default="{}",
|
||||
help="JSON spread over a job handler's keyword arguments")
|
||||
|
||||
worker = sub.add_parser("worker", help="drain the job table with no web app")
|
||||
worker.add_argument("--workers", default="", help="resource=count,... e.g. cpu=2")
|
||||
worker.add_argument("--until-idle", action="store_true",
|
||||
help="run what is queued, then exit")
|
||||
|
||||
companies = sub.add_parser("companies", help="the target company list")
|
||||
companies_sub = companies.add_subparsers(dest="action", required=True)
|
||||
@@ -60,52 +84,89 @@ def main(argv: list[str] | None = None) -> int:
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
try:
|
||||
if args.command == "serve":
|
||||
from app.web import serve
|
||||
config_mod.load(args.config) # a config error here, not in uvicorn
|
||||
return serve(args.host, args.port, args.config)
|
||||
return serve(args)
|
||||
if args.command == "job":
|
||||
return run_job(args)
|
||||
if args.command == "worker":
|
||||
return run_worker(args)
|
||||
if args.command == "companies":
|
||||
return run_companies(args)
|
||||
return init_db(args)
|
||||
except FileNotFoundError as exc:
|
||||
except (FileNotFoundError, config_mod.ConfigError) as exc:
|
||||
print(exc, file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def serve(args: argparse.Namespace) -> int:
|
||||
import uvicorn
|
||||
|
||||
config = config_mod.load(args.config) # a config error here, not in uvicorn
|
||||
host = args.host or config.server.host
|
||||
port = args.port or config.server.port
|
||||
level = str(config.log_level).lower()
|
||||
if args.reload:
|
||||
# The reloader builds the app in a fresh process from an import
|
||||
# string, and that process reads the config from $APP_CONFIG.
|
||||
if args.config:
|
||||
os.environ["APP_CONFIG"] = str(args.config)
|
||||
uvicorn.run("app.main:app", host=host, port=port, reload=True, log_level=level)
|
||||
return 0
|
||||
from app.main import create_app
|
||||
|
||||
uvicorn.run(create_app(config), host=host, port=port, log_level=level)
|
||||
return 0
|
||||
|
||||
|
||||
def run_job(args: argparse.Namespace) -> int:
|
||||
"""Queue `kind` — a job, or a task from `pipeline.TASKS` such as `pull` —
|
||||
then run everything queued until the queue is idle, and print how it
|
||||
ended: one line per task, from each task's final job. `pull` is a
|
||||
cascade, so it prints a line for every stage it went through."""
|
||||
from pipeline import FINAL, JobQueue, Jobs, TASKS
|
||||
"""Queue `kind` (a cascade such as `pull`, a task from `pipeline.TASKS`,
|
||||
or a job), run everything queued until the queue is idle, and print how
|
||||
it ended: one line per task, from each task's final job. `pull` is a
|
||||
cascade, so it prints a line for every stage it went through.
|
||||
|
||||
The queue fires no schedules: the command asked for one piece of work."""
|
||||
from app import jobs
|
||||
from app.storage import Store
|
||||
from pipeline import FINAL, JOBS, TASKS, JobQueue, Jobs
|
||||
|
||||
cascade = jobs.cascade
|
||||
|
||||
try:
|
||||
payload: dict[str, Any] = json.loads(args.payload or "{}")
|
||||
except json.JSONDecodeError as exc:
|
||||
print(f"--payload is not JSON: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
chained = args.kind in cascade.CASCADES
|
||||
if not chained and args.kind not in TASKS and args.kind not in JOBS:
|
||||
known = sorted(set(JOBS) | set(TASKS))
|
||||
print(f"no job kind {args.kind!r}. Have: {', '.join(known)}", file=sys.stderr)
|
||||
return 2
|
||||
config = config_mod.load(args.config)
|
||||
store = Store(config.database)
|
||||
queue = JobQueue(Jobs(config.database), config, store,
|
||||
queue = JobQueue(Jobs(str(config.database)), config, store,
|
||||
workers=config.workers.model_dump(),
|
||||
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=config.path)
|
||||
if args.kind in TASKS:
|
||||
task_id = queue.start_task(TASKS[args.kind]())
|
||||
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=str(config.path or ""),
|
||||
tz=config.timezone, schedules=False)
|
||||
if chained or args.kind not in JOBS:
|
||||
task_id = queue.start_task(cascade.chain(args.kind) if chained
|
||||
else TASKS[args.kind]())
|
||||
queue.drain()
|
||||
started = queue.jobs.of_task(task_id)
|
||||
if not started:
|
||||
print(json.dumps({"task": args.kind, "result": "nothing to do"}))
|
||||
return 0
|
||||
cascade = queue.jobs.of_cascade(started[0].cascade_id)
|
||||
finals = [j for j in cascade if j.kind == FINAL]
|
||||
whole = queue.jobs.of_cascade(started[0].cascade_id)
|
||||
finals = [j for j in whole if j.kind == FINAL]
|
||||
for final in finals:
|
||||
# A task's own jobs say what the work did; the final job only
|
||||
# tallies how they ended. Up to five, so a pull of twenty board
|
||||
# batches stays one readable line.
|
||||
work = [j.result for j in cascade if j.task_id == final.task_id
|
||||
work = [j.result for j in whole if j.task_id == final.task_id
|
||||
and j.kind != FINAL][:5]
|
||||
print(json.dumps({"id": final.id, "state": final.state,
|
||||
"result": final.result, "work": work,
|
||||
"error": final.error}))
|
||||
return 0 if all(f.state == "done" for f in finals) else 1
|
||||
queued = queue.enqueue(args.kind)
|
||||
queued = queue.enqueue(args.kind, payload)
|
||||
queue.drain()
|
||||
if queued is not None:
|
||||
done = queue.jobs.get(queued.id)
|
||||
@@ -116,6 +177,24 @@ def run_job(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def run_worker(args: argparse.Namespace) -> int:
|
||||
"""`python -m pipeline.worker` with this app's settings filled in: the
|
||||
bootstrap, the pools from `workers:`, and the timezone, without which a
|
||||
`daily at 06:00` schedule would run at 06:00 on the machine's clock."""
|
||||
from app.jobs import BOOTSTRAP
|
||||
from pipeline.worker import main as worker_main
|
||||
|
||||
config = config_mod.load(args.config)
|
||||
argv = ["--db", str(config.database), "--bootstrap", BOOTSTRAP,
|
||||
"--bootstrap-arg", str(args.config or config.path or ""),
|
||||
"--timezone", str(config.timezone)]
|
||||
argv += ["--workers", args.workers or
|
||||
",".join(f"{name}={n}" for name, n in config.workers.model_dump().items())]
|
||||
if args.until_idle:
|
||||
argv.append("--until-idle")
|
||||
return worker_main(argv)
|
||||
|
||||
|
||||
def run_companies(args: argparse.Namespace) -> int:
|
||||
from app.storage import Store
|
||||
|
||||
|
||||
Reference in new issue
Block a user