Wire the app like the boilerplate: door, queue, schedules, eight tabs
app/main.py assembles it in the boilerplate's order, auth.connect first, then the queue with the scheduler and the pages. TABS is Roles, Applications, Agents, Companies, Jobs, Schedules, Messages and Account: the alpha page's three views become templates on the boilerplate's base.html, driven by static/roles.js (the old app.js) through api() and toasts.js, over the unchanged JSON routes. Companies lists every board with its open roles and the affinity control. A 422 is one sentence, so a toast never reads [object Object]. Config takes the boilerplate's shape: APP_CONFIG or config.yaml, a missing file is the defaults, an unknown key is an error. roles.yaml is gone; config.yaml is the committed default and carries the starter agents. pull is now a job that asks for the pull cascade, because a schedule enqueues a job kind; its default schedule, every 6h, is declared beside it in app/jobs/cascade.py. pull_batch's targets are keyword-only so the Jobs tab offers no batch button. `app worker` is the boilerplate's; `app job` still drains a queue of its own and fires no schedules. test_coupling holds pipeline, auth and desk apart (desk may import pipeline) and plugins to desk; test_architecture keeps app's layers, now with main. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
24508dd6fc
commit
59a301c351
42 files changed
+2407
-998
No files matched your search
+10
-6
@@ -1,15 +1,18 @@
|
||||
"""The work roledesk queues. Importing this package registers every job and
|
||||
every task in `pipeline.TASKS`.
|
||||
"""The work roledesk queues. Importing this package registers every job, every
|
||||
task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
|
||||
|
||||
pull.py read every configured plugin and store the roles it yields
|
||||
extract.py title and description into the gate columns, on Haiku
|
||||
embed.py one vector per open role, for fit
|
||||
score.py every open role scored for every enabled search agent
|
||||
cascade.py pull → extract → embed → score, and what `TASKS` offers
|
||||
cascade.py pull → extract → embed → score, what `TASKS` offers, the
|
||||
`pull` job a schedule fires, and that schedule
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.jobs import cascade, embed, extract, pull, score
|
||||
from app.jobs.score import seed_agents
|
||||
|
||||
@@ -17,9 +20,10 @@ from app.jobs.score import seed_agents
|
||||
BOOTSTRAP = "app.jobs:bootstrap"
|
||||
|
||||
|
||||
def bootstrap(config_path: str = "") -> tuple[object, object]:
|
||||
"""roledesk's `(config, store)` from the config file alone, for a job run
|
||||
in its own process or by `python -m pipeline.worker`."""
|
||||
def bootstrap(config_path: str = "") -> tuple[Any, Any]:
|
||||
"""`(config, store)` from the config file alone, for a job run in its own
|
||||
process or by `python -m pipeline.worker`. Importing this module is what
|
||||
fills `JOBS`, which is why the worker is handed a name rather than a queue."""
|
||||
from app import config as config_mod
|
||||
from app.storage import Store
|
||||
|
||||
|
||||
+36
-5
@@ -15,6 +15,10 @@ too, since embedding does not read its columns and the order costs nothing.
|
||||
A batch that failed, or a stage somebody killed, does not stop the chain: the
|
||||
work comes round on the next pull, and a rescore of what did arrive is worth
|
||||
having either way.
|
||||
|
||||
`pull` is a job as well as a cascade, because a schedule enqueues a job kind
|
||||
and a cascade is started by one: the `pull` job's only work is to ask for the
|
||||
chain. Its default schedule, `every 6h`, is declared here beside it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -24,9 +28,9 @@ from dataclasses import replace
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
from desk.tools import agent
|
||||
from pipeline import TASKS, Tally, Task
|
||||
from app.jobs import embed, extract, pull, score # noqa: F401 -- registers the jobs
|
||||
from desk.tools import agent
|
||||
from pipeline import NULL, TASKS, RunContext, Tally, Task, declare, job
|
||||
|
||||
STEP = "app.jobs.cascade:step"
|
||||
|
||||
@@ -98,8 +102,35 @@ def step(config: Any, store: Any, tally: Tally, payload: dict[str, Any], *,
|
||||
return out
|
||||
|
||||
|
||||
#: What `app job <name>` and the Jobs view offer. `pull` is the whole
|
||||
#: chain; the others are one stage each, on their own.
|
||||
TASKS["pull"] = partial(task, "pull", cascade="pull")
|
||||
def chain(name: str) -> Task:
|
||||
"""The first stage of cascade `name`, carrying `step`, so the rest follow."""
|
||||
return task(CASCADES[name][0], cascade=name)
|
||||
|
||||
|
||||
@job("pull", "Pulling every board", resource="http")
|
||||
def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""Ask for the pull cascade once this job is done. Its own work is nil.
|
||||
|
||||
Under `NULL` there is no queue to take the chain, so this raises rather
|
||||
than report a pull that never happened; `app job pull` starts the chain
|
||||
on a queue of its own instead of calling this.
|
||||
"""
|
||||
if not context.task(chain("pull")):
|
||||
raise RuntimeError("no queue to start the pull cascade on")
|
||||
return {"started": " > ".join(CASCADES["pull"])}
|
||||
|
||||
|
||||
#: What `app job <name>` and the Jobs tab offer besides `pull`: one stage
|
||||
#: each, on its own. `pull` is not a task here, since it is a job: a name that
|
||||
#: is both would be two buttons on the Jobs tab.
|
||||
for _name in ("extract", "embed", "score"):
|
||||
TASKS[_name] = partial(task, _name)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- schedules
|
||||
|
||||
# The default, next to the job it fires. A starting point rather than a
|
||||
# setting: a cadence edited or switched off on the Schedules tab survives
|
||||
# every restart (see pipeline/schedule.py).
|
||||
declare("pull", "pull", "every 6h",
|
||||
label="Pull every board, then extract, embed and score")
|
||||
+8
-4
@@ -28,7 +28,7 @@ from app.storage import Store
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
#: As many boards in the air per job as there are `http` workers by default
|
||||
#: (see `roles.yaml`), so one pull keeps the pool busy without one job
|
||||
#: (see `config.yaml`), so one pull keeps the pool busy without one job
|
||||
#: holding all 190-odd boards' worth of retries against a single attempt.
|
||||
BATCH_SIZE = 10
|
||||
|
||||
@@ -52,10 +52,14 @@ def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
|
||||
|
||||
|
||||
@job("pull_batch", "Reading boards", resource="http")
|
||||
def run_batch(config: Config, store: Store, targets: list[str], *,
|
||||
context: RunContext = NULL) -> dict[str, Any]:
|
||||
def run_batch(config: Config, store: Store, *, targets: list[str],
|
||||
context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""Fetch this batch's boards and `upsert_role` every item they list.
|
||||
|
||||
`targets` is keyword-only without a default, which is what makes it a
|
||||
required payload key to the queue: the Jobs tab offers no button for a
|
||||
batch that could only fail without one.
|
||||
|
||||
Builds the `ats` plugin scoped to just these targets — `Source.targets()`
|
||||
reads `options[subscription_field]`, so handing it only this batch's
|
||||
entries is what keeps one job from re-fetching every other job's boards.
|
||||
@@ -105,6 +109,6 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
|
||||
def pull_task() -> Task:
|
||||
"""The pull on its own. `app.jobs.cascade` makes it the first stage
|
||||
of the chain that `TASKS["pull"]` starts."""
|
||||
of the chain the `pull` job starts."""
|
||||
return Task(kind="pull_batch", finder=_boards, name="pull",
|
||||
label="Reading every board", settle=SETTLE)
|
||||
+2
-2
@@ -197,7 +197,7 @@ def run_score(config: Config, store: Store, *,
|
||||
# ------------------------------------------------------------------ seeding
|
||||
|
||||
def seed_agents(config: Config, store: Store) -> list[str]:
|
||||
"""Import `roles.yaml`'s `agents:` into an empty table, once. The state
|
||||
"""Import `config.yaml`'s `agents:` into an empty table, once. The state
|
||||
marker is what makes it once: deleting every agent afterwards is a
|
||||
choice, and a restart must not undo it."""
|
||||
if store.state.get(SEEDED) or store.agents():
|
||||
@@ -213,5 +213,5 @@ def seed_agents(config: Config, store: Store) -> list[str]:
|
||||
added.append(seed.name)
|
||||
if added:
|
||||
store.state[SEEDED] = store.now()
|
||||
log.info("imported %d agents from %s", len(added), config.path or "roles.yaml")
|
||||
log.info("imported %d agents from %s", len(added), config.path or "the defaults")
|
||||
return added
|
||||
Reference in new issue
Block a user