app/main.py assembles it in the boilerplate's order, auth.connect first, then the queue with the scheduler and the pages. TABS is Roles, Applications, Agents, Companies, Jobs, Schedules, Messages and Account: the alpha page's three views become templates on the boilerplate's base.html, driven by static/roles.js (the old app.js) through api() and toasts.js, over the unchanged JSON routes. Companies lists every board with its open roles and the affinity control. A 422 is one sentence, so a toast never reads [object Object]. Config takes the boilerplate's shape: APP_CONFIG or config.yaml, a missing file is the defaults, an unknown key is an error. roles.yaml is gone; config.yaml is the committed default and carries the starter agents. pull is now a job that asks for the pull cascade, because a schedule enqueues a job kind; its default schedule, every 6h, is declared beside it in app/jobs/cascade.py. pull_batch's targets are keyword-only so the Jobs tab offers no batch button. `app worker` is the boilerplate's; `app job` still drains a queue of its own and fires no schedules. test_coupling holds pipeline, auth and desk apart (desk may import pipeline) and plugins to desk; test_architecture keeps app's layers, now with main. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
115 lines
4.7 KiB
Python
115 lines
4.7 KiB
Python
"""`pull`: fetch every active `ats` board and store the roles it lists.
|
|
|
|
A task, not a single job: ten boards per `pull_batch` job on the `http`
|
|
resource pool, because a board is one HTTP round trip and that pool is sized
|
|
for exactly this. The final job's `settle` then closes every board a
|
|
complete fetch of it did not list again — see `close_boards`.
|
|
|
|
Closure is read from `self.state`, which the `ats` plugin itself stamps per
|
|
board on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`), rather
|
|
than from what each `pull_batch` job happens to return. A batch job that
|
|
raises partway through still leaves the boards it had already finished
|
|
stamped, so one bad board in a batch of ten does not cost the other nine
|
|
their closure.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
from pipeline import NULL, RunContext, Tally, Task, job
|
|
|
|
from app.config import Config
|
|
from app.storage import Store
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: As many boards in the air per job as there are `http` workers by default
|
|
#: (see `config.yaml`), so one pull keeps the pool busy without one job
|
|
#: holding all 190-odd boards' worth of retries against a single attempt.
|
|
BATCH_SIZE = 10
|
|
|
|
SETTLE = "app.jobs.pull:close_boards"
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now(UTC).isoformat(timespec="seconds")
|
|
|
|
|
|
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
|
|
"""Every fetchable `ats` subscription, ten to a batch. Stamps the run
|
|
marker `close_boards` will look for, before any batch runs — finders run
|
|
synchronously, in whoever starts the task, ahead of every job it queues."""
|
|
subs = store.subscriptions(source="ats", fetchable=True)
|
|
if not subs:
|
|
return []
|
|
store.state["ats:run"] = _now()
|
|
return [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]]}
|
|
for i in range(0, len(subs), BATCH_SIZE)]
|
|
|
|
|
|
@job("pull_batch", "Reading boards", resource="http")
|
|
def run_batch(config: Config, store: Store, *, targets: list[str],
|
|
context: RunContext = NULL) -> dict[str, Any]:
|
|
"""Fetch this batch's boards and `upsert_role` every item they list.
|
|
|
|
`targets` is keyword-only without a default, which is what makes it a
|
|
required payload key to the queue: the Jobs tab offers no button for a
|
|
batch that could only fail without one.
|
|
|
|
Builds the `ats` plugin scoped to just these targets — `Source.targets()`
|
|
reads `options[subscription_field]`, so handing it only this batch's
|
|
entries is what keeps one job from re-fetching every other job's boards.
|
|
"""
|
|
from plugins.ats import ATSSource
|
|
|
|
subs = {s.target: s for s in store.subscriptions(source="ats") if s.target in targets}
|
|
options = dict(config.sources.get("ats") or {})
|
|
options["boards"] = [{"url": target, "label": subs[target].company or subs[target].label}
|
|
for target in targets if target in subs]
|
|
source = ATSSource(options)
|
|
source.state = store.state
|
|
|
|
stored = 0
|
|
for item in source.fetch():
|
|
item.source = "ats"
|
|
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
|
|
store.upsert_role(item, subs.get(target))
|
|
stored += 1
|
|
return {"roles": stored}
|
|
|
|
|
|
def close_boards(config: Config, store: Store, tally: Tally, *,
|
|
context: RunContext = NULL) -> dict[str, Any]:
|
|
"""The run's settle: close every board this run answered in full and
|
|
stopped listing a role from, and tally what happened.
|
|
|
|
Reads `self.state` rather than the batch jobs' own results, because a
|
|
`pull_batch` job that raised partway through still leaves the boards it
|
|
had already parsed stamped — see the module docstring. A board whose
|
|
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
|
whole batch job never got to it — is never closed.
|
|
"""
|
|
run_marker = store.state.get("ats:run", "")
|
|
answered = failed = closed = 0
|
|
for sub in store.subscriptions(source="ats", fetchable=True):
|
|
stamp = store.state.get(f"ats:ok:{sub.target}", "")
|
|
if stamp and stamp == run_marker:
|
|
answered += 1
|
|
seen_ids = json.loads(store.state.get(f"ats:ids:{sub.target}", "[]"))
|
|
closed += store.mark_closed("ats", seen_ids, sub.id)
|
|
else:
|
|
failed += 1
|
|
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
|
return {"boards": answered, "failed": failed, "roles": roles, "closed": closed}
|
|
|
|
|
|
def pull_task() -> Task:
|
|
"""The pull on its own. `app.jobs.cascade` makes it the first stage
|
|
of the chain the `pull` job starts."""
|
|
return Task(kind="pull_batch", finder=_boards, name="pull",
|
|
label="Reading every board", settle=SETTLE)
|