Files
jobboard/app/jobs/pull.py
T
WolfandClaude Opus 5.5 59a301c351 Wire the app like the boilerplate: door, queue, schedules, eight tabs
app/main.py assembles it in the boilerplate's order, auth.connect first,
then the queue with the scheduler and the pages. TABS is Roles,
Applications, Agents, Companies, Jobs, Schedules, Messages and Account:
the alpha page's three views become templates on the boilerplate's
base.html, driven by static/roles.js (the old app.js) through api() and
toasts.js, over the unchanged JSON routes. Companies lists every board with
its open roles and the affinity control. A 422 is one sentence, so a toast
never reads [object Object].

Config takes the boilerplate's shape: APP_CONFIG or config.yaml, a missing
file is the defaults, an unknown key is an error. roles.yaml is gone;
config.yaml is the committed default and carries the starter agents.

pull is now a job that asks for the pull cascade, because a schedule
enqueues a job kind; its default schedule, every 6h, is declared beside it
in app/jobs/cascade.py. pull_batch's targets are keyword-only so the Jobs
tab offers no batch button. `app worker` is the boilerplate's; `app job`
still drains a queue of its own and fires no schedules.

test_coupling holds pipeline, auth and desk apart (desk may import
pipeline) and plugins to desk; test_architecture keeps app's layers, now
with main.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:48:12 +02:00

115 lines
4.7 KiB
Python

"""`pull`: fetch every active `ats` board and store the roles it lists.
A task, not a single job: ten boards per `pull_batch` job on the `http`
resource pool, because a board is one HTTP round trip and that pool is sized
for exactly this. The final job's `settle` then closes every board a
complete fetch of it did not list again — see `close_boards`.
Closure is read from `self.state`, which the `ats` plugin itself stamps per
board on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`), rather
than from what each `pull_batch` job happens to return. A batch job that
raises partway through still leaves the boards it had already finished
stamped, so one bad board in a batch of ten does not cost the other nine
their closure.
"""
from __future__ import annotations
import json
import logging
from datetime import UTC, datetime
from typing import Any
from pipeline import NULL, RunContext, Tally, Task, job
from app.config import Config
from app.storage import Store
log = logging.getLogger(__name__)
#: As many boards in the air per job as there are `http` workers by default
#: (see `config.yaml`), so one pull keeps the pool busy without one job
#: holding all 190-odd boards' worth of retries against a single attempt.
BATCH_SIZE = 10
SETTLE = "app.jobs.pull:close_boards"
def _now() -> str:
return datetime.now(UTC).isoformat(timespec="seconds")
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
"""Every fetchable `ats` subscription, ten to a batch. Stamps the run
marker `close_boards` will look for, before any batch runs — finders run
synchronously, in whoever starts the task, ahead of every job it queues."""
subs = store.subscriptions(source="ats", fetchable=True)
if not subs:
return []
store.state["ats:run"] = _now()
return [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]]}
for i in range(0, len(subs), BATCH_SIZE)]
@job("pull_batch", "Reading boards", resource="http")
def run_batch(config: Config, store: Store, *, targets: list[str],
context: RunContext = NULL) -> dict[str, Any]:
"""Fetch this batch's boards and `upsert_role` every item they list.
`targets` is keyword-only without a default, which is what makes it a
required payload key to the queue: the Jobs tab offers no button for a
batch that could only fail without one.
Builds the `ats` plugin scoped to just these targets — `Source.targets()`
reads `options[subscription_field]`, so handing it only this batch's
entries is what keeps one job from re-fetching every other job's boards.
"""
from plugins.ats import ATSSource
subs = {s.target: s for s in store.subscriptions(source="ats") if s.target in targets}
options = dict(config.sources.get("ats") or {})
options["boards"] = [{"url": target, "label": subs[target].company or subs[target].label}
for target in targets if target in subs]
source = ATSSource(options)
source.state = store.state
stored = 0
for item in source.fetch():
item.source = "ats"
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
store.upsert_role(item, subs.get(target))
stored += 1
return {"roles": stored}
def close_boards(config: Config, store: Store, tally: Tally, *,
context: RunContext = NULL) -> dict[str, Any]:
"""The run's settle: close every board this run answered in full and
stopped listing a role from, and tally what happened.
Reads `self.state` rather than the batch jobs' own results, because a
`pull_batch` job that raised partway through still leaves the boards it
had already parsed stamped — see the module docstring. A board whose
`ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed.
"""
run_marker = store.state.get("ats:run", "")
answered = failed = closed = 0
for sub in store.subscriptions(source="ats", fetchable=True):
stamp = store.state.get(f"ats:ok:{sub.target}", "")
if stamp and stamp == run_marker:
answered += 1
seen_ids = json.loads(store.state.get(f"ats:ids:{sub.target}", "[]"))
closed += store.mark_closed("ats", seen_ids, sub.id)
else:
failed += 1
roles = sum(int(result.get("roles", 0)) for result in tally.results)
return {"boards": answered, "failed": failed, "roles": roles, "closed": closed}
def pull_task() -> Task:
"""The pull on its own. `app.jobs.cascade` makes it the first stage
of the chain the `pull` job starts."""
return Task(kind="pull_batch", finder=_boards, name="pull",
label="Reading every board", settle=SETTLE)