Files
jobboard/app/jobs/pull.py
T
WolfandClaude Opus 5.5 7a63311714 Rename the package roledesk to app
The boilerplate's rule: the deploy kit's Dockerfile, the APP_CONFIG,
APP_PASSWORD and APP_DATA variables and its commands all assume a package
called app, so a project renames its display title and nothing else. The
console script is `app`, and the launch entry runs `uv run app serve --port
8790`. The domain noun stays `role`.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:30:10 +02:00

111 lines
4.5 KiB
Python

"""`pull`: fetch every active `ats` board and store the roles it lists.
A task, not a single job: ten boards per `pull_batch` job on the `http`
resource pool, because a board is one HTTP round trip and that pool is sized
for exactly this. The final job's `settle` then closes every board a
complete fetch of it did not list again — see `close_boards`.
Closure is read from `self.state`, which the `ats` plugin itself stamps per
board on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`), rather
than from what each `pull_batch` job happens to return. A batch job that
raises partway through still leaves the boards it had already finished
stamped, so one bad board in a batch of ten does not cost the other nine
their closure.
"""
from __future__ import annotations
import json
import logging
from datetime import UTC, datetime
from typing import Any
from pipeline import NULL, RunContext, Tally, Task, job
from app.config import Config
from app.storage import Store
log = logging.getLogger(__name__)
#: As many boards in the air per job as there are `http` workers by default
#: (see `roles.yaml`), so one pull keeps the pool busy without one job
#: holding all 190-odd boards' worth of retries against a single attempt.
BATCH_SIZE = 10
SETTLE = "app.jobs.pull:close_boards"
def _now() -> str:
return datetime.now(UTC).isoformat(timespec="seconds")
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
"""Every fetchable `ats` subscription, ten to a batch. Stamps the run
marker `close_boards` will look for, before any batch runs — finders run
synchronously, in whoever starts the task, ahead of every job it queues."""
subs = store.subscriptions(source="ats", fetchable=True)
if not subs:
return []
store.state["ats:run"] = _now()
return [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]]}
for i in range(0, len(subs), BATCH_SIZE)]
@job("pull_batch", "Reading boards", resource="http")
def run_batch(config: Config, store: Store, targets: list[str], *,
context: RunContext = NULL) -> dict[str, Any]:
"""Fetch this batch's boards and `upsert_role` every item they list.
Builds the `ats` plugin scoped to just these targets — `Source.targets()`
reads `options[subscription_field]`, so handing it only this batch's
entries is what keeps one job from re-fetching every other job's boards.
"""
from plugins.ats import ATSSource
subs = {s.target: s for s in store.subscriptions(source="ats") if s.target in targets}
options = dict(config.sources.get("ats") or {})
options["boards"] = [{"url": target, "label": subs[target].company or subs[target].label}
for target in targets if target in subs]
source = ATSSource(options)
source.state = store.state
stored = 0
for item in source.fetch():
item.source = "ats"
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
store.upsert_role(item, subs.get(target))
stored += 1
return {"roles": stored}
def close_boards(config: Config, store: Store, tally: Tally, *,
context: RunContext = NULL) -> dict[str, Any]:
"""The run's settle: close every board this run answered in full and
stopped listing a role from, and tally what happened.
Reads `self.state` rather than the batch jobs' own results, because a
`pull_batch` job that raised partway through still leaves the boards it
had already parsed stamped — see the module docstring. A board whose
`ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed.
"""
run_marker = store.state.get("ats:run", "")
answered = failed = closed = 0
for sub in store.subscriptions(source="ats", fetchable=True):
stamp = store.state.get(f"ats:ok:{sub.target}", "")
if stamp and stamp == run_marker:
answered += 1
seen_ids = json.loads(store.state.get(f"ats:ids:{sub.target}", "[]"))
closed += store.mark_closed("ats", seen_ids, sub.id)
else:
failed += 1
roles = sum(int(result.get("roles", 0)) for result in tally.results)
return {"boards": answered, "failed": failed, "roles": roles, "closed": closed}
def pull_task() -> Task:
"""The pull on its own. `app.jobs.cascade` makes it the first stage
of the chain that `TASKS["pull"]` starts."""
return Task(kind="pull_batch", finder=_boards, name="pull",
label="Reading every board", settle=SETTLE)