"""`pull`: fetch every active `ats` board and store the roles it lists. A task, not a single job: ten boards per `pull_batch` job on the `http` resource pool, because a board is one HTTP round trip and that pool is sized for exactly this. The final job's `settle` then closes every board a complete fetch of it did not list again — see `close_boards`. Closure is read from `self.state`, which the `ats` plugin itself stamps per board on a successful parse (`ats:ok:`, `ats:ids:`), rather than from what each `pull_batch` job happens to return. A batch job that raises partway through still leaves the boards it had already finished stamped, so one bad board in a batch of ten does not cost the other nine their closure. """ from __future__ import annotations import json import logging from datetime import UTC, datetime from typing import Any from pipeline import NULL, RunContext, Tally, Task, job from app.config import Config from app.storage import Store log = logging.getLogger(__name__) #: As many boards in the air per job as there are `http` workers by default #: (see `config.yaml`), so one pull keeps the pool busy without one job #: holding all 190-odd boards' worth of retries against a single attempt. BATCH_SIZE = 10 SETTLE = "app.jobs.pull:close_boards" def _now() -> str: return datetime.now(UTC).isoformat(timespec="seconds") def _boards(config: Config, store: Store) -> list[dict[str, Any]]: """Every fetchable `ats` subscription, ten to a batch. Stamps the run marker `close_boards` will look for, before any batch runs — finders run synchronously, in whoever starts the task, ahead of every job it queues.""" subs = store.subscriptions(source="ats", fetchable=True) if not subs: return [] store.state["ats:run"] = _now() return [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]]} for i in range(0, len(subs), BATCH_SIZE)] @job("pull_batch", "Reading boards", resource="http") def run_batch(config: Config, store: Store, *, targets: list[str], context: RunContext = NULL) -> dict[str, Any]: """Fetch this batch's boards and `upsert_role` every item they list. `targets` is keyword-only without a default, which is what makes it a required payload key to the queue: the Jobs tab offers no button for a batch that could only fail without one. Builds the `ats` plugin scoped to just these targets — `Source.targets()` reads `options[subscription_field]`, so handing it only this batch's entries is what keeps one job from re-fetching every other job's boards. """ from plugins.ats import ATSSource subs = {s.target: s for s in store.subscriptions(source="ats") if s.target in targets} options = dict(config.sources.get("ats") or {}) options["boards"] = [{"url": target, "label": subs[target].company or subs[target].label} for target in targets if target in subs] source = ATSSource(options) source.state = store.state stored = 0 for item in source.fetch(): item.source = "ats" target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}" store.upsert_role(item, subs.get(target)) stored += 1 return {"roles": stored} def close_boards(config: Config, store: Store, tally: Tally, *, context: RunContext = NULL) -> dict[str, Any]: """The run's settle: close every board this run answered in full and stopped listing a role from, and tally what happened. Reads `self.state` rather than the batch jobs' own results, because a `pull_batch` job that raised partway through still leaves the boards it had already parsed stamped — see the module docstring. A board whose `ats:ok:` was not written during *this* run — it failed, or its whole batch job never got to it — is never closed. """ run_marker = store.state.get("ats:run", "") answered = failed = closed = 0 for sub in store.subscriptions(source="ats", fetchable=True): stamp = store.state.get(f"ats:ok:{sub.target}", "") if stamp and stamp == run_marker: answered += 1 seen_ids = json.loads(store.state.get(f"ats:ids:{sub.target}", "[]")) closed += store.mark_closed("ats", seen_ids, sub.id) else: failed += 1 roles = sum(int(result.get("roles", 0)) for result in tally.results) return {"boards": answered, "failed": failed, "roles": roles, "closed": closed} def pull_task() -> Task: """The pull on its own. `app.jobs.cascade` makes it the first stage of the chain the `pull` job starts.""" return Task(kind="pull_batch", finder=_boards, name="pull", label="Reading every board", settle=SETTLE)