"""`pull`: fetch every active `ats` board and `search`, and store the roles they list. A task, not a single job: ten boards or searches per `pull_batch` job on the `http` resource pool, because a board is one HTTP round trip and that pool is sized for exactly this. The final job's `settle` then closes every board a complete fetch of it did not list again — see `close_boards`. Closure is read from `self.state`, which each plugin itself stamps per target on a successful parse (`ats:ok:`, `ats:complete:`, `ats:ids:`, and the same under `search:`), rather than from what each `pull_batch` job happens to return. A batch job that raises partway through still leaves the boards it had already finished stamped, so one bad board in a batch of ten does not cost the other nine their closure. A board that answered but was not read to its stated total (a paged host that hit a 429 or the page cap) stores what it read and closes nothing. A search closes the same way when it listed everything that matches (see `plugins/search.py` for how each provider knows). One that did not closes its roles only once no search has seen them for `STALE_DAYS`. A board that has failed on every fetch for `STALE_DAYS` (`ats:failing:`) closes all of its roles, and its next answer reopens what it lists. """ from __future__ import annotations import json import logging from datetime import UTC, datetime, timedelta from typing import Any from pipeline import NULL, RunContext, Tally, Task, job from app.config import Config from app.models import company_key from app.storage import Store log = logging.getLogger(__name__) #: As many boards in the air per job as there are `http` workers by default #: (see `config.yaml`), so one pull keeps the pool busy without one job #: holding all 190-odd boards' worth of retries against a single attempt. BATCH_SIZE = 10 SETTLE = "app.jobs.pull:close_boards" #: The plugins a pull reads, by the `source` their subscriptions carry. SOURCES = ("ats", "search") #: How long a search role stays open once no search lists it any more, and #: how long a board may fail on every fetch before its roles close. STALE_DAYS = 21 #: The `state` prefix the `ats` plugin records a failing board under: #: `ats:failing:` -> `{"since", "reason"}`, removed on its next answer. FAILING = "ats:failing:" #: The `state` key that records the starter searches were imported. SEARCHES_SEEDED = "searches:seeded" def _now() -> str: """A run marker: to the microsecond, so two pulls a second apart are two runs and a stamp from one never matches the other.""" return datetime.now(UTC).isoformat(timespec="microseconds") def _boards(config: Config, store: Store) -> list[dict[str, Any]]: """Every fetchable `ats` and `search` subscription, ten to a batch of one source. Stamps the run marker `close_boards` will look for, before any batch runs — finders run synchronously, in whoever starts the task, ahead of every job it queues.""" seed_searches(config, store) batches: list[dict[str, Any]] = [] for source in SOURCES: subs = store.subscriptions(source=source, fetchable=True) if not subs: continue store.state[f"{source}:run"] = _now() batches += [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]], "source": source} for i in range(0, len(subs), BATCH_SIZE)] return batches @job("pull_batch", "Reading boards", resource="http") def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats", context: RunContext = NULL) -> dict[str, Any]: """Fetch this batch's boards or searches and `upsert_role` every item they list. `targets` is keyword-only without a default, which is what makes it a required payload key to the queue: `/api/jobs/kinds` lists no batch that could only fail without one. One plugin per target, scoped to just that entry — `Source.targets()` reads `options[subscription_field]` — so one job never re-fetches another job's boards, and `context.check()` runs before each fetch: a Kill lands between two boards rather than after the tenth. """ from plugins.ats import ATSSource from plugins.search import SearchSource # Fetchable now, not when the pull was queued: a board switched off on the # Job boards or Company boards since then is skipped. subs = {s.target: s for s in store.subscriptions(source=source, fetchable=True) if s.target in targets} stored = 0 for target in targets: sub = subs.get(target) if sub is None: continue context.check() options = dict(config.sources.get(source) or {}) plugin: ATSSource | SearchSource if source == "search": options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}] plugin = SearchSource(options) else: options["boards"] = [{"url": target, "label": sub.company or sub.label}] plugin = ATSSource(options) plugin.state = store.state for item in plugin.fetch(): item.source = source store.upsert_role(item, sub) stored += 1 return {"roles": stored} def close_boards(config: Config, store: Store, tally: Tally, *, context: RunContext = NULL) -> dict[str, Any]: """The run's settle: close every board this run answered in full and stopped listing a role from, and tally what happened. Reads `self.state` rather than the batch jobs' own results, because a `pull_batch` job that raised partway through still leaves the boards it had already parsed stamped — see the module docstring. A board whose `ats:ok:` was not written during *this* run — it failed, or its whole batch job never got to it — is never closed by its list, and a board that answered incompletely is not either. A search closes by its list only when that list was complete, and otherwise by `STALE_DAYS`. A board that has failed on every fetch since `STALE_DAYS` ago closes all its roles. A board or search switched off while its batch was in flight had its roles reopened by that batch, after the switch closed them, so every switched-off one is closed again here. """ answered = searched = failed = incomplete = closed = 0 stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds") failing = failing_boards(store) for source in SOURCES: for sub in store.subscriptions(source=source): if not (sub.subscribed and sub.fetch): closed += store.close_subscription(sub.id) run_marker = store.state.get(f"{source}:run", "") for sub in store.subscriptions(source=source, fetchable=True): stamp = store.state.get(f"{source}:ok:{sub.target}", "") if not (stamp and stamp == run_marker): failed += 1 since = failing.get(sub.target, ("", ""))[0] if source == "ats" else "" if since and since < stale: closed += store.close_subscription(sub.id) continue if source == "search": searched += 1 else: answered += 1 if not store.state.get(f"{source}:complete:{sub.target}", ""): if source == "ats": incomplete += 1 continue seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]")) closed += store.mark_closed(source, seen_ids, sub.id) if searched: closed += store.close_unseen("search", stale) roles = sum(int(result.get("roles", 0)) for result in tally.results) return {"boards": answered, "searches": searched, "failed": failed, "incomplete": incomplete, "roles": roles, "closed": closed} def failing_boards(store: Store) -> dict[str, tuple[str, str]]: """`target -> (since, reason)` for every `ats` board whose last fetch failed: when the failures began, and what the latest one said.""" out: dict[str, tuple[str, str]] = {} for key, value in store.state_items(FAILING).items(): try: record = json.loads(value) except ValueError: continue out[key[len(FAILING):]] = (str(record.get("since") or ""), str(record.get("reason") or "")) return out def search_target(provider: str, name: str) -> str: """The subscription target of a saved search: `jobindex:copenhagen-backend`. The query itself lives in `params`, so editing it keeps the target.""" return f"{provider}:{company_key(name)}" def seed_searches(config: Config, store: Store) -> list[str]: """Import `config.yaml`'s `searches:` as `search` subscriptions, once, and only while there are none: the same rule as `seed_agents`.""" if store.state.get(SEARCHES_SEEDED) or store.subscriptions(source="search"): return [] added = [] for seed in config.searches: try: store.add_subscription("search", search_target(seed.provider, seed.name), label=seed.name, board=seed.provider, params=seed.params) except ValueError as exc: log.warning("searches: %s", exc) continue added.append(seed.name) if added: store.state[SEARCHES_SEEDED] = store.now() log.info("imported %d searches from %s", len(added), config.path or "the defaults") return added def pull_task() -> Task: """The pull on its own. `app.jobs.cascade` makes it the first stage of the chain the `pull` job starts.""" return Task(kind="pull_batch", finder=_boards, name="pull", label="Reading every board", settle=SETTLE)