"""`pull`: fetch every active `ats` board and `search`, and store the roles they list. A task, not a single job: ten boards or searches per `pull_batch` job on the `http` resource pool, because a board is one HTTP round trip and that pool is sized for exactly this. The final job's `settle` then closes every board a complete fetch of it did not list again — see `close_boards`. Closure is read from `self.state`, which each plugin itself stamps per target on a successful parse (`ats:ok:`, `ats:ids:`, and the same under `search:`), rather than from what each `pull_batch` job happens to return. A batch job that raises partway through still leaves the boards it had already finished stamped, so one bad board in a batch of ten does not cost the other nine their closure. A search closes the same way when its results ran out before its page limit, since it then listed everything that matches. One that hit the limit did not, so its roles close only once no search has seen them for `STALE_DAYS`. """ from __future__ import annotations import json import logging from datetime import UTC, datetime, timedelta from typing import Any from pipeline import NULL, RunContext, Tally, Task, job from app.config import Config from app.models import company_key from app.storage import Store log = logging.getLogger(__name__) #: As many boards in the air per job as there are `http` workers by default #: (see `config.yaml`), so one pull keeps the pool busy without one job #: holding all 190-odd boards' worth of retries against a single attempt. BATCH_SIZE = 10 SETTLE = "app.jobs.pull:close_boards" #: The plugins a pull reads, by the `source` their subscriptions carry. SOURCES = ("ats", "search") #: How long a search role stays open once no search lists it any more. STALE_DAYS = 21 #: The `state` key that records the starter searches were imported. SEARCHES_SEEDED = "searches:seeded" def _now() -> str: return datetime.now(UTC).isoformat(timespec="seconds") def _boards(config: Config, store: Store) -> list[dict[str, Any]]: """Every fetchable `ats` and `search` subscription, ten to a batch of one source. Stamps the run marker `close_boards` will look for, before any batch runs — finders run synchronously, in whoever starts the task, ahead of every job it queues.""" seed_searches(config, store) batches: list[dict[str, Any]] = [] for source in SOURCES: subs = store.subscriptions(source=source, fetchable=True) if not subs: continue store.state[f"{source}:run"] = _now() batches += [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]], "source": source} for i in range(0, len(subs), BATCH_SIZE)] return batches @job("pull_batch", "Reading boards", resource="http") def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats", context: RunContext = NULL) -> dict[str, Any]: """Fetch this batch's boards or searches and `upsert_role` every item they list. `targets` is keyword-only without a default, which is what makes it a required payload key to the queue: `/api/jobs/kinds` lists no batch that could only fail without one. One plugin per target, scoped to just that entry — `Source.targets()` reads `options[subscription_field]` — so one job never re-fetches another job's boards, and `context.check()` runs before each fetch: a Kill lands between two boards rather than after the tenth. """ from plugins.ats import ATSSource from plugins.search import SearchSource # Fetchable now, not when the pull was queued: a board switched off on the # Job boards or Company boards since then is skipped. subs = {s.target: s for s in store.subscriptions(source=source, fetchable=True) if s.target in targets} stored = 0 for target in targets: sub = subs.get(target) if sub is None: continue context.check() options = dict(config.sources.get(source) or {}) plugin: ATSSource | SearchSource if source == "search": options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}] plugin = SearchSource(options) else: options["boards"] = [{"url": target, "label": sub.company or sub.label}] plugin = ATSSource(options) plugin.state = store.state for item in plugin.fetch(): item.source = source store.upsert_role(item, sub) stored += 1 return {"roles": stored} def close_boards(config: Config, store: Store, tally: Tally, *, context: RunContext = NULL) -> dict[str, Any]: """The run's settle: close every board this run answered in full and stopped listing a role from, and tally what happened. Reads `self.state` rather than the batch jobs' own results, because a `pull_batch` job that raised partway through still leaves the boards it had already parsed stamped — see the module docstring. A board whose `ats:ok:` was not written during *this* run — it failed, or its whole batch job never got to it — is never closed. A search closes by its list only when that list was complete, and otherwise by `STALE_DAYS`. A board or search switched off while its batch was in flight had its roles reopened by that batch, after the switch closed them, so every switched-off one is closed again here. """ answered = searched = failed = closed = 0 for source in SOURCES: for sub in store.subscriptions(source=source): if not (sub.subscribed and sub.fetch): closed += store.close_subscription(sub.id) run_marker = store.state.get(f"{source}:run", "") for sub in store.subscriptions(source=source, fetchable=True): stamp = store.state.get(f"{source}:ok:{sub.target}", "") if not (stamp and stamp == run_marker): failed += 1 continue if source == "search": searched += 1 if not store.state.get(f"search:complete:{sub.target}", ""): continue else: answered += 1 seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]")) closed += store.mark_closed(source, seen_ids, sub.id) if searched: stale = datetime.now(UTC) - timedelta(days=STALE_DAYS) closed += store.close_unseen("search", stale.isoformat(timespec="seconds")) roles = sum(int(result.get("roles", 0)) for result in tally.results) return {"boards": answered, "searches": searched, "failed": failed, "roles": roles, "closed": closed} def search_target(provider: str, name: str) -> str: """The subscription target of a saved search: `jobindex:copenhagen-backend`. The query itself lives in `params`, so editing it keeps the target.""" return f"{provider}:{company_key(name)}" def seed_searches(config: Config, store: Store) -> list[str]: """Import `config.yaml`'s `searches:` as `search` subscriptions, once, and only while there are none: the same rule as `seed_agents`.""" if store.state.get(SEARCHES_SEEDED) or store.subscriptions(source="search"): return [] added = [] for seed in config.searches: try: store.add_subscription("search", search_target(seed.provider, seed.name), label=seed.name, board=seed.provider, params=seed.params) except ValueError as exc: log.warning("searches: %s", exc) continue added.append(seed.name) if added: store.state[SEARCHES_SEEDED] = store.now() log.info("imported %d searches from %s", len(added), config.path or "the defaults") return added def pull_task() -> Task: """The pull on its own. `app.jobs.cascade` makes it the first stage of the chain the `pull` job starts.""" return Task(kind="pull_batch", finder=_boards, name="pull", label="Reading every board", settle=SETTLE)