Files
jobboard/app/jobs/pull.py
T
WolfandClaude Opus c7da12b93d Paged boards are read to their stated total, and a board failing for 21 days closes
Workable and SmartRecruiters page their lists, and both parsers read page one
only, so the settle closed every role past it (Delivery Hero: 980 stated, 100
open). `_paged` follows nextPage and offset, and the plugin stamps
`ats:complete` only when the distinct ids read equal the stated total; a 429
after page one keeps what was read and closes nothing. Adzuna completes on its
`count`, Jobindex counts raw items and raises on an answer with no channel.

A board whose fetch raises is recorded under `ats:failing:<target>` with the
first failure's time and the latest reason; after STALE_DAYS the settle closes
its roles, and its next answer reopens them. `/api/companies`, the Companies
list and `app companies check` show it.

Co-Authored-By: Claude Opus <noreply@anthropic.com>
2026-09-29 14:02:49 +02:00

225 lines
9.7 KiB
Python

"""`pull`: fetch every active `ats` board and `search`, and store the roles
they list.
A task, not a single job: ten boards or searches per `pull_batch` job on the
`http` resource pool, because a board is one HTTP round trip and that pool is
sized for exactly this. The final job's `settle` then closes every board a
complete fetch of it did not list again — see `close_boards`.
Closure is read from `self.state`, which each plugin itself stamps per target
on a successful parse (`ats:ok:<target>`, `ats:complete:<target>`,
`ats:ids:<target>`, and the same under `search:`), rather than from what each
`pull_batch` job happens to return. A batch job that raises partway through
still leaves the boards it had already finished stamped, so one bad board in a
batch of ten does not cost the other nine their closure. A board that answered
but was not read to its stated total (a paged host that hit a 429 or the page
cap) stores what it read and closes nothing.
A search closes the same way when it listed everything that matches (see
`plugins/search.py` for how each provider knows). One that did not closes its
roles only once no search has seen them for `STALE_DAYS`. A board
that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`)
closes all of its roles, and its next answer reopens what it lists.
"""
from __future__ import annotations
import json
import logging
from datetime import UTC, datetime, timedelta
from typing import Any
from pipeline import NULL, RunContext, Tally, Task, job
from app.config import Config
from app.models import company_key
from app.storage import Store
log = logging.getLogger(__name__)
#: As many boards in the air per job as there are `http` workers by default
#: (see `config.yaml`), so one pull keeps the pool busy without one job
#: holding all 190-odd boards' worth of retries against a single attempt.
BATCH_SIZE = 10
SETTLE = "app.jobs.pull:close_boards"
#: The plugins a pull reads, by the `source` their subscriptions carry.
SOURCES = ("ats", "search")
#: How long a search role stays open once no search lists it any more, and
#: how long a board may fail on every fetch before its roles close.
STALE_DAYS = 21
#: The `state` prefix the `ats` plugin records a failing board under:
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
FAILING = "ats:failing:"
#: The `state` key that records the starter searches were imported.
SEARCHES_SEEDED = "searches:seeded"
def _now() -> str:
"""A run marker: to the microsecond, so two pulls a second apart are two
runs and a stamp from one never matches the other."""
return datetime.now(UTC).isoformat(timespec="microseconds")
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
"""Every fetchable `ats` and `search` subscription, ten to a batch of one
source. Stamps the run marker `close_boards` will look for, before any
batch runs — finders run synchronously, in whoever starts the task, ahead
of every job it queues."""
seed_searches(config, store)
batches: list[dict[str, Any]] = []
for source in SOURCES:
subs = store.subscriptions(source=source, fetchable=True)
if not subs:
continue
store.state[f"{source}:run"] = _now()
batches += [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]], "source": source}
for i in range(0, len(subs), BATCH_SIZE)]
return batches
@job("pull_batch", "Reading boards", resource="http")
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
context: RunContext = NULL) -> dict[str, Any]:
"""Fetch this batch's boards or searches and `upsert_role` every item
they list.
`targets` is keyword-only without a default, which is what makes it a
required payload key to the queue: `/api/jobs/kinds` lists no batch that
could only fail without one.
One plugin per target, scoped to just that entry — `Source.targets()`
reads `options[subscription_field]` — so one job never re-fetches another
job's boards, and `context.check()` runs before each fetch: a Kill lands
between two boards rather than after the tenth.
"""
from plugins.ats import ATSSource
from plugins.search import SearchSource
# Fetchable now, not when the pull was queued: a board switched off on the
# Job boards or Company boards since then is skipped.
subs = {s.target: s for s in store.subscriptions(source=source, fetchable=True)
if s.target in targets}
stored = 0
for target in targets:
sub = subs.get(target)
if sub is None:
continue
context.check()
options = dict(config.sources.get(source) or {})
plugin: ATSSource | SearchSource
if source == "search":
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
plugin = SearchSource(options)
else:
options["boards"] = [{"url": target, "label": sub.company or sub.label}]
plugin = ATSSource(options)
plugin.state = store.state
for item in plugin.fetch():
item.source = source
store.upsert_role(item, sub)
stored += 1
return {"roles": stored}
def close_boards(config: Config, store: Store, tally: Tally, *,
context: RunContext = NULL) -> dict[str, Any]:
"""The run's settle: close every board this run answered in full and
stopped listing a role from, and tally what happened.
Reads `self.state` rather than the batch jobs' own results, because a
`pull_batch` job that raised partway through still leaves the boards it
had already parsed stamped — see the module docstring. A board whose
`ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed by its list, and a board
that answered incompletely is not either. A search closes by its list only
when that list was complete, and otherwise by `STALE_DAYS`. A board that
has failed on every fetch since `STALE_DAYS` ago closes all its roles.
A board or search switched off while its batch was in flight had its
roles reopened by that batch, after the switch closed them, so every
switched-off one is closed again here.
"""
answered = searched = failed = incomplete = closed = 0
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
failing = failing_boards(store)
for source in SOURCES:
for sub in store.subscriptions(source=source):
if not (sub.subscribed and sub.fetch):
closed += store.close_subscription(sub.id)
run_marker = store.state.get(f"{source}:run", "")
for sub in store.subscriptions(source=source, fetchable=True):
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
if not (stamp and stamp == run_marker):
failed += 1
since = failing.get(sub.target, ("", ""))[0] if source == "ats" else ""
if since and since < stale:
closed += store.close_subscription(sub.id)
continue
if source == "search":
searched += 1
else:
answered += 1
if not store.state.get(f"{source}:complete:{sub.target}", ""):
if source == "ats":
incomplete += 1
continue
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
closed += store.mark_closed(source, seen_ids, sub.id)
if searched:
closed += store.close_unseen("search", stale)
roles = sum(int(result.get("roles", 0)) for result in tally.results)
return {"boards": answered, "searches": searched, "failed": failed,
"incomplete": incomplete, "roles": roles, "closed": closed}
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
"""`target -> (since, reason)` for every `ats` board whose last fetch
failed: when the failures began, and what the latest one said."""
out: dict[str, tuple[str, str]] = {}
for key, value in store.state_items(FAILING).items():
try:
record = json.loads(value)
except ValueError:
continue
out[key[len(FAILING):]] = (str(record.get("since") or ""),
str(record.get("reason") or ""))
return out
def search_target(provider: str, name: str) -> str:
"""The subscription target of a saved search: `jobindex:copenhagen-backend`.
The query itself lives in `params`, so editing it keeps the target."""
return f"{provider}:{company_key(name)}"
def seed_searches(config: Config, store: Store) -> list[str]:
"""Import `config.yaml`'s `searches:` as `search` subscriptions, once,
and only while there are none: the same rule as `seed_agents`."""
if store.state.get(SEARCHES_SEEDED) or store.subscriptions(source="search"):
return []
added = []
for seed in config.searches:
try:
store.add_subscription("search", search_target(seed.provider, seed.name),
label=seed.name, board=seed.provider, params=seed.params)
except ValueError as exc:
log.warning("searches: %s", exc)
continue
added.append(seed.name)
if added:
store.state[SEARCHES_SEEDED] = store.now()
log.info("imported %d searches from %s", len(added), config.path or "the defaults")
return added
def pull_task() -> Task:
"""The pull on its own. `app.jobs.cascade` makes it the first stage
of the chain the `pull` job starts."""
return Task(kind="pull_batch", finder=_boards, name="pull",
label="Reading every board", settle=SETTLE)