Files
jobboard/app/jobs/pull.py
T

413 lines
20 KiB
Python

"""`pull`: fetch every active `ats` board and `search`, and store the roles
they list.
A task, not a single job: ten boards or searches per `pull_batch` job on the
`http` resource pool, because a board is one HTTP round trip and that pool is
sized for exactly this. The final job's `settle` then closes every board a
complete fetch of it did not list again — see `close_boards`.
Closure is read from `self.state`, which each plugin itself stamps per target
on a successful parse (`ats:ok:<target>`, `ats:complete:<target>`,
`ats:ids:<target>`, and the same under `search:`), rather than from what each
`pull_batch` job happens to return. A batch job that raises partway through
still leaves the boards it had already finished stamped, so one bad board in a
batch of ten does not cost the other nine their closure. A board that answered
but was not read to its stated total (a paged host that hit a 429 or the page
cap) stores what it read and closes nothing; a paged board over the page cap
is swept across pulls and closes once the sweep reached its end (see
`plugins/readers/_runner.py`).
A board read with a filter (`ats:filtered:<target>`, from the agents' reach,
`app.jobs.reach`, or its subscription's `params`) lists only the roles in reach, so its close keeps open the roles a
person acted on: those with a vote or an application (`_close_unlisted`). A
filtered read, or one of a reader that declares `empty="suspect"`, that lists
under a tenth of what the same read listed last time is recorded by the
runner as a failure, not as a list to close against.
A board swept across pulls (`ats:swept:<target>`, a complete read that finished
a sweep an earlier pull began) closes a role only when two finished sweeps in a
row missed it (`_close_swept`): offset paging drifts when roles close between
pulls, so one sweep can miss a role that is still open. The ids a finished
sweep missed are kept in `ats:missing:<target>`, and a complete read that was
not swept clears them.
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
the roles matching a word, which says nothing of the roles it did not list,
so it closes nothing; only an unfiltered or a place-filtered complete read
closes.
A search closes the same way when it listed everything that matches (see
`plugins/search.py` for how each provider knows). One that did not closes its
roles only once no search has seen them for `STALE_DAYS`. A board
that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`)
closes all of its roles, and its next answer reopens what it lists.
One pull at a time. A run is claimed in the database (`claim_run`: `pull:run`
set only while the previous run is settled, in one statement), so the
schedule, the Run button and `app job pull` in another process cannot start a
second fetch while one is in flight; a claim older than `ABANDONED` with no
settle is a run that died and is taken over. The marker travels in every
batch's payload, the plugins stamp with the marker they were handed, and the
settle compares against the marker its own batches carried, so neither a
later run's marker nor its id lists reach this run's closure.
"""
from __future__ import annotations
import json
import logging
from datetime import UTC, datetime, timedelta
from functools import partial
from typing import Any
from pipeline import NULL, RunContext, Tally, Task, job
from app.config import Config
from app.jobs.reach import reaches
from app.models import company_key
from app.storage import Store
log = logging.getLogger(__name__)
#: Targets per `pull_batch` job. A batch reads its boards one after another,
#: so the parallelism is the number of batches the `http` pool runs at once
#: (8 workers by default, see `config.yaml`), not this number. On the live
#: database of 2026-09-29, 175 boards and 4 searches make 19 batches, enough
#: to keep eight workers busy, while a batch that fails or is killed costs at
#: most ten targets' progress. A board's request is bounded by
#: `plugins.ats.DEADLINE` plus one read timeout.
BATCH_SIZE = 10
SETTLE = "app.jobs.pull:close_boards"
#: The plugins a pull reads, by the `source` their subscriptions carry.
SOURCES = ("ats", "search")
#: How long a search role stays open once no search lists it any more, and
#: how long a board may fail on every fetch before its roles close.
STALE_DAYS = 21
#: The `state` prefix the `ats` plugin records a failing board under:
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
FAILING = "ats:failing:"
#: The `state` key that records the starter searches were imported.
SEARCHES_SEEDED = "searches:seeded"
#: The marker of the pull run claimed last, and of the one settled last. A run
#: is in flight while the two differ.
RUN = "pull:run"
SETTLED = "pull:settled"
#: How old an unsettled claim may be before it counts as a run that died (a
#: killed process, a killed cascade) and the next pull takes over. A pull of
#: every board takes minutes.
ABANDONED = timedelta(hours=1)
#: What a refused pull says, in its result and its notice.
ALREADY_RUNNING = "a pull is already running"
def _now() -> str:
"""A run marker: to the microsecond, so two pulls a second apart are two
runs and a stamp from one never matches the other."""
return datetime.now(UTC).isoformat(timespec="microseconds")
def claim_run(store: Store) -> str:
"""Start a pull run: a new marker in `pull:run`, returned, unless another
run is still unsettled and younger than `ABANDONED`, which returns "".
One conditional write, so two starters in two processes cannot both win."""
marker = _now()
stale = (datetime.now(UTC) - ABANDONED).isoformat(timespec="microseconds")
return marker if store.claim_state(RUN, marker, released=SETTLED, stale=stale) else ""
def running(store: Store) -> str:
"""The marker of the pull run in flight, or "" when the last one settled."""
run = store.state.get(RUN, "")
return run if run and run != store.state.get(SETTLED, "") else ""
def release_run(store: Store, run: str) -> None:
"""Mark `run` settled, unless a later run has taken over since: letting go
of a claim that is no longer ours would release the one that is."""
if store.state.get(RUN, "") == run:
store.state[SETTLED] = run
def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]:
"""Every fetchable `ats` and `search` subscription, ten to a batch of one
source, each batch carrying the run marker its plugin stamps with and
`close_boards` compares against. `run` is a marker the caller claimed
already (the `pull` job, `app job pull`); without one the finder claims
its own, and queues nothing when another run is in flight. A run with
nothing to read is settled here, since no settle will see a batch of it."""
seed_searches(config, store)
subs = {source: store.subscriptions(source=source, fetchable=True) for source in SOURCES}
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
if not any(found for _, found in targets):
if run:
release_run(store, run)
return []
run = run or claim_run(store)
if not run:
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
running(store))
return []
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
reach_of = reaches(store, subs["ats"])
batches = [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
for batch in batches:
if batch["source"] == "ats":
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
return batches
@job("pull_batch", "Reading boards", resource="http")
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
run: str = "", reach: dict[str, Any] | None = None,
context: RunContext = NULL) -> dict[str, Any]:
"""Fetch this batch's boards or searches and `upsert_role` every item
they list.
`targets` is keyword-only without a default, which is what makes it a
required payload key to the queue: `/api/jobs/kinds` lists no batch that
could only fail without one.
One plugin per target, scoped to just that entry — `Source.targets()`
reads `options[subscription_field]` — so one job never re-fetches another
job's boards, and `context.check()` runs before each fetch: a Kill lands
between two boards rather than after the tenth.
`run` is the marker of the run that queued this batch, handed to the
plugin as its `run` option, so its stamps belong to that run whatever the
stored marker says by the time the batch runs. `reach` is `target ->` the
reach the run computed for a board (`app.jobs.reach`), handed to the
plugin as its `reach` option.
"""
from plugins.ats import ATSSource
from plugins.search import SearchSource
# Fetchable now, not when the pull was queued: a board switched off on the
# Job boards or Company boards since then is skipped.
subs = {s.target: s for s in store.subscriptions(source=source, fetchable=True)
if s.target in targets}
stored = 0
for target in targets:
sub = subs.get(target)
if sub is None:
continue
context.check()
options = dict(config.sources.get(source) or {}) | {"run": run}
plugin: ATSSource | SearchSource
if source == "search":
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
plugin = SearchSource(options)
else:
# A board's params are its filter; a reader's key is kept with
# the searches' (`sources.search.<name>`).
options["boards"] = [{"url": target, "label": sub.company or sub.label,
"params": sub.params}]
options["keys"] = dict(config.sources.get("search") or {})
options["reach"] = (reach or {}).get(target)
plugin = ATSSource(options)
plugin.state = store.state
for item in plugin.fetch():
item.source = source
store.upsert_role(item, sub)
stored += 1
return {"roles": stored, "run": run}
def _run_of(tally: Tally, context: RunContext) -> str:
"""The marker this settle's own batches were queued with: off their
payloads, which a batch that raised still has, else off their results
(the only thing a settle called directly is given)."""
for batch in context.waited_for:
run = (getattr(batch, "payload", None) or {}).get("run")
if run:
return str(run)
return next((str(r["run"]) for r in tally.results if r.get("run")), "")
def close_boards(config: Config, store: Store, tally: Tally, *,
context: RunContext = NULL) -> dict[str, Any]:
"""The run's settle: close every board this run answered in full and
stopped listing a role from, and tally what happened.
Reads `self.state` rather than the batch jobs' own results, because a
`pull_batch` job that raised partway through still leaves the boards it
had already parsed stamped — see the module docstring. A board whose
`ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed by its list, and a board
that answered incompletely is not either. A search closes by its list only
when that list was complete, and otherwise by `STALE_DAYS`. A board that
has failed on every fetch since `STALE_DAYS` ago closes all its roles.
The stamps are compared with this run's own marker, the one its batches
carried, never with the stored `pull:run`, which a later run may have
replaced. A settle with no batch behind it (a refused pull, nothing to
read) closes nothing by list or by age. Last, the run is marked settled,
which lets the next pull start.
A board or search switched off while its batch was in flight had its
roles reopened by that batch, after the switch closed them, so every
switched-off one is closed again here.
The `prep` stage that follows maps what this run stored and folds the
copies (`Store.rebuild_dupes`), so a closed kept role hands its group on
and the roles this run added join theirs.
"""
answered = searched = failed = incomplete = closed = 0
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
failing = failing_boards(store)
run_marker = _run_of(tally, context)
for source in SOURCES:
for sub in store.subscriptions(source=source):
if not (sub.subscribed and sub.fetch):
closed += store.close_subscription(sub.id)
if not run_marker:
continue
for sub in store.subscriptions(source=source, fetchable=True):
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
if not (stamp and stamp == run_marker):
failed += 1
since = failing.get(sub.target, ("", ""))[0] if source == "ats" else ""
if since and since < stale:
closed += store.close_subscription(sub.id)
continue
if source == "search":
searched += 1
else:
answered += 1
if not store.state.get(f"{source}:complete:{sub.target}", ""):
if source == "ats":
incomplete += 1
continue
if store.state.get(f"{source}:narrowed:{sub.target}", ""):
continue
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
is_filtered = bool(store.state.get(f"{source}:filtered:{sub.target}", ""))
if store.state.get(f"{source}:swept:{sub.target}", ""):
closed += _close_swept(store, source, sub, seen_ids, is_filtered)
continue
store.state.pop(f"{source}:missing:{sub.target}", None)
if is_filtered:
closed += _close_unlisted(store, source, seen_ids, sub.id)
else:
closed += store.mark_closed(source, seen_ids, sub.id)
if searched:
closed += store.close_unseen("search", stale)
if run_marker:
release_run(store, run_marker)
roles = sum(int(result.get("roles", 0)) for result in tally.results)
return {"boards": answered, "searches": searched, "failed": failed,
"incomplete": incomplete, "roles": roles, "closed": closed}
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int:
"""`Store.mark_closed` for a filtered read: close every open role of the
board it did not list, except a role someone acted on, one with a vote
(any, neutral included) or an application (which holds the notes). A
filtered read cannot tell a role gone from a role outside its filter, and
a role acted on stays until the board is read without the filter or
fails for `STALE_DAYS`."""
with store.transaction() as conn:
return conn.execute(
"""UPDATE roles SET closed_at = ?
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
AND source_id NOT IN (SELECT value FROM json_each(?))
AND uid NOT IN (SELECT uid FROM role_votes)
AND uid NOT IN (SELECT role_uid FROM applications
WHERE role_uid IS NOT NULL)""",
(store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount
def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
filtered: bool) -> int:
"""The close of a board whose complete read finished a sweep across
pulls: its open roles missing from `seen_ids` that also are in
`<source>:missing:<target>`, the ones the previous finished sweep missed.
The ones missing now are kept there for the next sweep. A role one sweep
missed may only have shifted back past the cursor when roles closed
between pulls, so one sweep is not evidence that it is gone. A filtered
read keeps its exceptions as `_close_unlisted` does: a role acted on
stays open, and stays in `missing`."""
key = f"{source}:missing:{sub.target}"
try:
earlier = {str(i) for i in json.loads(store.state.get(key) or "[]")}
except (TypeError, ValueError):
earlier = set()
acted_on = """ AND uid NOT IN (SELECT uid FROM role_votes)
AND uid NOT IN (SELECT role_uid FROM applications
WHERE role_uid IS NOT NULL)""" if filtered else ""
with store.transaction() as conn:
missed = [row[0] for row in conn.execute(
"""SELECT source_id FROM roles
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
AND source_id NOT IN (SELECT value FROM json_each(?))""",
(source, sub.id, json.dumps(list(seen_ids))))]
closing = [row[0] for row in conn.execute(
f"""SELECT source_id FROM roles
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
AND source_id IN (SELECT value FROM json_each(?))
AND source_id IN (SELECT value FROM json_each(?)){acted_on}""",
(source, sub.id, json.dumps(sorted(earlier)), json.dumps(missed)))]
count = conn.execute(
"""UPDATE roles SET closed_at = ?
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
AND source_id IN (SELECT value FROM json_each(?))""",
(store.now(), source, sub.id, json.dumps(closing))).rowcount
store.state[key] = json.dumps(sorted(set(missed) - set(closing)))
return count
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
"""`target -> (since, reason)` for every `ats` board whose last fetch
failed: when the failures began, and what the latest one said."""
out: dict[str, tuple[str, str]] = {}
for key, value in store.state_items(FAILING).items():
try:
record = json.loads(value)
except ValueError:
continue
out[key[len(FAILING):]] = (str(record.get("since") or ""),
str(record.get("reason") or ""))
return out
def search_target(provider: str, name: str) -> str:
"""The subscription target of a saved search: `jobindex:copenhagen-backend`.
The query itself lives in `params`, so editing it keeps the target."""
return f"{provider}:{company_key(name)}"
def seed_searches(config: Config, store: Store) -> list[str]:
"""Import `config.yaml`'s `searches:` as `search` subscriptions, once,
and only while there are none: the same rule as `seed_agents`."""
if store.state.get(SEARCHES_SEEDED) or store.subscriptions(source="search"):
return []
added = []
for seed in config.searches:
try:
store.add_subscription("search", search_target(seed.provider, seed.name),
label=seed.name, board=seed.provider, params=seed.params)
except ValueError as exc:
log.warning("searches: %s", exc)
continue
added.append(seed.name)
if added:
store.state[SEARCHES_SEEDED] = store.now()
log.info("imported %d searches from %s", len(added), config.path or "the defaults")
return added
def pull_task(run: str = "") -> Task:
"""The pull on its own. `app.jobs.cascade` makes it the first stage
of the chain the `pull` job starts. `run` is a marker `claim_run` already
gave the caller; without one the finder claims its own."""
return Task(kind="pull_batch", finder=partial(_boards, run=run), name="pull",
label="Reading every board", settle=SETTLE)