A second pull (schedule, Run button, `app job pull` in another process) could start while one was fetching and overwrite the stored run marker, so the first run's boards failed their stamp check or a settle closed on the other run's id lists. `claim_run` sets `pull:run` in one conditional statement, only while it equals `pull:settled` or is older than an hour; a refused `pull` job queues no stage and says so in its result and a notice, and the CLI exits 1. Batches carry the marker in their payload, the plugins stamp with it, and the settle reads it off its own batches' payloads before releasing the run. Co-Authored-By: Claude Opus <noreply@anthropic.com>
302 lines
13 KiB
Python
302 lines
13 KiB
Python
"""`pull`: fetch every active `ats` board and `search`, and store the roles
|
|
they list.
|
|
|
|
A task, not a single job: ten boards or searches per `pull_batch` job on the
|
|
`http` resource pool, because a board is one HTTP round trip and that pool is
|
|
sized for exactly this. The final job's `settle` then closes every board a
|
|
complete fetch of it did not list again — see `close_boards`.
|
|
|
|
Closure is read from `self.state`, which each plugin itself stamps per target
|
|
on a successful parse (`ats:ok:<target>`, `ats:complete:<target>`,
|
|
`ats:ids:<target>`, and the same under `search:`), rather than from what each
|
|
`pull_batch` job happens to return. A batch job that raises partway through
|
|
still leaves the boards it had already finished stamped, so one bad board in a
|
|
batch of ten does not cost the other nine their closure. A board that answered
|
|
but was not read to its stated total (a paged host that hit a 429 or the page
|
|
cap) stores what it read and closes nothing.
|
|
|
|
A search closes the same way when it listed everything that matches (see
|
|
`plugins/search.py` for how each provider knows). One that did not closes its
|
|
roles only once no search has seen them for `STALE_DAYS`. A board
|
|
that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`)
|
|
closes all of its roles, and its next answer reopens what it lists.
|
|
|
|
One pull at a time. A run is claimed in the database (`claim_run`: `pull:run`
|
|
set only while the previous run is settled, in one statement), so the
|
|
schedule, the Run button and `app job pull` in another process cannot start a
|
|
second fetch while one is in flight; a claim older than `ABANDONED` with no
|
|
settle is a run that died and is taken over. The marker travels in every
|
|
batch's payload, the plugins stamp with the marker they were handed, and the
|
|
settle compares against the marker its own batches carried, so neither a
|
|
later run's marker nor its id lists reach this run's closure.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from datetime import UTC, datetime, timedelta
|
|
from functools import partial
|
|
from typing import Any
|
|
|
|
from pipeline import NULL, RunContext, Tally, Task, job
|
|
|
|
from app.config import Config
|
|
from app.models import company_key
|
|
from app.storage import Store
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: As many boards in the air per job as there are `http` workers by default
|
|
#: (see `config.yaml`), so one pull keeps the pool busy without one job
|
|
#: holding all 190-odd boards' worth of retries against a single attempt.
|
|
BATCH_SIZE = 10
|
|
|
|
SETTLE = "app.jobs.pull:close_boards"
|
|
|
|
#: The plugins a pull reads, by the `source` their subscriptions carry.
|
|
SOURCES = ("ats", "search")
|
|
|
|
#: How long a search role stays open once no search lists it any more, and
|
|
#: how long a board may fail on every fetch before its roles close.
|
|
STALE_DAYS = 21
|
|
|
|
#: The `state` prefix the `ats` plugin records a failing board under:
|
|
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
|
|
FAILING = "ats:failing:"
|
|
|
|
#: The `state` key that records the starter searches were imported.
|
|
SEARCHES_SEEDED = "searches:seeded"
|
|
|
|
#: The marker of the pull run claimed last, and of the one settled last. A run
|
|
#: is in flight while the two differ.
|
|
RUN = "pull:run"
|
|
SETTLED = "pull:settled"
|
|
|
|
#: How old an unsettled claim may be before it counts as a run that died (a
|
|
#: killed process, a killed cascade) and the next pull takes over. A pull of
|
|
#: every board takes minutes.
|
|
ABANDONED = timedelta(hours=1)
|
|
|
|
#: What a refused pull says, in its result and its notice.
|
|
ALREADY_RUNNING = "a pull is already running"
|
|
|
|
|
|
def _now() -> str:
|
|
"""A run marker: to the microsecond, so two pulls a second apart are two
|
|
runs and a stamp from one never matches the other."""
|
|
return datetime.now(UTC).isoformat(timespec="microseconds")
|
|
|
|
|
|
def claim_run(store: Store) -> str:
|
|
"""Start a pull run: a new marker in `pull:run`, returned, unless another
|
|
run is still unsettled and younger than `ABANDONED`, which returns "".
|
|
One conditional write, so two starters in two processes cannot both win."""
|
|
marker = _now()
|
|
stale = (datetime.now(UTC) - ABANDONED).isoformat(timespec="microseconds")
|
|
return marker if store.claim_state(RUN, marker, released=SETTLED, stale=stale) else ""
|
|
|
|
|
|
def running(store: Store) -> str:
|
|
"""The marker of the pull run in flight, or "" when the last one settled."""
|
|
run = store.state.get(RUN, "")
|
|
return run if run and run != store.state.get(SETTLED, "") else ""
|
|
|
|
|
|
def release_run(store: Store, run: str) -> None:
|
|
"""Mark `run` settled, unless a later run has taken over since: letting go
|
|
of a claim that is no longer ours would release the one that is."""
|
|
if store.state.get(RUN, "") == run:
|
|
store.state[SETTLED] = run
|
|
|
|
|
|
def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]:
|
|
"""Every fetchable `ats` and `search` subscription, ten to a batch of one
|
|
source, each batch carrying the run marker its plugin stamps with and
|
|
`close_boards` compares against. `run` is a marker the caller claimed
|
|
already (the `pull` job, `app job pull`); without one the finder claims
|
|
its own, and queues nothing when another run is in flight. A run with
|
|
nothing to read is settled here, since no settle will see a batch of it."""
|
|
seed_searches(config, store)
|
|
targets = [(source, [s.target for s in store.subscriptions(source=source, fetchable=True)])
|
|
for source in SOURCES]
|
|
if not any(found for _, found in targets):
|
|
if run:
|
|
release_run(store, run)
|
|
return []
|
|
run = run or claim_run(store)
|
|
if not run:
|
|
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
|
|
running(store))
|
|
return []
|
|
return [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
|
|
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
|
|
|
|
|
|
@job("pull_batch", "Reading boards", resource="http")
|
|
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
|
|
run: str = "", context: RunContext = NULL) -> dict[str, Any]:
|
|
"""Fetch this batch's boards or searches and `upsert_role` every item
|
|
they list.
|
|
|
|
`targets` is keyword-only without a default, which is what makes it a
|
|
required payload key to the queue: `/api/jobs/kinds` lists no batch that
|
|
could only fail without one.
|
|
|
|
One plugin per target, scoped to just that entry — `Source.targets()`
|
|
reads `options[subscription_field]` — so one job never re-fetches another
|
|
job's boards, and `context.check()` runs before each fetch: a Kill lands
|
|
between two boards rather than after the tenth.
|
|
|
|
`run` is the marker of the run that queued this batch, handed to the
|
|
plugin as its `run` option, so its stamps belong to that run whatever the
|
|
stored marker says by the time the batch runs.
|
|
"""
|
|
from plugins.ats import ATSSource
|
|
from plugins.search import SearchSource
|
|
|
|
# Fetchable now, not when the pull was queued: a board switched off on the
|
|
# Job boards or Company boards since then is skipped.
|
|
subs = {s.target: s for s in store.subscriptions(source=source, fetchable=True)
|
|
if s.target in targets}
|
|
stored = 0
|
|
for target in targets:
|
|
sub = subs.get(target)
|
|
if sub is None:
|
|
continue
|
|
context.check()
|
|
options = dict(config.sources.get(source) or {}) | {"run": run}
|
|
plugin: ATSSource | SearchSource
|
|
if source == "search":
|
|
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
|
|
plugin = SearchSource(options)
|
|
else:
|
|
options["boards"] = [{"url": target, "label": sub.company or sub.label}]
|
|
plugin = ATSSource(options)
|
|
plugin.state = store.state
|
|
for item in plugin.fetch():
|
|
item.source = source
|
|
store.upsert_role(item, sub)
|
|
stored += 1
|
|
return {"roles": stored, "run": run}
|
|
|
|
|
|
def _run_of(tally: Tally, context: RunContext) -> str:
|
|
"""The marker this settle's own batches were queued with: off their
|
|
payloads, which a batch that raised still has, else off their results
|
|
(the only thing a settle called directly is given)."""
|
|
for batch in context.waited_for:
|
|
run = (getattr(batch, "payload", None) or {}).get("run")
|
|
if run:
|
|
return str(run)
|
|
return next((str(r["run"]) for r in tally.results if r.get("run")), "")
|
|
|
|
|
|
def close_boards(config: Config, store: Store, tally: Tally, *,
|
|
context: RunContext = NULL) -> dict[str, Any]:
|
|
"""The run's settle: close every board this run answered in full and
|
|
stopped listing a role from, and tally what happened.
|
|
|
|
Reads `self.state` rather than the batch jobs' own results, because a
|
|
`pull_batch` job that raised partway through still leaves the boards it
|
|
had already parsed stamped — see the module docstring. A board whose
|
|
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
|
whole batch job never got to it — is never closed by its list, and a board
|
|
that answered incompletely is not either. A search closes by its list only
|
|
when that list was complete, and otherwise by `STALE_DAYS`. A board that
|
|
has failed on every fetch since `STALE_DAYS` ago closes all its roles.
|
|
|
|
The stamps are compared with this run's own marker, the one its batches
|
|
carried, never with the stored `pull:run`, which a later run may have
|
|
replaced. A settle with no batch behind it (a refused pull, nothing to
|
|
read) closes nothing by list or by age. Last, the run is marked settled,
|
|
which lets the next pull start.
|
|
|
|
A board or search switched off while its batch was in flight had its
|
|
roles reopened by that batch, after the switch closed them, so every
|
|
switched-off one is closed again here.
|
|
"""
|
|
answered = searched = failed = incomplete = closed = 0
|
|
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
|
|
failing = failing_boards(store)
|
|
run_marker = _run_of(tally, context)
|
|
for source in SOURCES:
|
|
for sub in store.subscriptions(source=source):
|
|
if not (sub.subscribed and sub.fetch):
|
|
closed += store.close_subscription(sub.id)
|
|
if not run_marker:
|
|
continue
|
|
for sub in store.subscriptions(source=source, fetchable=True):
|
|
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
|
|
if not (stamp and stamp == run_marker):
|
|
failed += 1
|
|
since = failing.get(sub.target, ("", ""))[0] if source == "ats" else ""
|
|
if since and since < stale:
|
|
closed += store.close_subscription(sub.id)
|
|
continue
|
|
if source == "search":
|
|
searched += 1
|
|
else:
|
|
answered += 1
|
|
if not store.state.get(f"{source}:complete:{sub.target}", ""):
|
|
if source == "ats":
|
|
incomplete += 1
|
|
continue
|
|
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
|
closed += store.mark_closed(source, seen_ids, sub.id)
|
|
if searched:
|
|
closed += store.close_unseen("search", stale)
|
|
if run_marker:
|
|
release_run(store, run_marker)
|
|
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
|
return {"boards": answered, "searches": searched, "failed": failed,
|
|
"incomplete": incomplete, "roles": roles, "closed": closed}
|
|
|
|
|
|
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
|
|
"""`target -> (since, reason)` for every `ats` board whose last fetch
|
|
failed: when the failures began, and what the latest one said."""
|
|
out: dict[str, tuple[str, str]] = {}
|
|
for key, value in store.state_items(FAILING).items():
|
|
try:
|
|
record = json.loads(value)
|
|
except ValueError:
|
|
continue
|
|
out[key[len(FAILING):]] = (str(record.get("since") or ""),
|
|
str(record.get("reason") or ""))
|
|
return out
|
|
|
|
|
|
def search_target(provider: str, name: str) -> str:
|
|
"""The subscription target of a saved search: `jobindex:copenhagen-backend`.
|
|
The query itself lives in `params`, so editing it keeps the target."""
|
|
return f"{provider}:{company_key(name)}"
|
|
|
|
|
|
def seed_searches(config: Config, store: Store) -> list[str]:
|
|
"""Import `config.yaml`'s `searches:` as `search` subscriptions, once,
|
|
and only while there are none: the same rule as `seed_agents`."""
|
|
if store.state.get(SEARCHES_SEEDED) or store.subscriptions(source="search"):
|
|
return []
|
|
added = []
|
|
for seed in config.searches:
|
|
try:
|
|
store.add_subscription("search", search_target(seed.provider, seed.name),
|
|
label=seed.name, board=seed.provider, params=seed.params)
|
|
except ValueError as exc:
|
|
log.warning("searches: %s", exc)
|
|
continue
|
|
added.append(seed.name)
|
|
if added:
|
|
store.state[SEARCHES_SEEDED] = store.now()
|
|
log.info("imported %d searches from %s", len(added), config.path or "the defaults")
|
|
return added
|
|
|
|
|
|
def pull_task(run: str = "") -> Task:
|
|
"""The pull on its own. `app.jobs.cascade` makes it the first stage
|
|
of the chain the `pull` job starts. `run` is a marker `claim_run` already
|
|
gave the caller; without one the finder claims its own."""
|
|
return Task(kind="pull_batch", finder=partial(_boards, run=run), name="pull",
|
|
label="Reading every board", settle=SETTLE)
|