diff --git a/app/cli.py b/app/cli.py index f73bd40..a109450 100644 --- a/app/cli.py +++ b/app/cli.py @@ -176,8 +176,17 @@ def run_job(args: argparse.Namespace) -> int: bootstrap=jobs.BOOTSTRAP, bootstrap_arg=str(config.path or ""), tz=config.timezone, schedules=False) if chained or args.kind not in JOBS: - task_id = queue.start_task(cascade.chain(args.kind) if chained - else TASKS[args.kind]()) + first = cascade.chain(args.kind) if chained else TASKS[args.kind]() + if args.kind == "pull": + # The same claim the `pull` job makes, so a pull here and one in + # the server's worker cannot both fetch. + run = jobs.pull.claim_run(store) + if not run: + print(json.dumps({"task": "pull", "result": jobs.pull.ALREADY_RUNNING, + "running": jobs.pull.running(store)})) + return 1 + first = cascade.pull_chain(run) + task_id = queue.start_task(first) queue.drain() started = queue.jobs.of_task(task_id) if not started: diff --git a/app/jobs/cascade.py b/app/jobs/cascade.py index 13ecc61..a0f35b5 100644 --- a/app/jobs/cascade.py +++ b/app/jobs/cascade.py @@ -34,7 +34,7 @@ from typing import Any from app.jobs import embed, extract, pull, score, summarise # noqa: F401 -- registers the jobs from desk.tools import agent -from pipeline import NULL, TASKS, RunContext, Tally, Task, declare, job +from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job STEP = "app.jobs.cascade:step" @@ -129,17 +129,44 @@ def chain(name: str) -> Task: return task(CASCADES[name][0], cascade=name, at=0) +def pull_chain(run: str) -> Task: + """The pull cascade for a run `pull.claim_run` has claimed: its finder + reads under that marker rather than claiming one of its own.""" + return replace(chain("pull"), finder=pull.pull_task(run).finder) + + +def already_running(since: str) -> Notice: + """The notice for a pull refused because another is in flight.""" + return Notice( + code="pull.running", level="info", kind="expected", key="pull", + text=f"Not started: {pull.ALREADY_RUNNING}" + + (f" (since {since[:19].replace('T', ' ')} UTC)." if since else "."), + where="app.jobs.pull") + + @job("pull", "Pulling every board", resource="http") def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]: - """Ask for the pull cascade once this job is done. Its own work is nil. + """Claim a pull run and ask for the pull cascade once this job is done. + Its own work is nil. + + A pull whose claim is refused, because another run is still unsettled, + queues nothing, embed to summarise included, and says so in its result + and a notice: a second cascade would fetch every board again and could + send the same roles to the model twice. Under `NULL` there is no queue to take the chain, so this raises rather - than report a pull that never happened; `app job pull` starts the chain - on a queue of its own instead of calling this. + than report a pull that never happened; `app job pull` claims the run and + starts the chain on a queue of its own instead of calling this. """ - if not context.task(chain("pull")): + run = pull.claim_run(store) + if not run: + since = pull.running(store) + context.notice(already_running(since)) + return {"skipped": pull.ALREADY_RUNNING, "running": since} + if not context.task(pull_chain(run)): + pull.release_run(store, run) raise RuntimeError("no queue to start the pull cascade on") - return {"started": " > ".join(CASCADES["pull"])} + return {"started": " > ".join(CASCADES["pull"]), "run": run} #: What `app job ` and `/api/jobs` start besides `pull`: one stage diff --git a/app/jobs/pull.py b/app/jobs/pull.py index 75c9091..d0038b7 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -20,6 +20,15 @@ A search closes the same way when it listed everything that matches (see roles only once no search has seen them for `STALE_DAYS`. A board that has failed on every fetch for `STALE_DAYS` (`ats:failing:`) closes all of its roles, and its next answer reopens what it lists. + +One pull at a time. A run is claimed in the database (`claim_run`: `pull:run` +set only while the previous run is settled, in one statement), so the +schedule, the Run button and `app job pull` in another process cannot start a +second fetch while one is in flight; a claim older than `ABANDONED` with no +settle is a run that died and is taken over. The marker travels in every +batch's payload, the plugins stamp with the marker they were handed, and the +settle compares against the marker its own batches carried, so neither a +later run's marker nor its id lists reach this run's closure. """ from __future__ import annotations @@ -27,6 +36,7 @@ from __future__ import annotations import json import logging from datetime import UTC, datetime, timedelta +from functools import partial from typing import Any from pipeline import NULL, RunContext, Tally, Task, job @@ -58,6 +68,19 @@ FAILING = "ats:failing:" #: The `state` key that records the starter searches were imported. SEARCHES_SEEDED = "searches:seeded" +#: The marker of the pull run claimed last, and of the one settled last. A run +#: is in flight while the two differ. +RUN = "pull:run" +SETTLED = "pull:settled" + +#: How old an unsettled claim may be before it counts as a run that died (a +#: killed process, a killed cascade) and the next pull takes over. A pull of +#: every board takes minutes. +ABANDONED = timedelta(hours=1) + +#: What a refused pull says, in its result and its notice. +ALREADY_RUNNING = "a pull is already running" + def _now() -> str: """A run marker: to the microsecond, so two pulls a second apart are two @@ -65,26 +88,54 @@ def _now() -> str: return datetime.now(UTC).isoformat(timespec="microseconds") -def _boards(config: Config, store: Store) -> list[dict[str, Any]]: +def claim_run(store: Store) -> str: + """Start a pull run: a new marker in `pull:run`, returned, unless another + run is still unsettled and younger than `ABANDONED`, which returns "". + One conditional write, so two starters in two processes cannot both win.""" + marker = _now() + stale = (datetime.now(UTC) - ABANDONED).isoformat(timespec="microseconds") + return marker if store.claim_state(RUN, marker, released=SETTLED, stale=stale) else "" + + +def running(store: Store) -> str: + """The marker of the pull run in flight, or "" when the last one settled.""" + run = store.state.get(RUN, "") + return run if run and run != store.state.get(SETTLED, "") else "" + + +def release_run(store: Store, run: str) -> None: + """Mark `run` settled, unless a later run has taken over since: letting go + of a claim that is no longer ours would release the one that is.""" + if store.state.get(RUN, "") == run: + store.state[SETTLED] = run + + +def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]: """Every fetchable `ats` and `search` subscription, ten to a batch of one - source. Stamps the run marker `close_boards` will look for, before any - batch runs — finders run synchronously, in whoever starts the task, ahead - of every job it queues.""" + source, each batch carrying the run marker its plugin stamps with and + `close_boards` compares against. `run` is a marker the caller claimed + already (the `pull` job, `app job pull`); without one the finder claims + its own, and queues nothing when another run is in flight. A run with + nothing to read is settled here, since no settle will see a batch of it.""" seed_searches(config, store) - batches: list[dict[str, Any]] = [] - for source in SOURCES: - subs = store.subscriptions(source=source, fetchable=True) - if not subs: - continue - store.state[f"{source}:run"] = _now() - batches += [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]], "source": source} - for i in range(0, len(subs), BATCH_SIZE)] - return batches + targets = [(source, [s.target for s in store.subscriptions(source=source, fetchable=True)]) + for source in SOURCES] + if not any(found for _, found in targets): + if run: + release_run(store, run) + return [] + run = run or claim_run(store) + if not run: + log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING, + running(store)) + return [] + return [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run} + for source, found in targets for i in range(0, len(found), BATCH_SIZE)] @job("pull_batch", "Reading boards", resource="http") def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats", - context: RunContext = NULL) -> dict[str, Any]: + run: str = "", context: RunContext = NULL) -> dict[str, Any]: """Fetch this batch's boards or searches and `upsert_role` every item they list. @@ -96,6 +147,10 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str = reads `options[subscription_field]` — so one job never re-fetches another job's boards, and `context.check()` runs before each fetch: a Kill lands between two boards rather than after the tenth. + + `run` is the marker of the run that queued this batch, handed to the + plugin as its `run` option, so its stamps belong to that run whatever the + stored marker says by the time the batch runs. """ from plugins.ats import ATSSource from plugins.search import SearchSource @@ -110,7 +165,7 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str = if sub is None: continue context.check() - options = dict(config.sources.get(source) or {}) + options = dict(config.sources.get(source) or {}) | {"run": run} plugin: ATSSource | SearchSource if source == "search": options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}] @@ -123,7 +178,18 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str = item.source = source store.upsert_role(item, sub) stored += 1 - return {"roles": stored} + return {"roles": stored, "run": run} + + +def _run_of(tally: Tally, context: RunContext) -> str: + """The marker this settle's own batches were queued with: off their + payloads, which a batch that raised still has, else off their results + (the only thing a settle called directly is given).""" + for batch in context.waited_for: + run = (getattr(batch, "payload", None) or {}).get("run") + if run: + return str(run) + return next((str(r["run"]) for r in tally.results if r.get("run")), "") def close_boards(config: Config, store: Store, tally: Tally, *, @@ -140,6 +206,12 @@ def close_boards(config: Config, store: Store, tally: Tally, *, when that list was complete, and otherwise by `STALE_DAYS`. A board that has failed on every fetch since `STALE_DAYS` ago closes all its roles. + The stamps are compared with this run's own marker, the one its batches + carried, never with the stored `pull:run`, which a later run may have + replaced. A settle with no batch behind it (a refused pull, nothing to + read) closes nothing by list or by age. Last, the run is marked settled, + which lets the next pull start. + A board or search switched off while its batch was in flight had its roles reopened by that batch, after the switch closed them, so every switched-off one is closed again here. @@ -147,11 +219,13 @@ def close_boards(config: Config, store: Store, tally: Tally, *, answered = searched = failed = incomplete = closed = 0 stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds") failing = failing_boards(store) + run_marker = _run_of(tally, context) for source in SOURCES: for sub in store.subscriptions(source=source): if not (sub.subscribed and sub.fetch): closed += store.close_subscription(sub.id) - run_marker = store.state.get(f"{source}:run", "") + if not run_marker: + continue for sub in store.subscriptions(source=source, fetchable=True): stamp = store.state.get(f"{source}:ok:{sub.target}", "") if not (stamp and stamp == run_marker): @@ -172,6 +246,8 @@ def close_boards(config: Config, store: Store, tally: Tally, *, closed += store.mark_closed(source, seen_ids, sub.id) if searched: closed += store.close_unseen("search", stale) + if run_marker: + release_run(store, run_marker) roles = sum(int(result.get("roles", 0)) for result in tally.results) return {"boards": answered, "searches": searched, "failed": failed, "incomplete": incomplete, "roles": roles, "closed": closed} @@ -217,8 +293,9 @@ def seed_searches(config: Config, store: Store) -> list[str]: return added -def pull_task() -> Task: +def pull_task(run: str = "") -> Task: """The pull on its own. `app.jobs.cascade` makes it the first stage - of the chain the `pull` job starts.""" - return Task(kind="pull_batch", finder=_boards, name="pull", + of the chain the `pull` job starts. `run` is a marker `claim_run` already + gave the caller; without one the finder claims its own.""" + return Task(kind="pull_batch", finder=partial(_boards, run=run), name="pull", label="Reading every board", settle=SETTLE) diff --git a/app/storage/store.py b/app/storage/store.py index a007148..2518d99 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -1237,6 +1237,19 @@ class Store: "SELECT key, value FROM state WHERE substr(key, 1, ?) = ? ORDER BY key", (len(prefix), prefix))} + def claim_state(self, key: str, value: str, *, released: str, stale: str) -> bool: + """Set `key` to `value` only while it is free: unset, equal to the + value under `released` (the claim last let go of), or no later than + `stale`. Returns whether it was set. One statement, so of two + processes claiming at once exactly one wins.""" + return self._write( + """INSERT INTO state (key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value + WHERE state.value <= ? + OR state.value = COALESCE( + (SELECT r.value FROM state AS r WHERE r.key = ?), '')""", + (key, value, stale, released)) > 0 + def _check_status(status: str) -> None: if status not in STATUSES: diff --git a/docs/decisions.md b/docs/decisions.md index 030e993..36cd7f7 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -38,7 +38,9 @@ What was decided for roledesk and the one reason for each. The longer reasoning ## Ingestion -**Closure is decided from `self.state`, not from what a `pull_batch` job returns.** The `ats` plugin stamps each board it parses without raising with the run's marker, the ids it saw and whether they are the whole board, and the pull's settle closes a board only when its stamp matches the marker written before any batch ran and the read was complete. So a batch that raises part way still closes the boards it finished, and a board no batch reached stays open instead of being closed on an empty id list. +**Closure is decided from `self.state`, not from what a `pull_batch` job returns.** The `ats` plugin stamps each board it parses without raising with the run's marker, the ids it saw and whether they are the whole board, and the pull's settle closes a board only when its stamp matches the marker its own batches carried and the read was complete. So a batch that raises part way still closes the boards it finished, and a board no batch reached stays open instead of being closed on an empty id list. + +**One pull runs at a time, claimed in the database.** The schedule, the Run button and `app job pull`, which drains a queue of its own in another process, could each start a pull while one was fetching; the second overwrote the stored run marker, so the first run's boards failed their stamp check and stayed open, or a settle closed on the other run's id lists, and with extract on two cascades could send the same roles to the model twice. `claim_run` sets `pull:run` to a new marker in one conditional statement, only while it equals `pull:settled` or is older than an hour, so of two starters in two processes one wins; the settle writes `pull:settled` when it ends. A refused `pull` job queues no stage and says "a pull is already running" in its result and a notice, and the CLI prints the same and exits 1. The marker travels in every batch's payload, the plugins stamp with the marker they were handed, and the settle reads its marker off its own batches' payloads, which a batch that raised still has. The hour is the escape for a run that died: a killed process or a killed cascade never settles, and blocking every pull until someone clears a key would be worse than one duplicated fetch after an hour. The guard lives in `app` because `pipeline`'s queue dedups a job against a live job of its kind, not a cascade against a live cascade, and `solo=True` on the `pull` job would not help, since that job ends in milliseconds once it has asked for the chain. **Boards follow companies; searches find them.** `ats` only reads companies someone listed, so it cannot surface a role at a company nobody knew of, and the seed list came from a US-centred project: of 29,000 open roles, 31 were in Denmark. The `search` plugin runs saved queries on boards that span companies, as `search` subscriptions whose `params` are the query, and pulls them in the same task as the boards. diff --git a/plugins/ats.py b/plugins/ats.py index 6bf1929..2e8bdf6 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -644,10 +644,12 @@ class ATSSource(Source): def fetch(self) -> Iterator[Item]: with httpx.Client() as client: - # Written by the pull job before any board is fetched (see - # `app/jobs/pull.py`); a plugin run on its own — a test, a dry - # run — has no run to stamp itself against, so it makes one up. - run_marker = self.state.get("ats:run", "") or datetime.now(UTC).isoformat( + # Handed in by the batch that runs this plugin, as the marker of + # the pull run that queued it (see `app/jobs/pull.py`), and never + # read from state, where a later run may have replaced it. A + # plugin run on its own — a test, a dry run — has no run to stamp + # itself against, so it makes one up. + run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat( timespec="seconds") for target, label in self.targets(): diff --git a/plugins/search.py b/plugins/search.py index 26e4608..0c6b2f3 100644 --- a/plugins/search.py +++ b/plugins/search.py @@ -340,7 +340,8 @@ class SearchSource(Source): def fetch(self) -> Iterator[Item]: with httpx.Client() as client: - run_marker = self.state.get("search:run", "") or datetime.now(UTC).isoformat( + # The pull run's marker, handed in by its batch as for `ats`. + run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat( timespec="seconds") for target, label in self.targets(): provider = provider_of(target) diff --git a/tests/test_ats.py b/tests/test_ats.py index 299ef2c..2d9d294 100644 --- a/tests/test_ats.py +++ b/tests/test_ats.py @@ -310,8 +310,8 @@ def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource: return answer monkeypatch.setattr(ats, "_fetch_rows", fake) - source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}]}) - source.state = {"ats:run": "run-1"} + source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}], + "run": "run-1"}) return source diff --git a/tests/test_pull.py b/tests/test_pull.py index 2efbdac..d7aa695 100644 --- a/tests/test_pull.py +++ b/tests/test_pull.py @@ -6,6 +6,8 @@ from __future__ import annotations import json import sys +import threading +from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime, timedelta from pathlib import Path from typing import ClassVar @@ -17,10 +19,10 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from desk.models import Item # noqa: E402 -from pipeline import Cancelled, RunContext # noqa: E402 +from pipeline import Cancelled, RunContext, Tally # noqa: E402 from plugins import ats # noqa: E402 from plugins.ats import ATSSource # noqa: E402 -from app.jobs import pull # noqa: E402 +from app.jobs import cascade, pull # noqa: E402 from app.storage import Store # noqa: E402 @@ -55,12 +57,14 @@ def test_boards_batches_fetchable_subscriptions_and_stamps_a_run(store: Store) - batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] assert sum(len(b["targets"]) for b in batches) == 3 - assert store.state.get("ats:run", "") != "" + run = store.state.get(pull.RUN, "") + assert run and {b["run"] for b in batches} == {run} + assert pull.running(store) == run def test_boards_is_empty_and_stamps_nothing_with_no_subscriptions(store: Store) -> None: assert pull._boards(FakeConfig(), store) == [] # type: ignore[arg-type] - assert store.state.get("ats:run", "") == "" + assert store.state.get(pull.RUN, "") == "" def test_run_batch_upserts_every_item_the_plugin_yields( @@ -71,15 +75,15 @@ def test_run_batch_upserts_every_item_the_plugin_yields( def fake_fetch(self: ATSSource) -> object: yield _item("greenhouse", "acme", "1") yield _item("greenhouse", "acme", "2") - self.state["ats:ok:greenhouse:acme"] = self.state.get("ats:run", "") + self.state["ats:ok:greenhouse:acme"] = self.options["run"] self.state["ats:ids:greenhouse:acme"] = '["1", "2"]' monkeypatch.setattr(ATSSource, "fetch", fake_fetch) - store.state["ats:run"] = "run-1" - result = pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme"]) + result = pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme"], run="run-1") - assert result == {"roles": 2} + assert result == {"roles": 2, "run": "run-1"} + assert store.state["ats:ok:greenhouse:acme"] == "run-1" assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:acme:2"} assert store.roles()[0].subscription_id is not None @@ -126,13 +130,12 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists( # acme answered this run and now lists only "1"; other's board failed # (or its batch never got to it) and stamped nothing. - store.state["ats:run"] = "run-1" store.state["ats:ok:greenhouse:acme"] = "run-1" store.state["ats:complete:greenhouse:acme"] = "1" store.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]' class FakeTally: - results = ({"roles": 1},) + results = ({"roles": 1, "run": "run-1"},) result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] @@ -149,12 +152,11 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run( # A stale stamp from a run that is not this one — e.g. this run's batch # for this board never finished. - store.state["ats:run"] = "run-2" store.state["ats:ok:greenhouse:acme"] = "run-1" store.state["ats:ids:greenhouse:acme"] = "[]" class FakeTally: - results: tuple[dict[str, int], ...] = () + results = ({"roles": 0, "run": "run-2"},) result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] @@ -182,13 +184,9 @@ def _answer(monkeypatch: pytest.MonkeyPatch, answer: object) -> None: def _pull(store: Store) -> dict[str, object]: - for batch in pull._boards(FakeConfig(), store): # type: ignore[arg-type] - pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] - - class Tally: - results: ClassVar[list[dict[str, object]]] = [] - - return pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type] + results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] + for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type] + return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] def _open(store: Store) -> set[str]: @@ -246,3 +244,117 @@ def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens( _pull(acme) assert _open(acme) == {"workable:acme:2"} assert pull.failing_boards(acme) == {} + + +# ------------------------------------------------------ one pull at a time + +def _ago(**delta: float) -> str: + return (datetime.now(UTC) - timedelta(**delta)).isoformat(timespec="microseconds") + + +def test_a_pull_during_a_running_one_is_refused_and_queues_nothing(acme: Store) -> None: + first = pull.claim_run(acme) + assert first and pull.running(acme) == first + + asked: list[object] = [] + said: list[object] = [] + context = RunContext(on_task=lambda task: asked.append(task) or True, + on_notice=said.append) + result = cascade.run_pull(FakeConfig(), acme, context=context) + + assert result == {"skipped": pull.ALREADY_RUNNING, "running": first} + assert asked == [] + assert [n.code for n in said] == ["pull.running"] # type: ignore[attr-defined] + # A chain started without the job's claim (a test, a script) finds + # nothing to queue either, and the running run's marker stands. + assert pull._boards(FakeConfig(), acme) == [] # type: ignore[arg-type] + assert acme.state[pull.RUN] == first + + +def test_a_pull_starts_once_the_last_one_settled(acme: Store, + monkeypatch: pytest.MonkeyPatch) -> None: + _answer(monkeypatch, ([_row("1")], True)) + _pull(acme) + assert pull.running(acme) == "" + + asked: list[object] = [] + context = RunContext(on_task=lambda task: asked.append(task) or True) + result = cascade.run_pull(FakeConfig(), acme, context=context) + assert result["run"] == acme.state[pull.RUN] and len(asked) == 1 + # The chain's finder reads under the job's claim instead of claiming again. + [batch] = asked[0].finder(FakeConfig(), acme) # type: ignore[attr-defined] + assert batch["run"] == result["run"] + + +def test_starters_claiming_at_once_give_one_run(tmp_path: Path) -> None: + """Eight starters, each on its own connection to one file, as two + processes would be, released together: one claim wins.""" + Store(tmp_path / "shared.db") + stores = [Store(tmp_path / "shared.db") for _ in range(8)] + barrier = threading.Barrier(len(stores)) + + def claim(store: Store) -> str: + barrier.wait() + return pull.claim_run(store) + + with ThreadPoolExecutor(len(stores)) as pool: + claims = list(pool.map(claim, stores)) + assert sum(bool(c) for c in claims) == 1 + + +def test_an_abandoned_run_is_taken_over_after_the_threshold(acme: Store) -> None: + hours = pull.ABANDONED.total_seconds() / 3600 + acme.state[pull.RUN] = _ago(hours=hours / 2) + assert pull.claim_run(acme) == "" + acme.state[pull.RUN] = _ago(hours=hours + 0.1) + taken = pull.claim_run(acme) + assert taken and acme.state[pull.RUN] == taken + + +def test_a_batch_stamps_with_the_marker_it_was_handed( + acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _answer(monkeypatch, ([_row("1")], True)) + [batch] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type] + acme.state[pull.RUN] = "a later run" + pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type] + assert acme.state["ats:ok:workable:acme"] == batch["run"] + + +def test_run_a_settle_does_not_close_on_run_b_ids( + acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: + """Run A read the board in full; run B, which took over, read it again and + listed less before A's settle ran. A's settle compares with A's marker, so + B's stamp and B's ids close nothing, and A's settle leaves B's claim held; + B's own settle is the one that closes.""" + _answer(monkeypatch, ([_row("1"), _row("2")], True)) + [a] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type] + a_result = pull.run_batch(FakeConfig(), acme, **a) # type: ignore[arg-type] + + acme.state[pull.RUN] = _ago(hours=2) # A looks abandoned + _answer(monkeypatch, ([_row("1")], True)) + [b] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type] + assert b["run"] != a["run"] + b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type] + + result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type] + assert (result["failed"], result["closed"]) == (1, 0) + assert _open(acme) == {"workable:acme:1", "workable:acme:2"} + assert pull.running(acme) == b["run"] + + result = pull.close_boards(FakeConfig(), acme, Tally(results=(b_result,))) # type: ignore[arg-type] + assert result["closed"] == 1 + assert _open(acme) == {"workable:acme:1"} + assert pull.running(acme) == "" + + +def test_the_settle_reads_its_marker_off_its_batches_payloads(acme: Store) -> None: + """A batch that raised has no result, but its payload still names the run.""" + run = pull.claim_run(acme) + acme.state["ats:ok:workable:acme"] = run + acme.state["ats:complete:workable:acme"] = "1" + acme.state["ats:ids:workable:acme"] = "[]" + failed = type("Job", (), {"payload": {"targets": ["workable:acme"], "run": run}, + "result": None, "state": "failed"})() + context = RunContext(waited_for=(failed,)) + result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type] + assert result["boards"] == 1 and pull.running(acme) == "" diff --git a/tests/test_score.py b/tests/test_score.py index 83daa34..fe51854 100644 --- a/tests/test_score.py +++ b/tests/test_score.py @@ -491,7 +491,7 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise( published_at=datetime(2026, 9, 20, tzinfo=UTC), raw={"company": "Acme", "board": "greenhouse", "slug": "acme"}) yield item - self.state["ats:ok:greenhouse:acme"] = self.state.get("ats:run", "") + self.state["ats:ok:greenhouse:acme"] = self.options["run"] self.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]' monkeypatch.setattr(ATSSource, "fetch", fake_fetch) @@ -516,6 +516,28 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise( assert work == ["pull", "pull_batch", "embed", "score", "score"] assert all(j.state == "done" for j in jobs) assert len(store.scores(agent.id)) == 1 + # The board was stamped with the run the `pull` job claimed, and the + # settle let that run go, so the next pull may start. + run = queue.jobs.get(started.id).result["run"] # type: ignore[union-attr] + assert store.state["ats:ok:greenhouse:acme"] == run == store.state["pull:settled"] + + +def test_the_pull_job_queues_nothing_while_another_pull_runs( + store: Store, config: Config, tmp_path: Path) -> None: + from app.jobs import pull + + store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme", + board="greenhouse") + running = pull.claim_run(store) + queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config, store, + workers={"cpu": 1, "http": 1, "llm": 1}) + started = queue.enqueue("pull") + queue.drain(timeout=60) + + done = queue.jobs.get(started.id) # type: ignore[union-attr] + assert done is not None and done.state == "done" + assert done.result == {"skipped": pull.ALREADY_RUNNING, "running": running} + assert [j.kind for j in queue.jobs.of_cascade(done.cascade_id)] == ["pull"] def test_pull_is_a_job_on_a_six_hour_schedule_and_the_stages_are_tasks() -> None: diff --git a/tests/test_search.py b/tests/test_search.py index 8f5e16d..a18f404 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -21,6 +21,7 @@ from plugins import search # noqa: E402 from plugins.search import SearchSource # noqa: E402 from app.config import SearchSeed # noqa: E402 from app.jobs import pull # noqa: E402 +from pipeline import Tally # noqa: E402 from app.models import country_of, remote_of # noqa: E402 from app.storage import Store # noqa: E402 @@ -229,14 +230,9 @@ def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool def _pull(store: Store) -> dict[str, Any]: - batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] - for batch in batches: - pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] - - class Tally: - results: ClassVar[list[dict[str, Any]]] = [] - - return pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type] + results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] + for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type] + return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] def test_a_complete_search_closes_what_it_stopped_listing( @@ -306,13 +302,10 @@ def test_a_search_switched_off_mid_pull_ends_the_pull_closed( board="jobindex", params={"q": "backend"}) _stub_search(monkeypatch, ["h1"], complete=False) batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] - pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type] + result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type] store.update_subscription(sub.id, subscribed=False, fetch=False) - class Tally: - results: ClassVar[list[dict[str, Any]]] = [] - - pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type] + pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type] assert store.roles() == [] # And a batch that starts after the switch reads nothing. - assert pull.run_batch(FakeConfig(), store, **batches[0]) == {"roles": 0} # type: ignore[arg-type] + assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]