One pull at a time: the run is claimed in the database and its marker rides in each batch
A second pull (schedule, Run button, `app job pull` in another process) could start while one was fetching and overwrite the stored run marker, so the first run's boards failed their stamp check or a settle closed on the other run's id lists. `claim_run` sets `pull:run` in one conditional statement, only while it equals `pull:settled` or is older than an hour; a refused `pull` job queues no stage and says so in its result and a notice, and the CLI exits 1. Batches carry the marker in their payload, the plugins stamp with it, and the settle reads it off its own batches' payloads before releasing the run. Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
1 parent
c7da12b93d
commit
b43fc54565
11 files changed
+328
-70
No files matched your search
+2
-2
@@ -310,8 +310,8 @@ def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource:
|
||||
return answer
|
||||
|
||||
monkeypatch.setattr(ats, "_fetch_rows", fake)
|
||||
source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}]})
|
||||
source.state = {"ats:run": "run-1"}
|
||||
source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}],
|
||||
"run": "run-1"})
|
||||
return source
|
||||
|
||||
|
||||
|
||||
+131
-19
@@ -6,6 +6,8 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
@@ -17,10 +19,10 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from desk.models import Item # noqa: E402
|
||||
from pipeline import Cancelled, RunContext # noqa: E402
|
||||
from pipeline import Cancelled, RunContext, Tally # noqa: E402
|
||||
from plugins import ats # noqa: E402
|
||||
from plugins.ats import ATSSource # noqa: E402
|
||||
from app.jobs import pull # noqa: E402
|
||||
from app.jobs import cascade, pull # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
|
||||
|
||||
@@ -55,12 +57,14 @@ def test_boards_batches_fetchable_subscriptions_and_stamps_a_run(store: Store) -
|
||||
|
||||
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
|
||||
assert sum(len(b["targets"]) for b in batches) == 3
|
||||
assert store.state.get("ats:run", "") != ""
|
||||
run = store.state.get(pull.RUN, "")
|
||||
assert run and {b["run"] for b in batches} == {run}
|
||||
assert pull.running(store) == run
|
||||
|
||||
|
||||
def test_boards_is_empty_and_stamps_nothing_with_no_subscriptions(store: Store) -> None:
|
||||
assert pull._boards(FakeConfig(), store) == [] # type: ignore[arg-type]
|
||||
assert store.state.get("ats:run", "") == ""
|
||||
assert store.state.get(pull.RUN, "") == ""
|
||||
|
||||
|
||||
def test_run_batch_upserts_every_item_the_plugin_yields(
|
||||
@@ -71,15 +75,15 @@ def test_run_batch_upserts_every_item_the_plugin_yields(
|
||||
def fake_fetch(self: ATSSource) -> object:
|
||||
yield _item("greenhouse", "acme", "1")
|
||||
yield _item("greenhouse", "acme", "2")
|
||||
self.state["ats:ok:greenhouse:acme"] = self.state.get("ats:run", "")
|
||||
self.state["ats:ok:greenhouse:acme"] = self.options["run"]
|
||||
self.state["ats:ids:greenhouse:acme"] = '["1", "2"]'
|
||||
|
||||
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
||||
store.state["ats:run"] = "run-1"
|
||||
|
||||
result = pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme"])
|
||||
result = pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme"], run="run-1")
|
||||
|
||||
assert result == {"roles": 2}
|
||||
assert result == {"roles": 2, "run": "run-1"}
|
||||
assert store.state["ats:ok:greenhouse:acme"] == "run-1"
|
||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:acme:2"}
|
||||
assert store.roles()[0].subscription_id is not None
|
||||
|
||||
@@ -126,13 +130,12 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
|
||||
|
||||
# acme answered this run and now lists only "1"; other's board failed
|
||||
# (or its batch never got to it) and stamped nothing.
|
||||
store.state["ats:run"] = "run-1"
|
||||
store.state["ats:ok:greenhouse:acme"] = "run-1"
|
||||
store.state["ats:complete:greenhouse:acme"] = "1"
|
||||
store.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
|
||||
|
||||
class FakeTally:
|
||||
results = ({"roles": 1},)
|
||||
results = ({"roles": 1, "run": "run-1"},)
|
||||
|
||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||
|
||||
@@ -149,12 +152,11 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
|
||||
|
||||
# A stale stamp from a run that is not this one — e.g. this run's batch
|
||||
# for this board never finished.
|
||||
store.state["ats:run"] = "run-2"
|
||||
store.state["ats:ok:greenhouse:acme"] = "run-1"
|
||||
store.state["ats:ids:greenhouse:acme"] = "[]"
|
||||
|
||||
class FakeTally:
|
||||
results: tuple[dict[str, int], ...] = ()
|
||||
results = ({"roles": 0, "run": "run-2"},)
|
||||
|
||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||
|
||||
@@ -182,13 +184,9 @@ def _answer(monkeypatch: pytest.MonkeyPatch, answer: object) -> None:
|
||||
|
||||
|
||||
def _pull(store: Store) -> dict[str, object]:
|
||||
for batch in pull._boards(FakeConfig(), store): # type: ignore[arg-type]
|
||||
pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
||||
|
||||
class Tally:
|
||||
results: ClassVar[list[dict[str, object]]] = []
|
||||
|
||||
return pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
|
||||
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
||||
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
|
||||
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _open(store: Store) -> set[str]:
|
||||
@@ -246,3 +244,117 @@ def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens(
|
||||
_pull(acme)
|
||||
assert _open(acme) == {"workable:acme:2"}
|
||||
assert pull.failing_boards(acme) == {}
|
||||
|
||||
|
||||
# ------------------------------------------------------ one pull at a time
|
||||
|
||||
def _ago(**delta: float) -> str:
|
||||
return (datetime.now(UTC) - timedelta(**delta)).isoformat(timespec="microseconds")
|
||||
|
||||
|
||||
def test_a_pull_during_a_running_one_is_refused_and_queues_nothing(acme: Store) -> None:
|
||||
first = pull.claim_run(acme)
|
||||
assert first and pull.running(acme) == first
|
||||
|
||||
asked: list[object] = []
|
||||
said: list[object] = []
|
||||
context = RunContext(on_task=lambda task: asked.append(task) or True,
|
||||
on_notice=said.append)
|
||||
result = cascade.run_pull(FakeConfig(), acme, context=context)
|
||||
|
||||
assert result == {"skipped": pull.ALREADY_RUNNING, "running": first}
|
||||
assert asked == []
|
||||
assert [n.code for n in said] == ["pull.running"] # type: ignore[attr-defined]
|
||||
# A chain started without the job's claim (a test, a script) finds
|
||||
# nothing to queue either, and the running run's marker stands.
|
||||
assert pull._boards(FakeConfig(), acme) == [] # type: ignore[arg-type]
|
||||
assert acme.state[pull.RUN] == first
|
||||
|
||||
|
||||
def test_a_pull_starts_once_the_last_one_settled(acme: Store,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row("1")], True))
|
||||
_pull(acme)
|
||||
assert pull.running(acme) == ""
|
||||
|
||||
asked: list[object] = []
|
||||
context = RunContext(on_task=lambda task: asked.append(task) or True)
|
||||
result = cascade.run_pull(FakeConfig(), acme, context=context)
|
||||
assert result["run"] == acme.state[pull.RUN] and len(asked) == 1
|
||||
# The chain's finder reads under the job's claim instead of claiming again.
|
||||
[batch] = asked[0].finder(FakeConfig(), acme) # type: ignore[attr-defined]
|
||||
assert batch["run"] == result["run"]
|
||||
|
||||
|
||||
def test_starters_claiming_at_once_give_one_run(tmp_path: Path) -> None:
|
||||
"""Eight starters, each on its own connection to one file, as two
|
||||
processes would be, released together: one claim wins."""
|
||||
Store(tmp_path / "shared.db")
|
||||
stores = [Store(tmp_path / "shared.db") for _ in range(8)]
|
||||
barrier = threading.Barrier(len(stores))
|
||||
|
||||
def claim(store: Store) -> str:
|
||||
barrier.wait()
|
||||
return pull.claim_run(store)
|
||||
|
||||
with ThreadPoolExecutor(len(stores)) as pool:
|
||||
claims = list(pool.map(claim, stores))
|
||||
assert sum(bool(c) for c in claims) == 1
|
||||
|
||||
|
||||
def test_an_abandoned_run_is_taken_over_after_the_threshold(acme: Store) -> None:
|
||||
hours = pull.ABANDONED.total_seconds() / 3600
|
||||
acme.state[pull.RUN] = _ago(hours=hours / 2)
|
||||
assert pull.claim_run(acme) == ""
|
||||
acme.state[pull.RUN] = _ago(hours=hours + 0.1)
|
||||
taken = pull.claim_run(acme)
|
||||
assert taken and acme.state[pull.RUN] == taken
|
||||
|
||||
|
||||
def test_a_batch_stamps_with_the_marker_it_was_handed(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row("1")], True))
|
||||
[batch] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
|
||||
acme.state[pull.RUN] = "a later run"
|
||||
pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type]
|
||||
assert acme.state["ats:ok:workable:acme"] == batch["run"]
|
||||
|
||||
|
||||
def test_run_a_settle_does_not_close_on_run_b_ids(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Run A read the board in full; run B, which took over, read it again and
|
||||
listed less before A's settle ran. A's settle compares with A's marker, so
|
||||
B's stamp and B's ids close nothing, and A's settle leaves B's claim held;
|
||||
B's own settle is the one that closes."""
|
||||
_answer(monkeypatch, ([_row("1"), _row("2")], True))
|
||||
[a] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
|
||||
a_result = pull.run_batch(FakeConfig(), acme, **a) # type: ignore[arg-type]
|
||||
|
||||
acme.state[pull.RUN] = _ago(hours=2) # A looks abandoned
|
||||
_answer(monkeypatch, ([_row("1")], True))
|
||||
[b] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
|
||||
assert b["run"] != a["run"]
|
||||
b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type]
|
||||
|
||||
result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type]
|
||||
assert (result["failed"], result["closed"]) == (1, 0)
|
||||
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
||||
assert pull.running(acme) == b["run"]
|
||||
|
||||
result = pull.close_boards(FakeConfig(), acme, Tally(results=(b_result,))) # type: ignore[arg-type]
|
||||
assert result["closed"] == 1
|
||||
assert _open(acme) == {"workable:acme:1"}
|
||||
assert pull.running(acme) == ""
|
||||
|
||||
|
||||
def test_the_settle_reads_its_marker_off_its_batches_payloads(acme: Store) -> None:
|
||||
"""A batch that raised has no result, but its payload still names the run."""
|
||||
run = pull.claim_run(acme)
|
||||
acme.state["ats:ok:workable:acme"] = run
|
||||
acme.state["ats:complete:workable:acme"] = "1"
|
||||
acme.state["ats:ids:workable:acme"] = "[]"
|
||||
failed = type("Job", (), {"payload": {"targets": ["workable:acme"], "run": run},
|
||||
"result": None, "state": "failed"})()
|
||||
context = RunContext(waited_for=(failed,))
|
||||
result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type]
|
||||
assert result["boards"] == 1 and pull.running(acme) == ""
|
||||
+23
-1
@@ -491,7 +491,7 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
|
||||
published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
||||
raw={"company": "Acme", "board": "greenhouse", "slug": "acme"})
|
||||
yield item
|
||||
self.state["ats:ok:greenhouse:acme"] = self.state.get("ats:run", "")
|
||||
self.state["ats:ok:greenhouse:acme"] = self.options["run"]
|
||||
self.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
|
||||
|
||||
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
||||
@@ -516,6 +516,28 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
|
||||
assert work == ["pull", "pull_batch", "embed", "score", "score"]
|
||||
assert all(j.state == "done" for j in jobs)
|
||||
assert len(store.scores(agent.id)) == 1
|
||||
# The board was stamped with the run the `pull` job claimed, and the
|
||||
# settle let that run go, so the next pull may start.
|
||||
run = queue.jobs.get(started.id).result["run"] # type: ignore[union-attr]
|
||||
assert store.state["ats:ok:greenhouse:acme"] == run == store.state["pull:settled"]
|
||||
|
||||
|
||||
def test_the_pull_job_queues_nothing_while_another_pull_runs(
|
||||
store: Store, config: Config, tmp_path: Path) -> None:
|
||||
from app.jobs import pull
|
||||
|
||||
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
||||
board="greenhouse")
|
||||
running = pull.claim_run(store)
|
||||
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config, store,
|
||||
workers={"cpu": 1, "http": 1, "llm": 1})
|
||||
started = queue.enqueue("pull")
|
||||
queue.drain(timeout=60)
|
||||
|
||||
done = queue.jobs.get(started.id) # type: ignore[union-attr]
|
||||
assert done is not None and done.state == "done"
|
||||
assert done.result == {"skipped": pull.ALREADY_RUNNING, "running": running}
|
||||
assert [j.kind for j in queue.jobs.of_cascade(done.cascade_id)] == ["pull"]
|
||||
|
||||
|
||||
def test_pull_is_a_job_on_a_six_hour_schedule_and_the_stages_are_tasks() -> None:
|
||||
|
||||
+7
-14
@@ -21,6 +21,7 @@ from plugins import search # noqa: E402
|
||||
from plugins.search import SearchSource # noqa: E402
|
||||
from app.config import SearchSeed # noqa: E402
|
||||
from app.jobs import pull # noqa: E402
|
||||
from pipeline import Tally # noqa: E402
|
||||
from app.models import country_of, remote_of # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
|
||||
@@ -229,14 +230,9 @@ def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool
|
||||
|
||||
|
||||
def _pull(store: Store) -> dict[str, Any]:
|
||||
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
|
||||
for batch in batches:
|
||||
pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
||||
|
||||
class Tally:
|
||||
results: ClassVar[list[dict[str, Any]]] = []
|
||||
|
||||
return pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
|
||||
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
||||
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
|
||||
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_a_complete_search_closes_what_it_stopped_listing(
|
||||
@@ -306,13 +302,10 @@ def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
|
||||
board="jobindex", params={"q": "backend"})
|
||||
_stub_search(monkeypatch, ["h1"], complete=False)
|
||||
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
|
||||
pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
|
||||
result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
|
||||
store.update_subscription(sub.id, subscribed=False, fetch=False)
|
||||
|
||||
class Tally:
|
||||
results: ClassVar[list[dict[str, Any]]] = []
|
||||
|
||||
pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
|
||||
pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type]
|
||||
assert store.roles() == []
|
||||
# And a batch that starts after the switch reads nothing.
|
||||
assert pull.run_batch(FakeConfig(), store, **batches[0]) == {"roles": 0} # type: ignore[arg-type]
|
||||
assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]
|
||||
Reference in new issue
Block a user