One pull at a time: the run is claimed in the database and its marker rides in each batch

A second pull (schedule, Run button, `app job pull` in another process) could
start while one was fetching and overwrite the stored run marker, so the first
run's boards failed their stamp check or a settle closed on the other run's id
lists. `claim_run` sets `pull:run` in one conditional statement, only while it
equals `pull:settled` or is older than an hour; a refused `pull` job queues no
stage and says so in its result and a notice, and the CLI exits 1. Batches
carry the marker in their payload, the plugins stamp with it, and the settle
reads it off its own batches' payloads before releasing the run.

Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus committed 2026-09-29 14:12:17 +02:00
1 parent c7da12b93d
commit b43fc54565
11 files changed
+328 -70

No files matched your search

+23 -1
View File
@@ -491,7 +491,7 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"company": "Acme", "board": "greenhouse", "slug": "acme"})
yield item
self.state["ats:ok:greenhouse:acme"] = self.state.get("ats:run", "")
self.state["ats:ok:greenhouse:acme"] = self.options["run"]
self.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
@@ -516,6 +516,28 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
assert work == ["pull", "pull_batch", "embed", "score", "score"]
assert all(j.state == "done" for j in jobs)
assert len(store.scores(agent.id)) == 1
# The board was stamped with the run the `pull` job claimed, and the
# settle let that run go, so the next pull may start.
run = queue.jobs.get(started.id).result["run"] # type: ignore[union-attr]
assert store.state["ats:ok:greenhouse:acme"] == run == store.state["pull:settled"]
def test_the_pull_job_queues_nothing_while_another_pull_runs(
store: Store, config: Config, tmp_path: Path) -> None:
from app.jobs import pull
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
board="greenhouse")
running = pull.claim_run(store)
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config, store,
workers={"cpu": 1, "http": 1, "llm": 1})
started = queue.enqueue("pull")
queue.drain(timeout=60)
done = queue.jobs.get(started.id) # type: ignore[union-attr]
assert done is not None and done.state == "done"
assert done.result == {"skipped": pull.ALREADY_RUNNING, "running": running}
assert [j.kind for j in queue.jobs.of_cascade(done.cascade_id)] == ["pull"]
def test_pull_is_a_job_on_a_six_hour_schedule_and_the_stages_are_tasks() -> None: