One pull at a time: the run is claimed in the database and its marker rides in each batch

A second pull (schedule, Run button, `app job pull` in another process) could
start while one was fetching and overwrite the stored run marker, so the first
run's boards failed their stamp check or a settle closed on the other run's id
lists. `claim_run` sets `pull:run` in one conditional statement, only while it
equals `pull:settled` or is older than an hour; a refused `pull` job queues no
stage and says so in its result and a notice, and the CLI exits 1. Batches
carry the marker in their payload, the plugins stamp with it, and the settle
reads it off its own batches' payloads before releasing the run.

Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus committed 2026-09-29 14:12:17 +02:00
1 parent c7da12b93d
commit b43fc54565
11 files changed
+328 -70

No files matched your search

+7 -14
View File
@@ -21,6 +21,7 @@ from plugins import search # noqa: E402
from plugins.search import SearchSource # noqa: E402
from app.config import SearchSeed # noqa: E402
from app.jobs import pull # noqa: E402
from pipeline import Tally # noqa: E402
from app.models import country_of, remote_of # noqa: E402
from app.storage import Store # noqa: E402
@@ -229,14 +230,9 @@ def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool
def _pull(store: Store) -> dict[str, Any]:
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
for batch in batches:
pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
class Tally:
results: ClassVar[list[dict[str, Any]]] = []
return pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
def test_a_complete_search_closes_what_it_stopped_listing(
@@ -306,13 +302,10 @@ def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
board="jobindex", params={"q": "backend"})
_stub_search(monkeypatch, ["h1"], complete=False)
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
store.update_subscription(sub.id, subscribed=False, fetch=False)
class Tally:
results: ClassVar[list[dict[str, Any]]] = []
pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type]
assert store.roles() == []
# And a batch that starts after the switch reads nothing.
assert pull.run_batch(FakeConfig(), store, **batches[0]) == {"roles": 0} # type: ignore[arg-type]
assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]