One pull at a time: the run is claimed in the database and its marker rides in each batch

A second pull (schedule, Run button, `app job pull` in another process) could
start while one was fetching and overwrite the stored run marker, so the first
run's boards failed their stamp check or a settle closed on the other run's id
lists. `claim_run` sets `pull:run` in one conditional statement, only while it
equals `pull:settled` or is older than an hour; a refused `pull` job queues no
stage and says so in its result and a notice, and the CLI exits 1. Batches
carry the marker in their payload, the plugins stamp with it, and the settle
reads it off its own batches' payloads before releasing the run.

Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus committed 2026-09-29 14:12:17 +02:00
1 parent c7da12b93d
commit b43fc54565
11 files changed
+328 -70

No files matched your search

+11 -2
View File
@@ -176,8 +176,17 @@ def run_job(args: argparse.Namespace) -> int:
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=str(config.path or ""),
tz=config.timezone, schedules=False)
if chained or args.kind not in JOBS:
task_id = queue.start_task(cascade.chain(args.kind) if chained
else TASKS[args.kind]())
first = cascade.chain(args.kind) if chained else TASKS[args.kind]()
if args.kind == "pull":
# The same claim the `pull` job makes, so a pull here and one in
# the server's worker cannot both fetch.
run = jobs.pull.claim_run(store)
if not run:
print(json.dumps({"task": "pull", "result": jobs.pull.ALREADY_RUNNING,
"running": jobs.pull.running(store)}))
return 1
first = cascade.pull_chain(run)
task_id = queue.start_task(first)
queue.drain()
started = queue.jobs.of_task(task_id)
if not started:
+33 -6
View File
@@ -34,7 +34,7 @@ from typing import Any
from app.jobs import embed, extract, pull, score, summarise # noqa: F401 -- registers the jobs
from desk.tools import agent
from pipeline import NULL, TASKS, RunContext, Tally, Task, declare, job
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
STEP = "app.jobs.cascade:step"
@@ -129,17 +129,44 @@ def chain(name: str) -> Task:
return task(CASCADES[name][0], cascade=name, at=0)
def pull_chain(run: str) -> Task:
"""The pull cascade for a run `pull.claim_run` has claimed: its finder
reads under that marker rather than claiming one of its own."""
return replace(chain("pull"), finder=pull.pull_task(run).finder)
def already_running(since: str) -> Notice:
"""The notice for a pull refused because another is in flight."""
return Notice(
code="pull.running", level="info", kind="expected", key="pull",
text=f"Not started: {pull.ALREADY_RUNNING}"
+ (f" (since {since[:19].replace('T', ' ')} UTC)." if since else "."),
where="app.jobs.pull")
@job("pull", "Pulling every board", resource="http")
def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
"""Ask for the pull cascade once this job is done. Its own work is nil.
"""Claim a pull run and ask for the pull cascade once this job is done.
Its own work is nil.
A pull whose claim is refused, because another run is still unsettled,
queues nothing, embed to summarise included, and says so in its result
and a notice: a second cascade would fetch every board again and could
send the same roles to the model twice.
Under `NULL` there is no queue to take the chain, so this raises rather
than report a pull that never happened; `app job pull` starts the chain
on a queue of its own instead of calling this.
than report a pull that never happened; `app job pull` claims the run and
starts the chain on a queue of its own instead of calling this.
"""
if not context.task(chain("pull")):
run = pull.claim_run(store)
if not run:
since = pull.running(store)
context.notice(already_running(since))
return {"skipped": pull.ALREADY_RUNNING, "running": since}
if not context.task(pull_chain(run)):
pull.release_run(store, run)
raise RuntimeError("no queue to start the pull cascade on")
return {"started": " > ".join(CASCADES["pull"])}
return {"started": " > ".join(CASCADES["pull"]), "run": run}
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
+97 -20
View File
@@ -20,6 +20,15 @@ A search closes the same way when it listed everything that matches (see
roles only once no search has seen them for `STALE_DAYS`. A board
that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`)
closes all of its roles, and its next answer reopens what it lists.
One pull at a time. A run is claimed in the database (`claim_run`: `pull:run`
set only while the previous run is settled, in one statement), so the
schedule, the Run button and `app job pull` in another process cannot start a
second fetch while one is in flight; a claim older than `ABANDONED` with no
settle is a run that died and is taken over. The marker travels in every
batch's payload, the plugins stamp with the marker they were handed, and the
settle compares against the marker its own batches carried, so neither a
later run's marker nor its id lists reach this run's closure.
"""
from __future__ import annotations
@@ -27,6 +36,7 @@ from __future__ import annotations
import json
import logging
from datetime import UTC, datetime, timedelta
from functools import partial
from typing import Any
from pipeline import NULL, RunContext, Tally, Task, job
@@ -58,6 +68,19 @@ FAILING = "ats:failing:"
#: The `state` key that records the starter searches were imported.
SEARCHES_SEEDED = "searches:seeded"
#: The marker of the pull run claimed last, and of the one settled last. A run
#: is in flight while the two differ.
RUN = "pull:run"
SETTLED = "pull:settled"
#: How old an unsettled claim may be before it counts as a run that died (a
#: killed process, a killed cascade) and the next pull takes over. A pull of
#: every board takes minutes.
ABANDONED = timedelta(hours=1)
#: What a refused pull says, in its result and its notice.
ALREADY_RUNNING = "a pull is already running"
def _now() -> str:
"""A run marker: to the microsecond, so two pulls a second apart are two
@@ -65,26 +88,54 @@ def _now() -> str:
return datetime.now(UTC).isoformat(timespec="microseconds")
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
def claim_run(store: Store) -> str:
"""Start a pull run: a new marker in `pull:run`, returned, unless another
run is still unsettled and younger than `ABANDONED`, which returns "".
One conditional write, so two starters in two processes cannot both win."""
marker = _now()
stale = (datetime.now(UTC) - ABANDONED).isoformat(timespec="microseconds")
return marker if store.claim_state(RUN, marker, released=SETTLED, stale=stale) else ""
def running(store: Store) -> str:
"""The marker of the pull run in flight, or "" when the last one settled."""
run = store.state.get(RUN, "")
return run if run and run != store.state.get(SETTLED, "") else ""
def release_run(store: Store, run: str) -> None:
"""Mark `run` settled, unless a later run has taken over since: letting go
of a claim that is no longer ours would release the one that is."""
if store.state.get(RUN, "") == run:
store.state[SETTLED] = run
def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]:
"""Every fetchable `ats` and `search` subscription, ten to a batch of one
source. Stamps the run marker `close_boards` will look for, before any
batch runs — finders run synchronously, in whoever starts the task, ahead
of every job it queues."""
source, each batch carrying the run marker its plugin stamps with and
`close_boards` compares against. `run` is a marker the caller claimed
already (the `pull` job, `app job pull`); without one the finder claims
its own, and queues nothing when another run is in flight. A run with
nothing to read is settled here, since no settle will see a batch of it."""
seed_searches(config, store)
batches: list[dict[str, Any]] = []
for source in SOURCES:
subs = store.subscriptions(source=source, fetchable=True)
if not subs:
continue
store.state[f"{source}:run"] = _now()
batches += [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]], "source": source}
for i in range(0, len(subs), BATCH_SIZE)]
return batches
targets = [(source, [s.target for s in store.subscriptions(source=source, fetchable=True)])
for source in SOURCES]
if not any(found for _, found in targets):
if run:
release_run(store, run)
return []
run = run or claim_run(store)
if not run:
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
running(store))
return []
return [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
@job("pull_batch", "Reading boards", resource="http")
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
context: RunContext = NULL) -> dict[str, Any]:
run: str = "", context: RunContext = NULL) -> dict[str, Any]:
"""Fetch this batch's boards or searches and `upsert_role` every item
they list.
@@ -96,6 +147,10 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
reads `options[subscription_field]` — so one job never re-fetches another
job's boards, and `context.check()` runs before each fetch: a Kill lands
between two boards rather than after the tenth.
`run` is the marker of the run that queued this batch, handed to the
plugin as its `run` option, so its stamps belong to that run whatever the
stored marker says by the time the batch runs.
"""
from plugins.ats import ATSSource
from plugins.search import SearchSource
@@ -110,7 +165,7 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
if sub is None:
continue
context.check()
options = dict(config.sources.get(source) or {})
options = dict(config.sources.get(source) or {}) | {"run": run}
plugin: ATSSource | SearchSource
if source == "search":
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
@@ -123,7 +178,18 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
item.source = source
store.upsert_role(item, sub)
stored += 1
return {"roles": stored}
return {"roles": stored, "run": run}
def _run_of(tally: Tally, context: RunContext) -> str:
"""The marker this settle's own batches were queued with: off their
payloads, which a batch that raised still has, else off their results
(the only thing a settle called directly is given)."""
for batch in context.waited_for:
run = (getattr(batch, "payload", None) or {}).get("run")
if run:
return str(run)
return next((str(r["run"]) for r in tally.results if r.get("run")), "")
def close_boards(config: Config, store: Store, tally: Tally, *,
@@ -140,6 +206,12 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
when that list was complete, and otherwise by `STALE_DAYS`. A board that
has failed on every fetch since `STALE_DAYS` ago closes all its roles.
The stamps are compared with this run's own marker, the one its batches
carried, never with the stored `pull:run`, which a later run may have
replaced. A settle with no batch behind it (a refused pull, nothing to
read) closes nothing by list or by age. Last, the run is marked settled,
which lets the next pull start.
A board or search switched off while its batch was in flight had its
roles reopened by that batch, after the switch closed them, so every
switched-off one is closed again here.
@@ -147,11 +219,13 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
answered = searched = failed = incomplete = closed = 0
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
failing = failing_boards(store)
run_marker = _run_of(tally, context)
for source in SOURCES:
for sub in store.subscriptions(source=source):
if not (sub.subscribed and sub.fetch):
closed += store.close_subscription(sub.id)
run_marker = store.state.get(f"{source}:run", "")
if not run_marker:
continue
for sub in store.subscriptions(source=source, fetchable=True):
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
if not (stamp and stamp == run_marker):
@@ -172,6 +246,8 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
closed += store.mark_closed(source, seen_ids, sub.id)
if searched:
closed += store.close_unseen("search", stale)
if run_marker:
release_run(store, run_marker)
roles = sum(int(result.get("roles", 0)) for result in tally.results)
return {"boards": answered, "searches": searched, "failed": failed,
"incomplete": incomplete, "roles": roles, "closed": closed}
@@ -217,8 +293,9 @@ def seed_searches(config: Config, store: Store) -> list[str]:
return added
def pull_task() -> Task:
def pull_task(run: str = "") -> Task:
"""The pull on its own. `app.jobs.cascade` makes it the first stage
of the chain the `pull` job starts."""
return Task(kind="pull_batch", finder=_boards, name="pull",
of the chain the `pull` job starts. `run` is a marker `claim_run` already
gave the caller; without one the finder claims its own."""
return Task(kind="pull_batch", finder=partial(_boards, run=run), name="pull",
label="Reading every board", settle=SETTLE)
+13
View File
@@ -1237,6 +1237,19 @@ class Store:
"SELECT key, value FROM state WHERE substr(key, 1, ?) = ? ORDER BY key",
(len(prefix), prefix))}
def claim_state(self, key: str, value: str, *, released: str, stale: str) -> bool:
"""Set `key` to `value` only while it is free: unset, equal to the
value under `released` (the claim last let go of), or no later than
`stale`. Returns whether it was set. One statement, so of two
processes claiming at once exactly one wins."""
return self._write(
"""INSERT INTO state (key, value) VALUES (?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value
WHERE state.value <= ?
OR state.value = COALESCE(
(SELECT r.value FROM state AS r WHERE r.key = ?), '')""",
(key, value, stale, released)) > 0
def _check_status(status: str) -> None:
if status not in STATUSES: