Close a role of a swept board only when two finished sweeps in a row missed it

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 13:23:57 +02:00
1 parent a19c528e3f
commit a91032b60f
5 files changed
+373 -11

No files matched your search

+21 -7
View File
@@ -23,6 +23,11 @@ State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
count `{"asks", "count"}`, the roles the last whole read listed, for
the 10% guard
facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL`
swept "1" when the last stamped listing finished a sweep carried over
from an earlier pull (`Listing.swept`), else "". Offset paging
drifts when roles close between pulls, so `app.jobs.pull` closes
a role of such a board only when two finished sweeps in a row
missed it (it keeps the ids the last one missed in `missing`)
"""
from __future__ import annotations
@@ -126,7 +131,10 @@ class Listing:
state to keep for the next pull, None when no sweep is under way.
`filtered` says the read asked the host for the roles in some places,
`narrowed` for the roles matching words (`narrowed()`): the ids of
neither are the whole board."""
neither are the whole board. `swept` says a complete listing finished a
sweep carried over from an earlier pull: `ids` were read across pulls,
and roles closing meanwhile may have shifted an offset-paged host's later
roles back past the cursor, so `ids` can miss a role that is still open."""
rows: list[Row]
complete: bool
@@ -135,6 +143,7 @@ class Listing:
sweep: dict[str, Any] | None = None
asks: str = ""
narrowed: bool = False
swept: bool = False
# --------------------------------------------------------------- modules
@@ -521,8 +530,9 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns
the `sweep` to keep, and the next pull, handed it back, continues from
its series and cursor; the sweep is complete when it reaches the end,
and `ids` then hold every id it listed. A read at the reader's `ceiling`
is not swept: the host answers nothing past it."""
and `ids` then hold every id it listed, and the listing is `swept`. A
read at the reader's `ceiling` is not swept: the host answers nothing
past it."""
limits = limits or Limits()
ask = ask or Ask()
asks = series(module, ask, facets)
@@ -539,7 +549,8 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
never = reads is not None and reads.complete == "never"
ceiling = reads.ceiling if reads is not None else None
start, cursor, seen, earlier, whole = 0, None, set(), [], True
if sweep and sweep.get("asks") == key:
resumed = bool(sweep and sweep.get("asks") == key)
if sweep and resumed:
start, cursor = int(sweep.get("series") or 0), sweep.get("cursor")
seen, earlier = set(sweep.get("seen") or []), list(sweep.get("rows") or [])
whole = bool(sweep.get("whole", True))
@@ -591,7 +602,8 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
listed = list(dict.fromkeys(earlier + list(rows)))
if not stop:
return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words)
return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words,
resumed and whole)
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
module.NAME, slug, pages, stop)
at_ceiling = ceiling is not None and len(seen) >= ceiling
@@ -674,10 +686,12 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
"""Record a read whose rows are stored: `ok` (the run's marker),
`complete`, `ids` (the compound ids, what the pull closes against),
`filtered`, `narrowed` (a read by terms, which the pull never closes
against), the sweep to continue or none, and the count for the 10%
guard once a read is whole."""
against), whether it finished a sweep from earlier pulls (`swept`), the
sweep to continue or none, and the count for the 10% guard once a read
is whole."""
state[f"{prefix}:ok:{target}"] = run
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""