Merge two strikes for boards swept across pulls (wp-b-strikes)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
19401cfc7a
5 files changed
+373
-11
No files matched your search
@@ -23,6 +23,11 @@ State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
|
||||
count `{"asks", "count"}`, the roles the last whole read listed, for
|
||||
the 10% guard
|
||||
facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL`
|
||||
swept "1" when the last stamped listing finished a sweep carried over
|
||||
from an earlier pull (`Listing.swept`), else "". Offset paging
|
||||
drifts when roles close between pulls, so `app.jobs.pull` closes
|
||||
a role of such a board only when two finished sweeps in a row
|
||||
missed it (it keeps the ids the last one missed in `missing`)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -126,7 +131,10 @@ class Listing:
|
||||
state to keep for the next pull, None when no sweep is under way.
|
||||
`filtered` says the read asked the host for the roles in some places,
|
||||
`narrowed` for the roles matching words (`narrowed()`): the ids of
|
||||
neither are the whole board."""
|
||||
neither are the whole board. `swept` says a complete listing finished a
|
||||
sweep carried over from an earlier pull: `ids` were read across pulls,
|
||||
and roles closing meanwhile may have shifted an offset-paged host's later
|
||||
roles back past the cursor, so `ids` can miss a role that is still open."""
|
||||
|
||||
rows: list[Row]
|
||||
complete: bool
|
||||
@@ -135,6 +143,7 @@ class Listing:
|
||||
sweep: dict[str, Any] | None = None
|
||||
asks: str = ""
|
||||
narrowed: bool = False
|
||||
swept: bool = False
|
||||
|
||||
|
||||
# --------------------------------------------------------------- modules
|
||||
@@ -521,8 +530,9 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns
|
||||
the `sweep` to keep, and the next pull, handed it back, continues from
|
||||
its series and cursor; the sweep is complete when it reaches the end,
|
||||
and `ids` then hold every id it listed. A read at the reader's `ceiling`
|
||||
is not swept: the host answers nothing past it."""
|
||||
and `ids` then hold every id it listed, and the listing is `swept`. A
|
||||
read at the reader's `ceiling` is not swept: the host answers nothing
|
||||
past it."""
|
||||
limits = limits or Limits()
|
||||
ask = ask or Ask()
|
||||
asks = series(module, ask, facets)
|
||||
@@ -539,7 +549,8 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
never = reads is not None and reads.complete == "never"
|
||||
ceiling = reads.ceiling if reads is not None else None
|
||||
start, cursor, seen, earlier, whole = 0, None, set(), [], True
|
||||
if sweep and sweep.get("asks") == key:
|
||||
resumed = bool(sweep and sweep.get("asks") == key)
|
||||
if sweep and resumed:
|
||||
start, cursor = int(sweep.get("series") or 0), sweep.get("cursor")
|
||||
seen, earlier = set(sweep.get("seen") or []), list(sweep.get("rows") or [])
|
||||
whole = bool(sweep.get("whole", True))
|
||||
@@ -591,7 +602,8 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
|
||||
listed = list(dict.fromkeys(earlier + list(rows)))
|
||||
if not stop:
|
||||
return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words)
|
||||
return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words,
|
||||
resumed and whole)
|
||||
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
|
||||
module.NAME, slug, pages, stop)
|
||||
at_ceiling = ceiling is not None and len(seen) >= ceiling
|
||||
@@ -674,10 +686,12 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
||||
"""Record a read whose rows are stored: `ok` (the run's marker),
|
||||
`complete`, `ids` (the compound ids, what the pull closes against),
|
||||
`filtered`, `narrowed` (a read by terms, which the pull never closes
|
||||
against), the sweep to continue or none, and the count for the 10%
|
||||
guard once a read is whole."""
|
||||
against), whether it finished a sweep from earlier pulls (`swept`), the
|
||||
sweep to continue or none, and the count for the 10% guard once a read
|
||||
is whole."""
|
||||
state[f"{prefix}:ok:{target}"] = run
|
||||
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
||||
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
|
||||
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
|
||||
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
||||
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
|
||||
|
||||
Reference in new issue
Block a user