Let the runner drop and count a row without an id, a title or a link, as ADR Plugins 3.8 has it, in one place for whole, paged and search reads: the count is stamped per target, logged, named on the board's row, and over 10% of the rows fails the read; the 28 readers hand every listing over instead of skipping it
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
38c31c830b
commit
911dfe8584
58 files changed
+479
-203
No files matched your search
+98
-20
@@ -4,10 +4,12 @@ the reader declares in `READS`.
|
||||
A reader says how its host answers; this module reads it: the guarded fetch
|
||||
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
|
||||
to (`ask_for`) and its series (`series`), the listing whole or paged, one
|
||||
series per `one`-filter value unioned by id (`read_listing`), completeness,
|
||||
the sweep of an oversized paged board across pulls, the 10% guard on every
|
||||
read that is not a sweep (`read`), the per-role detail reads
|
||||
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
|
||||
series per `one`-filter value unioned by id (`read_listing`), the rows it
|
||||
keeps (`valid`: a row without an id, a title or a link is dropped and
|
||||
counted here, never by the reader), completeness, the sweep of an oversized
|
||||
paged board across pulls, the two 10% guards (`read`: on what a read that
|
||||
is not a sweep listed, and on what any read dropped), the per-role detail
|
||||
reads (`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
|
||||
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
|
||||
`plugins/search.py` are thin `Source` classes over it: they pass their own
|
||||
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
|
||||
@@ -30,6 +32,8 @@ State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
|
||||
drifts when roles close between pulls, so `app.jobs.pull` closes
|
||||
a role of such a board only when two finished sweeps in a row
|
||||
missed it (it keeps the ids the last one missed in `missing`)
|
||||
dropped the rows the last stamped read dropped (`Listing.dropped`), as a
|
||||
number, "" for none; `app.jobs.limits` names it on the board's row
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -52,7 +56,7 @@ import httpx
|
||||
from desk.sources import USER_AGENT
|
||||
|
||||
from ._contract import Ask, Key, Place, Reads
|
||||
from ._shared import Get, Row
|
||||
from ._shared import Get, Row, is_web_url
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -96,7 +100,8 @@ SUSPECT_SHARE = 0.1
|
||||
#: The last count from which the guard holds every read. A read that is
|
||||
#: filtered, narrowed by terms or of an `empty="suspect"` reader is held from
|
||||
#: a count of 1, as it was before every read was; a whole read of a board
|
||||
#: that listed a handful of roles can drop to none by itself.
|
||||
#: that listed a handful of roles can drop to none by itself. It is also the
|
||||
#: rows a read must hand over before the share it dropped can fail it.
|
||||
SUSPECT_MIN = 10
|
||||
|
||||
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
|
||||
@@ -113,8 +118,8 @@ class TooLarge(ValueError):
|
||||
|
||||
|
||||
class Suspect(Exception):
|
||||
"""A read that listed under `SUSPECT_SHARE` of what it listed last time
|
||||
(see `read`)."""
|
||||
"""A read that listed under `SUSPECT_SHARE` of what it listed last time,
|
||||
or dropped more than that share of its rows (see `read`)."""
|
||||
|
||||
|
||||
class MissingKey(ValueError):
|
||||
@@ -147,7 +152,9 @@ class Listing:
|
||||
and roles closing meanwhile may have shifted an offset-paged host's later
|
||||
roles back past the cursor, so `ids` can miss a role that is still open.
|
||||
`remote_unasked` says a filtered read was meant to ask for remote roles
|
||||
as well and could not, so its `ids` say nothing of the remote roles."""
|
||||
as well and could not, so its `ids` say nothing of the remote roles.
|
||||
`handed` counts the rows the reader handed over this pull, `dropped`
|
||||
those of them `valid` refused, which are in neither `rows` nor `ids`."""
|
||||
|
||||
rows: list[Row]
|
||||
complete: bool
|
||||
@@ -158,6 +165,8 @@ class Listing:
|
||||
narrowed: bool = False
|
||||
swept: bool = False
|
||||
remote_unasked: bool = False
|
||||
handed: int = 0
|
||||
dropped: int = 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------- modules
|
||||
@@ -524,8 +533,42 @@ def _result_ids(results: Iterable[dict[str, Any]]) -> set[str]:
|
||||
return {str(r.get("id") or r.get("shortcode") or "") for r in results}
|
||||
|
||||
|
||||
def valid(row: Mapping[str, Any]) -> bool:
|
||||
"""Whether the runner keeps a row: a non-empty `source_id`, a non-empty
|
||||
`title` and a web URL (`is_web_url`). A reader hands over every listing
|
||||
as far as it can make it out; a row without one of the three cannot be
|
||||
stored or opened, and the runner drops it and counts it."""
|
||||
try:
|
||||
link = is_web_url(str(row.get("url") or ""))
|
||||
except (httpx.InvalidURL, ValueError): # a string httpx cannot read as a URL
|
||||
link = False
|
||||
return (link and bool(str(row.get("source_id") or "").strip())
|
||||
and bool(str(row.get("title") or "").strip()))
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _Tally:
|
||||
"""The rows a read was handed and dropped, across its series and pages."""
|
||||
|
||||
handed: int = 0
|
||||
dropped: int = 0
|
||||
|
||||
|
||||
def _keep(found: Iterable[Row], rows: dict[str, Row], tally: _Tally) -> None:
|
||||
"""The rows a reader's `parse` or `rows` returned, into `rows` by
|
||||
`source_id`, the first of an id kept; those `valid` refuses are counted
|
||||
in `tally` and left out. The one place a row is checked, for the whole
|
||||
and the paged listing alike."""
|
||||
for row in found:
|
||||
tally.handed += 1
|
||||
if not valid(row):
|
||||
tally.dropped += 1
|
||||
continue
|
||||
rows.setdefault(row["source_id"], row)
|
||||
|
||||
|
||||
def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
||||
limits: Limits) -> dict[str, Row]:
|
||||
limits: Limits, tally: _Tally) -> dict[str, Row]:
|
||||
"""A whole listing, one request a series, unioned by `source_id`. A body
|
||||
over the cap gets one retry at the reader's `fallback_url`."""
|
||||
rows: dict[str, Row] = {}
|
||||
@@ -541,8 +584,7 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
||||
log.warning("readers: %s:%s: body over the cap; reading the smaller feed",
|
||||
module.NAME, slug)
|
||||
body = get(module.fallback_url(slug))
|
||||
for row in module.parse(body, slug):
|
||||
rows.setdefault(row["source_id"], row)
|
||||
_keep(module.parse(body, slug), rows, tally)
|
||||
return rows
|
||||
|
||||
|
||||
@@ -580,7 +622,29 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
only: a search's places are what the search is, as its query is.
|
||||
`remote` says the reach or the params wanted remote roles; when no series
|
||||
asked for them (a reader without a remote filter, or one whose facets
|
||||
state none beside places) the listing is `remote_unasked`."""
|
||||
state none beside places) the listing is `remote_unasked`.
|
||||
|
||||
Every row `rows` or `parse` returns goes through `valid`: one without a
|
||||
`source_id`, a title or a web URL is dropped here, counted in
|
||||
`Listing.dropped` and logged once for the read. Completeness counts the
|
||||
distinct result ids `page` returns beside its rows, not the rows kept, so
|
||||
a dropped row still counts as read; a whole listing is complete whenever
|
||||
it parses, whatever it dropped."""
|
||||
tally = _Tally()
|
||||
listing = _listing(module, slug, get, ask, limits=limits, sweep=sweep,
|
||||
can_sweep=can_sweep, facets=facets, remote=remote, tally=tally)
|
||||
listing.handed, listing.dropped = tally.handed, tally.dropped
|
||||
if tally.dropped:
|
||||
log.warning("readers: %s:%s: dropped %d of %d listings: no id, title or link",
|
||||
module.NAME, slug, tally.dropped, tally.handed)
|
||||
return listing
|
||||
|
||||
|
||||
def _listing(module: ModuleType, slug: str, get: Get, ask: Ask | None, *,
|
||||
limits: Limits | None, sweep: Mapping[str, Any] | None, can_sweep: bool,
|
||||
facets: Mapping[str, Mapping[str, str]] | None, remote: bool,
|
||||
tally: _Tally) -> Listing:
|
||||
"""`read_listing` before its count of dropped rows, which `tally` keeps."""
|
||||
limits = limits or Limits()
|
||||
ask = ask or Ask()
|
||||
reads: Reads = module.READS
|
||||
@@ -595,7 +659,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
"nothing asked, not closing", module.NAME, slug)
|
||||
return Listing([], False, [], True, None, key, by_words, remote_unasked=unasked)
|
||||
if not hasattr(module, "page"):
|
||||
whole_rows = _whole(module, slug, get, asks, limits)
|
||||
whole_rows = _whole(module, slug, get, asks, limits, tally)
|
||||
return Listing(list(whole_rows.values()), True, list(whole_rows), is_filtered, None,
|
||||
key, by_words, remote_unasked=unasked)
|
||||
|
||||
@@ -632,8 +696,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
pages += 1
|
||||
before = len(seen)
|
||||
seen |= _result_ids(results)
|
||||
for row in module.rows(data, slug):
|
||||
rows.setdefault(row["source_id"], row)
|
||||
_keep(module.rows(data, slug), rows, tally)
|
||||
if cursor is None or (isinstance(total, int) and len(seen) >= total):
|
||||
ended = True
|
||||
break
|
||||
@@ -747,13 +810,26 @@ def read(state: MutableMapping[str, str], prefix: str, target: str, module: Modu
|
||||
that really shrank is accepted once the target has been failing for
|
||||
`limits.stale_days` (its `<prefix>:failing:<target>` record), and its
|
||||
stamp then resets the count; a Recheck resets it at once
|
||||
(`plugins.ats.keep_size`)."""
|
||||
(`plugins.ats.keep_size`).
|
||||
|
||||
A read, a sweep's included, that dropped more than `SUSPECT_SHARE` of
|
||||
the rows its reader handed over (`valid`), once it was handed at least
|
||||
`SUSPECT_MIN`, raises `Suspect` the same way: a page whose shape changed
|
||||
parses to rows without a title or a link, and the roles it hid would
|
||||
close. It gives way after `limits.stale_days` of failing as well."""
|
||||
limits = limits or Limits()
|
||||
ask = ask or Ask()
|
||||
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
|
||||
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
|
||||
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets,
|
||||
remote=remote)
|
||||
if listing.handed >= SUSPECT_MIN and listing.dropped > SUSPECT_SHARE * listing.handed:
|
||||
why = (f"dropped {listing.dropped} of {listing.handed} listings, over "
|
||||
f"{SUSPECT_SHARE:.0%}: no id, title or link")
|
||||
if not _failing_for(state, f"{prefix}:failing:{target}", limits.stale_days):
|
||||
raise Suspect(why)
|
||||
log.warning("readers: %s:%s: %s, accepted after %d days of failing",
|
||||
module.NAME, slug, why, limits.stale_days)
|
||||
held = listing.filtered or listing.narrowed or module.READS.empty == "suspect"
|
||||
last = _load(state, f"{prefix}:count:{target}")
|
||||
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
|
||||
@@ -788,9 +864,10 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
||||
`complete`, `ids` (the compound ids, what the pull closes against),
|
||||
`filtered`, `narrowed` (a read by terms, which the pull never closes
|
||||
against), whether it finished a sweep from earlier pulls (`swept`),
|
||||
whether a filtered read left remote unasked (`remote_unasked`), the
|
||||
sweep to continue or none, and the count for the 10% guard once a read
|
||||
is not a sweep. A read that asked nothing keeps the count it had."""
|
||||
whether a filtered read left remote unasked (`remote_unasked`), the rows
|
||||
it dropped (`dropped`, "" for none), the sweep to continue or none, and
|
||||
the count for the 10% guard once a read is not a sweep. A read that
|
||||
asked nothing keeps the count it had."""
|
||||
state[f"{prefix}:ok:{target}"] = run
|
||||
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
||||
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
|
||||
@@ -798,6 +875,7 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
||||
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
||||
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
|
||||
state[f"{prefix}:remote_unasked:{target}"] = "1" if listing.remote_unasked else ""
|
||||
state[f"{prefix}:dropped:{target}"] = str(listing.dropped) if listing.dropped else ""
|
||||
if listing.sweep is not None:
|
||||
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
|
||||
return
|
||||
|
||||
Reference in new issue
Block a user