Let the runner drop and count a row without an id, a title or a link, as ADR Plugins 3.8 has it, in one place for whole, paged and search reads: the count is stamped per target, logged, named on the board's row, and over 10% of the rows fails the read; the 28 readers hand every listing over instead of skipping it

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-10-06 14:04:33 +02:00
1 parent 38c31c830b
commit 911dfe8584
58 files changed
+479 -203

No files matched your search

+107 -2
View File
@@ -1,7 +1,7 @@
"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a
subscription resolves to, series and their union, whole and paged listings,
completeness from the declaration, the sweep across pulls, the 10% guard,
the facets kept in state, the detail reads and the `raw` an item carries. No
completeness from the declaration, the rows kept and dropped, the sweep
across pulls, the 10% guards, the facets kept in state, the detail reads and the `raw` an item carries. No
network: `get` is a dict of URL -> body or exception."""
from __future__ import annotations
@@ -426,6 +426,93 @@ def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
assert filtered.ids == ["1"] and filtered.filtered
# --------------------------------------------------------- the rows kept
def _handed(i: str) -> dict[str, Any]:
"""A row as a reader hands it over: whole, or, for an id of `-id`,
`-title`, `-link` or `-url`, without that value or with a link that is
no web URL (`javascript:`, or one httpx cannot read)."""
broken = {"-id": {"source_id": ""}, "-title": {"title": " "},
"-link": {"url": "javascript:alert(1)"}, "-url": {"url": "http://[::1"}}
return _row(i, **broken.get(i.split(":")[0], {}))
def test_a_row_is_kept_with_an_id_a_title_and_a_web_link() -> None:
assert _runner.valid(_handed("1"))
assert not any(_runner.valid(_handed(i)) for i in ("-id", "-title", "-link", "-url"))
assert not _runner.valid({"source_id": "1", "title": "Role"})
def test_a_whole_listing_drops_the_rows_without_and_counts_them(
caplog: pytest.LogCaptureFixture) -> None:
module = _whole()
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
web = Web({"https://jobs.fake.example/acme/list":
b'["1", "-id", "2", "-title", "-link", "-url"]'})
with caplog.at_level("WARNING", logger=_runner.__name__):
listing = _runner.read_listing(module, "acme", web)
assert [r["source_id"] for r in listing.rows] == ["1", "2"]
assert (listing.ids, listing.complete) == (["1", "2"], True)
assert (listing.handed, listing.dropped) == (6, 4)
[line] = [r.getMessage() for r in caplog.records if "dropped" in r.getMessage()]
assert "fake:acme" in line and "dropped 4 of 6 listings" in line
def test_a_paged_listing_drops_on_every_page_and_counts_distinct_results(
caplog: pytest.LogCaptureFixture) -> None:
"""Completeness counts the result ids `page` returns, not the rows kept,
so the dropped rows still count as read; the drop is logged once."""
pages = [["1", "-title:a"], ["2", "-link:b"]]
module = _paged(pages, total=4)
module.rows = lambda ids, slug: [_handed(i) for i in ids]
with caplog.at_level("WARNING", logger=_runner.__name__):
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2"], True)
assert (listing.handed, listing.dropped) == (4, 2)
assert sum("dropped" in r.getMessage() for r in caplog.records) == 1
def test_the_stamp_keeps_the_rows_dropped() -> None:
state: dict[str, str] = {}
listing = _runner.Listing([_row("1")], True, ["1"], handed=3, dropped=2)
_runner.stamp(state, "ats", "t", "r", listing, ["fake:acme:1"])
assert state["ats:dropped:t"] == "2"
_runner.stamp(state, "ats", "t", "r", _runner.Listing([], True, []), [])
assert state["ats:dropped:t"] == ""
def _dropping(handed: int, dropped: int) -> tuple[Any, Web]:
"""A whole board whose feed hands over `handed` rows, `dropped` of them
without a title."""
module = _whole()
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
ids = [f"-title:{n}" for n in range(dropped)] + [str(n) for n in range(handed - dropped)]
return module, Web({"https://jobs.fake.example/acme/list": json.dumps(ids).encode()})
@pytest.mark.parametrize(("handed", "dropped", "fails"), [
(20, 3, True), (20, 2, False), (9, 5, False), (10, 2, True)])
def test_a_read_that_drops_over_a_tenth_of_its_rows_fails(
handed: int, dropped: int, fails: bool) -> None:
"""Over `SUSPECT_SHARE` of the rows, once `SUSPECT_MIN` were handed over:
a page whose shape changed, which must close nothing."""
module, web = _dropping(handed, dropped)
if not fails:
assert _runner.read({}, "ats", "t", module, "acme", web).dropped == dropped
return
with pytest.raises(_runner.Suspect, match=f"dropped {dropped} of {handed} listings"):
_runner.read({}, "ats", "t", module, "acme", web)
def test_a_read_that_drops_too_many_is_accepted_after_the_stale_days() -> None:
module, web = _dropping(20, 5)
since = (datetime.now(UTC) - timedelta(days=30)).isoformat(timespec="seconds")
state = {"ats:failing:t": json.dumps({"since": since, "reason": "x"})}
listing = _runner.read(state, "ats", "t", module, "acme", web,
limits=_runner.Limits(page_delay=0, stale_days=21))
assert (len(listing.ids), listing.dropped) == (15, 5)
# ------------------------------------------------------ through ATSSource
@pytest.fixture
@@ -479,6 +566,24 @@ def test_ats_source_hands_the_board_params_to_the_reader(board: Any) -> None:
assert state["ats:filtered:fake:acme"] == "1"
def test_ats_source_stamps_the_rows_dropped_and_fails_on_too_many(board: Any) -> None:
board.rows = lambda ids, slug: [_handed(i) for i in ids]
board.web.answers = {"https://jobs.fake.example/acme/page/0":
json.dumps([str(n) for n in range(19)] + ["-link:x"]).encode()}
board.READS = board.READS.model_copy(update={"complete": "end"})
state: dict[str, str] = {}
assert len(list(_source(state).fetch())) == 19
assert state["ats:dropped:fake:acme"] == "1"
board.web.answers = {"https://jobs.fake.example/acme/page/0":
json.dumps([str(n) for n in range(15)]
+ [f"-id:{n}" for n in range(5)]).encode()}
state = {}
assert list(_source(state).fetch()) == []
assert "dropped 5 of 20" in json.loads(state["ats:failing:fake:acme"])["reason"]
assert "ats:ok:fake:acme" not in state
# -------------------------------------------------------------------- raw
def test_raw_keeps_every_stated_key_and_the_places() -> None: