Let the runner drop and count a row without an id, a title or a link, as ADR Plugins 3.8 has it, in one place for whole, paged and search reads: the count is stamped per target, logged, named on the board's row, and over 10% of the rows fails the read; the 28 readers hand every listing over instead of skipping it
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
38c31c830b
commit
911dfe8584
58 files changed
+479
-203
No files matched your search
+107
-2
@@ -1,7 +1,7 @@
|
||||
"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a
|
||||
subscription resolves to, series and their union, whole and paged listings,
|
||||
completeness from the declaration, the sweep across pulls, the 10% guard,
|
||||
the facets kept in state, the detail reads and the `raw` an item carries. No
|
||||
completeness from the declaration, the rows kept and dropped, the sweep
|
||||
across pulls, the 10% guards, the facets kept in state, the detail reads and the `raw` an item carries. No
|
||||
network: `get` is a dict of URL -> body or exception."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -426,6 +426,93 @@ def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
|
||||
assert filtered.ids == ["1"] and filtered.filtered
|
||||
|
||||
|
||||
# --------------------------------------------------------- the rows kept
|
||||
|
||||
def _handed(i: str) -> dict[str, Any]:
|
||||
"""A row as a reader hands it over: whole, or, for an id of `-id`,
|
||||
`-title`, `-link` or `-url`, without that value or with a link that is
|
||||
no web URL (`javascript:`, or one httpx cannot read)."""
|
||||
broken = {"-id": {"source_id": ""}, "-title": {"title": " "},
|
||||
"-link": {"url": "javascript:alert(1)"}, "-url": {"url": "http://[::1"}}
|
||||
return _row(i, **broken.get(i.split(":")[0], {}))
|
||||
|
||||
|
||||
def test_a_row_is_kept_with_an_id_a_title_and_a_web_link() -> None:
|
||||
assert _runner.valid(_handed("1"))
|
||||
assert not any(_runner.valid(_handed(i)) for i in ("-id", "-title", "-link", "-url"))
|
||||
assert not _runner.valid({"source_id": "1", "title": "Role"})
|
||||
|
||||
|
||||
def test_a_whole_listing_drops_the_rows_without_and_counts_them(
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
module = _whole()
|
||||
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
|
||||
web = Web({"https://jobs.fake.example/acme/list":
|
||||
b'["1", "-id", "2", "-title", "-link", "-url"]'})
|
||||
with caplog.at_level("WARNING", logger=_runner.__name__):
|
||||
listing = _runner.read_listing(module, "acme", web)
|
||||
assert [r["source_id"] for r in listing.rows] == ["1", "2"]
|
||||
assert (listing.ids, listing.complete) == (["1", "2"], True)
|
||||
assert (listing.handed, listing.dropped) == (6, 4)
|
||||
[line] = [r.getMessage() for r in caplog.records if "dropped" in r.getMessage()]
|
||||
assert "fake:acme" in line and "dropped 4 of 6 listings" in line
|
||||
|
||||
|
||||
def test_a_paged_listing_drops_on_every_page_and_counts_distinct_results(
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
"""Completeness counts the result ids `page` returns, not the rows kept,
|
||||
so the dropped rows still count as read; the drop is logged once."""
|
||||
pages = [["1", "-title:a"], ["2", "-link:b"]]
|
||||
module = _paged(pages, total=4)
|
||||
module.rows = lambda ids, slug: [_handed(i) for i in ids]
|
||||
with caplog.at_level("WARNING", logger=_runner.__name__):
|
||||
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
|
||||
assert (listing.ids, listing.complete) == (["1", "2"], True)
|
||||
assert (listing.handed, listing.dropped) == (4, 2)
|
||||
assert sum("dropped" in r.getMessage() for r in caplog.records) == 1
|
||||
|
||||
|
||||
def test_the_stamp_keeps_the_rows_dropped() -> None:
|
||||
state: dict[str, str] = {}
|
||||
listing = _runner.Listing([_row("1")], True, ["1"], handed=3, dropped=2)
|
||||
_runner.stamp(state, "ats", "t", "r", listing, ["fake:acme:1"])
|
||||
assert state["ats:dropped:t"] == "2"
|
||||
_runner.stamp(state, "ats", "t", "r", _runner.Listing([], True, []), [])
|
||||
assert state["ats:dropped:t"] == ""
|
||||
|
||||
|
||||
def _dropping(handed: int, dropped: int) -> tuple[Any, Web]:
|
||||
"""A whole board whose feed hands over `handed` rows, `dropped` of them
|
||||
without a title."""
|
||||
module = _whole()
|
||||
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
|
||||
ids = [f"-title:{n}" for n in range(dropped)] + [str(n) for n in range(handed - dropped)]
|
||||
return module, Web({"https://jobs.fake.example/acme/list": json.dumps(ids).encode()})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("handed", "dropped", "fails"), [
|
||||
(20, 3, True), (20, 2, False), (9, 5, False), (10, 2, True)])
|
||||
def test_a_read_that_drops_over_a_tenth_of_its_rows_fails(
|
||||
handed: int, dropped: int, fails: bool) -> None:
|
||||
"""Over `SUSPECT_SHARE` of the rows, once `SUSPECT_MIN` were handed over:
|
||||
a page whose shape changed, which must close nothing."""
|
||||
module, web = _dropping(handed, dropped)
|
||||
if not fails:
|
||||
assert _runner.read({}, "ats", "t", module, "acme", web).dropped == dropped
|
||||
return
|
||||
with pytest.raises(_runner.Suspect, match=f"dropped {dropped} of {handed} listings"):
|
||||
_runner.read({}, "ats", "t", module, "acme", web)
|
||||
|
||||
|
||||
def test_a_read_that_drops_too_many_is_accepted_after_the_stale_days() -> None:
|
||||
module, web = _dropping(20, 5)
|
||||
since = (datetime.now(UTC) - timedelta(days=30)).isoformat(timespec="seconds")
|
||||
state = {"ats:failing:t": json.dumps({"since": since, "reason": "x"})}
|
||||
listing = _runner.read(state, "ats", "t", module, "acme", web,
|
||||
limits=_runner.Limits(page_delay=0, stale_days=21))
|
||||
assert (len(listing.ids), listing.dropped) == (15, 5)
|
||||
|
||||
|
||||
# ------------------------------------------------------ through ATSSource
|
||||
|
||||
@pytest.fixture
|
||||
@@ -479,6 +566,24 @@ def test_ats_source_hands_the_board_params_to_the_reader(board: Any) -> None:
|
||||
assert state["ats:filtered:fake:acme"] == "1"
|
||||
|
||||
|
||||
def test_ats_source_stamps_the_rows_dropped_and_fails_on_too_many(board: Any) -> None:
|
||||
board.rows = lambda ids, slug: [_handed(i) for i in ids]
|
||||
board.web.answers = {"https://jobs.fake.example/acme/page/0":
|
||||
json.dumps([str(n) for n in range(19)] + ["-link:x"]).encode()}
|
||||
board.READS = board.READS.model_copy(update={"complete": "end"})
|
||||
state: dict[str, str] = {}
|
||||
assert len(list(_source(state).fetch())) == 19
|
||||
assert state["ats:dropped:fake:acme"] == "1"
|
||||
|
||||
board.web.answers = {"https://jobs.fake.example/acme/page/0":
|
||||
json.dumps([str(n) for n in range(15)]
|
||||
+ [f"-id:{n}" for n in range(5)]).encode()}
|
||||
state = {}
|
||||
assert list(_source(state).fetch()) == []
|
||||
assert "dropped 5 of 20" in json.loads(state["ats:failing:fake:acme"])["reason"]
|
||||
assert "ats:ok:fake:acme" not in state
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- raw
|
||||
|
||||
def test_raw_keeps_every_stated_key_and_the_places() -> None:
|
||||
|
||||
Reference in new issue
Block a user