Count a page of ids already read as a stall only where the host states no total or the reader hands the page over empty, so a resumed sweep reads on when the list grew at the top

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-10-05 09:56:06 +02:00
1 parent 7f37270eb3
commit c4c17fb019
3 files changed
+39 -23

No files matched your search

+12 -8
View File
@@ -556,10 +556,11 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
A whole listing is complete whenever it parses. A paged one reads each
series' pages, `limits.page_delay` apart, until the host has no next
page, the stated total is reached, a page brings no new id while the
reader still hands a cursor (a stall: a host that ignores the cursor or
sends a page again, with or without a total) or `limits.max_pages`
pages in all. A series is complete when it read as many distinct result
page, the stated total is reached, the read stalls or `limits.max_pages`
pages in all. A stall is a page that brings no new id while the reader
still hands a cursor: any such page on a host that states no total (it
ignores the cursor), and with a total one the reader hands over empty
(it saw the page come again). A series is complete when it read as many distinct result
ids as its stated total, or, without a total, when it ended on the
host's last page; never after a stall, and never for
`complete="never"`. The read is complete when every series is. A 429
@@ -636,10 +637,13 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
if cursor is None or (isinstance(total, int) and len(seen) >= total):
ended = True
break
# Nothing new while the reader says more follow: the host ignores
# the cursor or sent a page again. A stall, whatever the total: the
# series ends incomplete rather than loop to the page cap.
if len(seen) == before:
# Nothing new while the reader says more follow. Without a total
# the host ignores the cursor, and the read would never end. With
# one, only a page the reader hands over empty is a stall (it saw
# the page come again, as SuccessFactors does): a page of ids
# already read is what a resumed sweep starts on when the list
# grew at the top since, and the read goes on from there.
if len(seen) == before and (total is None or not results):
stalled = True
break
if stop:
+1 -7
View File
@@ -618,13 +618,7 @@ def test_the_page_cap_leaves_a_board_incomplete(
def fake_get(client, url, host, slug, *, method="GET", payload=None):
calls.append(payload)
# Always another page of roles not read yet: a page of nothing new
# would be a stall, which ends the read before the cap.
page = json.loads(_page("workable.json"))
page["total"] = 100
for result in page["results"]:
result["id"] = f"{result['id']}-{len(calls)}"
return json.dumps(page).encode()
return _page("workable.json") # always another page
monkeypatch.setattr(ats, "MAX_PAGES", 3)
monkeypatch.setattr(ats, "_get", fake_get)
+26 -8
View File
@@ -215,15 +215,33 @@ def test_a_page_with_nothing_new_ends_the_read_incomplete() -> None:
assert [index for index, _ in module.asked] == [0, 1]
def test_a_page_with_nothing_new_is_a_stall_with_a_stated_total_too() -> None:
"""A host that ignores the cursor and states a total used to be read on
to the page cap; the second page, which adds nothing while the reader
hands a cursor, ends it incomplete."""
pages = [["1", "2"]] * 5
module = _paged(pages, total=5)
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
def test_an_empty_page_with_a_cursor_is_a_stall_with_a_stated_total_too() -> None:
"""A reader that sees a page come again hands it over empty with a
cursor, as SuccessFactors does. That ends the read incomplete, also when
the host states a total, where it used to run on to the page cap."""
module = _paged([["1", "2"]] * 5, total=5)
seen: list[Any] = []
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
seen.append(cursor)
return (["1", "2"], [{"id": "1"}, {"id": "2"}], 5, 1) if cursor is None \
else ([], [], 5, cursor)
module.page = page
listing = _runner.read_listing(module, "acme", Web(), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2"], False)
assert [index for index, _ in module.asked] == [0, 1]
assert seen == [None, 1]
def test_a_page_of_ids_already_read_is_no_stall_with_a_stated_total() -> None:
"""What a resumed sweep starts on when the list grew at the top since:
the page at the saved cursor holds only roles the sweep has. The read
goes on to the roles behind them."""
pages = [["1", "2"], ["1", "2"], ["3"]]
module = _paged(pages, total=3)
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2", "3"], True)
assert [index for index, _ in module.asked] == [0, 1, 2]
def test_an_empty_last_page_ends_a_read_without_a_total_complete() -> None: