diff --git a/plugins/readers/_runner.py b/plugins/readers/_runner.py index 3e5011f..7f9c698 100644 --- a/plugins/readers/_runner.py +++ b/plugins/readers/_runner.py @@ -556,10 +556,11 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None A whole listing is complete whenever it parses. A paged one reads each series' pages, `limits.page_delay` apart, until the host has no next - page, the stated total is reached, a page brings no new id while the - reader still hands a cursor (a stall: a host that ignores the cursor or - sends a page again, with or without a total) or `limits.max_pages` - pages in all. A series is complete when it read as many distinct result + page, the stated total is reached, the read stalls or `limits.max_pages` + pages in all. A stall is a page that brings no new id while the reader + still hands a cursor: any such page on a host that states no total (it + ignores the cursor), and with a total one the reader hands over empty + (it saw the page come again). A series is complete when it read as many distinct result ids as its stated total, or, without a total, when it ended on the host's last page; never after a stall, and never for `complete="never"`. The read is complete when every series is. A 429 @@ -636,10 +637,13 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None if cursor is None or (isinstance(total, int) and len(seen) >= total): ended = True break - # Nothing new while the reader says more follow: the host ignores - # the cursor or sent a page again. A stall, whatever the total: the - # series ends incomplete rather than loop to the page cap. - if len(seen) == before: + # Nothing new while the reader says more follow. Without a total + # the host ignores the cursor, and the read would never end. With + # one, only a page the reader hands over empty is a stall (it saw + # the page come again, as SuccessFactors does): a page of ids + # already read is what a resumed sweep starts on when the list + # grew at the top since, and the read goes on from there. + if len(seen) == before and (total is None or not results): stalled = True break if stop: diff --git a/tests/test_ats.py b/tests/test_ats.py index 8fc033e..e82808c 100644 --- a/tests/test_ats.py +++ b/tests/test_ats.py @@ -618,13 +618,7 @@ def test_the_page_cap_leaves_a_board_incomplete( def fake_get(client, url, host, slug, *, method="GET", payload=None): calls.append(payload) - # Always another page of roles not read yet: a page of nothing new - # would be a stall, which ends the read before the cap. - page = json.loads(_page("workable.json")) - page["total"] = 100 - for result in page["results"]: - result["id"] = f"{result['id']}-{len(calls)}" - return json.dumps(page).encode() + return _page("workable.json") # always another page monkeypatch.setattr(ats, "MAX_PAGES", 3) monkeypatch.setattr(ats, "_get", fake_get) diff --git a/tests/test_runner.py b/tests/test_runner.py index 9d5cc41..41380c2 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -215,15 +215,33 @@ def test_a_page_with_nothing_new_ends_the_read_incomplete() -> None: assert [index for index, _ in module.asked] == [0, 1] -def test_a_page_with_nothing_new_is_a_stall_with_a_stated_total_too() -> None: - """A host that ignores the cursor and states a total used to be read on - to the page cap; the second page, which adds nothing while the reader - hands a cursor, ends it incomplete.""" - pages = [["1", "2"]] * 5 - module = _paged(pages, total=5) - listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY) +def test_an_empty_page_with_a_cursor_is_a_stall_with_a_stated_total_too() -> None: + """A reader that sees a page come again hands it over empty with a + cursor, as SuccessFactors does. That ends the read incomplete, also when + the host states a total, where it used to run on to the page cap.""" + module = _paged([["1", "2"]] * 5, total=5) + seen: list[Any] = [] + + def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any: + seen.append(cursor) + return (["1", "2"], [{"id": "1"}, {"id": "2"}], 5, 1) if cursor is None \ + else ([], [], 5, cursor) + + module.page = page + listing = _runner.read_listing(module, "acme", Web(), limits=NO_DELAY) assert (listing.ids, listing.complete) == (["1", "2"], False) - assert [index for index, _ in module.asked] == [0, 1] + assert seen == [None, 1] + + +def test_a_page_of_ids_already_read_is_no_stall_with_a_stated_total() -> None: + """What a resumed sweep starts on when the list grew at the top since: + the page at the saved cursor holds only roles the sweep has. The read + goes on to the roles behind them.""" + pages = [["1", "2"], ["1", "2"], ["3"]] + module = _paged(pages, total=3) + listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY) + assert (listing.ids, listing.complete) == (["1", "2", "3"], True) + assert [index for index, _ in module.asked] == [0, 1, 2] def test_an_empty_last_page_ends_a_read_without_a_total_complete() -> None: