Keep the sweep of a reader whose cursor is a tuple, so Workday and SuccessFactors boards over 100 pages finish across pulls, and end a read incomplete on a page with nothing new while a cursor follows, whatever the total, which SuccessFactors now hands back on a repeated page

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-10-05 09:51:16 +02:00
1 parent 85aed0807b
commit 7f37270eb3
6 files changed
+219 -18

No files matched your search

+25
View File
@@ -251,6 +251,31 @@ def test_a_first_page_stating_no_total_is_read_to_a_page_that_adds_nothing(
assert len(rows) == 30 and not complete
def test_a_board_over_the_page_cap_is_swept_from_its_cursor_as_json_gives_it_back(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
"""The cursor `(offset, total)` is a tuple, kept in state as a list, and
the total later pages state as 0 is carried in it. The sweep is kept,
and the next pull reads on from it to the first page's total."""
tenant = Tenant(45)
monkeypatch.setattr(ats, "_get", tenant)
monkeypatch.setattr(ats, "MAX_PAGES", 2)
target = f"workday:{SLUG}"
state: dict[str, str] = {}
def pull_once() -> list[Any]:
source = ats.ATSSource({"boards": [{"url": target, "label": "Zendesk", "params": {}}],
"run": "r"})
source.state = state
return list(source.fetch())
assert len(pull_once()) == 40
assert json.loads(state[f"ats:sweep:{target}"])["cursor"] == [40, 45]
assert len(pull_once()) == 5
assert (state[f"ats:complete:{target}"], state[f"ats:swept:{target}"]) == ("1", "1")
assert len(json.loads(state[f"ats:ids:{target}"])) == 45
assert [c[2]["offset"] for c in tenant.calls] == [0, 20, 40]
def test_a_page_that_is_not_json_fails_the_read(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
monkeypatch.setattr(ats, "_get", lambda *a, **k: b"<html>maintenance</html>")