Keep the sweep of a reader whose cursor is a tuple, so Workday and SuccessFactors boards over 100 pages finish across pulls, and end a read incomplete on a page with nothing new while a cursor follows, whatever the total, which SuccessFactors now hands back on a repeated page
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
85aed0807b
commit
7f37270eb3
6 files changed
+219
-18
No files matched your search
@@ -251,6 +251,31 @@ def test_a_first_page_stating_no_total_is_read_to_a_page_that_adds_nothing(
|
||||
assert len(rows) == 30 and not complete
|
||||
|
||||
|
||||
def test_a_board_over_the_page_cap_is_swept_from_its_cursor_as_json_gives_it_back(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
"""The cursor `(offset, total)` is a tuple, kept in state as a list, and
|
||||
the total later pages state as 0 is carried in it. The sweep is kept,
|
||||
and the next pull reads on from it to the first page's total."""
|
||||
tenant = Tenant(45)
|
||||
monkeypatch.setattr(ats, "_get", tenant)
|
||||
monkeypatch.setattr(ats, "MAX_PAGES", 2)
|
||||
target = f"workday:{SLUG}"
|
||||
state: dict[str, str] = {}
|
||||
|
||||
def pull_once() -> list[Any]:
|
||||
source = ats.ATSSource({"boards": [{"url": target, "label": "Zendesk", "params": {}}],
|
||||
"run": "r"})
|
||||
source.state = state
|
||||
return list(source.fetch())
|
||||
|
||||
assert len(pull_once()) == 40
|
||||
assert json.loads(state[f"ats:sweep:{target}"])["cursor"] == [40, 45]
|
||||
assert len(pull_once()) == 5
|
||||
assert (state[f"ats:complete:{target}"], state[f"ats:swept:{target}"]) == ("1", "1")
|
||||
assert len(json.loads(state[f"ats:ids:{target}"])) == 45
|
||||
assert [c[2]["offset"] for c in tenant.calls] == [0, 20, 40]
|
||||
|
||||
|
||||
def test_a_page_that_is_not_json_fails_the_read(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
monkeypatch.setattr(ats, "_get", lambda *a, **k: b"<html>maintenance</html>")
|
||||
|
||||
Reference in new issue
Block a user