Keep the sweep of a reader whose cursor is a tuple, so Workday and SuccessFactors boards over 100 pages finish across pulls, and end a read incomplete on a page with nothing new while a cursor follows, whatever the total, which SuccessFactors now hands back on a repeated page
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
85aed0807b
commit
7f37270eb3
6 files changed
+219
-18
No files matched your search
@@ -215,6 +215,28 @@ def test_a_page_with_nothing_new_ends_the_read_incomplete() -> None:
|
||||
assert [index for index, _ in module.asked] == [0, 1]
|
||||
|
||||
|
||||
def test_a_page_with_nothing_new_is_a_stall_with_a_stated_total_too() -> None:
|
||||
"""A host that ignores the cursor and states a total used to be read on
|
||||
to the page cap; the second page, which adds nothing while the reader
|
||||
hands a cursor, ends it incomplete."""
|
||||
pages = [["1", "2"]] * 5
|
||||
module = _paged(pages, total=5)
|
||||
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
|
||||
assert (listing.ids, listing.complete) == (["1", "2"], False)
|
||||
assert [index for index, _ in module.asked] == [0, 1]
|
||||
|
||||
|
||||
def test_an_empty_last_page_ends_a_read_without_a_total_complete() -> None:
|
||||
"""`complete="end"`: the page past the last role is empty and hands no
|
||||
cursor. That is the end, not a stall."""
|
||||
pages = [["1"], ["2"], []]
|
||||
module = _paged(pages)
|
||||
assert module.READS.complete == "end"
|
||||
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
|
||||
assert (listing.ids, listing.complete) == (["1", "2"], True)
|
||||
assert [index for index, _ in module.asked] == [0, 1, 2]
|
||||
|
||||
|
||||
def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None:
|
||||
pages = [["1"], ["2"]]
|
||||
web = _pages_web(pages)
|
||||
@@ -260,6 +282,50 @@ def test_a_board_over_the_page_cap_is_swept_across_pulls() -> None:
|
||||
assert state["ats:swept:fake:acme"] == "1"
|
||||
|
||||
|
||||
def _tupled(pages: list[list[str]], total: int) -> Any:
|
||||
"""`_paged` with a cursor of Workday's kind, `(next page, total)`, which a
|
||||
sweep hands back from state as a list. `cursors` are the ones it was
|
||||
handed."""
|
||||
module = _paged(pages, total=total)
|
||||
module.cursors = []
|
||||
|
||||
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
||||
module.cursors.append(cursor)
|
||||
index, stated = cursor or (0, total)
|
||||
ids = pages[index]
|
||||
more = index + 1 < len(pages)
|
||||
return ids, [{"id": i} for i in ids], stated, (index + 1, stated) if more else None
|
||||
|
||||
module.page = page
|
||||
return module
|
||||
|
||||
|
||||
def test_a_board_whose_cursor_is_a_tuple_is_swept_across_pulls() -> None:
|
||||
"""A tuple comes back from JSON as a list. The sweep is kept all the
|
||||
same, the reader gets the list, and the second pull finishes it."""
|
||||
module = _tupled([["1"], ["2"], ["3"]], total=3)
|
||||
state: dict[str, str] = {}
|
||||
limits = _runner.Limits(max_pages=2, page_delay=0)
|
||||
first = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits)
|
||||
_runner.stamp(state, "ats", "fake:acme", "r", first, first.ids)
|
||||
assert (first.ids, first.complete) == (["1", "2"], False)
|
||||
assert json.loads(state["ats:sweep:fake:acme"])["cursor"] == [2, 3]
|
||||
|
||||
second = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits)
|
||||
_runner.stamp(state, "ats", "fake:acme", "r", second, second.ids)
|
||||
assert (second.ids, second.complete, second.swept) == (["1", "2", "3"], True, True)
|
||||
assert module.cursors == [None, (1, 3), [2, 3]]
|
||||
assert "ats:sweep:fake:acme" not in state
|
||||
|
||||
|
||||
def test_a_cursor_is_storable_when_json_gives_it_back_save_tuples_as_lists() -> None:
|
||||
assert _runner._storable(7) and _runner._storable("token") and _runner._storable(None)
|
||||
assert _runner._storable((40, 45)) and _runner._storable((4, "1431772533"))
|
||||
assert _runner._storable({"after": (1, [2, (3,)])})
|
||||
assert not _runner._storable({1: "a"}) # the key comes back as "1"
|
||||
assert not _runner._storable(object())
|
||||
|
||||
|
||||
def test_a_board_read_in_one_pull_is_not_swept() -> None:
|
||||
pages = [["1"], ["2"]]
|
||||
state: dict[str, str] = {}
|
||||
|
||||
Reference in new issue
Block a user