Keep the sweep of a reader whose cursor is a tuple, so Workday and SuccessFactors boards over 100 pages finish across pulls, and end a read incomplete on a page with nothing new while a cursor follows, whatever the total, which SuccessFactors now hands back on a repeated page
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
85aed0807b
commit
7f37270eb3
6 files changed
+219
-18
No files matched your search
@@ -269,10 +269,14 @@ def _entries(text: str) -> list[dict[str, Any]]:
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
||||
"""One search page, in the one country `ask` names, matching its `query`. The cursor is `(start row, first role of the page
|
||||
before)`: the next start row is what this page returned, and a page that
|
||||
starts with the same role as the last one is the site ignoring `startrow`,
|
||||
which ends the read (incomplete) instead of looping."""
|
||||
"""One search page, in the one country `ask` names, matching its `query`.
|
||||
The cursor is `(start row, first role of the page before)`, a list when
|
||||
a sweep hands it back from state: the next start row is what this page
|
||||
returned. A page that starts with the same role as the last one is the
|
||||
site ignoring `startrow`: it comes back with no results and the cursor
|
||||
it was asked with, which the runner reads as a stall that ends the read
|
||||
incomplete. A cursor of None there would read as the last page, and,
|
||||
on a site that states no total, as a complete read."""
|
||||
start, previous = cursor or (0, None)
|
||||
country = ask.countries[0] if ask.countries else ""
|
||||
text = get(_search_url(slug, start, country, ask.query)).decode("utf-8", "replace")
|
||||
@@ -281,7 +285,7 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
||||
found = _entries(text)
|
||||
total = _total(text)
|
||||
if found and found[0]["id"] == previous:
|
||||
return text, [], total, None
|
||||
return text, [], total, (start, previous)
|
||||
more = bool(found) and (total is None or start + len(found) < total)
|
||||
return (text, [{"id": entry["id"]} for entry in found], total,
|
||||
(start + len(found), found[0]["id"]) if more else None)
|
||||
|
||||
Reference in new issue
Block a user