Merge sweeps that resume on a tuple cursor, a stalled read counted incomplete, searches closing by age only, and Lever and Oracle failing on a wrong-shaped answer (wp-rev-runner)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
commit
62cf1c738c
11 files changed
+364
-47
No files matched your search
+33
-10
@@ -556,17 +556,21 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
|
||||
A whole listing is complete whenever it parses. A paged one reads each
|
||||
series' pages, `limits.page_delay` apart, until the host has no next
|
||||
page, the stated total is reached, a page brings nothing new (a host
|
||||
that ignores the cursor, incomplete) or `limits.max_pages` pages in all.
|
||||
A series is complete when it read as many distinct result ids as its
|
||||
stated total, or, without a total, when it ended on the host's last
|
||||
page; never for `complete="never"`. The read is complete when every
|
||||
series is. A 429 after the first page ends the read, incomplete.
|
||||
page, the stated total is reached, the read stalls or `limits.max_pages`
|
||||
pages in all. A stall is a page that brings no new id while the reader
|
||||
still hands a cursor: any such page on a host that states no total (it
|
||||
ignores the cursor), and with a total one the reader hands over empty
|
||||
(it saw the page come again). A series is complete when it read as many distinct result
|
||||
ids as its stated total, or, without a total, when it ended on the
|
||||
host's last page; never after a stall, and never for
|
||||
`complete="never"`. The read is complete when every series is. A 429
|
||||
after the first page ends the read, incomplete.
|
||||
|
||||
With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns
|
||||
the `sweep` to keep, and the next pull, handed it back, continues from
|
||||
its series and cursor; the sweep is complete when it reaches the end,
|
||||
and `ids` then hold every id it listed, and the listing is `swept`. A
|
||||
and `ids` then hold every id it listed, and the listing is `swept`. The
|
||||
cursor comes back as JSON gives it, a tuple as a list (`_storable`). A
|
||||
read at the reader's `ceiling` is not swept: the host answers nothing
|
||||
past it.
|
||||
|
||||
@@ -633,7 +637,13 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
if cursor is None or (isinstance(total, int) and len(seen) >= total):
|
||||
ended = True
|
||||
break
|
||||
if total is None and len(seen) == before:
|
||||
# Nothing new while the reader says more follow. Without a total
|
||||
# the host ignores the cursor, and the read would never end. With
|
||||
# one, only a page the reader hands over empty is a stall (it saw
|
||||
# the page come again, as SuccessFactors does): a page of ids
|
||||
# already read is what a resumed sweep starts on when the list
|
||||
# grew at the top since, and the read goes on from there.
|
||||
if len(seen) == before and (total is None or not results):
|
||||
stalled = True
|
||||
break
|
||||
if stop:
|
||||
@@ -663,13 +673,26 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
|
||||
|
||||
def _storable(cursor: Any) -> bool:
|
||||
"""Whether a cursor survives a trip through JSON in state."""
|
||||
"""Whether a cursor survives a trip through JSON in state. A tuple comes
|
||||
back as a list (Workday's `(offset, total)`, SuccessFactors' `(start
|
||||
row, first id)`), which counts as surviving: the next pull hands the
|
||||
reader its cursor as JSON gives it back, and every paged reader takes
|
||||
the list as it took the tuple."""
|
||||
try:
|
||||
return json.loads(json.dumps(cursor)) == cursor
|
||||
return json.loads(json.dumps(cursor)) == _as_json(cursor)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def _as_json(value: Any) -> Any:
|
||||
"""`value` with every tuple in it a list, as JSON hands it back."""
|
||||
if isinstance(value, tuple | list):
|
||||
return [_as_json(item) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {key: _as_json(item) for key, item in value.items()}
|
||||
return value
|
||||
|
||||
|
||||
# ------------------------------------------------------------ with state
|
||||
|
||||
def _load(state: Mapping[str, str], key: str) -> dict[str, Any]:
|
||||
|
||||
@@ -112,9 +112,13 @@ def _places(categories: dict[str, Any]) -> list[Place]:
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"""The board's postings, a JSON list. Any other answer (an error
|
||||
document sent with a 200) raises, so the read fails and closes nothing
|
||||
rather than reading as a board with no roles."""
|
||||
data = json.loads(body)
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
raise ValueError(f"Lever's answer is a JSON {type(data).__name__}, "
|
||||
"not a list of postings")
|
||||
rows = []
|
||||
for j in data:
|
||||
title = (j.get("text") or "").strip()
|
||||
|
||||
@@ -110,20 +110,31 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
||||
`items[0]`. Without the `expand` only the facets come back; asking for
|
||||
`requisitionList.secondaryLocations` gives the roles and each one's other
|
||||
locations, which the plain `requisitionList` leaves out. Past the last role
|
||||
the list is empty."""
|
||||
the list is empty; an answer without the wrapper raises (`_search`)."""
|
||||
offset = int(cursor or 0)
|
||||
site = _parts(slug)[1]
|
||||
data = json.loads(get(
|
||||
f"{_api(slug)}/recruitingCEJobRequisitions?onlyData=true"
|
||||
f"&expand=requisitionList.secondaryLocations"
|
||||
f"&finder=findReqs;siteNumber={site},{_selected(ask)}limit={PAGE_SIZE},offset={offset}"))
|
||||
search = (data.get("items") or [{}])[0]
|
||||
search = _search(data)
|
||||
roles = search.get("requisitionList") or []
|
||||
# `_paged` counts `id`, which Oracle spells `Id`.
|
||||
results = [{"id": str(role.get("Id") or "")} for role in roles]
|
||||
return search, results, search.get("TotalJobsCount"), offset + len(roles) if roles else None
|
||||
|
||||
|
||||
def _search(data: Any) -> dict[str, Any]:
|
||||
"""The search wrapper of a listing's answer, `items[0]`. Every answer has
|
||||
one, past the last role too; one without (an error document sent with a
|
||||
200) raises, so the read fails and closes nothing rather than reading as
|
||||
an empty last page with no total, which is a complete read."""
|
||||
items = data.get("items") if isinstance(data, dict) else None
|
||||
if not isinstance(items, list) or not items or not isinstance(items[0], dict):
|
||||
raise ValueError("Oracle's answer has no search wrapper in items")
|
||||
return items[0]
|
||||
|
||||
|
||||
#: How many countries `locationsFacet` lists at most; a list this long may
|
||||
#: leave some out.
|
||||
FACET_LIMIT = 10
|
||||
@@ -162,7 +173,7 @@ def facets(get: Get, slug: str) -> dict[str, dict[str, str]]:
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
return rows((json.loads(body).get("items") or [{}])[0], slug)
|
||||
return rows(_search(json.loads(body)), slug)
|
||||
|
||||
|
||||
def _text(value: Any) -> str:
|
||||
|
||||
@@ -269,10 +269,14 @@ def _entries(text: str) -> list[dict[str, Any]]:
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
||||
"""One search page, in the one country `ask` names, matching its `query`. The cursor is `(start row, first role of the page
|
||||
before)`: the next start row is what this page returned, and a page that
|
||||
starts with the same role as the last one is the site ignoring `startrow`,
|
||||
which ends the read (incomplete) instead of looping."""
|
||||
"""One search page, in the one country `ask` names, matching its `query`.
|
||||
The cursor is `(start row, first role of the page before)`, a list when
|
||||
a sweep hands it back from state: the next start row is what this page
|
||||
returned. A page that starts with the same role as the last one is the
|
||||
site ignoring `startrow`: it comes back with no results and the cursor
|
||||
it was asked with, which the runner reads as a stall that ends the read
|
||||
incomplete. A cursor of None there would read as the last page, and,
|
||||
on a site that states no total, as a complete read."""
|
||||
start, previous = cursor or (0, None)
|
||||
country = ask.countries[0] if ask.countries else ""
|
||||
text = get(_search_url(slug, start, country, ask.query)).decode("utf-8", "replace")
|
||||
@@ -281,7 +285,7 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
||||
found = _entries(text)
|
||||
total = _total(text)
|
||||
if found and found[0]["id"] == previous:
|
||||
return text, [], total, None
|
||||
return text, [], total, (start, previous)
|
||||
more = bool(found) and (total is None or start + len(found) < total)
|
||||
return (text, [{"id": entry["id"]} for entry in found], total,
|
||||
(start + len(found), found[0]["id"]) if more else None)
|
||||
|
||||
Reference in new issue
Block a user