Keep the sweep of a reader whose cursor is a tuple, so Workday and SuccessFactors boards over 100 pages finish across pulls, and end a read incomplete on a page with nothing new while a cursor follows, whatever the total, which SuccessFactors now hands back on a repeated page

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-10-05 09:51:16 +02:00
1 parent 85aed0807b
commit 7f37270eb3
6 files changed
+219 -18

No files matched your search

+29 -10
View File
@@ -556,17 +556,20 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
A whole listing is complete whenever it parses. A paged one reads each A whole listing is complete whenever it parses. A paged one reads each
series' pages, `limits.page_delay` apart, until the host has no next series' pages, `limits.page_delay` apart, until the host has no next
page, the stated total is reached, a page brings nothing new (a host page, the stated total is reached, a page brings no new id while the
that ignores the cursor, incomplete) or `limits.max_pages` pages in all. reader still hands a cursor (a stall: a host that ignores the cursor or
A series is complete when it read as many distinct result ids as its sends a page again, with or without a total) or `limits.max_pages`
stated total, or, without a total, when it ended on the host's last pages in all. A series is complete when it read as many distinct result
page; never for `complete="never"`. The read is complete when every ids as its stated total, or, without a total, when it ended on the
series is. A 429 after the first page ends the read, incomplete. host's last page; never after a stall, and never for
`complete="never"`. The read is complete when every series is. A 429
after the first page ends the read, incomplete.
With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns
the `sweep` to keep, and the next pull, handed it back, continues from the `sweep` to keep, and the next pull, handed it back, continues from
its series and cursor; the sweep is complete when it reaches the end, its series and cursor; the sweep is complete when it reaches the end,
and `ids` then hold every id it listed, and the listing is `swept`. A and `ids` then hold every id it listed, and the listing is `swept`. The
cursor comes back as JSON gives it, a tuple as a list (`_storable`). A
read at the reader's `ceiling` is not swept: the host answers nothing read at the reader's `ceiling` is not swept: the host answers nothing
past it. past it.
@@ -633,7 +636,10 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
if cursor is None or (isinstance(total, int) and len(seen) >= total): if cursor is None or (isinstance(total, int) and len(seen) >= total):
ended = True ended = True
break break
if total is None and len(seen) == before: # Nothing new while the reader says more follow: the host ignores
# the cursor or sent a page again. A stall, whatever the total: the
# series ends incomplete rather than loop to the page cap.
if len(seen) == before:
stalled = True stalled = True
break break
if stop: if stop:
@@ -663,13 +669,26 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
def _storable(cursor: Any) -> bool: def _storable(cursor: Any) -> bool:
"""Whether a cursor survives a trip through JSON in state.""" """Whether a cursor survives a trip through JSON in state. A tuple comes
back as a list (Workday's `(offset, total)`, SuccessFactors' `(start
row, first id)`), which counts as surviving: the next pull hands the
reader its cursor as JSON gives it back, and every paged reader takes
the list as it took the tuple."""
try: try:
return json.loads(json.dumps(cursor)) == cursor return json.loads(json.dumps(cursor)) == _as_json(cursor)
except (TypeError, ValueError): except (TypeError, ValueError):
return False return False
def _as_json(value: Any) -> Any:
"""`value` with every tuple in it a list, as JSON hands it back."""
if isinstance(value, tuple | list):
return [_as_json(item) for item in value]
if isinstance(value, dict):
return {key: _as_json(item) for key, item in value.items()}
return value
# ------------------------------------------------------------ with state # ------------------------------------------------------------ with state
def _load(state: Mapping[str, str], key: str) -> dict[str, Any]: def _load(state: Mapping[str, str], key: str) -> dict[str, Any]:
+9 -5
View File
@@ -269,10 +269,14 @@ def _entries(text: str) -> list[dict[str, Any]]:
def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
"""One search page, in the one country `ask` names, matching its `query`. The cursor is `(start row, first role of the page """One search page, in the one country `ask` names, matching its `query`.
before)`: the next start row is what this page returned, and a page that The cursor is `(start row, first role of the page before)`, a list when
starts with the same role as the last one is the site ignoring `startrow`, a sweep hands it back from state: the next start row is what this page
which ends the read (incomplete) instead of looping.""" returned. A page that starts with the same role as the last one is the
site ignoring `startrow`: it comes back with no results and the cursor
it was asked with, which the runner reads as a stall that ends the read
incomplete. A cursor of None there would read as the last page, and,
on a site that states no total, as a complete read."""
start, previous = cursor or (0, None) start, previous = cursor or (0, None)
country = ask.countries[0] if ask.countries else "" country = ask.countries[0] if ask.countries else ""
text = get(_search_url(slug, start, country, ask.query)).decode("utf-8", "replace") text = get(_search_url(slug, start, country, ask.query)).decode("utf-8", "replace")
@@ -281,7 +285,7 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
found = _entries(text) found = _entries(text)
total = _total(text) total = _total(text)
if found and found[0]["id"] == previous: if found and found[0]["id"] == previous:
return text, [], total, None return text, [], total, (start, previous)
more = bool(found) and (total is None or start + len(found) < total) more = bool(found) and (total is None or start + len(found) < total)
return (text, [{"id": entry["id"]} for entry in found], total, return (text, [{"id": entry["id"]} for entry in found], total,
(start + len(found), found[0]["id"]) if more else None) (start + len(found), found[0]["id"]) if more else None)
+7 -1
View File
@@ -618,7 +618,13 @@ def test_the_page_cap_leaves_a_board_incomplete(
def fake_get(client, url, host, slug, *, method="GET", payload=None): def fake_get(client, url, host, slug, *, method="GET", payload=None):
calls.append(payload) calls.append(payload)
return _page("workable.json") # always another page # Always another page of roles not read yet: a page of nothing new
# would be a stall, which ends the read before the cap.
page = json.loads(_page("workable.json"))
page["total"] = 100
for result in page["results"]:
result["id"] = f"{result['id']}-{len(calls)}"
return json.dumps(page).encode()
monkeypatch.setattr(ats, "MAX_PAGES", 3) monkeypatch.setattr(ats, "MAX_PAGES", 3)
monkeypatch.setattr(ats, "_get", fake_get) monkeypatch.setattr(ats, "_get", fake_get)
+83 -2
View File
@@ -1,15 +1,17 @@
"""The SuccessFactors host: `plugins/readers/successfactors.py` against saved """The SuccessFactors host: `plugins/readers/successfactors.py` against saved
career-site pages. No network. The fixtures are trimmed pages of Vestas and career-site pages, and against written ones through a sweep and a pull with
its settle. No network. The fixtures are trimmed pages of Vestas and
SAP (table layout), Demant and Scania (tile layout), Teradyne (a shared site SAP (table layout), Demant and Scania (tile layout), Teradyne (a shared site
with a path prefix), and the job pages of Vestas (`addressCountry`), with a path prefix), and the job pages of Vestas (`addressCountry`),
Systematic (`streetAddress`) and Novo Nordisk (no microdata).""" Systematic (`streetAddress`) and Novo Nordisk (no microdata)."""
from __future__ import annotations from __future__ import annotations
import json
import sys import sys
from datetime import UTC, datetime from datetime import UTC, datetime
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any, ClassVar
import httpx import httpx
import pytest import pytest
@@ -18,13 +20,28 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path: if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from pipeline import Tally # noqa: E402
from app.jobs import pull # noqa: E402
from app.jobs.discover import boards_in # noqa: E402 from app.jobs.discover import boards_in # noqa: E402
from app.storage import Store # noqa: E402
from plugins import ats, readers # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.readers import successfactors # noqa: E402 from plugins.readers import successfactors # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
class FakeConfig:
sources: ClassVar[dict[str, object]] = {}
searches: ClassVar[list[object]] = []
def _pull(store: Store) -> dict[str, Any]:
"""One pull of every board, batches and settle, as `test_sweep_strikes`."""
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
def _read(name: str) -> str: def _read(name: str) -> str:
return (FIXTURES / name).read_text(encoding="utf-8") return (FIXTURES / name).read_text(encoding="utf-8")
@@ -264,6 +281,70 @@ def test_a_site_that_ignores_startrow_does_not_loop(monkeypatch: pytest.MonkeyPa
assert len(asked) == 2 assert len(asked) == 2
def _untotalled(ids: list[int]) -> str:
"""A table page that states no total."""
return _table_page(ids, 0).split("</span>", 1)[1]
def test_the_repeated_page_hands_back_a_cursor_with_no_results() -> None:
text = _untotalled([1_000_000_001, 1_000_000_002])
_decoded, results, total, cursor = successfactors.page(
lambda url, **kw: text.encode(), "careers.acme.com", [2, "1000000001"])
assert (results, total, cursor) == ([], None, (2, "1000000001"))
def test_a_site_that_ignores_startrow_and_states_no_total_closes_nothing(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""The repeated first page used to end the read with no cursor, which a
site that states no total made complete, and the settle closed every
role past the first page."""
store = Store(tmp_path / "roledesk.db")
store.add_subscription("ats", "successfactors:careers.acme.com", label="Acme",
company="Acme", board="successfactors")
ids = [1_000_000_000 + n for n in range(4)]
site = {"ignores_startrow": False}
def fake_get(client, url, host, slug, **kw):
start = 0 if site["ignores_startrow"] else int(dict(httpx.URL(url).params)["startrow"])
return _untotalled(ids[start:start + 2]).encode()
monkeypatch.setattr(ats, "_get", fake_get)
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
_pull(store)
assert store.state["ats:complete:successfactors:careers.acme.com"] == "1"
assert len(store.roles()) == 4
site["ignores_startrow"] = True
result = _pull(store)
assert store.state["ats:complete:successfactors:careers.acme.com"] == ""
assert (result["incomplete"], result["closed"], len(store.roles())) == (1, 0, 4)
def test_a_site_over_the_page_cap_is_swept_from_its_cursor_as_json_gives_it_back(
monkeypatch: pytest.MonkeyPatch) -> None:
"""The cursor `(start row, first id)` is a tuple, kept in state as a
list. The sweep is kept, and the next pull reads on from it."""
asked = _site(monkeypatch, 5, 2)
monkeypatch.setattr(ats, "MAX_PAGES", 2)
target = "successfactors:careers.acme.com"
state: dict[str, str] = {}
def pull_once() -> list[Any]:
source = ats.ATSSource({"boards": [{"url": target, "label": "Acme", "params": {}}],
"run": "r"})
source.state = state
return list(source.fetch())
assert len(pull_once()) == 4
assert state[f"ats:complete:{target}"] == ""
assert json.loads(state[f"ats:sweep:{target}"])["cursor"] == [4, "1000000002"]
assert len(pull_once()) == 1
assert (state[f"ats:complete:{target}"], state[f"ats:swept:{target}"]) == ("1", "1")
assert json.loads(state[f"ats:ids:{target}"]) == [
f"{target}:{1_000_000_000 + n}" for n in range(5)]
assert [int(u.rsplit("=", 1)[1]) for u in asked] == [0, 2, 4]
def test_a_board_with_no_roles_is_complete_at_zero(monkeypatch: pytest.MonkeyPatch) -> None: def test_a_board_with_no_roles_is_complete_at_zero(monkeypatch: pytest.MonkeyPatch) -> None:
_site(monkeypatch, 0, 10) _site(monkeypatch, 0, 10)
assert ats._fetch_rows(object(), "successfactors:careers.acme.com") == ([], True) # type: ignore[arg-type] assert ats._fetch_rows(object(), "successfactors:careers.acme.com") == ([], True) # type: ignore[arg-type]
+25
View File
@@ -251,6 +251,31 @@ def test_a_first_page_stating_no_total_is_read_to_a_page_that_adds_nothing(
assert len(rows) == 30 and not complete assert len(rows) == 30 and not complete
def test_a_board_over_the_page_cap_is_swept_from_its_cursor_as_json_gives_it_back(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
"""The cursor `(offset, total)` is a tuple, kept in state as a list, and
the total later pages state as 0 is carried in it. The sweep is kept,
and the next pull reads on from it to the first page's total."""
tenant = Tenant(45)
monkeypatch.setattr(ats, "_get", tenant)
monkeypatch.setattr(ats, "MAX_PAGES", 2)
target = f"workday:{SLUG}"
state: dict[str, str] = {}
def pull_once() -> list[Any]:
source = ats.ATSSource({"boards": [{"url": target, "label": "Zendesk", "params": {}}],
"run": "r"})
source.state = state
return list(source.fetch())
assert len(pull_once()) == 40
assert json.loads(state[f"ats:sweep:{target}"])["cursor"] == [40, 45]
assert len(pull_once()) == 5
assert (state[f"ats:complete:{target}"], state[f"ats:swept:{target}"]) == ("1", "1")
assert len(json.loads(state[f"ats:ids:{target}"])) == 45
assert [c[2]["offset"] for c in tenant.calls] == [0, 20, 40]
def test_a_page_that_is_not_json_fails_the_read( def test_a_page_that_is_not_json_fails_the_read(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None: monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
monkeypatch.setattr(ats, "_get", lambda *a, **k: b"<html>maintenance</html>") monkeypatch.setattr(ats, "_get", lambda *a, **k: b"<html>maintenance</html>")
+66
View File
@@ -215,6 +215,28 @@ def test_a_page_with_nothing_new_ends_the_read_incomplete() -> None:
assert [index for index, _ in module.asked] == [0, 1] assert [index for index, _ in module.asked] == [0, 1]
def test_a_page_with_nothing_new_is_a_stall_with_a_stated_total_too() -> None:
"""A host that ignores the cursor and states a total used to be read on
to the page cap; the second page, which adds nothing while the reader
hands a cursor, ends it incomplete."""
pages = [["1", "2"]] * 5
module = _paged(pages, total=5)
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2"], False)
assert [index for index, _ in module.asked] == [0, 1]
def test_an_empty_last_page_ends_a_read_without_a_total_complete() -> None:
"""`complete="end"`: the page past the last role is empty and hands no
cursor. That is the end, not a stall."""
pages = [["1"], ["2"], []]
module = _paged(pages)
assert module.READS.complete == "end"
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2"], True)
assert [index for index, _ in module.asked] == [0, 1, 2]
def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None: def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None:
pages = [["1"], ["2"]] pages = [["1"], ["2"]]
web = _pages_web(pages) web = _pages_web(pages)
@@ -260,6 +282,50 @@ def test_a_board_over_the_page_cap_is_swept_across_pulls() -> None:
assert state["ats:swept:fake:acme"] == "1" assert state["ats:swept:fake:acme"] == "1"
def _tupled(pages: list[list[str]], total: int) -> Any:
"""`_paged` with a cursor of Workday's kind, `(next page, total)`, which a
sweep hands back from state as a list. `cursors` are the ones it was
handed."""
module = _paged(pages, total=total)
module.cursors = []
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
module.cursors.append(cursor)
index, stated = cursor or (0, total)
ids = pages[index]
more = index + 1 < len(pages)
return ids, [{"id": i} for i in ids], stated, (index + 1, stated) if more else None
module.page = page
return module
def test_a_board_whose_cursor_is_a_tuple_is_swept_across_pulls() -> None:
"""A tuple comes back from JSON as a list. The sweep is kept all the
same, the reader gets the list, and the second pull finishes it."""
module = _tupled([["1"], ["2"], ["3"]], total=3)
state: dict[str, str] = {}
limits = _runner.Limits(max_pages=2, page_delay=0)
first = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits)
_runner.stamp(state, "ats", "fake:acme", "r", first, first.ids)
assert (first.ids, first.complete) == (["1", "2"], False)
assert json.loads(state["ats:sweep:fake:acme"])["cursor"] == [2, 3]
second = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits)
_runner.stamp(state, "ats", "fake:acme", "r", second, second.ids)
assert (second.ids, second.complete, second.swept) == (["1", "2", "3"], True, True)
assert module.cursors == [None, (1, 3), [2, 3]]
assert "ats:sweep:fake:acme" not in state
def test_a_cursor_is_storable_when_json_gives_it_back_save_tuples_as_lists() -> None:
assert _runner._storable(7) and _runner._storable("token") and _runner._storable(None)
assert _runner._storable((40, 45)) and _runner._storable((4, "1431772533"))
assert _runner._storable({"after": (1, [2, (3,)])})
assert not _runner._storable({1: "a"}) # the key comes back as "1"
assert not _runner._storable(object())
def test_a_board_read_in_one_pull_is_not_swept() -> None: def test_a_board_read_in_one_pull_is_not_swept() -> None:
pages = [["1"], ["2"]] pages = [["1"], ["2"]]
state: dict[str, str] = {} state: dict[str, str] = {}