From 7f37270eb33beda9271bf45693c6dd115058cfcf Mon Sep 17 00:00:00 2001 From: Wolf Date: Mon, 5 Oct 2026 09:51:16 +0200 Subject: [PATCH] Keep the sweep of a reader whose cursor is a tuple, so Workday and SuccessFactors boards over 100 pages finish across pulls, and end a read incomplete on a page with nothing new while a cursor follows, whatever the total, which SuccessFactors now hands back on a repeated page Co-Authored-By: Claude Fable 5.1 --- plugins/readers/_runner.py | 39 ++++++++++---- plugins/readers/successfactors.py | 14 +++-- tests/test_ats.py | 8 ++- tests/test_ats_successfactors.py | 85 ++++++++++++++++++++++++++++++- tests/test_ats_workday.py | 25 +++++++++ tests/test_runner.py | 66 ++++++++++++++++++++++++ 6 files changed, 219 insertions(+), 18 deletions(-) diff --git a/plugins/readers/_runner.py b/plugins/readers/_runner.py index cb13887..3e5011f 100644 --- a/plugins/readers/_runner.py +++ b/plugins/readers/_runner.py @@ -556,17 +556,20 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None A whole listing is complete whenever it parses. A paged one reads each series' pages, `limits.page_delay` apart, until the host has no next - page, the stated total is reached, a page brings nothing new (a host - that ignores the cursor, incomplete) or `limits.max_pages` pages in all. - A series is complete when it read as many distinct result ids as its - stated total, or, without a total, when it ended on the host's last - page; never for `complete="never"`. The read is complete when every - series is. A 429 after the first page ends the read, incomplete. + page, the stated total is reached, a page brings no new id while the + reader still hands a cursor (a stall: a host that ignores the cursor or + sends a page again, with or without a total) or `limits.max_pages` + pages in all. A series is complete when it read as many distinct result + ids as its stated total, or, without a total, when it ended on the + host's last page; never after a stall, and never for + `complete="never"`. The read is complete when every series is. A 429 + after the first page ends the read, incomplete. With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns the `sweep` to keep, and the next pull, handed it back, continues from its series and cursor; the sweep is complete when it reaches the end, - and `ids` then hold every id it listed, and the listing is `swept`. A + and `ids` then hold every id it listed, and the listing is `swept`. The + cursor comes back as JSON gives it, a tuple as a list (`_storable`). A read at the reader's `ceiling` is not swept: the host answers nothing past it. @@ -633,7 +636,10 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None if cursor is None or (isinstance(total, int) and len(seen) >= total): ended = True break - if total is None and len(seen) == before: + # Nothing new while the reader says more follow: the host ignores + # the cursor or sent a page again. A stall, whatever the total: the + # series ends incomplete rather than loop to the page cap. + if len(seen) == before: stalled = True break if stop: @@ -663,13 +669,26 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None def _storable(cursor: Any) -> bool: - """Whether a cursor survives a trip through JSON in state.""" + """Whether a cursor survives a trip through JSON in state. A tuple comes + back as a list (Workday's `(offset, total)`, SuccessFactors' `(start + row, first id)`), which counts as surviving: the next pull hands the + reader its cursor as JSON gives it back, and every paged reader takes + the list as it took the tuple.""" try: - return json.loads(json.dumps(cursor)) == cursor + return json.loads(json.dumps(cursor)) == _as_json(cursor) except (TypeError, ValueError): return False +def _as_json(value: Any) -> Any: + """`value` with every tuple in it a list, as JSON hands it back.""" + if isinstance(value, tuple | list): + return [_as_json(item) for item in value] + if isinstance(value, dict): + return {key: _as_json(item) for key, item in value.items()} + return value + + # ------------------------------------------------------------ with state def _load(state: Mapping[str, str], key: str) -> dict[str, Any]: diff --git a/plugins/readers/successfactors.py b/plugins/readers/successfactors.py index 03dfee2..3ad376d 100644 --- a/plugins/readers/successfactors.py +++ b/plugins/readers/successfactors.py @@ -269,10 +269,14 @@ def _entries(text: str) -> list[dict[str, Any]]: def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: - """One search page, in the one country `ask` names, matching its `query`. The cursor is `(start row, first role of the page - before)`: the next start row is what this page returned, and a page that - starts with the same role as the last one is the site ignoring `startrow`, - which ends the read (incomplete) instead of looping.""" + """One search page, in the one country `ask` names, matching its `query`. + The cursor is `(start row, first role of the page before)`, a list when + a sweep hands it back from state: the next start row is what this page + returned. A page that starts with the same role as the last one is the + site ignoring `startrow`: it comes back with no results and the cursor + it was asked with, which the runner reads as a stall that ends the read + incomplete. A cursor of None there would read as the last page, and, + on a site that states no total, as a complete read.""" start, previous = cursor or (0, None) country = ask.countries[0] if ask.countries else "" text = get(_search_url(slug, start, country, ask.query)).decode("utf-8", "replace") @@ -281,7 +285,7 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: found = _entries(text) total = _total(text) if found and found[0]["id"] == previous: - return text, [], total, None + return text, [], total, (start, previous) more = bool(found) and (total is None or start + len(found) < total) return (text, [{"id": entry["id"]} for entry in found], total, (start + len(found), found[0]["id"]) if more else None) diff --git a/tests/test_ats.py b/tests/test_ats.py index e82808c..8fc033e 100644 --- a/tests/test_ats.py +++ b/tests/test_ats.py @@ -618,7 +618,13 @@ def test_the_page_cap_leaves_a_board_incomplete( def fake_get(client, url, host, slug, *, method="GET", payload=None): calls.append(payload) - return _page("workable.json") # always another page + # Always another page of roles not read yet: a page of nothing new + # would be a stall, which ends the read before the cap. + page = json.loads(_page("workable.json")) + page["total"] = 100 + for result in page["results"]: + result["id"] = f"{result['id']}-{len(calls)}" + return json.dumps(page).encode() monkeypatch.setattr(ats, "MAX_PAGES", 3) monkeypatch.setattr(ats, "_get", fake_get) diff --git a/tests/test_ats_successfactors.py b/tests/test_ats_successfactors.py index c3b4e4b..ea41762 100644 --- a/tests/test_ats_successfactors.py +++ b/tests/test_ats_successfactors.py @@ -1,15 +1,17 @@ """The SuccessFactors host: `plugins/readers/successfactors.py` against saved -career-site pages. No network. The fixtures are trimmed pages of Vestas and +career-site pages, and against written ones through a sweep and a pull with +its settle. No network. The fixtures are trimmed pages of Vestas and SAP (table layout), Demant and Scania (tile layout), Teradyne (a shared site with a path prefix), and the job pages of Vestas (`addressCountry`), Systematic (`streetAddress`) and Novo Nordisk (no microdata).""" from __future__ import annotations +import json import sys from datetime import UTC, datetime from pathlib import Path -from typing import Any +from typing import Any, ClassVar import httpx import pytest @@ -18,13 +20,28 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from pipeline import Tally # noqa: E402 +from app.jobs import pull # noqa: E402 from app.jobs.discover import boards_in # noqa: E402 +from app.storage import Store # noqa: E402 from plugins import ats, readers # noqa: E402 from plugins.readers import successfactors # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" +class FakeConfig: + sources: ClassVar[dict[str, object]] = {} + searches: ClassVar[list[object]] = [] + + +def _pull(store: Store) -> dict[str, Any]: + """One pull of every board, batches and settle, as `test_sweep_strikes`.""" + results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] + for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type] + return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] + + def _read(name: str) -> str: return (FIXTURES / name).read_text(encoding="utf-8") @@ -264,6 +281,70 @@ def test_a_site_that_ignores_startrow_does_not_loop(monkeypatch: pytest.MonkeyPa assert len(asked) == 2 +def _untotalled(ids: list[int]) -> str: + """A table page that states no total.""" + return _table_page(ids, 0).split("", 1)[1] + + +def test_the_repeated_page_hands_back_a_cursor_with_no_results() -> None: + text = _untotalled([1_000_000_001, 1_000_000_002]) + _decoded, results, total, cursor = successfactors.page( + lambda url, **kw: text.encode(), "careers.acme.com", [2, "1000000001"]) + assert (results, total, cursor) == ([], None, (2, "1000000001")) + + +def test_a_site_that_ignores_startrow_and_states_no_total_closes_nothing( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """The repeated first page used to end the read with no cursor, which a + site that states no total made complete, and the settle closed every + role past the first page.""" + store = Store(tmp_path / "roledesk.db") + store.add_subscription("ats", "successfactors:careers.acme.com", label="Acme", + company="Acme", board="successfactors") + ids = [1_000_000_000 + n for n in range(4)] + site = {"ignores_startrow": False} + + def fake_get(client, url, host, slug, **kw): + start = 0 if site["ignores_startrow"] else int(dict(httpx.URL(url).params)["startrow"]) + return _untotalled(ids[start:start + 2]).encode() + + monkeypatch.setattr(ats, "_get", fake_get) + monkeypatch.setattr(ats, "PAGE_DELAY", 0) + _pull(store) + assert store.state["ats:complete:successfactors:careers.acme.com"] == "1" + assert len(store.roles()) == 4 + + site["ignores_startrow"] = True + result = _pull(store) + assert store.state["ats:complete:successfactors:careers.acme.com"] == "" + assert (result["incomplete"], result["closed"], len(store.roles())) == (1, 0, 4) + + +def test_a_site_over_the_page_cap_is_swept_from_its_cursor_as_json_gives_it_back( + monkeypatch: pytest.MonkeyPatch) -> None: + """The cursor `(start row, first id)` is a tuple, kept in state as a + list. The sweep is kept, and the next pull reads on from it.""" + asked = _site(monkeypatch, 5, 2) + monkeypatch.setattr(ats, "MAX_PAGES", 2) + target = "successfactors:careers.acme.com" + state: dict[str, str] = {} + + def pull_once() -> list[Any]: + source = ats.ATSSource({"boards": [{"url": target, "label": "Acme", "params": {}}], + "run": "r"}) + source.state = state + return list(source.fetch()) + + assert len(pull_once()) == 4 + assert state[f"ats:complete:{target}"] == "" + assert json.loads(state[f"ats:sweep:{target}"])["cursor"] == [4, "1000000002"] + assert len(pull_once()) == 1 + assert (state[f"ats:complete:{target}"], state[f"ats:swept:{target}"]) == ("1", "1") + assert json.loads(state[f"ats:ids:{target}"]) == [ + f"{target}:{1_000_000_000 + n}" for n in range(5)] + assert [int(u.rsplit("=", 1)[1]) for u in asked] == [0, 2, 4] + + def test_a_board_with_no_roles_is_complete_at_zero(monkeypatch: pytest.MonkeyPatch) -> None: _site(monkeypatch, 0, 10) assert ats._fetch_rows(object(), "successfactors:careers.acme.com") == ([], True) # type: ignore[arg-type] diff --git a/tests/test_ats_workday.py b/tests/test_ats_workday.py index a089e9b..9598b75 100644 --- a/tests/test_ats_workday.py +++ b/tests/test_ats_workday.py @@ -251,6 +251,31 @@ def test_a_first_page_stating_no_total_is_read_to_a_page_that_adds_nothing( assert len(rows) == 30 and not complete +def test_a_board_over_the_page_cap_is_swept_from_its_cursor_as_json_gives_it_back( + monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None: + """The cursor `(offset, total)` is a tuple, kept in state as a list, and + the total later pages state as 0 is carried in it. The sweep is kept, + and the next pull reads on from it to the first page's total.""" + tenant = Tenant(45) + monkeypatch.setattr(ats, "_get", tenant) + monkeypatch.setattr(ats, "MAX_PAGES", 2) + target = f"workday:{SLUG}" + state: dict[str, str] = {} + + def pull_once() -> list[Any]: + source = ats.ATSSource({"boards": [{"url": target, "label": "Zendesk", "params": {}}], + "run": "r"}) + source.state = state + return list(source.fetch()) + + assert len(pull_once()) == 40 + assert json.loads(state[f"ats:sweep:{target}"])["cursor"] == [40, 45] + assert len(pull_once()) == 5 + assert (state[f"ats:complete:{target}"], state[f"ats:swept:{target}"]) == ("1", "1") + assert len(json.loads(state[f"ats:ids:{target}"])) == 45 + assert [c[2]["offset"] for c in tenant.calls] == [0, 20, 40] + + def test_a_page_that_is_not_json_fails_the_read( monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None: monkeypatch.setattr(ats, "_get", lambda *a, **k: b"maintenance") diff --git a/tests/test_runner.py b/tests/test_runner.py index 124cf4f..9d5cc41 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -215,6 +215,28 @@ def test_a_page_with_nothing_new_ends_the_read_incomplete() -> None: assert [index for index, _ in module.asked] == [0, 1] +def test_a_page_with_nothing_new_is_a_stall_with_a_stated_total_too() -> None: + """A host that ignores the cursor and states a total used to be read on + to the page cap; the second page, which adds nothing while the reader + hands a cursor, ends it incomplete.""" + pages = [["1", "2"]] * 5 + module = _paged(pages, total=5) + listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY) + assert (listing.ids, listing.complete) == (["1", "2"], False) + assert [index for index, _ in module.asked] == [0, 1] + + +def test_an_empty_last_page_ends_a_read_without_a_total_complete() -> None: + """`complete="end"`: the page past the last role is empty and hands no + cursor. That is the end, not a stall.""" + pages = [["1"], ["2"], []] + module = _paged(pages) + assert module.READS.complete == "end" + listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY) + assert (listing.ids, listing.complete) == (["1", "2"], True) + assert [index for index, _ in module.asked] == [0, 1, 2] + + def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None: pages = [["1"], ["2"]] web = _pages_web(pages) @@ -260,6 +282,50 @@ def test_a_board_over_the_page_cap_is_swept_across_pulls() -> None: assert state["ats:swept:fake:acme"] == "1" +def _tupled(pages: list[list[str]], total: int) -> Any: + """`_paged` with a cursor of Workday's kind, `(next page, total)`, which a + sweep hands back from state as a list. `cursors` are the ones it was + handed.""" + module = _paged(pages, total=total) + module.cursors = [] + + def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any: + module.cursors.append(cursor) + index, stated = cursor or (0, total) + ids = pages[index] + more = index + 1 < len(pages) + return ids, [{"id": i} for i in ids], stated, (index + 1, stated) if more else None + + module.page = page + return module + + +def test_a_board_whose_cursor_is_a_tuple_is_swept_across_pulls() -> None: + """A tuple comes back from JSON as a list. The sweep is kept all the + same, the reader gets the list, and the second pull finishes it.""" + module = _tupled([["1"], ["2"], ["3"]], total=3) + state: dict[str, str] = {} + limits = _runner.Limits(max_pages=2, page_delay=0) + first = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits) + _runner.stamp(state, "ats", "fake:acme", "r", first, first.ids) + assert (first.ids, first.complete) == (["1", "2"], False) + assert json.loads(state["ats:sweep:fake:acme"])["cursor"] == [2, 3] + + second = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits) + _runner.stamp(state, "ats", "fake:acme", "r", second, second.ids) + assert (second.ids, second.complete, second.swept) == (["1", "2", "3"], True, True) + assert module.cursors == [None, (1, 3), [2, 3]] + assert "ats:sweep:fake:acme" not in state + + +def test_a_cursor_is_storable_when_json_gives_it_back_save_tuples_as_lists() -> None: + assert _runner._storable(7) and _runner._storable("token") and _runner._storable(None) + assert _runner._storable((40, 45)) and _runner._storable((4, "1431772533")) + assert _runner._storable({"after": (1, [2, (3,)])}) + assert not _runner._storable({1: "a"}) # the key comes back as "1" + assert not _runner._storable(object()) + + def test_a_board_read_in_one_pull_is_not_swept() -> None: pages = [["1"], ["2"]] state: dict[str, str] = {}