Keep the sweep of a reader whose cursor is a tuple, so Workday and SuccessFactors boards over 100 pages finish across pulls, and end a read incomplete on a page with nothing new while a cursor follows, whatever the total, which SuccessFactors now hands back on a repeated page
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
85aed0807b
commit
7f37270eb3
6 files changed
+219
-18
No files matched your search
+29
-10
@@ -556,17 +556,20 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
|
||||
A whole listing is complete whenever it parses. A paged one reads each
|
||||
series' pages, `limits.page_delay` apart, until the host has no next
|
||||
page, the stated total is reached, a page brings nothing new (a host
|
||||
that ignores the cursor, incomplete) or `limits.max_pages` pages in all.
|
||||
A series is complete when it read as many distinct result ids as its
|
||||
stated total, or, without a total, when it ended on the host's last
|
||||
page; never for `complete="never"`. The read is complete when every
|
||||
series is. A 429 after the first page ends the read, incomplete.
|
||||
page, the stated total is reached, a page brings no new id while the
|
||||
reader still hands a cursor (a stall: a host that ignores the cursor or
|
||||
sends a page again, with or without a total) or `limits.max_pages`
|
||||
pages in all. A series is complete when it read as many distinct result
|
||||
ids as its stated total, or, without a total, when it ended on the
|
||||
host's last page; never after a stall, and never for
|
||||
`complete="never"`. The read is complete when every series is. A 429
|
||||
after the first page ends the read, incomplete.
|
||||
|
||||
With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns
|
||||
the `sweep` to keep, and the next pull, handed it back, continues from
|
||||
its series and cursor; the sweep is complete when it reaches the end,
|
||||
and `ids` then hold every id it listed, and the listing is `swept`. A
|
||||
and `ids` then hold every id it listed, and the listing is `swept`. The
|
||||
cursor comes back as JSON gives it, a tuple as a list (`_storable`). A
|
||||
read at the reader's `ceiling` is not swept: the host answers nothing
|
||||
past it.
|
||||
|
||||
@@ -633,7 +636,10 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
if cursor is None or (isinstance(total, int) and len(seen) >= total):
|
||||
ended = True
|
||||
break
|
||||
if total is None and len(seen) == before:
|
||||
# Nothing new while the reader says more follow: the host ignores
|
||||
# the cursor or sent a page again. A stall, whatever the total: the
|
||||
# series ends incomplete rather than loop to the page cap.
|
||||
if len(seen) == before:
|
||||
stalled = True
|
||||
break
|
||||
if stop:
|
||||
@@ -663,13 +669,26 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
|
||||
|
||||
def _storable(cursor: Any) -> bool:
|
||||
"""Whether a cursor survives a trip through JSON in state."""
|
||||
"""Whether a cursor survives a trip through JSON in state. A tuple comes
|
||||
back as a list (Workday's `(offset, total)`, SuccessFactors' `(start
|
||||
row, first id)`), which counts as surviving: the next pull hands the
|
||||
reader its cursor as JSON gives it back, and every paged reader takes
|
||||
the list as it took the tuple."""
|
||||
try:
|
||||
return json.loads(json.dumps(cursor)) == cursor
|
||||
return json.loads(json.dumps(cursor)) == _as_json(cursor)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def _as_json(value: Any) -> Any:
|
||||
"""`value` with every tuple in it a list, as JSON hands it back."""
|
||||
if isinstance(value, tuple | list):
|
||||
return [_as_json(item) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {key: _as_json(item) for key, item in value.items()}
|
||||
return value
|
||||
|
||||
|
||||
# ------------------------------------------------------------ with state
|
||||
|
||||
def _load(state: Mapping[str, str], key: str) -> dict[str, Any]:
|
||||
|
||||
@@ -269,10 +269,14 @@ def _entries(text: str) -> list[dict[str, Any]]:
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
||||
"""One search page, in the one country `ask` names, matching its `query`. The cursor is `(start row, first role of the page
|
||||
before)`: the next start row is what this page returned, and a page that
|
||||
starts with the same role as the last one is the site ignoring `startrow`,
|
||||
which ends the read (incomplete) instead of looping."""
|
||||
"""One search page, in the one country `ask` names, matching its `query`.
|
||||
The cursor is `(start row, first role of the page before)`, a list when
|
||||
a sweep hands it back from state: the next start row is what this page
|
||||
returned. A page that starts with the same role as the last one is the
|
||||
site ignoring `startrow`: it comes back with no results and the cursor
|
||||
it was asked with, which the runner reads as a stall that ends the read
|
||||
incomplete. A cursor of None there would read as the last page, and,
|
||||
on a site that states no total, as a complete read."""
|
||||
start, previous = cursor or (0, None)
|
||||
country = ask.countries[0] if ask.countries else ""
|
||||
text = get(_search_url(slug, start, country, ask.query)).decode("utf-8", "replace")
|
||||
@@ -281,7 +285,7 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
||||
found = _entries(text)
|
||||
total = _total(text)
|
||||
if found and found[0]["id"] == previous:
|
||||
return text, [], total, None
|
||||
return text, [], total, (start, previous)
|
||||
more = bool(found) and (total is None or start + len(found) < total)
|
||||
return (text, [{"id": entry["id"]} for entry in found], total,
|
||||
(start + len(found), found[0]["id"]) if more else None)
|
||||
|
||||
+7
-1
@@ -618,7 +618,13 @@ def test_the_page_cap_leaves_a_board_incomplete(
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
calls.append(payload)
|
||||
return _page("workable.json") # always another page
|
||||
# Always another page of roles not read yet: a page of nothing new
|
||||
# would be a stall, which ends the read before the cap.
|
||||
page = json.loads(_page("workable.json"))
|
||||
page["total"] = 100
|
||||
for result in page["results"]:
|
||||
result["id"] = f"{result['id']}-{len(calls)}"
|
||||
return json.dumps(page).encode()
|
||||
|
||||
monkeypatch.setattr(ats, "MAX_PAGES", 3)
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
|
||||
@@ -1,15 +1,17 @@
|
||||
"""The SuccessFactors host: `plugins/readers/successfactors.py` against saved
|
||||
career-site pages. No network. The fixtures are trimmed pages of Vestas and
|
||||
career-site pages, and against written ones through a sweep and a pull with
|
||||
its settle. No network. The fixtures are trimmed pages of Vestas and
|
||||
SAP (table layout), Demant and Scania (tile layout), Teradyne (a shared site
|
||||
with a path prefix), and the job pages of Vestas (`addressCountry`),
|
||||
Systematic (`streetAddress`) and Novo Nordisk (no microdata)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
@@ -18,13 +20,28 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from pipeline import Tally # noqa: E402
|
||||
from app.jobs import pull # noqa: E402
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import successfactors # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
|
||||
class FakeConfig:
|
||||
sources: ClassVar[dict[str, object]] = {}
|
||||
searches: ClassVar[list[object]] = []
|
||||
|
||||
|
||||
def _pull(store: Store) -> dict[str, Any]:
|
||||
"""One pull of every board, batches and settle, as `test_sweep_strikes`."""
|
||||
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
||||
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
|
||||
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _read(name: str) -> str:
|
||||
return (FIXTURES / name).read_text(encoding="utf-8")
|
||||
|
||||
@@ -264,6 +281,70 @@ def test_a_site_that_ignores_startrow_does_not_loop(monkeypatch: pytest.MonkeyPa
|
||||
assert len(asked) == 2
|
||||
|
||||
|
||||
def _untotalled(ids: list[int]) -> str:
|
||||
"""A table page that states no total."""
|
||||
return _table_page(ids, 0).split("</span>", 1)[1]
|
||||
|
||||
|
||||
def test_the_repeated_page_hands_back_a_cursor_with_no_results() -> None:
|
||||
text = _untotalled([1_000_000_001, 1_000_000_002])
|
||||
_decoded, results, total, cursor = successfactors.page(
|
||||
lambda url, **kw: text.encode(), "careers.acme.com", [2, "1000000001"])
|
||||
assert (results, total, cursor) == ([], None, (2, "1000000001"))
|
||||
|
||||
|
||||
def test_a_site_that_ignores_startrow_and_states_no_total_closes_nothing(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""The repeated first page used to end the read with no cursor, which a
|
||||
site that states no total made complete, and the settle closed every
|
||||
role past the first page."""
|
||||
store = Store(tmp_path / "roledesk.db")
|
||||
store.add_subscription("ats", "successfactors:careers.acme.com", label="Acme",
|
||||
company="Acme", board="successfactors")
|
||||
ids = [1_000_000_000 + n for n in range(4)]
|
||||
site = {"ignores_startrow": False}
|
||||
|
||||
def fake_get(client, url, host, slug, **kw):
|
||||
start = 0 if site["ignores_startrow"] else int(dict(httpx.URL(url).params)["startrow"])
|
||||
return _untotalled(ids[start:start + 2]).encode()
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||
_pull(store)
|
||||
assert store.state["ats:complete:successfactors:careers.acme.com"] == "1"
|
||||
assert len(store.roles()) == 4
|
||||
|
||||
site["ignores_startrow"] = True
|
||||
result = _pull(store)
|
||||
assert store.state["ats:complete:successfactors:careers.acme.com"] == ""
|
||||
assert (result["incomplete"], result["closed"], len(store.roles())) == (1, 0, 4)
|
||||
|
||||
|
||||
def test_a_site_over_the_page_cap_is_swept_from_its_cursor_as_json_gives_it_back(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The cursor `(start row, first id)` is a tuple, kept in state as a
|
||||
list. The sweep is kept, and the next pull reads on from it."""
|
||||
asked = _site(monkeypatch, 5, 2)
|
||||
monkeypatch.setattr(ats, "MAX_PAGES", 2)
|
||||
target = "successfactors:careers.acme.com"
|
||||
state: dict[str, str] = {}
|
||||
|
||||
def pull_once() -> list[Any]:
|
||||
source = ats.ATSSource({"boards": [{"url": target, "label": "Acme", "params": {}}],
|
||||
"run": "r"})
|
||||
source.state = state
|
||||
return list(source.fetch())
|
||||
|
||||
assert len(pull_once()) == 4
|
||||
assert state[f"ats:complete:{target}"] == ""
|
||||
assert json.loads(state[f"ats:sweep:{target}"])["cursor"] == [4, "1000000002"]
|
||||
assert len(pull_once()) == 1
|
||||
assert (state[f"ats:complete:{target}"], state[f"ats:swept:{target}"]) == ("1", "1")
|
||||
assert json.loads(state[f"ats:ids:{target}"]) == [
|
||||
f"{target}:{1_000_000_000 + n}" for n in range(5)]
|
||||
assert [int(u.rsplit("=", 1)[1]) for u in asked] == [0, 2, 4]
|
||||
|
||||
|
||||
def test_a_board_with_no_roles_is_complete_at_zero(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_site(monkeypatch, 0, 10)
|
||||
assert ats._fetch_rows(object(), "successfactors:careers.acme.com") == ([], True) # type: ignore[arg-type]
|
||||
|
||||
@@ -251,6 +251,31 @@ def test_a_first_page_stating_no_total_is_read_to_a_page_that_adds_nothing(
|
||||
assert len(rows) == 30 and not complete
|
||||
|
||||
|
||||
def test_a_board_over_the_page_cap_is_swept_from_its_cursor_as_json_gives_it_back(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
"""The cursor `(offset, total)` is a tuple, kept in state as a list, and
|
||||
the total later pages state as 0 is carried in it. The sweep is kept,
|
||||
and the next pull reads on from it to the first page's total."""
|
||||
tenant = Tenant(45)
|
||||
monkeypatch.setattr(ats, "_get", tenant)
|
||||
monkeypatch.setattr(ats, "MAX_PAGES", 2)
|
||||
target = f"workday:{SLUG}"
|
||||
state: dict[str, str] = {}
|
||||
|
||||
def pull_once() -> list[Any]:
|
||||
source = ats.ATSSource({"boards": [{"url": target, "label": "Zendesk", "params": {}}],
|
||||
"run": "r"})
|
||||
source.state = state
|
||||
return list(source.fetch())
|
||||
|
||||
assert len(pull_once()) == 40
|
||||
assert json.loads(state[f"ats:sweep:{target}"])["cursor"] == [40, 45]
|
||||
assert len(pull_once()) == 5
|
||||
assert (state[f"ats:complete:{target}"], state[f"ats:swept:{target}"]) == ("1", "1")
|
||||
assert len(json.loads(state[f"ats:ids:{target}"])) == 45
|
||||
assert [c[2]["offset"] for c in tenant.calls] == [0, 20, 40]
|
||||
|
||||
|
||||
def test_a_page_that_is_not_json_fails_the_read(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
monkeypatch.setattr(ats, "_get", lambda *a, **k: b"<html>maintenance</html>")
|
||||
|
||||
@@ -215,6 +215,28 @@ def test_a_page_with_nothing_new_ends_the_read_incomplete() -> None:
|
||||
assert [index for index, _ in module.asked] == [0, 1]
|
||||
|
||||
|
||||
def test_a_page_with_nothing_new_is_a_stall_with_a_stated_total_too() -> None:
|
||||
"""A host that ignores the cursor and states a total used to be read on
|
||||
to the page cap; the second page, which adds nothing while the reader
|
||||
hands a cursor, ends it incomplete."""
|
||||
pages = [["1", "2"]] * 5
|
||||
module = _paged(pages, total=5)
|
||||
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
|
||||
assert (listing.ids, listing.complete) == (["1", "2"], False)
|
||||
assert [index for index, _ in module.asked] == [0, 1]
|
||||
|
||||
|
||||
def test_an_empty_last_page_ends_a_read_without_a_total_complete() -> None:
|
||||
"""`complete="end"`: the page past the last role is empty and hands no
|
||||
cursor. That is the end, not a stall."""
|
||||
pages = [["1"], ["2"], []]
|
||||
module = _paged(pages)
|
||||
assert module.READS.complete == "end"
|
||||
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
|
||||
assert (listing.ids, listing.complete) == (["1", "2"], True)
|
||||
assert [index for index, _ in module.asked] == [0, 1, 2]
|
||||
|
||||
|
||||
def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None:
|
||||
pages = [["1"], ["2"]]
|
||||
web = _pages_web(pages)
|
||||
@@ -260,6 +282,50 @@ def test_a_board_over_the_page_cap_is_swept_across_pulls() -> None:
|
||||
assert state["ats:swept:fake:acme"] == "1"
|
||||
|
||||
|
||||
def _tupled(pages: list[list[str]], total: int) -> Any:
|
||||
"""`_paged` with a cursor of Workday's kind, `(next page, total)`, which a
|
||||
sweep hands back from state as a list. `cursors` are the ones it was
|
||||
handed."""
|
||||
module = _paged(pages, total=total)
|
||||
module.cursors = []
|
||||
|
||||
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
||||
module.cursors.append(cursor)
|
||||
index, stated = cursor or (0, total)
|
||||
ids = pages[index]
|
||||
more = index + 1 < len(pages)
|
||||
return ids, [{"id": i} for i in ids], stated, (index + 1, stated) if more else None
|
||||
|
||||
module.page = page
|
||||
return module
|
||||
|
||||
|
||||
def test_a_board_whose_cursor_is_a_tuple_is_swept_across_pulls() -> None:
|
||||
"""A tuple comes back from JSON as a list. The sweep is kept all the
|
||||
same, the reader gets the list, and the second pull finishes it."""
|
||||
module = _tupled([["1"], ["2"], ["3"]], total=3)
|
||||
state: dict[str, str] = {}
|
||||
limits = _runner.Limits(max_pages=2, page_delay=0)
|
||||
first = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits)
|
||||
_runner.stamp(state, "ats", "fake:acme", "r", first, first.ids)
|
||||
assert (first.ids, first.complete) == (["1", "2"], False)
|
||||
assert json.loads(state["ats:sweep:fake:acme"])["cursor"] == [2, 3]
|
||||
|
||||
second = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits)
|
||||
_runner.stamp(state, "ats", "fake:acme", "r", second, second.ids)
|
||||
assert (second.ids, second.complete, second.swept) == (["1", "2", "3"], True, True)
|
||||
assert module.cursors == [None, (1, 3), [2, 3]]
|
||||
assert "ats:sweep:fake:acme" not in state
|
||||
|
||||
|
||||
def test_a_cursor_is_storable_when_json_gives_it_back_save_tuples_as_lists() -> None:
|
||||
assert _runner._storable(7) and _runner._storable("token") and _runner._storable(None)
|
||||
assert _runner._storable((40, 45)) and _runner._storable((4, "1431772533"))
|
||||
assert _runner._storable({"after": (1, [2, (3,)])})
|
||||
assert not _runner._storable({1: "a"}) # the key comes back as "1"
|
||||
assert not _runner._storable(object())
|
||||
|
||||
|
||||
def test_a_board_read_in_one_pull_is_not_swept() -> None:
|
||||
pages = [["1"], ["2"]]
|
||||
state: dict[str, str] = {}
|
||||
|
||||
Reference in new issue
Block a user