Merge sweeps that resume on a tuple cursor, a stalled read counted incomplete, searches closing by age only, and Lever and Oracle failing on a wrong-shaped answer (wp-rev-runner)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-10-05 09:56:07 +02:00
commit 62cf1c738c
11 files changed
+364 -47

No files matched your search

+26 -1
View File
@@ -1,15 +1,20 @@
"""The Lever host module's places, country and employment, against a small
saved board. No network. The pay and the plain row are in `test_ats.py`."""
saved board, and an answer that is no board. No network. The pay and the
plain row are in `test_ats.py`."""
from __future__ import annotations
import json
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import ats # noqa: E402
from plugins.readers import Ask, lever # noqa: E402
BOARD = (ROOT / "tests" / "fixtures" / "ats" / "lever-places.json").read_bytes()
@@ -33,6 +38,26 @@ def test_the_commitment_is_the_hosts_word() -> None:
assert several["employment"] == "Full-time" and one["employment"] == "Permanent"
def test_an_answer_that_is_not_a_list_raises_and_an_empty_list_is_no_roles() -> None:
with pytest.raises(ValueError, match="JSON dict, not a list"):
lever.parse(b'{"ok": false, "error": "Document not found"}', "acme")
assert lever.parse(b"[]", "acme") == []
def test_an_error_document_with_a_200_fails_the_read_and_stamps_nothing(
monkeypatch: pytest.MonkeyPatch) -> None:
"""It used to read as a complete board with no roles, which the settle
closed whole under ten roles, where the 10% guard does not hold."""
monkeypatch.setattr(ats, "_get", lambda *a, **kw: b'{"ok": false}')
source = ats.ATSSource({"boards": [{"url": "lever:acme", "label": "Acme", "params": {}}],
"run": "r"})
source.state = {}
assert list(source.fetch()) == []
assert "ats:ok:lever:acme" not in source.state
assert json.loads(source.state["ats:failing:lever:acme"])["reason"].startswith(
"ValueError: Lever's answer is a JSON dict")
def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None:
reads = lever.READS
assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many")
+22
View File
@@ -165,6 +165,28 @@ def test_an_empty_board_is_complete(monkeypatch: pytest.MonkeyPatch, no_delay: N
assert ats._fetch_rows(object(), f"oracle:{SLUG}") == ([], True) # type: ignore[arg-type]
@pytest.mark.parametrize("answer", [
{}, {"items": []}, {"items": {}}, {"items": [None]}, [], {"title": "Unauthorized"}])
def test_an_answer_without_the_search_wrapper_fails_the_read(
monkeypatch: pytest.MonkeyPatch, no_delay: None, answer: Any) -> None:
"""It used to read as an empty last page with no total: a complete
read of a board with no roles."""
monkeypatch.setattr(ats, "_get", lambda *a, **k: json.dumps(answer).encode())
with pytest.raises(ValueError, match="no search wrapper in items"):
ats._fetch_rows(object(), f"oracle:{SLUG}") # type: ignore[arg-type]
def test_a_board_answering_without_the_wrapper_is_recorded_failing(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
monkeypatch.setattr(ats, "_get", lambda *a, **k: b'{"title": "Unauthorized"}')
source = ats.ATSSource({"boards": [{"url": f"oracle:{SLUG}", "label": "Nokia",
"params": {}}], "run": "r"})
source.state = {}
assert list(source.fetch()) == []
assert f"ats:ok:oracle:{SLUG}" not in source.state
assert "no search wrapper" in json.loads(source.state[f"ats:failing:oracle:{SLUG}"])["reason"]
# ----------------------------------------------------------------- detail
def _detail_get(body: bytes, asked: list[str]) -> Any:
+83 -2
View File
@@ -1,15 +1,17 @@
"""The SuccessFactors host: `plugins/readers/successfactors.py` against saved
career-site pages. No network. The fixtures are trimmed pages of Vestas and
career-site pages, and against written ones through a sweep and a pull with
its settle. No network. The fixtures are trimmed pages of Vestas and
SAP (table layout), Demant and Scania (tile layout), Teradyne (a shared site
with a path prefix), and the job pages of Vestas (`addressCountry`),
Systematic (`streetAddress`) and Novo Nordisk (no microdata)."""
from __future__ import annotations
import json
import sys
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from typing import Any, ClassVar
import httpx
import pytest
@@ -18,13 +20,28 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from pipeline import Tally # noqa: E402
from app.jobs import pull # noqa: E402
from app.jobs.discover import boards_in # noqa: E402
from app.storage import Store # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import successfactors # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
class FakeConfig:
sources: ClassVar[dict[str, object]] = {}
searches: ClassVar[list[object]] = []
def _pull(store: Store) -> dict[str, Any]:
"""One pull of every board, batches and settle, as `test_sweep_strikes`."""
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
def _read(name: str) -> str:
return (FIXTURES / name).read_text(encoding="utf-8")
@@ -264,6 +281,70 @@ def test_a_site_that_ignores_startrow_does_not_loop(monkeypatch: pytest.MonkeyPa
assert len(asked) == 2
def _untotalled(ids: list[int]) -> str:
"""A table page that states no total."""
return _table_page(ids, 0).split("</span>", 1)[1]
def test_the_repeated_page_hands_back_a_cursor_with_no_results() -> None:
text = _untotalled([1_000_000_001, 1_000_000_002])
_decoded, results, total, cursor = successfactors.page(
lambda url, **kw: text.encode(), "careers.acme.com", [2, "1000000001"])
assert (results, total, cursor) == ([], None, (2, "1000000001"))
def test_a_site_that_ignores_startrow_and_states_no_total_closes_nothing(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""The repeated first page used to end the read with no cursor, which a
site that states no total made complete, and the settle closed every
role past the first page."""
store = Store(tmp_path / "roledesk.db")
store.add_subscription("ats", "successfactors:careers.acme.com", label="Acme",
company="Acme", board="successfactors")
ids = [1_000_000_000 + n for n in range(4)]
site = {"ignores_startrow": False}
def fake_get(client, url, host, slug, **kw):
start = 0 if site["ignores_startrow"] else int(dict(httpx.URL(url).params)["startrow"])
return _untotalled(ids[start:start + 2]).encode()
monkeypatch.setattr(ats, "_get", fake_get)
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
_pull(store)
assert store.state["ats:complete:successfactors:careers.acme.com"] == "1"
assert len(store.roles()) == 4
site["ignores_startrow"] = True
result = _pull(store)
assert store.state["ats:complete:successfactors:careers.acme.com"] == ""
assert (result["incomplete"], result["closed"], len(store.roles())) == (1, 0, 4)
def test_a_site_over_the_page_cap_is_swept_from_its_cursor_as_json_gives_it_back(
monkeypatch: pytest.MonkeyPatch) -> None:
"""The cursor `(start row, first id)` is a tuple, kept in state as a
list. The sweep is kept, and the next pull reads on from it."""
asked = _site(monkeypatch, 5, 2)
monkeypatch.setattr(ats, "MAX_PAGES", 2)
target = "successfactors:careers.acme.com"
state: dict[str, str] = {}
def pull_once() -> list[Any]:
source = ats.ATSSource({"boards": [{"url": target, "label": "Acme", "params": {}}],
"run": "r"})
source.state = state
return list(source.fetch())
assert len(pull_once()) == 4
assert state[f"ats:complete:{target}"] == ""
assert json.loads(state[f"ats:sweep:{target}"])["cursor"] == [4, "1000000002"]
assert len(pull_once()) == 1
assert (state[f"ats:complete:{target}"], state[f"ats:swept:{target}"]) == ("1", "1")
assert json.loads(state[f"ats:ids:{target}"]) == [
f"{target}:{1_000_000_000 + n}" for n in range(5)]
assert [int(u.rsplit("=", 1)[1]) for u in asked] == [0, 2, 4]
def test_a_board_with_no_roles_is_complete_at_zero(monkeypatch: pytest.MonkeyPatch) -> None:
_site(monkeypatch, 0, 10)
assert ats._fetch_rows(object(), "successfactors:careers.acme.com") == ([], True) # type: ignore[arg-type]
+25
View File
@@ -251,6 +251,31 @@ def test_a_first_page_stating_no_total_is_read_to_a_page_that_adds_nothing(
assert len(rows) == 30 and not complete
def test_a_board_over_the_page_cap_is_swept_from_its_cursor_as_json_gives_it_back(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
"""The cursor `(offset, total)` is a tuple, kept in state as a list, and
the total later pages state as 0 is carried in it. The sweep is kept,
and the next pull reads on from it to the first page's total."""
tenant = Tenant(45)
monkeypatch.setattr(ats, "_get", tenant)
monkeypatch.setattr(ats, "MAX_PAGES", 2)
target = f"workday:{SLUG}"
state: dict[str, str] = {}
def pull_once() -> list[Any]:
source = ats.ATSSource({"boards": [{"url": target, "label": "Zendesk", "params": {}}],
"run": "r"})
source.state = state
return list(source.fetch())
assert len(pull_once()) == 40
assert json.loads(state[f"ats:sweep:{target}"])["cursor"] == [40, 45]
assert len(pull_once()) == 5
assert (state[f"ats:complete:{target}"], state[f"ats:swept:{target}"]) == ("1", "1")
assert len(json.loads(state[f"ats:ids:{target}"])) == 45
assert [c[2]["offset"] for c in tenant.calls] == [0, 20, 40]
def test_a_page_that_is_not_json_fails_the_read(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
monkeypatch.setattr(ats, "_get", lambda *a, **k: b"<html>maintenance</html>")
+84
View File
@@ -215,6 +215,46 @@ def test_a_page_with_nothing_new_ends_the_read_incomplete() -> None:
assert [index for index, _ in module.asked] == [0, 1]
def test_an_empty_page_with_a_cursor_is_a_stall_with_a_stated_total_too() -> None:
"""A reader that sees a page come again hands it over empty with a
cursor, as SuccessFactors does. That ends the read incomplete, also when
the host states a total, where it used to run on to the page cap."""
module = _paged([["1", "2"]] * 5, total=5)
seen: list[Any] = []
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
seen.append(cursor)
return (["1", "2"], [{"id": "1"}, {"id": "2"}], 5, 1) if cursor is None \
else ([], [], 5, cursor)
module.page = page
listing = _runner.read_listing(module, "acme", Web(), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2"], False)
assert seen == [None, 1]
def test_a_page_of_ids_already_read_is_no_stall_with_a_stated_total() -> None:
"""What a resumed sweep starts on when the list grew at the top since:
the page at the saved cursor holds only roles the sweep has. The read
goes on to the roles behind them."""
pages = [["1", "2"], ["1", "2"], ["3"]]
module = _paged(pages, total=3)
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2", "3"], True)
assert [index for index, _ in module.asked] == [0, 1, 2]
def test_an_empty_last_page_ends_a_read_without_a_total_complete() -> None:
"""`complete="end"`: the page past the last role is empty and hands no
cursor. That is the end, not a stall."""
pages = [["1"], ["2"], []]
module = _paged(pages)
assert module.READS.complete == "end"
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2"], True)
assert [index for index, _ in module.asked] == [0, 1, 2]
def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None:
pages = [["1"], ["2"]]
web = _pages_web(pages)
@@ -260,6 +300,50 @@ def test_a_board_over_the_page_cap_is_swept_across_pulls() -> None:
assert state["ats:swept:fake:acme"] == "1"
def _tupled(pages: list[list[str]], total: int) -> Any:
"""`_paged` with a cursor of Workday's kind, `(next page, total)`, which a
sweep hands back from state as a list. `cursors` are the ones it was
handed."""
module = _paged(pages, total=total)
module.cursors = []
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
module.cursors.append(cursor)
index, stated = cursor or (0, total)
ids = pages[index]
more = index + 1 < len(pages)
return ids, [{"id": i} for i in ids], stated, (index + 1, stated) if more else None
module.page = page
return module
def test_a_board_whose_cursor_is_a_tuple_is_swept_across_pulls() -> None:
"""A tuple comes back from JSON as a list. The sweep is kept all the
same, the reader gets the list, and the second pull finishes it."""
module = _tupled([["1"], ["2"], ["3"]], total=3)
state: dict[str, str] = {}
limits = _runner.Limits(max_pages=2, page_delay=0)
first = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits)
_runner.stamp(state, "ats", "fake:acme", "r", first, first.ids)
assert (first.ids, first.complete) == (["1", "2"], False)
assert json.loads(state["ats:sweep:fake:acme"])["cursor"] == [2, 3]
second = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits)
_runner.stamp(state, "ats", "fake:acme", "r", second, second.ids)
assert (second.ids, second.complete, second.swept) == (["1", "2", "3"], True, True)
assert module.cursors == [None, (1, 3), [2, 3]]
assert "ats:sweep:fake:acme" not in state
def test_a_cursor_is_storable_when_json_gives_it_back_save_tuples_as_lists() -> None:
assert _runner._storable(7) and _runner._storable("token") and _runner._storable(None)
assert _runner._storable((40, 45)) and _runner._storable((4, "1431772533"))
assert _runner._storable({"after": (1, [2, (3,)])})
assert not _runner._storable({1: "a"}) # the key comes back as "1"
assert not _runner._storable(object())
def test_a_board_read_in_one_pull_is_not_swept() -> None:
pages = [["1"], ["2"]]
state: dict[str, str] = {}
+35 -2
View File
@@ -26,7 +26,7 @@ from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
from plugins.search import SearchSource # noqa: E402
from app.config import SearchSeed # noqa: E402
from app.jobs import pull # noqa: E402
from pipeline import Tally # noqa: E402
from pipeline import RunContext, Tally # noqa: E402
from app.models import country_of, remote_of # noqa: E402
from app.storage import Store # noqa: E402
from prepped import upsert # noqa: E402
@@ -291,8 +291,12 @@ def _pull(store: Store) -> dict[str, Any]:
return closed
def test_a_complete_search_closes_what_it_stopped_listing(
def test_a_complete_search_closes_nothing_by_its_list_only_by_age(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""A search is never a company's whole list: one that read as many
results as its provider stated is stamped complete, and a role it
stopped listing still stays open until no search has listed it for
`STALE_DAYS`."""
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"})
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
@@ -304,10 +308,39 @@ def test_a_complete_search_closes_what_it_stopped_listing(
_fake_provider(monkeypatch, ["h2"], complete=True)
result = _pull(store)
assert store.state["search:complete:fake:cph"] == "1"
assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"}
assert (result["searches"], result["closed"]) == (1, 0)
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat()
with store.transaction() as conn:
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,))
result = _pull(store)
assert {r.source_id for r in store.roles()} == {"fake:h2"}
assert (result["searches"], result["closed"]) == (1, 1)
def test_a_complete_search_that_lists_far_fewer_is_not_named_in_the_notice(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""Fifteen of twenty roles dropping out of a complete search closed
them, and named the search as closing many; now they stay open and the
settle has nothing to say."""
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"})
twenty = [f"h{n}" for n in range(20)]
_fake_provider(monkeypatch, twenty, complete=True)
_pull(store)
_fake_provider(monkeypatch, twenty[:5], complete=True)
said: list[object] = []
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
result = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results)), # type: ignore[arg-type]
context=RunContext(on_notice=said.append))
assert (result["searches"], result["unanswered"], result["closed"]) == (1, 0, 0)
assert len(store.roles()) == 20
assert said == []
def test_an_incomplete_search_closes_nothing_until_stale(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",