From a91032b60f09d739f2fdc645562c5395de7f3274 Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 13:23:57 +0200 Subject: [PATCH] Close a role of a swept board only when two finished sweeps in a row missed it Co-Authored-By: Claude Opus 5.5 --- app/jobs/pull.py | 53 +++++++++- plugins/readers/_runner.py | 28 +++-- tests/test_pull.py | 81 ++++++++++++++- tests/test_runner.py | 22 ++++ tests/test_sweep_strikes.py | 200 ++++++++++++++++++++++++++++++++++++ 5 files changed, 373 insertions(+), 11 deletions(-) create mode 100644 tests/test_sweep_strikes.py diff --git a/app/jobs/pull.py b/app/jobs/pull.py index fd6b652..09368d2 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -24,6 +24,13 @@ filtered read, or one of a reader that declares `empty="suspect"`, that lists under a tenth of what the same read listed last time is recorded by the runner as a failure, not as a list to close against. +A board swept across pulls (`ats:swept:`, a complete read that finished +a sweep an earlier pull began) closes a role only when two finished sweeps in a +row missed it (`_close_swept`): offset paging drifts when roles close between +pulls, so one sweep can miss a role that is still open. The ids a finished +sweep missed are kept in `ats:missing:`, and a complete read that was +not swept clears them. + A read narrowed by the agents' fetch terms (`ats:narrowed:`) lists the roles matching a word, which says nothing of the roles it did not list, so it closes nothing; only an unfiltered or a place-filtered complete read @@ -282,7 +289,12 @@ def close_boards(config: Config, store: Store, tally: Tally, *, if store.state.get(f"{source}:narrowed:{sub.target}", ""): continue seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]")) - if store.state.get(f"{source}:filtered:{sub.target}", ""): + is_filtered = bool(store.state.get(f"{source}:filtered:{sub.target}", "")) + if store.state.get(f"{source}:swept:{sub.target}", ""): + closed += _close_swept(store, source, sub, seen_ids, is_filtered) + continue + store.state.pop(f"{source}:missing:{sub.target}", None) + if is_filtered: closed += _close_unlisted(store, source, seen_ids, sub.id) else: closed += store.mark_closed(source, seen_ids, sub.id) @@ -313,6 +325,45 @@ def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription (store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount +def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str], + filtered: bool) -> int: + """The close of a board whose complete read finished a sweep across + pulls: its open roles missing from `seen_ids` that also are in + `:missing:`, the ones the previous finished sweep missed. + The ones missing now are kept there for the next sweep. A role one sweep + missed may only have shifted back past the cursor when roles closed + between pulls, so one sweep is not evidence that it is gone. A filtered + read keeps its exceptions as `_close_unlisted` does: a role acted on + stays open, and stays in `missing`.""" + key = f"{source}:missing:{sub.target}" + try: + earlier = {str(i) for i in json.loads(store.state.get(key) or "[]")} + except (TypeError, ValueError): + earlier = set() + acted_on = """ AND uid NOT IN (SELECT uid FROM role_votes) + AND uid NOT IN (SELECT role_uid FROM applications + WHERE role_uid IS NOT NULL)""" if filtered else "" + with store.transaction() as conn: + missed = [row[0] for row in conn.execute( + """SELECT source_id FROM roles + WHERE source = ? AND subscription_id = ? AND closed_at IS NULL + AND source_id NOT IN (SELECT value FROM json_each(?))""", + (source, sub.id, json.dumps(list(seen_ids))))] + closing = [row[0] for row in conn.execute( + f"""SELECT source_id FROM roles + WHERE source = ? AND subscription_id = ? AND closed_at IS NULL + AND source_id IN (SELECT value FROM json_each(?)) + AND source_id IN (SELECT value FROM json_each(?)){acted_on}""", + (source, sub.id, json.dumps(sorted(earlier)), json.dumps(missed)))] + count = conn.execute( + """UPDATE roles SET closed_at = ? + WHERE source = ? AND subscription_id = ? AND closed_at IS NULL + AND source_id IN (SELECT value FROM json_each(?))""", + (store.now(), source, sub.id, json.dumps(closing))).rowcount + store.state[key] = json.dumps(sorted(set(missed) - set(closing))) + return count + + def failing_boards(store: Store) -> dict[str, tuple[str, str]]: """`target -> (since, reason)` for every `ats` board whose last fetch failed: when the failures began, and what the latest one said.""" diff --git a/plugins/readers/_runner.py b/plugins/readers/_runner.py index 74ef71f..a4272c6 100644 --- a/plugins/readers/_runner.py +++ b/plugins/readers/_runner.py @@ -23,6 +23,11 @@ State keys, under the Source's prefix and the target (`ats:sweep:`): count `{"asks", "count"}`, the roles the last whole read listed, for the 10% guard facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL` + swept "1" when the last stamped listing finished a sweep carried over + from an earlier pull (`Listing.swept`), else "". Offset paging + drifts when roles close between pulls, so `app.jobs.pull` closes + a role of such a board only when two finished sweeps in a row + missed it (it keeps the ids the last one missed in `missing`) """ from __future__ import annotations @@ -126,7 +131,10 @@ class Listing: state to keep for the next pull, None when no sweep is under way. `filtered` says the read asked the host for the roles in some places, `narrowed` for the roles matching words (`narrowed()`): the ids of - neither are the whole board.""" + neither are the whole board. `swept` says a complete listing finished a + sweep carried over from an earlier pull: `ids` were read across pulls, + and roles closing meanwhile may have shifted an offset-paged host's later + roles back past the cursor, so `ids` can miss a role that is still open.""" rows: list[Row] complete: bool @@ -135,6 +143,7 @@ class Listing: sweep: dict[str, Any] | None = None asks: str = "" narrowed: bool = False + swept: bool = False # --------------------------------------------------------------- modules @@ -521,8 +530,9 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns the `sweep` to keep, and the next pull, handed it back, continues from its series and cursor; the sweep is complete when it reaches the end, - and `ids` then hold every id it listed. A read at the reader's `ceiling` - is not swept: the host answers nothing past it.""" + and `ids` then hold every id it listed, and the listing is `swept`. A + read at the reader's `ceiling` is not swept: the host answers nothing + past it.""" limits = limits or Limits() ask = ask or Ask() asks = series(module, ask, facets) @@ -539,7 +549,8 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None never = reads is not None and reads.complete == "never" ceiling = reads.ceiling if reads is not None else None start, cursor, seen, earlier, whole = 0, None, set(), [], True - if sweep and sweep.get("asks") == key: + resumed = bool(sweep and sweep.get("asks") == key) + if sweep and resumed: start, cursor = int(sweep.get("series") or 0), sweep.get("cursor") seen, earlier = set(sweep.get("seen") or []), list(sweep.get("rows") or []) whole = bool(sweep.get("whole", True)) @@ -591,7 +602,8 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None listed = list(dict.fromkeys(earlier + list(rows))) if not stop: - return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words) + return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words, + resumed and whole) log.warning("readers: %s:%s: stopped after %d pages (%s); not complete", module.NAME, slug, pages, stop) at_ceiling = ceiling is not None and len(seen) >= ceiling @@ -674,10 +686,12 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str, """Record a read whose rows are stored: `ok` (the run's marker), `complete`, `ids` (the compound ids, what the pull closes against), `filtered`, `narrowed` (a read by terms, which the pull never closes - against), the sweep to continue or none, and the count for the 10% - guard once a read is whole.""" + against), whether it finished a sweep from earlier pulls (`swept`), the + sweep to continue or none, and the count for the 10% guard once a read + is whole.""" state[f"{prefix}:ok:{target}"] = run state[f"{prefix}:complete:{target}"] = "1" if listing.complete else "" + state[f"{prefix}:swept:{target}"] = "1" if listing.swept else "" state[f"{prefix}:ids:{target}"] = json.dumps(ids) state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else "" state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else "" diff --git a/tests/test_pull.py b/tests/test_pull.py index 2001dbb..6a94718 100644 --- a/tests/test_pull.py +++ b/tests/test_pull.py @@ -206,14 +206,17 @@ def _row(job_id: str) -> dict[str, object]: "description": "", "remote": None, "salary_text": ""} -def _answer(monkeypatch: pytest.MonkeyPatch, answer: object, filtered: bool = False) -> None: +def _answer(monkeypatch: pytest.MonkeyPatch, answer: object, filtered: bool = False, + swept: bool = False) -> None: """Every board's listing is `answer`, `(rows, complete)`, or raises it; - with `filtered`, a read the board's filter narrowed.""" + with `filtered`, a read the board's filter narrowed, with `swept` one that + finished a sweep across pulls.""" def fake(*args: object, **kwargs: object) -> _runner.Listing: if isinstance(answer, Exception): raise answer rows, complete = answer # type: ignore[misc] - return _runner.Listing(rows, complete, [r["source_id"] for r in rows], filtered) + return _runner.Listing(rows, complete, [r["source_id"] for r in rows], filtered, + swept=swept) monkeypatch.setattr(_runner, "read_listing", fake) @@ -302,6 +305,78 @@ def test_a_filtered_read_closes_what_it_left_out_except_roles_acted_on( assert _open(acme) == {"workable:acme:1"} +def test_a_swept_read_closes_only_what_two_sweeps_in_a_row_missed( + acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _answer(monkeypatch, ([_row(i) for i in "123"], True)) + _pull(acme) + + _answer(monkeypatch, ([_row("1")], True), swept=True) + result = _pull(acme) + assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"} + assert result["closed"] == 0 + assert json.loads(acme.state["ats:missing:workable:acme"]) == [ + "workable:acme:2", "workable:acme:3"] + + # 3 came back, 2 did not: 2 has two strikes, 3 starts over. + _answer(monkeypatch, ([_row("1"), _row("3")], True), swept=True) + result = _pull(acme) + assert _open(acme) == {"workable:acme:1", "workable:acme:3"} + assert result["closed"] == 1 + assert json.loads(acme.state["ats:missing:workable:acme"]) == [] + + +def test_an_incomplete_read_between_two_sweeps_keeps_the_strike( + acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _answer(monkeypatch, ([_row(i) for i in "12"], True)) + _pull(acme) + _answer(monkeypatch, ([_row("1")], True), swept=True) + _pull(acme) + _answer(monkeypatch, ([_row("1")], False)) + _pull(acme) + _answer(monkeypatch, ValueError("down")) + _pull(acme) + assert json.loads(acme.state["ats:missing:workable:acme"]) == ["workable:acme:2"] + + _answer(monkeypatch, ([_row("1")], True), swept=True) + _pull(acme) + assert _open(acme) == {"workable:acme:1"} + + +def test_a_filtered_swept_read_keeps_the_roles_acted_on( + acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _answer(monkeypatch, ([_row(i) for i in "1234"], True)) + _pull(acme) + uid = {r.source_id: r.uid for r in acme.roles()} + agent = acme.save_agent("Hunt") + acme.set_vote(agent.id, uid["workable:acme:2"], 0) + acme.save_application(Application(role_uid=uid["workable:acme:3"], role_title="Engineer")) + + _answer(monkeypatch, ([_row("1")], True), filtered=True, swept=True) + assert _pull(acme)["closed"] == 0 + assert _open(acme) == {f"workable:acme:{i}" for i in "1234"} + + result = _pull(acme) + assert result["closed"] == 1 + assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"} + assert json.loads(acme.state["ats:missing:workable:acme"]) == [ + "workable:acme:2", "workable:acme:3"] + + +def test_a_narrowed_swept_read_closes_nothing_and_keeps_the_record( + acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _answer(monkeypatch, ([_row(i) for i in "12"], True)) + _pull(acme) + _answer(monkeypatch, ([_row("1")], True), swept=True) + _pull(acme) + + def fake(*args: object, **kwargs: object) -> _runner.Listing: + return _runner.Listing([_row("1")], True, ["1"], swept=True, narrowed=True) + + monkeypatch.setattr(_runner, "read_listing", fake) + assert _pull(acme)["closed"] == 0 + assert json.loads(acme.state["ats:missing:workable:acme"]) == ["workable:acme:2"] + + def test_a_board_subscriptions_params_reach_the_runner_as_its_ask( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("ats", "workable:acme", label="Acme", company="Acme", diff --git a/tests/test_runner.py b/tests/test_runner.py index 5a0ba43..69fd80a 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -269,6 +269,28 @@ def test_a_board_over_the_page_cap_is_swept_across_pulls() -> None: assert [index for index, _ in module.asked] == [0, 1, 2, 3, 4] assert "ats:sweep:fake:acme" not in state assert json.loads(state["ats:count:fake:acme"])["count"] == 5 + assert state["ats:swept:fake:acme"] == "1" + + +def test_a_board_read_in_one_pull_is_not_swept() -> None: + pages = [["1"], ["2"]] + state: dict[str, str] = {} + listing = _runner.read(state, "ats", "fake:acme", _paged(pages, total=2), "acme", + _pages_web(pages), limits=NO_DELAY) + _runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids) + assert (listing.complete, listing.swept, state["ats:swept:fake:acme"]) == (True, False, "") + + +def test_a_sweep_restarted_by_a_changed_ask_is_not_swept() -> None: + pages = [["1"], ["2"], ["3"]] + module = _paged(pages, filters=Filters(country="many")) + web, state = _pages_web(pages), {} + limits = _runner.Limits(max_pages=2, page_delay=0) + first = _runner.read(state, "ats", "t", module, "acme", web, Ask(), limits=limits) + _runner.stamp(state, "ats", "t", "r", first, first.ids) + again = _runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",)), + limits=NO_DELAY) + assert (again.complete, again.swept) == (True, False) def test_a_sweep_starts_again_when_the_ask_changed() -> None: diff --git a/tests/test_sweep_strikes.py b/tests/test_sweep_strikes.py new file mode 100644 index 0000000..ec7f13a --- /dev/null +++ b/tests/test_sweep_strikes.py @@ -0,0 +1,200 @@ +"""Two strikes for swept boards, end to end: a fake offset-paged host read +across pulls by the real `ATSSource`, run by the real `pull` batch and +settle. Roles closing on the host between pulls shift later roles back past +the saved cursor, so a finished sweep misses a role that is still open: it +closes only when two finished sweeps in a row missed it. No network.""" + +from __future__ import annotations + +import json +import sys +import types +from pathlib import Path +from typing import Any, ClassVar + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from pipeline import Tally # noqa: E402 +from app.jobs import pull # noqa: E402 +from app.storage import Store # noqa: E402 +from plugins import ats, readers # noqa: E402 +from plugins.readers import Reads, _runner # noqa: E402 + +PAGE = 2 +BOARD = "fake:acme" + + +class FakeConfig: + sources: ClassVar[dict[str, object]] = {} + searches: ClassVar[list[object]] = [] + + +class Host: + """An offset-paged board over `live`, two roles a page, stating its + current total.""" + + def __init__(self, live: list[str]) -> None: + self.live = live + + def module(self) -> Any: + module: Any = types.ModuleType("fake") + module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () + module.slug_ok = lambda slug: slug.isalnum() + module.allowed = lambda slug: lambda h: h == "jobs.fake.example" + module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" + module.READS = Reads(scope="board", listing="paged", text="listing", complete="stated", + places="many", robots="test", page_size=PAGE) + + def page(get: Any, slug: str, cursor: Any, ask: Any) -> Any: + offset = int(cursor or 0) + ids = self.live[offset:offset + PAGE] + more = offset + PAGE < len(self.live) + return ids, [{"id": i} for i in ids], len(self.live), offset + PAGE if more else None + + module.page = page + module.rows = lambda ids, slug: [ + {"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}", + "places": [{"text": "Berlin", "remote": None}]} for i in ids] + return module + + +@pytest.fixture +def host(monkeypatch: pytest.MonkeyPatch) -> Host: + fake = Host([str(i) for i in range(1, 7)]) + monkeypatch.setitem(readers.HOSTS, "fake", fake.module()) + monkeypatch.setattr(ats, "PAGE_DELAY", 0) + monkeypatch.setattr(ats, "MAX_PAGES", 1) + monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: b"") + return fake + + +@pytest.fixture +def store(tmp_path: Path, host: Host) -> Store: + found = Store(tmp_path / "roledesk.db") + found.add_subscription("ats", BOARD, label="Acme", company="Acme", board="fake") + return found + + +def _pull(store: Store) -> dict[str, Any]: + results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] + for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type] + return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] + + +def _open(store: Store) -> list[str]: + return sorted(r.source_id.rpartition(":")[2] for r in store.roles()) + + +def _missing(store: Store) -> list[str] | None: + kept = store.state.get(f"ats:missing:{BOARD}") + return None if kept is None else sorted(i.rpartition(":")[2] for i in json.loads(kept)) + + +def _sweep(store: Store) -> dict[str, Any]: + """Pull until a sweep finishes, three pulls of one page each.""" + result: dict[str, Any] = {} + for _ in range(3): + result = _pull(store) + if store.state.get(f"ats:complete:{BOARD}"): + break + assert store.state.get(f"ats:complete:{BOARD}") == "1" + return result + + +def _whole_read(store: Store, monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: + monkeypatch.setattr(ats, "MAX_PAGES", 10) + result = _pull(store) + monkeypatch.setattr(ats, "MAX_PAGES", 1) + return result + + +def test_a_role_one_sweep_missed_stays_open_and_two_close_it( + store: Store, host: Host, monkeypatch: pytest.MonkeyPatch) -> None: + _whole_read(store, monkeypatch) + assert _open(store) == list("123456") + assert store.state[f"ats:swept:{BOARD}"] == "" + + # Sweep one: role 1 closes on the host after page one, role 3 shifts back + # past the cursor and is not listed, though it is open. + _pull(store) + assert store.state[f"ats:complete:{BOARD}"] == "" + host.live.remove("1") + _sweep(store) + assert store.state[f"ats:swept:{BOARD}"] == "1" + assert json.loads(store.state[f"ats:ids:{BOARD}"]) == [ + f"{BOARD}:{i}" for i in "12456"] + assert _open(store) == list("123456") + assert _missing(store) == ["3"] + + # Sweep two lists 3 again and misses 1, which the host closed: one strike. + _sweep(store) + assert _open(store) == list("123456") + assert _missing(store) == ["1"] + + # Sweep three misses 1 again: the second strike closes it, only it. + result = _sweep(store) + assert _open(store) == list("23456") + assert result["closed"] == 1 + assert _missing(store) == [] + + +def test_a_role_that_comes_back_between_sweeps_loses_its_strike( + store: Store, host: Host, monkeypatch: pytest.MonkeyPatch) -> None: + _whole_read(store, monkeypatch) + host.live.remove("6") + _sweep(store) + assert _open(store) == list("123456") and _missing(store) == ["6"] + + host.live.append("6") + _sweep(store) + assert _open(store) == list("123456") and _missing(store) == [] + + +def test_a_complete_read_that_was_not_swept_clears_the_record( + store: Store, host: Host, monkeypatch: pytest.MonkeyPatch) -> None: + _whole_read(store, monkeypatch) + host.live.remove("6") + _sweep(store) + assert _missing(store) == ["6"] + + host.live.remove("5") + result = _whole_read(store, monkeypatch) + assert store.state[f"ats:swept:{BOARD}"] == "" + assert _missing(store) is None + # Not swept, so as before: one complete read closes what it did not list. + assert _open(store) == list("1234") and result["closed"] == 2 + + +def test_a_sweep_that_is_not_finished_leaves_the_record_alone( + store: Store, host: Host, monkeypatch: pytest.MonkeyPatch) -> None: + _whole_read(store, monkeypatch) + host.live.remove("6") + _sweep(store) + assert _missing(store) == ["6"] + _pull(store) + assert store.state[f"ats:complete:{BOARD}"] == "" + assert _missing(store) == ["6"] + assert _open(store) == list("123456") + + +def test_the_listing_says_swept_only_for_a_sweep_carried_over(host: Host) -> None: + module = host.module() + limits = _runner.Limits(max_pages=1, page_delay=0) + state: dict[str, str] = {} + first = _runner.read(state, "ats", BOARD, module, "acme", lambda url, **kw: b"", + limits=limits) + assert (first.complete, first.swept) == (False, False) + _runner.stamp(state, "ats", BOARD, "r", first, first.ids) + assert state[f"ats:swept:{BOARD}"] == "" + while not first.complete: + first = _runner.read(state, "ats", BOARD, module, "acme", lambda url, **kw: b"", + limits=limits) + _runner.stamp(state, "ats", BOARD, "r", first, first.ids) + assert first.swept and state[f"ats:swept:{BOARD}"] == "1" + whole = _runner.read({}, "ats", BOARD, module, "acme", lambda url, **kw: b"", + limits=_runner.Limits(max_pages=10, page_delay=0)) + assert (whole.complete, whole.swept) == (True, False)