"""Two strikes for swept boards, end to end: a fake offset-paged host read across pulls by the real `ATSSource`, run by the real `pull` batch and settle. Roles closing on the host between pulls shift later roles back past the saved cursor, so a finished sweep misses a role that is still open: it closes only when two finished sweeps in a row missed it. No network.""" from __future__ import annotations import json import sys import types from pathlib import Path from typing import Any, ClassVar import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from pipeline import Tally # noqa: E402 from app.jobs import pull # noqa: E402 from app.storage import Store # noqa: E402 from plugins import ats, readers # noqa: E402 from plugins.readers import Reads, _runner # noqa: E402 PAGE = 2 BOARD = "fake:acme" class FakeConfig: sources: ClassVar[dict[str, object]] = {} searches: ClassVar[list[object]] = [] class Host: """An offset-paged board over `live`, two roles a page, stating its current total.""" def __init__(self, live: list[str]) -> None: self.live = live def module(self) -> Any: module: Any = types.ModuleType("fake") module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () module.slug_ok = lambda slug: slug.isalnum() module.allowed = lambda slug: lambda h: h == "jobs.fake.example" module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" module.READS = Reads(scope="board", listing="paged", text="listing", complete="stated", places="many", robots="test", page_size=PAGE) def page(get: Any, slug: str, cursor: Any, ask: Any) -> Any: offset = int(cursor or 0) ids = self.live[offset:offset + PAGE] more = offset + PAGE < len(self.live) return ids, [{"id": i} for i in ids], len(self.live), offset + PAGE if more else None module.page = page module.rows = lambda ids, slug: [ {"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}", "places": [{"text": "Berlin", "remote": None}]} for i in ids] return module @pytest.fixture def host(monkeypatch: pytest.MonkeyPatch) -> Host: fake = Host([str(i) for i in range(1, 7)]) monkeypatch.setitem(readers.HOSTS, "fake", fake.module()) monkeypatch.setattr(ats, "PAGE_DELAY", 0) monkeypatch.setattr(ats, "MAX_PAGES", 1) monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: b"") return fake @pytest.fixture def store(tmp_path: Path, host: Host) -> Store: found = Store(tmp_path / "roledesk.db") found.add_subscription("ats", BOARD, label="Acme", company="Acme", board="fake") return found def _pull(store: Store) -> dict[str, Any]: results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type] return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] def _open(store: Store) -> list[str]: return sorted(r.source_id.rpartition(":")[2] for r in store.roles()) def _missing(store: Store) -> list[str] | None: kept = store.state.get(f"ats:missing:{BOARD}") return None if kept is None else sorted(i.rpartition(":")[2] for i in json.loads(kept)) def _sweep(store: Store) -> dict[str, Any]: """Pull until a sweep finishes, three pulls of one page each.""" result: dict[str, Any] = {} for _ in range(3): result = _pull(store) if store.state.get(f"ats:complete:{BOARD}"): break assert store.state.get(f"ats:complete:{BOARD}") == "1" return result def _whole_read(store: Store, monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: monkeypatch.setattr(ats, "MAX_PAGES", 10) result = _pull(store) monkeypatch.setattr(ats, "MAX_PAGES", 1) return result def test_a_role_one_sweep_missed_stays_open_and_two_close_it( store: Store, host: Host, monkeypatch: pytest.MonkeyPatch) -> None: _whole_read(store, monkeypatch) assert _open(store) == list("123456") assert store.state[f"ats:swept:{BOARD}"] == "" # Sweep one: role 1 closes on the host after page one, role 3 shifts back # past the cursor and is not listed, though it is open. _pull(store) assert store.state[f"ats:complete:{BOARD}"] == "" host.live.remove("1") _sweep(store) assert store.state[f"ats:swept:{BOARD}"] == "1" assert json.loads(store.state[f"ats:ids:{BOARD}"]) == [ f"{BOARD}:{i}" for i in "12456"] assert _open(store) == list("123456") assert _missing(store) == ["3"] # Sweep two lists 3 again and misses 1, which the host closed: one strike. _sweep(store) assert _open(store) == list("123456") assert _missing(store) == ["1"] # Sweep three misses 1 again: the second strike closes it, only it. result = _sweep(store) assert _open(store) == list("23456") assert result["closed"] == 1 assert _missing(store) == [] def test_a_role_that_comes_back_between_sweeps_loses_its_strike( store: Store, host: Host, monkeypatch: pytest.MonkeyPatch) -> None: _whole_read(store, monkeypatch) host.live.remove("6") _sweep(store) assert _open(store) == list("123456") and _missing(store) == ["6"] host.live.append("6") _sweep(store) assert _open(store) == list("123456") and _missing(store) == [] def test_a_complete_read_that_was_not_swept_clears_the_record( store: Store, host: Host, monkeypatch: pytest.MonkeyPatch) -> None: _whole_read(store, monkeypatch) host.live.remove("6") _sweep(store) assert _missing(store) == ["6"] host.live.remove("5") result = _whole_read(store, monkeypatch) assert store.state[f"ats:swept:{BOARD}"] == "" assert _missing(store) is None # Not swept, so as before: one complete read closes what it did not list. assert _open(store) == list("1234") and result["closed"] == 2 def test_a_sweep_that_is_not_finished_leaves_the_record_alone( store: Store, host: Host, monkeypatch: pytest.MonkeyPatch) -> None: _whole_read(store, monkeypatch) host.live.remove("6") _sweep(store) assert _missing(store) == ["6"] _pull(store) assert store.state[f"ats:complete:{BOARD}"] == "" assert _missing(store) == ["6"] assert _open(store) == list("123456") def test_the_listing_says_swept_only_for_a_sweep_carried_over(host: Host) -> None: module = host.module() limits = _runner.Limits(max_pages=1, page_delay=0) state: dict[str, str] = {} first = _runner.read(state, "ats", BOARD, module, "acme", lambda url, **kw: b"", limits=limits) assert (first.complete, first.swept) == (False, False) _runner.stamp(state, "ats", BOARD, "r", first, first.ids) assert state[f"ats:swept:{BOARD}"] == "" while not first.complete: first = _runner.read(state, "ats", BOARD, module, "acme", lambda url, **kw: b"", limits=limits) _runner.stamp(state, "ats", BOARD, "r", first, first.ids) assert first.swept and state[f"ats:swept:{BOARD}"] == "1" whole = _runner.read({}, "ats", BOARD, module, "acme", lambda url, **kw: b"", limits=_runner.Limits(max_pages=10, page_delay=0)) assert (whole.complete, whole.swept) == (True, False)