"""The `pull` task: batching, the batch job's upsert, and the settle's closure — with the `ats` plugin's own `fetch()` stubbed out, so nothing here touches the network.""" from __future__ import annotations import json import sys import threading import types from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime, timedelta from pathlib import Path from typing import ClassVar import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from desk.models import Item # noqa: E402 from pipeline import Cancelled, RunContext, Tally # noqa: E402 from app.models import Application # noqa: E402 from plugins import ats, readers # noqa: E402 from plugins.readers import Filters, Reads, _runner # noqa: E402 from plugins.ats import ATSSource # noqa: E402 from app import prep # noqa: E402 from app.jobs import cascade, pull # noqa: E402 from app.storage import Store # noqa: E402 class FakeConfig: sources: ClassVar[dict[str, object]] = {} searches: ClassVar[list[object]] = [] def _item(board: str, slug: str, job_id: str, company: str = "Acme") -> Item: item = Item( source_id=f"{board}:{slug}:{job_id}", location=f"https://x.example/{job_id}", title="Engineer", body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw={"company": company, "place": "Berlin", "department": "", "function": "engineering", "remote": None, "salary_text": "", "board": board, "slug": slug}) item.source = "ats" return item @pytest.fixture def store(tmp_path: Path) -> Store: return Store(tmp_path / "roledesk.db") def test_boards_batches_fetchable_subscriptions_and_stamps_a_run(store: Store) -> None: for i in range(3): store.add_subscription("ats", f"greenhouse:c{i}", label=f"C{i}", company=f"C{i}", board="greenhouse") off = store.add_subscription("ats", "greenhouse:off", label="Off", company="Off", board="greenhouse") store.update_subscription(off.id, fetch=False) batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] assert sum(len(b["targets"]) for b in batches) == 3 run = store.state.get(pull.RUN, "") assert run and {b["run"] for b in batches} == {run} assert pull.running(store) == run def test_boards_is_empty_and_stamps_nothing_with_no_subscriptions(store: Store) -> None: assert pull._boards(FakeConfig(), store) == [] # type: ignore[arg-type] assert store.state.get(pull.RUN, "") == "" def test_run_batch_upserts_every_item_the_plugin_yields( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme", board="greenhouse") def fake_fetch(self: ATSSource) -> object: yield _item("greenhouse", "acme", "1") yield _item("greenhouse", "acme", "2") self.state["ats:ok:greenhouse:acme"] = self.options["run"] self.state["ats:ids:greenhouse:acme"] = '["1", "2"]' monkeypatch.setattr(ATSSource, "fetch", fake_fetch) result = pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme"], run="run-1") assert result == {"roles": 2, "run": "run-1"} assert store.state["ats:ok:greenhouse:acme"] == "run-1" assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:acme:2"} assert store.roles()[0].subscription_id is not None def test_a_kill_stops_the_batch_before_the_next_board( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: """`check()` before each board: a Kill pressed while the first is read ends the job before the second is fetched, and keeps what the first stored.""" for slug in ("acme", "beta"): store.add_subscription("ats", f"greenhouse:{slug}", label=slug.title(), company=slug.title(), board="greenhouse") fetched: list[str] = [] killed = False def fake_fetch(self: ATSSource) -> object: nonlocal killed [(target, _)] = self.targets() fetched.append(target) yield _item("greenhouse", target.partition(":")[2], "1") killed = True monkeypatch.setattr(ATSSource, "fetch", fake_fetch) context = RunContext(is_cancelled=lambda: killed) with pytest.raises(Cancelled): pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme", "greenhouse:beta"], context=context) assert fetched == ["greenhouse:acme"] assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"} def test_close_boards_closes_what_a_completed_board_no_longer_lists( store: Store) -> None: sub = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme", board="greenhouse") other = store.add_subscription("ats", "greenhouse:other", label="Other", company="Other", board="greenhouse") store.upsert_role(_item("greenhouse", "acme", "1"), sub) store.upsert_role(_item("greenhouse", "acme", "2"), sub) store.upsert_role(_item("greenhouse", "other", "9", company="Other"), other) # acme answered this run and now lists only "1"; other's board failed # (or its batch never got to it) and stamped nothing. store.state["ats:ok:greenhouse:acme"] = "run-1" store.state["ats:complete:greenhouse:acme"] = "1" store.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]' class FakeTally: results = ({"roles": 1, "run": "run-1"},) result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] assert result == {"boards": 1, "searches": 0, "unanswered": 1, "incomplete": 0, "roles": 1, "closed": 1} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"} def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run( store: Store) -> None: sub = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme", board="greenhouse") store.upsert_role(_item("greenhouse", "acme", "1"), sub) # A stale stamp from a run that is not this one — e.g. this run's batch # for this board never finished. store.state["ats:ok:greenhouse:acme"] = "run-1" store.state["ats:ids:greenhouse:acme"] = "[]" class FakeTally: results = ({"roles": 0, "run": "run-2"},) result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] assert result == {"boards": 0, "searches": 0, "unanswered": 1, "incomplete": 0, "roles": 0, "closed": 0} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"} def test_the_prep_after_the_settle_rebuilds_the_copies(store: Store) -> None: """The prep that follows the settle folds: a board's repeats of one job are one role, and when the kept one closes the next copy is kept in its place.""" sub = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme", board="greenhouse") for job_id in ("1", "2", "3"): store.upsert_role(_item("greenhouse", "acme", job_id), sub) store.state["ats:ok:greenhouse:acme"] = "run-1" store.state["ats:complete:greenhouse:acme"] = "1" store.state["ats:ids:greenhouse:acme"] = json.dumps( ["greenhouse:acme:1", "greenhouse:acme:2", "greenhouse:acme:3"]) class FakeTally: results = ({"roles": 3, "run": "run-1"},) pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] assert prep.run(store).copies == 2 [kept] = store.roles_page() assert kept.copies == 2 gone = store.role(kept.uid).source_id # type: ignore[union-attr] store.state["ats:ids:greenhouse:acme"] = json.dumps( [f"greenhouse:acme:{n}" for n in "123" if f"greenhouse:acme:{n}" != gone]) result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] assert (result["closed"], prep.run(store).copies) == (1, 1) [after] = store.roles_page() assert after.uid != kept.uid and after.copies == 1 # ---------------------------------------------- whole pulls, fetch stubbed def _row(job_id: str) -> dict[str, object]: return {"source_id": job_id, "title": f"Engineer {job_id}", "location": "Berlin", "url": f"https://x.example/{job_id}", "posted": None, "department": "", "description": "", "remote": None, "salary_text": ""} def _answer(monkeypatch: pytest.MonkeyPatch, answer: object, filtered: bool = False, swept: bool = False) -> None: """Every board's listing is `answer`, `(rows, complete)`, or raises it; with `filtered`, a read the board's filter narrowed, with `swept` one that finished a sweep across pulls.""" def fake(*args: object, **kwargs: object) -> _runner.Listing: if isinstance(answer, Exception): raise answer rows, complete = answer # type: ignore[misc] return _runner.Listing(rows, complete, [r["source_id"] for r in rows], filtered, swept=swept) monkeypatch.setattr(_runner, "read_listing", fake) def _pull(store: Store) -> dict[str, object]: results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type] return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] def _open(store: Store) -> set[str]: return {r.source_id for r in store.roles()} @pytest.fixture def acme(store: Store) -> Store: store.add_subscription("ats", "workable:acme", label="Acme", company="Acme", board="workable") return store def test_an_incomplete_board_stores_what_it_read_and_closes_nothing( acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: _answer(monkeypatch, ([_row("1"), _row("2")], True)) _pull(acme) # A 429 on page two: page one's role is stored again, the other stays open. _answer(monkeypatch, ([_row("1"), _row("3")], False)) result = _pull(acme) assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"} assert (result["boards"], result["incomplete"], result["closed"]) == (1, 1, 0) def _aged(store: Store, source_id: str) -> None: """Make a role look unlisted for longer than `STALE_DAYS`.""" old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(timespec="seconds") with store.transaction() as conn: conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = ?", (old, source_id)) def test_an_incomplete_read_closes_what_no_read_has_listed_for_stale_days( acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: """A board past its ceiling or page cap is never read complete; a role it has not listed for three weeks still closes (ADR 1.3).""" _answer(monkeypatch, ([_row("1"), _row("2"), _row("3")], True)) _pull(acme) _aged(acme, "workable:acme:3") _answer(monkeypatch, ([_row("1")], False)) result = _pull(acme) # 2 was listed two pulls ago and stays; 3 has been unlisted for 22 days. assert _open(acme) == {"workable:acme:1", "workable:acme:2"} assert (result["incomplete"], result["closed"]) == (1, 1) def test_a_filtered_read_closes_a_spared_role_once_it_is_stale( acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: """A filtered read spares a role acted on from its list; after three weeks unlisted the role closes by age like any other.""" _answer(monkeypatch, ([_row("1"), _row("2")], True)) _pull(acme) uid = {r.source_id: r.uid for r in acme.roles()} agent = acme.save_agent("Hunt") acme.set_vote(agent.id, uid["workable:acme:2"], 1) _answer(monkeypatch, ([_row("1")], True), filtered=True) assert _pull(acme)["closed"] == 0 assert _open(acme) == {"workable:acme:1", "workable:acme:2"} _aged(acme, "workable:acme:2") assert _pull(acme)["closed"] == 1 assert _open(acme) == {"workable:acme:1"} def test_a_failing_board_keeps_its_roles_before_stale_days( acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: _answer(monkeypatch, ([_row("1")], True)) _pull(acme) _answer(monkeypatch, ValueError("gone")) result = _pull(acme) assert _open(acme) == {"workable:acme:1"} assert (result["unanswered"], result["closed"]) == (1, 0) since, reason = pull.failing_boards(acme)["workable:acme"] assert since and reason == "ValueError: gone" def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens( acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: _answer(monkeypatch, ([_row("1"), _row("2")], True)) _pull(acme) _answer(monkeypatch, ValueError("404 for good")) _pull(acme) old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat( timespec="seconds") acme.state[f"{pull.FAILING}workable:acme"] = json.dumps( {"since": old, "reason": "404 Not Found"}) result = _pull(acme) assert _open(acme) == set() assert result["closed"] == 2 # Still failing, and still the date the failures began. assert pull.failing_boards(acme)["workable:acme"][0] == old _answer(monkeypatch, ([_row("2")], True)) _pull(acme) assert _open(acme) == {"workable:acme:2"} assert pull.failing_boards(acme) == {} def test_a_filtered_read_closes_what_it_left_out_except_roles_acted_on( acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: _answer(monkeypatch, ([_row(i) for i in "1234"], True)) _pull(acme) uid = {r.source_id: r.uid for r in acme.roles()} agent = acme.save_agent("Hunt") acme.set_vote(agent.id, uid["workable:acme:2"], 0) acme.save_application(Application(role_uid=uid["workable:acme:3"], role_title="Engineer")) _answer(monkeypatch, ([_row("1")], True), filtered=True) result = _pull(acme) assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"} assert result["closed"] == 1 assert acme.state["ats:filtered:workable:acme"] == "1" # Read without the filter again, the board closes as before. _answer(monkeypatch, ([_row("1")], True)) _pull(acme) assert _open(acme) == {"workable:acme:1"} def test_a_swept_read_closes_only_what_two_sweeps_in_a_row_missed( acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: _answer(monkeypatch, ([_row(i) for i in "123"], True)) _pull(acme) _answer(monkeypatch, ([_row("1")], True), swept=True) result = _pull(acme) assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"} assert result["closed"] == 0 assert json.loads(acme.state["ats:missing:workable:acme"]) == [ "workable:acme:2", "workable:acme:3"] # 3 came back, 2 did not: 2 has two strikes, 3 starts over. _answer(monkeypatch, ([_row("1"), _row("3")], True), swept=True) result = _pull(acme) assert _open(acme) == {"workable:acme:1", "workable:acme:3"} assert result["closed"] == 1 assert json.loads(acme.state["ats:missing:workable:acme"]) == [] def test_an_incomplete_read_between_two_sweeps_keeps_the_strike( acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: _answer(monkeypatch, ([_row(i) for i in "12"], True)) _pull(acme) _answer(monkeypatch, ([_row("1")], True), swept=True) _pull(acme) _answer(monkeypatch, ([_row("1")], False)) _pull(acme) _answer(monkeypatch, ValueError("down")) _pull(acme) assert json.loads(acme.state["ats:missing:workable:acme"]) == ["workable:acme:2"] _answer(monkeypatch, ([_row("1")], True), swept=True) _pull(acme) assert _open(acme) == {"workable:acme:1"} def test_a_filtered_swept_read_keeps_the_roles_acted_on( acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: _answer(monkeypatch, ([_row(i) for i in "1234"], True)) _pull(acme) uid = {r.source_id: r.uid for r in acme.roles()} agent = acme.save_agent("Hunt") acme.set_vote(agent.id, uid["workable:acme:2"], 0) acme.save_application(Application(role_uid=uid["workable:acme:3"], role_title="Engineer")) _answer(monkeypatch, ([_row("1")], True), filtered=True, swept=True) assert _pull(acme)["closed"] == 0 assert _open(acme) == {f"workable:acme:{i}" for i in "1234"} result = _pull(acme) assert result["closed"] == 1 assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"} assert json.loads(acme.state["ats:missing:workable:acme"]) == [ "workable:acme:2", "workable:acme:3"] def test_a_narrowed_swept_read_closes_nothing_and_keeps_the_record( acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: _answer(monkeypatch, ([_row(i) for i in "12"], True)) _pull(acme) _answer(monkeypatch, ([_row("1")], True), swept=True) _pull(acme) def fake(*args: object, **kwargs: object) -> _runner.Listing: return _runner.Listing([_row("1")], True, ["1"], swept=True, narrowed=True) monkeypatch.setattr(_runner, "read_listing", fake) assert _pull(acme)["closed"] == 0 assert json.loads(acme.state["ats:missing:workable:acme"]) == ["workable:acme:2"] def test_a_board_subscriptions_params_reach_the_runner_as_its_ask( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("ats", "workable:acme", label="Acme", company="Acme", board="workable", params={"countries": ["dk"], "remote": True}) asked: list[object] = [] def fake(module: object, slug: str, get: object, ask: object, **kwargs: object ) -> _runner.Listing: asked.append(ask) return _runner.Listing([_row("1")], True, ["1"], _runner.filtered(ask)) # type: ignore[arg-type] monkeypatch.setattr(_runner, "read_listing", fake) _pull(store) [ask] = asked declared = readers.HOSTS["workable"].READS.filters.declared() assert ask.countries == (("DK",) if "country" in declared else ()) # type: ignore[attr-defined] # ------------------------------------------------------ one pull at a time def _ago(**delta: float) -> str: return (datetime.now(UTC) - timedelta(**delta)).isoformat(timespec="microseconds") def test_a_pull_during_a_running_one_is_refused_and_queues_nothing(acme: Store) -> None: first = pull.claim_run(acme) assert first and pull.running(acme) == first asked: list[object] = [] said: list[object] = [] context = RunContext(on_task=lambda task: asked.append(task) or True, on_notice=said.append) result = cascade.run_pull(FakeConfig(), acme, context=context) assert result == {"skipped": pull.ALREADY_RUNNING, "running": first} assert asked == [] assert [n.code for n in said] == ["pull.running"] # type: ignore[attr-defined] # A chain started without the job's claim (a test, a script) finds # nothing to queue either, and the running run's marker stands. assert pull._boards(FakeConfig(), acme) == [] # type: ignore[arg-type] assert acme.state[pull.RUN] == first def test_a_pull_starts_once_the_last_one_settled(acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: _answer(monkeypatch, ([_row("1")], True)) _pull(acme) assert pull.running(acme) == "" asked: list[object] = [] context = RunContext(on_task=lambda task: asked.append(task) or True) result = cascade.run_pull(FakeConfig(), acme, context=context) assert result["run"] == acme.state[pull.RUN] and len(asked) == 1 # The chain's finder reads under the job's claim instead of claiming again. [batch] = asked[0].finder(FakeConfig(), acme) # type: ignore[attr-defined] assert batch["run"] == result["run"] def test_starters_claiming_at_once_give_one_run(tmp_path: Path) -> None: """Eight starters, each on its own connection to one file, as two processes would be, released together: one claim wins.""" Store(tmp_path / "shared.db") stores = [Store(tmp_path / "shared.db") for _ in range(8)] barrier = threading.Barrier(len(stores)) def claim(store: Store) -> str: barrier.wait() return pull.claim_run(store) with ThreadPoolExecutor(len(stores)) as pool: claims = list(pool.map(claim, stores)) assert sum(bool(c) for c in claims) == 1 def test_an_abandoned_run_is_taken_over_after_the_threshold(acme: Store) -> None: hours = pull.ABANDONED.total_seconds() / 3600 acme.state[pull.RUN] = _ago(hours=hours / 2) assert pull.claim_run(acme) == "" acme.state[pull.RUN] = _ago(hours=hours + 0.1) taken = pull.claim_run(acme) assert taken and acme.state[pull.RUN] == taken def test_a_batch_stamps_with_the_marker_it_was_handed( acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: _answer(monkeypatch, ([_row("1")], True)) [batch] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type] acme.state[pull.RUN] = "a later run" pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type] assert acme.state["ats:ok:workable:acme"] == batch["run"] def test_run_a_settle_does_not_close_on_run_b_ids( acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: """Run A read the board in full; run B, which took over, read it again and listed less before A's settle ran. A's settle compares with A's marker, so B's stamp and B's ids close nothing, and A's settle leaves B's claim held; B's own settle is the one that closes.""" _answer(monkeypatch, ([_row("1"), _row("2")], True)) [a] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type] a_result = pull.run_batch(FakeConfig(), acme, **a) # type: ignore[arg-type] acme.state[pull.RUN] = _ago(hours=2) # A looks abandoned _answer(monkeypatch, ([_row("1")], True)) [b] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type] assert b["run"] != a["run"] b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type] result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type] assert (result["unanswered"], result["closed"]) == (1, 0) assert _open(acme) == {"workable:acme:1", "workable:acme:2"} assert pull.running(acme) == b["run"] result = pull.close_boards(FakeConfig(), acme, Tally(results=(b_result,))) # type: ignore[arg-type] assert result["closed"] == 1 assert _open(acme) == {"workable:acme:1"} assert pull.running(acme) == "" def test_the_settle_reads_its_marker_off_its_batches_payloads(acme: Store) -> None: """A batch that raised has no result, but its payload still names the run.""" run = pull.claim_run(acme) acme.state["ats:ok:workable:acme"] = run acme.state["ats:complete:workable:acme"] = "1" acme.state["ats:ids:workable:acme"] = "[]" failed = type("Job", (), {"payload": {"targets": ["workable:acme"], "run": run}, "result": None, "state": "failed"})() context = RunContext(waited_for=(failed,)) result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type] assert result["boards"] == 1 and pull.running(acme) == "" # -------------------------------------------------------- losses, guarded def _days_ago(days: float) -> str: return (datetime.now(UTC) - timedelta(days=days)).isoformat(timespec="seconds") def _set(store: Store, column: str, value: str, *source_ids: str) -> None: with store.transaction() as conn: conn.executemany(f"UPDATE roles SET {column} = ? WHERE source_id = ?", [(value, i) for i in source_ids]) @pytest.fixture def big(store: Store) -> Store: store.add_subscription("ats", "greenhouse:big", label="Big", company="Big", board="greenhouse") return store def test_a_whole_read_of_nothing_after_forty_keeps_the_roles_and_fails( big: Store, monkeypatch: pytest.MonkeyPatch) -> None: _answer(monkeypatch, ([_row(str(n)) for n in range(40)], True)) _pull(big) _answer(monkeypatch, ([], True)) result = _pull(big) assert len(_open(big)) == 40 assert (result["unanswered"], result["closed"]) == (1, 0) assert "under 10% of the 40" in pull.failing_boards(big)["greenhouse:big"][1] def test_a_whole_read_of_nothing_after_five_closes_the_board( big: Store, monkeypatch: pytest.MonkeyPatch) -> None: _answer(monkeypatch, ([_row(str(n)) for n in range(5)], True)) _pull(big) _answer(monkeypatch, ([], True)) assert _pull(big)["closed"] == 5 assert _open(big) == set() def test_a_suspect_answer_is_accepted_after_stale_days( big: Store, monkeypatch: pytest.MonkeyPatch) -> None: _answer(monkeypatch, ([_row(str(n)) for n in range(40)], True)) _pull(big) _answer(monkeypatch, ([_row("1")], True)) _pull(big) assert len(_open(big)) == 40 record = json.loads(big.state[f"{pull.FAILING}greenhouse:big"]) big.state[f"{pull.FAILING}greenhouse:big"] = json.dumps( record | {"since": _days_ago(pull.STALE_DAYS + 1)}) result = _pull(big) assert _open(big) == {"greenhouse:big:1"} assert (result["unanswered"], result["closed"]) == (0, 39) assert pull.failing_boards(big) == {} assert json.loads(big.state["ats:count:greenhouse:big"])["count"] == 1 def _remote_host(monkeypatch: pytest.MonkeyPatch, remote: str) -> list[str]: """A whole-listing host `fake` that filters by country and, per `remote`, on remote; its feed lists the ids in the returned list.""" listed: list[str] = [] module: object = types.ModuleType("fake") module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () # type: ignore[attr-defined] module.slug_ok = lambda slug: slug.isalnum() # type: ignore[attr-defined] module.allowed = lambda slug: lambda h: h == "jobs.fake.example" # type: ignore[attr-defined] module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined] module.READS = Reads( # type: ignore[attr-defined] scope="board", listing="whole", text="listing", complete="whole", places="one", robots="test", filters=Filters(country="many", remote=remote)) # type: ignore[arg-type] module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined] module.parse = lambda body, slug: [_row(i) for i in json.loads(body)] # type: ignore[attr-defined] monkeypatch.setitem(readers.HOSTS, "fake", module) monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: json.dumps( listed).encode()) return listed def test_a_filtered_read_that_could_not_ask_remote_keeps_remote_roles_until_stale( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: listed = _remote_host(monkeypatch, remote="none") store.add_subscription("ats", "fake:acme", label="Acme", company="Acme", board="fake") reach = {"fake:acme": {"countries": ["DK"], "remote": True}} def pull_once() -> dict[str, object]: run = pull.claim_run(store) result = pull.run_batch(FakeConfig(), store, targets=["fake:acme"], run=run, # type: ignore[arg-type] reach=reach) return pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type] listed[:] = ["1", "2", "3", "4"] pull_once() _set(store, "remote", "remote", "fake:acme:3", "fake:acme:4") listed[:] = ["1"] assert pull_once()["closed"] == 1 assert _open(store) == {"fake:acme:1", "fake:acme:3", "fake:acme:4"} assert store.state["ats:remote_unasked:fake:acme"] == "1" _set(store, "last_seen", _days_ago(pull.STALE_DAYS + 1), "fake:acme:4") assert pull_once()["closed"] == 1 assert _open(store) == {"fake:acme:1", "fake:acme:3"} def test_a_narrowed_read_closes_what_it_has_not_listed_for_stale_days( acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: _answer(monkeypatch, ([_row(i) for i in "123"], True)) _pull(acme) def fake(*args: object, **kwargs: object) -> _runner.Listing: return _runner.Listing([_row("1")], True, ["1"], narrowed=True) monkeypatch.setattr(_runner, "read_listing", fake) assert _pull(acme)["closed"] == 0 _set(acme, "last_seen", _days_ago(pull.STALE_DAYS + 1), "workable:acme:3") assert _pull(acme)["closed"] == 1 assert _open(acme) == {"workable:acme:1", "workable:acme:2"} def test_a_target_that_raises_after_its_read_is_recorded_and_the_batch_goes_on( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: targets = [f"greenhouse:{slug}" for slug in ("a", "b", "c")] for target in targets: slug = target.partition(":")[2] store.add_subscription("ats", target, label=slug, company=slug, board="greenhouse") def fake_fetch(self: ATSSource) -> object: [(target, _)] = self.targets() yield _item("greenhouse", target.partition(":")[2], "1") self.state[f"ats:ok:{target}"] = self.options["run"] real = Store.upsert_role def upsert(self: Store, item: Item, sub: object = None) -> bool: if item.source_id.startswith("greenhouse:b:"): raise ValueError("a row the store refuses") return real(self, item, sub) # type: ignore[arg-type] monkeypatch.setattr(ATSSource, "fetch", fake_fetch) monkeypatch.setattr(Store, "upsert_role", upsert) result = pull.run_batch(FakeConfig(), store, targets=targets, run="r") # type: ignore[arg-type] assert result["roles"] == 2 assert _open(store) == {"greenhouse:a:1", "greenhouse:c:1"} assert "ats:ok:greenhouse:b" not in store.state since, reason = pull.failing_boards(store)["greenhouse:b"] assert since and reason == "ValueError: a row the store refuses" assert set(pull.failing_boards(store)) == {"greenhouse:b"} def test_a_board_whose_rows_do_not_store_keeps_its_failing_record( acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: since = _days_ago(2) acme.state[f"{pull.FAILING}workable:acme"] = json.dumps( {"since": since, "reason": "404 Not Found"}) _answer(monkeypatch, ([_row("1")], True)) def refuse(self: Store, item: Item, sub: object = None) -> bool: raise ValueError("no") monkeypatch.setattr(Store, "upsert_role", refuse) _pull(acme) assert pull.failing_boards(acme)["workable:acme"] == (since, "ValueError: no") assert "ats:ok:workable:acme" not in acme.state # ------------------------------------------------------- the settle speaks def test_a_settle_names_the_unanswered_board_and_the_one_that_closed_many( store: Store) -> None: sub = store.add_subscription("ats", "greenhouse:big", label="Big", company="Big", board="greenhouse") store.add_subscription("ats", "greenhouse:gone", label="Gone", company="Gone", board="greenhouse") for n in range(80): store.upsert_role(_item("greenhouse", "big", str(n), company="Big"), sub) run = pull.claim_run(store) store.state["ats:ok:greenhouse:big"] = run store.state["ats:complete:greenhouse:big"] = "1" store.state["ats:ids:greenhouse:big"] = json.dumps( [f"greenhouse:big:{n}" for n in range(20)]) store.state[f"{pull.FAILING}greenhouse:gone"] = json.dumps( {"since": _days_ago(1), "reason": "404 Not Found"}) said: list[object] = [] result = pull.close_boards(FakeConfig(), store, Tally(results=({"roles": 20, "run": run},)), # type: ignore[arg-type] context=RunContext(on_notice=said.append)) assert (result["unanswered"], result["closed"]) == (1, 60) [notice] = said assert notice.code == "pull.settled" # type: ignore[attr-defined] assert "greenhouse:gone (404 Not Found)" in notice.text # type: ignore[attr-defined] assert "greenhouse:big 60 of 80" in notice.text # type: ignore[attr-defined] assert "closed 60 roles" in notice.text # type: ignore[attr-defined] def test_a_clean_settle_says_nothing(acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: _answer(monkeypatch, ([_row(i) for i in "12"], True)) said: list[object] = [] results = [pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type] for batch in pull._boards(FakeConfig(), acme)] # type: ignore[arg-type] pull.close_boards(FakeConfig(), acme, Tally(results=tuple(results)), # type: ignore[arg-type] context=RunContext(on_notice=said.append)) assert said == [] def test_a_finder_that_raises_releases_the_pull_claim( acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: def broken(store: Store, subs: object) -> dict[str, object]: raise RuntimeError("reach broke") monkeypatch.setattr(pull, "reaches", broken) # The worker runs the chain's finder once the job is done; here at once. context = RunContext(on_task=lambda task: task.finder(FakeConfig(), acme)) with pytest.raises(RuntimeError, match="reach broke"): cascade.run_pull(FakeConfig(), acme, context=context) assert pull.running(acme) == "" # And the job itself lets go when the chain cannot be built. def no_chain(run: str) -> object: raise RuntimeError("no chain") monkeypatch.setattr(cascade, "pull_chain", no_chain) with pytest.raises(RuntimeError, match="no chain"): cascade.run_pull(FakeConfig(), acme, context=RunContext(on_task=lambda task: True)) assert pull.running(acme) == ""