"""A board's size, set once: `ats.board_size` per kind of host, `set_board` and the Recheck (route and CLI) that keep it, the pull that sizes a board with none before it reads it, and the Companies line that names what a site's own ids leave out. No network: `ats._get` or the runner's listing is stubbed.""" from __future__ import annotations import json import sys from collections.abc import Iterator from datetime import UTC, datetime from pathlib import Path from typing import Any, ClassVar from urllib.parse import parse_qs, urlsplit import pytest from fastapi import FastAPI from fastapi.testclient import TestClient ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from pipeline import Tally # noqa: E402 from app.cli import main # noqa: E402 from app.config import Config, ServerConfig # noqa: E402 from app.jobs import pull, reach, sources # noqa: E402 from app.main import create_app # noqa: E402 from app.models import Gate # noqa: E402 from app.storage import Store # noqa: E402 from plugins import ats # noqa: E402 from plugins.readers import Ask, _runner, greenhouse, workday # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" BOSCH = "smartrecruiters:BoschGroup" TODAY = datetime.now(UTC).date().isoformat() class FakeConfig: sources: ClassVar[dict[str, object]] = {} searches: ClassVar[list[object]] = [] @pytest.fixture def store(tmp_path: Path) -> Store: return Store(tmp_path / "t.db") @pytest.fixture(autouse=True) def _no_pause(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(ats, "PAGE_DELAY", 0) def _query(url: str) -> dict[str, list[str]]: return parse_qs(urlsplit(url).query) def _posting(job_id: str, country: str = "de") -> dict[str, Any]: return {"id": job_id, "name": f"Engineer {job_id}", "location": {"city": "Berlin", "country": country}} class SmartRecruiters: """A SmartRecruiters board of `total` roles that answers each request with one page of three and the stated total, or, per country, the roles there. Remembers every URL asked.""" def __init__(self, total: int | None, *, down: bool = False) -> None: self.total, self.down = total, down self.asked: list[str] = [] def __call__(self, client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes: self.asked.append(url) if self.down: raise OSError("connection refused") query = _query(url) if int(query.get("offset", ["0"])[0]): return json.dumps({"content": []}).encode() country = query.get("country", [""])[0] content = [_posting(f"{country}{n}", country or "de") for n in range(3)] answer: dict[str, Any] = {"content": content} if self.total is not None: answer["totalFound"] = 3 if country else self.total return json.dumps(answer).encode() def _teamtailor(count: int) -> Any: """A Teamtailor site of `count` roles, served 50 a page by offset.""" asked: list[str] = [] def get(client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes: asked.append(url) offset = int(_query(url)["offset"][0]) items = "".join( f"Role {n}https://acme.teamtailor.com/jobs/{n}" f"{n}" for n in range(offset, min(offset + 50, count))) return f"{items}".encode() get.asked = asked # type: ignore[attr-defined] return get def _wolf(store: Store, countries: tuple[str, ...] = ("DK", "SE"), remote: bool = True) -> None: gates = [Gate(field="country", values=list(countries), mode="any")] if remote: gates.append(Gate(field="remote", values=["remote"], mode="any")) store.save_agent("Wolf", gates=gates) def _board(store: Store, target: str, company: str, size: int | None = None, params: dict[str, object] | None = None) -> None: store.add_subscription("ats", target, label=company, company=company, board=target.partition(":")[0], params=params or {}) store.add_company(company) if size is not None: store.state[f"ats:size:{target}"] = str(size) def _sub(store: Store, target: str) -> Any: return next(s for s in store.subscriptions(source="ats") if s.target == target) def _pull(store: Store) -> list[dict[str, Any]]: batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] results = [pull.run_batch(FakeConfig(), store, **batch) for batch in batches] pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] return batches # ------------------------------------------------------------- board_size def test_a_host_that_states_a_total_is_sized_by_page_one( monkeypatch: pytest.MonkeyPatch) -> None: board = SmartRecruiters(999) monkeypatch.setattr(ats, "_get", board) assert ats.board_size(BOSCH) == 999 # One unfiltered request: no country, no terms. [url] = board.asked assert "country" not in _query(url) and "q" not in _query(url) def test_a_host_that_states_no_total_is_read_to_the_end( monkeypatch: pytest.MonkeyPatch) -> None: get = _teamtailor(120) monkeypatch.setattr(ats, "_get", get) assert ats.board_size("teamtailor:acme") == 120 assert [_query(u)["offset"][0] for u in get.asked] == ["0", "50", "100", "120"] def test_a_stated_host_that_leaves_out_its_total_is_read_to_the_end( monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(ats, "_get", SmartRecruiters(None)) assert ats.board_size(BOSCH) == 3 def test_a_whole_host_is_sized_by_its_one_answer(monkeypatch: pytest.MonkeyPatch) -> None: body = (FIXTURES / "greenhouse.json").read_bytes() monkeypatch.setattr(ats, "_get", lambda *args, **kwargs: body) assert ats.board_size("greenhouse:acme") == len(greenhouse.parse(body, "acme")) def test_board_size_refuses_an_unknown_board_and_raises_on_no_answer( monkeypatch: pytest.MonkeyPatch) -> None: with pytest.raises(ValueError, match="host:slug"): ats.board_size("taleo:acme") monkeypatch.setattr(ats, "_get", SmartRecruiters(5, down=True)) with pytest.raises(OSError): ats.board_size(BOSCH) # -------------------------------------------------------------- set_board def test_set_board_keeps_the_size_and_the_day_without_reading_the_board_whole( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: board = SmartRecruiters(4830) monkeypatch.setattr(ats, "_get", board) sub, found, _closed = sources.set_board(store, "bosch", BOSCH, name="Bosch") assert found == 4830 and len(board.asked) == 1 assert store.state[f"ats:size:{BOSCH}"] == "4830" assert store.state[f"ats:sizechecked:{BOSCH}"] == TODAY assert reach.plan(store).why(sub)[1] == "agents" def test_the_seed_keeps_no_size_and_a_detached_board_forgets_it( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(ats, "_get", SmartRecruiters(4830, down=True)) sources.set_board(store, "bosch", BOSCH, name="Bosch", check=False) assert f"ats:size:{BOSCH}" not in store.state ats.keep_size(store.state, BOSCH, 4830) sources.detach_board(store, "bosch") assert f"ats:size:{BOSCH}" not in store.state assert f"ats:sizechecked:{BOSCH}" not in store.state # ------------------------------------------------------ the unsized reach def test_a_board_with_no_size_is_handed_the_agents_reach_marked_unsized(store: Store) -> None: _wolf(store) _board(store, BOSCH, "Bosch") _board(store, "workday:acme.wd3/Jobs", "Acme", params={"countries": ["DE"]}) _board(store, "smartrecruiters:tiny", "Tiny", size=12) plan = reach.plan(store) assert plan.for_subscription(_sub(store, BOSCH)) == { "countries": ["DK", "SE"], "remote": True, "unsized": True, "small": reach.SMALL} # A board with its own reach carries only the marks; a sized one none. assert plan.for_subscription(_sub(store, "workday:acme.wd3/Jobs")) == { "unsized": True, "small": reach.SMALL} assert plan.for_subscription(_sub(store, "smartrecruiters:tiny")) is None assert reach.line(plan, _sub(store, BOSCH)) == "size not known yet: sized on the next pull" def test_a_big_board_with_no_size_is_sized_then_read_with_the_reach( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: _wolf(store) _board(store, BOSCH, "Bosch") board = SmartRecruiters(4830) monkeypatch.setattr(ats, "_get", board) _pull(store) assert store.state[f"ats:size:{BOSCH}"] == "4830" assert store.state[f"ats:sizechecked:{BOSCH}"] == TODAY # The first request sizes the board; the read then asks for DK and SE. assert "country" not in _query(board.asked[0]) assert sorted(_query(u).get("country", [""])[0] for u in board.asked[1:]) == ["dk", "se"] assert store.state[f"ats:filtered:{BOSCH}"] == "1" # The next pull finds the size and does not ask for it again. board.asked.clear() _pull(store) assert all("country" in _query(u) for u in board.asked) def test_a_small_board_with_no_size_is_sized_then_read_whole( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: _wolf(store) _board(store, BOSCH, "Bosch") board = SmartRecruiters(3) monkeypatch.setattr(ats, "_get", board) _pull(store) assert store.state[f"ats:size:{BOSCH}"] == "3" assert all("country" not in _query(u) for u in board.asked) assert store.state[f"ats:filtered:{BOSCH}"] == "" def test_a_board_that_does_not_answer_its_sizing_fails_as_any_read( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: _wolf(store) _board(store, BOSCH, "Bosch") monkeypatch.setattr(ats, "_get", SmartRecruiters(4830, down=True)) _pull(store) assert f"ats:size:{BOSCH}" not in store.state assert "connection refused" in json.loads(store.state[f"ats:failing:{BOSCH}"])["reason"] def test_a_board_read_whole_anyway_is_sized_by_that_read_once_it_is_complete( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: _wolf(store) _board(store, "greenhouse:acme", "Acme") rows = [{"source_id": str(n), "title": f"Role {n}", "url": f"https://x.example/{n}", "places": [], "description": ""} for n in range(4)] answer = {"complete": False} asked: list[Ask] = [] def fake(module: object, slug: str, get: object, ask: Ask, **kwargs: object ) -> _runner.Listing: asked.append(ask) return _runner.Listing(rows, answer["complete"], [r["source_id"] for r in rows]) # type: ignore[misc] monkeypatch.setattr(_runner, "read_listing", fake) monkeypatch.setattr(ats, "board_size", lambda *a, **k: pytest.fail("sized apart")) _pull(store) # Greenhouse cannot filter: one read, and an incomplete one sizes nothing. assert asked == [Ask()] assert "ats:size:greenhouse:acme" not in store.state answer["complete"] = True _pull(store) assert store.state["ats:size:greenhouse:acme"] == "4" assert store.state["ats:sizechecked:greenhouse:acme"] == TODAY def test_a_pull_leaves_a_kept_size_alone( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: _wolf(store) _board(store, "greenhouse:acme", "Acme", size=250) def fake(module: object, slug: str, get: object, ask: Ask, **kwargs: object ) -> _runner.Listing: return _runner.Listing([], True, []) monkeypatch.setattr(_runner, "read_listing", fake) _pull(store) assert store.state["ats:size:greenhouse:acme"] == "250" # ---------------------------------------------------- the line and facets WORKDAY = "workday:acme.wd3/Jobs" @pytest.mark.parametrize("facets, line", [ (None, "follows agents: DK, SE, remote · all roles"), ({"country": {"DK": "c=dk", "SE": "c=se"}, "remote": {"remote": "r=1"}}, "follows agents: DK, SE, remote · all roles"), ({"country": {"DK": "c=dk"}}, "follows agents: DK · SE not on this site · no remote option · all roles"), ({"country": {"DK": "c=dk"}, "remote": {"remote": "r=1"}}, "follows agents: DK, remote · SE not on this site · all roles"), ({"country": {"DE": "c=de"}, "remote": {"remote": "r=1"}}, "follows agents: remote · DK, SE not on this site · all roles"), ({"country": {"DE": "c=de"}}, "site lists none of DK, SE, remote: nothing asked, not closing"), ], ids=["none kept", "all known", "one country, no remote", "one country and remote", "remote only", "none"]) def test_the_line_names_what_a_site_does_not_know( store: Store, facets: dict[str, dict[str, str]] | None, line: str) -> None: _wolf(store) _board(store, WORKDAY, "Acme", size=1000) if facets is not None: store.state[f"ats:facets:{WORKDAY}"] = json.dumps( {"checked": "2026-09-30T06:00:00+00:00", "facets": facets}) plan = reach.plan(store) sub = _sub(store, WORKDAY) assert reach.line(plan, sub) == line # What the line says is left to ask is what the runner asks. ask = _runner.ask_for(workday, sub.params, plan.for_subscription(sub)) assert (_runner.series(workday, ask, facets) == []) == line.endswith("nothing asked, not closing") def test_the_line_says_nothing_is_asked_in_the_plans_words(store: Store) -> None: _wolf(store, remote=False) _board(store, WORKDAY, "Acme", size=1000) store.state[f"ats:facets:{WORKDAY}"] = json.dumps( {"checked": "2026-09-30T06:00:00+00:00", "facets": {"country": {"DE": "c=de"}}}) assert reach.line(reach.plan(store), _sub(store, WORKDAY)) == \ "site lists none of DK, SE: nothing asked, not closing" def test_a_boards_own_reach_is_named_the_same_and_remote_alone_is_read_whole( store: Store) -> None: _wolf(store) _board(store, WORKDAY, "Acme", size=1000, params={"countries": ["DK", "NO"]}) _board(store, "workday:other.wd3/Jobs", "Other", size=1000, params={"remote": True}) for target in (WORKDAY, "workday:other.wd3/Jobs"): store.state[f"ats:facets:{target}"] = json.dumps( {"checked": "2026-09-30T06:00:00+00:00", "facets": {"country": {"DK": "c=dk"}}}) plan = reach.plan(store) assert reach.line(plan, _sub(store, WORKDAY)) == \ "set here: DK · NO not on this site · all roles" assert reach.line(plan, _sub(store, "workday:other.wd3/Jobs")) == \ "all places, no remote option on this site · all roles" # ------------------------------------------------------ Recheck and the CLI @pytest.fixture def app(tmp_path: Path) -> FastAPI: return create_app(Config(server=ServerConfig(host="127.0.0.1", port=0), database=tmp_path / "roledesk.db")) @pytest.fixture def client(app: FastAPI) -> Iterator[TestClient]: with TestClient(app) as c: yield c def test_recheck_sizes_the_board_again_and_queues_a_pull( app: FastAPI, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: store: Store = app.state.store _wolf(store) _board(store, BOSCH, "Bosch", size=120) store.state[f"ats:sizechecked:{BOSCH}"] = "2026-09-01" sub_id = _sub(store, BOSCH).id [row] = client.get("/api/companies").json() assert (row["size"], row["size_checked"], row["reach"]) == ( 120, "2026-09-01", "under 300 roles: read whole") monkeypatch.setattr(ats, "_get", SmartRecruiters(4830)) res = client.post(f"/api/sources/{sub_id}/size") assert res.status_code == 200, res.text assert res.json() == {"size": 4830, "size_checked": TODAY, "reach": "follows agents: DK, SE · remote not asked on SmartRecruiters · all roles"} assert app.state.queue.jobs.last("pull", states=("queued", "running", "done")) is not None [row] = client.get("/api/companies").json() assert (row["size"], row["size_checked"]) == (4830, TODAY) # A board that does not answer keeps the size it had. monkeypatch.setattr(ats, "_get", SmartRecruiters(5, down=True)) res = client.post(f"/api/sources/{sub_id}/size") assert res.status_code == 422 and "did not answer" in res.json()["detail"] assert store.state[f"ats:size:{BOSCH}"] == "4830" assert client.post("/api/sources/9999/size").status_code == 404 def _cli(tmp_path: Path, *argv: str) -> int: config = tmp_path / "config.yaml" if not config.exists(): config.write_text(f"database: {tmp_path / 't.db'}\n") return main(["-c", str(config), "companies", *argv]) def test_the_cli_sizes_one_board_or_every_board_without_a_size( store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: _wolf(store) _board(store, BOSCH, "Bosch", size=12) _board(store, "smartrecruiters:tiny", "Tiny") _board(store, "smartrecruiters:gone", "Gone") monkeypatch.setattr(ats, "_get", SmartRecruiters(4830)) assert _cli(tmp_path, "size", "bosch") == 0 assert json.loads(capsys.readouterr().out) == { "board": BOSCH, "size": 4830, "reach": "follows agents: DK, SE · remote not asked on SmartRecruiters · all roles"} def some(client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes: if slug == "gone": raise OSError("connection refused") return SmartRecruiters(40)(client, url, host, slug) monkeypatch.setattr(ats, "_get", some) assert _cli(tmp_path, "size", "--missing") == 1 out, err = capsys.readouterr() assert out.splitlines() == [ json.dumps({"board": "smartrecruiters:tiny", "size": 40, "reach": "under 300 roles: read whole"}), "sized 1 of 2 boards without a size"] assert "smartrecruiters:gone did not answer" in err assert store.state[f"ats:size:{BOSCH}"] == "4830" assert _cli(tmp_path, "size") == 2 assert _cli(tmp_path, "size", "nobody") == 1