"""The host modules of `plugins/readers/` and what `plugins/ats.py` does for every host: the registry and its contract check, the fetch a host is handed, the per-role detail reads the `enrich` stage makes, and Lever's EU instance. No network — a fake host stands in where a behaviour belongs to no real host yet.""" from __future__ import annotations import json import os import subprocess import sys import types from collections.abc import Callable from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any import httpx import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from plugins import ats, readers # noqa: E402 from plugins.readers import _runner # noqa: E402 from plugins.readers import lever # noqa: E402 from plugins.readers._contract import Ask, Filters, Reads # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" # --------------------------------------------------------------- registry def test_the_registry_finds_every_module() -> None: names = sorted(p.stem for p in (ROOT / "plugins" / "readers").glob("*.py") if not p.stem.startswith("_")) assert sorted(readers.READERS) == names assert set(readers.HOSTS) | set(readers.PROVIDERS) == set(names) assert set(names) >= {"greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee", "personio", "teamtailor"} assert all(not readers.problems(module, name) for name, module in readers.READERS.items()) # The Sources page's select and the plugin's description follow the modules. assert {name: module.LABEL for name, module in readers.HOSTS.items()} == ats.HOSTS assert "Teamtailor" in ats.ATSSource.description def _package(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str, modules: dict[str, str]) -> list[str]: package = tmp_path / name package.mkdir() (package / "__init__.py").write_text("") for module, source in modules.items(): (package / f"{module}.py").write_text(source) monkeypatch.syspath_prepend(str(tmp_path)) return [str(package)] _GOOD = """ NAME = "{name}" LABEL = "Good" PATTERNS = (r"good\\.example/([a-z]+)",) def slug_ok(slug): return True def allowed(slug): return lambda h: h == "good.example" def careers_url(slug): return "https://good.example/" + slug def feed_url(slug): return "https://good.example/" + slug + ".json" def parse(body, slug): return [] """ def test_the_registry_loads_ats_by_path_without_the_repo_root_on_the_path( tmp_path: Path) -> None: """As `desk.sources.registry` does in a process the app did not set up: another working directory, and `plugins` not importable at first.""" code = ( "import importlib.util, sys\n" "assert importlib.util.find_spec('plugins') is None, 'plugins is importable'\n" "from desk.sources import registry\n" f"print(sorted(registry.available([{str(ROOT / 'plugins')!r}])))\n") env = {k: v for k, v in os.environ.items() if k != "PYTHONPATH"} done = subprocess.run([sys.executable, "-c", code], cwd=tmp_path, env=env, capture_output=True, text=True, timeout=60, check=False) assert done.returncode == 0, done.stderr assert "'ats'" in done.stdout and "'search'" in done.stdout def test_the_registry_loads_a_package_by_name_and_skips_private_modules( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: path = _package(tmp_path, monkeypatch, "hostsok", { "zeta": _GOOD.format(name="zeta"), "alpha": _GOOD.format(name="alpha"), "_helpers": "X = 1\n"}) assert list(readers.load(path, "hostsok")) == ["alpha", "zeta"] def test_the_registry_orders_by_rank_then_name( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: path = _package(tmp_path, monkeypatch, "hostsranked", { "alpha": _GOOD.format(name="alpha"), # the default, 100 "beta": _GOOD.format(name="beta") + "RANK = 5\n", "gamma": _GOOD.format(name="gamma") + "RANK = 100\n"}) assert list(readers.load(path, "hostsranked")) == ["beta", "alpha", "gamma"] def test_the_eight_first_hosts_keep_the_order_they_had_before_modules() -> None: """The Sources page's select opens on Greenhouse, which its hint names, and a guessed slug on two hosts goes to the one probed first, as before.""" first = ["greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee", "personio", "teamtailor"] assert list(ats.HOSTS)[:8] == first assert ats.ATSSource.description.startswith( "Company job boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, " "Recruitee, Personio, Teamtailor") assert all(module.RANK < readers.DEFAULT_RANK for name, module in readers.HOSTS.items() if name in first) def test_the_registry_refuses_a_module_missing_a_required_name( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: broken = _GOOD.format(name="broken").replace("def careers_url", "def not_careers_url") path = _package(tmp_path, monkeypatch, "hostsbad", {"broken": broken}) with pytest.raises(ImportError, match=r"hostsbad\.broken .*no careers_url"): readers.load(path, "hostsbad") def test_the_contract_check_names_each_problem() -> None: module: Any = types.ModuleType("odd") module.NAME, module.LABEL, module.PATTERNS = "other", "", ("(",) module.slug_ok = module.allowed = module.careers_url = lambda slug: "" module.page = lambda get, slug, cursor: None module.probe_ok = lambda body: True module.RANK, module.KEEP_CASE = "1", "yes" found = readers.problems(module, "odd") assert "NAME is 'other', not the module's name 'odd'" in found assert "LABEL is not a non-empty string" in found assert any(p.startswith("PATTERNS has '('") for p in found) assert "page without rows" in found assert "probe_ok without probe_url" in found assert "RANK is not an int" in found and "KEEP_CASE is not a bool" in found @pytest.mark.parametrize("host", ["greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee", "personio", "teamtailor"]) def test_a_slug_is_matched_whole(host: str) -> None: assert ats.is_known_target(f"{host}:acme") for slug in ("acme\n", "acme\r\n", " acme", "acme ", "ac me", "acme/x", "acme?x", "acme#x"): assert not ats.is_known_target(f"{host}:{slug}"), slug def test_a_slug_in_the_host_name_is_allowed_whatever_its_case() -> None: for host, api in (("recruitee", "acme.recruitee.com"), ("personio", "acme.jobs.personio.de")): allow = ats._allowed_host(host, "Acme") assert allow(api) and allow(api.upper()) assert not allow(api.replace("acme", "other")) # ------------------------------------------------------------ the fetch def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client: return httpx.Client(transport=httpx.MockTransport(handler)) def test_get_sends_extra_headers_on_top_of_its_own() -> None: seen: list[httpx.Headers] = [] def handler(request: httpx.Request) -> httpx.Response: seen.append(request.headers) return httpx.Response(200, content=b"ok") with _client(handler) as client: get = ats._getter(client, "lever", "acme") assert get("https://api.lever.co/v0/postings/acme", headers={"Accept": "text/plain", "X-Board": "acme"}) == b"ok" get("https://api.lever.co/v0/postings/acme") assert seen[0]["accept"] == "text/plain" and seen[0]["x-board"] == "acme" assert seen[0]["user-agent"] == seen[1]["user-agent"] == ats.USER_AGENT assert seen[1]["accept"] == "application/json, text/xml" assert "x-board" not in seen[1] @pytest.mark.parametrize("site", [ "metadata.google.internal", "printer.local", "router.lan", "nas.home.arpa", "app.localhost", "foo.0x7f000001", "printer.local.", ]) def test_get_refuses_a_local_name_before_any_request(site: str) -> None: """A Teamtailor slug may be any host name, so the allow-list admits these; `_get` refuses them before a request is made.""" asked: list[str] = [] def handler(request: httpx.Request) -> httpx.Response: asked.append(str(request.url)) return httpx.Response(200, content=b"") with _client(handler) as client: # A trailing dot is no Teamtailor slug at all; the rest are, and are refused. with pytest.raises(ValueError if site.endswith(".") else ats._Refused): ats._fetch_rows(client, f"teamtailor:{site}") for url in (f"https://{site}/jobs.rss", "https://127.0.0.1/", "https://[::1]/", "https://2130706433/", "https://127.1/", "https://localhost/"): with pytest.raises(ats._Refused): ats._get(client, url, "teamtailor", site) assert asked == [] @pytest.mark.parametrize("location", [ "https://api.lever.co:8443/x", "https://127.0.0.1/x", "https://[::1]/x", "https://localhost/x", ]) def test_get_refuses_a_redirect_to_another_port_or_a_local_host(location: str) -> None: def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(302, headers={"location": location}) with _client(handler) as client, pytest.raises(ats._Refused): ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme") def test_get_takes_port_443_written_out_and_refuses_another() -> None: with _client(lambda request: httpx.Response(200, content=b"[]")) as client: assert ats._get(client, "https://api.lever.co:443/v0/postings/acme", "lever", "acme") == b"[]" with pytest.raises(ats._Refused): ats._get(client, "https://api.lever.co:8443/v0/postings/acme", "lever", "acme") def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatch) -> None: """Bound before the replacement, and still reaching it: `_get` is looked up when a request is made.""" client = object() get = ats._getter(client, "lever", "acme") # type: ignore[arg-type] calls: list[tuple[Any, ...]] = [] def replaced(*args: Any, **kwargs: Any) -> bytes: calls.append((args, kwargs)) return b"" monkeypatch.setattr(ats, "_get", replaced) get("https://api.lever.co/v0/postings/acme", headers={"X": "1"}) get("https://api.lever.co/v0/postings/acme", method="POST", payload={}) assert calls == [ ((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"), {"method": "GET", "headers": {"X": "1"}}), ((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"), {"method": "POST", "payload": {}})] # ------------------------------------------------------- a fake host def _fake_host(*, detail: Callable[..., Any] | None = None) -> types.ModuleType: """A host `fake` whose feed is a JSON list of ids, each role with a teaser for a description, and, with `detail`, a per-role page.""" module: Any = types.ModuleType("fake") module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () module.slug_ok = lambda slug: slug.isalnum() module.allowed = lambda slug: lambda h: h == "jobs.fake.example" module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" module.READS = Reads(scope="board", listing="whole", complete="whole", places="one", text="detail" if detail is not None else "listing", robots="test") module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}/list" module.parse = lambda body, slug: [ {"source_id": i, "title": f"Role {i}", "location": "", "department": "", "url": f"https://jobs.fake.example/{slug}/{i}", "posted": None, "description": "teaser", "remote": None, "salary_text": ""} for i in json.loads(body)] if detail is not None: module.detail = detail return module def _status(code: int, url: str = "https://jobs.fake.example/") -> httpx.HTTPStatusError: request = httpx.Request("GET", url) return httpx.HTTPStatusError(str(code), request=request, response=httpx.Response(code, request=request)) @pytest.fixture def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: """`ats._get` answering from a dict of URL -> body or exception, and remembering what it was asked. No delay between requests.""" answers: dict[str, Any] = {} asked: list[str] = [] def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None, www_twin=False): asked.append(url) if not ats._allowed_host(host, slug)(httpx.URL(url).host): raise ats._Refused(url) answer = answers.get(url, _status(404, url)) if isinstance(answer, BaseException): raise answer return answer monkeypatch.setattr(ats, "_get", fake_get) monkeypatch.setattr(ats, "PAGE_DELAY", 0) return {"answers": answers, "asked": asked} def _run(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType, state: dict[str, str] | None = None) -> tuple[list[Any], Any]: monkeypatch.setitem(readers.HOSTS, "fake", module) source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"}) if state is not None: source.state = state return list(source.fetch()), source.state def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]: """The fake host's detail: the role's own page is its description.""" return {"description": get(f"https://jobs.fake.example/{slug}/{row['source_id']}").decode(), "location": "Copenhagen"} # ---------------------------------------------------------------- detail def test_a_pull_reads_no_role_page( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: """The pages are `enrich`'s: the pull reads the listing only, and sends every row of a host with `detail` without a description.""" base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one in full"}) items, state = _run(monkeypatch, _fake_host(detail=_detail)) assert web["asked"] == [f"{base}/list"] assert [(i.source_id, i.body) for i in items] == [("fake:acme:1", ""), ("fake:acme:2", "")] assert "ats:detailed:fake:acme" not in state assert state["ats:complete:fake:acme"] == "1" def test_a_role_read_on_its_page_keeps_the_place_its_page_gave( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: """The listing states a place on every pull, and a filled value replaces a filled column, so for a role already read it is withheld.""" base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["1", "3"]'}) module = _fake_host(detail=_detail) listing = module.parse module.parse = lambda body, slug: [row | {"location": "2 Locations", "remote": "hybrid"} for row in listing(body, slug)] state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"]), "ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])} items, _state = _run(monkeypatch, module, state) by_id = {i.source_id: (i.body, i.raw["place"], i.raw["remote"]) for i in items} assert by_id["fake:acme:1"] == ("", "", None) # Never read: the listing's stand until its page is. assert by_id["fake:acme:3"] == ("", "2 Locations", "hybrid") def test_ids_no_longer_listed_leave_the_detail_lists( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["2"]'}) state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"]), "ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} _items, state = _run(monkeypatch, _fake_host(detail=_detail), state) assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"] assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"] def test_an_incomplete_listing_keeps_the_ids_it_did_not_list( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: """A listing cut short says nothing about the roles it left out, so their ids stay, and they are not read on their pages again.""" monkeypatch.setattr(_runner, "read_listing", lambda *args, **kwargs: _runner.Listing( readers.HOSTS["fake"].parse(b'["2"]', "acme"), False, ["2"])) state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} _items, state = _run(monkeypatch, _fake_host(detail=_detail), state) assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"] assert state["ats:complete:fake:acme"] == "" def test_a_pull_stopped_half_way_through_a_board_drops_no_detail_ids( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["2"]'}) monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail)) source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"}) source.state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])} roles: Any = source.fetch() # a generator, which `close` stops assert next(roles).body == "" roles.close() assert json.loads(source.state["ats:detailed:fake:acme"]) == ["fake:acme:1"] def test_fetch_board_reads_the_listing_only( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": b"one"}) monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail)) rows, complete = ats.fetch_board("fake:acme") assert ([r["description"] for r in rows], complete) == (["teaser"], True) assert web["asked"] == [f"{base}/list"] def test_a_host_without_detail_keeps_the_listings_description( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]' items, state = _run(monkeypatch, _fake_host()) assert [i.body for i in items] == ["teaser"] assert "ats:detailed:fake:acme" not in state # -------------------------------------------------- read_board_details NOW = datetime(2026, 9, 30, 12, tzinfo=UTC) def _details(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType, state: dict[str, str], ids: list[str], **kwargs: Any) -> list[dict[str, Any]]: """`ats.read_board_details` over the fake host's roles `ids`, stored without text, in that order.""" monkeypatch.setitem(readers.HOSTS, "fake", module) rows = [row | {"description": ""} for row in module.parse(json.dumps(ids).encode(), "acme")] return ats.read_board_details(state, "fake:acme", rows, client=object(), # type: ignore[arg-type] now=kwargs.pop("now", NOW), **kwargs) def test_details_are_read_in_the_order_given_for_unread_roles_only( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/1": b"one in full", f"{base}/2": b"two in full", f"{base}/3": b"three in full"}) state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]), "ats:detailempty:fake:acme": json.dumps({"fake:acme:2": NOW.isoformat()})} filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["3", "2", "1"]) assert web["asked"] == [f"{base}/3", f"{base}/1"] assert [(r["source_id"], r["description"], r["location"]) for r in filled] == [ ("3", "three in full", "Copenhagen"), ("1", "one in full", "Copenhagen")] assert json.loads(state["ats:detailed:fake:acme"]) == [ "fake:acme:1", "fake:acme:2", "fake:acme:3"] def test_details_stop_at_the_cap_and_the_rest_wait( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" ids = [str(n) for n in range(5)] web["answers"].update({f"{base}/{i}": f"full {i}".encode() for i in ids}) module = _fake_host(detail=_detail) state: dict[str, str] = {} filled = _details(monkeypatch, module, state, ids, cap=2) assert [r["description"] for r in filled] == ["full 0", "full 1"] web["asked"].clear() filled = _details(monkeypatch, module, state, ids[2:], cap=2) assert web["asked"] == [f"{base}/2", f"{base}/3"] def test_a_detail_that_fails_skips_the_role_and_a_429_stops_the_board( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/1": b"one", f"{base}/2": ValueError("no such page shape"), f"{base}/3": _status(429), f"{base}/4": b"four"}) state: dict[str, str] = {} filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3", "4"]) assert web["asked"] == [f"{base}/1", f"{base}/2", f"{base}/3"] assert [r["source_id"] for r in filled] == ["1"] assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"] # A 429 is not the role's fault. assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"] @pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")]) def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None: """Roles never tried go before those that failed, within the cap.""" base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/1": failure, f"{base}/2": failure, f"{base}/3": b"three", f"{base}/4": b"four"}) module = _fake_host(detail=_detail) state: dict[str, str] = {} _details(monkeypatch, module, state, ["1", "2"]) assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"] web["asked"].clear() _details(monkeypatch, module, state, ["1", "2", "3", "4"], cap=2) assert web["asked"] == [f"{base}/3", f"{base}/4"] web["asked"].clear() web["answers"][f"{base}/1"] = b"one at last" filled = _details(monkeypatch, module, state, ["1", "2"], cap=2) assert [r["description"] for r in filled] == ["one at last"] assert json.loads(state["ats:detailed:fake:acme"]) == [ "fake:acme:1", "fake:acme:3", "fake:acme:4"] assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"] def test_a_detail_answering_404_or_410_counts_as_read( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/2": _status(410), f"{base}/3": b"three"}) # 1 answers 404 state: dict[str, str] = {} filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3"]) assert [r["source_id"] for r in filled] == ["3"] assert json.loads(state["ats:detailed:fake:acme"]) == [ "fake:acme:1", "fake:acme:2", "fake:acme:3"] assert json.loads(state["ats:detailfailed:fake:acme"]) == [] def test_a_read_without_text_is_read_again_after_some_days( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: """Finding 18: an Oracle requisition not yet published has an empty page today and a full one next week.""" base = "https://jobs.fake.example/acme" web["answers"][f"{base}/1"] = b"" module = _fake_host(detail=lambda get, slug, row: { "description": get(f"{base}/{row['source_id']}").decode()}) state: dict[str, str] = {} assert _details(monkeypatch, module, state, ["1"]) == [] assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"] assert json.loads(state["ats:detailempty:fake:acme"]) == { "fake:acme:1": NOW.isoformat(timespec="seconds")} web["asked"].clear() later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS - 1) assert _details(monkeypatch, module, state, ["1"], now=later) == [] assert web["asked"] == [] web["answers"][f"{base}/1"] = b"published now" later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS) [filled] = _details(monkeypatch, module, state, ["1"], now=later) assert filled["description"] == "published now" assert json.loads(state["ats:detailempty:fake:acme"]) == {} assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"] def test_a_role_read_before_the_dates_were_kept_is_read_again( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: web["answers"]["https://jobs.fake.example/acme/1"] = b"one" state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])} [filled] = _details(monkeypatch, _fake_host(detail=_detail), state, ["1"]) assert filled["description"] == "one" def test_detail_fills_only_the_fields_a_later_pull_keeps( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], caplog: pytest.LogCaptureFixture) -> None: """`salary_text`, `department` and `title` would be rewritten from the listing on the next pull, so detail does not fill them.""" def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]: return {"description": "in full", "remote": "hybrid", "currency": "EUR", "salary_min": 60000, "salary_max": 70000, "posted": datetime(2026, 9, 1, tzinfo=UTC), "salary_text": "EUR 60,000 to 70,000 per year", "department": "Sales", "title": "Another title"} caplog.set_level("DEBUG", logger=_runner.log.name) [row] = _details(monkeypatch, _fake_host(detail=detail), {}, ["1"]) item = ats.item_of(row, "fake:acme", "Acme") assert (item.body, item.title, item.published_at) == ( "in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC)) assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"], item.raw["salary_max"]) == ("hybrid", "EUR", 60000, 70000) assert (item.raw["salary_text"], item.raw["department"]) == ("", "") assert "department, salary_text, title left out" in caplog.text assert set(ats.DETAIL_FIELDS) == {"description", "location", "places", "teaser", "remote", "country", "employment", "currency", "salary_min", "salary_max", "posted"} # ----------------------------------------------------------------- Lever EU def test_a_lever_slug_may_name_the_eu_instance() -> None: assert ats.is_known_target("lever:acme") and ats.is_known_target("lever:eu/tomtom") assert not ats.is_known_target("lever:us/acme") assert not ats.is_known_target("lever:eu/") assert not ats.is_known_target("lever:eu/a/b") assert ats.careers_url("lever:acme") == "https://jobs.lever.co/acme" assert ats.careers_url("lever:eu/tomtom") == "https://jobs.eu.lever.co/tomtom" assert lever.feed_url("eu/tomtom", Ask()) == "https://api.eu.lever.co/v0/postings/tomtom?mode=json" eu = ats._allowed_host("lever", "eu/tomtom") assert eu("api.eu.lever.co") and not eu("api.lever.co") assert not ats._allowed_host("lever", "tomtom")("api.eu.lever.co") def test_a_lever_eu_board_is_read_from_the_eu_api(monkeypatch: pytest.MonkeyPatch) -> None: urls: list[str] = [] def fake_get(client, url, host, slug, *, method="GET"): urls.append(url) assert ats._allowed_host(host, slug)(httpx.URL(url).host) return (FIXTURES / "lever.json").read_bytes() monkeypatch.setattr(ats, "_get", fake_get) rows, complete = ats._fetch_rows(object(), "lever:eu/tomtom") # type: ignore[arg-type] assert urls == ["https://api.eu.lever.co/v0/postings/tomtom?mode=json"] assert rows and complete def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None: def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(302, headers={"location": "https://api.lever.co/v0/postings/x"}) with _client(handler) as client, pytest.raises(ats._Refused): ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x") # ------------------------------------------- the guard, asks, remote unasked def _listing_host(**reads: Any) -> Any: """A declared whole-listing board `fake` whose feed lists `module.ids`, whatever is asked; `reads` override its declaration.""" module: Any = types.ModuleType("fake") module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () module.slug_ok = lambda slug: slug.isalnum() module.allowed = lambda slug: lambda h: h == "jobs.fake.example" module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" module.READS = Reads(**({"scope": "board", "listing": "whole", "text": "listing", "complete": "whole", "places": "one", "robots": "test"} | reads)) module.ids = [] module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}/list" module.parse = lambda body, slug: [ {"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}", "places": [{"text": "Berlin", "remote": None}]} for i in json.loads(body)] return module def _listed(module: Any) -> Callable[..., bytes]: return lambda url, **kwargs: json.dumps(module.ids).encode() def _read_and_stamp(state: dict[str, str], module: Any, limits: _runner.Limits | None = None) -> _runner.Listing: listing = _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module), limits=limits) _runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids) return listing def test_an_unfiltered_whole_read_of_nothing_after_forty_is_suspect() -> None: """A Greenhouse-style board, trusted, unfiltered: a changed answer that parses to no rows is refused rather than read as a board that closed.""" module = _listing_host() state: dict[str, str] = {} module.ids = [str(n) for n in range(40)] _read_and_stamp(state, module) module.ids = [] with pytest.raises(_runner.Suspect, match="listed 0 roles, under 10% of the 40"): _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module)) def test_a_whole_read_under_suspect_min_is_not_held() -> None: module = _listing_host() state: dict[str, str] = {} module.ids = [str(n) for n in range(5)] _read_and_stamp(state, module) module.ids = [] assert _read_and_stamp(state, module).ids == [] assert json.loads(state["ats:count:fake:acme"])["count"] == 0 def test_the_same_suspect_answer_is_accepted_after_stale_days() -> None: module = _listing_host() state: dict[str, str] = {} module.ids = [str(n) for n in range(40)] _read_and_stamp(state, module) module.ids = ["1"] failing = "ats:failing:fake:acme" recent = (datetime.now(UTC) - timedelta(days=20)).isoformat(timespec="seconds") state[failing] = json.dumps({"since": recent, "reason": "Suspect: listed 1 roles"}) limits = _runner.Limits(page_delay=0, stale_days=21) with pytest.raises(_runner.Suspect): _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module), limits=limits) old = (datetime.now(UTC) - timedelta(days=22)).isoformat(timespec="seconds") state[failing] = json.dumps({"since": old, "reason": "Suspect: listed 1 roles"}) # Outside a pull (no `stale_days`) the guard never gives way. with pytest.raises(_runner.Suspect): _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module)) assert _read_and_stamp(state, module, limits).ids == ["1"] assert json.loads(state["ats:count:fake:acme"])["count"] == 1 def test_keep_size_resets_the_guards_count_unless_told_not_to() -> None: state = {"ats:count:fake:acme": json.dumps({"asks": "[]", "count": 40})} ats.keep_size(state, "fake:acme", 40, reset=False) assert "ats:count:fake:acme" in state ats.keep_size(state, "fake:acme", 3) assert "ats:count:fake:acme" not in state assert state["ats:size:fake:acme"] == "3" def test_a_filter_that_maps_to_no_host_id_asks_nothing_and_is_incomplete( monkeypatch: pytest.MonkeyPatch) -> None: module = _listing_host(filters=Filters(country="ids")) module.facets = lambda get, slug: {"country": {}} module.ids = ["1", "2"] state: dict[str, str] = {} listing = _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module), Ask(countries=("DK",))) assert (listing.rows, listing.ids, listing.complete, listing.filtered) == ( [], [], False, True) # Through the plugin: stamped, but not complete, so the settle closes nothing. monkeypatch.setitem(readers.HOSTS, "fake", module) monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: b'["1"]') source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme", "params": {"countries": ["DK"]}}], "run": "r"}) assert list(source.fetch()) == [] assert (source.state["ats:ok:fake:acme"], source.state["ats:complete:fake:acme"]) == ( "r", "") def test_a_filtered_read_that_cannot_ask_remote_says_so() -> None: reach = {"countries": ["DK"], "remote": True} blind = _listing_host(filters=Filters(country="many")) ask = _runner.ask_for(blind, None, reach) assert (ask.countries, ask.remote) == (("DK",), None) assert _runner.remote_wanted(blind, None, reach) listing = _runner.read_listing(blind, "acme", _listed(blind), ask, remote=_runner.remote_wanted(blind, None, reach)) assert listing.filtered and listing.remote_unasked state: dict[str, str] = {} _runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids) assert state["ats:remote_unasked:fake:acme"] == "1" # A host that filters on remote asks it as a series of its own. sees = _listing_host(filters=Filters(country="many", remote="many")) ask = _runner.ask_for(sees, None, reach) listing = _runner.read_listing(sees, "acme", _listed(sees), ask, remote=True) assert listing.filtered and not listing.remote_unasked # A params' remote of False asks nothing and wants nothing. assert not _runner.remote_wanted(blind, {"remote": False}, reach)