"""The reader contract, held for every module of `plugins/readers/`. Per module: it declares `READS`, the declaration matches its functions, and its rows keep what the contract says they keep. No network: fixtures stand in for the host. `UNDECLARED` lists the modules that do not declare `READS` yet; their tests are marked xfail (not strict), and the planner removes a name once the module declares. A module not in the list must pass. Fixtures, in `tests/fixtures/ats/` for boards and `tests/fixtures/search/` for searches, named after the module: - `-places.`: one listing answer with at least one role at two or more places. For `listing="whole"` it is the body `parse` gets; for `listing="paged"` it is what every request of the first `page` call returns, and `rows` reads that page. Required by the test of a module that declares `places="many"`, which is skipped until the fixture exists. - `-places-detail.`: optional, for a host whose places come from the role's own page: what `detail`'s requests return; its `places` replace the row's. - `-teaser.`: one listing answer with a teaser, read the same way. Required by the test of a module with `text="excerpt"` or `teaser=True`, skipped until it exists. - `.slug`: optional, the slug the fixtures are read with, one line; `acme` without it. """ from __future__ import annotations import sys import types from collections.abc import Callable from pathlib import Path from typing import Any import pydantic import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from plugins import readers # noqa: E402 from plugins.readers import Ask, Fields, Filters, Key, Reads # noqa: E402 from plugins.readers import _contract # noqa: E402 FIXTURE_DIRS = (ROOT / "tests" / "fixtures" / "ats", ROOT / "tests" / "fixtures" / "search") #: The modules without `READS` yet. Shrinks as D1 (a to l), D2 (o to w) and #: C (the search providers, which arrive declared) land. UNDECLARED = frozenset({ "ashby", "bamboohr", "deel", "eightfold", "gem", "greenhouse", "hibob", "hrmanager", "jibe", "jobylon", "lever", "oracle", "personio", "phenom", "pinpoint", "recruitee", "rippling", "smartrecruiters", "successfactors", "teamtailor", "workable", "workday", }) def _params() -> list[Any]: return [pytest.param(name, marks=pytest.mark.xfail(reason="no READS yet", strict=False)) if name in UNDECLARED else name for name in readers.READERS] def _declared(name: str) -> tuple[types.ModuleType, Reads]: module = readers.READERS[name] reads = getattr(module, "READS", None) assert isinstance(reads, Reads), f"{name} declares no READS" return module, reads def _fixture(name: str, kind: str) -> Path | None: for folder in FIXTURE_DIRS: found = sorted(folder.glob(f"{name}-{kind}.*")) if found: return found[0] return None def _slug(name: str) -> str: for folder in FIXTURE_DIRS: if (path := folder / f"{name}.slug").exists(): return path.read_text(encoding="utf-8").strip() return "acme" def _answer(body: bytes) -> Callable[..., bytes]: return lambda url, **kwargs: body def read_fixture(module: types.ModuleType, body: bytes, slug: str, detail_body: bytes | None = None) -> list[dict[str, Any]]: """The rows `module` makes of one listing answer, as the runner reads them: `parse` for a whole listing, the first `page` and its `rows` for a paged one, and, given `detail_body`, each row's `places` from `detail`.""" reads: Reads = module.READS if reads.listing == "whole": rows = list(module.parse(body, slug)) else: decoded, _results, _total, _cursor = module.page(_answer(body), slug, None, Ask()) rows = list(module.rows(decoded, slug)) if detail_body is not None: for row in rows: found = module.detail(_answer(detail_body), slug, row) if "places" in found: row["places"] = found["places"] return rows def place_problems(rows: list[dict[str, Any]], places: str) -> list[str]: """What rows get wrong about `places` under the declared trait.""" out = [] for row in rows: found = row.get("places") rid = row.get("source_id") if not isinstance(found, list): out.append(f"{rid}: places is not a list") continue for place in found: if not (isinstance(place, dict) and set(place) == {"text", "remote"} and isinstance(place["text"], str) and place["text"].strip() and place["remote"] in (True, False, None)): out.append(f"{rid}: {place!r} is not a Place") if places in ("one", "copies") and len(found) > 1: out.append(f'{rid}: {len(found)} places under places="{places}"') if places == "many" and not any(len(row.get("places") or ()) >= 2 for row in rows): out.append('no row has two places: a reader with places="many" keeps every place') return out # ------------------------------------------------------------ every module def test_the_undeclared_list_names_only_readers() -> None: assert set(readers.READERS) >= UNDECLARED def test_undeclared_lists_the_modules_without_reads() -> None: assert set(readers.undeclared()) == {name for name, module in readers.READERS.items() if not hasattr(module, "READS")} @pytest.mark.parametrize("name", _params()) def test_a_reader_declares_reads(name: str) -> None: _declared(name) @pytest.mark.parametrize("name", _params()) def test_a_declaration_matches_the_functions(name: str) -> None: module, _reads = _declared(name) assert readers.problems(module, name) == [] expected = "search" if module.READS.scope == "search" else "board" assert name in (readers.PROVIDERS if expected == "search" else readers.HOSTS) @pytest.mark.parametrize("name", _params()) def test_a_reader_keeps_every_place(name: str) -> None: module, reads = _declared(name) fixture = _fixture(name, "places") if fixture is None: if reads.places == "many": pytest.skip(f"no fixture {name}-places.* yet") pytest.skip(f'places="{reads.places}" and no fixture') detail = _fixture(name, "places-detail") rows = read_fixture(module, fixture.read_bytes(), _slug(name), detail.read_bytes() if detail else None) assert rows, f"{fixture.name} gave no rows" assert place_problems(rows, reads.places) == [] @pytest.mark.parametrize("name", _params()) def test_a_reader_keeps_the_teaser(name: str) -> None: module, reads = _declared(name) if not (reads.text == "excerpt" or reads.teaser): pytest.skip("the list carries no teaser") fixture = _fixture(name, "teaser") if fixture is None: pytest.skip(f"no fixture {name}-teaser.* yet") rows = read_fixture(module, fixture.read_bytes(), _slug(name)) assert any(isinstance(row.get("teaser"), str) and row["teaser"].strip() for row in rows) # ------------------------------------------------------------ the checks def _reader(**functions: Any) -> Any: module: Any = types.ModuleType("fake") module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", (r"fake\.example/([a-z]+)",) module.slug_ok = lambda slug: True module.allowed = lambda slug: lambda host: host == "fake.example" module.careers_url = lambda slug: "https://fake.example/" + slug for name, function in functions.items(): setattr(module, name, function) return module def _whole(**functions: Any) -> Any: return _reader(**{"feed_url": lambda slug, ask: "https://fake.example/" + slug, "parse": lambda body, slug: []} | functions) def _paged(**functions: Any) -> Any: return _reader(**{"page": lambda get, slug, cursor, ask: (None, [], None, None), "rows": lambda decoded, slug: []} | functions) _ROBOTS = "read: no robots.txt rule on /api/ (checked 2026-09-30)" def _reads(**traits: Any) -> Reads: base: dict[str, Any] = {"scope": "board", "listing": "whole", "text": "listing", "complete": "whole", "places": "one", "robots": _ROBOTS} return Reads(**(base | traits)) def test_a_good_declaration_has_no_problems() -> None: module = _whole() module.READS = _reads() assert readers.problems(module, "fake") == [] module = _paged(detail=lambda get, slug, row: {}, facets=lambda get, slug: {"country": {"DE": "1"}}) module.READS = _reads(listing="paged", page_size=20, ceiling=2000, text="detail", teaser=True, complete="stated", filters=Filters(country="ids"), places="many", fields=Fields(posted="listed", remote="stated"), empty="suspect", title="provisional") assert readers.problems(module, "fake") == [] def test_a_module_without_reads_is_checked_as_before() -> None: module = _reader(feed_url=lambda slug: "", parse=lambda body, slug: []) assert readers.problems(module, "fake") == [] assert readers.scope(module) == "board" assert readers.undeclared({"fake": module}) == ["fake"] @pytest.mark.parametrize(("module", "reads", "problem"), [ (_reader(feed_url=lambda slug, ask: "", parse=lambda b, s: []), {"listing": "paged", "complete": "stated"}, 'listing="paged" without page and rows'), (_paged(), {}, 'listing="whole" without feed_url and parse'), (_whole(), {"complete": "stated"}, 'complete="whole" goes with listing="whole"'), (_whole(), {"page_size": 20}, 'page_size and ceiling are for listing="paged"'), (_paged(), {"listing": "paged", "complete": "end", "page_size": 100, "ceiling": 50}, "ceiling is below page_size"), (_paged(fallback_url=lambda slug: ""), {"listing": "paged", "complete": "end"}, 'fallback_url without listing="whole"'), (_whole(), {"text": "detail"}, 'text="detail" without detail'), (_whole(detail=lambda g, s, r: {}), {}, 'detail with text="listing"'), (_whole(), {"teaser": True, "text": "excerpt"}, "teaser is for text=\"detail\""), (_whole(), {"title": "provisional"}, 'title="provisional" without detail'), (_whole(), {"filters": Filters(country="ids")}, "an ids filter (country) without facets"), (_whole(facets=lambda g, s: {}), {"filters": Filters(country="one")}, "facets without an ids filter"), (_whole(facets=lambda g, s: {}), {"filters": Filters(query="ids")}, 'query cannot be "ids"'), (_paged(), {"scope": "search", "listing": "paged", "complete": "end"}, 'scope="search" needs complete="never" or "stated"'), (_paged(), {"listing": "paged", "complete": "never"}, 'scope="board" cannot be complete="never"'), (_whole(), {"key": Key(name="Adzuna Key", fields=("app_id",))}, "key names no config entry"), (_whole(), {"key": Key(name="adzuna", fields=())}, "key has no fields"), (_whole(), {"key": Key(name="adzuna", fields=("app_id", "app_key"), env=("ADZUNA_APP_ID",))}, "key.env does not match key.fields"), (_whole(), {"robots": " "}, "robots is empty"), (_reader(feed_url=lambda slug: "", parse=lambda b, s: []), {}, "feed_url does not take (slug, ask)"), (_reader(page=lambda get, slug, cursor: None, rows=lambda d, s: []), {"listing": "paged", "complete": "end"}, "page does not take (get, slug, cursor, ask)"), ]) def test_the_declaration_check_names_each_problem( module: Any, reads: dict[str, Any], problem: str) -> None: module.READS = _reads(**reads) found = readers.problems(module, "fake") assert any(p.startswith(problem) for p in found), found def test_reads_that_is_not_a_reads_is_refused() -> None: module = _whole() module.READS = {"scope": "board"} assert "READS is not a Reads" in readers.problems(module, "fake") def test_a_search_needs_fewer_names_than_a_board() -> None: module: Any = types.ModuleType("fake") module.NAME, module.LABEL = "fake", "Fake" module.allowed = lambda slug: lambda host: True module.page = lambda get, slug, cursor, ask: (None, [], 0, None) module.rows = lambda decoded, slug: [] module.READS = _reads(scope="search", listing="paged", complete="never", filters=Filters(query="one"), key=Key(name="adzuna", fields=("app_id", "app_key"), env=("ADZUNA_APP_ID", "ADZUNA_APP_KEY"))) assert readers.problems(module, "fake") == [] assert readers.scope(module) == "search" del module.READS assert "no careers_url" in readers.problems(module, "fake") def test_a_misspelt_trait_fails_where_it_is_declared() -> None: with pytest.raises(pydantic.ValidationError): _reads(listing="pages") with pytest.raises(pydantic.ValidationError): Filters(country="some") with pytest.raises(pydantic.ValidationError): _reads(robots=None) with pytest.raises(pydantic.ValidationError): Reads(scope="board", listing="whole", text="listing", complete="whole", places="one") # type: ignore[call-arg] # robots is required with pytest.raises(pydantic.ValidationError): _reads(colour="blue") # extra keys are refused def test_a_declaration_is_frozen() -> None: reads = _reads() with pytest.raises(pydantic.ValidationError): reads.places = "many" # type: ignore[misc] def test_the_loader_refuses_a_module_whose_declaration_breaks_the_contract( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: package = tmp_path / "readersbad" package.mkdir() (package / "__init__.py").write_text("") (package / "odd.py").write_text( "from plugins.readers import Reads\n" "NAME = 'odd'\nLABEL = 'Odd'\nPATTERNS = ()\n" "def slug_ok(slug): return True\n" "def allowed(slug): return lambda h: True\n" "def careers_url(slug): return ''\n" "def feed_url(slug, ask): return ''\n" "def parse(body, slug): return []\n" "READS = Reads(scope='board', listing='whole', text='detail', complete='whole',\n" " places='many', robots='read (checked 2026-09-30)')\n") monkeypatch.syspath_prepend(str(tmp_path)) with pytest.raises(ImportError, match=r'readersbad\.odd .*text="detail" without detail'): readers.load([str(package)], "readersbad") def test_filters_list_what_is_filtered() -> None: assert Filters(country="one", query="many").declared() == {"country": "one", "query": "many"} # ------------------------------------------------------------ the fixture check def _fake_many(rows: list[dict[str, Any]]) -> Any: module = _reader(feed_url=lambda slug, ask: "", parse=lambda body, slug: rows) module.READS = _reads(places="many") return module def test_the_place_check_passes_a_role_at_two_places() -> None: rows = [{"source_id": "1", "places": [{"text": "Berlin", "remote": False}, {"text": "Remote (DE)", "remote": True}]}] assert place_problems(read_fixture(_fake_many(rows), b"", "acme"), "many") == [] def test_the_place_check_fails_a_reader_that_drops_or_joins_places() -> None: joined = [{"source_id": "1", "places": [{"text": "Berlin; Munich", "remote": None}]}] assert place_problems(joined, "many") == [ 'no row has two places: a reader with places="many" keeps every place'] assert place_problems([{"source_id": "2", "location": "Berlin"}], "one") == [ "2: places is not a list"] assert place_problems([{"source_id": "3", "places": [("Berlin", False)]}], "one") == [ "3: ('Berlin', False) is not a Place"] def test_the_fixture_reader_drives_a_paged_module_and_detail() -> None: seen: list[bytes] = [] def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any: seen.append(get("https://fake.example/list")) return ([{"id": "1"}], [{"id": "1"}], 1, None) module = _paged(page=page, rows=lambda decoded, slug: [{"source_id": d["id"], "places": []} for d in decoded], detail=lambda get, slug, row: {"places": [ {"text": get("https://fake.example/1").decode(), "remote": None}, {"text": "Munich", "remote": None}]}) module.READS = _reads(listing="paged", complete="stated", text="detail", places="many") rows = read_fixture(module, b"Berlin", "acme", detail_body=b"Berlin") assert seen == [b"Berlin"] assert place_problems(rows, "many") == [] def test_the_contract_types_are_what_the_package_exports() -> None: assert readers.Reads is _contract.Reads and readers.Row is _contract.Row assert _contract.Place(text="Berlin", remote=None) == {"text": "Berlin", "remote": None}