"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a subscription resolves to, series and their union, whole and paged listings, completeness from the declaration, the sweep across pulls, the 10% guard, the facets kept in state, the detail reads and the `raw` an item carries. No network: `get` is a dict of URL -> body or exception.""" from __future__ import annotations import json import sys import types from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any import httpx import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from plugins import ats, readers # noqa: E402 from plugins.readers import Ask, Filters, Key, Reads, _runner # noqa: E402 NO_DELAY = _runner.Limits(page_delay=0) def _status(code: int) -> httpx.HTTPStatusError: request = httpx.Request("GET", "https://jobs.fake.example/") return httpx.HTTPStatusError(str(code), request=request, response=httpx.Response(code, request=request)) class Web: """A `get` answering from a dict of URL -> body or exception.""" def __init__(self, answers: dict[str, Any] | None = None) -> None: self.answers = answers or {} self.asked: list[str] = [] def __call__(self, url: str, **kwargs: Any) -> bytes: self.asked.append(url) answer = self.answers.get(url, _status(404)) if isinstance(answer, BaseException): raise answer return answer def _row(i: str, **extra: Any) -> dict[str, Any]: return {"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}", "places": [{"text": "Berlin", "remote": None}]} | extra def _whole(**reads: Any) -> Any: """A declared whole-listing board: the feed is a JSON list of ids, one feed per `ask.countries` value.""" module: Any = types.ModuleType("fake") module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () module.slug_ok = lambda slug: slug.isalnum() module.allowed = lambda slug: lambda h: h == "jobs.fake.example" module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" module.READS = Reads(**({"scope": "board", "listing": "whole", "text": "listing", "complete": "whole", "places": "many", "robots": "test"} | reads)) module.feed_url = lambda slug, ask: (f"https://jobs.fake.example/{slug}/list" + "".join(f"?c={c}" for c in ask.countries)) module.parse = lambda body, slug: [_row(i) for i in json.loads(body)] return module def _paged(pages: list[list[str]], total: int | None = None, **reads: Any) -> Any: """A declared paged board over `pages`, the cursor being the page index.""" module = _whole(**({"listing": "paged", "complete": "stated" if total is not None else "end"} | reads)) del module.feed_url, module.parse module.asked = [] def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any: index = int(cursor or 0) module.asked.append((index, ask.countries)) body = get(f"https://jobs.fake.example/{slug}/page/{index}") ids = json.loads(body) if body else pages[index] more = index + 1 < len(pages) and bool(ids) return ids, [{"id": i} for i in ids], total, index + 1 if more else None module.page = page module.rows = lambda ids, slug: [_row(i) for i in ids] return module def _pages_web(pages: list[list[str]]) -> Web: return Web({f"https://jobs.fake.example/acme/page/{n}": json.dumps(ids).encode() for n, ids in enumerate(pages)}) # -------------------------------------------------------------------- ask def test_ask_for_narrows_the_params_to_the_declared_filters() -> None: board = _whole(filters=Filters(country="one")) ask = _runner.ask_for(board, {"countries": ["dk", "se"], "remote": True, "places": ["X"]}) assert (ask.countries, ask.remote, ask.places, ask.params) == (("DK", "SE"), None, (), {}) # The params override the reach per dimension; the reach fills the rest. both = _whole(filters=Filters(country="many", remote="many")) ask = _runner.ask_for(both, {"countries": ["DE"]}, {"countries": ["DK"], "remote": True}) assert (ask.countries, ask.remote) == (("DE",), True) # A module without READS reads the whole board. assert _runner.ask_for(types.ModuleType("old"), {"countries": ["DK"]}) == Ask() def test_a_searchs_params_go_whole_into_the_ask_with_its_key( monkeypatch: pytest.MonkeyPatch) -> None: module = _whole(scope="search", listing="paged", complete="stated", text="excerpt", filters=Filters(query="one", place="one"), key=Key(name="fake", fields=("id", "secret"), env=("FAKE_ID", "FAKE_SECRET"))) module.PARAMS = {"q": "query", "area": "place"} monkeypatch.delenv("FAKE_ID", raising=False) monkeypatch.setenv("FAKE_SECRET", "s") ask = _runner.ask_for(module, {"q": "python", "area": 1, "pages": 2}, keys={"fake": {"id": "i"}}) assert (ask.query, ask.places) == ("python", ("1",)) assert ask.params == {"q": "python", "area": 1, "pages": 2, "key": {"id": "i", "secret": "s"}} with pytest.raises(ValueError, match=r"sources\.search\.fake: id, secret, or FAKE_ID"): _runner.ask_for(module, {"q": "x"}, keys={}) def test_series_are_one_a_value_for_one_and_one_for_many() -> None: ask = Ask(countries=("DK", "SE"), places=("Aarhus",)) one = _whole(filters=Filters(country="one", place="many")) assert [(a.countries, a.places) for a in _runner.series(one, ask)] == [ (("DK",), ("Aarhus",)), (("SE",), ("Aarhus",))] many = _whole(filters=Filters(country="many")) assert _runner.series(many, ask) == [ask] def test_ids_filters_are_mapped_through_the_facets() -> None: module = _whole(filters=Filters(country="ids")) module.facets = lambda get, slug: {} ask = Ask(countries=("DK", "XX")) [mapped] = _runner.series(module, ask, {"country": {"DK": "h-1", "SE": "h-2"}}) assert mapped.countries == ("h-1",) # None known: the host has nothing in reach, and the read is empty. assert _runner.series(module, Ask(countries=("XX",)), {"country": {"DK": "h-1"}}) == [] listing = _runner.read_listing(module, "acme", Web(), Ask(countries=("XX",)), facets={"country": {}}) assert (listing.rows, listing.complete, listing.filtered) == ([], True, True) def test_facets_are_asked_once_a_day_and_only_for_an_ids_filter() -> None: module = _whole(filters=Filters(country="ids")) calls: list[str] = [] module.facets = lambda get, slug: calls.append(slug) or {"country": {"DK": "7"}} state: dict[str, str] = {} assert _runner.facets_for(module, "acme", Web(), Ask(), state, "k") == {} for _ in range(2): found = _runner.facets_for(module, "acme", Web(), Ask(countries=("DK",)), state, "k") assert found == {"country": {"DK": "7"}} assert calls == ["acme"] kept = json.loads(state["k"]) kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat() state["k"] = json.dumps(kept) _runner.facets_for(module, "acme", Web(), Ask(countries=("DK",)), state, "k") assert calls == ["acme", "acme"] # ---------------------------------------------------------------- listing def test_a_whole_listing_reads_one_feed_a_series_and_unions_by_id() -> None: module = _whole(filters=Filters(country="one")) web = Web({"https://jobs.fake.example/acme/list?c=DK": b'["1", "2"]', "https://jobs.fake.example/acme/list?c=SE": b'["2", "3"]'}) listing = _runner.read_listing(module, "acme", web, Ask(countries=("DK", "SE")), limits=NO_DELAY) assert [r["source_id"] for r in listing.rows] == ["1", "2", "3"] assert (listing.complete, listing.filtered, listing.ids) == (True, True, ["1", "2", "3"]) def test_a_body_over_the_cap_is_read_again_at_the_fallback_feed() -> None: """Greenhouse's shape: the feed with the text, and one without it.""" module = _whole() module.fallback_url = lambda slug: f"https://jobs.fake.example/{slug}/small" web = Web({"https://jobs.fake.example/acme/list": _runner.TooLarge("over"), "https://jobs.fake.example/acme/small": b'["1"]'}) listing = _runner.read_listing(module, "acme", web) assert ([r["source_id"] for r in listing.rows], listing.complete) == (["1"], True) # A reader without a fallback fails. del module.fallback_url with pytest.raises(_runner.TooLarge): _runner.read_listing(module, "acme", web) def test_a_module_without_reads_gets_the_older_signatures() -> None: module: Any = types.ModuleType("old") module.NAME = "old" module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list" module.parse = lambda body, slug: [_row(i) for i in json.loads(body)] listing = _runner.read_listing(module, "acme", Web({"https://jobs.fake.example/acme/list": b'["1"]'}), Ask(countries=("DK",))) assert [r["source_id"] for r in listing.rows] == ["1"] and listing.complete def test_a_paged_board_with_a_total_is_complete_when_it_read_as_many() -> None: pages = [["1", "2"], ["3"]] listing = _runner.read_listing(_paged(pages, total=3), "acme", _pages_web(pages), limits=NO_DELAY) assert (listing.ids, listing.complete) == (["1", "2", "3"], True) short = _runner.read_listing(_paged(pages, total=4), "acme", _pages_web(pages), limits=NO_DELAY) assert not short.complete def test_the_page_cap_leaves_a_board_incomplete() -> None: pages = [["1"], ["2"], ["3"]] listing = _runner.read_listing(_paged(pages), "acme", _pages_web(pages), limits=_runner.Limits(max_pages=2, page_delay=0)) assert (listing.ids, listing.complete, listing.sweep) == (["1", "2"], False, None) def test_a_page_with_nothing_new_ends_the_read_incomplete() -> None: """A host that ignores the cursor answers page one again; without a total the read would never end, so a page with nothing new stops it.""" pages = [["1", "2"], ["1", "2"], ["3"]] module = _paged(pages) listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY) assert (listing.ids, listing.complete) == (["1", "2"], False) assert [index for index, _ in module.asked] == [0, 1] def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None: pages = [["1"], ["2"]] web = _pages_web(pages) web.answers["https://jobs.fake.example/acme/page/1"] = _status(429) listing = _runner.read_listing(_paged(pages), "acme", web, limits=NO_DELAY) assert (listing.ids, listing.complete) == (["1"], False) web.answers["https://jobs.fake.example/acme/page/0"] = _status(429) with pytest.raises(httpx.HTTPStatusError): _runner.read_listing(_paged(pages), "acme", web, limits=NO_DELAY) def test_a_search_is_never_complete_when_it_says_so() -> None: pages = [["1"]] module = _paged(pages, scope="search", complete="never", text="excerpt") assert not _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY).complete def test_paged_series_each_read_to_the_end_and_union() -> None: pages = [["1", "2"]] module = _paged(pages, filters=Filters(country="one")) listing = _runner.read_listing(module, "acme", _pages_web(pages), Ask(countries=("DK", "SE")), limits=NO_DELAY) assert module.asked == [(0, ("DK",)), (0, ("SE",))] assert (listing.ids, listing.complete) == (["1", "2"], True) # ------------------------------------------------------------------ sweep def test_a_board_over_the_page_cap_is_swept_across_pulls() -> None: pages = [["1"], ["2"], ["3"], ["4"], ["5"]] module = _paged(pages, total=5) web, state = _pages_web(pages), {} limits = _runner.Limits(max_pages=2, page_delay=0) seen: list[tuple[list[str], bool]] = [] for _pull in range(3): listing = _runner.read(state, "ats", "fake:acme", module, "acme", web, limits=limits) _runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids) seen.append((listing.ids, listing.complete)) assert seen == [(["1", "2"], False), (["3", "4"], False), (["1", "2", "3", "4", "5"], True)] assert [index for index, _ in module.asked] == [0, 1, 2, 3, 4] assert "ats:sweep:fake:acme" not in state assert json.loads(state["ats:count:fake:acme"])["count"] == 5 def test_a_sweep_starts_again_when_the_ask_changed() -> None: pages = [["1"], ["2"], ["3"]] module = _paged(pages, filters=Filters(country="many")) web, state = _pages_web(pages), {} limits = _runner.Limits(max_pages=2, page_delay=0) first = _runner.read(state, "ats", "t", module, "acme", web, Ask(), limits=limits) _runner.stamp(state, "ats", "t", "r", first, first.ids) module.asked.clear() _runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",)), limits=limits) assert module.asked[0][0] == 0 def test_a_board_at_its_ceiling_is_not_swept() -> None: pages = [["1", "2"], ["3", "4"], ["5"]] module = _paged(pages, total=5, page_size=2, ceiling=4) listing = _runner.read({}, "ats", "t", module, "acme", _pages_web(pages), limits=_runner.Limits(max_pages=2, page_delay=0)) assert (listing.complete, listing.sweep) == (False, None) def test_a_search_is_not_swept() -> None: pages = [["1"], ["2"]] module = _paged(pages, total=2) listing = _runner.read_listing(module, "acme", _pages_web(pages), can_sweep=False, limits=_runner.Limits(max_pages=1, page_delay=0)) assert listing.sweep is None # ----------------------------------------------------------- the 10% guard def test_a_suspect_read_far_under_the_last_count_fails() -> None: module = _whole(empty="suspect") web = Web({"https://jobs.fake.example/acme/list": json.dumps( [str(n) for n in range(20)]).encode()}) state: dict[str, str] = {} listing = _runner.read(state, "ats", "t", module, "acme", web) _runner.stamp(state, "ats", "t", "r", listing, listing.ids) web.answers["https://jobs.fake.example/acme/list"] = b'["1"]' with pytest.raises(_runner.Suspect, match="under 10% of the 20"): _runner.read(state, "ats", "t", module, "acme", web) # A reader whose empty answer is trusted reads it as it is. module.READS = module.READS.model_copy(update={"empty": "trusted"}) assert _runner.read(state, "ats", "t", module, "acme", web).ids == ["1"] def test_the_guard_compares_only_reads_that_asked_the_same() -> None: """A board read whole last time and with a filter now lists far fewer roles; that is the filter, not a failure.""" module = _whole(filters=Filters(country="one")) web = Web({"https://jobs.fake.example/acme/list": json.dumps( [str(n) for n in range(50)]).encode(), "https://jobs.fake.example/acme/list?c=DK": b'["1"]'}) state: dict[str, str] = {} listing = _runner.read(state, "ats", "t", module, "acme", web) _runner.stamp(state, "ats", "t", "r", listing, listing.ids) filtered = _runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",))) assert filtered.ids == ["1"] and filtered.filtered # ------------------------------------------------------ through ATSSource @pytest.fixture def board(monkeypatch: pytest.MonkeyPatch) -> Any: pages = [["1"], ["2"], ["3"]] module = _paged(pages, total=3, empty="suspect") monkeypatch.setitem(readers.HOSTS, "fake", module) monkeypatch.setattr(ats, "PAGE_DELAY", 0) monkeypatch.setattr(ats, "MAX_PAGES", 2) web = _pages_web(pages) monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: web(url)) module.web = web return module def _source(state: dict[str, str], params: dict[str, Any] | None = None) -> ats.ATSSource: source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme", "params": params or {}}], "run": "r"}) source.state = state return source def test_ats_source_sweeps_and_stamps_every_id_once_the_sweep_ends(board: Any) -> None: state: dict[str, str] = {} items = list(_source(state).fetch()) assert [i.source_id for i in items] == ["fake:acme:1", "fake:acme:2"] assert state["ats:complete:fake:acme"] == "" assert json.loads(state["ats:sweep:fake:acme"])["cursor"] == 2 items = list(_source(state).fetch()) assert [i.source_id for i in items] == ["fake:acme:3"] assert state["ats:complete:fake:acme"] == "1" assert json.loads(state["ats:ids:fake:acme"]) == [ "fake:acme:1", "fake:acme:2", "fake:acme:3"] def test_ats_source_records_a_suspect_read_as_failing(board: Any) -> None: state = {"ats:count:fake:acme": json.dumps( {"asks": _runner.fingerprint([Ask()]), "count": 100})} board.web.answers = {"https://jobs.fake.example/acme/page/0": b"[]"} assert list(_source(state).fetch()) == [] assert "under 10%" in json.loads(state["ats:failing:fake:acme"])["reason"] assert "ats:ok:fake:acme" not in state def test_ats_source_hands_the_board_params_to_the_reader(board: Any) -> None: board.READS = board.READS.model_copy(update={"filters": Filters(country="many")}) state: dict[str, str] = {} list(_source(state, {"countries": ["dk"]}).fetch()) assert board.asked[0] == (0, ("DK",)) assert state["ats:filtered:fake:acme"] == "1" # -------------------------------------------------------------------- raw def test_raw_keeps_every_stated_key_and_the_places() -> None: row = _row("1", places=[{"text": "Berlin", "remote": False}, {"text": "Remote (DE)", "remote": True}], teaser="Short.", seniority="senior", employment="full_time", country="DE", description="Long.", posted=None, location="Berlin; Remote (DE)") raw = _runner.raw_of(row) assert raw["places"] == row["places"] assert raw["place"] == "Berlin; Remote (DE)" assert (raw["teaser"], raw["seniority"], raw["employment"], raw["country"]) == ( "Short.", "senior", "full_time", "DE") assert raw["location"] == "Berlin; Remote (DE)" assert not {"description", "title", "url", "posted", "source_id"} & set(raw) assert (raw["department"], raw["salary_text"], raw["currency"], raw["salary_min"]) == ( "", "", "", None) def test_a_row_without_places_has_its_location_as_the_one_place() -> None: raw = _runner.raw_of({"source_id": "1", "title": "t", "url": "u", "location": "Berlin, Munich"}) assert raw["places"] == [{"text": "Berlin, Munich", "remote": None}] assert raw["place"] == "Berlin, Munich" assert _runner.raw_of({"source_id": "1", "title": "t", "url": "u"})["places"] == [] def test_a_detail_location_without_places_becomes_the_one_place() -> None: module = _whole(text="detail") module.detail = lambda get, slug, row: {"location": "Aarhus", "employment": "contract", "description": "Full."} rows, done, failed = _runner.read_details(module, "fake", "acme", Web(), [_row("1")], set(), set(), limits=NO_DELAY) assert rows[0]["places"] == [{"text": "Aarhus", "remote": None}] assert (rows[0]["employment"], rows[0]["description"]) == ("contract", "Full.") assert (done, failed) == ({"fake:acme:1"}, set())