"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a subscription resolves to, series and their union, whole and paged listings, completeness from the declaration, the sweep across pulls, the 10% guard, the facets kept in state, the detail reads and the `raw` an item carries. No network: `get` is a dict of URL -> body or exception.""" from __future__ import annotations import json import sys import types from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any import httpx import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from plugins import ats, readers # noqa: E402 from plugins.readers import Ask, Filters, Key, Reads, _runner # noqa: E402 NO_DELAY = _runner.Limits(page_delay=0) def _status(code: int) -> httpx.HTTPStatusError: request = httpx.Request("GET", "https://jobs.fake.example/") return httpx.HTTPStatusError(str(code), request=request, response=httpx.Response(code, request=request)) class Web: """A `get` answering from a dict of URL -> body or exception.""" def __init__(self, answers: dict[str, Any] | None = None) -> None: self.answers = answers or {} self.asked: list[str] = [] def __call__(self, url: str, **kwargs: Any) -> bytes: self.asked.append(url) answer = self.answers.get(url, _status(404)) if isinstance(answer, BaseException): raise answer return answer def _row(i: str, **extra: Any) -> dict[str, Any]: return {"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}", "places": [{"text": "Berlin", "remote": None}]} | extra def _whole(**reads: Any) -> Any: """A declared whole-listing board: the feed is a JSON list of ids, one feed per `ask.countries` value.""" module: Any = types.ModuleType("fake") module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () module.slug_ok = lambda slug: slug.isalnum() module.allowed = lambda slug: lambda h: h == "jobs.fake.example" module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" module.READS = Reads(**({"scope": "board", "listing": "whole", "text": "listing", "complete": "whole", "places": "many", "robots": "test"} | reads)) module.feed_url = lambda slug, ask: (f"https://jobs.fake.example/{slug}/list" + "".join(f"?c={c}" for c in ask.countries)) module.parse = lambda body, slug: [_row(i) for i in json.loads(body)] return module def _paged(pages: list[list[str]], total: int | None = None, **reads: Any) -> Any: """A declared paged board over `pages`, the cursor being the page index.""" module = _whole(**({"listing": "paged", "complete": "stated" if total is not None else "end"} | reads)) del module.feed_url, module.parse module.asked = [] def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any: index = int(cursor or 0) module.asked.append((index, ask.countries)) body = get(f"https://jobs.fake.example/{slug}/page/{index}") ids = json.loads(body) if body else pages[index] more = index + 1 < len(pages) and bool(ids) return ids, [{"id": i} for i in ids], total, index + 1 if more else None module.page = page module.rows = lambda ids, slug: [_row(i) for i in ids] return module def _pages_web(pages: list[list[str]]) -> Web: return Web({f"https://jobs.fake.example/acme/page/{n}": json.dumps(ids).encode() for n, ids in enumerate(pages)}) # -------------------------------------------------------------------- ask def test_ask_for_narrows_the_params_to_the_declared_filters() -> None: board = _whole(filters=Filters(country="one")) ask = _runner.ask_for(board, {"countries": ["dk", "se"], "remote": True, "places": ["X"]}) assert (ask.countries, ask.remote, ask.places, ask.params) == (("DK", "SE"), None, (), {}) # The params override the reach per dimension; the reach fills the rest. both = _whole(filters=Filters(country="many", remote="many")) ask = _runner.ask_for(both, {"countries": ["DE"]}, {"countries": ["DK"], "remote": True}) assert (ask.countries, ask.remote) == (("DE",), True) def test_a_searchs_params_go_whole_into_the_ask_with_its_key( monkeypatch: pytest.MonkeyPatch) -> None: module = _whole(scope="search", listing="paged", complete="stated", text="excerpt", filters=Filters(query="one", place="one"), key=Key(name="fake", fields=("id", "secret"), env=("FAKE_ID", "FAKE_SECRET"))) module.PARAMS = {"q": "query", "area": "place"} monkeypatch.delenv("FAKE_ID", raising=False) monkeypatch.setenv("FAKE_SECRET", "s") ask = _runner.ask_for(module, {"q": "python", "area": 1, "pages": 2}, keys={"fake": {"id": "i"}}) assert (ask.query, ask.places) == ("python", ("1",)) assert ask.params == {"q": "python", "area": 1, "pages": 2, "key": {"id": "i", "secret": "s"}} with pytest.raises(ValueError, match=r"sources\.search\.fake: id, secret, or FAKE_ID"): _runner.ask_for(module, {"q": "x"}, keys={}) def test_series_are_one_a_value_for_one_and_one_for_many() -> None: ask = Ask(countries=("DK", "SE"), places=("Aarhus",)) one = _whole(filters=Filters(country="one", place="many")) assert [(a.countries, a.places) for a in _runner.series(one, ask)] == [ (("DK",), ("Aarhus",)), (("SE",), ("Aarhus",))] many = _whole(filters=Filters(country="many")) assert _runner.series(many, ask) == [ask] def test_ids_filters_are_mapped_through_the_facets() -> None: module = _whole(filters=Filters(country="ids")) module.facets = lambda get, slug: {} ask = Ask(countries=("DK", "XX")) [mapped] = _runner.series(module, ask, {"country": {"DK": "h-1", "SE": "h-2"}}) assert mapped.countries == ("h-1",) # None known: nothing is asked, which is no evidence, so the read is # incomplete and closes nothing. assert _runner.series(module, Ask(countries=("XX",)), {"country": {"DK": "h-1"}}) == [] listing = _runner.read_listing(module, "acme", Web(), Ask(countries=("XX",)), facets={"country": {}}) assert (listing.rows, listing.complete, listing.filtered) == ([], False, True) def test_facets_are_asked_once_a_day_and_only_for_an_ids_filter() -> None: module = _whole(filters=Filters(country="ids")) calls: list[str] = [] module.facets = lambda get, slug: calls.append(slug) or {"country": {"DK": "7"}} state: dict[str, str] = {} assert _runner.facets_for(module, "acme", Web(), Ask(), state, "k") == {} for _ in range(2): found = _runner.facets_for(module, "acme", Web(), Ask(countries=("DK",)), state, "k") assert found == {"country": {"DK": "7"}} assert calls == ["acme"] kept = json.loads(state["k"]) kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat() state["k"] = json.dumps(kept) _runner.facets_for(module, "acme", Web(), Ask(countries=("DK",)), state, "k") assert calls == ["acme", "acme"] # ---------------------------------------------------------------- listing def test_a_whole_listing_reads_one_feed_a_series_and_unions_by_id() -> None: module = _whole(filters=Filters(country="one")) web = Web({"https://jobs.fake.example/acme/list?c=DK": b'["1", "2"]', "https://jobs.fake.example/acme/list?c=SE": b'["2", "3"]'}) listing = _runner.read_listing(module, "acme", web, Ask(countries=("DK", "SE")), limits=NO_DELAY) assert [r["source_id"] for r in listing.rows] == ["1", "2", "3"] assert (listing.complete, listing.filtered, listing.ids) == (True, True, ["1", "2", "3"]) def test_a_body_over_the_cap_is_read_again_at_the_fallback_feed() -> None: """Greenhouse's shape: the feed with the text, and one without it.""" module = _whole() module.fallback_url = lambda slug: f"https://jobs.fake.example/{slug}/small" web = Web({"https://jobs.fake.example/acme/list": _runner.TooLarge("over"), "https://jobs.fake.example/acme/small": b'["1"]'}) listing = _runner.read_listing(module, "acme", web) assert ([r["source_id"] for r in listing.rows], listing.complete) == (["1"], True) # A reader without a fallback fails. del module.fallback_url with pytest.raises(_runner.TooLarge): _runner.read_listing(module, "acme", web) def test_a_paged_board_with_a_total_is_complete_when_it_read_as_many() -> None: pages = [["1", "2"], ["3"]] listing = _runner.read_listing(_paged(pages, total=3), "acme", _pages_web(pages), limits=NO_DELAY) assert (listing.ids, listing.complete) == (["1", "2", "3"], True) short = _runner.read_listing(_paged(pages, total=4), "acme", _pages_web(pages), limits=NO_DELAY) assert not short.complete def test_the_page_cap_leaves_a_board_incomplete() -> None: pages = [["1"], ["2"], ["3"]] listing = _runner.read_listing(_paged(pages), "acme", _pages_web(pages), limits=_runner.Limits(max_pages=2, page_delay=0)) assert (listing.ids, listing.complete, listing.sweep) == (["1", "2"], False, None) def test_a_page_with_nothing_new_ends_the_read_incomplete() -> None: """A host that ignores the cursor answers page one again; without a total the read would never end, so a page with nothing new stops it.""" pages = [["1", "2"], ["1", "2"], ["3"]] module = _paged(pages) listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY) assert (listing.ids, listing.complete) == (["1", "2"], False) assert [index for index, _ in module.asked] == [0, 1] def test_a_page_with_nothing_new_is_a_stall_with_a_stated_total_too() -> None: """A host that ignores the cursor and states a total used to be read on to the page cap; the second page, which adds nothing while the reader hands a cursor, ends it incomplete.""" pages = [["1", "2"]] * 5 module = _paged(pages, total=5) listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY) assert (listing.ids, listing.complete) == (["1", "2"], False) assert [index for index, _ in module.asked] == [0, 1] def test_an_empty_last_page_ends_a_read_without_a_total_complete() -> None: """`complete="end"`: the page past the last role is empty and hands no cursor. That is the end, not a stall.""" pages = [["1"], ["2"], []] module = _paged(pages) assert module.READS.complete == "end" listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY) assert (listing.ids, listing.complete) == (["1", "2"], True) assert [index for index, _ in module.asked] == [0, 1, 2] def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None: pages = [["1"], ["2"]] web = _pages_web(pages) web.answers["https://jobs.fake.example/acme/page/1"] = _status(429) listing = _runner.read_listing(_paged(pages), "acme", web, limits=NO_DELAY) assert (listing.ids, listing.complete) == (["1"], False) web.answers["https://jobs.fake.example/acme/page/0"] = _status(429) with pytest.raises(httpx.HTTPStatusError): _runner.read_listing(_paged(pages), "acme", web, limits=NO_DELAY) def test_a_search_is_never_complete_when_it_says_so() -> None: pages = [["1"]] module = _paged(pages, scope="search", complete="never", text="excerpt") assert not _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY).complete def test_paged_series_each_read_to_the_end_and_union() -> None: pages = [["1", "2"]] module = _paged(pages, filters=Filters(country="one")) listing = _runner.read_listing(module, "acme", _pages_web(pages), Ask(countries=("DK", "SE")), limits=NO_DELAY) assert module.asked == [(0, ("DK",)), (0, ("SE",))] assert (listing.ids, listing.complete) == (["1", "2"], True) # ------------------------------------------------------------------ sweep def test_a_board_over_the_page_cap_is_swept_across_pulls() -> None: pages = [["1"], ["2"], ["3"], ["4"], ["5"]] module = _paged(pages, total=5) web, state = _pages_web(pages), {} limits = _runner.Limits(max_pages=2, page_delay=0) seen: list[tuple[list[str], bool]] = [] for _pull in range(3): listing = _runner.read(state, "ats", "fake:acme", module, "acme", web, limits=limits) _runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids) seen.append((listing.ids, listing.complete)) assert seen == [(["1", "2"], False), (["3", "4"], False), (["1", "2", "3", "4", "5"], True)] assert [index for index, _ in module.asked] == [0, 1, 2, 3, 4] assert "ats:sweep:fake:acme" not in state assert json.loads(state["ats:count:fake:acme"])["count"] == 5 assert state["ats:swept:fake:acme"] == "1" def _tupled(pages: list[list[str]], total: int) -> Any: """`_paged` with a cursor of Workday's kind, `(next page, total)`, which a sweep hands back from state as a list. `cursors` are the ones it was handed.""" module = _paged(pages, total=total) module.cursors = [] def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any: module.cursors.append(cursor) index, stated = cursor or (0, total) ids = pages[index] more = index + 1 < len(pages) return ids, [{"id": i} for i in ids], stated, (index + 1, stated) if more else None module.page = page return module def test_a_board_whose_cursor_is_a_tuple_is_swept_across_pulls() -> None: """A tuple comes back from JSON as a list. The sweep is kept all the same, the reader gets the list, and the second pull finishes it.""" module = _tupled([["1"], ["2"], ["3"]], total=3) state: dict[str, str] = {} limits = _runner.Limits(max_pages=2, page_delay=0) first = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits) _runner.stamp(state, "ats", "fake:acme", "r", first, first.ids) assert (first.ids, first.complete) == (["1", "2"], False) assert json.loads(state["ats:sweep:fake:acme"])["cursor"] == [2, 3] second = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits) _runner.stamp(state, "ats", "fake:acme", "r", second, second.ids) assert (second.ids, second.complete, second.swept) == (["1", "2", "3"], True, True) assert module.cursors == [None, (1, 3), [2, 3]] assert "ats:sweep:fake:acme" not in state def test_a_cursor_is_storable_when_json_gives_it_back_save_tuples_as_lists() -> None: assert _runner._storable(7) and _runner._storable("token") and _runner._storable(None) assert _runner._storable((40, 45)) and _runner._storable((4, "1431772533")) assert _runner._storable({"after": (1, [2, (3,)])}) assert not _runner._storable({1: "a"}) # the key comes back as "1" assert not _runner._storable(object()) def test_a_board_read_in_one_pull_is_not_swept() -> None: pages = [["1"], ["2"]] state: dict[str, str] = {} listing = _runner.read(state, "ats", "fake:acme", _paged(pages, total=2), "acme", _pages_web(pages), limits=NO_DELAY) _runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids) assert (listing.complete, listing.swept, state["ats:swept:fake:acme"]) == (True, False, "") def test_a_sweep_restarted_by_a_changed_ask_is_not_swept() -> None: pages = [["1"], ["2"], ["3"]] module = _paged(pages, filters=Filters(country="many")) web, state = _pages_web(pages), {} limits = _runner.Limits(max_pages=2, page_delay=0) first = _runner.read(state, "ats", "t", module, "acme", web, Ask(), limits=limits) _runner.stamp(state, "ats", "t", "r", first, first.ids) again = _runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",)), limits=NO_DELAY) assert (again.complete, again.swept) == (True, False) def test_a_sweep_starts_again_when_the_ask_changed() -> None: pages = [["1"], ["2"], ["3"]] module = _paged(pages, filters=Filters(country="many")) web, state = _pages_web(pages), {} limits = _runner.Limits(max_pages=2, page_delay=0) first = _runner.read(state, "ats", "t", module, "acme", web, Ask(), limits=limits) _runner.stamp(state, "ats", "t", "r", first, first.ids) module.asked.clear() _runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",)), limits=limits) assert module.asked[0][0] == 0 def test_a_board_at_its_ceiling_is_not_swept() -> None: pages = [["1", "2"], ["3", "4"], ["5"]] module = _paged(pages, total=5, page_size=2, ceiling=4) listing = _runner.read({}, "ats", "t", module, "acme", _pages_web(pages), limits=_runner.Limits(max_pages=2, page_delay=0)) assert (listing.complete, listing.sweep) == (False, None) def test_a_search_is_not_swept() -> None: pages = [["1"], ["2"]] module = _paged(pages, total=2) listing = _runner.read_listing(module, "acme", _pages_web(pages), can_sweep=False, limits=_runner.Limits(max_pages=1, page_delay=0)) assert listing.sweep is None # ----------------------------------------------------------- the 10% guard def test_a_suspect_read_far_under_the_last_count_fails() -> None: module = _whole(empty="suspect") web = Web({"https://jobs.fake.example/acme/list": json.dumps( [str(n) for n in range(20)]).encode()}) state: dict[str, str] = {} listing = _runner.read(state, "ats", "t", module, "acme", web) _runner.stamp(state, "ats", "t", "r", listing, listing.ids) web.answers["https://jobs.fake.example/acme/list"] = b'["1"]' with pytest.raises(_runner.Suspect, match="under 10% of the 20"): _runner.read(state, "ats", "t", module, "acme", web) # A reader whose empty answer is trusted is held too, from a count of # `SUSPECT_MIN`. module.READS = module.READS.model_copy(update={"empty": "trusted"}) with pytest.raises(_runner.Suspect): _runner.read(state, "ats", "t", module, "acme", web) def test_the_guard_compares_only_reads_that_asked_the_same() -> None: """A board read whole last time and with a filter now lists far fewer roles; that is the filter, not a failure.""" module = _whole(filters=Filters(country="one")) web = Web({"https://jobs.fake.example/acme/list": json.dumps( [str(n) for n in range(50)]).encode(), "https://jobs.fake.example/acme/list?c=DK": b'["1"]'}) state: dict[str, str] = {} listing = _runner.read(state, "ats", "t", module, "acme", web) _runner.stamp(state, "ats", "t", "r", listing, listing.ids) filtered = _runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",))) assert filtered.ids == ["1"] and filtered.filtered # ------------------------------------------------------ through ATSSource @pytest.fixture def board(monkeypatch: pytest.MonkeyPatch) -> Any: pages = [["1"], ["2"], ["3"]] module = _paged(pages, total=3, empty="suspect") monkeypatch.setitem(readers.HOSTS, "fake", module) monkeypatch.setattr(ats, "PAGE_DELAY", 0) monkeypatch.setattr(ats, "MAX_PAGES", 2) web = _pages_web(pages) monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: web(url)) module.web = web return module def _source(state: dict[str, str], params: dict[str, Any] | None = None) -> ats.ATSSource: source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme", "params": params or {}}], "run": "r"}) source.state = state return source def test_ats_source_sweeps_and_stamps_every_id_once_the_sweep_ends(board: Any) -> None: state: dict[str, str] = {} items = list(_source(state).fetch()) assert [i.source_id for i in items] == ["fake:acme:1", "fake:acme:2"] assert state["ats:complete:fake:acme"] == "" assert json.loads(state["ats:sweep:fake:acme"])["cursor"] == 2 items = list(_source(state).fetch()) assert [i.source_id for i in items] == ["fake:acme:3"] assert state["ats:complete:fake:acme"] == "1" assert json.loads(state["ats:ids:fake:acme"]) == [ "fake:acme:1", "fake:acme:2", "fake:acme:3"] def test_ats_source_records_a_suspect_read_as_failing(board: Any) -> None: state = {"ats:count:fake:acme": json.dumps( {"asks": _runner.fingerprint([Ask()]), "count": 100})} board.web.answers = {"https://jobs.fake.example/acme/page/0": b"[]"} assert list(_source(state).fetch()) == [] assert "under 10%" in json.loads(state["ats:failing:fake:acme"])["reason"] assert "ats:ok:fake:acme" not in state def test_ats_source_hands_the_board_params_to_the_reader(board: Any) -> None: board.READS = board.READS.model_copy(update={"filters": Filters(country="many")}) state: dict[str, str] = {} list(_source(state, {"countries": ["dk"]}).fetch()) assert board.asked[0] == (0, ("DK",)) assert state["ats:filtered:fake:acme"] == "1" # -------------------------------------------------------------------- raw def test_raw_keeps_every_stated_key_and_the_places() -> None: row = _row("1", places=[{"text": "Berlin", "remote": False}, {"text": "Remote (DE)", "remote": True}], teaser="Short.", seniority="senior", employment="full_time", country="DE", description="Long.", posted=None, location="Berlin; Remote (DE)") raw = _runner.raw_of(row) assert raw["places"] == row["places"] assert raw["place"] == "Berlin; Remote (DE)" assert (raw["teaser"], raw["seniority"], raw["employment"], raw["country"]) == ( "Short.", "senior", "full_time", "DE") assert raw["location"] == "Berlin; Remote (DE)" assert not {"description", "title", "url", "posted", "source_id"} & set(raw) assert (raw["department"], raw["salary_text"], raw["currency"], raw["salary_min"]) == ( "", "", "", None) def test_a_row_without_places_has_its_location_as_the_one_place() -> None: raw = _runner.raw_of({"source_id": "1", "title": "t", "url": "u", "location": "Berlin, Munich"}) assert raw["places"] == [{"text": "Berlin, Munich", "remote": None}] assert raw["place"] == "Berlin, Munich" assert _runner.raw_of({"source_id": "1", "title": "t", "url": "u"})["places"] == [] def test_a_detail_location_without_places_becomes_the_one_place() -> None: module = _whole(text="detail") module.detail = lambda get, slug, row: {"location": "Aarhus", "employment": "contract", "description": "Full."} rows, done, failed = _runner.read_details(module, "fake", "acme", Web(), [_row("1")], set(), set(), limits=NO_DELAY) assert rows[0]["places"] == [{"text": "Aarhus", "remote": None}] assert (rows[0]["employment"], rows[0]["description"]) == ("contract", "Full.") assert (done, failed) == ({"fake:acme:1"}, set())