Files
jobboard/tests/test_runner.py
T

499 lines
23 KiB
Python

"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a
subscription resolves to, series and their union, whole and paged listings,
completeness from the declaration, the sweep across pulls, the 10% guard,
the facets kept in state, the detail reads and the `raw` an item carries. No
network: `get` is a dict of URL -> body or exception."""
from __future__ import annotations
import json
import sys
import types
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
import httpx
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, Filters, Key, Reads, _runner # noqa: E402
NO_DELAY = _runner.Limits(page_delay=0)
def _status(code: int) -> httpx.HTTPStatusError:
request = httpx.Request("GET", "https://jobs.fake.example/")
return httpx.HTTPStatusError(str(code), request=request,
response=httpx.Response(code, request=request))
class Web:
"""A `get` answering from a dict of URL -> body or exception."""
def __init__(self, answers: dict[str, Any] | None = None) -> None:
self.answers = answers or {}
self.asked: list[str] = []
def __call__(self, url: str, **kwargs: Any) -> bytes:
self.asked.append(url)
answer = self.answers.get(url, _status(404))
if isinstance(answer, BaseException):
raise answer
return answer
def _row(i: str, **extra: Any) -> dict[str, Any]:
return {"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}",
"places": [{"text": "Berlin", "remote": None}]} | extra
def _whole(**reads: Any) -> Any:
"""A declared whole-listing board: the feed is a JSON list of ids, one
feed per `ask.countries` value."""
module: Any = types.ModuleType("fake")
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
module.slug_ok = lambda slug: slug.isalnum()
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
module.READS = Reads(**({"scope": "board", "listing": "whole", "text": "listing",
"complete": "whole", "places": "many", "robots": "test"} | reads))
module.feed_url = lambda slug, ask: (f"https://jobs.fake.example/{slug}/list"
+ "".join(f"?c={c}" for c in ask.countries))
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)]
return module
def _paged(pages: list[list[str]], total: int | None = None, **reads: Any) -> Any:
"""A declared paged board over `pages`, the cursor being the page index."""
module = _whole(**({"listing": "paged", "complete": "stated" if total is not None
else "end"} | reads))
del module.feed_url, module.parse
module.asked = []
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
index = int(cursor or 0)
module.asked.append((index, ask.countries))
body = get(f"https://jobs.fake.example/{slug}/page/{index}")
ids = json.loads(body) if body else pages[index]
more = index + 1 < len(pages) and bool(ids)
return ids, [{"id": i} for i in ids], total, index + 1 if more else None
module.page = page
module.rows = lambda ids, slug: [_row(i) for i in ids]
return module
def _pages_web(pages: list[list[str]]) -> Web:
return Web({f"https://jobs.fake.example/acme/page/{n}": json.dumps(ids).encode()
for n, ids in enumerate(pages)})
# -------------------------------------------------------------------- ask
def test_ask_for_narrows_the_params_to_the_declared_filters() -> None:
board = _whole(filters=Filters(country="one"))
ask = _runner.ask_for(board, {"countries": ["dk", "se"], "remote": True, "places": ["X"]})
assert (ask.countries, ask.remote, ask.places, ask.params) == (("DK", "SE"), None, (), {})
# The params override the reach per dimension; the reach fills the rest.
both = _whole(filters=Filters(country="many", remote="many"))
ask = _runner.ask_for(both, {"countries": ["DE"]}, {"countries": ["DK"], "remote": True})
assert (ask.countries, ask.remote) == (("DE",), True)
def test_a_searchs_params_go_whole_into_the_ask_with_its_key(
monkeypatch: pytest.MonkeyPatch) -> None:
module = _whole(scope="search", listing="paged", complete="stated", text="excerpt",
filters=Filters(query="one", place="one"),
key=Key(name="fake", fields=("id", "secret"), env=("FAKE_ID", "FAKE_SECRET")))
module.PARAMS = {"q": "query", "area": "place"}
monkeypatch.delenv("FAKE_ID", raising=False)
monkeypatch.setenv("FAKE_SECRET", "s")
ask = _runner.ask_for(module, {"q": "python", "area": 1, "pages": 2},
keys={"fake": {"id": "i"}})
assert (ask.query, ask.places) == ("python", ("1",))
assert ask.params == {"q": "python", "area": 1, "pages": 2,
"key": {"id": "i", "secret": "s"}}
with pytest.raises(ValueError, match=r"sources\.search\.fake: id, secret, or FAKE_ID"):
_runner.ask_for(module, {"q": "x"}, keys={})
def test_series_are_one_a_value_for_one_and_one_for_many() -> None:
ask = Ask(countries=("DK", "SE"), places=("Aarhus",))
one = _whole(filters=Filters(country="one", place="many"))
assert [(a.countries, a.places) for a in _runner.series(one, ask)] == [
(("DK",), ("Aarhus",)), (("SE",), ("Aarhus",))]
many = _whole(filters=Filters(country="many"))
assert _runner.series(many, ask) == [ask]
def test_ids_filters_are_mapped_through_the_facets() -> None:
module = _whole(filters=Filters(country="ids"))
module.facets = lambda get, slug: {}
ask = Ask(countries=("DK", "XX"))
[mapped] = _runner.series(module, ask, {"country": {"DK": "h-1", "SE": "h-2"}})
assert mapped.countries == ("h-1",)
# None known: nothing is asked, which is no evidence, so the read is
# incomplete and closes nothing.
assert _runner.series(module, Ask(countries=("XX",)), {"country": {"DK": "h-1"}}) == []
listing = _runner.read_listing(module, "acme", Web(), Ask(countries=("XX",)),
facets={"country": {}})
assert (listing.rows, listing.complete, listing.filtered) == ([], False, True)
def test_facets_are_asked_once_a_day_and_only_for_an_ids_filter() -> None:
module = _whole(filters=Filters(country="ids"))
calls: list[str] = []
module.facets = lambda get, slug: calls.append(slug) or {"country": {"DK": "7"}}
state: dict[str, str] = {}
assert _runner.facets_for(module, "acme", Web(), Ask(), state, "k") == {}
for _ in range(2):
found = _runner.facets_for(module, "acme", Web(), Ask(countries=("DK",)), state, "k")
assert found == {"country": {"DK": "7"}}
assert calls == ["acme"]
kept = json.loads(state["k"])
kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat()
state["k"] = json.dumps(kept)
_runner.facets_for(module, "acme", Web(), Ask(countries=("DK",)), state, "k")
assert calls == ["acme", "acme"]
# ---------------------------------------------------------------- listing
def test_a_whole_listing_reads_one_feed_a_series_and_unions_by_id() -> None:
module = _whole(filters=Filters(country="one"))
web = Web({"https://jobs.fake.example/acme/list?c=DK": b'["1", "2"]',
"https://jobs.fake.example/acme/list?c=SE": b'["2", "3"]'})
listing = _runner.read_listing(module, "acme", web, Ask(countries=("DK", "SE")),
limits=NO_DELAY)
assert [r["source_id"] for r in listing.rows] == ["1", "2", "3"]
assert (listing.complete, listing.filtered, listing.ids) == (True, True, ["1", "2", "3"])
def test_a_body_over_the_cap_is_read_again_at_the_fallback_feed() -> None:
"""Greenhouse's shape: the feed with the text, and one without it."""
module = _whole()
module.fallback_url = lambda slug: f"https://jobs.fake.example/{slug}/small"
web = Web({"https://jobs.fake.example/acme/list": _runner.TooLarge("over"),
"https://jobs.fake.example/acme/small": b'["1"]'})
listing = _runner.read_listing(module, "acme", web)
assert ([r["source_id"] for r in listing.rows], listing.complete) == (["1"], True)
# A reader without a fallback fails.
del module.fallback_url
with pytest.raises(_runner.TooLarge):
_runner.read_listing(module, "acme", web)
def test_a_paged_board_with_a_total_is_complete_when_it_read_as_many() -> None:
pages = [["1", "2"], ["3"]]
listing = _runner.read_listing(_paged(pages, total=3), "acme", _pages_web(pages),
limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2", "3"], True)
short = _runner.read_listing(_paged(pages, total=4), "acme", _pages_web(pages),
limits=NO_DELAY)
assert not short.complete
def test_the_page_cap_leaves_a_board_incomplete() -> None:
pages = [["1"], ["2"], ["3"]]
listing = _runner.read_listing(_paged(pages), "acme", _pages_web(pages),
limits=_runner.Limits(max_pages=2, page_delay=0))
assert (listing.ids, listing.complete, listing.sweep) == (["1", "2"], False, None)
def test_a_page_with_nothing_new_ends_the_read_incomplete() -> None:
"""A host that ignores the cursor answers page one again; without a total
the read would never end, so a page with nothing new stops it."""
pages = [["1", "2"], ["1", "2"], ["3"]]
module = _paged(pages)
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2"], False)
assert [index for index, _ in module.asked] == [0, 1]
def test_a_page_with_nothing_new_is_a_stall_with_a_stated_total_too() -> None:
"""A host that ignores the cursor and states a total used to be read on
to the page cap; the second page, which adds nothing while the reader
hands a cursor, ends it incomplete."""
pages = [["1", "2"]] * 5
module = _paged(pages, total=5)
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2"], False)
assert [index for index, _ in module.asked] == [0, 1]
def test_an_empty_last_page_ends_a_read_without_a_total_complete() -> None:
"""`complete="end"`: the page past the last role is empty and hands no
cursor. That is the end, not a stall."""
pages = [["1"], ["2"], []]
module = _paged(pages)
assert module.READS.complete == "end"
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2"], True)
assert [index for index, _ in module.asked] == [0, 1, 2]
def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None:
pages = [["1"], ["2"]]
web = _pages_web(pages)
web.answers["https://jobs.fake.example/acme/page/1"] = _status(429)
listing = _runner.read_listing(_paged(pages), "acme", web, limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1"], False)
web.answers["https://jobs.fake.example/acme/page/0"] = _status(429)
with pytest.raises(httpx.HTTPStatusError):
_runner.read_listing(_paged(pages), "acme", web, limits=NO_DELAY)
def test_a_search_is_never_complete_when_it_says_so() -> None:
pages = [["1"]]
module = _paged(pages, scope="search", complete="never", text="excerpt")
assert not _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY).complete
def test_paged_series_each_read_to_the_end_and_union() -> None:
pages = [["1", "2"]]
module = _paged(pages, filters=Filters(country="one"))
listing = _runner.read_listing(module, "acme", _pages_web(pages),
Ask(countries=("DK", "SE")), limits=NO_DELAY)
assert module.asked == [(0, ("DK",)), (0, ("SE",))]
assert (listing.ids, listing.complete) == (["1", "2"], True)
# ------------------------------------------------------------------ sweep
def test_a_board_over_the_page_cap_is_swept_across_pulls() -> None:
pages = [["1"], ["2"], ["3"], ["4"], ["5"]]
module = _paged(pages, total=5)
web, state = _pages_web(pages), {}
limits = _runner.Limits(max_pages=2, page_delay=0)
seen: list[tuple[list[str], bool]] = []
for _pull in range(3):
listing = _runner.read(state, "ats", "fake:acme", module, "acme", web, limits=limits)
_runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
seen.append((listing.ids, listing.complete))
assert seen == [(["1", "2"], False), (["3", "4"], False), (["1", "2", "3", "4", "5"], True)]
assert [index for index, _ in module.asked] == [0, 1, 2, 3, 4]
assert "ats:sweep:fake:acme" not in state
assert json.loads(state["ats:count:fake:acme"])["count"] == 5
assert state["ats:swept:fake:acme"] == "1"
def _tupled(pages: list[list[str]], total: int) -> Any:
"""`_paged` with a cursor of Workday's kind, `(next page, total)`, which a
sweep hands back from state as a list. `cursors` are the ones it was
handed."""
module = _paged(pages, total=total)
module.cursors = []
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
module.cursors.append(cursor)
index, stated = cursor or (0, total)
ids = pages[index]
more = index + 1 < len(pages)
return ids, [{"id": i} for i in ids], stated, (index + 1, stated) if more else None
module.page = page
return module
def test_a_board_whose_cursor_is_a_tuple_is_swept_across_pulls() -> None:
"""A tuple comes back from JSON as a list. The sweep is kept all the
same, the reader gets the list, and the second pull finishes it."""
module = _tupled([["1"], ["2"], ["3"]], total=3)
state: dict[str, str] = {}
limits = _runner.Limits(max_pages=2, page_delay=0)
first = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits)
_runner.stamp(state, "ats", "fake:acme", "r", first, first.ids)
assert (first.ids, first.complete) == (["1", "2"], False)
assert json.loads(state["ats:sweep:fake:acme"])["cursor"] == [2, 3]
second = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits)
_runner.stamp(state, "ats", "fake:acme", "r", second, second.ids)
assert (second.ids, second.complete, second.swept) == (["1", "2", "3"], True, True)
assert module.cursors == [None, (1, 3), [2, 3]]
assert "ats:sweep:fake:acme" not in state
def test_a_cursor_is_storable_when_json_gives_it_back_save_tuples_as_lists() -> None:
assert _runner._storable(7) and _runner._storable("token") and _runner._storable(None)
assert _runner._storable((40, 45)) and _runner._storable((4, "1431772533"))
assert _runner._storable({"after": (1, [2, (3,)])})
assert not _runner._storable({1: "a"}) # the key comes back as "1"
assert not _runner._storable(object())
def test_a_board_read_in_one_pull_is_not_swept() -> None:
pages = [["1"], ["2"]]
state: dict[str, str] = {}
listing = _runner.read(state, "ats", "fake:acme", _paged(pages, total=2), "acme",
_pages_web(pages), limits=NO_DELAY)
_runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
assert (listing.complete, listing.swept, state["ats:swept:fake:acme"]) == (True, False, "")
def test_a_sweep_restarted_by_a_changed_ask_is_not_swept() -> None:
pages = [["1"], ["2"], ["3"]]
module = _paged(pages, filters=Filters(country="many"))
web, state = _pages_web(pages), {}
limits = _runner.Limits(max_pages=2, page_delay=0)
first = _runner.read(state, "ats", "t", module, "acme", web, Ask(), limits=limits)
_runner.stamp(state, "ats", "t", "r", first, first.ids)
again = _runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",)),
limits=NO_DELAY)
assert (again.complete, again.swept) == (True, False)
def test_a_sweep_starts_again_when_the_ask_changed() -> None:
pages = [["1"], ["2"], ["3"]]
module = _paged(pages, filters=Filters(country="many"))
web, state = _pages_web(pages), {}
limits = _runner.Limits(max_pages=2, page_delay=0)
first = _runner.read(state, "ats", "t", module, "acme", web, Ask(), limits=limits)
_runner.stamp(state, "ats", "t", "r", first, first.ids)
module.asked.clear()
_runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",)), limits=limits)
assert module.asked[0][0] == 0
def test_a_board_at_its_ceiling_is_not_swept() -> None:
pages = [["1", "2"], ["3", "4"], ["5"]]
module = _paged(pages, total=5, page_size=2, ceiling=4)
listing = _runner.read({}, "ats", "t", module, "acme", _pages_web(pages),
limits=_runner.Limits(max_pages=2, page_delay=0))
assert (listing.complete, listing.sweep) == (False, None)
def test_a_search_is_not_swept() -> None:
pages = [["1"], ["2"]]
module = _paged(pages, total=2)
listing = _runner.read_listing(module, "acme", _pages_web(pages), can_sweep=False,
limits=_runner.Limits(max_pages=1, page_delay=0))
assert listing.sweep is None
# ----------------------------------------------------------- the 10% guard
def test_a_suspect_read_far_under_the_last_count_fails() -> None:
module = _whole(empty="suspect")
web = Web({"https://jobs.fake.example/acme/list": json.dumps(
[str(n) for n in range(20)]).encode()})
state: dict[str, str] = {}
listing = _runner.read(state, "ats", "t", module, "acme", web)
_runner.stamp(state, "ats", "t", "r", listing, listing.ids)
web.answers["https://jobs.fake.example/acme/list"] = b'["1"]'
with pytest.raises(_runner.Suspect, match="under 10% of the 20"):
_runner.read(state, "ats", "t", module, "acme", web)
# A reader whose empty answer is trusted is held too, from a count of
# `SUSPECT_MIN`.
module.READS = module.READS.model_copy(update={"empty": "trusted"})
with pytest.raises(_runner.Suspect):
_runner.read(state, "ats", "t", module, "acme", web)
def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
"""A board read whole last time and with a filter now lists far fewer
roles; that is the filter, not a failure."""
module = _whole(filters=Filters(country="one"))
web = Web({"https://jobs.fake.example/acme/list": json.dumps(
[str(n) for n in range(50)]).encode(),
"https://jobs.fake.example/acme/list?c=DK": b'["1"]'})
state: dict[str, str] = {}
listing = _runner.read(state, "ats", "t", module, "acme", web)
_runner.stamp(state, "ats", "t", "r", listing, listing.ids)
filtered = _runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",)))
assert filtered.ids == ["1"] and filtered.filtered
# ------------------------------------------------------ through ATSSource
@pytest.fixture
def board(monkeypatch: pytest.MonkeyPatch) -> Any:
pages = [["1"], ["2"], ["3"]]
module = _paged(pages, total=3, empty="suspect")
monkeypatch.setitem(readers.HOSTS, "fake", module)
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
monkeypatch.setattr(ats, "MAX_PAGES", 2)
web = _pages_web(pages)
monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: web(url))
module.web = web
return module
def _source(state: dict[str, str], params: dict[str, Any] | None = None) -> ats.ATSSource:
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme",
"params": params or {}}], "run": "r"})
source.state = state
return source
def test_ats_source_sweeps_and_stamps_every_id_once_the_sweep_ends(board: Any) -> None:
state: dict[str, str] = {}
items = list(_source(state).fetch())
assert [i.source_id for i in items] == ["fake:acme:1", "fake:acme:2"]
assert state["ats:complete:fake:acme"] == ""
assert json.loads(state["ats:sweep:fake:acme"])["cursor"] == 2
items = list(_source(state).fetch())
assert [i.source_id for i in items] == ["fake:acme:3"]
assert state["ats:complete:fake:acme"] == "1"
assert json.loads(state["ats:ids:fake:acme"]) == [
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
def test_ats_source_records_a_suspect_read_as_failing(board: Any) -> None:
state = {"ats:count:fake:acme": json.dumps(
{"asks": _runner.fingerprint([Ask()]), "count": 100})}
board.web.answers = {"https://jobs.fake.example/acme/page/0": b"[]"}
assert list(_source(state).fetch()) == []
assert "under 10%" in json.loads(state["ats:failing:fake:acme"])["reason"]
assert "ats:ok:fake:acme" not in state
def test_ats_source_hands_the_board_params_to_the_reader(board: Any) -> None:
board.READS = board.READS.model_copy(update={"filters": Filters(country="many")})
state: dict[str, str] = {}
list(_source(state, {"countries": ["dk"]}).fetch())
assert board.asked[0] == (0, ("DK",))
assert state["ats:filtered:fake:acme"] == "1"
# -------------------------------------------------------------------- raw
def test_raw_keeps_every_stated_key_and_the_places() -> None:
row = _row("1", places=[{"text": "Berlin", "remote": False},
{"text": "Remote (DE)", "remote": True}],
teaser="Short.", seniority="senior", employment="full_time", country="DE",
description="Long.", posted=None, location="Berlin; Remote (DE)")
raw = _runner.raw_of(row)
assert raw["places"] == row["places"]
assert raw["place"] == "Berlin; Remote (DE)"
assert (raw["teaser"], raw["seniority"], raw["employment"], raw["country"]) == (
"Short.", "senior", "full_time", "DE")
assert raw["location"] == "Berlin; Remote (DE)"
assert not {"description", "title", "url", "posted", "source_id"} & set(raw)
assert (raw["department"], raw["salary_text"], raw["currency"], raw["salary_min"]) == (
"", "", "", None)
def test_a_row_without_places_has_its_location_as_the_one_place() -> None:
raw = _runner.raw_of({"source_id": "1", "title": "t", "url": "u",
"location": "Berlin, Munich"})
assert raw["places"] == [{"text": "Berlin, Munich", "remote": None}]
assert raw["place"] == "Berlin, Munich"
assert _runner.raw_of({"source_id": "1", "title": "t", "url": "u"})["places"] == []
def test_a_detail_location_without_places_becomes_the_one_place() -> None:
module = _whole(text="detail")
module.detail = lambda get, slug, row: {"location": "Aarhus", "employment": "contract",
"description": "Full."}
rows, done, failed = _runner.read_details(module, "fake", "acme", Web(), [_row("1")],
set(), set(), limits=NO_DELAY)
assert rows[0]["places"] == [{"text": "Aarhus", "remote": None}]
assert (rows[0]["employment"], rows[0]["description"]) == ("contract", "Full.")
assert (done, failed) == ({"fake:acme:1"}, set())