"""The host modules of `plugins/atshosts/` and what `plugins/ats.py` does for every host: the registry and its contract check, the fetch a host is handed, the per-role detail reads, robots.txt, and Lever's EU instance. No network — a fake host stands in where a behaviour belongs to no real host yet.""" from __future__ import annotations import json import sys import types from collections.abc import Callable from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any import httpx import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from plugins import ats, atshosts # noqa: E402 from plugins.atshosts import lever # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" # --------------------------------------------------------------- registry def test_the_registry_finds_every_module() -> None: names = sorted(p.stem for p in (ROOT / "plugins" / "atshosts").glob("*.py") if not p.stem.startswith("_")) assert sorted(atshosts.HOSTS) == names assert set(names) >= {"greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee", "personio", "teamtailor"} assert all(not atshosts.problems(module, name) for name, module in atshosts.HOSTS.items()) # The Sources page's select and the plugin's description follow the modules. assert {name: module.LABEL for name, module in atshosts.HOSTS.items()} == ats.HOSTS assert "Teamtailor" in ats.ATSSource.description def _package(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str, modules: dict[str, str]) -> list[str]: package = tmp_path / name package.mkdir() (package / "__init__.py").write_text("") for module, source in modules.items(): (package / f"{module}.py").write_text(source) monkeypatch.syspath_prepend(str(tmp_path)) return [str(package)] _GOOD = """ NAME = "{name}" LABEL = "Good" PATTERNS = (r"good\\.example/([a-z]+)",) def slug_ok(slug): return True def allowed(slug): return lambda h: h == "good.example" def careers_url(slug): return "https://good.example/" + slug def feed_url(slug): return "https://good.example/" + slug + ".json" def parse(body, slug): return [] """ def test_the_registry_loads_a_package_by_name_and_skips_private_modules( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: path = _package(tmp_path, monkeypatch, "hostsok", { "zeta": _GOOD.format(name="zeta"), "alpha": _GOOD.format(name="alpha"), "_helpers": "X = 1\n"}) assert list(atshosts.load(path, "hostsok")) == ["alpha", "zeta"] def test_the_registry_orders_by_rank_then_name( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: path = _package(tmp_path, monkeypatch, "hostsranked", { "alpha": _GOOD.format(name="alpha"), # the default, 100 "beta": _GOOD.format(name="beta") + "RANK = 5\n", "gamma": _GOOD.format(name="gamma") + "RANK = 100\n"}) assert list(atshosts.load(path, "hostsranked")) == ["beta", "alpha", "gamma"] def test_the_eight_first_hosts_keep_the_order_they_had_before_modules() -> None: """The Sources page's select opens on Greenhouse, which its hint names, and a guessed slug on two hosts goes to the one probed first, as before.""" first = ["greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee", "personio", "teamtailor"] assert list(ats.HOSTS)[:8] == first assert ats.ATSSource.description.startswith( "Company job boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, " "Recruitee, Personio, Teamtailor") assert all(module.RANK < atshosts.DEFAULT_RANK for name, module in atshosts.HOSTS.items() if name in first) def test_the_registry_refuses_a_module_missing_a_required_name( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: broken = _GOOD.format(name="broken").replace("def careers_url", "def not_careers_url") path = _package(tmp_path, monkeypatch, "hostsbad", {"broken": broken}) with pytest.raises(ImportError, match=r"hostsbad\.broken .*no careers_url"): atshosts.load(path, "hostsbad") def test_the_contract_check_names_each_problem() -> None: module = types.ModuleType("odd") module.NAME, module.LABEL, module.PATTERNS = "other", "", ("(",) module.slug_ok = module.allowed = module.careers_url = lambda slug: "" module.page = lambda get, slug, cursor: None module.probe_ok = lambda body: True module.RANK, module.KEEP_CASE = "1", "yes" found = atshosts.problems(module, "odd") assert "NAME is 'other', not the module's name 'odd'" in found assert "LABEL is not a non-empty string" in found assert any(p.startswith("PATTERNS has '('") for p in found) assert "page without rows" in found assert "probe_ok without probe_url" in found assert "RANK is not an int" in found and "KEEP_CASE is not a bool" in found @pytest.mark.parametrize("host", ["greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee", "personio", "teamtailor"]) def test_a_slug_is_matched_whole(host: str) -> None: assert ats.is_known_target(f"{host}:acme") for slug in ("acme\n", "acme\r\n", " acme", "acme ", "ac me", "acme/x", "acme?x", "acme#x"): assert not ats.is_known_target(f"{host}:{slug}"), slug def test_a_slug_in_the_host_name_is_allowed_whatever_its_case() -> None: for host, api in (("recruitee", "acme.recruitee.com"), ("personio", "acme.jobs.personio.de")): allow = ats._allowed_host(host, "Acme") assert allow(api) and allow(api.upper()) assert not allow(api.replace("acme", "other")) # ------------------------------------------------------------ the fetch def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client: return httpx.Client(transport=httpx.MockTransport(handler)) def test_get_sends_extra_headers_on_top_of_its_own() -> None: seen: list[httpx.Headers] = [] def handler(request: httpx.Request) -> httpx.Response: seen.append(request.headers) return httpx.Response(200, content=b"ok") with _client(handler) as client: get = ats._getter(client, "lever", "acme") assert get("https://api.lever.co/v0/postings/acme", headers={"Accept": "text/plain", "X-Board": "acme"}) == b"ok" get("https://api.lever.co/v0/postings/acme") assert seen[0]["accept"] == "text/plain" and seen[0]["x-board"] == "acme" assert seen[0]["user-agent"] == seen[1]["user-agent"] == ats.USER_AGENT assert seen[1]["accept"] == "application/json, text/xml" assert "x-board" not in seen[1] def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatch) -> None: """Bound before the replacement, and still reaching it: `_get` is looked up when a request is made.""" client = object() get = ats._getter(client, "lever", "acme") # type: ignore[arg-type] calls: list[tuple[Any, ...]] = [] def replaced(*args: Any, **kwargs: Any) -> bytes: calls.append((args, kwargs)) return b"" monkeypatch.setattr(ats, "_get", replaced) get("https://api.lever.co/v0/postings/acme", headers={"X": "1"}) get("https://api.lever.co/v0/postings/acme", method="POST", payload={}) assert calls == [ ((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"), {"method": "GET", "headers": {"X": "1"}}), ((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"), {"method": "POST", "payload": {}})] # ------------------------------------------------------- a fake host def _fake_host(*, detail: Callable[..., Any] | None = None, robots: bool = False) -> types.ModuleType: """A host `fake` whose feed is a JSON list of ids, each role with a teaser for a description, and, with `detail`, a per-role page; with `robots`, its listing is the page robots.txt must allow.""" module = types.ModuleType("fake") module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () module.slug_ok = lambda slug: slug.isalnum() module.allowed = lambda slug: lambda h: h == "jobs.fake.example" module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list" module.parse = lambda body, slug: [ {"source_id": i, "title": f"Role {i}", "location": "", "department": "", "url": f"https://jobs.fake.example/{slug}/{i}", "posted": None, "description": "teaser", "remote": None, "salary_text": ""} for i in json.loads(body)] if detail is not None: module.detail = detail if robots: module.robots_page = lambda slug: f"https://jobs.fake.example/{slug}/list" return module def _status(code: int, url: str = "https://jobs.fake.example/") -> httpx.HTTPStatusError: request = httpx.Request("GET", url) return httpx.HTTPStatusError(str(code), request=request, response=httpx.Response(code, request=request)) @pytest.fixture def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: """`ats._get` answering from a dict of URL -> body or exception, and remembering what it was asked. No delay between requests.""" answers: dict[str, Any] = {} asked: list[str] = [] def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None, www_twin=False): asked.append(url) if not ats._allowed_host(host, slug)(httpx.URL(url).host): raise ats._Refused(url) answer = answers.get(url, _status(404, url)) if isinstance(answer, BaseException): raise answer return answer monkeypatch.setattr(ats, "_get", fake_get) monkeypatch.setattr(ats, "PAGE_DELAY", 0) return {"answers": answers, "asked": asked} def _run(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType, state: dict[str, str] | None = None) -> tuple[list[Any], dict[str, str]]: monkeypatch.setitem(atshosts.HOSTS, "fake", module) source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"}) if state is not None: source.state = state return list(source.fetch()), source.state def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]: """The fake host's detail: the role's own page is its description.""" return {"description": get(f"https://jobs.fake.example/{slug}/{row['source_id']}").decode(), "location": "Copenhagen"} # ---------------------------------------------------------------- detail def test_detail_is_read_for_unread_roles_only( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one in full", f"{base}/2": b"two in full", f"{base}/3": b"three in full"}) module = _fake_host(detail=_detail) state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"])} items, state = _run(monkeypatch, module, state) assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/3"] assert [(i.source_id, i.body) for i in items] == [ ("fake:acme:1", "one in full"), ("fake:acme:2", ""), ("fake:acme:3", "three in full")] # What detail returns replaces the listing's field; a role not read keeps the listing's. assert items[0].raw["place"] == "Copenhagen" and items[1].raw["place"] == "" assert json.loads(state["ats:detailed:fake:acme"]) == [ "fake:acme:1", "fake:acme:2", "fake:acme:3"] assert state["ats:complete:fake:acme"] == "1" def test_detail_reads_stop_at_the_cap_and_the_rest_wait( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" ids = [str(n) for n in range(5)] web["answers"].update({f"{base}/list": json.dumps(ids).encode()} | {f"{base}/{i}": f"full {i}".encode() for i in ids}) monkeypatch.setattr(ats, "DETAIL_CAP", 2) module = _fake_host(detail=_detail) items, state = _run(monkeypatch, module) assert [i.body for i in items] == ["full 0", "full 1", "", "", ""] assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:0", "fake:acme:1"] web["asked"].clear() items, state = _run(monkeypatch, module, state) assert web["asked"] == [f"{base}/list", f"{base}/2", f"{base}/3"] assert [i.body for i in items] == ["", "", "full 2", "full 3", ""] def test_a_detail_that_fails_skips_the_role_and_not_the_board( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one", f"{base}/2": ValueError("no such page shape"), f"{base}/3": b"three"}) items, state = _run(monkeypatch, _fake_host(detail=_detail)) assert [(i.source_id, i.body) for i in items] == [ ("fake:acme:1", "one"), ("fake:acme:2", ""), ("fake:acme:3", "three")] assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:3"] assert "ats:failing:fake:acme" not in state and state["ats:ok:fake:acme"] == "r" assert state["ats:complete:fake:acme"] == "1" def test_a_429_on_a_detail_stops_the_reads_for_the_pull( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one", f"{base}/2": _status(429), f"{base}/3": b"three"}) items, state = _run(monkeypatch, _fake_host(detail=_detail)) assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"] assert [i.body for i in items] == ["one", "", ""] assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"] assert state["ats:complete:fake:acme"] == "1" def test_ids_no_longer_listed_leave_the_detailed_list( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["2"]'}) state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} _items, state = _run(monkeypatch, _fake_host(detail=_detail), state) assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"] def test_an_incomplete_listing_keeps_the_ids_it_did_not_list( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: """A listing cut short says nothing about the roles it left out, so their ids stay, and the next pull does not read them in detail again.""" base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["2"]'}) monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: ( atshosts.HOSTS["fake"].parse(b'["2"]', "acme"), False)) state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} _items, state = _run(monkeypatch, _fake_host(detail=_detail), state) assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"] assert state["ats:complete:fake:acme"] == "" def test_fetch_board_reads_the_listing_only( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": b"one"}) monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail)) rows, complete = ats.fetch_board("fake:acme") assert ([r["description"] for r in rows], complete) == (["teaser"], True) assert web["asked"] == [f"{base}/list"] def test_a_host_without_detail_keeps_the_listings_description( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]' items, state = _run(monkeypatch, _fake_host()) assert [i.body for i in items] == ["teaser"] assert "ats:detailed:fake:acme" not in state def test_a_detail_answering_404_or_410_counts_as_read( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": _status(410), f"{base}/3": b"three"}) # 1 answers 404 items, state = _run(monkeypatch, _fake_host(detail=_detail)) assert [i.body for i in items] == ["", "", "three"] assert json.loads(state["ats:detailed:fake:acme"]) == [ "fake:acme:1", "fake:acme:2", "fake:acme:3"] assert json.loads(state["ats:detailfailed:fake:acme"]) == [] @pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")]) def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None: """Two roles that always fail and a cap of two: the third is read on the second pull, since roles never tried go before those that failed.""" base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["1", "2", "3", "4"]', f"{base}/1": failure, f"{base}/2": failure, f"{base}/3": b"three", f"{base}/4": b"four"}) monkeypatch.setattr(ats, "DETAIL_CAP", 2) module = _fake_host(detail=_detail) _items, state = _run(monkeypatch, module) assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"] web["asked"].clear() items, state = _run(monkeypatch, module, state) assert web["asked"] == [f"{base}/list", f"{base}/3", f"{base}/4"] assert [i.body for i in items] == ["", "", "three", "four"] # With nothing new left, the failed ones are tried again, within the cap. web["asked"].clear() web["answers"][f"{base}/1"] = b"one at last" items, state = _run(monkeypatch, module, state) assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"] assert [i.body for i in items] == ["one at last", "", "", ""] assert json.loads(state["ats:detailed:fake:acme"]) == [ "fake:acme:1", "fake:acme:3", "fake:acme:4"] assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"] def test_a_429_on_a_detail_does_not_mark_the_role_as_failing( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": _status(429)}) _items, state = _run(monkeypatch, _fake_host(detail=_detail)) assert json.loads(state["ats:detailfailed:fake:acme"]) == [] assert json.loads(state["ats:detailed:fake:acme"]) == [] def test_failed_ids_no_longer_listed_leave_the_failed_list( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["2"]', f"{base}/2": ValueError("odd page")}) state = {"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} _items, state = _run(monkeypatch, _fake_host(detail=_detail), state) assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"] def test_detail_fills_only_the_fields_a_later_pull_keeps( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], caplog: pytest.LogCaptureFixture) -> None: """`salary_text`, `department` and `title` would be rewritten from the listing on the next pull, so detail does not fill them.""" base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["1"]'}) def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]: return {"description": "in full", "remote": "hybrid", "currency": "EUR", "salary_min": 60000, "salary_max": 70000, "posted": datetime(2026, 9, 1, tzinfo=UTC), "salary_text": "EUR 60,000 to 70,000 per year", "department": "Sales", "title": "Another title"} caplog.set_level("DEBUG", logger=ats.log.name) [item], _state = _run(monkeypatch, _fake_host(detail=detail)) assert (item.body, item.title, item.published_at) == ( "in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC)) assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"], item.raw["salary_max"]) == ("hybrid", "EUR", 60000, 70000) assert (item.raw["salary_text"], item.raw["department"]) == ("", "") assert "department, salary_text, title left out" in caplog.text assert set(ats.DETAIL_FIELDS) == {"description", "location", "remote", "currency", "salary_min", "salary_max", "posted"} def _paged_host() -> types.ModuleType: """A host `fake` that pages its listing, one id a page, and has `detail`.""" module = _fake_host(detail=_detail) del module.feed_url, module.parse def page(get: Any, slug: str, cursor: Any) -> tuple[Any, list[dict[str, Any]], Any, Any]: index = int(cursor or 0) ids = json.loads(get(f"https://jobs.fake.example/{slug}/list?page={index}")) return ids, [{"id": i} for i in ids], 2, index + 1 if ids else None module.page = page module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug) assert not atshosts.problems(module, "fake") return module def test_a_paged_host_reads_every_page_then_each_role_once( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": b'["2"]', f"{base}/1": b"one", f"{base}/2": b"two"}) items, state = _run(monkeypatch, _paged_host()) assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1", f"{base}/1", f"{base}/2"] assert [i.body for i in items] == ["one", "two"] assert state["ats:complete:fake:acme"] == "1" web["asked"].clear() items, state = _run(monkeypatch, _paged_host(), state) assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1"] assert [i.body for i in items] == ["", ""] def test_a_paged_listing_cut_short_keeps_the_ids_it_did_not_reach( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": _status(429), f"{base}/1": b"one"}) state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]), "ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])} _items, state = _run(monkeypatch, _paged_host(), state) assert state["ats:complete:fake:acme"] == "" assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"] assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:3"] def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: """The consumer stops after the first role, so the second, read but not stored, is read again on the next pull.""" base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"}) monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail)) source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"}) roles = source.fetch() assert next(roles).body == "one" roles.close() assert "ats:detailed:fake:acme" not in source.state # ------------------------------------------------------------ robots.txt ROBOTS = "https://jobs.fake.example/robots.txt" def test_robots_that_allow_the_page_let_the_board_be_read( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /admin/\n", "https://jobs.fake.example/acme/list": b'["1"]'}) items, state = _run(monkeypatch, _fake_host(robots=True)) assert len(items) == 1 assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True @pytest.mark.parametrize("robots", [ b"User-agent: *\nDisallow: /acme/\n", # A rule for this app by name counts as well as the one for everyone. f"User-agent: {ats.USER_AGENT.split('/')[0]}\nDisallow: /\n\nUser-agent: *\nAllow: /\n" .encode(), ]) def test_robots_that_disallow_the_page_fail_the_board( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], robots: bytes) -> None: web["answers"].update({ROBOTS: robots, "https://jobs.fake.example/acme/list": b'["1"]'}) items, state = _run(monkeypatch, _fake_host(robots=True)) assert items == [] assert web["asked"] == [ROBOTS] # the listing is never fetched assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it" assert "ats:ok:fake:acme" not in state @pytest.mark.parametrize(("robots", "page", "allowed"), [ # The longest matching rule wins, whatever the order of the lines. ("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", "https://jobs.infineon.com/api/pcsx/search?domain=infineon.com", True), ("User-agent: *\nAllow: /careers\nDisallow: /\n", "https://x.example/careers/list", True), ("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", "https://x.example/other", False), # An allow wins a tie. ("User-agent: *\nDisallow: /jobs\nAllow: /jobs\n", "https://x.example/jobs", True), # `*` inside a path, and the query counts as part of it. ("User-agent: *\nDisallow: /*company\n", "https://career5.successfactors.eu/career?company=x", False), # `$` anchors the end. ("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json", False), ("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json?page=2", True), # An empty Disallow allows. ("User-agent: *\nDisallow:\n", "https://x.example/anything", True), # A byte order mark and CRLF line ends. ("\ufeffUser-agent: *\r\nDisallow: /\r\n", "https://x.example/jobs", False), # Paths are case-sensitive. ("User-agent: *\nDisallow: /Novonesis_Careers/\n", "https://novonesis.wd103.myworkdayjobs.com/Novonesis_Careers/", False), ("User-agent: *\nDisallow: /Novonesis_Careers/\n", "https://novonesis.wd103.myworkdayjobs.com/novonesis_careers/", True), # A group for this app by name counts, a group for someone else does # not, and a token matches whole, not as a substring. (f"User-agent: {ats.ROBOTS_TOKEN.upper()}\nDisallow: /\n\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", False), ("User-agent: Googlebot\nDisallow: /\n", "https://x.example/jobs", True), ("User-agent: sk\nDisallow: /\n\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True), # Two agents share one group, and a comment is no rule. ("User-agent: a\nUser-agent: *\nDisallow: /jobs # not these\n", "https://x.example/jobs/1", False), # A group for this app that allows does not undo the `*` group's refusal. (f"User-agent: {ats.ROBOTS_TOKEN}\nAllow: /\n\nUser-agent: *\nDisallow: /\n", "https://x.example/jobs", False), # Rules before any user-agent line belong to no group. ("Disallow: /\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True), ]) def test_robots_rules_follow_rfc_9309(robots: str, page: str, allowed: bool) -> None: assert ats._robots_allows(robots.encode(), page) is allowed def test_a_robots_verdict_is_kept_for_a_day( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n", "https://jobs.fake.example/acme/list": b'["1"]'}) module = _fake_host(robots=True) _items, state = _run(monkeypatch, module) assert web["asked"] == [ROBOTS] # Within the day the kept verdict holds, even though the site changed. web["answers"][ROBOTS] = b"User-agent: *\nAllow: /\n" web["asked"].clear() items, state = _run(monkeypatch, module, state) assert (items, web["asked"]) == ([], []) # A day on, robots.txt is read again. kept = json.loads(state["ats:robots:fake:acme"]) kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat() state["ats:robots:fake:acme"] = json.dumps(kept) items, state = _run(monkeypatch, module, state) assert web["asked"] == [ROBOTS, "https://jobs.fake.example/acme/list"] assert len(items) == 1 and "ats:failing:fake:acme" not in state @pytest.mark.parametrize("code", [404, 410, 401, 403]) def test_a_robots_txt_answering_4xx_allows_and_is_kept( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], code: int) -> None: web["answers"].update({ROBOTS: _status(code), "https://jobs.fake.example/acme/list": b'["1"]'}) items, state = _run(monkeypatch, _fake_host(robots=True)) assert len(items) == 1 assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True @pytest.mark.parametrize("failure", [ _status(500), _status(503), _status(429), httpx.ConnectTimeout("timed out"), httpx.ConnectError("refused"), ats._Refused("https://elsewhere.example/robots.txt"), ]) def test_a_robots_txt_that_cannot_be_read_fails_the_board_and_is_asked_again( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None: web["answers"].update({ROBOTS: failure, "https://jobs.fake.example/acme/list": b'["1"]'}) items, state = _run(monkeypatch, _fake_host(robots=True)) assert items == [] and web["asked"] == [ROBOTS] assert json.loads(state["ats:failing:fake:acme"])["reason"] == \ "robots.txt could not be read" assert "ats:robots:fake:acme" not in state def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]: """A site whose robots.txt on jobs.fake.example redirects to `location`, where a robots.txt that disallows everything is served.""" def handler(request: httpx.Request) -> httpx.Response: if str(request.url) == ROBOTS: return httpx.Response(301, headers={"location": location}) if request.url.path == "/robots.txt": return httpx.Response(200, text="User-agent: *\nDisallow: /\n") return httpx.Response(200, text='["1"]') return handler def test_a_robots_txt_redirect_to_the_www_twin_is_followed( monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \ pytest.raises(ats._Disallowed): ats._check_robots(client, "fake:acme", {}) @pytest.mark.parametrize("location", [ "https://evil.example/robots.txt", "https://other.fake.example/robots.txt", "http://www.jobs.fake.example/robots.txt", ]) def test_a_robots_txt_redirect_anywhere_else_is_unreachable( monkeypatch: pytest.MonkeyPatch, location: str) -> None: monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable): ats._check_robots(client, "fake:acme", {}) def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \ pytest.raises(ats._Refused): ats._get(client, ROBOTS, "fake", "acme") def test_fetch_board_honours_robots_txt( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n", "https://jobs.fake.example/acme/list": b'["1"]'}) monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) with pytest.raises(ats._Disallowed) as caught: ats.fetch_board("fake:acme") assert ats.failure_reason(caught.value) == "robots.txt disallows it" # ----------------------------------------------------------------- Lever EU def test_a_lever_slug_may_name_the_eu_instance() -> None: assert ats.is_known_target("lever:acme") and ats.is_known_target("lever:eu/tomtom") assert not ats.is_known_target("lever:us/acme") assert not ats.is_known_target("lever:eu/") assert not ats.is_known_target("lever:eu/a/b") assert ats.careers_url("lever:acme") == "https://jobs.lever.co/acme" assert ats.careers_url("lever:eu/tomtom") == "https://jobs.eu.lever.co/tomtom" assert lever.feed_url("eu/tomtom") == "https://api.eu.lever.co/v0/postings/tomtom?mode=json" eu = ats._allowed_host("lever", "eu/tomtom") assert eu("api.eu.lever.co") and not eu("api.lever.co") assert not ats._allowed_host("lever", "tomtom")("api.eu.lever.co") def test_a_lever_eu_board_is_read_from_the_eu_api(monkeypatch: pytest.MonkeyPatch) -> None: urls: list[str] = [] def fake_get(client, url, host, slug, *, method="GET"): urls.append(url) assert ats._allowed_host(host, slug)(httpx.URL(url).host) return (FIXTURES / "lever.json").read_bytes() monkeypatch.setattr(ats, "_get", fake_get) rows, complete = ats._fetch_rows(object(), "lever:eu/tomtom") # type: ignore[arg-type] assert urls == ["https://api.eu.lever.co/v0/postings/tomtom?mode=json"] assert rows and complete def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None: def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(302, headers={"location": "https://api.lever.co/v0/postings/x"}) with _client(handler) as client, pytest.raises(ats._Refused): ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x")