From 8b14d8575b356977a16dafddb7d46aed4dd4893a Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 17:05:44 +0200 Subject: [PATCH] Read a role's own page once for a host with detail, and fail a board whose robots.txt disallows it Co-Authored-By: Claude Opus 5.5 --- plugins/ats.py | 159 ++++++++++++++- tests/test_atshosts.py | 425 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 578 insertions(+), 6 deletions(-) create mode 100644 tests/test_atshosts.py diff --git a/plugins/ats.py b/plugins/ats.py index bc18502..5c3f94c 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -6,7 +6,8 @@ unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that drops off the board is *closed*, which is the cheapest staleness check there is. What a host says and how its feed reads is its module's; this file holds what every host shares: the fetch and its allow-list, paging and -completeness, the classifier, and the plugin itself. +completeness, the per-role detail reads, robots.txt, the classifier, and the +plugin itself. The allow-list and the classifier are ported from `openings.go` and `cmd/seedgen/main.go` in @@ -32,9 +33,10 @@ import json import logging import re import time -from collections.abc import Callable, Iterator -from datetime import UTC, datetime +from collections.abc import Callable, Iterator, MutableMapping +from datetime import UTC, datetime, timedelta from typing import Any +from urllib.robotparser import RobotFileParser import httpx @@ -110,6 +112,11 @@ class _TooLarge(ValueError): retry at the host's `fallback_url`.""" +class _Disallowed(Exception): + """The site's robots.txt disallows the page a board is read from (see + `_check_robots`).""" + + def _get(client: httpx.Client, url: str, host: str, slug: str, *, method: str = "GET", payload: dict[str, Any] | None = None, headers: dict[str, str] | None = None) -> bytes: @@ -329,7 +336,8 @@ def careers_url(target: str) -> str: #: Pages read from one paged board at most, and the pause between two of #: them. Workable answers ten roles a page, SmartRecruiters a hundred and #: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause -#: keeps Workable, which answers a burst with 429, from seeing one. +#: keeps Workable, which answers a burst with 429, from seeing one. The same +#: pause separates two detail reads. MAX_PAGES = 100 PAGE_DELAY = 0.5 @@ -408,18 +416,142 @@ def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]: return module.parse(body, slug), True +# -------------------------------------------------------------- robots.txt + +#: How long a robots.txt answer holds for one board before it is read again. +ROBOTS_TTL = timedelta(days=1) + + +def _robots_allows(body: bytes, page: str) -> bool: + """Whether a robots.txt lets `page` be fetched: by its rules for every + agent (`*`), and by any it has for this app's own User-Agent token, so a + site that shuts out this app by name is honoured as well.""" + parser = RobotFileParser() + parser.parse(body.decode("utf-8", "replace").splitlines()) + return parser.can_fetch("*", page) and parser.can_fetch(USER_AGENT, page) + + +def _check_robots(client: httpx.Client, target: str, + state: MutableMapping[str, str] | None = None) -> None: + """Raise `_Disallowed` when the site of a host with a `robots_page` has a + robots.txt that disallows that page. Read through `_get`, so from the + board's own allowed hosts only, and at most once per `ROBOTS_TTL` for a + board when `state` is given: the verdict and when it was reached are kept + under `ats:robots:`. A robots.txt that is missing (404 or 410) + allows, and that verdict is kept like any other; one that cannot be read + for another reason allows too, but is not kept, so the next pull asks + again.""" + host, _, slug = target.partition(":") + module = atshosts.HOSTS.get(host) + if module is None or not hasattr(module, "robots_page") or not is_known_target(target): + return + page = module.robots_page(slug) + key = f"ats:robots:{target}" + now = datetime.now(UTC) + if state is not None: + try: + kept = json.loads(state.get(key) or "{}") + fresh = now - datetime.fromisoformat(kept["checked"]) < ROBOTS_TTL + except (ValueError, KeyError, TypeError): + kept, fresh = {}, False + if fresh: + if not kept.get("allowed", True): + raise _Disallowed(f"robots.txt disallows {page}") + return + url = f"https://{httpx.URL(page).host}/robots.txt" + try: + body = _getter(client, host, slug)(url, headers={"Accept": "text/plain"}) + except httpx.HTTPStatusError as exc: + if exc.response.status_code not in (404, 410): + log.warning("ats: %s: robots.txt answered %s; allowed until the next pull", + target, exc.response.status_code) + return + allowed = True + except Exception as exc: + log.warning("ats: %s: robots.txt not read (%s); allowed until the next pull", + target, exc) + return + else: + allowed = _robots_allows(body, page) + if state is not None: + state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"), + "allowed": allowed}) + if not allowed: + raise _Disallowed(f"robots.txt disallows {page}") + + +# ---------------------------------------------------------------- details + +#: Roles read in detail from one board in one pull at most. The rest of a +#: large new board is read over the next pulls, 200 at a time. +DETAIL_CAP = 200 + + +def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool, + detailed: set[str]) -> tuple[list[Row], set[str]]: + """The rows of a board whose host has `detail`, each role not yet in + `detailed` (compound ids) filled from its own page, at most `DETAIL_CAP` + of them and `PAGE_DELAY` apart. Returns the rows and the ids read in + detail so far, for `ats:detailed:`. + + Every row not read in detail in this call goes out with an empty + description: a role read on an earlier pull keeps the one it has, since + `upsert_role` never overwrites a filled column with an empty value, and a + teaser from the listing must not replace it. A detail that raises is + logged and the role left for the next pull; a 429 stops the reads for this + board until then. None of this touches completeness, which is the + listing's alone. The ids of roles no longer listed are dropped, but only + after a complete listing, so one cut short does not make the next pull + read its missing roles again.""" + host, _, slug = target.partition(":") + module = atshosts.HOSTS[host] + get = _getter(client, host, slug) + done = set(detailed) + out: list[Row] = [] + tried = 0 + stopped = False + for row in rows: + compound = f"{host}:{slug}:{row['source_id']}" + if compound in done or stopped or tried >= DETAIL_CAP: + out.append(row | {"description": ""}) + continue + time.sleep(PAGE_DELAY) + tried += 1 + try: + filled = module.detail(get, slug, row) + except Exception as exc: + if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 429: + log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull", + target) + stopped = True + else: + log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc) + out.append(row | {"description": ""}) + continue + out.append(row | filled | {"source_id": row["source_id"]}) + done.add(compound) + if complete: + done &= {f"{host}:{slug}:{row['source_id']}" for row in rows} + return out, done + + def fetch_board(target: str) -> tuple[list[Row], bool]: """Fetch and parse one board directly: no `Source`, no `self.state`, just "does this board answer, what does it list, and is that all of it". For `app companies add|check`, which ask that question about one board at a - time outside any pull. Raises the same way `_fetch_rows` does.""" + time outside any pull. The listing only, with no detail reads, and + robots.txt read afresh. Raises the same way `_fetch_rows` does, and + `_Disallowed` for a board robots.txt shuts out.""" with httpx.Client() as client: + _check_robots(client, target) return _fetch_rows(client, target) def failure_reason(exc: BaseException) -> str: """One short line for why a board did not answer: the status of an HTTP error, else the exception's class and the first line of its message.""" + if isinstance(exc, _Disallowed): + return "robots.txt disallows it" if isinstance(exc, httpx.HTTPStatusError): return f"{exc.response.status_code} {exc.response.reason_phrase}".strip() lines = str(exc).strip().splitlines() @@ -465,6 +597,7 @@ class ATSSource(Source): host, _, slug = target.partition(":") failing = f"ats:failing:{target}" try: + _check_robots(client, target, self.state) rows, complete = _fetch_rows(client, target) except Exception as exc: log.warning("ats: %s failed (%s)", target, exc) @@ -481,6 +614,15 @@ class ATSSource(Source): continue self.state.pop(failing, None) + detailed_key = f"ats:detailed:{target}" + detailed: set[str] | None = None + if hasattr(atshosts.HOSTS.get(host), "detail"): + try: + detailed = set(json.loads(self.state.get(detailed_key) or "[]")) + except (ValueError, TypeError): + detailed = set() + rows, detailed = _read_details(client, target, rows, complete, detailed) + seen_ids = [] for row in rows: # The compound id, matching `roles.source_id` — what @@ -520,7 +662,12 @@ class ATSSource(Source): # board that blew up half way through is neither marked ok nor # has a partial id list treated as the whole of it. `complete` # says whether the ids are the whole board, which the pull - # needs before it closes what they leave out. + # needs before it closes what they leave out. The ids read in + # detail are kept here too, after every row went out, so a + # pull stopped half way reads the rest again rather than + # losing their descriptions. self.state[f"ats:ok:{target}"] = run_marker self.state[f"ats:complete:{target}"] = "1" if complete else "" self.state[f"ats:ids:{target}"] = json.dumps(seen_ids) + if detailed is not None: + self.state[detailed_key] = json.dumps(sorted(detailed)) diff --git a/tests/test_atshosts.py b/tests/test_atshosts.py new file mode 100644 index 0000000..4965d6d --- /dev/null +++ b/tests/test_atshosts.py @@ -0,0 +1,425 @@ +"""The host modules of `plugins/atshosts/` and what `plugins/ats.py` does for +every host: the registry and its contract check, the fetch a host is handed, +the per-role detail reads, robots.txt, and Lever's EU instance. No network — +a fake host stands in where a behaviour belongs to no real host yet.""" + +from __future__ import annotations + +import json +import sys +import types +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import httpx +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from plugins import ats, atshosts # noqa: E402 +from plugins.atshosts import lever # noqa: E402 + +FIXTURES = ROOT / "tests" / "fixtures" / "ats" + + +# --------------------------------------------------------------- registry + +def test_the_registry_finds_every_module_in_order_of_name() -> None: + names = sorted(p.stem for p in (ROOT / "plugins" / "atshosts").glob("*.py") + if not p.stem.startswith("_")) + assert list(atshosts.HOSTS) == names + assert set(names) >= {"greenhouse", "lever", "ashby", "workable", "smartrecruiters", + "recruitee", "personio", "teamtailor"} + assert all(not atshosts.problems(module, name) for name, module in atshosts.HOSTS.items()) + # The Sources page's select and the plugin's description follow the modules. + assert {name: module.LABEL for name, module in atshosts.HOSTS.items()} == ats.HOSTS + assert "Teamtailor" in ats.ATSSource.description + + +def _package(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str, + modules: dict[str, str]) -> list[str]: + package = tmp_path / name + package.mkdir() + (package / "__init__.py").write_text("") + for module, source in modules.items(): + (package / f"{module}.py").write_text(source) + monkeypatch.syspath_prepend(str(tmp_path)) + return [str(package)] + + +_GOOD = """ +NAME = "{name}" +LABEL = "Good" +PATTERNS = (r"good\\.example/([a-z]+)",) +def slug_ok(slug): return True +def allowed(slug): return lambda h: h == "good.example" +def careers_url(slug): return "https://good.example/" + slug +def feed_url(slug): return "https://good.example/" + slug + ".json" +def parse(body, slug): return [] +""" + + +def test_the_registry_loads_a_package_by_name_and_skips_private_modules( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + path = _package(tmp_path, monkeypatch, "hostsok", { + "zeta": _GOOD.format(name="zeta"), "alpha": _GOOD.format(name="alpha"), + "_helpers": "X = 1\n"}) + assert list(atshosts.load(path, "hostsok")) == ["alpha", "zeta"] + + +def test_the_registry_refuses_a_module_missing_a_required_name( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + broken = _GOOD.format(name="broken").replace("def careers_url", "def not_careers_url") + path = _package(tmp_path, monkeypatch, "hostsbad", {"broken": broken}) + with pytest.raises(ImportError, match=r"hostsbad\.broken .*no careers_url"): + atshosts.load(path, "hostsbad") + + +def test_the_contract_check_names_each_problem() -> None: + module = types.ModuleType("odd") + module.NAME, module.LABEL, module.PATTERNS = "other", "", ("(",) + module.slug_ok = module.allowed = module.careers_url = lambda slug: "" + module.page = lambda get, slug, cursor: None + module.probe_ok = lambda body: True + found = atshosts.problems(module, "odd") + assert "NAME is 'other', not the module's name 'odd'" in found + assert "LABEL is not a non-empty string" in found + assert any(p.startswith("PATTERNS has '('") for p in found) + assert "page without rows" in found + assert "probe_ok without probe_url" in found + + +# ------------------------------------------------------------ the fetch + +def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client: + return httpx.Client(transport=httpx.MockTransport(handler)) + + +def test_get_sends_extra_headers_on_top_of_its_own() -> None: + seen: list[httpx.Headers] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request.headers) + return httpx.Response(200, content=b"ok") + + with _client(handler) as client: + get = ats._getter(client, "lever", "acme") + assert get("https://api.lever.co/v0/postings/acme", + headers={"Accept": "text/plain", "X-Board": "acme"}) == b"ok" + get("https://api.lever.co/v0/postings/acme") + assert seen[0]["accept"] == "text/plain" and seen[0]["x-board"] == "acme" + assert seen[0]["user-agent"] == seen[1]["user-agent"] == ats.USER_AGENT + assert seen[1]["accept"] == "application/json, text/xml" + assert "x-board" not in seen[1] + + +def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatch) -> None: + """Bound before the replacement, and still reaching it: `_get` is looked + up when a request is made.""" + client = object() + get = ats._getter(client, "lever", "acme") # type: ignore[arg-type] + calls: list[tuple[Any, ...]] = [] + + def replaced(*args: Any, **kwargs: Any) -> bytes: + calls.append((args, kwargs)) + return b"" + + monkeypatch.setattr(ats, "_get", replaced) + get("https://api.lever.co/v0/postings/acme", headers={"X": "1"}) + get("https://api.lever.co/v0/postings/acme", method="POST", payload={}) + assert calls == [ + ((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"), + {"method": "GET", "headers": {"X": "1"}}), + ((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"), + {"method": "POST", "payload": {}})] + + +# ------------------------------------------------------- a fake host + +def _fake_host(*, detail: Callable[..., Any] | None = None, + robots: bool = False) -> types.ModuleType: + """A host `fake` whose feed is a JSON list of ids, each role with a + teaser for a description, and, with `detail`, a per-role page; with + `robots`, its listing is the page robots.txt must allow.""" + module = types.ModuleType("fake") + module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () + module.slug_ok = lambda slug: slug.isalnum() + module.allowed = lambda slug: lambda h: h == "jobs.fake.example" + module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" + module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list" + module.parse = lambda body, slug: [ + {"source_id": i, "title": f"Role {i}", "location": "", "department": "", + "url": f"https://jobs.fake.example/{slug}/{i}", "posted": None, + "description": "teaser", "remote": None, "salary_text": ""} + for i in json.loads(body)] + if detail is not None: + module.detail = detail + if robots: + module.robots_page = lambda slug: f"https://jobs.fake.example/{slug}/list" + return module + + +def _status(code: int, url: str = "https://jobs.fake.example/") -> httpx.HTTPStatusError: + request = httpx.Request("GET", url) + return httpx.HTTPStatusError(str(code), request=request, + response=httpx.Response(code, request=request)) + + +@pytest.fixture +def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: + """`ats._get` answering from a dict of URL -> body or exception, and + remembering what it was asked. No delay between requests.""" + answers: dict[str, Any] = {} + asked: list[str] = [] + + def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None): + asked.append(url) + if not ats._allowed_host(host, slug)(httpx.URL(url).host): + raise ats._Refused(url) + answer = answers.get(url, _status(404, url)) + if isinstance(answer, BaseException): + raise answer + return answer + + monkeypatch.setattr(ats, "_get", fake_get) + monkeypatch.setattr(ats, "PAGE_DELAY", 0) + return {"answers": answers, "asked": asked} + + +def _run(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType, + state: dict[str, str] | None = None) -> tuple[list[Any], dict[str, str]]: + monkeypatch.setitem(atshosts.HOSTS, "fake", module) + source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"}) + if state is not None: + source.state = state + return list(source.fetch()), source.state + + +def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]: + """The fake host's detail: the role's own page is its description.""" + return {"description": get(f"https://jobs.fake.example/{slug}/{row['source_id']}").decode(), + "location": "Copenhagen"} + + +# ---------------------------------------------------------------- detail + +def test_detail_is_read_for_unread_roles_only( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one in full", + f"{base}/2": b"two in full", f"{base}/3": b"three in full"}) + module = _fake_host(detail=_detail) + state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"])} + items, state = _run(monkeypatch, module, state) + + assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/3"] + assert [(i.source_id, i.body) for i in items] == [ + ("fake:acme:1", "one in full"), ("fake:acme:2", ""), ("fake:acme:3", "three in full")] + # What detail returns replaces the listing's field; a role not read keeps the listing's. + assert items[0].raw["place"] == "Copenhagen" and items[1].raw["place"] == "" + assert json.loads(state["ats:detailed:fake:acme"]) == [ + "fake:acme:1", "fake:acme:2", "fake:acme:3"] + assert state["ats:complete:fake:acme"] == "1" + + +def test_detail_reads_stop_at_the_cap_and_the_rest_wait( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + ids = [str(n) for n in range(5)] + web["answers"].update({f"{base}/list": json.dumps(ids).encode()} + | {f"{base}/{i}": f"full {i}".encode() for i in ids}) + monkeypatch.setattr(ats, "DETAIL_CAP", 2) + module = _fake_host(detail=_detail) + items, state = _run(monkeypatch, module) + assert [i.body for i in items] == ["full 0", "full 1", "", "", ""] + assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:0", "fake:acme:1"] + + web["asked"].clear() + items, state = _run(monkeypatch, module, state) + assert web["asked"] == [f"{base}/list", f"{base}/2", f"{base}/3"] + assert [i.body for i in items] == ["", "", "full 2", "full 3", ""] + + +def test_a_detail_that_fails_skips_the_role_and_not_the_board( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one", + f"{base}/2": ValueError("no such page shape"), f"{base}/3": b"three"}) + items, state = _run(monkeypatch, _fake_host(detail=_detail)) + assert [(i.source_id, i.body) for i in items] == [ + ("fake:acme:1", "one"), ("fake:acme:2", ""), ("fake:acme:3", "three")] + assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:3"] + assert "ats:failing:fake:acme" not in state and state["ats:ok:fake:acme"] == "r" + assert state["ats:complete:fake:acme"] == "1" + + +def test_a_429_on_a_detail_stops_the_reads_for_the_pull( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one", + f"{base}/2": _status(429), f"{base}/3": b"three"}) + items, state = _run(monkeypatch, _fake_host(detail=_detail)) + assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"] + assert [i.body for i in items] == ["one", "", ""] + assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"] + assert state["ats:complete:fake:acme"] == "1" + + +def test_ids_no_longer_listed_leave_the_detailed_list( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["2"]'}) + state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} + _items, state = _run(monkeypatch, _fake_host(detail=_detail), state) + assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"] + + +def test_an_incomplete_listing_keeps_the_ids_it_did_not_list( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + """A listing cut short says nothing about the roles it left out, so their + ids stay, and the next pull does not read them in detail again.""" + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["2"]'}) + monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: ( + atshosts.HOSTS["fake"].parse(b'["2"]', "acme"), False)) + state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} + _items, state = _run(monkeypatch, _fake_host(detail=_detail), state) + assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"] + assert state["ats:complete:fake:acme"] == "" + + +def test_fetch_board_reads_the_listing_only( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": b"one"}) + monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail)) + rows, complete = ats.fetch_board("fake:acme") + assert ([r["description"] for r in rows], complete) == (["teaser"], True) + assert web["asked"] == [f"{base}/list"] + + +def test_a_host_without_detail_keeps_the_listings_description( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]' + items, state = _run(monkeypatch, _fake_host()) + assert [i.body for i in items] == ["teaser"] + assert "ats:detailed:fake:acme" not in state + + +# ------------------------------------------------------------ robots.txt + +ROBOTS = "https://jobs.fake.example/robots.txt" + + +def test_robots_that_allow_the_page_let_the_board_be_read( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /admin/\n", + "https://jobs.fake.example/acme/list": b'["1"]'}) + items, state = _run(monkeypatch, _fake_host(robots=True)) + assert len(items) == 1 + assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True + + +@pytest.mark.parametrize("robots", [ + b"User-agent: *\nDisallow: /acme/\n", + # A rule for this app by name counts as well as the one for everyone. + f"User-agent: {ats.USER_AGENT.split('/')[0]}\nDisallow: /\n\nUser-agent: *\nAllow: /\n" + .encode(), +]) +def test_robots_that_disallow_the_page_fail_the_board( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], robots: bytes) -> None: + web["answers"].update({ROBOTS: robots, "https://jobs.fake.example/acme/list": b'["1"]'}) + items, state = _run(monkeypatch, _fake_host(robots=True)) + assert items == [] + assert web["asked"] == [ROBOTS] # the listing is never fetched + assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it" + assert "ats:ok:fake:acme" not in state + + +def test_a_robots_verdict_is_kept_for_a_day( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n", + "https://jobs.fake.example/acme/list": b'["1"]'}) + module = _fake_host(robots=True) + _items, state = _run(monkeypatch, module) + assert web["asked"] == [ROBOTS] + + # Within the day the kept verdict holds, even though the site changed. + web["answers"][ROBOTS] = b"User-agent: *\nAllow: /\n" + web["asked"].clear() + items, state = _run(monkeypatch, module, state) + assert (items, web["asked"]) == ([], []) + + # A day on, robots.txt is read again. + kept = json.loads(state["ats:robots:fake:acme"]) + kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat() + state["ats:robots:fake:acme"] = json.dumps(kept) + items, state = _run(monkeypatch, module, state) + assert web["asked"] == [ROBOTS, "https://jobs.fake.example/acme/list"] + assert len(items) == 1 and "ats:failing:fake:acme" not in state + + +def test_a_missing_robots_txt_allows_and_is_kept( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]' # robots.txt: 404 + items, state = _run(monkeypatch, _fake_host(robots=True)) + assert len(items) == 1 + assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True + + +def test_a_robots_txt_that_cannot_be_read_allows_and_is_asked_again( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + web["answers"].update({ROBOTS: _status(503), "https://jobs.fake.example/acme/list": b'["1"]'}) + items, state = _run(monkeypatch, _fake_host(robots=True)) + assert len(items) == 1 and "ats:robots:fake:acme" not in state + + +def test_fetch_board_honours_robots_txt( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n", + "https://jobs.fake.example/acme/list": b'["1"]'}) + monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) + with pytest.raises(ats._Disallowed) as caught: + ats.fetch_board("fake:acme") + assert ats.failure_reason(caught.value) == "robots.txt disallows it" + + +# ----------------------------------------------------------------- Lever EU + +def test_a_lever_slug_may_name_the_eu_instance() -> None: + assert ats.is_known_target("lever:acme") and ats.is_known_target("lever:eu/tomtom") + assert not ats.is_known_target("lever:us/acme") + assert not ats.is_known_target("lever:eu/") + assert not ats.is_known_target("lever:eu/a/b") + assert ats.careers_url("lever:acme") == "https://jobs.lever.co/acme" + assert ats.careers_url("lever:eu/tomtom") == "https://jobs.eu.lever.co/tomtom" + assert lever.feed_url("eu/tomtom") == "https://api.eu.lever.co/v0/postings/tomtom?mode=json" + eu = ats._allowed_host("lever", "eu/tomtom") + assert eu("api.eu.lever.co") and not eu("api.lever.co") + assert not ats._allowed_host("lever", "tomtom")("api.eu.lever.co") + + +def test_a_lever_eu_board_is_read_from_the_eu_api(monkeypatch: pytest.MonkeyPatch) -> None: + urls: list[str] = [] + + def fake_get(client, url, host, slug, *, method="GET"): + urls.append(url) + assert ats._allowed_host(host, slug)(httpx.URL(url).host) + return (FIXTURES / "lever.json").read_bytes() + + monkeypatch.setattr(ats, "_get", fake_get) + rows, complete = ats._fetch_rows(object(), "lever:eu/tomtom") # type: ignore[arg-type] + assert urls == ["https://api.eu.lever.co/v0/postings/tomtom?mode=json"] + assert rows and complete + + +def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(302, headers={"location": "https://api.lever.co/v0/postings/x"}) + + with _client(handler) as client, pytest.raises(ats._Refused): + ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x")