Read a role's own page once for a host with detail, and fail a board whose robots.txt disallows it

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-29 17:05:44 +02:00
1 parent b02b72b4aa
commit 8b14d8575b
2 files changed
+578 -6

No files matched your search

+153 -6
View File
@@ -6,7 +6,8 @@ unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that
drops off the board is *closed*, which is the cheapest staleness check there
is. What a host says and how its feed reads is its module's; this file holds
what every host shares: the fetch and its allow-list, paging and
completeness, the classifier, and the plugin itself.
completeness, the per-role detail reads, robots.txt, the classifier, and the
plugin itself.
The allow-list and the classifier are ported from `openings.go` and
`cmd/seedgen/main.go` in
@@ -32,9 +33,10 @@ import json
import logging
import re
import time
from collections.abc import Callable, Iterator
from datetime import UTC, datetime
from collections.abc import Callable, Iterator, MutableMapping
from datetime import UTC, datetime, timedelta
from typing import Any
from urllib.robotparser import RobotFileParser
import httpx
@@ -110,6 +112,11 @@ class _TooLarge(ValueError):
retry at the host's `fallback_url`."""
class _Disallowed(Exception):
"""The site's robots.txt disallows the page a board is read from (see
`_check_robots`)."""
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None) -> bytes:
@@ -329,7 +336,8 @@ def careers_url(target: str) -> str:
#: Pages read from one paged board at most, and the pause between two of
#: them. Workable answers ten roles a page, SmartRecruiters a hundred and
#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause
#: keeps Workable, which answers a burst with 429, from seeing one.
#: keeps Workable, which answers a burst with 429, from seeing one. The same
#: pause separates two detail reads.
MAX_PAGES = 100
PAGE_DELAY = 0.5
@@ -408,18 +416,142 @@ def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
return module.parse(body, slug), True
# -------------------------------------------------------------- robots.txt
#: How long a robots.txt answer holds for one board before it is read again.
ROBOTS_TTL = timedelta(days=1)
def _robots_allows(body: bytes, page: str) -> bool:
"""Whether a robots.txt lets `page` be fetched: by its rules for every
agent (`*`), and by any it has for this app's own User-Agent token, so a
site that shuts out this app by name is honoured as well."""
parser = RobotFileParser()
parser.parse(body.decode("utf-8", "replace").splitlines())
return parser.can_fetch("*", page) and parser.can_fetch(USER_AGENT, page)
def _check_robots(client: httpx.Client, target: str,
state: MutableMapping[str, str] | None = None) -> None:
"""Raise `_Disallowed` when the site of a host with a `robots_page` has a
robots.txt that disallows that page. Read through `_get`, so from the
board's own allowed hosts only, and at most once per `ROBOTS_TTL` for a
board when `state` is given: the verdict and when it was reached are kept
under `ats:robots:<target>`. A robots.txt that is missing (404 or 410)
allows, and that verdict is kept like any other; one that cannot be read
for another reason allows too, but is not kept, so the next pull asks
again."""
host, _, slug = target.partition(":")
module = atshosts.HOSTS.get(host)
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
return
page = module.robots_page(slug)
key = f"ats:robots:{target}"
now = datetime.now(UTC)
if state is not None:
try:
kept = json.loads(state.get(key) or "{}")
fresh = now - datetime.fromisoformat(kept["checked"]) < ROBOTS_TTL
except (ValueError, KeyError, TypeError):
kept, fresh = {}, False
if fresh:
if not kept.get("allowed", True):
raise _Disallowed(f"robots.txt disallows {page}")
return
url = f"https://{httpx.URL(page).host}/robots.txt"
try:
body = _getter(client, host, slug)(url, headers={"Accept": "text/plain"})
except httpx.HTTPStatusError as exc:
if exc.response.status_code not in (404, 410):
log.warning("ats: %s: robots.txt answered %s; allowed until the next pull",
target, exc.response.status_code)
return
allowed = True
except Exception as exc:
log.warning("ats: %s: robots.txt not read (%s); allowed until the next pull",
target, exc)
return
else:
allowed = _robots_allows(body, page)
if state is not None:
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
"allowed": allowed})
if not allowed:
raise _Disallowed(f"robots.txt disallows {page}")
# ---------------------------------------------------------------- details
#: Roles read in detail from one board in one pull at most. The rest of a
#: large new board is read over the next pulls, 200 at a time.
DETAIL_CAP = 200
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
detailed: set[str]) -> tuple[list[Row], set[str]]:
"""The rows of a board whose host has `detail`, each role not yet in
`detailed` (compound ids) filled from its own page, at most `DETAIL_CAP`
of them and `PAGE_DELAY` apart. Returns the rows and the ids read in
detail so far, for `ats:detailed:<target>`.
Every row not read in detail in this call goes out with an empty
description: a role read on an earlier pull keeps the one it has, since
`upsert_role` never overwrites a filled column with an empty value, and a
teaser from the listing must not replace it. A detail that raises is
logged and the role left for the next pull; a 429 stops the reads for this
board until then. None of this touches completeness, which is the
listing's alone. The ids of roles no longer listed are dropped, but only
after a complete listing, so one cut short does not make the next pull
read its missing roles again."""
host, _, slug = target.partition(":")
module = atshosts.HOSTS[host]
get = _getter(client, host, slug)
done = set(detailed)
out: list[Row] = []
tried = 0
stopped = False
for row in rows:
compound = f"{host}:{slug}:{row['source_id']}"
if compound in done or stopped or tried >= DETAIL_CAP:
out.append(row | {"description": ""})
continue
time.sleep(PAGE_DELAY)
tried += 1
try:
filled = module.detail(get, slug, row)
except Exception as exc:
if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 429:
log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull",
target)
stopped = True
else:
log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc)
out.append(row | {"description": ""})
continue
out.append(row | filled | {"source_id": row["source_id"]})
done.add(compound)
if complete:
done &= {f"{host}:{slug}:{row['source_id']}" for row in rows}
return out, done
def fetch_board(target: str) -> tuple[list[Row], bool]:
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
"does this board answer, what does it list, and is that all of it". For
`app companies add|check`, which ask that question about one board at a
time outside any pull. Raises the same way `_fetch_rows` does."""
time outside any pull. The listing only, with no detail reads, and
robots.txt read afresh. Raises the same way `_fetch_rows` does, and
`_Disallowed` for a board robots.txt shuts out."""
with httpx.Client() as client:
_check_robots(client, target)
return _fetch_rows(client, target)
def failure_reason(exc: BaseException) -> str:
"""One short line for why a board did not answer: the status of an HTTP
error, else the exception's class and the first line of its message."""
if isinstance(exc, _Disallowed):
return "robots.txt disallows it"
if isinstance(exc, httpx.HTTPStatusError):
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
lines = str(exc).strip().splitlines()
@@ -465,6 +597,7 @@ class ATSSource(Source):
host, _, slug = target.partition(":")
failing = f"ats:failing:{target}"
try:
_check_robots(client, target, self.state)
rows, complete = _fetch_rows(client, target)
except Exception as exc:
log.warning("ats: %s failed (%s)", target, exc)
@@ -481,6 +614,15 @@ class ATSSource(Source):
continue
self.state.pop(failing, None)
detailed_key = f"ats:detailed:{target}"
detailed: set[str] | None = None
if hasattr(atshosts.HOSTS.get(host), "detail"):
try:
detailed = set(json.loads(self.state.get(detailed_key) or "[]"))
except (ValueError, TypeError):
detailed = set()
rows, detailed = _read_details(client, target, rows, complete, detailed)
seen_ids = []
for row in rows:
# The compound id, matching `roles.source_id` — what
@@ -520,7 +662,12 @@ class ATSSource(Source):
# board that blew up half way through is neither marked ok nor
# has a partial id list treated as the whole of it. `complete`
# says whether the ids are the whole board, which the pull
# needs before it closes what they leave out.
# needs before it closes what they leave out. The ids read in
# detail are kept here too, after every row went out, so a
# pull stopped half way reads the rest again rather than
# losing their descriptions.
self.state[f"ats:ok:{target}"] = run_marker
self.state[f"ats:complete:{target}"] = "1" if complete else ""
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
if detailed is not None:
self.state[detailed_key] = json.dumps(sorted(detailed))
+425
View File
@@ -0,0 +1,425 @@
"""The host modules of `plugins/atshosts/` and what `plugins/ats.py` does for
every host: the registry and its contract check, the fetch a host is handed,
the per-role detail reads, robots.txt, and Lever's EU instance. No network —
a fake host stands in where a behaviour belongs to no real host yet."""
from __future__ import annotations
import json
import sys
import types
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
import httpx
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import ats, atshosts # noqa: E402
from plugins.atshosts import lever # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
# --------------------------------------------------------------- registry
def test_the_registry_finds_every_module_in_order_of_name() -> None:
names = sorted(p.stem for p in (ROOT / "plugins" / "atshosts").glob("*.py")
if not p.stem.startswith("_"))
assert list(atshosts.HOSTS) == names
assert set(names) >= {"greenhouse", "lever", "ashby", "workable", "smartrecruiters",
"recruitee", "personio", "teamtailor"}
assert all(not atshosts.problems(module, name) for name, module in atshosts.HOSTS.items())
# The Sources page's select and the plugin's description follow the modules.
assert {name: module.LABEL for name, module in atshosts.HOSTS.items()} == ats.HOSTS
assert "Teamtailor" in ats.ATSSource.description
def _package(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str,
modules: dict[str, str]) -> list[str]:
package = tmp_path / name
package.mkdir()
(package / "__init__.py").write_text("")
for module, source in modules.items():
(package / f"{module}.py").write_text(source)
monkeypatch.syspath_prepend(str(tmp_path))
return [str(package)]
_GOOD = """
NAME = "{name}"
LABEL = "Good"
PATTERNS = (r"good\\.example/([a-z]+)",)
def slug_ok(slug): return True
def allowed(slug): return lambda h: h == "good.example"
def careers_url(slug): return "https://good.example/" + slug
def feed_url(slug): return "https://good.example/" + slug + ".json"
def parse(body, slug): return []
"""
def test_the_registry_loads_a_package_by_name_and_skips_private_modules(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
path = _package(tmp_path, monkeypatch, "hostsok", {
"zeta": _GOOD.format(name="zeta"), "alpha": _GOOD.format(name="alpha"),
"_helpers": "X = 1\n"})
assert list(atshosts.load(path, "hostsok")) == ["alpha", "zeta"]
def test_the_registry_refuses_a_module_missing_a_required_name(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
broken = _GOOD.format(name="broken").replace("def careers_url", "def not_careers_url")
path = _package(tmp_path, monkeypatch, "hostsbad", {"broken": broken})
with pytest.raises(ImportError, match=r"hostsbad\.broken .*no careers_url"):
atshosts.load(path, "hostsbad")
def test_the_contract_check_names_each_problem() -> None:
module = types.ModuleType("odd")
module.NAME, module.LABEL, module.PATTERNS = "other", "", ("(",)
module.slug_ok = module.allowed = module.careers_url = lambda slug: ""
module.page = lambda get, slug, cursor: None
module.probe_ok = lambda body: True
found = atshosts.problems(module, "odd")
assert "NAME is 'other', not the module's name 'odd'" in found
assert "LABEL is not a non-empty string" in found
assert any(p.startswith("PATTERNS has '('") for p in found)
assert "page without rows" in found
assert "probe_ok without probe_url" in found
# ------------------------------------------------------------ the fetch
def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client:
return httpx.Client(transport=httpx.MockTransport(handler))
def test_get_sends_extra_headers_on_top_of_its_own() -> None:
seen: list[httpx.Headers] = []
def handler(request: httpx.Request) -> httpx.Response:
seen.append(request.headers)
return httpx.Response(200, content=b"ok")
with _client(handler) as client:
get = ats._getter(client, "lever", "acme")
assert get("https://api.lever.co/v0/postings/acme",
headers={"Accept": "text/plain", "X-Board": "acme"}) == b"ok"
get("https://api.lever.co/v0/postings/acme")
assert seen[0]["accept"] == "text/plain" and seen[0]["x-board"] == "acme"
assert seen[0]["user-agent"] == seen[1]["user-agent"] == ats.USER_AGENT
assert seen[1]["accept"] == "application/json, text/xml"
assert "x-board" not in seen[1]
def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatch) -> None:
"""Bound before the replacement, and still reaching it: `_get` is looked
up when a request is made."""
client = object()
get = ats._getter(client, "lever", "acme") # type: ignore[arg-type]
calls: list[tuple[Any, ...]] = []
def replaced(*args: Any, **kwargs: Any) -> bytes:
calls.append((args, kwargs))
return b""
monkeypatch.setattr(ats, "_get", replaced)
get("https://api.lever.co/v0/postings/acme", headers={"X": "1"})
get("https://api.lever.co/v0/postings/acme", method="POST", payload={})
assert calls == [
((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"),
{"method": "GET", "headers": {"X": "1"}}),
((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"),
{"method": "POST", "payload": {}})]
# ------------------------------------------------------- a fake host
def _fake_host(*, detail: Callable[..., Any] | None = None,
robots: bool = False) -> types.ModuleType:
"""A host `fake` whose feed is a JSON list of ids, each role with a
teaser for a description, and, with `detail`, a per-role page; with
`robots`, its listing is the page robots.txt must allow."""
module = types.ModuleType("fake")
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
module.slug_ok = lambda slug: slug.isalnum()
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
module.parse = lambda body, slug: [
{"source_id": i, "title": f"Role {i}", "location": "", "department": "",
"url": f"https://jobs.fake.example/{slug}/{i}", "posted": None,
"description": "teaser", "remote": None, "salary_text": ""}
for i in json.loads(body)]
if detail is not None:
module.detail = detail
if robots:
module.robots_page = lambda slug: f"https://jobs.fake.example/{slug}/list"
return module
def _status(code: int, url: str = "https://jobs.fake.example/") -> httpx.HTTPStatusError:
request = httpx.Request("GET", url)
return httpx.HTTPStatusError(str(code), request=request,
response=httpx.Response(code, request=request))
@pytest.fixture
def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
"""`ats._get` answering from a dict of URL -> body or exception, and
remembering what it was asked. No delay between requests."""
answers: dict[str, Any] = {}
asked: list[str] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None):
asked.append(url)
if not ats._allowed_host(host, slug)(httpx.URL(url).host):
raise ats._Refused(url)
answer = answers.get(url, _status(404, url))
if isinstance(answer, BaseException):
raise answer
return answer
monkeypatch.setattr(ats, "_get", fake_get)
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
return {"answers": answers, "asked": asked}
def _run(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType,
state: dict[str, str] | None = None) -> tuple[list[Any], dict[str, str]]:
monkeypatch.setitem(atshosts.HOSTS, "fake", module)
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
if state is not None:
source.state = state
return list(source.fetch()), source.state
def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
"""The fake host's detail: the role's own page is its description."""
return {"description": get(f"https://jobs.fake.example/{slug}/{row['source_id']}").decode(),
"location": "Copenhagen"}
# ---------------------------------------------------------------- detail
def test_detail_is_read_for_unread_roles_only(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one in full",
f"{base}/2": b"two in full", f"{base}/3": b"three in full"})
module = _fake_host(detail=_detail)
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"])}
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/3"]
assert [(i.source_id, i.body) for i in items] == [
("fake:acme:1", "one in full"), ("fake:acme:2", ""), ("fake:acme:3", "three in full")]
# What detail returns replaces the listing's field; a role not read keeps the listing's.
assert items[0].raw["place"] == "Copenhagen" and items[1].raw["place"] == ""
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
assert state["ats:complete:fake:acme"] == "1"
def test_detail_reads_stop_at_the_cap_and_the_rest_wait(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
ids = [str(n) for n in range(5)]
web["answers"].update({f"{base}/list": json.dumps(ids).encode()}
| {f"{base}/{i}": f"full {i}".encode() for i in ids})
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
module = _fake_host(detail=_detail)
items, state = _run(monkeypatch, module)
assert [i.body for i in items] == ["full 0", "full 1", "", "", ""]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:0", "fake:acme:1"]
web["asked"].clear()
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/2", f"{base}/3"]
assert [i.body for i in items] == ["", "", "full 2", "full 3", ""]
def test_a_detail_that_fails_skips_the_role_and_not_the_board(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
f"{base}/2": ValueError("no such page shape"), f"{base}/3": b"three"})
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert [(i.source_id, i.body) for i in items] == [
("fake:acme:1", "one"), ("fake:acme:2", ""), ("fake:acme:3", "three")]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:3"]
assert "ats:failing:fake:acme" not in state and state["ats:ok:fake:acme"] == "r"
assert state["ats:complete:fake:acme"] == "1"
def test_a_429_on_a_detail_stops_the_reads_for_the_pull(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
f"{base}/2": _status(429), f"{base}/3": b"three"})
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
assert [i.body for i in items] == ["one", "", ""]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
assert state["ats:complete:fake:acme"] == "1"
def test_ids_no_longer_listed_leave_the_detailed_list(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]'})
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"]
def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""A listing cut short says nothing about the roles it left out, so their
ids stay, and the next pull does not read them in detail again."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]'})
monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: (
atshosts.HOSTS["fake"].parse(b'["2"]', "acme"), False))
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
assert state["ats:complete:fake:acme"] == ""
def test_fetch_board_reads_the_listing_only(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": b"one"})
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail))
rows, complete = ats.fetch_board("fake:acme")
assert ([r["description"] for r in rows], complete) == (["teaser"], True)
assert web["asked"] == [f"{base}/list"]
def test_a_host_without_detail_keeps_the_listings_description(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]'
items, state = _run(monkeypatch, _fake_host())
assert [i.body for i in items] == ["teaser"]
assert "ats:detailed:fake:acme" not in state
# ------------------------------------------------------------ robots.txt
ROBOTS = "https://jobs.fake.example/robots.txt"
def test_robots_that_allow_the_page_let_the_board_be_read(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /admin/\n",
"https://jobs.fake.example/acme/list": b'["1"]'})
items, state = _run(monkeypatch, _fake_host(robots=True))
assert len(items) == 1
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
@pytest.mark.parametrize("robots", [
b"User-agent: *\nDisallow: /acme/\n",
# A rule for this app by name counts as well as the one for everyone.
f"User-agent: {ats.USER_AGENT.split('/')[0]}\nDisallow: /\n\nUser-agent: *\nAllow: /\n"
.encode(),
])
def test_robots_that_disallow_the_page_fail_the_board(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], robots: bytes) -> None:
web["answers"].update({ROBOTS: robots, "https://jobs.fake.example/acme/list": b'["1"]'})
items, state = _run(monkeypatch, _fake_host(robots=True))
assert items == []
assert web["asked"] == [ROBOTS] # the listing is never fetched
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
assert "ats:ok:fake:acme" not in state
def test_a_robots_verdict_is_kept_for_a_day(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
"https://jobs.fake.example/acme/list": b'["1"]'})
module = _fake_host(robots=True)
_items, state = _run(monkeypatch, module)
assert web["asked"] == [ROBOTS]
# Within the day the kept verdict holds, even though the site changed.
web["answers"][ROBOTS] = b"User-agent: *\nAllow: /\n"
web["asked"].clear()
items, state = _run(monkeypatch, module, state)
assert (items, web["asked"]) == ([], [])
# A day on, robots.txt is read again.
kept = json.loads(state["ats:robots:fake:acme"])
kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat()
state["ats:robots:fake:acme"] = json.dumps(kept)
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [ROBOTS, "https://jobs.fake.example/acme/list"]
assert len(items) == 1 and "ats:failing:fake:acme" not in state
def test_a_missing_robots_txt_allows_and_is_kept(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]' # robots.txt: 404
items, state = _run(monkeypatch, _fake_host(robots=True))
assert len(items) == 1
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
def test_a_robots_txt_that_cannot_be_read_allows_and_is_asked_again(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: _status(503), "https://jobs.fake.example/acme/list": b'["1"]'})
items, state = _run(monkeypatch, _fake_host(robots=True))
assert len(items) == 1 and "ats:robots:fake:acme" not in state
def test_fetch_board_honours_robots_txt(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
"https://jobs.fake.example/acme/list": b'["1"]'})
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
with pytest.raises(ats._Disallowed) as caught:
ats.fetch_board("fake:acme")
assert ats.failure_reason(caught.value) == "robots.txt disallows it"
# ----------------------------------------------------------------- Lever EU
def test_a_lever_slug_may_name_the_eu_instance() -> None:
assert ats.is_known_target("lever:acme") and ats.is_known_target("lever:eu/tomtom")
assert not ats.is_known_target("lever:us/acme")
assert not ats.is_known_target("lever:eu/")
assert not ats.is_known_target("lever:eu/a/b")
assert ats.careers_url("lever:acme") == "https://jobs.lever.co/acme"
assert ats.careers_url("lever:eu/tomtom") == "https://jobs.eu.lever.co/tomtom"
assert lever.feed_url("eu/tomtom") == "https://api.eu.lever.co/v0/postings/tomtom?mode=json"
eu = ats._allowed_host("lever", "eu/tomtom")
assert eu("api.eu.lever.co") and not eu("api.lever.co")
assert not ats._allowed_host("lever", "tomtom")("api.eu.lever.co")
def test_a_lever_eu_board_is_read_from_the_eu_api(monkeypatch: pytest.MonkeyPatch) -> None:
urls: list[str] = []
def fake_get(client, url, host, slug, *, method="GET"):
urls.append(url)
assert ats._allowed_host(host, slug)(httpx.URL(url).host)
return (FIXTURES / "lever.json").read_bytes()
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "lever:eu/tomtom") # type: ignore[arg-type]
assert urls == ["https://api.eu.lever.co/v0/postings/tomtom?mode=json"]
assert rows and complete
def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None:
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(302, headers={"location": "https://api.lever.co/v0/postings/x"})
with _client(handler) as client, pytest.raises(ats._Refused):
ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x")