762 lines
36 KiB
Python
762 lines
36 KiB
Python
"""The host modules of `plugins/atshosts/` and what `plugins/ats.py` does for
|
|
every host: the registry and its contract check, the fetch a host is handed,
|
|
the per-role detail reads, robots.txt, and Lever's EU instance. No network —
|
|
a fake host stands in where a behaviour belongs to no real host yet."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import types
|
|
from collections.abc import Callable
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from plugins import ats, atshosts # noqa: E402
|
|
from plugins.atshosts import lever # noqa: E402
|
|
|
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
|
|
|
|
|
# --------------------------------------------------------------- registry
|
|
|
|
def test_the_registry_finds_every_module() -> None:
|
|
names = sorted(p.stem for p in (ROOT / "plugins" / "atshosts").glob("*.py")
|
|
if not p.stem.startswith("_"))
|
|
assert sorted(atshosts.HOSTS) == names
|
|
assert set(names) >= {"greenhouse", "lever", "ashby", "workable", "smartrecruiters",
|
|
"recruitee", "personio", "teamtailor"}
|
|
assert all(not atshosts.problems(module, name) for name, module in atshosts.HOSTS.items())
|
|
# The Sources page's select and the plugin's description follow the modules.
|
|
assert {name: module.LABEL for name, module in atshosts.HOSTS.items()} == ats.HOSTS
|
|
assert "Teamtailor" in ats.ATSSource.description
|
|
|
|
|
|
def _package(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str,
|
|
modules: dict[str, str]) -> list[str]:
|
|
package = tmp_path / name
|
|
package.mkdir()
|
|
(package / "__init__.py").write_text("")
|
|
for module, source in modules.items():
|
|
(package / f"{module}.py").write_text(source)
|
|
monkeypatch.syspath_prepend(str(tmp_path))
|
|
return [str(package)]
|
|
|
|
|
|
_GOOD = """
|
|
NAME = "{name}"
|
|
LABEL = "Good"
|
|
PATTERNS = (r"good\\.example/([a-z]+)",)
|
|
def slug_ok(slug): return True
|
|
def allowed(slug): return lambda h: h == "good.example"
|
|
def careers_url(slug): return "https://good.example/" + slug
|
|
def feed_url(slug): return "https://good.example/" + slug + ".json"
|
|
def parse(body, slug): return []
|
|
"""
|
|
|
|
|
|
def test_the_registry_loads_ats_by_path_without_the_repo_root_on_the_path(
|
|
tmp_path: Path) -> None:
|
|
"""As `desk.sources.registry` does in a process the app did not set up:
|
|
another working directory, and `plugins` not importable at first."""
|
|
code = (
|
|
"import importlib.util, sys\n"
|
|
"assert importlib.util.find_spec('plugins') is None, 'plugins is importable'\n"
|
|
"from desk.sources import registry\n"
|
|
f"print(sorted(registry.available([{str(ROOT / 'plugins')!r}])))\n")
|
|
env = {k: v for k, v in os.environ.items() if k != "PYTHONPATH"}
|
|
done = subprocess.run([sys.executable, "-c", code], cwd=tmp_path, env=env,
|
|
capture_output=True, text=True, timeout=60, check=False)
|
|
assert done.returncode == 0, done.stderr
|
|
assert "'ats'" in done.stdout and "'search'" in done.stdout
|
|
|
|
|
|
def test_the_registry_loads_a_package_by_name_and_skips_private_modules(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
path = _package(tmp_path, monkeypatch, "hostsok", {
|
|
"zeta": _GOOD.format(name="zeta"), "alpha": _GOOD.format(name="alpha"),
|
|
"_helpers": "X = 1\n"})
|
|
assert list(atshosts.load(path, "hostsok")) == ["alpha", "zeta"]
|
|
|
|
|
|
def test_the_registry_orders_by_rank_then_name(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
path = _package(tmp_path, monkeypatch, "hostsranked", {
|
|
"alpha": _GOOD.format(name="alpha"), # the default, 100
|
|
"beta": _GOOD.format(name="beta") + "RANK = 5\n",
|
|
"gamma": _GOOD.format(name="gamma") + "RANK = 100\n"})
|
|
assert list(atshosts.load(path, "hostsranked")) == ["beta", "alpha", "gamma"]
|
|
|
|
|
|
def test_the_eight_first_hosts_keep_the_order_they_had_before_modules() -> None:
|
|
"""The Sources page's select opens on Greenhouse, which its hint names,
|
|
and a guessed slug on two hosts goes to the one probed first, as before."""
|
|
first = ["greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee",
|
|
"personio", "teamtailor"]
|
|
assert list(ats.HOSTS)[:8] == first
|
|
assert ats.ATSSource.description.startswith(
|
|
"Company job boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, "
|
|
"Recruitee, Personio, Teamtailor")
|
|
assert all(module.RANK < atshosts.DEFAULT_RANK
|
|
for name, module in atshosts.HOSTS.items() if name in first)
|
|
|
|
|
|
def test_the_registry_refuses_a_module_missing_a_required_name(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
broken = _GOOD.format(name="broken").replace("def careers_url", "def not_careers_url")
|
|
path = _package(tmp_path, monkeypatch, "hostsbad", {"broken": broken})
|
|
with pytest.raises(ImportError, match=r"hostsbad\.broken .*no careers_url"):
|
|
atshosts.load(path, "hostsbad")
|
|
|
|
|
|
def test_the_contract_check_names_each_problem() -> None:
|
|
module: Any = types.ModuleType("odd")
|
|
module.NAME, module.LABEL, module.PATTERNS = "other", "", ("(",)
|
|
module.slug_ok = module.allowed = module.careers_url = lambda slug: ""
|
|
module.page = lambda get, slug, cursor: None
|
|
module.probe_ok = lambda body: True
|
|
module.RANK, module.KEEP_CASE = "1", "yes"
|
|
found = atshosts.problems(module, "odd")
|
|
assert "NAME is 'other', not the module's name 'odd'" in found
|
|
assert "LABEL is not a non-empty string" in found
|
|
assert any(p.startswith("PATTERNS has '('") for p in found)
|
|
assert "page without rows" in found
|
|
assert "probe_ok without probe_url" in found
|
|
assert "RANK is not an int" in found and "KEEP_CASE is not a bool" in found
|
|
|
|
|
|
@pytest.mark.parametrize("host", ["greenhouse", "lever", "ashby", "workable", "smartrecruiters",
|
|
"recruitee", "personio", "teamtailor"])
|
|
def test_a_slug_is_matched_whole(host: str) -> None:
|
|
assert ats.is_known_target(f"{host}:acme")
|
|
for slug in ("acme\n", "acme\r\n", " acme", "acme ", "ac me", "acme/x", "acme?x", "acme#x"):
|
|
assert not ats.is_known_target(f"{host}:{slug}"), slug
|
|
|
|
|
|
def test_a_slug_in_the_host_name_is_allowed_whatever_its_case() -> None:
|
|
for host, api in (("recruitee", "acme.recruitee.com"), ("personio", "acme.jobs.personio.de")):
|
|
allow = ats._allowed_host(host, "Acme")
|
|
assert allow(api) and allow(api.upper())
|
|
assert not allow(api.replace("acme", "other"))
|
|
|
|
|
|
# ------------------------------------------------------------ the fetch
|
|
|
|
def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client:
|
|
return httpx.Client(transport=httpx.MockTransport(handler))
|
|
|
|
|
|
def test_get_sends_extra_headers_on_top_of_its_own() -> None:
|
|
seen: list[httpx.Headers] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
seen.append(request.headers)
|
|
return httpx.Response(200, content=b"ok")
|
|
|
|
with _client(handler) as client:
|
|
get = ats._getter(client, "lever", "acme")
|
|
assert get("https://api.lever.co/v0/postings/acme",
|
|
headers={"Accept": "text/plain", "X-Board": "acme"}) == b"ok"
|
|
get("https://api.lever.co/v0/postings/acme")
|
|
assert seen[0]["accept"] == "text/plain" and seen[0]["x-board"] == "acme"
|
|
assert seen[0]["user-agent"] == seen[1]["user-agent"] == ats.USER_AGENT
|
|
assert seen[1]["accept"] == "application/json, text/xml"
|
|
assert "x-board" not in seen[1]
|
|
|
|
|
|
@pytest.mark.parametrize("site", [
|
|
"metadata.google.internal", "printer.local", "router.lan", "nas.home.arpa",
|
|
"app.localhost", "foo.0x7f000001", "printer.local.",
|
|
])
|
|
def test_get_refuses_a_local_name_before_any_request(site: str) -> None:
|
|
"""A Teamtailor slug may be any host name, so the allow-list admits
|
|
these; `_get` refuses them before a request is made."""
|
|
asked: list[str] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
asked.append(str(request.url))
|
|
return httpx.Response(200, content=b"<rss><channel/></rss>")
|
|
|
|
with _client(handler) as client:
|
|
# A trailing dot is no Teamtailor slug at all; the rest are, and are refused.
|
|
with pytest.raises(ValueError if site.endswith(".") else ats._Refused):
|
|
ats._fetch_rows(client, f"teamtailor:{site}")
|
|
for url in (f"https://{site}/jobs.rss", "https://127.0.0.1/", "https://[::1]/",
|
|
"https://2130706433/", "https://127.1/", "https://localhost/"):
|
|
with pytest.raises(ats._Refused):
|
|
ats._get(client, url, "teamtailor", site)
|
|
assert asked == []
|
|
|
|
|
|
@pytest.mark.parametrize("location", [
|
|
"https://api.lever.co:8443/x", "https://127.0.0.1/x", "https://[::1]/x",
|
|
"https://localhost/x",
|
|
])
|
|
def test_get_refuses_a_redirect_to_another_port_or_a_local_host(location: str) -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(302, headers={"location": location})
|
|
|
|
with _client(handler) as client, pytest.raises(ats._Refused):
|
|
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
|
|
|
|
|
|
def test_get_takes_port_443_written_out_and_refuses_another() -> None:
|
|
with _client(lambda request: httpx.Response(200, content=b"[]")) as client:
|
|
assert ats._get(client, "https://api.lever.co:443/v0/postings/acme",
|
|
"lever", "acme") == b"[]"
|
|
with pytest.raises(ats._Refused):
|
|
ats._get(client, "https://api.lever.co:8443/v0/postings/acme", "lever", "acme")
|
|
|
|
|
|
def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Bound before the replacement, and still reaching it: `_get` is looked
|
|
up when a request is made."""
|
|
client = object()
|
|
get = ats._getter(client, "lever", "acme") # type: ignore[arg-type]
|
|
calls: list[tuple[Any, ...]] = []
|
|
|
|
def replaced(*args: Any, **kwargs: Any) -> bytes:
|
|
calls.append((args, kwargs))
|
|
return b""
|
|
|
|
monkeypatch.setattr(ats, "_get", replaced)
|
|
get("https://api.lever.co/v0/postings/acme", headers={"X": "1"})
|
|
get("https://api.lever.co/v0/postings/acme", method="POST", payload={})
|
|
assert calls == [
|
|
((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"),
|
|
{"method": "GET", "headers": {"X": "1"}}),
|
|
((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"),
|
|
{"method": "POST", "payload": {}})]
|
|
|
|
|
|
# ------------------------------------------------------- a fake host
|
|
|
|
def _fake_host(*, detail: Callable[..., Any] | None = None,
|
|
robots: bool = False) -> types.ModuleType:
|
|
"""A host `fake` whose feed is a JSON list of ids, each role with a
|
|
teaser for a description, and, with `detail`, a per-role page; with
|
|
`robots`, its listing is the page robots.txt must allow."""
|
|
module: Any = types.ModuleType("fake")
|
|
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
|
|
module.slug_ok = lambda slug: slug.isalnum()
|
|
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
|
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
|
|
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
|
module.parse = lambda body, slug: [
|
|
{"source_id": i, "title": f"Role {i}", "location": "", "department": "",
|
|
"url": f"https://jobs.fake.example/{slug}/{i}", "posted": None,
|
|
"description": "teaser", "remote": None, "salary_text": ""}
|
|
for i in json.loads(body)]
|
|
if detail is not None:
|
|
module.detail = detail
|
|
if robots:
|
|
module.robots_page = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
|
return module
|
|
|
|
|
|
def _status(code: int, url: str = "https://jobs.fake.example/") -> httpx.HTTPStatusError:
|
|
request = httpx.Request("GET", url)
|
|
return httpx.HTTPStatusError(str(code), request=request,
|
|
response=httpx.Response(code, request=request))
|
|
|
|
|
|
@pytest.fixture
|
|
def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
|
|
"""`ats._get` answering from a dict of URL -> body or exception, and
|
|
remembering what it was asked. No delay between requests."""
|
|
answers: dict[str, Any] = {}
|
|
asked: list[str] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None,
|
|
www_twin=False):
|
|
asked.append(url)
|
|
if not ats._allowed_host(host, slug)(httpx.URL(url).host):
|
|
raise ats._Refused(url)
|
|
answer = answers.get(url, _status(404, url))
|
|
if isinstance(answer, BaseException):
|
|
raise answer
|
|
return answer
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
|
return {"answers": answers, "asked": asked}
|
|
|
|
|
|
def _run(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType,
|
|
state: dict[str, str] | None = None) -> tuple[list[Any], Any]:
|
|
monkeypatch.setitem(atshosts.HOSTS, "fake", module)
|
|
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
|
if state is not None:
|
|
source.state = state
|
|
return list(source.fetch()), source.state
|
|
|
|
|
|
def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
|
|
"""The fake host's detail: the role's own page is its description."""
|
|
return {"description": get(f"https://jobs.fake.example/{slug}/{row['source_id']}").decode(),
|
|
"location": "Copenhagen"}
|
|
|
|
|
|
# ---------------------------------------------------------------- detail
|
|
|
|
def test_detail_is_read_for_unread_roles_only(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one in full",
|
|
f"{base}/2": b"two in full", f"{base}/3": b"three in full"})
|
|
module = _fake_host(detail=_detail)
|
|
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"])}
|
|
items, state = _run(monkeypatch, module, state)
|
|
|
|
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/3"]
|
|
assert [(i.source_id, i.body) for i in items] == [
|
|
("fake:acme:1", "one in full"), ("fake:acme:2", ""), ("fake:acme:3", "three in full")]
|
|
# What detail returns replaces the listing's field; a role not read keeps the listing's.
|
|
assert items[0].raw["place"] == "Copenhagen" and items[1].raw["place"] == ""
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
|
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
|
assert state["ats:complete:fake:acme"] == "1"
|
|
|
|
|
|
def test_detail_reads_stop_at_the_cap_and_the_rest_wait(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
ids = [str(n) for n in range(5)]
|
|
web["answers"].update({f"{base}/list": json.dumps(ids).encode()}
|
|
| {f"{base}/{i}": f"full {i}".encode() for i in ids})
|
|
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
|
|
module = _fake_host(detail=_detail)
|
|
items, state = _run(monkeypatch, module)
|
|
assert [i.body for i in items] == ["full 0", "full 1", "", "", ""]
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:0", "fake:acme:1"]
|
|
|
|
web["asked"].clear()
|
|
items, state = _run(monkeypatch, module, state)
|
|
assert web["asked"] == [f"{base}/list", f"{base}/2", f"{base}/3"]
|
|
assert [i.body for i in items] == ["", "", "full 2", "full 3", ""]
|
|
|
|
|
|
def test_a_detail_that_fails_skips_the_role_and_not_the_board(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
|
|
f"{base}/2": ValueError("no such page shape"), f"{base}/3": b"three"})
|
|
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
|
assert [(i.source_id, i.body) for i in items] == [
|
|
("fake:acme:1", "one"), ("fake:acme:2", ""), ("fake:acme:3", "three")]
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:3"]
|
|
assert "ats:failing:fake:acme" not in state and state["ats:ok:fake:acme"] == "r"
|
|
assert state["ats:complete:fake:acme"] == "1"
|
|
|
|
|
|
def test_a_429_on_a_detail_stops_the_reads_for_the_pull(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
|
|
f"{base}/2": _status(429), f"{base}/3": b"three"})
|
|
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
|
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
|
|
assert [i.body for i in items] == ["one", "", ""]
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
|
assert state["ats:complete:fake:acme"] == "1"
|
|
|
|
|
|
def test_ids_no_longer_listed_leave_the_detailed_list(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list": b'["2"]'})
|
|
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
|
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"]
|
|
|
|
|
|
def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
"""A listing cut short says nothing about the roles it left out, so their
|
|
ids stay, and the next pull does not read them in detail again."""
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list": b'["2"]'})
|
|
monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: (
|
|
atshosts.HOSTS["fake"].parse(b'["2"]', "acme"), False))
|
|
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
|
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
|
assert state["ats:complete:fake:acme"] == ""
|
|
|
|
|
|
def test_fetch_board_reads_the_listing_only(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": b"one"})
|
|
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail))
|
|
rows, complete = ats.fetch_board("fake:acme")
|
|
assert ([r["description"] for r in rows], complete) == (["teaser"], True)
|
|
assert web["asked"] == [f"{base}/list"]
|
|
|
|
|
|
def test_a_host_without_detail_keeps_the_listings_description(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]'
|
|
items, state = _run(monkeypatch, _fake_host())
|
|
assert [i.body for i in items] == ["teaser"]
|
|
assert "ats:detailed:fake:acme" not in state
|
|
|
|
|
|
def test_a_detail_answering_404_or_410_counts_as_read(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": _status(410),
|
|
f"{base}/3": b"three"}) # 1 answers 404
|
|
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
|
assert [i.body for i in items] == ["", "", "three"]
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
|
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
|
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
|
|
|
|
|
|
@pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")])
|
|
def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
|
|
"""Two roles that always fail and a cap of two: the third is read on the
|
|
second pull, since roles never tried go before those that failed."""
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list": b'["1", "2", "3", "4"]', f"{base}/1": failure,
|
|
f"{base}/2": failure, f"{base}/3": b"three", f"{base}/4": b"four"})
|
|
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
|
|
module = _fake_host(detail=_detail)
|
|
_items, state = _run(monkeypatch, module)
|
|
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
|
|
|
web["asked"].clear()
|
|
items, state = _run(monkeypatch, module, state)
|
|
assert web["asked"] == [f"{base}/list", f"{base}/3", f"{base}/4"]
|
|
assert [i.body for i in items] == ["", "", "three", "four"]
|
|
|
|
# With nothing new left, the failed ones are tried again, within the cap.
|
|
web["asked"].clear()
|
|
web["answers"][f"{base}/1"] = b"one at last"
|
|
items, state = _run(monkeypatch, module, state)
|
|
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
|
|
assert [i.body for i in items] == ["one at last", "", "", ""]
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
|
"fake:acme:1", "fake:acme:3", "fake:acme:4"]
|
|
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
|
|
|
|
|
def test_a_429_on_a_detail_does_not_mark_the_role_as_failing(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": _status(429)})
|
|
_items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
|
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == []
|
|
|
|
|
|
def test_failed_ids_no_longer_listed_leave_the_failed_list(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list": b'["2"]', f"{base}/2": ValueError("odd page")})
|
|
state = {"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
|
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
|
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
|
|
|
|
|
def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any],
|
|
caplog: pytest.LogCaptureFixture) -> None:
|
|
"""`salary_text`, `department` and `title` would be rewritten from the
|
|
listing on the next pull, so detail does not fill them."""
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list": b'["1"]'})
|
|
|
|
def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
|
|
return {"description": "in full", "remote": "hybrid", "currency": "EUR",
|
|
"salary_min": 60000, "salary_max": 70000,
|
|
"posted": datetime(2026, 9, 1, tzinfo=UTC),
|
|
"salary_text": "EUR 60,000 to 70,000 per year", "department": "Sales",
|
|
"title": "Another title"}
|
|
|
|
caplog.set_level("DEBUG", logger=ats.log.name)
|
|
[item], _state = _run(monkeypatch, _fake_host(detail=detail))
|
|
assert (item.body, item.title, item.published_at) == (
|
|
"in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC))
|
|
assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"],
|
|
item.raw["salary_max"]) == ("hybrid", "EUR", 60000, 70000)
|
|
assert (item.raw["salary_text"], item.raw["department"]) == ("", "")
|
|
assert "department, salary_text, title left out" in caplog.text
|
|
assert set(ats.DETAIL_FIELDS) == {"description", "location", "remote", "currency",
|
|
"salary_min", "salary_max", "posted"}
|
|
|
|
|
|
def _paged_host() -> types.ModuleType:
|
|
"""A host `fake` that pages its listing, one id a page, and has `detail`."""
|
|
module: Any = _fake_host(detail=_detail)
|
|
del module.feed_url, module.parse
|
|
|
|
def page(get: Any, slug: str, cursor: Any) -> tuple[Any, list[dict[str, Any]], Any, Any]:
|
|
index = int(cursor or 0)
|
|
ids = json.loads(get(f"https://jobs.fake.example/{slug}/list?page={index}"))
|
|
return ids, [{"id": i} for i in ids], 2, index + 1 if ids else None
|
|
|
|
module.page = page
|
|
module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug)
|
|
assert not atshosts.problems(module, "fake")
|
|
return module
|
|
|
|
|
|
def test_a_paged_host_reads_every_page_then_each_role_once(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": b'["2"]',
|
|
f"{base}/1": b"one", f"{base}/2": b"two"})
|
|
items, state = _run(monkeypatch, _paged_host())
|
|
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1",
|
|
f"{base}/1", f"{base}/2"]
|
|
assert [i.body for i in items] == ["one", "two"]
|
|
assert state["ats:complete:fake:acme"] == "1"
|
|
|
|
web["asked"].clear()
|
|
items, state = _run(monkeypatch, _paged_host(), state)
|
|
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1"]
|
|
assert [i.body for i in items] == ["", ""]
|
|
|
|
|
|
def test_a_paged_listing_cut_short_keeps_the_ids_it_did_not_reach(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": _status(429),
|
|
f"{base}/1": b"one"})
|
|
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
|
|
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])}
|
|
_items, state = _run(monkeypatch, _paged_host(), state)
|
|
assert state["ats:complete:fake:acme"] == ""
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
|
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:3"]
|
|
|
|
|
|
def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
"""The consumer stops after the first role, so the second, read but not
|
|
stored, is read again on the next pull."""
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"})
|
|
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail))
|
|
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
|
roles: Any = source.fetch() # a generator, which `close` stops
|
|
assert next(roles).body == "one"
|
|
roles.close()
|
|
assert "ats:detailed:fake:acme" not in source.state
|
|
|
|
|
|
# ------------------------------------------------------------ robots.txt
|
|
|
|
ROBOTS = "https://jobs.fake.example/robots.txt"
|
|
|
|
|
|
def test_robots_that_allow_the_page_let_the_board_be_read(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /admin/\n",
|
|
"https://jobs.fake.example/acme/list": b'["1"]'})
|
|
items, state = _run(monkeypatch, _fake_host(robots=True))
|
|
assert len(items) == 1
|
|
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
|
|
|
|
|
|
@pytest.mark.parametrize("robots", [
|
|
b"User-agent: *\nDisallow: /acme/\n",
|
|
# A rule for this app by name counts as well as the one for everyone.
|
|
f"User-agent: {ats.USER_AGENT.split('/')[0]}\nDisallow: /\n\nUser-agent: *\nAllow: /\n"
|
|
.encode(),
|
|
])
|
|
def test_robots_that_disallow_the_page_fail_the_board(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], robots: bytes) -> None:
|
|
web["answers"].update({ROBOTS: robots, "https://jobs.fake.example/acme/list": b'["1"]'})
|
|
items, state = _run(monkeypatch, _fake_host(robots=True))
|
|
assert items == []
|
|
assert web["asked"] == [ROBOTS] # the listing is never fetched
|
|
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
|
|
assert "ats:ok:fake:acme" not in state
|
|
|
|
|
|
@pytest.mark.parametrize(("robots", "page", "allowed"), [
|
|
# The longest matching rule wins, whatever the order of the lines.
|
|
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",
|
|
"https://jobs.infineon.com/api/pcsx/search?domain=infineon.com", True),
|
|
("User-agent: *\nAllow: /careers\nDisallow: /\n", "https://x.example/careers/list", True),
|
|
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", "https://x.example/other", False),
|
|
# An allow wins a tie.
|
|
("User-agent: *\nDisallow: /jobs\nAllow: /jobs\n", "https://x.example/jobs", True),
|
|
# `*` inside a path, and the query counts as part of it.
|
|
("User-agent: *\nDisallow: /*company\n",
|
|
"https://career5.successfactors.eu/career?company=x", False),
|
|
# `$` anchors the end.
|
|
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json", False),
|
|
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json?page=2", True),
|
|
# An empty Disallow allows.
|
|
("User-agent: *\nDisallow:\n", "https://x.example/anything", True),
|
|
# A byte order mark and CRLF line ends.
|
|
("\ufeffUser-agent: *\r\nDisallow: /\r\n", "https://x.example/jobs", False),
|
|
# Paths are case-sensitive.
|
|
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
|
|
"https://novonesis.wd103.myworkdayjobs.com/Novonesis_Careers/", False),
|
|
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
|
|
"https://novonesis.wd103.myworkdayjobs.com/novonesis_careers/", True),
|
|
# A group for this app by name counts, a group for someone else does
|
|
# not, and a token matches whole, not as a substring.
|
|
(f"User-agent: {ats.ROBOTS_TOKEN.upper()}\nDisallow: /\n\nUser-agent: *\nAllow: /\n",
|
|
"https://x.example/jobs", False),
|
|
("User-agent: Googlebot\nDisallow: /\n", "https://x.example/jobs", True),
|
|
("User-agent: sk\nDisallow: /\n\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
|
|
# Two agents share one group, and a comment is no rule.
|
|
("User-agent: a\nUser-agent: *\nDisallow: /jobs # not these\n",
|
|
"https://x.example/jobs/1", False),
|
|
# A group for this app that allows does not undo the `*` group's refusal.
|
|
(f"User-agent: {ats.ROBOTS_TOKEN}\nAllow: /\n\nUser-agent: *\nDisallow: /\n",
|
|
"https://x.example/jobs", False),
|
|
# Rules before any user-agent line belong to no group.
|
|
("Disallow: /\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
|
|
])
|
|
def test_robots_rules_follow_rfc_9309(robots: str, page: str, allowed: bool) -> None:
|
|
assert ats._robots_allows(robots.encode(), page) is allowed
|
|
|
|
|
|
def test_a_robots_verdict_is_kept_for_a_day(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
|
|
"https://jobs.fake.example/acme/list": b'["1"]'})
|
|
module = _fake_host(robots=True)
|
|
_items, state = _run(monkeypatch, module)
|
|
assert web["asked"] == [ROBOTS]
|
|
|
|
# Within the day the kept verdict holds, even though the site changed.
|
|
web["answers"][ROBOTS] = b"User-agent: *\nAllow: /\n"
|
|
web["asked"].clear()
|
|
items, state = _run(monkeypatch, module, state)
|
|
assert (items, web["asked"]) == ([], [])
|
|
|
|
# A day on, robots.txt is read again.
|
|
kept = json.loads(state["ats:robots:fake:acme"])
|
|
kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat()
|
|
state["ats:robots:fake:acme"] = json.dumps(kept)
|
|
items, state = _run(monkeypatch, module, state)
|
|
assert web["asked"] == [ROBOTS, "https://jobs.fake.example/acme/list"]
|
|
assert len(items) == 1 and "ats:failing:fake:acme" not in state
|
|
|
|
|
|
@pytest.mark.parametrize("code", [404, 410, 401, 403])
|
|
def test_a_robots_txt_answering_4xx_allows_and_is_kept(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], code: int) -> None:
|
|
web["answers"].update({ROBOTS: _status(code), "https://jobs.fake.example/acme/list": b'["1"]'})
|
|
items, state = _run(monkeypatch, _fake_host(robots=True))
|
|
assert len(items) == 1
|
|
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
|
|
|
|
|
|
@pytest.mark.parametrize("failure", [
|
|
_status(500), _status(503), _status(429), httpx.ConnectTimeout("timed out"),
|
|
httpx.ConnectError("refused"), ats._Refused("https://elsewhere.example/robots.txt"),
|
|
])
|
|
def test_a_robots_txt_that_cannot_be_read_fails_the_board_and_is_asked_again(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
|
|
web["answers"].update({ROBOTS: failure, "https://jobs.fake.example/acme/list": b'["1"]'})
|
|
items, state = _run(monkeypatch, _fake_host(robots=True))
|
|
assert items == [] and web["asked"] == [ROBOTS]
|
|
assert json.loads(state["ats:failing:fake:acme"])["reason"] == \
|
|
"robots.txt could not be read"
|
|
assert "ats:robots:fake:acme" not in state
|
|
|
|
|
|
def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]:
|
|
"""A site whose robots.txt on jobs.fake.example redirects to `location`,
|
|
where a robots.txt that disallows everything is served."""
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if str(request.url) == ROBOTS:
|
|
return httpx.Response(301, headers={"location": location})
|
|
if request.url.path == "/robots.txt":
|
|
return httpx.Response(200, text="User-agent: *\nDisallow: /\n")
|
|
return httpx.Response(200, text='["1"]')
|
|
return handler
|
|
|
|
|
|
def test_a_robots_txt_redirect_to_the_www_twin_is_followed(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
|
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
|
pytest.raises(ats._Disallowed):
|
|
ats._check_robots(client, "fake:acme", {})
|
|
|
|
|
|
@pytest.mark.parametrize("location", [
|
|
"https://evil.example/robots.txt", "https://other.fake.example/robots.txt",
|
|
"http://www.jobs.fake.example/robots.txt",
|
|
])
|
|
def test_a_robots_txt_redirect_anywhere_else_is_unreachable(
|
|
monkeypatch: pytest.MonkeyPatch, location: str) -> None:
|
|
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
|
with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable):
|
|
ats._check_robots(client, "fake:acme", {})
|
|
|
|
|
|
def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
|
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
|
pytest.raises(ats._Refused):
|
|
ats._get(client, ROBOTS, "fake", "acme")
|
|
|
|
|
|
def test_fetch_board_honours_robots_txt(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
|
|
"https://jobs.fake.example/acme/list": b'["1"]'})
|
|
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
|
with pytest.raises(ats._Disallowed) as caught:
|
|
ats.fetch_board("fake:acme")
|
|
assert ats.failure_reason(caught.value) == "robots.txt disallows it"
|
|
|
|
|
|
# ----------------------------------------------------------------- Lever EU
|
|
|
|
def test_a_lever_slug_may_name_the_eu_instance() -> None:
|
|
assert ats.is_known_target("lever:acme") and ats.is_known_target("lever:eu/tomtom")
|
|
assert not ats.is_known_target("lever:us/acme")
|
|
assert not ats.is_known_target("lever:eu/")
|
|
assert not ats.is_known_target("lever:eu/a/b")
|
|
assert ats.careers_url("lever:acme") == "https://jobs.lever.co/acme"
|
|
assert ats.careers_url("lever:eu/tomtom") == "https://jobs.eu.lever.co/tomtom"
|
|
assert lever.feed_url("eu/tomtom") == "https://api.eu.lever.co/v0/postings/tomtom?mode=json"
|
|
eu = ats._allowed_host("lever", "eu/tomtom")
|
|
assert eu("api.eu.lever.co") and not eu("api.lever.co")
|
|
assert not ats._allowed_host("lever", "tomtom")("api.eu.lever.co")
|
|
|
|
|
|
def test_a_lever_eu_board_is_read_from_the_eu_api(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
urls: list[str] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET"):
|
|
urls.append(url)
|
|
assert ats._allowed_host(host, slug)(httpx.URL(url).host)
|
|
return (FIXTURES / "lever.json").read_bytes()
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "lever:eu/tomtom") # type: ignore[arg-type]
|
|
assert urls == ["https://api.eu.lever.co/v0/postings/tomtom?mode=json"]
|
|
assert rows and complete
|
|
|
|
|
|
def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(302, headers={"location": "https://api.lever.co/v0/postings/x"})
|
|
|
|
with _client(handler) as client, pytest.raises(ats._Refused):
|
|
ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x")
|