The pull reads listings only. `enrich` (app/jobs/enrich.py) takes the open roles without text that an enabled agent ranks ungated in its best `enrich.top` at `enrich.min_score` or above, best score first, on boards whose reader has `detail`, within `enrich.per_board` and `enrich.per_run`, one batch per board. `plugins.ats.read_board_details` reads them and keeps the per-board state; a read that gave no text is read again after `enrich.retry_days` (finding 18). Filled roles are upserted, which drops their vectors, and embed and score follow; `cascade.step` passes over both when nothing was filled. The pull cascade is pull, embed, score, enrich, embed, score, extract, score, summarise. Saving an agent and changing a company's affinity queue the `enrich` job (enrich, embed, score) behind their rescore; a vote does not. The stage has a switch, `stage:enrich`, over `enrich.enabled`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
593 lines
28 KiB
Python
593 lines
28 KiB
Python
"""The host modules of `plugins/readers/` and what `plugins/ats.py` does for
|
|
every host: the registry and its contract check, the fetch a host is handed,
|
|
the per-role detail reads the `enrich` stage makes, and Lever's EU
|
|
instance. No network —
|
|
a fake host stands in where a behaviour belongs to no real host yet."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import types
|
|
from collections.abc import Callable
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from plugins import ats, readers # noqa: E402
|
|
from plugins.readers import _runner # noqa: E402
|
|
from plugins.readers import lever # noqa: E402
|
|
from plugins.readers._contract import Ask # noqa: E402
|
|
|
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
|
|
|
|
|
# --------------------------------------------------------------- registry
|
|
|
|
def test_the_registry_finds_every_module() -> None:
|
|
names = sorted(p.stem for p in (ROOT / "plugins" / "readers").glob("*.py")
|
|
if not p.stem.startswith("_"))
|
|
assert sorted(readers.READERS) == names
|
|
assert set(readers.HOSTS) | set(readers.PROVIDERS) == set(names)
|
|
assert set(names) >= {"greenhouse", "lever", "ashby", "workable", "smartrecruiters",
|
|
"recruitee", "personio", "teamtailor"}
|
|
assert all(not readers.problems(module, name) for name, module in readers.READERS.items())
|
|
# The Sources page's select and the plugin's description follow the modules.
|
|
assert {name: module.LABEL for name, module in readers.HOSTS.items()} == ats.HOSTS
|
|
assert "Teamtailor" in ats.ATSSource.description
|
|
|
|
|
|
def _package(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str,
|
|
modules: dict[str, str]) -> list[str]:
|
|
package = tmp_path / name
|
|
package.mkdir()
|
|
(package / "__init__.py").write_text("")
|
|
for module, source in modules.items():
|
|
(package / f"{module}.py").write_text(source)
|
|
monkeypatch.syspath_prepend(str(tmp_path))
|
|
return [str(package)]
|
|
|
|
|
|
_GOOD = """
|
|
NAME = "{name}"
|
|
LABEL = "Good"
|
|
PATTERNS = (r"good\\.example/([a-z]+)",)
|
|
def slug_ok(slug): return True
|
|
def allowed(slug): return lambda h: h == "good.example"
|
|
def careers_url(slug): return "https://good.example/" + slug
|
|
def feed_url(slug): return "https://good.example/" + slug + ".json"
|
|
def parse(body, slug): return []
|
|
"""
|
|
|
|
|
|
def test_the_registry_loads_ats_by_path_without_the_repo_root_on_the_path(
|
|
tmp_path: Path) -> None:
|
|
"""As `desk.sources.registry` does in a process the app did not set up:
|
|
another working directory, and `plugins` not importable at first."""
|
|
code = (
|
|
"import importlib.util, sys\n"
|
|
"assert importlib.util.find_spec('plugins') is None, 'plugins is importable'\n"
|
|
"from desk.sources import registry\n"
|
|
f"print(sorted(registry.available([{str(ROOT / 'plugins')!r}])))\n")
|
|
env = {k: v for k, v in os.environ.items() if k != "PYTHONPATH"}
|
|
done = subprocess.run([sys.executable, "-c", code], cwd=tmp_path, env=env,
|
|
capture_output=True, text=True, timeout=60, check=False)
|
|
assert done.returncode == 0, done.stderr
|
|
assert "'ats'" in done.stdout and "'search'" in done.stdout
|
|
|
|
|
|
def test_the_registry_loads_a_package_by_name_and_skips_private_modules(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
path = _package(tmp_path, monkeypatch, "hostsok", {
|
|
"zeta": _GOOD.format(name="zeta"), "alpha": _GOOD.format(name="alpha"),
|
|
"_helpers": "X = 1\n"})
|
|
assert list(readers.load(path, "hostsok")) == ["alpha", "zeta"]
|
|
|
|
|
|
def test_the_registry_orders_by_rank_then_name(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
path = _package(tmp_path, monkeypatch, "hostsranked", {
|
|
"alpha": _GOOD.format(name="alpha"), # the default, 100
|
|
"beta": _GOOD.format(name="beta") + "RANK = 5\n",
|
|
"gamma": _GOOD.format(name="gamma") + "RANK = 100\n"})
|
|
assert list(readers.load(path, "hostsranked")) == ["beta", "alpha", "gamma"]
|
|
|
|
|
|
def test_the_eight_first_hosts_keep_the_order_they_had_before_modules() -> None:
|
|
"""The Sources page's select opens on Greenhouse, which its hint names,
|
|
and a guessed slug on two hosts goes to the one probed first, as before."""
|
|
first = ["greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee",
|
|
"personio", "teamtailor"]
|
|
assert list(ats.HOSTS)[:8] == first
|
|
assert ats.ATSSource.description.startswith(
|
|
"Company job boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, "
|
|
"Recruitee, Personio, Teamtailor")
|
|
assert all(module.RANK < readers.DEFAULT_RANK
|
|
for name, module in readers.HOSTS.items() if name in first)
|
|
|
|
|
|
def test_the_registry_refuses_a_module_missing_a_required_name(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
broken = _GOOD.format(name="broken").replace("def careers_url", "def not_careers_url")
|
|
path = _package(tmp_path, monkeypatch, "hostsbad", {"broken": broken})
|
|
with pytest.raises(ImportError, match=r"hostsbad\.broken .*no careers_url"):
|
|
readers.load(path, "hostsbad")
|
|
|
|
|
|
def test_the_contract_check_names_each_problem() -> None:
|
|
module: Any = types.ModuleType("odd")
|
|
module.NAME, module.LABEL, module.PATTERNS = "other", "", ("(",)
|
|
module.slug_ok = module.allowed = module.careers_url = lambda slug: ""
|
|
module.page = lambda get, slug, cursor: None
|
|
module.probe_ok = lambda body: True
|
|
module.RANK, module.KEEP_CASE = "1", "yes"
|
|
found = readers.problems(module, "odd")
|
|
assert "NAME is 'other', not the module's name 'odd'" in found
|
|
assert "LABEL is not a non-empty string" in found
|
|
assert any(p.startswith("PATTERNS has '('") for p in found)
|
|
assert "page without rows" in found
|
|
assert "probe_ok without probe_url" in found
|
|
assert "RANK is not an int" in found and "KEEP_CASE is not a bool" in found
|
|
|
|
|
|
@pytest.mark.parametrize("host", ["greenhouse", "lever", "ashby", "workable", "smartrecruiters",
|
|
"recruitee", "personio", "teamtailor"])
|
|
def test_a_slug_is_matched_whole(host: str) -> None:
|
|
assert ats.is_known_target(f"{host}:acme")
|
|
for slug in ("acme\n", "acme\r\n", " acme", "acme ", "ac me", "acme/x", "acme?x", "acme#x"):
|
|
assert not ats.is_known_target(f"{host}:{slug}"), slug
|
|
|
|
|
|
def test_a_slug_in_the_host_name_is_allowed_whatever_its_case() -> None:
|
|
for host, api in (("recruitee", "acme.recruitee.com"), ("personio", "acme.jobs.personio.de")):
|
|
allow = ats._allowed_host(host, "Acme")
|
|
assert allow(api) and allow(api.upper())
|
|
assert not allow(api.replace("acme", "other"))
|
|
|
|
|
|
# ------------------------------------------------------------ the fetch
|
|
|
|
def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client:
|
|
return httpx.Client(transport=httpx.MockTransport(handler))
|
|
|
|
|
|
def test_get_sends_extra_headers_on_top_of_its_own() -> None:
|
|
seen: list[httpx.Headers] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
seen.append(request.headers)
|
|
return httpx.Response(200, content=b"ok")
|
|
|
|
with _client(handler) as client:
|
|
get = ats._getter(client, "lever", "acme")
|
|
assert get("https://api.lever.co/v0/postings/acme",
|
|
headers={"Accept": "text/plain", "X-Board": "acme"}) == b"ok"
|
|
get("https://api.lever.co/v0/postings/acme")
|
|
assert seen[0]["accept"] == "text/plain" and seen[0]["x-board"] == "acme"
|
|
assert seen[0]["user-agent"] == seen[1]["user-agent"] == ats.USER_AGENT
|
|
assert seen[1]["accept"] == "application/json, text/xml"
|
|
assert "x-board" not in seen[1]
|
|
|
|
|
|
@pytest.mark.parametrize("site", [
|
|
"metadata.google.internal", "printer.local", "router.lan", "nas.home.arpa",
|
|
"app.localhost", "foo.0x7f000001", "printer.local.",
|
|
])
|
|
def test_get_refuses_a_local_name_before_any_request(site: str) -> None:
|
|
"""A Teamtailor slug may be any host name, so the allow-list admits
|
|
these; `_get` refuses them before a request is made."""
|
|
asked: list[str] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
asked.append(str(request.url))
|
|
return httpx.Response(200, content=b"<rss><channel/></rss>")
|
|
|
|
with _client(handler) as client:
|
|
# A trailing dot is no Teamtailor slug at all; the rest are, and are refused.
|
|
with pytest.raises(ValueError if site.endswith(".") else ats._Refused):
|
|
ats._fetch_rows(client, f"teamtailor:{site}")
|
|
for url in (f"https://{site}/jobs.rss", "https://127.0.0.1/", "https://[::1]/",
|
|
"https://2130706433/", "https://127.1/", "https://localhost/"):
|
|
with pytest.raises(ats._Refused):
|
|
ats._get(client, url, "teamtailor", site)
|
|
assert asked == []
|
|
|
|
|
|
@pytest.mark.parametrize("location", [
|
|
"https://api.lever.co:8443/x", "https://127.0.0.1/x", "https://[::1]/x",
|
|
"https://localhost/x",
|
|
])
|
|
def test_get_refuses_a_redirect_to_another_port_or_a_local_host(location: str) -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(302, headers={"location": location})
|
|
|
|
with _client(handler) as client, pytest.raises(ats._Refused):
|
|
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
|
|
|
|
|
|
def test_get_takes_port_443_written_out_and_refuses_another() -> None:
|
|
with _client(lambda request: httpx.Response(200, content=b"[]")) as client:
|
|
assert ats._get(client, "https://api.lever.co:443/v0/postings/acme",
|
|
"lever", "acme") == b"[]"
|
|
with pytest.raises(ats._Refused):
|
|
ats._get(client, "https://api.lever.co:8443/v0/postings/acme", "lever", "acme")
|
|
|
|
|
|
def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Bound before the replacement, and still reaching it: `_get` is looked
|
|
up when a request is made."""
|
|
client = object()
|
|
get = ats._getter(client, "lever", "acme") # type: ignore[arg-type]
|
|
calls: list[tuple[Any, ...]] = []
|
|
|
|
def replaced(*args: Any, **kwargs: Any) -> bytes:
|
|
calls.append((args, kwargs))
|
|
return b""
|
|
|
|
monkeypatch.setattr(ats, "_get", replaced)
|
|
get("https://api.lever.co/v0/postings/acme", headers={"X": "1"})
|
|
get("https://api.lever.co/v0/postings/acme", method="POST", payload={})
|
|
assert calls == [
|
|
((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"),
|
|
{"method": "GET", "headers": {"X": "1"}}),
|
|
((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"),
|
|
{"method": "POST", "payload": {}})]
|
|
|
|
|
|
# ------------------------------------------------------- a fake host
|
|
|
|
def _fake_host(*, detail: Callable[..., Any] | None = None) -> types.ModuleType:
|
|
"""A host `fake` whose feed is a JSON list of ids, each role with a
|
|
teaser for a description, and, with `detail`, a per-role page."""
|
|
module: Any = types.ModuleType("fake")
|
|
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
|
|
module.slug_ok = lambda slug: slug.isalnum()
|
|
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
|
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
|
|
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
|
module.parse = lambda body, slug: [
|
|
{"source_id": i, "title": f"Role {i}", "location": "", "department": "",
|
|
"url": f"https://jobs.fake.example/{slug}/{i}", "posted": None,
|
|
"description": "teaser", "remote": None, "salary_text": ""}
|
|
for i in json.loads(body)]
|
|
if detail is not None:
|
|
module.detail = detail
|
|
return module
|
|
|
|
|
|
def _status(code: int, url: str = "https://jobs.fake.example/") -> httpx.HTTPStatusError:
|
|
request = httpx.Request("GET", url)
|
|
return httpx.HTTPStatusError(str(code), request=request,
|
|
response=httpx.Response(code, request=request))
|
|
|
|
|
|
@pytest.fixture
|
|
def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
|
|
"""`ats._get` answering from a dict of URL -> body or exception, and
|
|
remembering what it was asked. No delay between requests."""
|
|
answers: dict[str, Any] = {}
|
|
asked: list[str] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None,
|
|
www_twin=False):
|
|
asked.append(url)
|
|
if not ats._allowed_host(host, slug)(httpx.URL(url).host):
|
|
raise ats._Refused(url)
|
|
answer = answers.get(url, _status(404, url))
|
|
if isinstance(answer, BaseException):
|
|
raise answer
|
|
return answer
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
|
return {"answers": answers, "asked": asked}
|
|
|
|
|
|
def _run(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType,
|
|
state: dict[str, str] | None = None) -> tuple[list[Any], Any]:
|
|
monkeypatch.setitem(readers.HOSTS, "fake", module)
|
|
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
|
if state is not None:
|
|
source.state = state
|
|
return list(source.fetch()), source.state
|
|
|
|
|
|
def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
|
|
"""The fake host's detail: the role's own page is its description."""
|
|
return {"description": get(f"https://jobs.fake.example/{slug}/{row['source_id']}").decode(),
|
|
"location": "Copenhagen"}
|
|
|
|
|
|
# ---------------------------------------------------------------- detail
|
|
|
|
def test_a_pull_reads_no_role_page(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
"""The pages are `enrich`'s: the pull reads the listing only, and sends
|
|
every row of a host with `detail` without a description."""
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one in full"})
|
|
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
|
assert web["asked"] == [f"{base}/list"]
|
|
assert [(i.source_id, i.body) for i in items] == [("fake:acme:1", ""), ("fake:acme:2", "")]
|
|
assert "ats:detailed:fake:acme" not in state
|
|
assert state["ats:complete:fake:acme"] == "1"
|
|
|
|
|
|
def test_a_role_read_on_its_page_keeps_the_place_its_page_gave(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
"""The listing states a place on every pull, and a filled value replaces
|
|
a filled column, so for a role already read it is withheld."""
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list": b'["1", "3"]'})
|
|
module = _fake_host(detail=_detail)
|
|
listing = module.parse
|
|
module.parse = lambda body, slug: [row | {"location": "2 Locations", "remote": "hybrid"}
|
|
for row in listing(body, slug)]
|
|
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"]),
|
|
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])}
|
|
items, _state = _run(monkeypatch, module, state)
|
|
|
|
by_id = {i.source_id: (i.body, i.raw["place"], i.raw["remote"]) for i in items}
|
|
assert by_id["fake:acme:1"] == ("", "", None)
|
|
# Never read: the listing's stand until its page is.
|
|
assert by_id["fake:acme:3"] == ("", "2 Locations", "hybrid")
|
|
|
|
|
|
def test_ids_no_longer_listed_leave_the_detail_lists(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list": b'["2"]'})
|
|
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"]),
|
|
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
|
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"]
|
|
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
|
|
|
|
|
def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
"""A listing cut short says nothing about the roles it left out, so their
|
|
ids stay, and they are not read on their pages again."""
|
|
monkeypatch.setattr(_runner, "read_listing", lambda *args, **kwargs: _runner.Listing(
|
|
readers.HOSTS["fake"].parse(b'["2"]', "acme"), False, ["2"]))
|
|
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
|
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
|
assert state["ats:complete:fake:acme"] == ""
|
|
|
|
|
|
def test_a_pull_stopped_half_way_through_a_board_drops_no_detail_ids(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list": b'["2"]'})
|
|
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
|
|
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
|
source.state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])}
|
|
roles: Any = source.fetch() # a generator, which `close` stops
|
|
assert next(roles).body == ""
|
|
roles.close()
|
|
assert json.loads(source.state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
|
|
|
|
|
def test_fetch_board_reads_the_listing_only(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": b"one"})
|
|
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
|
|
rows, complete = ats.fetch_board("fake:acme")
|
|
assert ([r["description"] for r in rows], complete) == (["teaser"], True)
|
|
assert web["asked"] == [f"{base}/list"]
|
|
|
|
|
|
def test_a_host_without_detail_keeps_the_listings_description(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]'
|
|
items, state = _run(monkeypatch, _fake_host())
|
|
assert [i.body for i in items] == ["teaser"]
|
|
assert "ats:detailed:fake:acme" not in state
|
|
|
|
|
|
# -------------------------------------------------- read_board_details
|
|
|
|
NOW = datetime(2026, 9, 30, 12, tzinfo=UTC)
|
|
|
|
|
|
def _details(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType,
|
|
state: dict[str, str], ids: list[str], **kwargs: Any) -> list[dict[str, Any]]:
|
|
"""`ats.read_board_details` over the fake host's roles `ids`, stored
|
|
without text, in that order."""
|
|
monkeypatch.setitem(readers.HOSTS, "fake", module)
|
|
rows = [row | {"description": ""}
|
|
for row in module.parse(json.dumps(ids).encode(), "acme")]
|
|
return ats.read_board_details(state, "fake:acme", rows, client=object(), # type: ignore[arg-type]
|
|
now=kwargs.pop("now", NOW), **kwargs)
|
|
|
|
|
|
def test_details_are_read_in_the_order_given_for_unread_roles_only(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/1": b"one in full", f"{base}/2": b"two in full",
|
|
f"{base}/3": b"three in full"})
|
|
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
|
|
"ats:detailempty:fake:acme": json.dumps({"fake:acme:2": NOW.isoformat()})}
|
|
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["3", "2", "1"])
|
|
|
|
assert web["asked"] == [f"{base}/3", f"{base}/1"]
|
|
assert [(r["source_id"], r["description"], r["location"]) for r in filled] == [
|
|
("3", "three in full", "Copenhagen"), ("1", "one in full", "Copenhagen")]
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
|
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
|
|
|
|
|
def test_details_stop_at_the_cap_and_the_rest_wait(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
ids = [str(n) for n in range(5)]
|
|
web["answers"].update({f"{base}/{i}": f"full {i}".encode() for i in ids})
|
|
module = _fake_host(detail=_detail)
|
|
state: dict[str, str] = {}
|
|
filled = _details(monkeypatch, module, state, ids, cap=2)
|
|
assert [r["description"] for r in filled] == ["full 0", "full 1"]
|
|
|
|
web["asked"].clear()
|
|
filled = _details(monkeypatch, module, state, ids[2:], cap=2)
|
|
assert web["asked"] == [f"{base}/2", f"{base}/3"]
|
|
|
|
|
|
def test_a_detail_that_fails_skips_the_role_and_a_429_stops_the_board(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/1": b"one", f"{base}/2": ValueError("no such page shape"),
|
|
f"{base}/3": _status(429), f"{base}/4": b"four"})
|
|
state: dict[str, str] = {}
|
|
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3", "4"])
|
|
assert web["asked"] == [f"{base}/1", f"{base}/2", f"{base}/3"]
|
|
assert [r["source_id"] for r in filled] == ["1"]
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
|
# A 429 is not the role's fault.
|
|
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
|
|
|
|
|
@pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")])
|
|
def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
|
|
"""Roles never tried go before those that failed, within the cap."""
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/1": failure, f"{base}/2": failure,
|
|
f"{base}/3": b"three", f"{base}/4": b"four"})
|
|
module = _fake_host(detail=_detail)
|
|
state: dict[str, str] = {}
|
|
_details(monkeypatch, module, state, ["1", "2"])
|
|
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
|
|
|
web["asked"].clear()
|
|
_details(monkeypatch, module, state, ["1", "2", "3", "4"], cap=2)
|
|
assert web["asked"] == [f"{base}/3", f"{base}/4"]
|
|
|
|
web["asked"].clear()
|
|
web["answers"][f"{base}/1"] = b"one at last"
|
|
filled = _details(monkeypatch, module, state, ["1", "2"], cap=2)
|
|
assert [r["description"] for r in filled] == ["one at last"]
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
|
"fake:acme:1", "fake:acme:3", "fake:acme:4"]
|
|
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
|
|
|
|
|
def test_a_detail_answering_404_or_410_counts_as_read(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"].update({f"{base}/2": _status(410), f"{base}/3": b"three"}) # 1 answers 404
|
|
state: dict[str, str] = {}
|
|
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3"])
|
|
assert [r["source_id"] for r in filled] == ["3"]
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
|
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
|
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
|
|
|
|
|
|
def test_a_read_without_text_is_read_again_after_some_days(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
"""Finding 18: an Oracle requisition not yet published has an empty page
|
|
today and a full one next week."""
|
|
base = "https://jobs.fake.example/acme"
|
|
web["answers"][f"{base}/1"] = b""
|
|
module = _fake_host(detail=lambda get, slug, row: {
|
|
"description": get(f"{base}/{row['source_id']}").decode()})
|
|
state: dict[str, str] = {}
|
|
assert _details(monkeypatch, module, state, ["1"]) == []
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
|
assert json.loads(state["ats:detailempty:fake:acme"]) == {
|
|
"fake:acme:1": NOW.isoformat(timespec="seconds")}
|
|
|
|
web["asked"].clear()
|
|
later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS - 1)
|
|
assert _details(monkeypatch, module, state, ["1"], now=later) == []
|
|
assert web["asked"] == []
|
|
|
|
web["answers"][f"{base}/1"] = b"published now"
|
|
later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS)
|
|
[filled] = _details(monkeypatch, module, state, ["1"], now=later)
|
|
assert filled["description"] == "published now"
|
|
assert json.loads(state["ats:detailempty:fake:acme"]) == {}
|
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
|
|
|
|
|
def test_a_role_read_before_the_dates_were_kept_is_read_again(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
|
web["answers"]["https://jobs.fake.example/acme/1"] = b"one"
|
|
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])}
|
|
[filled] = _details(monkeypatch, _fake_host(detail=_detail), state, ["1"])
|
|
assert filled["description"] == "one"
|
|
|
|
|
|
def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any],
|
|
caplog: pytest.LogCaptureFixture) -> None:
|
|
"""`salary_text`, `department` and `title` would be rewritten from the
|
|
listing on the next pull, so detail does not fill them."""
|
|
def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
|
|
return {"description": "in full", "remote": "hybrid", "currency": "EUR",
|
|
"salary_min": 60000, "salary_max": 70000,
|
|
"posted": datetime(2026, 9, 1, tzinfo=UTC),
|
|
"salary_text": "EUR 60,000 to 70,000 per year", "department": "Sales",
|
|
"title": "Another title"}
|
|
|
|
caplog.set_level("DEBUG", logger=_runner.log.name)
|
|
[row] = _details(monkeypatch, _fake_host(detail=detail), {}, ["1"])
|
|
item = ats.item_of(row, "fake:acme", "Acme")
|
|
assert (item.body, item.title, item.published_at) == (
|
|
"in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC))
|
|
assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"],
|
|
item.raw["salary_max"]) == ("hybrid", "EUR", 60000, 70000)
|
|
assert (item.raw["salary_text"], item.raw["department"]) == ("", "")
|
|
assert "department, salary_text, title left out" in caplog.text
|
|
assert set(ats.DETAIL_FIELDS) == {"description", "location", "places", "teaser", "remote",
|
|
"country", "employment", "currency", "salary_min",
|
|
"salary_max", "posted"}
|
|
|
|
|
|
# ----------------------------------------------------------------- Lever EU
|
|
|
|
def test_a_lever_slug_may_name_the_eu_instance() -> None:
|
|
assert ats.is_known_target("lever:acme") and ats.is_known_target("lever:eu/tomtom")
|
|
assert not ats.is_known_target("lever:us/acme")
|
|
assert not ats.is_known_target("lever:eu/")
|
|
assert not ats.is_known_target("lever:eu/a/b")
|
|
assert ats.careers_url("lever:acme") == "https://jobs.lever.co/acme"
|
|
assert ats.careers_url("lever:eu/tomtom") == "https://jobs.eu.lever.co/tomtom"
|
|
assert lever.feed_url("eu/tomtom", Ask()) == "https://api.eu.lever.co/v0/postings/tomtom?mode=json"
|
|
eu = ats._allowed_host("lever", "eu/tomtom")
|
|
assert eu("api.eu.lever.co") and not eu("api.lever.co")
|
|
assert not ats._allowed_host("lever", "tomtom")("api.eu.lever.co")
|
|
|
|
|
|
def test_a_lever_eu_board_is_read_from_the_eu_api(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
urls: list[str] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET"):
|
|
urls.append(url)
|
|
assert ats._allowed_host(host, slug)(httpx.URL(url).host)
|
|
return (FIXTURES / "lever.json").read_bytes()
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "lever:eu/tomtom") # type: ignore[arg-type]
|
|
assert urls == ["https://api.eu.lever.co/v0/postings/tomtom?mode=json"]
|
|
assert rows and complete
|
|
|
|
|
|
def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(302, headers={"location": "https://api.lever.co/v0/postings/x"})
|
|
|
|
with _client(handler) as client, pytest.raises(ats._Refused):
|
|
ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x")
|