Files
jobboard/tests/test_ats.py
T
WolfandClaude Opus c7da12b93d Paged boards are read to their stated total, and a board failing for 21 days closes
Workable and SmartRecruiters page their lists, and both parsers read page one
only, so the settle closed every role past it (Delivery Hero: 980 stated, 100
open). `_paged` follows nextPage and offset, and the plugin stamps
`ats:complete` only when the distinct ids read equal the stated total; a 429
after page one keeps what was read and closes nothing. Adzuna completes on its
`count`, Jobindex counts raw items and raises on an answer with no channel.

A board whose fetch raises is recorded under `ats:failing:<target>` with the
first failure's time and the latest reason; after STALE_DAYS the settle closes
its roles, and its next answer reopens them. `/api/companies`, the Companies
list and `app companies check` show it.

Co-Authored-By: Claude Opus <noreply@anthropic.com>
2026-09-29 14:02:49 +02:00

345 lines
14 KiB
Python

"""The `ats` plugin: each host's parser against a saved feed, the function
classifier, and the fetch allow-list. No network — every fixture is a file
under `tests/fixtures/ats/`."""
from __future__ import annotations
import json
import sys
from pathlib import Path
from typing import Any
import httpx
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import ats # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
path = FIXTURES / f"{host}.{'xml' if host == 'personio' else 'json'}"
body = path.read_bytes()
return ats._PARSERS[host](body, slug)
# ------------------------------------------------------------------ hosts
def test_greenhouse_parses_titles_locations_and_html_description() -> None:
rows = _rows("greenhouse")
# The javascript: and data: rows are refused, same as the two real ones
# are kept: a feed is external content, never trusted as a URL.
assert [r["source_id"] for r in rows] == ["1001", "1002"]
first = rows[0]
assert first["title"] == "Senior Backend Engineer" # trimmed
assert first["location"] == "Berlin, Germany"
assert first["department"] == "Engineering"
assert first["description"] == "Build our core platform."
assert first["posted"] is not None and first["posted"].year == 2026
assert first["remote"] is None
def test_lever_parses_categories_and_workplace_type() -> None:
rows = _rows("lever")
assert [r["source_id"] for r in rows] == ["2001", "2002"]
first = rows[0]
assert first["department"] == "Engineering"
assert first["remote"] == "remote"
assert first["description"] == "Own our platform reliability."
# createdAt=0 means "no date", not epoch zero.
assert rows[1]["posted"] is None
def test_ashby_parses_is_remote_and_compensation() -> None:
rows = _rows("ashby")
assert [r["source_id"] for r in rows] == ["3001", "3002"]
assert rows[0]["remote"] is False
assert rows[0]["salary"] == "70,000-90,000 EUR"
assert rows[1]["remote"] is True
def test_workable_builds_the_job_url_and_reads_location_and_remote() -> None:
rows = _rows("workable", slug="huggingface")
assert [r["source_id"] for r in rows] == ["6125026", "6125041"]
assert rows[0]["url"] == "https://apply.workable.com/huggingface/j/DB4D7C0EC8/"
assert rows[0]["location"] == "France"
assert rows[0]["remote"] is True
assert rows[0]["department"] == "Product"
assert rows[1]["location"] == "Berlin, Germany"
assert rows[1]["remote"] is False
assert rows[0]["description"] == "" # the list endpoint carries no text
def test_smartrecruiters_builds_the_job_url_from_id_and_slug() -> None:
rows = _rows("smartrecruiters", slug="Acme")
assert [r["source_id"] for r in rows] == ["744000151326819", "744000147485112"]
assert rows[0]["url"] == "https://jobs.smartrecruiters.com/Acme/744000151326819"
assert rows[0]["location"] == "Berlin, Germany"
assert rows[1]["remote"] is True
def test_recruitee_parses_salary_bounds_and_careers_url() -> None:
rows = _rows("recruitee")
assert [r["source_id"] for r in rows] == ["5001", "5002"]
assert rows[0]["url"] == "https://acme.example/o/player-support-agent-german-1"
assert rows[0]["salary"] == "35000-45000 EUR"
assert rows[1]["salary"] == ""
def test_personio_builds_the_job_url_and_reads_cdata_description() -> None:
rows = _rows("personio", slug="acme")
assert [r["source_id"] for r in rows] == ["6001", "6002"]
assert rows[0]["url"] == "https://acme.jobs.personio.de/job/6001"
assert rows[0]["description"] == "Grow our pipeline."
assert rows[0]["remote"] is True # "Remote - Germany"
assert rows[1]["remote"] is None # "Munich"
# --------------------------------------------------------------- classify
def test_classify_matches_department_before_title() -> None:
assert ats.classify("Engineering", "Marketing Lead") == "engineering"
assert ats.classify("", "Senior Backend Engineer") == "engineering"
assert ats.classify("", "Product Manager, Payments") == "product"
assert ats.classify("", "Data Product Manager") == "product"
assert ats.classify("", "Machine Learning Engineer") == "data"
assert ats.classify("", "Executive Assistant to the CEO") == "finance/legal/ops"
assert ats.classify("", "") == "other"
# ------------------------------------------------------------- allow-list
def test_is_known_target_accepts_the_seven_hosts_and_refuses_others() -> None:
for host in ("greenhouse", "lever", "ashby", "workable", "smartrecruiters",
"recruitee", "personio"):
assert ats.is_known_target(f"{host}:acme")
assert not ats.is_known_target("linkedin:acme")
assert not ats.is_known_target("workday:acme")
assert not ats.is_known_target("greenhouse:")
assert not ats.is_known_target("nohostnoseparator")
def test_allowed_host_refuses_a_foreign_host_per_board() -> None:
allow = ats._allowed_host("greenhouse", "acme")
assert allow("boards-api.greenhouse.io")
assert not allow("boards-api.greenhouse.io.evil.example")
assert not allow("api.lever.co")
# recruitee and personio key off the company's own subdomain, so one
# company's slug must not pass for another's.
allow_recruitee = ats._allowed_host("recruitee", "acme")
assert allow_recruitee("acme.recruitee.com")
assert not allow_recruitee("other.recruitee.com")
def test_careers_url_is_filled_per_host_and_empty_for_unknown() -> None:
assert ats.careers_url("greenhouse:acme") == "https://job-boards.greenhouse.io/acme"
assert ats.careers_url("ashby:acme") == "https://jobs.ashbyhq.com/acme"
assert ats.careers_url("recruitee:acme") == "https://acme.recruitee.com"
assert ats.careers_url("workday:acme") == ""
# --------------------------------------------------------------- body cap
def test_max_bytes_is_96_mb() -> None:
"""Raised from openings.go's 8 MB: every URL here is on a fixed allow-list
of ATS API hosts, not an arbitrary one, and the real run of 2026-09-23
refused boards up to 41 MB under it."""
assert ats.MAX_BYTES == 96 << 20
def test_fetch_rows_retries_greenhouse_without_content_on_a_too_large_body(
monkeypatch: pytest.MonkeyPatch) -> None:
calls = []
def fake_get(client, url, host, slug, *, method="GET"):
calls.append(url)
if "?content=true" in url:
raise ats._TooLarge("response too large (100000000 bytes)")
return (FIXTURES / "greenhouse.json").read_bytes()
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type]
assert calls == [
"https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true",
"https://boards-api.greenhouse.io/v1/boards/acme/jobs",
]
assert [r["source_id"] for r in rows] == ["1001", "1002"]
assert complete # a host that does not page answers with the whole board
def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
monkeypatch: pytest.MonkeyPatch) -> None:
def fake_get(client, url, host, slug, *, method="GET"):
raise ats._TooLarge("response too large (100000000 bytes)")
monkeypatch.setattr(ats, "_get", fake_get)
with pytest.raises(ats._TooLarge):
ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]
# ----------------------------------------------------------------- paging
def _page(name: str) -> bytes:
return (FIXTURES / name).read_bytes()
def _status(code: int) -> httpx.HTTPStatusError:
request = httpx.Request("POST", "https://apply.workable.com/")
return httpx.HTTPStatusError(str(code), request=request,
response=httpx.Response(code, request=request))
@pytest.fixture
def no_delay(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
def test_workable_follows_next_page_to_the_stated_total(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
calls: list[dict[str, Any] | None] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None):
calls.append(payload)
return _page("workable-2.json" if payload else "workable.json")
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
assert calls == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}]
assert [r["source_id"] for r in rows] == ["6125026", "6125041", "6125077"]
assert complete
def test_smartrecruiters_pages_by_offset_to_total_found(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
urls: list[str] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None):
urls.append(url)
return _page("smartrecruiters-2.json" if "offset=2" in url else "smartrecruiters.json")
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
base = "https://api.smartrecruiters.com/v1/companies/Acme/postings"
assert urls == [f"{base}?offset=0&limit=2", f"{base}?offset=2&limit=2"]
assert len(rows) == 3 and rows[2]["location"] == "Tilburg, Netherlands"
assert complete
def test_a_board_that_ends_short_of_its_total_is_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
"""totalFound says 3; the second page is empty, so two were read."""
empty = json.dumps({"offset": 2, "limit": 2, "totalFound": 3, "content": []}).encode()
def fake_get(client, url, host, slug, *, method="GET", payload=None):
return empty if "offset=2" in url else _page("smartrecruiters.json")
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
assert (len(rows), complete) == (2, False)
def test_a_repeated_row_does_not_count_twice_toward_the_total(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
"""A page that shifted under an offset repeats a role and misses one: three
rows read, two distinct, against a total of three."""
def fake_get(client, url, host, slug, *, method="GET", payload=None):
page = json.loads(_page("smartrecruiters.json"))
if "offset=2" in url:
page["content"] = page["content"][1:]
return json.dumps(page).encode()
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
assert (len(rows), complete) == (2, False)
def test_the_page_cap_leaves_a_board_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
calls: list[dict[str, Any] | None] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None):
calls.append(payload)
return _page("workable.json") # always another page
monkeypatch.setattr(ats, "MAX_PAGES", 3)
monkeypatch.setattr(ats, "_get", fake_get)
_rows_read, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
assert (len(calls), complete) == (3, False)
def test_a_429_mid_way_keeps_what_was_read_and_is_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
def fake_get(client, url, host, slug, *, method="GET", payload=None):
if payload:
raise _status(429)
return _page("workable.json")
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
assert ([r["source_id"] for r in rows], complete) == (["6125026", "6125041"], False)
def test_a_429_on_the_first_page_is_a_failure(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
def fake_get(client, url, host, slug, *, method="GET", payload=None):
raise _status(429)
monkeypatch.setattr(ats, "_get", fake_get)
with pytest.raises(httpx.HTTPStatusError):
ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
# -------------------------------------------------------- failing boards
def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource:
"""One board, `greenhouse:gone`, whose fetch answers `answer` or raises it."""
def fake(client: object, target: str) -> object:
if isinstance(answer, Exception):
raise answer
return answer
monkeypatch.setattr(ats, "_fetch_rows", fake)
source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}]})
source.state = {"ats:run": "run-1"}
return source
def test_a_failing_board_records_since_and_the_latest_reason(
monkeypatch: pytest.MonkeyPatch) -> None:
source = _source(monkeypatch, _status(404))
assert list(source.fetch()) == []
first = json.loads(source.state["ats:failing:greenhouse:gone"])
assert first["reason"] == "404 Not Found" and first["since"]
assert "ats:ok:greenhouse:gone" not in source.state
# A later failure keeps when the failures began and updates the reason.
source.state["ats:failing:greenhouse:gone"] = json.dumps(
first | {"since": "2026-09-01T00:00:00+00:00"})
later = _source(monkeypatch, httpx.ConnectTimeout("timed out"))
later.state = source.state
list(later.fetch())
assert json.loads(source.state["ats:failing:greenhouse:gone"]) == {
"since": "2026-09-01T00:00:00+00:00", "reason": "ConnectTimeout: timed out"}
def test_an_answer_clears_the_failure_and_stamps_completeness(
monkeypatch: pytest.MonkeyPatch) -> None:
source = _source(monkeypatch, (_rows("greenhouse"), False))
source.state["ats:failing:greenhouse:gone"] = json.dumps(
{"since": "2026-09-01T00:00:00+00:00", "reason": "404 Not Found"})
assert len(list(source.fetch())) == 2
assert "ats:failing:greenhouse:gone" not in source.state
assert source.state["ats:ok:greenhouse:gone"] == "run-1"
assert source.state["ats:complete:greenhouse:gone"] == ""