A second pull (schedule, Run button, `app job pull` in another process) could start while one was fetching and overwrite the stored run marker, so the first run's boards failed their stamp check or a settle closed on the other run's id lists. `claim_run` sets `pull:run` in one conditional statement, only while it equals `pull:settled` or is older than an hour; a refused `pull` job queues no stage and says so in its result and a notice, and the CLI exits 1. Batches carry the marker in their payload, the plugins stamp with it, and the settle reads it off its own batches' payloads before releasing the run. Co-Authored-By: Claude Opus <noreply@anthropic.com>
345 lines
14 KiB
Python
345 lines
14 KiB
Python
"""The `ats` plugin: each host's parser against a saved feed, the function
|
|
classifier, and the fetch allow-list. No network — every fixture is a file
|
|
under `tests/fixtures/ats/`."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from plugins import ats # noqa: E402
|
|
|
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
|
|
|
|
|
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
|
|
path = FIXTURES / f"{host}.{'xml' if host == 'personio' else 'json'}"
|
|
body = path.read_bytes()
|
|
return ats._PARSERS[host](body, slug)
|
|
|
|
|
|
# ------------------------------------------------------------------ hosts
|
|
|
|
def test_greenhouse_parses_titles_locations_and_html_description() -> None:
|
|
rows = _rows("greenhouse")
|
|
# The javascript: and data: rows are refused, same as the two real ones
|
|
# are kept: a feed is external content, never trusted as a URL.
|
|
assert [r["source_id"] for r in rows] == ["1001", "1002"]
|
|
first = rows[0]
|
|
assert first["title"] == "Senior Backend Engineer" # trimmed
|
|
assert first["location"] == "Berlin, Germany"
|
|
assert first["department"] == "Engineering"
|
|
assert first["description"] == "Build our core platform."
|
|
assert first["posted"] is not None and first["posted"].year == 2026
|
|
assert first["remote"] is None
|
|
|
|
|
|
def test_lever_parses_categories_and_workplace_type() -> None:
|
|
rows = _rows("lever")
|
|
assert [r["source_id"] for r in rows] == ["2001", "2002"]
|
|
first = rows[0]
|
|
assert first["department"] == "Engineering"
|
|
assert first["remote"] == "remote"
|
|
assert first["description"] == "Own our platform reliability."
|
|
# createdAt=0 means "no date", not epoch zero.
|
|
assert rows[1]["posted"] is None
|
|
|
|
|
|
def test_ashby_parses_is_remote_and_compensation() -> None:
|
|
rows = _rows("ashby")
|
|
assert [r["source_id"] for r in rows] == ["3001", "3002"]
|
|
assert rows[0]["remote"] is False
|
|
assert rows[0]["salary"] == "70,000-90,000 EUR"
|
|
assert rows[1]["remote"] is True
|
|
|
|
|
|
def test_workable_builds_the_job_url_and_reads_location_and_remote() -> None:
|
|
rows = _rows("workable", slug="huggingface")
|
|
assert [r["source_id"] for r in rows] == ["6125026", "6125041"]
|
|
assert rows[0]["url"] == "https://apply.workable.com/huggingface/j/DB4D7C0EC8/"
|
|
assert rows[0]["location"] == "France"
|
|
assert rows[0]["remote"] is True
|
|
assert rows[0]["department"] == "Product"
|
|
assert rows[1]["location"] == "Berlin, Germany"
|
|
assert rows[1]["remote"] is False
|
|
assert rows[0]["description"] == "" # the list endpoint carries no text
|
|
|
|
|
|
def test_smartrecruiters_builds_the_job_url_from_id_and_slug() -> None:
|
|
rows = _rows("smartrecruiters", slug="Acme")
|
|
assert [r["source_id"] for r in rows] == ["744000151326819", "744000147485112"]
|
|
assert rows[0]["url"] == "https://jobs.smartrecruiters.com/Acme/744000151326819"
|
|
assert rows[0]["location"] == "Berlin, Germany"
|
|
assert rows[1]["remote"] is True
|
|
|
|
|
|
def test_recruitee_parses_salary_bounds_and_careers_url() -> None:
|
|
rows = _rows("recruitee")
|
|
assert [r["source_id"] for r in rows] == ["5001", "5002"]
|
|
assert rows[0]["url"] == "https://acme.example/o/player-support-agent-german-1"
|
|
assert rows[0]["salary"] == "35000-45000 EUR"
|
|
assert rows[1]["salary"] == ""
|
|
|
|
|
|
def test_personio_builds_the_job_url_and_reads_cdata_description() -> None:
|
|
rows = _rows("personio", slug="acme")
|
|
assert [r["source_id"] for r in rows] == ["6001", "6002"]
|
|
assert rows[0]["url"] == "https://acme.jobs.personio.de/job/6001"
|
|
assert rows[0]["description"] == "Grow our pipeline."
|
|
assert rows[0]["remote"] is True # "Remote - Germany"
|
|
assert rows[1]["remote"] is None # "Munich"
|
|
|
|
|
|
# --------------------------------------------------------------- classify
|
|
|
|
def test_classify_matches_department_before_title() -> None:
|
|
assert ats.classify("Engineering", "Marketing Lead") == "engineering"
|
|
assert ats.classify("", "Senior Backend Engineer") == "engineering"
|
|
assert ats.classify("", "Product Manager, Payments") == "product"
|
|
assert ats.classify("", "Data Product Manager") == "product"
|
|
assert ats.classify("", "Machine Learning Engineer") == "data"
|
|
assert ats.classify("", "Executive Assistant to the CEO") == "finance/legal/ops"
|
|
assert ats.classify("", "") == "other"
|
|
|
|
|
|
# ------------------------------------------------------------- allow-list
|
|
|
|
def test_is_known_target_accepts_the_seven_hosts_and_refuses_others() -> None:
|
|
for host in ("greenhouse", "lever", "ashby", "workable", "smartrecruiters",
|
|
"recruitee", "personio"):
|
|
assert ats.is_known_target(f"{host}:acme")
|
|
assert not ats.is_known_target("linkedin:acme")
|
|
assert not ats.is_known_target("workday:acme")
|
|
assert not ats.is_known_target("greenhouse:")
|
|
assert not ats.is_known_target("nohostnoseparator")
|
|
|
|
|
|
def test_allowed_host_refuses_a_foreign_host_per_board() -> None:
|
|
allow = ats._allowed_host("greenhouse", "acme")
|
|
assert allow("boards-api.greenhouse.io")
|
|
assert not allow("boards-api.greenhouse.io.evil.example")
|
|
assert not allow("api.lever.co")
|
|
|
|
# recruitee and personio key off the company's own subdomain, so one
|
|
# company's slug must not pass for another's.
|
|
allow_recruitee = ats._allowed_host("recruitee", "acme")
|
|
assert allow_recruitee("acme.recruitee.com")
|
|
assert not allow_recruitee("other.recruitee.com")
|
|
|
|
|
|
def test_careers_url_is_filled_per_host_and_empty_for_unknown() -> None:
|
|
assert ats.careers_url("greenhouse:acme") == "https://job-boards.greenhouse.io/acme"
|
|
assert ats.careers_url("ashby:acme") == "https://jobs.ashbyhq.com/acme"
|
|
assert ats.careers_url("recruitee:acme") == "https://acme.recruitee.com"
|
|
assert ats.careers_url("workday:acme") == ""
|
|
|
|
|
|
# --------------------------------------------------------------- body cap
|
|
|
|
def test_max_bytes_is_96_mb() -> None:
|
|
"""Raised from openings.go's 8 MB: every URL here is on a fixed allow-list
|
|
of ATS API hosts, not an arbitrary one, and the real run of 2026-09-23
|
|
refused boards up to 41 MB under it."""
|
|
assert ats.MAX_BYTES == 96 << 20
|
|
|
|
|
|
def test_fetch_rows_retries_greenhouse_without_content_on_a_too_large_body(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
calls = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET"):
|
|
calls.append(url)
|
|
if "?content=true" in url:
|
|
raise ats._TooLarge("response too large (100000000 bytes)")
|
|
return (FIXTURES / "greenhouse.json").read_bytes()
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type]
|
|
|
|
assert calls == [
|
|
"https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true",
|
|
"https://boards-api.greenhouse.io/v1/boards/acme/jobs",
|
|
]
|
|
assert [r["source_id"] for r in rows] == ["1001", "1002"]
|
|
assert complete # a host that does not page answers with the whole board
|
|
|
|
|
|
def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
def fake_get(client, url, host, slug, *, method="GET"):
|
|
raise ats._TooLarge("response too large (100000000 bytes)")
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
with pytest.raises(ats._TooLarge):
|
|
ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]
|
|
|
|
|
|
# ----------------------------------------------------------------- paging
|
|
|
|
def _page(name: str) -> bytes:
|
|
return (FIXTURES / name).read_bytes()
|
|
|
|
|
|
def _status(code: int) -> httpx.HTTPStatusError:
|
|
request = httpx.Request("POST", "https://apply.workable.com/")
|
|
return httpx.HTTPStatusError(str(code), request=request,
|
|
response=httpx.Response(code, request=request))
|
|
|
|
|
|
@pytest.fixture
|
|
def no_delay(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
|
|
|
|
|
def test_workable_follows_next_page_to_the_stated_total(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
calls: list[dict[str, Any] | None] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
calls.append(payload)
|
|
return _page("workable-2.json" if payload else "workable.json")
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
|
|
|
assert calls == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}]
|
|
assert [r["source_id"] for r in rows] == ["6125026", "6125041", "6125077"]
|
|
assert complete
|
|
|
|
|
|
def test_smartrecruiters_pages_by_offset_to_total_found(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
urls: list[str] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
urls.append(url)
|
|
return _page("smartrecruiters-2.json" if "offset=2" in url else "smartrecruiters.json")
|
|
|
|
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
|
|
|
base = "https://api.smartrecruiters.com/v1/companies/Acme/postings"
|
|
assert urls == [f"{base}?offset=0&limit=2", f"{base}?offset=2&limit=2"]
|
|
assert len(rows) == 3 and rows[2]["location"] == "Tilburg, Netherlands"
|
|
assert complete
|
|
|
|
|
|
def test_a_board_that_ends_short_of_its_total_is_incomplete(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
"""totalFound says 3; the second page is empty, so two were read."""
|
|
empty = json.dumps({"offset": 2, "limit": 2, "totalFound": 3, "content": []}).encode()
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
return empty if "offset=2" in url else _page("smartrecruiters.json")
|
|
|
|
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
|
assert (len(rows), complete) == (2, False)
|
|
|
|
|
|
def test_a_repeated_row_does_not_count_twice_toward_the_total(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
"""A page that shifted under an offset repeats a role and misses one: three
|
|
rows read, two distinct, against a total of three."""
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
page = json.loads(_page("smartrecruiters.json"))
|
|
if "offset=2" in url:
|
|
page["content"] = page["content"][1:]
|
|
return json.dumps(page).encode()
|
|
|
|
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
|
assert (len(rows), complete) == (2, False)
|
|
|
|
|
|
def test_the_page_cap_leaves_a_board_incomplete(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
calls: list[dict[str, Any] | None] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
calls.append(payload)
|
|
return _page("workable.json") # always another page
|
|
|
|
monkeypatch.setattr(ats, "MAX_PAGES", 3)
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
_rows_read, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
|
assert (len(calls), complete) == (3, False)
|
|
|
|
|
|
def test_a_429_mid_way_keeps_what_was_read_and_is_incomplete(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
if payload:
|
|
raise _status(429)
|
|
return _page("workable.json")
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
|
assert ([r["source_id"] for r in rows], complete) == (["6125026", "6125041"], False)
|
|
|
|
|
|
def test_a_429_on_the_first_page_is_a_failure(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
raise _status(429)
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
|
|
|
|
|
# -------------------------------------------------------- failing boards
|
|
|
|
def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource:
|
|
"""One board, `greenhouse:gone`, whose fetch answers `answer` or raises it."""
|
|
def fake(client: object, target: str) -> object:
|
|
if isinstance(answer, Exception):
|
|
raise answer
|
|
return answer
|
|
|
|
monkeypatch.setattr(ats, "_fetch_rows", fake)
|
|
source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}],
|
|
"run": "run-1"})
|
|
return source
|
|
|
|
|
|
def test_a_failing_board_records_since_and_the_latest_reason(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
source = _source(monkeypatch, _status(404))
|
|
assert list(source.fetch()) == []
|
|
first = json.loads(source.state["ats:failing:greenhouse:gone"])
|
|
assert first["reason"] == "404 Not Found" and first["since"]
|
|
assert "ats:ok:greenhouse:gone" not in source.state
|
|
|
|
# A later failure keeps when the failures began and updates the reason.
|
|
source.state["ats:failing:greenhouse:gone"] = json.dumps(
|
|
first | {"since": "2026-09-01T00:00:00+00:00"})
|
|
later = _source(monkeypatch, httpx.ConnectTimeout("timed out"))
|
|
later.state = source.state
|
|
list(later.fetch())
|
|
assert json.loads(source.state["ats:failing:greenhouse:gone"]) == {
|
|
"since": "2026-09-01T00:00:00+00:00", "reason": "ConnectTimeout: timed out"}
|
|
|
|
|
|
def test_an_answer_clears_the_failure_and_stamps_completeness(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
source = _source(monkeypatch, (_rows("greenhouse"), False))
|
|
source.state["ats:failing:greenhouse:gone"] = json.dumps(
|
|
{"since": "2026-09-01T00:00:00+00:00", "reason": "404 Not Found"})
|
|
assert len(list(source.fetch())) == 2
|
|
assert "ats:failing:greenhouse:gone" not in source.state
|
|
assert source.state["ats:ok:greenhouse:gone"] == "run-1"
|
|
assert source.state["ats:complete:greenhouse:gone"] == ""
|