Files
jobboard/tests/test_ats.py
T
WolfandClaude Opus 5.5 c497fe965d Keep a description's headings, bullets and paragraphs, and draw them in the reader
The readers flattened a posting's HTML into one line with strip_html, so
sections and bullets ran together. _shared.html_text gives a line per
paragraph, '• ' before a list item and '## ' before a heading; every
reader's description goes through it, sections read per role get their
own headings, and the reader pane draws the lines as h3, ul and p.
Excerpts leave the marks out.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 21:40:12 +02:00

795 lines
35 KiB
Python

"""The `ats` plugin: each host's parser against a saved feed, the function
classifier, and the fetch allow-list. No network — every fixture is a file
under `tests/fixtures/ats/`."""
from __future__ import annotations
import json
import sys
from datetime import UTC, datetime
from pathlib import Path
from collections.abc import Callable, Iterator
from typing import Any
import httpx
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import ats # noqa: E402
from plugins.readers import _runner, smartrecruiters # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
path = FIXTURES / f"{host}.{'xml' if host == 'personio' else 'json'}"
body = path.read_bytes()
return ats._PARSERS[host](body, slug)
# ------------------------------------------------------------------ hosts
def test_greenhouse_parses_titles_locations_and_html_description() -> None:
rows = _rows("greenhouse")
# The javascript: and data: rows are refused, same as the four real ones
# are kept: a feed is external content, never trusted as a URL.
assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"]
first = rows[0]
assert first["title"] == "Senior Backend Engineer" # trimmed
assert first["location"] == "Berlin, Germany"
assert first["department"] == "Engineering"
assert first["description"] == "Build our core platform."
assert first["posted"] is not None and first["posted"].year == 2026
assert first["remote"] is None
def test_greenhouse_reads_pay_input_ranges_and_skips_an_hourly_one() -> None:
rows = _rows("greenhouse")
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
(65000, 85000, "EUR")
# "SF Bay Area Hourly Rate" is skipped: a yearly figure invented from an
# hourly one would be wrong.
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
(None, None, "")
def test_greenhouse_salary_text_covers_the_single_hourly_and_zoned_cases() -> None:
"""`salary_text` is written whether or not the pay normalises: a single
yearly range reads plainly, the hourly one that the columns dropped
still reads as a line, several zones are joined and labelled with the
board's own titles, and a range with no currency says so."""
rows = _rows("greenhouse")
assert rows[0]["salary_text"] == "EUR 65,000 to 85,000 per year"
assert rows[1]["salary_text"] == "USD 30 to 45 per hour"
assert rows[2]["salary_text"] == ("Zone 1: USD 180,000 to 220,000 per year; "
"Zone 2: USD 160,000 to 200,000 per year")
assert rows[3]["salary_text"] == "65,000 to 85,000 per year (no currency stated)"
def test_lever_parses_categories_and_workplace_type() -> None:
rows = _rows("lever")
assert [r["source_id"] for r in rows] == ["2001", "2002", "2003"]
first = rows[0]
assert first["department"] == "Engineering"
assert first["remote"] == "remote"
assert first["description"] == "Own our platform reliability."
# createdAt=0 means "no date", not epoch zero.
assert rows[1]["posted"] is None
def test_lever_reads_salary_range_and_skips_an_hourly_wage() -> None:
rows = _rows("lever")
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
(90000, 120000, "EUR")
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
(None, None, "")
def test_lever_salary_text_covers_the_hourly_and_no_currency_cases() -> None:
rows = _rows("lever")
assert rows[0]["salary_text"] == "EUR 90,000 to 120,000 per year"
assert rows[1]["salary_text"] == "USD 25 to 35 per hour"
assert rows[2]["salary_text"] == "80,000 to 100,000 per year (no currency stated)"
def test_ashby_parses_is_remote_and_compensation() -> None:
rows = _rows("ashby")
assert [r["source_id"] for r in rows] == ["3001", "3002", "3003", "3004"]
assert rows[0]["remote"] is False
assert rows[0]["salary_text"] == "EUR 70,000 to 90,000 per year"
assert rows[1]["remote"] is True
def test_ashby_reads_summary_components_only_with_includecompensation() -> None:
# summaryComponents only appears with ?includeCompensation=true on the
# feed URL (see _BOARDS); the second job has no compensation key at all.
rows = _rows("ashby")
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
(70000, 90000, "EUR")
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
(None, None, "")
def test_ashby_salary_text_falls_back_to_prose_and_labels_several_components() -> None:
"""No component with a bound at all (job 3): Ashby's own prose summary is
what is left to show. An hourly Salary component beside a yearly Bonus
one (job 4): neither normalises (`_ashby_salary` only reads a yearly or
monthly `Salary` component), but both are stated, labelled since there
is more than one."""
rows = _rows("ashby")
assert rows[2]["salary_text"] == "Competitive OTE, details on request"
assert rows[3]["salary_text"] == \
"Salary: USD 45 to 60 per hour; Bonus: USD 5,000 to 10,000 per year"
assert (rows[3]["salary_min"], rows[3]["salary_max"], rows[3]["currency"]) == \
(None, None, "")
def test_workable_builds_the_job_url_and_reads_location_and_remote() -> None:
rows = _rows("workable", slug="huggingface")
assert [r["source_id"] for r in rows] == ["6125026", "6125041"]
assert rows[0]["url"] == "https://apply.workable.com/huggingface/j/DB4D7C0EC8/"
assert rows[0]["location"] == "France"
assert rows[0]["remote"] == "remote"
assert rows[0]["department"] == "Product"
assert rows[1]["location"] == "Berlin, Germany"
assert rows[1]["remote"] == "hybrid"
assert rows[0]["description"] == "" # the list endpoint carries no text
assert rows[0]["salary_text"] == "" # Workable's list endpoint states no pay at all
def test_smartrecruiters_builds_the_job_url_from_id_and_slug() -> None:
rows = _rows("smartrecruiters", slug="Acme")
assert [r["source_id"] for r in rows] == ["744000151326819", "744000147485112"]
assert rows[0]["url"] == "https://jobs.smartrecruiters.com/Acme/744000151326819"
assert rows[0]["location"] == "Berlin, Germany"
assert rows[1]["remote"] is True
assert rows[0]["salary_text"] == "" # SmartRecruiters' list endpoint states no pay at all
def test_recruitee_parses_salary_bounds_and_careers_url() -> None:
rows = _rows("recruitee")
assert [r["source_id"] for r in rows] == ["5001", "5002", "5003"]
assert rows[0]["url"] == "https://acme.example/o/player-support-agent-german-1"
assert rows[0]["salary_text"] == "EUR 35,000 to 45,000 per year"
assert rows[1]["salary_text"] == ""
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
(35000, 45000, "EUR")
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
(None, None, "")
def test_recruitee_salary_text_states_an_hourly_rate_with_no_currency() -> None:
rows = _rows("recruitee")
assert rows[2]["salary_text"] == "18 to 24 per hour (no currency stated)"
assert (rows[2]["salary_min"], rows[2]["salary_max"], rows[2]["currency"]) == \
(None, None, "")
def test_personio_builds_the_job_url_and_reads_cdata_description() -> None:
rows = _rows("personio", slug="acme")
assert [r["source_id"] for r in rows] == ["6001", "6002", "6003"]
assert rows[0]["url"] == "https://acme.jobs.personio.de/job/6001"
assert rows[0]["description"] == "Grow our pipeline."
assert rows[0]["places"][0]["remote"] is True # "Remote - Germany"
assert rows[1]["places"][0]["remote"] is None # "Munich"
assert "remote" not in rows[0] # the feed states none
def test_personio_reads_salary_information_when_present() -> None:
rows = _rows("personio", slug="acme")
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
(42000, 55000, "EUR")
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
(None, None, "")
assert rows[0]["salary_text"] == "EUR 42,000 to 55,000 per year"
assert rows[1]["salary_text"] == ""
def test_personio_salary_text_states_a_monthly_rate_with_no_currency() -> None:
rows = _rows("personio", slug="acme")
assert rows[2]["salary_text"] == "3,000 to 3,600 per month (no currency stated)"
# No currency: dropped from the columns like any other host's, even
# though the interval itself (monthly) is one _yearly_amount knows.
assert (rows[2]["salary_min"], rows[2]["salary_max"], rows[2]["currency"]) == \
(None, None, "")
# Trimmed from live feeds of 2026-09-29 (career.kry.se, careers.tobii.com): the
# `tt:` namespace, an item with several locations, one whose city is a country,
# an empty department with a role, and one whose link is not a web URL.
TEAMTAILOR = """<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:tt="https://teamtailor.com/locations">
<channel>
<title>Kry</title>
<description>Job openings</description>
<link>https://career.kry.se/jobs</link>
<item>
<title>Software Engineer</title>
<description>&lt;h1&gt;About the role&lt;/h1&gt;&lt;p&gt;Build &amp;amp; run care.&lt;/p&gt;</description>
<pubDate>Wed, 23 Sep 2026 12:59:53 +0200</pubDate>
<link>https://career.kry.se/jobs/8440833-software-engineer</link>
<remoteStatus>hybrid</remoteStatus>
<guid>39ac1fca-a904-4905-b780-18c599e99b07</guid>
<company_name>Kry</company_name>
<tt:locations>
<tt:location>
<tt:name>Sverige</tt:name><tt:address/><tt:zip/>
<tt:city>Sverige</tt:city><tt:country>Sweden</tt:country>
</tt:location>
<tt:location>
<tt:name>Stockholm, Sverige</tt:name><tt:address>Torsgatan 21</tt:address>
<tt:zip>113 21</tt:zip><tt:city>Stockholm</tt:city><tt:country>Sweden</tt:country>
</tt:location>
<tt:location>
<tt:name>Stockholm, Sverige</tt:name><tt:address>Torsgatan 21</tt:address>
<tt:zip>113 21</tt:zip><tt:city>Stockholm</tt:city><tt:country>Sweden</tt:country>
</tt:location>
</tt:locations>
<tt:department>Engineering</tt:department>
<tt:role/>
</item>
<item>
<title> Läkare </title>
<description>&lt;p&gt;Träffa patienter.&lt;/p&gt;</description>
<pubDate>Fri, 31 Jul 2026 14:51:42 +0200</pubDate>
<link>https://career.kry.se/jobs/8156121-lakare</link>
<remoteStatus>fully</remoteStatus>
<guid>9787a509-ff84-400a-ad68-84e22c4cf2fb</guid>
<tt:locations>
<tt:location>
<tt:name>Sweden</tt:name><tt:address>Karlsrovägen 2D</tt:address>
<tt:zip>182 53</tt:zip><tt:city>Danderyd</tt:city><tt:country>Sweden</tt:country>
</tt:location>
</tt:locations>
<tt:department/>
<tt:role>Software Development</tt:role>
</item>
<item>
<title>Sales Lead</title>
<description></description>
<link>https://career.kry.se/jobs/8156122-sales-lead</link>
<remoteStatus>none</remoteStatus>
<guid>0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb</guid>
<tt:locations/>
<tt:department>Sales</tt:department>
<tt:role/>
</item>
<item>
<title>Elsewhere</title>
<link>javascript:alert(1)</link>
<guid>bad</guid>
</item>
</channel>
</rss>
""".encode()
def test_teamtailor_reads_the_rss_feed_with_its_own_namespace() -> None:
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
# the javascript: link is refused like on the other hosts
assert [r["source_id"] for r in rows] == [
"39ac1fca-a904-4905-b780-18c599e99b07", "9787a509-ff84-400a-ad68-84e22c4cf2fb",
"0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb"]
first = rows[0]
assert first["url"] == "https://career.kry.se/jobs/8440833-software-engineer"
assert first["department"] == "Engineering"
assert first["description"] == "## About the role\nBuild & run care."
assert first["posted"] == datetime(2026, 9, 23, 10, 59, 53, tzinfo=UTC)
assert first["salary_text"] == ""
def test_teamtailor_joins_the_distinct_locations_for_country_of() -> None:
from app.models import country_of
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
# "Sverige" (a city that is a country) once, Stockholm's twin once
assert rows[0]["location"] == "Sverige, Sweden; Stockholm, Sweden"
assert rows[1]["location"] == "Danderyd, Sweden"
assert rows[2]["location"] == ""
assert country_of(rows[0]["location"]) == "SE"
def test_teamtailor_hands_on_remote_status_and_falls_back_to_role_for_department() -> None:
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
assert [r["remote"] for r in rows] == ["hybrid", "fully", "none"] # prep maps them
assert rows[1]["title"] == "Läkare" # trimmed
assert rows[1]["department"] == "Software Development"
assert ats.classify(rows[1]["department"], "Sales Lead") == "sales"
def test_teamtailor_refuses_a_page_that_is_not_a_feed() -> None:
with pytest.raises(ValueError):
ats._parse_teamtailor(b"<rss version='2.0'/>", "kry")
def test_parse_dt_reads_an_rss_date_and_still_refuses_junk() -> None:
assert ats._parse_dt("Tue, 29 Sep 2026 13:36:59 +0200") == \
datetime(2026, 9, 29, 11, 36, 59, tzinfo=UTC)
assert ats._parse_dt("last tuesday") is None
# ------------------------------------------------------------ salary math
def test_yearly_amount_converts_a_monthly_range_by_12() -> None:
assert ats._yearly_amount(3000, 4000, "eur", "month", {"year": 1, "month": 12}) == \
(36000, 48000, "EUR")
def test_yearly_amount_ignores_a_range_with_no_currency() -> None:
assert ats._yearly_amount(3000, 4000, "", "year", {"year": 1, "month": 12}) == \
(None, None, "")
def test_yearly_amount_skips_an_interval_outside_the_factor_table() -> None:
# An hourly or a daily rate: a yearly figure invented from it would be
# wrong, so it is left out rather than multiplied by a guess.
assert ats._yearly_amount(20, 30, "usd", "hour", {"year": 1, "month": 12}) == \
(None, None, "")
def test_greenhouse_salary_skips_more_than_one_range() -> None:
# Several ranges (one per office/zone) give no way to tell which one the
# role in hand is at, so all of them are left out rather than guessed.
ranges = [
{"min_cents": 100000, "max_cents": 200000, "currency_type": "EUR", "title": "Zone 1"},
{"min_cents": 90000, "max_cents": 180000, "currency_type": "EUR", "title": "Zone 2"},
]
assert ats._greenhouse_salary(ranges) == (None, None, "")
# --------------------------------------------------------------- classify
def test_classify_matches_title_before_department() -> None:
# A specific title beats a generic department (see classify's docstring
# for the hand-checked sample this is based on).
assert ats.classify("Engineering", "Marketing Lead") == "marketing"
assert ats.classify("Operations", "Senior Backend Engineer") == "engineering"
assert ats.classify("", "Product Manager, Payments") == "product"
assert ats.classify("", "Data Product Manager") == "product"
assert ats.classify("", "Machine Learning Engineer") == "data"
assert ats.classify("", "Executive Assistant to the CEO") == "finance/legal/ops"
# The department is still the fallback when the title matches nothing.
assert ats.classify("Engineering", "") == "engineering"
# Nothing matches either: '', not "other", which would claim a
# knowledge the classifier does not have.
assert ats.classify("", "") == ""
assert ats.classify("", "Chief Bottle Washer") == ""
def test_classify_matches_whole_words_not_substrings() -> None:
# "it" used to match inside "Italy"/"Italia" ("IT" meant the department,
# not the country), filing a sales role as engineering.
assert ats.classify("Commercial Italy", "Account Manager - Liguria") == "sales"
assert ats.classify("IT", "") == "engineering"
assert ats.classify("", "Marketing Lead, Seoul") == "marketing"
# A stem still matches its suffixed forms: "Engineering" the department,
# not just "engineer" the exact word.
assert ats.classify("Engineering", "") == "engineering"
assert ats.classify("", "Software Developers") == "engineering"
def test_classify_understands_a_modest_set_of_other_languages() -> None:
assert ats.classify("", "Senior Entwickler (m/w/d)") == "engineering"
assert ats.classify("", "Vertriebsmitarbeiter (m/w/d)") == "sales"
assert ats.classify("", "Ontwikkelaar") == "engineering"
assert ats.classify("", "Udvikler") == "engineering"
assert ats.classify("", "Ingénieur logiciel") == "engineering"
# ------------------------------------------------------------- allow-list
def test_is_known_target_accepts_the_eight_hosts_and_refuses_others() -> None:
for host in ("greenhouse", "lever", "ashby", "workable", "smartrecruiters",
"recruitee", "personio", "teamtailor"):
assert ats.is_known_target(f"{host}:acme")
assert not ats.is_known_target("linkedin:acme")
assert not ats.is_known_target("taleo:acme")
assert not ats.is_known_target("greenhouse:")
assert not ats.is_known_target("nohostnoseparator")
# a Teamtailor slug names a host: its own subdomain, or a whole domain
assert ats.is_known_target("teamtailor:careers.acme.com")
assert not ats.is_known_target("teamtailor:127.0.0.1")
assert not ats.is_known_target("teamtailor:acme.com/x")
assert not ats.is_known_target("teamtailor:acme com")
def test_allowed_host_refuses_a_foreign_host_per_board() -> None:
allow = ats._allowed_host("greenhouse", "acme")
assert allow("boards-api.greenhouse.io")
assert not allow("boards-api.greenhouse.io.evil.example")
assert not allow("api.lever.co")
# recruitee and personio key off the company's own subdomain, so one
# company's slug must not pass for another's.
allow_recruitee = ats._allowed_host("recruitee", "acme")
assert allow_recruitee("acme.recruitee.com")
assert not allow_recruitee("other.recruitee.com")
# teamtailor: the subdomain of a plain slug, the whole host of a dotted one
assert ats._allowed_host("teamtailor", "acme")("acme.teamtailor.com")
assert not ats._allowed_host("teamtailor", "acme")("other.teamtailor.com")
own = ats._allowed_host("teamtailor", "Careers.Acme.com")
assert own("careers.acme.com")
assert not own("acme.com") and not own("careers.acme.com.evil.example")
def test_careers_url_is_filled_per_host_and_empty_for_unknown() -> None:
assert ats.careers_url("greenhouse:acme") == "https://job-boards.greenhouse.io/acme"
assert ats.careers_url("ashby:acme") == "https://jobs.ashbyhq.com/acme"
assert ats.careers_url("recruitee:acme") == "https://acme.recruitee.com"
assert ats.careers_url("teamtailor:acme") == "https://acme.teamtailor.com/jobs"
assert ats.careers_url("teamtailor:careers.acme.com") == "https://careers.acme.com/jobs"
assert ats.careers_url("taleo:acme") == ""
# --------------------------------------------------------------- body cap
def test_max_bytes_is_96_mb() -> None:
"""Raised from openings.go's 8 MB: every URL here is on a fixed allow-list
of ATS API hosts, not an arbitrary one, and the real run of 2026-09-23
refused boards up to 41 MB under it."""
assert ats.MAX_BYTES == 96 << 20
def test_fetch_rows_retries_greenhouse_without_content_on_a_too_large_body(
monkeypatch: pytest.MonkeyPatch) -> None:
calls = []
def fake_get(client, url, host, slug, *, method="GET"):
calls.append(url)
if "?content=true" in url:
raise ats._TooLarge("response too large (100000000 bytes)")
return (FIXTURES / "greenhouse.json").read_bytes()
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type]
assert calls == [
"https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true&pay_transparency=true",
"https://boards-api.greenhouse.io/v1/boards/acme/jobs",
]
assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"]
assert complete # a host that does not page answers with the whole board
def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
monkeypatch: pytest.MonkeyPatch) -> None:
def fake_get(client, url, host, slug, *, method="GET"):
raise ats._TooLarge("response too large (100000000 bytes)")
monkeypatch.setattr(ats, "_get", fake_get)
with pytest.raises(ats._TooLarge):
ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]
def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client:
return httpx.Client(transport=httpx.MockTransport(handler))
def test_get_stops_reading_at_max_bytes_rather_than_after_the_body(
monkeypatch: pytest.MonkeyPatch) -> None:
sent: list[int] = []
def chunks() -> Iterator[bytes]:
for i in range(100):
sent.append(i)
yield b"x" * 10
monkeypatch.setattr(ats, "MAX_BYTES", 25)
with _client(lambda request: httpx.Response(200, content=chunks())) as client, \
pytest.raises(ats._TooLarge):
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
assert len(sent) == 3 # refused on the third chunk, not the hundredth
def test_get_gives_up_at_the_deadline_however_steadily_bytes_arrive(
monkeypatch: pytest.MonkeyPatch) -> None:
clock = iter(range(0, 1000, 10))
monkeypatch.setattr(_runner.time, "monotonic", lambda: float(next(clock)))
monkeypatch.setattr(ats, "DEADLINE", 25.0)
body = (b"x" for _ in range(100))
with _client(lambda request: httpx.Response(200, content=body)) as client, \
pytest.raises(httpx.ReadTimeout, match="within 25 s"):
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
def test_get_follows_an_allowed_redirect_and_refuses_a_foreign_one() -> None:
def handler(request: httpx.Request) -> httpx.Response:
if request.url.path == "/v0/postings/old":
return httpx.Response(301, headers={"location": "/v0/postings/acme"})
if request.url.path == "/v0/postings/away":
return httpx.Response(302, headers={"location": "https://evil.example/"})
return httpx.Response(200, content=b"[]")
with _client(handler) as client:
assert ats._get(client, "https://api.lever.co/v0/postings/old", "lever", "acme") == b"[]"
with pytest.raises(ats._Refused):
ats._get(client, "https://api.lever.co/v0/postings/away", "lever", "acme")
def test_paging_reads_through_the_streamed_get(no_delay: None) -> None:
"""`_paged` over the real `_get`: the POST body carries the cursor, and a
429 on a later page still reaches it as an `HTTPStatusError`."""
bodies: list[Any] = []
def handler(request: httpx.Request) -> httpx.Response:
payload = json.loads(request.content or b"{}")
bodies.append(payload)
if payload.get("token") == "stop":
return httpx.Response(429)
page = json.loads(_page("workable-2.json" if payload else "workable.json"))
if payload:
page["total"], page["nextPage"] = 4, "stop"
return httpx.Response(200, json=page)
with _client(handler) as client:
rows, complete = ats._fetch_rows(client, "workable:exotec")
assert bodies == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}, {"token": "stop"}]
assert (len(rows), complete) == (3, False)
# ----------------------------------------------------------------- paging
def _page(name: str) -> bytes:
return (FIXTURES / name).read_bytes()
def _status(code: int) -> httpx.HTTPStatusError:
request = httpx.Request("POST", "https://apply.workable.com/")
return httpx.HTTPStatusError(str(code), request=request,
response=httpx.Response(code, request=request))
@pytest.fixture
def no_delay(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
def test_workable_follows_next_page_to_the_stated_total(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
calls: list[dict[str, Any] | None] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None):
calls.append(payload)
return _page("workable-2.json" if payload else "workable.json")
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
assert calls == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}]
assert [r["source_id"] for r in rows] == ["6125026", "6125041", "6125077"]
assert complete
def test_smartrecruiters_pages_by_offset_to_total_found(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
urls: list[str] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None):
urls.append(url)
return _page("smartrecruiters-2.json" if "offset=2" in url else "smartrecruiters.json")
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
base = "https://api.smartrecruiters.com/v1/companies/Acme/postings"
assert urls == [f"{base}?offset=0&limit=2", f"{base}?offset=2&limit=2"]
assert len(rows) == 3 and rows[2]["location"] == "Tilburg, Netherlands"
assert complete
def test_a_board_that_ends_short_of_its_total_is_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
"""totalFound says 3; the second page is empty, so two were read."""
empty = json.dumps({"offset": 2, "limit": 2, "totalFound": 3, "content": []}).encode()
def fake_get(client, url, host, slug, *, method="GET", payload=None):
return empty if "offset=2" in url else _page("smartrecruiters.json")
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
assert (len(rows), complete) == (2, False)
def test_a_repeated_row_does_not_count_twice_toward_the_total(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
"""A page that shifted under an offset repeats a role and misses one: three
rows read, two distinct, against a total of three."""
def fake_get(client, url, host, slug, *, method="GET", payload=None):
page = json.loads(_page("smartrecruiters.json"))
if "offset=2" in url:
page["content"] = page["content"][1:]
return json.dumps(page).encode()
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
assert (len(rows), complete) == (2, False)
def test_the_page_cap_leaves_a_board_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
calls: list[dict[str, Any] | None] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None):
calls.append(payload)
return _page("workable.json") # always another page
monkeypatch.setattr(ats, "MAX_PAGES", 3)
monkeypatch.setattr(ats, "_get", fake_get)
_rows_read, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
assert (len(calls), complete) == (3, False)
def test_a_429_mid_way_keeps_what_was_read_and_is_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
def fake_get(client, url, host, slug, *, method="GET", payload=None):
if payload:
raise _status(429)
return _page("workable.json")
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
assert ([r["source_id"] for r in rows], complete) == (["6125026", "6125041"], False)
def test_a_429_on_the_first_page_is_a_failure(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
def fake_get(client, url, host, slug, *, method="GET", payload=None):
raise _status(429)
monkeypatch.setattr(ats, "_get", fake_get)
with pytest.raises(httpx.HTTPStatusError):
ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
def _feed(guids: list[str]) -> bytes:
"""A Teamtailor feed of `guids`, one role each."""
items = "".join(
f"<item><title>Role {g}</title><link>https://careers.acme.com/jobs/{g}</link>"
f"<guid>{g}</guid></item>" for g in guids)
return ('<rss xmlns:tt="https://teamtailor.com/locations"><channel>'
f"{items}</channel></rss>").encode()
def _teamtailor_web(monkeypatch: pytest.MonkeyPatch, guids: list[str],
page: int = 2) -> list[str]:
"""`_get` answering from `guids` by `offset` and `per_page`, as a
Teamtailor site does, returning the URLs it was asked."""
urls: list[str] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None):
urls.append(url)
query = dict(httpx.URL(url).params)
start, size = int(query["offset"]), min(int(query["per_page"]), page)
return _feed(guids[start:start + size])
monkeypatch.setattr(ats, "_get", fake_get)
return urls
def test_teamtailor_pages_by_offset_and_is_complete_on_an_empty_page(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
"""The host caps a page below what was asked; the offset advances by what
came back, and the read ends on the empty page after the last role."""
urls = _teamtailor_web(monkeypatch, ["a", "b", "c", "d", "e"])
rows, complete = ats._fetch_rows(object(), "teamtailor:careers.acme.com") # type: ignore[arg-type]
base = "https://careers.acme.com/jobs.rss?per_page=50"
assert urls == [f"{base}&offset=0", f"{base}&offset=2", f"{base}&offset=4",
f"{base}&offset=5"]
assert [r["source_id"] for r in rows] == ["a", "b", "c", "d", "e"]
assert complete
def test_a_teamtailor_slug_without_a_dot_is_a_teamtailor_subdomain(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
urls = _teamtailor_web(monkeypatch, ["a"])
ats._fetch_rows(object(), "teamtailor:acme") # type: ignore[arg-type]
assert urls[0].startswith("https://acme.teamtailor.com/jobs.rss?")
def test_a_board_with_no_roles_is_complete_at_zero(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
_teamtailor_web(monkeypatch, [])
assert ats._fetch_rows(object(), "teamtailor:acme") == ([], True) # type: ignore[arg-type]
def test_a_teamtailor_host_that_ignores_the_offset_is_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
"""The same page again: nothing new, so the read stops rather than loops."""
calls: list[str] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None):
calls.append(url)
return _feed(["a", "b"])
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "teamtailor:acme") # type: ignore[arg-type]
assert (len(calls), len(rows), complete) == (2, 2, False)
def test_the_page_cap_leaves_a_teamtailor_board_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
_teamtailor_web(monkeypatch, [str(n) for n in range(10)])
monkeypatch.setattr(ats, "MAX_PAGES", 2)
rows, complete = ats._fetch_rows(object(), "teamtailor:acme") # type: ignore[arg-type]
assert (len(rows), complete) == (4, False)
def test_a_teamtailor_429_after_the_first_page_is_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
def fake_get(client, url, host, slug, *, method="GET", payload=None):
if "offset=0" not in url:
raise _status(429)
return _feed(["a", "b"])
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "teamtailor:acme") # type: ignore[arg-type]
assert ([r["source_id"] for r in rows], complete) == (["a", "b"], False)
def test_teamtailor_is_read_through_the_streamed_get_and_a_redirect_elsewhere_is_refused(
no_delay: None) -> None:
def handler(request: httpx.Request) -> httpx.Response:
if request.url.host == "moved.acme.com":
return httpx.Response(302, headers={"location": "https://evil.example/jobs.rss"})
assert request.url.path == "/jobs.rss"
return httpx.Response(200, content=_feed(["a"] if request.url.params["offset"] == "0" else []))
with _client(handler) as client:
rows, complete = ats._fetch_rows(client, "teamtailor:careers.acme.com")
assert ([r["source_id"] for r in rows], complete) == (["a"], True)
with pytest.raises(ats._Refused):
ats._fetch_rows(client, "teamtailor:moved.acme.com")
# -------------------------------------------------------- failing boards
def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource:
"""One board, `greenhouse:gone`, whose fetch answers `answer` or raises it."""
def fake(*args: object, **kwargs: object) -> _runner.Listing:
if isinstance(answer, Exception):
raise answer
rows, complete = answer # type: ignore[misc]
return _runner.Listing(rows, complete, [row["source_id"] for row in rows])
monkeypatch.setattr(_runner, "read_listing", fake)
source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}],
"run": "run-1"})
return source
def test_a_failing_board_records_since_and_the_latest_reason(
monkeypatch: pytest.MonkeyPatch) -> None:
source = _source(monkeypatch, _status(404))
assert list(source.fetch()) == []
first = json.loads(source.state["ats:failing:greenhouse:gone"])
assert first["reason"] == "404 Not Found" and first["since"]
assert "ats:ok:greenhouse:gone" not in source.state
# A later failure keeps when the failures began and updates the reason.
source.state["ats:failing:greenhouse:gone"] = json.dumps(
first | {"since": "2026-09-01T00:00:00+00:00"})
later = _source(monkeypatch, httpx.ConnectTimeout("timed out"))
later.state = source.state
list(later.fetch())
assert json.loads(source.state["ats:failing:greenhouse:gone"]) == {
"since": "2026-09-01T00:00:00+00:00", "reason": "ConnectTimeout: timed out"}
def test_an_answer_clears_the_failure_and_stamps_completeness(
monkeypatch: pytest.MonkeyPatch) -> None:
source = _source(monkeypatch, (_rows("greenhouse"), False))
source.state["ats:failing:greenhouse:gone"] = json.dumps(
{"since": "2026-09-01T00:00:00+00:00", "reason": "404 Not Found"})
assert len(list(source.fetch())) == 4
assert "ats:failing:greenhouse:gone" not in source.state
assert source.state["ats:ok:greenhouse:gone"] == "run-1"
assert source.state["ats:complete:greenhouse:gone"] == ""