802 lines
35 KiB
Python
802 lines
35 KiB
Python
"""The `ats` plugin: each host's parser against a saved feed, the function
|
|
classifier, and the fetch allow-list. No network — every fixture is a file
|
|
under `tests/fixtures/ats/`."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from collections.abc import Callable, Iterator
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from plugins import ats # noqa: E402
|
|
from plugins.readers import _runner, smartrecruiters # noqa: E402
|
|
|
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
|
|
|
|
|
def _parsed(host: str, slug: str = "acme") -> list[ats.Row]:
|
|
path = FIXTURES / f"{host}.{'xml' if host == 'personio' else 'json'}"
|
|
return ats._PARSERS[host](path.read_bytes(), slug)
|
|
|
|
|
|
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
|
|
"""The rows of a host's fixture the runner keeps (`_runner.valid`)."""
|
|
return [row for row in _parsed(host, slug) if _runner.valid(row)]
|
|
|
|
|
|
# ------------------------------------------------------------------ hosts
|
|
|
|
def test_greenhouse_parses_titles_locations_and_html_description() -> None:
|
|
# The javascript: and data: rows are handed over and the runner drops
|
|
# them: a feed is external content, never trusted as a URL.
|
|
assert [r["source_id"] for r in _parsed("greenhouse")] == [
|
|
"1001", "1002", "1003", "1004", "1005", "1006"]
|
|
rows = _rows("greenhouse")
|
|
assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"]
|
|
first = rows[0]
|
|
assert first["title"] == "Senior Backend Engineer" # trimmed
|
|
assert first["location"] == "Berlin, Germany"
|
|
assert first["department"] == "Engineering"
|
|
assert first["description"] == "Build our core platform."
|
|
assert first["posted"] is not None and first["posted"].year == 2026
|
|
assert first["remote"] is None
|
|
|
|
|
|
def test_greenhouse_reads_pay_input_ranges_and_skips_an_hourly_one() -> None:
|
|
rows = _rows("greenhouse")
|
|
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
|
|
(65000, 85000, "EUR")
|
|
# "SF Bay Area Hourly Rate" is skipped: a yearly figure invented from an
|
|
# hourly one would be wrong.
|
|
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
|
|
(None, None, "")
|
|
|
|
|
|
def test_greenhouse_salary_text_covers_the_single_hourly_and_zoned_cases() -> None:
|
|
"""`salary_text` is written whether or not the pay normalises: a single
|
|
yearly range reads plainly, the hourly one that the columns dropped
|
|
still reads as a line, several zones are joined and labelled with the
|
|
board's own titles, and a range with no currency says so."""
|
|
rows = _rows("greenhouse")
|
|
assert rows[0]["salary_text"] == "EUR 65,000 to 85,000 per year"
|
|
assert rows[1]["salary_text"] == "USD 30 to 45 per hour"
|
|
assert rows[2]["salary_text"] == ("Zone 1: USD 180,000 to 220,000 per year; "
|
|
"Zone 2: USD 160,000 to 200,000 per year")
|
|
assert rows[3]["salary_text"] == "65,000 to 85,000 per year (no currency stated)"
|
|
|
|
|
|
def test_lever_parses_categories_and_workplace_type() -> None:
|
|
rows = _rows("lever")
|
|
assert [r["source_id"] for r in rows] == ["2001", "2002", "2003"]
|
|
first = rows[0]
|
|
assert first["department"] == "Engineering"
|
|
assert first["remote"] == "remote"
|
|
assert first["description"] == "Own our platform reliability."
|
|
# createdAt=0 means "no date", not epoch zero.
|
|
assert rows[1]["posted"] is None
|
|
|
|
|
|
def test_lever_reads_salary_range_and_skips_an_hourly_wage() -> None:
|
|
rows = _rows("lever")
|
|
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
|
|
(90000, 120000, "EUR")
|
|
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
|
|
(None, None, "")
|
|
|
|
|
|
def test_lever_salary_text_covers_the_hourly_and_no_currency_cases() -> None:
|
|
rows = _rows("lever")
|
|
assert rows[0]["salary_text"] == "EUR 90,000 to 120,000 per year"
|
|
assert rows[1]["salary_text"] == "USD 25 to 35 per hour"
|
|
assert rows[2]["salary_text"] == "80,000 to 100,000 per year (no currency stated)"
|
|
|
|
|
|
def test_ashby_parses_is_remote_and_compensation() -> None:
|
|
rows = _rows("ashby")
|
|
assert [r["source_id"] for r in rows] == ["3001", "3002", "3003", "3004"]
|
|
assert rows[0]["remote"] is False
|
|
assert rows[0]["salary_text"] == "EUR 70,000 to 90,000 per year"
|
|
assert rows[1]["remote"] is True
|
|
|
|
|
|
def test_ashby_reads_summary_components_only_with_includecompensation() -> None:
|
|
# summaryComponents only appears with ?includeCompensation=true on the
|
|
# feed URL (see _BOARDS); the second job has no compensation key at all.
|
|
rows = _rows("ashby")
|
|
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
|
|
(70000, 90000, "EUR")
|
|
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
|
|
(None, None, "")
|
|
|
|
|
|
def test_ashby_salary_text_falls_back_to_prose_and_labels_several_components() -> None:
|
|
"""No component with a bound at all (job 3): Ashby's own prose summary is
|
|
what is left to show. An hourly Salary component beside a yearly Bonus
|
|
one (job 4): neither normalises (`_ashby_salary` only reads a yearly or
|
|
monthly `Salary` component), but both are stated, labelled since there
|
|
is more than one."""
|
|
rows = _rows("ashby")
|
|
assert rows[2]["salary_text"] == "Competitive OTE, details on request"
|
|
assert rows[3]["salary_text"] == \
|
|
"Salary: USD 45 to 60 per hour; Bonus: USD 5,000 to 10,000 per year"
|
|
assert (rows[3]["salary_min"], rows[3]["salary_max"], rows[3]["currency"]) == \
|
|
(None, None, "")
|
|
|
|
|
|
def test_workable_builds_the_job_url_and_reads_location_and_remote() -> None:
|
|
rows = _rows("workable", slug="huggingface")
|
|
assert [r["source_id"] for r in rows] == ["6125026", "6125041"]
|
|
assert rows[0]["url"] == "https://apply.workable.com/huggingface/j/DB4D7C0EC8/"
|
|
assert rows[0]["location"] == "France"
|
|
assert rows[0]["remote"] == "remote"
|
|
assert rows[0]["department"] == "Product"
|
|
assert rows[1]["location"] == "Berlin, Germany"
|
|
assert rows[1]["remote"] == "hybrid"
|
|
assert rows[0]["description"] == "" # the list endpoint carries no text
|
|
assert rows[0]["salary_text"] == "" # Workable's list endpoint states no pay at all
|
|
|
|
|
|
def test_smartrecruiters_builds_the_job_url_from_id_and_slug() -> None:
|
|
rows = _rows("smartrecruiters", slug="Acme")
|
|
assert [r["source_id"] for r in rows] == ["744000151326819", "744000147485112"]
|
|
assert rows[0]["url"] == "https://jobs.smartrecruiters.com/Acme/744000151326819"
|
|
assert rows[0]["location"] == "Berlin, Germany"
|
|
assert rows[1]["remote"] is True
|
|
assert rows[0]["salary_text"] == "" # SmartRecruiters' list endpoint states no pay at all
|
|
|
|
|
|
def test_recruitee_parses_salary_bounds_and_careers_url() -> None:
|
|
rows = _rows("recruitee")
|
|
assert [r["source_id"] for r in rows] == ["5001", "5002", "5003"]
|
|
assert rows[0]["url"] == "https://acme.example/o/player-support-agent-german-1"
|
|
assert rows[0]["salary_text"] == "EUR 35,000 to 45,000 per year"
|
|
assert rows[1]["salary_text"] == ""
|
|
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
|
|
(35000, 45000, "EUR")
|
|
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
|
|
(None, None, "")
|
|
|
|
|
|
def test_recruitee_salary_text_states_an_hourly_rate_with_no_currency() -> None:
|
|
rows = _rows("recruitee")
|
|
assert rows[2]["salary_text"] == "18 to 24 per hour (no currency stated)"
|
|
assert (rows[2]["salary_min"], rows[2]["salary_max"], rows[2]["currency"]) == \
|
|
(None, None, "")
|
|
|
|
|
|
def test_personio_builds_the_job_url_and_reads_cdata_description() -> None:
|
|
rows = _rows("personio", slug="acme")
|
|
assert [r["source_id"] for r in rows] == ["6001", "6002", "6003"]
|
|
assert rows[0]["url"] == "https://acme.jobs.personio.de/job/6001"
|
|
assert rows[0]["description"] == "Grow our pipeline."
|
|
assert rows[0]["places"][0]["remote"] is True # "Remote - Germany"
|
|
assert rows[1]["places"][0]["remote"] is None # "Munich"
|
|
assert "remote" not in rows[0] # the feed states none
|
|
|
|
|
|
def test_personio_reads_salary_information_when_present() -> None:
|
|
rows = _rows("personio", slug="acme")
|
|
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
|
|
(42000, 55000, "EUR")
|
|
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
|
|
(None, None, "")
|
|
assert rows[0]["salary_text"] == "EUR 42,000 to 55,000 per year"
|
|
assert rows[1]["salary_text"] == ""
|
|
|
|
|
|
def test_personio_salary_text_states_a_monthly_rate_with_no_currency() -> None:
|
|
rows = _rows("personio", slug="acme")
|
|
assert rows[2]["salary_text"] == "3,000 to 3,600 per month (no currency stated)"
|
|
# No currency: dropped from the columns like any other host's, even
|
|
# though the interval itself (monthly) is one _yearly_amount knows.
|
|
assert (rows[2]["salary_min"], rows[2]["salary_max"], rows[2]["currency"]) == \
|
|
(None, None, "")
|
|
|
|
|
|
# Trimmed from live feeds of 2026-09-29 (career.kry.se, careers.tobii.com): the
|
|
# `tt:` namespace, an item with several locations, one whose city is a country,
|
|
# an empty department with a role, and one whose link is not a web URL.
|
|
TEAMTAILOR = """<?xml version="1.0" encoding="UTF-8"?>
|
|
<rss version="2.0" xmlns:tt="https://teamtailor.com/locations">
|
|
<channel>
|
|
<title>Kry</title>
|
|
<description>Job openings</description>
|
|
<link>https://career.kry.se/jobs</link>
|
|
<item>
|
|
<title>Software Engineer</title>
|
|
<description><h1>About the role</h1><p>Build &amp; run care.</p></description>
|
|
<pubDate>Wed, 23 Sep 2026 12:59:53 +0200</pubDate>
|
|
<link>https://career.kry.se/jobs/8440833-software-engineer</link>
|
|
<remoteStatus>hybrid</remoteStatus>
|
|
<guid>39ac1fca-a904-4905-b780-18c599e99b07</guid>
|
|
<company_name>Kry</company_name>
|
|
<tt:locations>
|
|
<tt:location>
|
|
<tt:name>Sverige</tt:name><tt:address/><tt:zip/>
|
|
<tt:city>Sverige</tt:city><tt:country>Sweden</tt:country>
|
|
</tt:location>
|
|
<tt:location>
|
|
<tt:name>Stockholm, Sverige</tt:name><tt:address>Torsgatan 21</tt:address>
|
|
<tt:zip>113 21</tt:zip><tt:city>Stockholm</tt:city><tt:country>Sweden</tt:country>
|
|
</tt:location>
|
|
<tt:location>
|
|
<tt:name>Stockholm, Sverige</tt:name><tt:address>Torsgatan 21</tt:address>
|
|
<tt:zip>113 21</tt:zip><tt:city>Stockholm</tt:city><tt:country>Sweden</tt:country>
|
|
</tt:location>
|
|
</tt:locations>
|
|
<tt:department>Engineering</tt:department>
|
|
<tt:role/>
|
|
</item>
|
|
<item>
|
|
<title> Läkare </title>
|
|
<description><p>Träffa patienter.</p></description>
|
|
<pubDate>Fri, 31 Jul 2026 14:51:42 +0200</pubDate>
|
|
<link>https://career.kry.se/jobs/8156121-lakare</link>
|
|
<remoteStatus>fully</remoteStatus>
|
|
<guid>9787a509-ff84-400a-ad68-84e22c4cf2fb</guid>
|
|
<tt:locations>
|
|
<tt:location>
|
|
<tt:name>Sweden</tt:name><tt:address>Karlsrovägen 2D</tt:address>
|
|
<tt:zip>182 53</tt:zip><tt:city>Danderyd</tt:city><tt:country>Sweden</tt:country>
|
|
</tt:location>
|
|
</tt:locations>
|
|
<tt:department/>
|
|
<tt:role>Software Development</tt:role>
|
|
</item>
|
|
<item>
|
|
<title>Sales Lead</title>
|
|
<description></description>
|
|
<link>https://career.kry.se/jobs/8156122-sales-lead</link>
|
|
<remoteStatus>none</remoteStatus>
|
|
<guid>0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb</guid>
|
|
<tt:locations/>
|
|
<tt:department>Sales</tt:department>
|
|
<tt:role/>
|
|
</item>
|
|
<item>
|
|
<title>Elsewhere</title>
|
|
<link>javascript:alert(1)</link>
|
|
<guid>bad</guid>
|
|
</item>
|
|
</channel>
|
|
</rss>
|
|
""".encode()
|
|
|
|
|
|
def test_teamtailor_reads_the_rss_feed_with_its_own_namespace() -> None:
|
|
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
|
# the javascript: link is handed over and refused by the runner
|
|
assert [r["source_id"] for r in rows] == [
|
|
"39ac1fca-a904-4905-b780-18c599e99b07", "9787a509-ff84-400a-ad68-84e22c4cf2fb",
|
|
"0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb", "bad"]
|
|
assert [_runner.valid(r) for r in rows] == [True, True, True, False]
|
|
first = rows[0]
|
|
assert first["url"] == "https://career.kry.se/jobs/8440833-software-engineer"
|
|
assert first["department"] == "Engineering"
|
|
assert first["description"] == "## About the role\nBuild & run care."
|
|
assert first["posted"] == datetime(2026, 9, 23, 10, 59, 53, tzinfo=UTC)
|
|
assert first["salary_text"] == ""
|
|
|
|
|
|
def test_teamtailor_joins_the_distinct_locations_for_country_of() -> None:
|
|
from app.models import country_of
|
|
|
|
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
|
# "Sverige" (a city that is a country) once, Stockholm's twin once
|
|
assert rows[0]["location"] == "Sverige, Sweden; Stockholm, Sweden"
|
|
assert rows[1]["location"] == "Danderyd, Sweden"
|
|
assert rows[2]["location"] == ""
|
|
assert country_of(rows[0]["location"]) == "SE"
|
|
|
|
|
|
def test_teamtailor_hands_on_remote_status_and_falls_back_to_role_for_department() -> None:
|
|
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
|
assert [r["remote"] for r in rows[:3]] == ["hybrid", "fully", "none"] # prep maps them
|
|
assert rows[1]["title"] == "Läkare" # trimmed
|
|
assert rows[1]["department"] == "Software Development"
|
|
assert ats.classify(rows[1]["department"], "Sales Lead") == "sales"
|
|
|
|
|
|
def test_teamtailor_refuses_a_page_that_is_not_a_feed() -> None:
|
|
with pytest.raises(ValueError):
|
|
ats._parse_teamtailor(b"<rss version='2.0'/>", "kry")
|
|
|
|
|
|
def test_parse_dt_reads_an_rss_date_and_still_refuses_junk() -> None:
|
|
assert ats._parse_dt("Tue, 29 Sep 2026 13:36:59 +0200") == \
|
|
datetime(2026, 9, 29, 11, 36, 59, tzinfo=UTC)
|
|
assert ats._parse_dt("last tuesday") is None
|
|
|
|
|
|
# ------------------------------------------------------------ salary math
|
|
|
|
def test_yearly_amount_converts_a_monthly_range_by_12() -> None:
|
|
assert ats._yearly_amount(3000, 4000, "eur", "month", {"year": 1, "month": 12}) == \
|
|
(36000, 48000, "EUR")
|
|
|
|
|
|
def test_yearly_amount_ignores_a_range_with_no_currency() -> None:
|
|
assert ats._yearly_amount(3000, 4000, "", "year", {"year": 1, "month": 12}) == \
|
|
(None, None, "")
|
|
|
|
|
|
def test_yearly_amount_skips_an_interval_outside_the_factor_table() -> None:
|
|
# An hourly or a daily rate: a yearly figure invented from it would be
|
|
# wrong, so it is left out rather than multiplied by a guess.
|
|
assert ats._yearly_amount(20, 30, "usd", "hour", {"year": 1, "month": 12}) == \
|
|
(None, None, "")
|
|
|
|
|
|
def test_greenhouse_salary_skips_more_than_one_range() -> None:
|
|
# Several ranges (one per office/zone) give no way to tell which one the
|
|
# role in hand is at, so all of them are left out rather than guessed.
|
|
ranges = [
|
|
{"min_cents": 100000, "max_cents": 200000, "currency_type": "EUR", "title": "Zone 1"},
|
|
{"min_cents": 90000, "max_cents": 180000, "currency_type": "EUR", "title": "Zone 2"},
|
|
]
|
|
assert ats._greenhouse_salary(ranges) == (None, None, "")
|
|
|
|
|
|
# --------------------------------------------------------------- classify
|
|
|
|
def test_classify_matches_title_before_department() -> None:
|
|
# A specific title beats a generic department (see classify's docstring
|
|
# for the hand-checked sample this is based on).
|
|
assert ats.classify("Engineering", "Marketing Lead") == "marketing"
|
|
assert ats.classify("Operations", "Senior Backend Engineer") == "engineering"
|
|
assert ats.classify("", "Product Manager, Payments") == "product"
|
|
assert ats.classify("", "Data Product Manager") == "product"
|
|
assert ats.classify("", "Machine Learning Engineer") == "data"
|
|
assert ats.classify("", "Executive Assistant to the CEO") == "finance/legal/ops"
|
|
# The department is still the fallback when the title matches nothing.
|
|
assert ats.classify("Engineering", "") == "engineering"
|
|
# Nothing matches either: '', not "other", which would claim a
|
|
# knowledge the classifier does not have.
|
|
assert ats.classify("", "") == ""
|
|
assert ats.classify("", "Chief Bottle Washer") == ""
|
|
|
|
|
|
def test_classify_matches_whole_words_not_substrings() -> None:
|
|
# "it" used to match inside "Italy"/"Italia" ("IT" meant the department,
|
|
# not the country), filing a sales role as engineering.
|
|
assert ats.classify("Commercial Italy", "Account Manager - Liguria") == "sales"
|
|
assert ats.classify("IT", "") == "engineering"
|
|
assert ats.classify("", "Marketing Lead, Seoul") == "marketing"
|
|
# A stem still matches its suffixed forms: "Engineering" the department,
|
|
# not just "engineer" the exact word.
|
|
assert ats.classify("Engineering", "") == "engineering"
|
|
assert ats.classify("", "Software Developers") == "engineering"
|
|
|
|
|
|
def test_classify_understands_a_modest_set_of_other_languages() -> None:
|
|
assert ats.classify("", "Senior Entwickler (m/w/d)") == "engineering"
|
|
assert ats.classify("", "Vertriebsmitarbeiter (m/w/d)") == "sales"
|
|
assert ats.classify("", "Ontwikkelaar") == "engineering"
|
|
assert ats.classify("", "Udvikler") == "engineering"
|
|
assert ats.classify("", "Ingénieur logiciel") == "engineering"
|
|
|
|
|
|
# ------------------------------------------------------------- allow-list
|
|
|
|
def test_is_known_target_accepts_the_eight_hosts_and_refuses_others() -> None:
|
|
for host in ("greenhouse", "lever", "ashby", "workable", "smartrecruiters",
|
|
"recruitee", "personio", "teamtailor"):
|
|
assert ats.is_known_target(f"{host}:acme")
|
|
assert not ats.is_known_target("linkedin:acme")
|
|
assert not ats.is_known_target("taleo:acme")
|
|
assert not ats.is_known_target("greenhouse:")
|
|
assert not ats.is_known_target("nohostnoseparator")
|
|
# a Teamtailor slug names a host: its own subdomain, or a whole domain
|
|
assert ats.is_known_target("teamtailor:careers.acme.com")
|
|
assert not ats.is_known_target("teamtailor:127.0.0.1")
|
|
assert not ats.is_known_target("teamtailor:acme.com/x")
|
|
assert not ats.is_known_target("teamtailor:acme com")
|
|
|
|
|
|
def test_allowed_host_refuses_a_foreign_host_per_board() -> None:
|
|
allow = ats._allowed_host("greenhouse", "acme")
|
|
assert allow("boards-api.greenhouse.io")
|
|
assert not allow("boards-api.greenhouse.io.evil.example")
|
|
assert not allow("api.lever.co")
|
|
|
|
# recruitee and personio key off the company's own subdomain, so one
|
|
# company's slug must not pass for another's.
|
|
allow_recruitee = ats._allowed_host("recruitee", "acme")
|
|
assert allow_recruitee("acme.recruitee.com")
|
|
assert not allow_recruitee("other.recruitee.com")
|
|
|
|
# teamtailor: the subdomain of a plain slug, the whole host of a dotted one
|
|
assert ats._allowed_host("teamtailor", "acme")("acme.teamtailor.com")
|
|
assert not ats._allowed_host("teamtailor", "acme")("other.teamtailor.com")
|
|
own = ats._allowed_host("teamtailor", "Careers.Acme.com")
|
|
assert own("careers.acme.com")
|
|
assert not own("acme.com") and not own("careers.acme.com.evil.example")
|
|
|
|
|
|
def test_careers_url_is_filled_per_host_and_empty_for_unknown() -> None:
|
|
assert ats.careers_url("greenhouse:acme") == "https://job-boards.greenhouse.io/acme"
|
|
assert ats.careers_url("ashby:acme") == "https://jobs.ashbyhq.com/acme"
|
|
assert ats.careers_url("recruitee:acme") == "https://acme.recruitee.com"
|
|
assert ats.careers_url("teamtailor:acme") == "https://acme.teamtailor.com/jobs"
|
|
assert ats.careers_url("teamtailor:careers.acme.com") == "https://careers.acme.com/jobs"
|
|
assert ats.careers_url("taleo:acme") == ""
|
|
|
|
|
|
# --------------------------------------------------------------- body cap
|
|
|
|
def test_max_bytes_is_96_mb() -> None:
|
|
"""Raised from openings.go's 8 MB: every URL here is on a fixed allow-list
|
|
of ATS API hosts, not an arbitrary one, and the real run of 2026-09-23
|
|
refused boards up to 41 MB under it."""
|
|
assert ats.MAX_BYTES == 96 << 20
|
|
|
|
|
|
def test_fetch_rows_retries_greenhouse_without_content_on_a_too_large_body(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
calls = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET"):
|
|
calls.append(url)
|
|
if "?content=true" in url:
|
|
raise ats._TooLarge("response too large (100000000 bytes)")
|
|
return (FIXTURES / "greenhouse.json").read_bytes()
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type]
|
|
|
|
assert calls == [
|
|
"https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true&pay_transparency=true",
|
|
"https://boards-api.greenhouse.io/v1/boards/acme/jobs",
|
|
]
|
|
assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"]
|
|
assert complete # a host that does not page answers with the whole board
|
|
|
|
|
|
def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
def fake_get(client, url, host, slug, *, method="GET"):
|
|
raise ats._TooLarge("response too large (100000000 bytes)")
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
with pytest.raises(ats._TooLarge):
|
|
ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]
|
|
|
|
|
|
def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client:
|
|
return httpx.Client(transport=httpx.MockTransport(handler))
|
|
|
|
|
|
def test_get_stops_reading_at_max_bytes_rather_than_after_the_body(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
sent: list[int] = []
|
|
|
|
def chunks() -> Iterator[bytes]:
|
|
for i in range(100):
|
|
sent.append(i)
|
|
yield b"x" * 10
|
|
|
|
monkeypatch.setattr(ats, "MAX_BYTES", 25)
|
|
with _client(lambda request: httpx.Response(200, content=chunks())) as client, \
|
|
pytest.raises(ats._TooLarge):
|
|
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
|
|
assert len(sent) == 3 # refused on the third chunk, not the hundredth
|
|
|
|
|
|
def test_get_gives_up_at_the_deadline_however_steadily_bytes_arrive(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
clock = iter(range(0, 1000, 10))
|
|
monkeypatch.setattr(_runner.time, "monotonic", lambda: float(next(clock)))
|
|
monkeypatch.setattr(ats, "DEADLINE", 25.0)
|
|
body = (b"x" for _ in range(100))
|
|
with _client(lambda request: httpx.Response(200, content=body)) as client, \
|
|
pytest.raises(httpx.ReadTimeout, match="within 25 s"):
|
|
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
|
|
|
|
|
|
def test_get_follows_an_allowed_redirect_and_refuses_a_foreign_one() -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if request.url.path == "/v0/postings/old":
|
|
return httpx.Response(301, headers={"location": "/v0/postings/acme"})
|
|
if request.url.path == "/v0/postings/away":
|
|
return httpx.Response(302, headers={"location": "https://evil.example/"})
|
|
return httpx.Response(200, content=b"[]")
|
|
|
|
with _client(handler) as client:
|
|
assert ats._get(client, "https://api.lever.co/v0/postings/old", "lever", "acme") == b"[]"
|
|
with pytest.raises(ats._Refused):
|
|
ats._get(client, "https://api.lever.co/v0/postings/away", "lever", "acme")
|
|
|
|
|
|
def test_paging_reads_through_the_streamed_get(no_delay: None) -> None:
|
|
"""`_paged` over the real `_get`: the POST body carries the cursor, and a
|
|
429 on a later page still reaches it as an `HTTPStatusError`."""
|
|
bodies: list[Any] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
payload = json.loads(request.content or b"{}")
|
|
bodies.append(payload)
|
|
if payload.get("token") == "stop":
|
|
return httpx.Response(429)
|
|
page = json.loads(_page("workable-2.json" if payload else "workable.json"))
|
|
if payload:
|
|
page["total"], page["nextPage"] = 4, "stop"
|
|
return httpx.Response(200, json=page)
|
|
|
|
with _client(handler) as client:
|
|
rows, complete = ats._fetch_rows(client, "workable:exotec")
|
|
assert bodies == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}, {"token": "stop"}]
|
|
assert (len(rows), complete) == (3, False)
|
|
|
|
|
|
# ----------------------------------------------------------------- paging
|
|
|
|
def _page(name: str) -> bytes:
|
|
return (FIXTURES / name).read_bytes()
|
|
|
|
|
|
def _status(code: int) -> httpx.HTTPStatusError:
|
|
request = httpx.Request("POST", "https://apply.workable.com/")
|
|
return httpx.HTTPStatusError(str(code), request=request,
|
|
response=httpx.Response(code, request=request))
|
|
|
|
|
|
@pytest.fixture
|
|
def no_delay(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
|
|
|
|
|
def test_workable_follows_next_page_to_the_stated_total(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
calls: list[dict[str, Any] | None] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
calls.append(payload)
|
|
return _page("workable-2.json" if payload else "workable.json")
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
|
|
|
assert calls == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}]
|
|
assert [r["source_id"] for r in rows] == ["6125026", "6125041", "6125077"]
|
|
assert complete
|
|
|
|
|
|
def test_smartrecruiters_pages_by_offset_to_total_found(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
urls: list[str] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
urls.append(url)
|
|
return _page("smartrecruiters-2.json" if "offset=2" in url else "smartrecruiters.json")
|
|
|
|
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
|
|
|
base = "https://api.smartrecruiters.com/v1/companies/Acme/postings"
|
|
assert urls == [f"{base}?offset=0&limit=2", f"{base}?offset=2&limit=2"]
|
|
assert len(rows) == 3 and rows[2]["location"] == "Tilburg, Netherlands"
|
|
assert complete
|
|
|
|
|
|
def test_a_board_that_ends_short_of_its_total_is_incomplete(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
"""totalFound says 3; the second page is empty, so two were read."""
|
|
empty = json.dumps({"offset": 2, "limit": 2, "totalFound": 3, "content": []}).encode()
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
return empty if "offset=2" in url else _page("smartrecruiters.json")
|
|
|
|
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
|
assert (len(rows), complete) == (2, False)
|
|
|
|
|
|
def test_a_repeated_row_does_not_count_twice_toward_the_total(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
"""A page that shifted under an offset repeats a role and misses one: three
|
|
rows read, two distinct, against a total of three."""
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
page = json.loads(_page("smartrecruiters.json"))
|
|
if "offset=2" in url:
|
|
page["content"] = page["content"][1:]
|
|
return json.dumps(page).encode()
|
|
|
|
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
|
assert (len(rows), complete) == (2, False)
|
|
|
|
|
|
def test_the_page_cap_leaves_a_board_incomplete(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
calls: list[dict[str, Any] | None] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
calls.append(payload)
|
|
return _page("workable.json") # always another page
|
|
|
|
monkeypatch.setattr(ats, "MAX_PAGES", 3)
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
_rows_read, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
|
assert (len(calls), complete) == (3, False)
|
|
|
|
|
|
def test_a_429_mid_way_keeps_what_was_read_and_is_incomplete(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
if payload:
|
|
raise _status(429)
|
|
return _page("workable.json")
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
|
assert ([r["source_id"] for r in rows], complete) == (["6125026", "6125041"], False)
|
|
|
|
|
|
def test_a_429_on_the_first_page_is_a_failure(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
raise _status(429)
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
|
|
|
|
|
def _feed(guids: list[str]) -> bytes:
|
|
"""A Teamtailor feed of `guids`, one role each."""
|
|
items = "".join(
|
|
f"<item><title>Role {g}</title><link>https://careers.acme.com/jobs/{g}</link>"
|
|
f"<guid>{g}</guid></item>" for g in guids)
|
|
return ('<rss xmlns:tt="https://teamtailor.com/locations"><channel>'
|
|
f"{items}</channel></rss>").encode()
|
|
|
|
|
|
def _teamtailor_web(monkeypatch: pytest.MonkeyPatch, guids: list[str],
|
|
page: int = 2) -> list[str]:
|
|
"""`_get` answering from `guids` by `offset` and `per_page`, as a
|
|
Teamtailor site does, returning the URLs it was asked."""
|
|
urls: list[str] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
urls.append(url)
|
|
query = dict(httpx.URL(url).params)
|
|
start, size = int(query["offset"]), min(int(query["per_page"]), page)
|
|
return _feed(guids[start:start + size])
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
return urls
|
|
|
|
|
|
def test_teamtailor_pages_by_offset_and_is_complete_on_an_empty_page(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
"""The host caps a page below what was asked; the offset advances by what
|
|
came back, and the read ends on the empty page after the last role."""
|
|
urls = _teamtailor_web(monkeypatch, ["a", "b", "c", "d", "e"])
|
|
rows, complete = ats._fetch_rows(object(), "teamtailor:careers.acme.com") # type: ignore[arg-type]
|
|
|
|
base = "https://careers.acme.com/jobs.rss?per_page=50"
|
|
assert urls == [f"{base}&offset=0", f"{base}&offset=2", f"{base}&offset=4",
|
|
f"{base}&offset=5"]
|
|
assert [r["source_id"] for r in rows] == ["a", "b", "c", "d", "e"]
|
|
assert complete
|
|
|
|
|
|
def test_a_teamtailor_slug_without_a_dot_is_a_teamtailor_subdomain(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
urls = _teamtailor_web(monkeypatch, ["a"])
|
|
ats._fetch_rows(object(), "teamtailor:acme") # type: ignore[arg-type]
|
|
assert urls[0].startswith("https://acme.teamtailor.com/jobs.rss?")
|
|
|
|
|
|
def test_a_board_with_no_roles_is_complete_at_zero(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
_teamtailor_web(monkeypatch, [])
|
|
assert ats._fetch_rows(object(), "teamtailor:acme") == ([], True) # type: ignore[arg-type]
|
|
|
|
|
|
def test_a_teamtailor_host_that_ignores_the_offset_is_incomplete(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
"""The same page again: nothing new, so the read stops rather than loops."""
|
|
calls: list[str] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
calls.append(url)
|
|
return _feed(["a", "b"])
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "teamtailor:acme") # type: ignore[arg-type]
|
|
assert (len(calls), len(rows), complete) == (2, 2, False)
|
|
|
|
|
|
def test_the_page_cap_leaves_a_teamtailor_board_incomplete(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
_teamtailor_web(monkeypatch, [str(n) for n in range(10)])
|
|
monkeypatch.setattr(ats, "MAX_PAGES", 2)
|
|
rows, complete = ats._fetch_rows(object(), "teamtailor:acme") # type: ignore[arg-type]
|
|
assert (len(rows), complete) == (4, False)
|
|
|
|
|
|
def test_a_teamtailor_429_after_the_first_page_is_incomplete(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
if "offset=0" not in url:
|
|
raise _status(429)
|
|
return _feed(["a", "b"])
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "teamtailor:acme") # type: ignore[arg-type]
|
|
assert ([r["source_id"] for r in rows], complete) == (["a", "b"], False)
|
|
|
|
|
|
def test_teamtailor_is_read_through_the_streamed_get_and_a_redirect_elsewhere_is_refused(
|
|
no_delay: None) -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if request.url.host == "moved.acme.com":
|
|
return httpx.Response(302, headers={"location": "https://evil.example/jobs.rss"})
|
|
assert request.url.path == "/jobs.rss"
|
|
return httpx.Response(200, content=_feed(["a"] if request.url.params["offset"] == "0" else []))
|
|
|
|
with _client(handler) as client:
|
|
rows, complete = ats._fetch_rows(client, "teamtailor:careers.acme.com")
|
|
assert ([r["source_id"] for r in rows], complete) == (["a"], True)
|
|
with pytest.raises(ats._Refused):
|
|
ats._fetch_rows(client, "teamtailor:moved.acme.com")
|
|
|
|
|
|
# -------------------------------------------------------- failing boards
|
|
|
|
def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource:
|
|
"""One board, `greenhouse:gone`, whose fetch answers `answer` or raises it."""
|
|
def fake(*args: object, **kwargs: object) -> _runner.Listing:
|
|
if isinstance(answer, Exception):
|
|
raise answer
|
|
rows, complete = answer # type: ignore[misc]
|
|
return _runner.Listing(rows, complete, [row["source_id"] for row in rows])
|
|
|
|
monkeypatch.setattr(_runner, "read_listing", fake)
|
|
source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}],
|
|
"run": "run-1"})
|
|
return source
|
|
|
|
|
|
def test_a_failing_board_records_since_and_the_latest_reason(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
source = _source(monkeypatch, _status(404))
|
|
assert list(source.fetch()) == []
|
|
first = json.loads(source.state["ats:failing:greenhouse:gone"])
|
|
assert first["reason"] == "404 Not Found" and first["since"]
|
|
assert "ats:ok:greenhouse:gone" not in source.state
|
|
|
|
# A later failure keeps when the failures began and updates the reason.
|
|
source.state["ats:failing:greenhouse:gone"] = json.dumps(
|
|
first | {"since": "2026-09-01T00:00:00+00:00"})
|
|
later = _source(monkeypatch, httpx.ConnectTimeout("timed out"))
|
|
later.state = source.state
|
|
list(later.fetch())
|
|
assert json.loads(source.state["ats:failing:greenhouse:gone"]) == {
|
|
"since": "2026-09-01T00:00:00+00:00", "reason": "ConnectTimeout: timed out"}
|
|
|
|
|
|
def test_an_answer_clears_the_failure_and_stamps_completeness(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
source = _source(monkeypatch, (_rows("greenhouse"), False))
|
|
source.state["ats:failing:greenhouse:gone"] = json.dumps(
|
|
{"since": "2026-09-01T00:00:00+00:00", "reason": "404 Not Found"})
|
|
assert len(list(source.fetch())) == 4
|
|
assert "ats:failing:greenhouse:gone" not in source.state
|
|
assert source.state["ats:ok:greenhouse:gone"] == "run-1"
|
|
assert source.state["ats:complete:greenhouse:gone"] == ""
|