The readers flattened a posting's HTML into one line with strip_html, so sections and bullets ran together. _shared.html_text gives a line per paragraph, '• ' before a list item and '## ' before a heading; every reader's description goes through it, sections read per role get their own headings, and the reader pane draws the lines as h3, ul and p. Excerpts leave the marks out. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
795 lines
35 KiB
Python
795 lines
35 KiB
Python
"""The `ats` plugin: each host's parser against a saved feed, the function
|
|
classifier, and the fetch allow-list. No network — every fixture is a file
|
|
under `tests/fixtures/ats/`."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from collections.abc import Callable, Iterator
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from plugins import ats # noqa: E402
|
|
from plugins.readers import _runner, smartrecruiters # noqa: E402
|
|
|
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
|
|
|
|
|
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
|
|
path = FIXTURES / f"{host}.{'xml' if host == 'personio' else 'json'}"
|
|
body = path.read_bytes()
|
|
return ats._PARSERS[host](body, slug)
|
|
|
|
|
|
# ------------------------------------------------------------------ hosts
|
|
|
|
def test_greenhouse_parses_titles_locations_and_html_description() -> None:
|
|
rows = _rows("greenhouse")
|
|
# The javascript: and data: rows are refused, same as the four real ones
|
|
# are kept: a feed is external content, never trusted as a URL.
|
|
assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"]
|
|
first = rows[0]
|
|
assert first["title"] == "Senior Backend Engineer" # trimmed
|
|
assert first["location"] == "Berlin, Germany"
|
|
assert first["department"] == "Engineering"
|
|
assert first["description"] == "Build our core platform."
|
|
assert first["posted"] is not None and first["posted"].year == 2026
|
|
assert first["remote"] is None
|
|
|
|
|
|
def test_greenhouse_reads_pay_input_ranges_and_skips_an_hourly_one() -> None:
|
|
rows = _rows("greenhouse")
|
|
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
|
|
(65000, 85000, "EUR")
|
|
# "SF Bay Area Hourly Rate" is skipped: a yearly figure invented from an
|
|
# hourly one would be wrong.
|
|
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
|
|
(None, None, "")
|
|
|
|
|
|
def test_greenhouse_salary_text_covers_the_single_hourly_and_zoned_cases() -> None:
|
|
"""`salary_text` is written whether or not the pay normalises: a single
|
|
yearly range reads plainly, the hourly one that the columns dropped
|
|
still reads as a line, several zones are joined and labelled with the
|
|
board's own titles, and a range with no currency says so."""
|
|
rows = _rows("greenhouse")
|
|
assert rows[0]["salary_text"] == "EUR 65,000 to 85,000 per year"
|
|
assert rows[1]["salary_text"] == "USD 30 to 45 per hour"
|
|
assert rows[2]["salary_text"] == ("Zone 1: USD 180,000 to 220,000 per year; "
|
|
"Zone 2: USD 160,000 to 200,000 per year")
|
|
assert rows[3]["salary_text"] == "65,000 to 85,000 per year (no currency stated)"
|
|
|
|
|
|
def test_lever_parses_categories_and_workplace_type() -> None:
|
|
rows = _rows("lever")
|
|
assert [r["source_id"] for r in rows] == ["2001", "2002", "2003"]
|
|
first = rows[0]
|
|
assert first["department"] == "Engineering"
|
|
assert first["remote"] == "remote"
|
|
assert first["description"] == "Own our platform reliability."
|
|
# createdAt=0 means "no date", not epoch zero.
|
|
assert rows[1]["posted"] is None
|
|
|
|
|
|
def test_lever_reads_salary_range_and_skips_an_hourly_wage() -> None:
|
|
rows = _rows("lever")
|
|
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
|
|
(90000, 120000, "EUR")
|
|
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
|
|
(None, None, "")
|
|
|
|
|
|
def test_lever_salary_text_covers_the_hourly_and_no_currency_cases() -> None:
|
|
rows = _rows("lever")
|
|
assert rows[0]["salary_text"] == "EUR 90,000 to 120,000 per year"
|
|
assert rows[1]["salary_text"] == "USD 25 to 35 per hour"
|
|
assert rows[2]["salary_text"] == "80,000 to 100,000 per year (no currency stated)"
|
|
|
|
|
|
def test_ashby_parses_is_remote_and_compensation() -> None:
|
|
rows = _rows("ashby")
|
|
assert [r["source_id"] for r in rows] == ["3001", "3002", "3003", "3004"]
|
|
assert rows[0]["remote"] is False
|
|
assert rows[0]["salary_text"] == "EUR 70,000 to 90,000 per year"
|
|
assert rows[1]["remote"] is True
|
|
|
|
|
|
def test_ashby_reads_summary_components_only_with_includecompensation() -> None:
|
|
# summaryComponents only appears with ?includeCompensation=true on the
|
|
# feed URL (see _BOARDS); the second job has no compensation key at all.
|
|
rows = _rows("ashby")
|
|
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
|
|
(70000, 90000, "EUR")
|
|
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
|
|
(None, None, "")
|
|
|
|
|
|
def test_ashby_salary_text_falls_back_to_prose_and_labels_several_components() -> None:
|
|
"""No component with a bound at all (job 3): Ashby's own prose summary is
|
|
what is left to show. An hourly Salary component beside a yearly Bonus
|
|
one (job 4): neither normalises (`_ashby_salary` only reads a yearly or
|
|
monthly `Salary` component), but both are stated, labelled since there
|
|
is more than one."""
|
|
rows = _rows("ashby")
|
|
assert rows[2]["salary_text"] == "Competitive OTE, details on request"
|
|
assert rows[3]["salary_text"] == \
|
|
"Salary: USD 45 to 60 per hour; Bonus: USD 5,000 to 10,000 per year"
|
|
assert (rows[3]["salary_min"], rows[3]["salary_max"], rows[3]["currency"]) == \
|
|
(None, None, "")
|
|
|
|
|
|
def test_workable_builds_the_job_url_and_reads_location_and_remote() -> None:
|
|
rows = _rows("workable", slug="huggingface")
|
|
assert [r["source_id"] for r in rows] == ["6125026", "6125041"]
|
|
assert rows[0]["url"] == "https://apply.workable.com/huggingface/j/DB4D7C0EC8/"
|
|
assert rows[0]["location"] == "France"
|
|
assert rows[0]["remote"] == "remote"
|
|
assert rows[0]["department"] == "Product"
|
|
assert rows[1]["location"] == "Berlin, Germany"
|
|
assert rows[1]["remote"] == "hybrid"
|
|
assert rows[0]["description"] == "" # the list endpoint carries no text
|
|
assert rows[0]["salary_text"] == "" # Workable's list endpoint states no pay at all
|
|
|
|
|
|
def test_smartrecruiters_builds_the_job_url_from_id_and_slug() -> None:
|
|
rows = _rows("smartrecruiters", slug="Acme")
|
|
assert [r["source_id"] for r in rows] == ["744000151326819", "744000147485112"]
|
|
assert rows[0]["url"] == "https://jobs.smartrecruiters.com/Acme/744000151326819"
|
|
assert rows[0]["location"] == "Berlin, Germany"
|
|
assert rows[1]["remote"] is True
|
|
assert rows[0]["salary_text"] == "" # SmartRecruiters' list endpoint states no pay at all
|
|
|
|
|
|
def test_recruitee_parses_salary_bounds_and_careers_url() -> None:
|
|
rows = _rows("recruitee")
|
|
assert [r["source_id"] for r in rows] == ["5001", "5002", "5003"]
|
|
assert rows[0]["url"] == "https://acme.example/o/player-support-agent-german-1"
|
|
assert rows[0]["salary_text"] == "EUR 35,000 to 45,000 per year"
|
|
assert rows[1]["salary_text"] == ""
|
|
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
|
|
(35000, 45000, "EUR")
|
|
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
|
|
(None, None, "")
|
|
|
|
|
|
def test_recruitee_salary_text_states_an_hourly_rate_with_no_currency() -> None:
|
|
rows = _rows("recruitee")
|
|
assert rows[2]["salary_text"] == "18 to 24 per hour (no currency stated)"
|
|
assert (rows[2]["salary_min"], rows[2]["salary_max"], rows[2]["currency"]) == \
|
|
(None, None, "")
|
|
|
|
|
|
def test_personio_builds_the_job_url_and_reads_cdata_description() -> None:
|
|
rows = _rows("personio", slug="acme")
|
|
assert [r["source_id"] for r in rows] == ["6001", "6002", "6003"]
|
|
assert rows[0]["url"] == "https://acme.jobs.personio.de/job/6001"
|
|
assert rows[0]["description"] == "Grow our pipeline."
|
|
assert rows[0]["places"][0]["remote"] is True # "Remote - Germany"
|
|
assert rows[1]["places"][0]["remote"] is None # "Munich"
|
|
assert "remote" not in rows[0] # the feed states none
|
|
|
|
|
|
def test_personio_reads_salary_information_when_present() -> None:
|
|
rows = _rows("personio", slug="acme")
|
|
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
|
|
(42000, 55000, "EUR")
|
|
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
|
|
(None, None, "")
|
|
assert rows[0]["salary_text"] == "EUR 42,000 to 55,000 per year"
|
|
assert rows[1]["salary_text"] == ""
|
|
|
|
|
|
def test_personio_salary_text_states_a_monthly_rate_with_no_currency() -> None:
|
|
rows = _rows("personio", slug="acme")
|
|
assert rows[2]["salary_text"] == "3,000 to 3,600 per month (no currency stated)"
|
|
# No currency: dropped from the columns like any other host's, even
|
|
# though the interval itself (monthly) is one _yearly_amount knows.
|
|
assert (rows[2]["salary_min"], rows[2]["salary_max"], rows[2]["currency"]) == \
|
|
(None, None, "")
|
|
|
|
|
|
# Trimmed from live feeds of 2026-09-29 (career.kry.se, careers.tobii.com): the
|
|
# `tt:` namespace, an item with several locations, one whose city is a country,
|
|
# an empty department with a role, and one whose link is not a web URL.
|
|
TEAMTAILOR = """<?xml version="1.0" encoding="UTF-8"?>
|
|
<rss version="2.0" xmlns:tt="https://teamtailor.com/locations">
|
|
<channel>
|
|
<title>Kry</title>
|
|
<description>Job openings</description>
|
|
<link>https://career.kry.se/jobs</link>
|
|
<item>
|
|
<title>Software Engineer</title>
|
|
<description><h1>About the role</h1><p>Build &amp; run care.</p></description>
|
|
<pubDate>Wed, 23 Sep 2026 12:59:53 +0200</pubDate>
|
|
<link>https://career.kry.se/jobs/8440833-software-engineer</link>
|
|
<remoteStatus>hybrid</remoteStatus>
|
|
<guid>39ac1fca-a904-4905-b780-18c599e99b07</guid>
|
|
<company_name>Kry</company_name>
|
|
<tt:locations>
|
|
<tt:location>
|
|
<tt:name>Sverige</tt:name><tt:address/><tt:zip/>
|
|
<tt:city>Sverige</tt:city><tt:country>Sweden</tt:country>
|
|
</tt:location>
|
|
<tt:location>
|
|
<tt:name>Stockholm, Sverige</tt:name><tt:address>Torsgatan 21</tt:address>
|
|
<tt:zip>113 21</tt:zip><tt:city>Stockholm</tt:city><tt:country>Sweden</tt:country>
|
|
</tt:location>
|
|
<tt:location>
|
|
<tt:name>Stockholm, Sverige</tt:name><tt:address>Torsgatan 21</tt:address>
|
|
<tt:zip>113 21</tt:zip><tt:city>Stockholm</tt:city><tt:country>Sweden</tt:country>
|
|
</tt:location>
|
|
</tt:locations>
|
|
<tt:department>Engineering</tt:department>
|
|
<tt:role/>
|
|
</item>
|
|
<item>
|
|
<title> Läkare </title>
|
|
<description><p>Träffa patienter.</p></description>
|
|
<pubDate>Fri, 31 Jul 2026 14:51:42 +0200</pubDate>
|
|
<link>https://career.kry.se/jobs/8156121-lakare</link>
|
|
<remoteStatus>fully</remoteStatus>
|
|
<guid>9787a509-ff84-400a-ad68-84e22c4cf2fb</guid>
|
|
<tt:locations>
|
|
<tt:location>
|
|
<tt:name>Sweden</tt:name><tt:address>Karlsrovägen 2D</tt:address>
|
|
<tt:zip>182 53</tt:zip><tt:city>Danderyd</tt:city><tt:country>Sweden</tt:country>
|
|
</tt:location>
|
|
</tt:locations>
|
|
<tt:department/>
|
|
<tt:role>Software Development</tt:role>
|
|
</item>
|
|
<item>
|
|
<title>Sales Lead</title>
|
|
<description></description>
|
|
<link>https://career.kry.se/jobs/8156122-sales-lead</link>
|
|
<remoteStatus>none</remoteStatus>
|
|
<guid>0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb</guid>
|
|
<tt:locations/>
|
|
<tt:department>Sales</tt:department>
|
|
<tt:role/>
|
|
</item>
|
|
<item>
|
|
<title>Elsewhere</title>
|
|
<link>javascript:alert(1)</link>
|
|
<guid>bad</guid>
|
|
</item>
|
|
</channel>
|
|
</rss>
|
|
""".encode()
|
|
|
|
|
|
def test_teamtailor_reads_the_rss_feed_with_its_own_namespace() -> None:
|
|
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
|
# the javascript: link is refused like on the other hosts
|
|
assert [r["source_id"] for r in rows] == [
|
|
"39ac1fca-a904-4905-b780-18c599e99b07", "9787a509-ff84-400a-ad68-84e22c4cf2fb",
|
|
"0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb"]
|
|
first = rows[0]
|
|
assert first["url"] == "https://career.kry.se/jobs/8440833-software-engineer"
|
|
assert first["department"] == "Engineering"
|
|
assert first["description"] == "## About the role\nBuild & run care."
|
|
assert first["posted"] == datetime(2026, 9, 23, 10, 59, 53, tzinfo=UTC)
|
|
assert first["salary_text"] == ""
|
|
|
|
|
|
def test_teamtailor_joins_the_distinct_locations_for_country_of() -> None:
|
|
from app.models import country_of
|
|
|
|
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
|
# "Sverige" (a city that is a country) once, Stockholm's twin once
|
|
assert rows[0]["location"] == "Sverige, Sweden; Stockholm, Sweden"
|
|
assert rows[1]["location"] == "Danderyd, Sweden"
|
|
assert rows[2]["location"] == ""
|
|
assert country_of(rows[0]["location"]) == "SE"
|
|
|
|
|
|
def test_teamtailor_hands_on_remote_status_and_falls_back_to_role_for_department() -> None:
|
|
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
|
assert [r["remote"] for r in rows] == ["hybrid", "fully", "none"] # prep maps them
|
|
assert rows[1]["title"] == "Läkare" # trimmed
|
|
assert rows[1]["department"] == "Software Development"
|
|
assert ats.classify(rows[1]["department"], "Sales Lead") == "sales"
|
|
|
|
|
|
def test_teamtailor_refuses_a_page_that_is_not_a_feed() -> None:
|
|
with pytest.raises(ValueError):
|
|
ats._parse_teamtailor(b"<rss version='2.0'/>", "kry")
|
|
|
|
|
|
def test_parse_dt_reads_an_rss_date_and_still_refuses_junk() -> None:
|
|
assert ats._parse_dt("Tue, 29 Sep 2026 13:36:59 +0200") == \
|
|
datetime(2026, 9, 29, 11, 36, 59, tzinfo=UTC)
|
|
assert ats._parse_dt("last tuesday") is None
|
|
|
|
|
|
# ------------------------------------------------------------ salary math
|
|
|
|
def test_yearly_amount_converts_a_monthly_range_by_12() -> None:
|
|
assert ats._yearly_amount(3000, 4000, "eur", "month", {"year": 1, "month": 12}) == \
|
|
(36000, 48000, "EUR")
|
|
|
|
|
|
def test_yearly_amount_ignores_a_range_with_no_currency() -> None:
|
|
assert ats._yearly_amount(3000, 4000, "", "year", {"year": 1, "month": 12}) == \
|
|
(None, None, "")
|
|
|
|
|
|
def test_yearly_amount_skips_an_interval_outside_the_factor_table() -> None:
|
|
# An hourly or a daily rate: a yearly figure invented from it would be
|
|
# wrong, so it is left out rather than multiplied by a guess.
|
|
assert ats._yearly_amount(20, 30, "usd", "hour", {"year": 1, "month": 12}) == \
|
|
(None, None, "")
|
|
|
|
|
|
def test_greenhouse_salary_skips_more_than_one_range() -> None:
|
|
# Several ranges (one per office/zone) give no way to tell which one the
|
|
# role in hand is at, so all of them are left out rather than guessed.
|
|
ranges = [
|
|
{"min_cents": 100000, "max_cents": 200000, "currency_type": "EUR", "title": "Zone 1"},
|
|
{"min_cents": 90000, "max_cents": 180000, "currency_type": "EUR", "title": "Zone 2"},
|
|
]
|
|
assert ats._greenhouse_salary(ranges) == (None, None, "")
|
|
|
|
|
|
# --------------------------------------------------------------- classify
|
|
|
|
def test_classify_matches_title_before_department() -> None:
|
|
# A specific title beats a generic department (see classify's docstring
|
|
# for the hand-checked sample this is based on).
|
|
assert ats.classify("Engineering", "Marketing Lead") == "marketing"
|
|
assert ats.classify("Operations", "Senior Backend Engineer") == "engineering"
|
|
assert ats.classify("", "Product Manager, Payments") == "product"
|
|
assert ats.classify("", "Data Product Manager") == "product"
|
|
assert ats.classify("", "Machine Learning Engineer") == "data"
|
|
assert ats.classify("", "Executive Assistant to the CEO") == "finance/legal/ops"
|
|
# The department is still the fallback when the title matches nothing.
|
|
assert ats.classify("Engineering", "") == "engineering"
|
|
# Nothing matches either: '', not "other", which would claim a
|
|
# knowledge the classifier does not have.
|
|
assert ats.classify("", "") == ""
|
|
assert ats.classify("", "Chief Bottle Washer") == ""
|
|
|
|
|
|
def test_classify_matches_whole_words_not_substrings() -> None:
|
|
# "it" used to match inside "Italy"/"Italia" ("IT" meant the department,
|
|
# not the country), filing a sales role as engineering.
|
|
assert ats.classify("Commercial Italy", "Account Manager - Liguria") == "sales"
|
|
assert ats.classify("IT", "") == "engineering"
|
|
assert ats.classify("", "Marketing Lead, Seoul") == "marketing"
|
|
# A stem still matches its suffixed forms: "Engineering" the department,
|
|
# not just "engineer" the exact word.
|
|
assert ats.classify("Engineering", "") == "engineering"
|
|
assert ats.classify("", "Software Developers") == "engineering"
|
|
|
|
|
|
def test_classify_understands_a_modest_set_of_other_languages() -> None:
|
|
assert ats.classify("", "Senior Entwickler (m/w/d)") == "engineering"
|
|
assert ats.classify("", "Vertriebsmitarbeiter (m/w/d)") == "sales"
|
|
assert ats.classify("", "Ontwikkelaar") == "engineering"
|
|
assert ats.classify("", "Udvikler") == "engineering"
|
|
assert ats.classify("", "Ingénieur logiciel") == "engineering"
|
|
|
|
|
|
# ------------------------------------------------------------- allow-list
|
|
|
|
def test_is_known_target_accepts_the_eight_hosts_and_refuses_others() -> None:
|
|
for host in ("greenhouse", "lever", "ashby", "workable", "smartrecruiters",
|
|
"recruitee", "personio", "teamtailor"):
|
|
assert ats.is_known_target(f"{host}:acme")
|
|
assert not ats.is_known_target("linkedin:acme")
|
|
assert not ats.is_known_target("taleo:acme")
|
|
assert not ats.is_known_target("greenhouse:")
|
|
assert not ats.is_known_target("nohostnoseparator")
|
|
# a Teamtailor slug names a host: its own subdomain, or a whole domain
|
|
assert ats.is_known_target("teamtailor:careers.acme.com")
|
|
assert not ats.is_known_target("teamtailor:127.0.0.1")
|
|
assert not ats.is_known_target("teamtailor:acme.com/x")
|
|
assert not ats.is_known_target("teamtailor:acme com")
|
|
|
|
|
|
def test_allowed_host_refuses_a_foreign_host_per_board() -> None:
|
|
allow = ats._allowed_host("greenhouse", "acme")
|
|
assert allow("boards-api.greenhouse.io")
|
|
assert not allow("boards-api.greenhouse.io.evil.example")
|
|
assert not allow("api.lever.co")
|
|
|
|
# recruitee and personio key off the company's own subdomain, so one
|
|
# company's slug must not pass for another's.
|
|
allow_recruitee = ats._allowed_host("recruitee", "acme")
|
|
assert allow_recruitee("acme.recruitee.com")
|
|
assert not allow_recruitee("other.recruitee.com")
|
|
|
|
# teamtailor: the subdomain of a plain slug, the whole host of a dotted one
|
|
assert ats._allowed_host("teamtailor", "acme")("acme.teamtailor.com")
|
|
assert not ats._allowed_host("teamtailor", "acme")("other.teamtailor.com")
|
|
own = ats._allowed_host("teamtailor", "Careers.Acme.com")
|
|
assert own("careers.acme.com")
|
|
assert not own("acme.com") and not own("careers.acme.com.evil.example")
|
|
|
|
|
|
def test_careers_url_is_filled_per_host_and_empty_for_unknown() -> None:
|
|
assert ats.careers_url("greenhouse:acme") == "https://job-boards.greenhouse.io/acme"
|
|
assert ats.careers_url("ashby:acme") == "https://jobs.ashbyhq.com/acme"
|
|
assert ats.careers_url("recruitee:acme") == "https://acme.recruitee.com"
|
|
assert ats.careers_url("teamtailor:acme") == "https://acme.teamtailor.com/jobs"
|
|
assert ats.careers_url("teamtailor:careers.acme.com") == "https://careers.acme.com/jobs"
|
|
assert ats.careers_url("taleo:acme") == ""
|
|
|
|
|
|
# --------------------------------------------------------------- body cap
|
|
|
|
def test_max_bytes_is_96_mb() -> None:
|
|
"""Raised from openings.go's 8 MB: every URL here is on a fixed allow-list
|
|
of ATS API hosts, not an arbitrary one, and the real run of 2026-09-23
|
|
refused boards up to 41 MB under it."""
|
|
assert ats.MAX_BYTES == 96 << 20
|
|
|
|
|
|
def test_fetch_rows_retries_greenhouse_without_content_on_a_too_large_body(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
calls = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET"):
|
|
calls.append(url)
|
|
if "?content=true" in url:
|
|
raise ats._TooLarge("response too large (100000000 bytes)")
|
|
return (FIXTURES / "greenhouse.json").read_bytes()
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type]
|
|
|
|
assert calls == [
|
|
"https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true&pay_transparency=true",
|
|
"https://boards-api.greenhouse.io/v1/boards/acme/jobs",
|
|
]
|
|
assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"]
|
|
assert complete # a host that does not page answers with the whole board
|
|
|
|
|
|
def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
def fake_get(client, url, host, slug, *, method="GET"):
|
|
raise ats._TooLarge("response too large (100000000 bytes)")
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
with pytest.raises(ats._TooLarge):
|
|
ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]
|
|
|
|
|
|
def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client:
|
|
return httpx.Client(transport=httpx.MockTransport(handler))
|
|
|
|
|
|
def test_get_stops_reading_at_max_bytes_rather_than_after_the_body(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
sent: list[int] = []
|
|
|
|
def chunks() -> Iterator[bytes]:
|
|
for i in range(100):
|
|
sent.append(i)
|
|
yield b"x" * 10
|
|
|
|
monkeypatch.setattr(ats, "MAX_BYTES", 25)
|
|
with _client(lambda request: httpx.Response(200, content=chunks())) as client, \
|
|
pytest.raises(ats._TooLarge):
|
|
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
|
|
assert len(sent) == 3 # refused on the third chunk, not the hundredth
|
|
|
|
|
|
def test_get_gives_up_at_the_deadline_however_steadily_bytes_arrive(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
clock = iter(range(0, 1000, 10))
|
|
monkeypatch.setattr(_runner.time, "monotonic", lambda: float(next(clock)))
|
|
monkeypatch.setattr(ats, "DEADLINE", 25.0)
|
|
body = (b"x" for _ in range(100))
|
|
with _client(lambda request: httpx.Response(200, content=body)) as client, \
|
|
pytest.raises(httpx.ReadTimeout, match="within 25 s"):
|
|
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
|
|
|
|
|
|
def test_get_follows_an_allowed_redirect_and_refuses_a_foreign_one() -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if request.url.path == "/v0/postings/old":
|
|
return httpx.Response(301, headers={"location": "/v0/postings/acme"})
|
|
if request.url.path == "/v0/postings/away":
|
|
return httpx.Response(302, headers={"location": "https://evil.example/"})
|
|
return httpx.Response(200, content=b"[]")
|
|
|
|
with _client(handler) as client:
|
|
assert ats._get(client, "https://api.lever.co/v0/postings/old", "lever", "acme") == b"[]"
|
|
with pytest.raises(ats._Refused):
|
|
ats._get(client, "https://api.lever.co/v0/postings/away", "lever", "acme")
|
|
|
|
|
|
def test_paging_reads_through_the_streamed_get(no_delay: None) -> None:
|
|
"""`_paged` over the real `_get`: the POST body carries the cursor, and a
|
|
429 on a later page still reaches it as an `HTTPStatusError`."""
|
|
bodies: list[Any] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
payload = json.loads(request.content or b"{}")
|
|
bodies.append(payload)
|
|
if payload.get("token") == "stop":
|
|
return httpx.Response(429)
|
|
page = json.loads(_page("workable-2.json" if payload else "workable.json"))
|
|
if payload:
|
|
page["total"], page["nextPage"] = 4, "stop"
|
|
return httpx.Response(200, json=page)
|
|
|
|
with _client(handler) as client:
|
|
rows, complete = ats._fetch_rows(client, "workable:exotec")
|
|
assert bodies == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}, {"token": "stop"}]
|
|
assert (len(rows), complete) == (3, False)
|
|
|
|
|
|
# ----------------------------------------------------------------- paging
|
|
|
|
def _page(name: str) -> bytes:
|
|
return (FIXTURES / name).read_bytes()
|
|
|
|
|
|
def _status(code: int) -> httpx.HTTPStatusError:
|
|
request = httpx.Request("POST", "https://apply.workable.com/")
|
|
return httpx.HTTPStatusError(str(code), request=request,
|
|
response=httpx.Response(code, request=request))
|
|
|
|
|
|
@pytest.fixture
|
|
def no_delay(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
|
|
|
|
|
def test_workable_follows_next_page_to_the_stated_total(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
calls: list[dict[str, Any] | None] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
calls.append(payload)
|
|
return _page("workable-2.json" if payload else "workable.json")
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
|
|
|
assert calls == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}]
|
|
assert [r["source_id"] for r in rows] == ["6125026", "6125041", "6125077"]
|
|
assert complete
|
|
|
|
|
|
def test_smartrecruiters_pages_by_offset_to_total_found(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
urls: list[str] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
urls.append(url)
|
|
return _page("smartrecruiters-2.json" if "offset=2" in url else "smartrecruiters.json")
|
|
|
|
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
|
|
|
base = "https://api.smartrecruiters.com/v1/companies/Acme/postings"
|
|
assert urls == [f"{base}?offset=0&limit=2", f"{base}?offset=2&limit=2"]
|
|
assert len(rows) == 3 and rows[2]["location"] == "Tilburg, Netherlands"
|
|
assert complete
|
|
|
|
|
|
def test_a_board_that_ends_short_of_its_total_is_incomplete(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
"""totalFound says 3; the second page is empty, so two were read."""
|
|
empty = json.dumps({"offset": 2, "limit": 2, "totalFound": 3, "content": []}).encode()
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
return empty if "offset=2" in url else _page("smartrecruiters.json")
|
|
|
|
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
|
assert (len(rows), complete) == (2, False)
|
|
|
|
|
|
def test_a_repeated_row_does_not_count_twice_toward_the_total(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
"""A page that shifted under an offset repeats a role and misses one: three
|
|
rows read, two distinct, against a total of three."""
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
page = json.loads(_page("smartrecruiters.json"))
|
|
if "offset=2" in url:
|
|
page["content"] = page["content"][1:]
|
|
return json.dumps(page).encode()
|
|
|
|
monkeypatch.setattr(smartrecruiters, "PAGE_SIZE", 2)
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
|
assert (len(rows), complete) == (2, False)
|
|
|
|
|
|
def test_the_page_cap_leaves_a_board_incomplete(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
calls: list[dict[str, Any] | None] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
calls.append(payload)
|
|
return _page("workable.json") # always another page
|
|
|
|
monkeypatch.setattr(ats, "MAX_PAGES", 3)
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
_rows_read, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
|
assert (len(calls), complete) == (3, False)
|
|
|
|
|
|
def test_a_429_mid_way_keeps_what_was_read_and_is_incomplete(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
if payload:
|
|
raise _status(429)
|
|
return _page("workable.json")
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
|
assert ([r["source_id"] for r in rows], complete) == (["6125026", "6125041"], False)
|
|
|
|
|
|
def test_a_429_on_the_first_page_is_a_failure(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
raise _status(429)
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
|
|
|
|
|
def _feed(guids: list[str]) -> bytes:
|
|
"""A Teamtailor feed of `guids`, one role each."""
|
|
items = "".join(
|
|
f"<item><title>Role {g}</title><link>https://careers.acme.com/jobs/{g}</link>"
|
|
f"<guid>{g}</guid></item>" for g in guids)
|
|
return ('<rss xmlns:tt="https://teamtailor.com/locations"><channel>'
|
|
f"{items}</channel></rss>").encode()
|
|
|
|
|
|
def _teamtailor_web(monkeypatch: pytest.MonkeyPatch, guids: list[str],
|
|
page: int = 2) -> list[str]:
|
|
"""`_get` answering from `guids` by `offset` and `per_page`, as a
|
|
Teamtailor site does, returning the URLs it was asked."""
|
|
urls: list[str] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
urls.append(url)
|
|
query = dict(httpx.URL(url).params)
|
|
start, size = int(query["offset"]), min(int(query["per_page"]), page)
|
|
return _feed(guids[start:start + size])
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
return urls
|
|
|
|
|
|
def test_teamtailor_pages_by_offset_and_is_complete_on_an_empty_page(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
"""The host caps a page below what was asked; the offset advances by what
|
|
came back, and the read ends on the empty page after the last role."""
|
|
urls = _teamtailor_web(monkeypatch, ["a", "b", "c", "d", "e"])
|
|
rows, complete = ats._fetch_rows(object(), "teamtailor:careers.acme.com") # type: ignore[arg-type]
|
|
|
|
base = "https://careers.acme.com/jobs.rss?per_page=50"
|
|
assert urls == [f"{base}&offset=0", f"{base}&offset=2", f"{base}&offset=4",
|
|
f"{base}&offset=5"]
|
|
assert [r["source_id"] for r in rows] == ["a", "b", "c", "d", "e"]
|
|
assert complete
|
|
|
|
|
|
def test_a_teamtailor_slug_without_a_dot_is_a_teamtailor_subdomain(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
urls = _teamtailor_web(monkeypatch, ["a"])
|
|
ats._fetch_rows(object(), "teamtailor:acme") # type: ignore[arg-type]
|
|
assert urls[0].startswith("https://acme.teamtailor.com/jobs.rss?")
|
|
|
|
|
|
def test_a_board_with_no_roles_is_complete_at_zero(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
_teamtailor_web(monkeypatch, [])
|
|
assert ats._fetch_rows(object(), "teamtailor:acme") == ([], True) # type: ignore[arg-type]
|
|
|
|
|
|
def test_a_teamtailor_host_that_ignores_the_offset_is_incomplete(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
"""The same page again: nothing new, so the read stops rather than loops."""
|
|
calls: list[str] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
calls.append(url)
|
|
return _feed(["a", "b"])
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "teamtailor:acme") # type: ignore[arg-type]
|
|
assert (len(calls), len(rows), complete) == (2, 2, False)
|
|
|
|
|
|
def test_the_page_cap_leaves_a_teamtailor_board_incomplete(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
_teamtailor_web(monkeypatch, [str(n) for n in range(10)])
|
|
monkeypatch.setattr(ats, "MAX_PAGES", 2)
|
|
rows, complete = ats._fetch_rows(object(), "teamtailor:acme") # type: ignore[arg-type]
|
|
assert (len(rows), complete) == (4, False)
|
|
|
|
|
|
def test_a_teamtailor_429_after_the_first_page_is_incomplete(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
|
if "offset=0" not in url:
|
|
raise _status(429)
|
|
return _feed(["a", "b"])
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
rows, complete = ats._fetch_rows(object(), "teamtailor:acme") # type: ignore[arg-type]
|
|
assert ([r["source_id"] for r in rows], complete) == (["a", "b"], False)
|
|
|
|
|
|
def test_teamtailor_is_read_through_the_streamed_get_and_a_redirect_elsewhere_is_refused(
|
|
no_delay: None) -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if request.url.host == "moved.acme.com":
|
|
return httpx.Response(302, headers={"location": "https://evil.example/jobs.rss"})
|
|
assert request.url.path == "/jobs.rss"
|
|
return httpx.Response(200, content=_feed(["a"] if request.url.params["offset"] == "0" else []))
|
|
|
|
with _client(handler) as client:
|
|
rows, complete = ats._fetch_rows(client, "teamtailor:careers.acme.com")
|
|
assert ([r["source_id"] for r in rows], complete) == (["a"], True)
|
|
with pytest.raises(ats._Refused):
|
|
ats._fetch_rows(client, "teamtailor:moved.acme.com")
|
|
|
|
|
|
# -------------------------------------------------------- failing boards
|
|
|
|
def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource:
|
|
"""One board, `greenhouse:gone`, whose fetch answers `answer` or raises it."""
|
|
def fake(*args: object, **kwargs: object) -> _runner.Listing:
|
|
if isinstance(answer, Exception):
|
|
raise answer
|
|
rows, complete = answer # type: ignore[misc]
|
|
return _runner.Listing(rows, complete, [row["source_id"] for row in rows])
|
|
|
|
monkeypatch.setattr(_runner, "read_listing", fake)
|
|
source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}],
|
|
"run": "run-1"})
|
|
return source
|
|
|
|
|
|
def test_a_failing_board_records_since_and_the_latest_reason(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
source = _source(monkeypatch, _status(404))
|
|
assert list(source.fetch()) == []
|
|
first = json.loads(source.state["ats:failing:greenhouse:gone"])
|
|
assert first["reason"] == "404 Not Found" and first["since"]
|
|
assert "ats:ok:greenhouse:gone" not in source.state
|
|
|
|
# A later failure keeps when the failures began and updates the reason.
|
|
source.state["ats:failing:greenhouse:gone"] = json.dumps(
|
|
first | {"since": "2026-09-01T00:00:00+00:00"})
|
|
later = _source(monkeypatch, httpx.ConnectTimeout("timed out"))
|
|
later.state = source.state
|
|
list(later.fetch())
|
|
assert json.loads(source.state["ats:failing:greenhouse:gone"]) == {
|
|
"since": "2026-09-01T00:00:00+00:00", "reason": "ConnectTimeout: timed out"}
|
|
|
|
|
|
def test_an_answer_clears_the_failure_and_stamps_completeness(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
source = _source(monkeypatch, (_rows("greenhouse"), False))
|
|
source.state["ats:failing:greenhouse:gone"] = json.dumps(
|
|
{"since": "2026-09-01T00:00:00+00:00", "reason": "404 Not Found"})
|
|
assert len(list(source.fetch())) == 4
|
|
assert "ats:failing:greenhouse:gone" not in source.state
|
|
assert source.state["ats:ok:greenhouse:gone"] == "run-1"
|
|
assert source.state["ats:complete:greenhouse:gone"] == ""
|