"""Board discovery: which ATS a careers site names, and in what order the site's pages are read.""" from __future__ import annotations import json import re import types from collections.abc import Set from pathlib import Path from typing import Any import httpx import pytest from app.jobs import discover as mod from app.jobs.discover import Discovery, boards_in, careers_links, discover, slug_guesses from app.jobs.sources import seed_companies from app.storage import Store # ------------------------------------------------------------------ parsing def test_boards_in_names_readable_hosts_with_their_slug() -> None: page = """ Open roles One role Workable SR Recruitee Personio """ assert [(h, s) for h, s, _url in boards_in(page)] == [ ("greenhouse", "pleo"), ("lever", "spotify"), ("ashby", "deepl"), ("workable", "acme"), ("smartrecruiters", "bosch"), ("recruitee", "acme"), ("personio", "acme-gmbh")] def test_boards_in_keeps_hosts_the_app_cannot_read() -> None: page = """ Taleo TT """ assert boards_in(page) == [ ("taleo", "", ".taleo.net"), ("teamtailor", "acme", "https://acme.teamtailor.com")] def test_boards_in_names_a_lever_eu_board_by_its_region() -> None: page = ('EU' '' 'US') assert [(h, s) for h, s, _ in boards_in(page)] == [("lever", "eu/tomtom"), ("lever", "whereby")] def test_boards_in_lowercases_a_slug_unless_the_host_keeps_its_case( monkeypatch: pytest.MonkeyPatch) -> None: """A Workday-shaped host with two groups and `KEEP_CASE`: the site keeps its case, and a site called `jobs` is a site, since only group 1 is checked against the hosts' own path words.""" workday = re.compile(r"https?://([a-z0-9-]+\.wd\d+)\.myworkdayjobs\.com/" r"(?:[a-z]{2}-[A-Z]{2}/)?([A-Za-z0-9_-]+)", re.IGNORECASE) monkeypatch.setattr(mod, "HOSTS", (("workday", workday), *(h for h in mod.HOSTS if h[0] != "workday"))) monkeypatch.setattr(mod, "_KEEP_CASE", frozenset({"workday"})) page = ('a' 'b' 'c' 'd') assert [(h, s) for h, s, _ in boards_in(page)] == [ ("workday", "lego.wd103/LEGO_External"), ("workday", "redhat.wd5/jobs"), ("smartrecruiters", "boschgroup"), ("lever", "eu/tomtom")] # A host's own path word in group 1 still names no board. assert boards_in('x') == [] def test_no_module_of_the_eight_keeps_its_case() -> None: """SmartRecruiters and Ashby ignore case (checked live, 2026-09-29), so a slug from a page stays lowercased as it always was. A host added since may keep it, as Workday does for its site names.""" assert not mod._KEEP_CASE & {"greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee", "personio", "teamtailor"} def test_a_host_that_gains_a_module_replaces_its_unreadable_pattern() -> None: """Wave by wave, a host named here becomes readable by its module alone.""" taleo: Any = types.ModuleType("taleo") taleo.PATTERNS = (r"https?://([a-z0-9-]+)\.taleo\.net",) hosts = mod._hosts({"taleo": taleo}, mod._UNREADABLE) assert [pattern.pattern for host, pattern in hosts if host == "taleo"] == [ r"https?://([a-z0-9-]+)\.taleo\.net"] assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None: page = ('a' 'b') assert [(h, s) for h, s, _ in boards_in(page)] == [("greenhouse", "acme")] assert boards_in("nothing here") == [] def test_careers_links_keeps_on_site_links_that_read_as_careers() -> None: html = """ About Careers We are hiring Partner jobs Karriere Mail Careers again """ assert careers_links(html, "https://www.acme.com/") == [ "https://www.acme.com/careers", "https://jobs.acme.com/", "https://www.acme.com/en/karriere"] def test_slug_guesses_come_from_the_key_and_the_domain() -> None: assert slug_guesses("Trade Republic Bank GmbH", "www.traderepublic.com") == [ "trade-republic-bank", "traderepublicbank", "traderepublic"] assert slug_guesses("Pleo", "pleo.io") == ["pleo"] # ------------------------------------------------------------------- discover class Site: """A fake web: `pages` maps a URL to `(final URL, html)`; anything else is a page that did not answer. Remembers the order of fetches.""" def __init__(self, pages: dict[str, tuple[str, str] | str], boards: Set[tuple[str, str]] = frozenset()) -> None: self.pages = {k: (v if isinstance(v, tuple) else (k, v)) for k, v in pages.items()} self.boards = boards self.fetched: list[str] = [] self.probed: list[tuple[str, str]] = [] def fetch(self, client: httpx.Client, url: str) -> tuple[str, str]: self.fetched.append(url) return self.pages.get(url, ("", "")) def probe(self, client: httpx.Client, host: str, slug: str) -> bool: self.probed.append((host, slug)) return (host, slug) in self.boards def run(site: Site, name: str = "Acme", domain: str = "acme.com") -> Discovery: with httpx.Client() as client: return discover(name, domain, fetch=site.fetch, probe_slug=site.probe, client=client) def test_a_board_named_on_the_home_page_ends_discovery_there() -> None: site = Site({"https://acme.com/": 'Jobs'}) assert run(site) == Discovery(board="lever:acme", ats="lever", ats_slug="acme", careers_url="https://acme.com/", evidence="jobs.lever.co/acme") assert site.fetched == ["https://acme.com/"] assert site.probed == [] def test_the_home_pages_own_careers_link_is_read_before_the_usual_paths() -> None: site = Site({ "https://acme.com/": 'Join us', "https://acme.com/company/join-us": ('https://acme.com/company/join-us', 'Open'), "https://acme.com/careers": 'no', }) found = run(site) assert found.board == "recruitee:acme" assert found.careers_url == "https://acme.com/company/join-us" assert site.fetched == ["https://acme.com/", "https://acme.com/company/join-us"] def test_a_careers_pages_own_link_is_followed_one_level_down() -> None: site = Site({ "https://acme.com/": "

no links

", "https://acme.com/careers": 'See open positions', "https://acme.com/careers/open-positions": '', }) assert run(site).board == "ashby:acme" assert site.fetched[:3] == ["https://acme.com/", "https://acme.com/careers", "https://acme.com/careers/open-positions"] def test_an_unreadable_host_named_by_a_page_is_the_answer_and_nothing_is_probed() -> None: site = Site({ "https://acme.com/": "

welcome

", "https://acme.com/careers": 'Jobs', }, boards={("greenhouse", "acme")}) found = run(site) assert found == Discovery(ats="taleo", ats_slug="", careers_url="https://acme.com/careers", evidence=".taleo.net") assert not found.readable # The page named the host: a probe could only find another tenant's # board of the same name, like the greenhouse:acme this fake web has. assert site.probed == [] def test_a_readable_board_on_a_later_page_still_beats_an_unreadable_host() -> None: site = Site({ "https://acme.com/": '' 'Careers', "https://acme.com/careers": 'Jobs', }) assert run(site).board == "lever:acme" assert site.probed == [] def test_a_teamtailor_link_is_a_board_even_after_the_cdn_named_it_first() -> None: """The CDN names teamtailor before any link does; the slug comes from the first link that carries one, the acme of acme.teamtailor.com.""" site = Site({ "https://acme.com/": '', "https://acme.com/careers": 'Jobs', }) found = run(site) assert (found.ats, found.ats_slug, found.board) == ("teamtailor", "acme-jobs", "teamtailor:acme-jobs") assert found.readable assert found.evidence == "https://acme-jobs.teamtailor.com" assert found.careers_url == "https://acme.com/careers" def test_a_teamtailor_site_on_its_own_domain_is_read_off_its_host() -> None: """Only the CDN is named, so the site's host is the slug, once that host serves the feed.""" site = Site({ "https://acme.com/": 'Careers', "https://careers.acme.com/": '', "https://careers.acme.com/jobs.rss": '', }) found = run(site) assert found == Discovery(board="teamtailor:careers.acme.com", ats="teamtailor", ats_slug="careers.acme.com", careers_url="https://careers.acme.com/", evidence="https://careers.acme.com/jobs.rss") assert site.probed == [] def test_a_page_that_only_loads_the_cdn_is_not_a_board_without_the_feed() -> None: site = Site({"https://acme.com/": ''}) found = run(site) assert (found.ats, found.ats_slug, found.board) == ("teamtailor", "", "") assert not found.readable assert "https://acme.com/jobs.rss" in site.fetched def test_the_admin_app_and_cdn_hosts_are_no_teamtailor_slug() -> None: page = 'a' assert [(h, s) for h, s, _ in boards_in(page)] == [("teamtailor", "")] def test_a_teamtailor_board_is_found_by_its_slug_probe() -> None: site = Site({"https://acme.com/": "

a script draws the jobs

"}, boards={("teamtailor", "acme")}) found = run(site) assert found.board == "teamtailor:acme" assert found.evidence == "https://acme.teamtailor.com/jobs.rss" def test_slug_probes_find_a_board_the_pages_never_link() -> None: site = Site({"https://acme.com/": "

a script draws the jobs

", "https://acme.com/jobs": "
"}, boards={("ashby", "acme")}) found = run(site) assert found.board == "ashby:acme" assert found.careers_url == "https://acme.com/jobs" assert found.evidence == "https://api.ashbyhq.com/posting-api/job-board/acme" @pytest.mark.parametrize(("boards", "found"), [ ({("greenhouse", "acme"), ("ashby", "acme")}, "greenhouse:acme"), ({("lever", "acme"), ("ashby", "acme")}, "lever:acme"), ({("smartrecruiters", "acme"), ("personio", "acme")}, "smartrecruiters:acme"), ({("lever", "eu/acme"), ("personio", "acme")}, "lever:eu/acme"), ]) def test_a_slug_on_two_hosts_goes_to_the_one_probed_first( boards: set[tuple[str, str]], found: str) -> None: site = Site({}, boards=boards) assert run(site).board == found def test_the_probe_order_is_the_one_before_modules() -> None: assert list(mod._PROBES)[:7] == ["greenhouse", "lever", "ashby", "smartrecruiters", "recruitee", "personio", "teamtailor"] site = Site({}) run(site) assert site.probed[:8] == [("greenhouse", "acme"), ("lever", "acme"), ("lever", "eu/acme"), ("ashby", "acme"), ("smartrecruiters", "acme"), ("recruitee", "acme"), ("personio", "acme"), ("teamtailor", "acme")] def test_a_lever_eu_board_is_found_by_its_probe() -> None: site = Site({"https://acme.com/": "

a script draws the jobs

"}, boards={("lever", "eu/acme")}) found = run(site) assert found.board == "lever:eu/acme" assert found.evidence == "https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json" assert site.probed.index(("lever", "acme")) + 1 == site.probed.index(("lever", "eu/acme")) def test_probe_asks_each_hosts_own_endpoint_and_body_check() -> None: asked: list[str] = [] def handler(request: httpx.Request) -> httpx.Response: asked.append(str(request.url)) if request.url.host == "api.eu.lever.co": return httpx.Response(200, json=[]) if request.url.host == "api.ashbyhq.com": return httpx.Response(200, content=b'{"jobs":[]}') return httpx.Response(404) with httpx.Client(transport=httpx.MockTransport(handler)) as client: assert mod.probe(client, "lever", "eu/acme") assert not mod.probe(client, "lever", "acme") assert not mod.probe(client, "ashby", "acme") # an empty listing is no board assert asked == ["https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json", "https://api.lever.co/v0/postings/acme?limit=1&mode=json", "https://api.ashbyhq.com/posting-api/job-board/acme"] assert "workable" not in mod._PROBES and "lever" in mod._PROBES def test_a_site_that_never_answers_gives_an_empty_discovery() -> None: site = Site({}) assert run(site) == Discovery() assert len(site.fetched) <= mod.MAX_PAGES + 1 def test_a_domain_is_folded_to_its_site() -> None: site = Site({"https://acme.com/": 'Jobs'}) with httpx.Client() as client: found = discover("Acme", "https://www.acme.com/some/page", fetch=site.fetch, probe_slug=site.probe, client=client) assert found.board == "lever:acme" assert discover("Acme", "", fetch=site.fetch, probe_slug=site.probe) == Discovery() # ---------------------------------------------------------------- the seed @pytest.fixture def store(tmp_path: Path) -> Store: return Store(tmp_path / "t.db") def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Path) -> None: seed = tmp_path / "companies.json" seed.write_text(json.dumps({"companies": [ {"name": "Pleo", "domain": "pleo.io", "country": "DK", "city": "Copenhagen", "board": "ashby:pleo"}, {"name": "Novo Nordisk", "domain": "novonordisk.com", "country": "DK", "city": "Bagsværd", "ats": "taleo", "careers_url": "https://www.novonordisk.com/careers.html"}, {"name": "Odd", "board": "nosuch:odd"}, ]})) result = seed_companies(store, seed) assert result == {"added": 3, "boards": 1, "skipped": 0, "total": 3, "problems": ["odd: 'nosuch:odd' is not a known board; kept without it"]} pleo = store.company("pleo") assert pleo is not None assert (pleo.ats, pleo.ats_slug, pleo.country, pleo.city, pleo.careers_url) == ( "ashby", "pleo", "DK", "Copenhagen", "https://jobs.ashbyhq.com/pleo") [sub] = store.subscriptions(source="ats") assert (sub.target, sub.company) == ("ashby:pleo", "Pleo") novo = store.company("novo-nordisk") assert novo is not None assert (novo.ats, novo.careers_url) == ("taleo", "https://www.novonordisk.com/careers.html") odd = store.company("odd") assert odd is not None and odd.ats == "" # Re-running adds nothing and edits nothing. assert seed_companies(store, seed)["skipped"] == 3