"""Board discovery: which ATS a careers site names, and in what order the site's pages are read.""" from __future__ import annotations import json import re import types from pathlib import Path import httpx import pytest from app.jobs import discover as mod from app.jobs.discover import Discovery, boards_in, careers_links, discover, slug_guesses from app.jobs.sources import seed_companies from app.storage import Store # ------------------------------------------------------------------ parsing def test_boards_in_names_readable_hosts_with_their_slug() -> None: page = """ Open roles One role Workable SR Recruitee Personio """ assert [(h, s) for h, s, _url in boards_in(page)] == [ ("greenhouse", "pleo"), ("lever", "spotify"), ("ashby", "deepl"), ("workable", "acme"), ("smartrecruiters", "bosch"), ("recruitee", "acme"), ("personio", "acme-gmbh")] def test_boards_in_keeps_hosts_the_app_cannot_read() -> None: page = """ Taleo TT """ assert boards_in(page) == [ ("taleo", "", ".taleo.net"), ("teamtailor", "acme", "https://acme.teamtailor.com")] def test_boards_in_names_a_lever_eu_board_by_its_region() -> None: page = ('EU' '' 'US') assert [(h, s) for h, s, _ in boards_in(page)] == [("lever", "eu/tomtom"), ("lever", "whereby")] def test_boards_in_lowercases_a_slug_unless_the_host_keeps_its_case( monkeypatch: pytest.MonkeyPatch) -> None: """A Workday-shaped host with two groups and `KEEP_CASE`: the site keeps its case, and a site called `jobs` is a site, since only group 1 is checked against the hosts' own path words.""" workday = re.compile(r"https?://([a-z0-9-]+\.wd\d+)\.myworkdayjobs\.com/" r"(?:[a-z]{2}-[A-Z]{2}/)?([A-Za-z0-9_-]+)", re.IGNORECASE) monkeypatch.setattr(mod, "HOSTS", (("workday", workday), *(h for h in mod.HOSTS if h[0] != "workday"))) monkeypatch.setattr(mod, "_KEEP_CASE", frozenset({"workday"})) page = ('a' 'b' 'c' 'd') assert [(h, s) for h, s, _ in boards_in(page)] == [ ("workday", "lego.wd103/LEGO_External"), ("workday", "redhat.wd5/jobs"), ("smartrecruiters", "boschgroup"), ("lever", "eu/tomtom")] # A host's own path word in group 1 still names no board. assert boards_in('x') == [] def test_no_module_of_the_eight_keeps_its_case() -> None: """SmartRecruiters and Ashby ignore case (checked live, 2026-09-29), so a slug from a page stays lowercased as it always was.""" assert not mod._KEEP_CASE def test_a_host_that_gains_a_module_replaces_its_unreadable_pattern() -> None: """Wave by wave, a host named here becomes readable by its module alone.""" taleo = types.ModuleType("taleo") taleo.PATTERNS = (r"https?://([a-z0-9-]+)\.taleo\.net",) hosts = mod._hosts({"taleo": taleo}, mod._UNREADABLE) assert [pattern.pattern for host, pattern in hosts if host == "taleo"] == [ r"https?://([a-z0-9-]+)\.taleo\.net"] assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None: page = ('a' 'b') assert [(h, s) for h, s, _ in boards_in(page)] == [("greenhouse", "acme")] assert boards_in("nothing here") == [] def test_careers_links_keeps_on_site_links_that_read_as_careers() -> None: html = """ About Careers We are hiring Partner jobs Karriere Mail Careers again """ assert careers_links(html, "https://www.acme.com/") == [ "https://www.acme.com/careers", "https://jobs.acme.com/", "https://www.acme.com/en/karriere"] def test_slug_guesses_come_from_the_key_and_the_domain() -> None: assert slug_guesses("Trade Republic Bank GmbH", "www.traderepublic.com") == [ "trade-republic-bank", "traderepublicbank", "traderepublic"] assert slug_guesses("Pleo", "pleo.io") == ["pleo"] # ------------------------------------------------------------------- discover class Site: """A fake web: `pages` maps a URL to `(final URL, html)`; anything else is a page that did not answer. Remembers the order of fetches.""" def __init__(self, pages: dict[str, tuple[str, str] | str], boards: set[tuple[str, str]] = frozenset()) -> None: self.pages = {k: (v if isinstance(v, tuple) else (k, v)) for k, v in pages.items()} self.boards = boards self.fetched: list[str] = [] self.probed: list[tuple[str, str]] = [] def fetch(self, client: httpx.Client, url: str) -> tuple[str, str]: self.fetched.append(url) return self.pages.get(url, ("", "")) def probe(self, client: httpx.Client, host: str, slug: str) -> bool: self.probed.append((host, slug)) return (host, slug) in self.boards def run(site: Site, name: str = "Acme", domain: str = "acme.com") -> Discovery: with httpx.Client() as client: return discover(name, domain, fetch=site.fetch, probe_slug=site.probe, client=client) def test_a_board_named_on_the_home_page_ends_discovery_there() -> None: site = Site({"https://acme.com/": 'Jobs'}) assert run(site) == Discovery(board="lever:acme", ats="lever", ats_slug="acme", careers_url="https://acme.com/", evidence="jobs.lever.co/acme") assert site.fetched == ["https://acme.com/"] assert site.probed == [] def test_the_home_pages_own_careers_link_is_read_before_the_usual_paths() -> None: site = Site({ "https://acme.com/": 'Join us', "https://acme.com/company/join-us": ('https://acme.com/company/join-us', 'Open'), "https://acme.com/careers": 'no', }) found = run(site) assert found.board == "recruitee:acme" assert found.careers_url == "https://acme.com/company/join-us" assert site.fetched == ["https://acme.com/", "https://acme.com/company/join-us"] def test_a_careers_pages_own_link_is_followed_one_level_down() -> None: site = Site({ "https://acme.com/": "

no links

", "https://acme.com/careers": 'See open positions', "https://acme.com/careers/open-positions": '', }) assert run(site).board == "ashby:acme" assert site.fetched[:3] == ["https://acme.com/", "https://acme.com/careers", "https://acme.com/careers/open-positions"] def test_an_unreadable_host_named_by_a_page_is_the_answer_and_nothing_is_probed() -> None: site = Site({ "https://acme.com/": "

welcome

", "https://acme.com/careers": 'Jobs', }, boards={("greenhouse", "acme")}) found = run(site) assert found == Discovery(ats="taleo", ats_slug="", careers_url="https://acme.com/careers", evidence=".taleo.net") assert not found.readable # The page named the host: a probe could only find another tenant's # board of the same name, like the greenhouse:acme this fake web has. assert site.probed == [] def test_a_readable_board_on_a_later_page_still_beats_an_unreadable_host() -> None: site = Site({ "https://acme.com/": '' 'Careers', "https://acme.com/careers": 'Jobs', }) assert run(site).board == "lever:acme" assert site.probed == [] def test_a_teamtailor_link_is_a_board_even_after_the_cdn_named_it_first() -> None: """The CDN names teamtailor before any link does; the slug comes from the first link that carries one, the acme of acme.teamtailor.com.""" site = Site({ "https://acme.com/": '', "https://acme.com/careers": 'Jobs', }) found = run(site) assert (found.ats, found.ats_slug, found.board) == ("teamtailor", "acme-jobs", "teamtailor:acme-jobs") assert found.readable assert found.evidence == "https://acme-jobs.teamtailor.com" assert found.careers_url == "https://acme.com/careers" def test_a_teamtailor_site_on_its_own_domain_is_read_off_its_host() -> None: """Only the CDN is named, so the site's host is the slug, once that host serves the feed.""" site = Site({ "https://acme.com/": 'Careers', "https://careers.acme.com/": '', "https://careers.acme.com/jobs.rss": '', }) found = run(site) assert found == Discovery(board="teamtailor:careers.acme.com", ats="teamtailor", ats_slug="careers.acme.com", careers_url="https://careers.acme.com/", evidence="https://careers.acme.com/jobs.rss") assert site.probed == [] def test_a_page_that_only_loads_the_cdn_is_not_a_board_without_the_feed() -> None: site = Site({"https://acme.com/": ''}) found = run(site) assert (found.ats, found.ats_slug, found.board) == ("teamtailor", "", "") assert not found.readable assert "https://acme.com/jobs.rss" in site.fetched def test_the_admin_app_and_cdn_hosts_are_no_teamtailor_slug() -> None: page = 'a' assert [(h, s) for h, s, _ in boards_in(page)] == [("teamtailor", "")] def test_a_teamtailor_board_is_found_by_its_slug_probe() -> None: site = Site({"https://acme.com/": "

a script draws the jobs

"}, boards={("teamtailor", "acme")}) found = run(site) assert found.board == "teamtailor:acme" assert found.evidence == "https://acme.teamtailor.com/jobs.rss" def test_slug_probes_find_a_board_the_pages_never_link() -> None: site = Site({"https://acme.com/": "

a script draws the jobs

", "https://acme.com/jobs": "
"}, boards={("ashby", "acme")}) found = run(site) assert found.board == "ashby:acme" assert found.careers_url == "https://acme.com/jobs" assert found.evidence == "https://api.ashbyhq.com/posting-api/job-board/acme" def test_a_lever_eu_board_is_found_by_its_probe() -> None: site = Site({"https://acme.com/": "

a script draws the jobs

"}, boards={("lever", "eu/acme")}) found = run(site) assert found.board == "lever:eu/acme" assert found.evidence == "https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json" assert site.probed.index(("lever", "acme")) + 1 == site.probed.index(("lever", "eu/acme")) def test_probe_asks_each_hosts_own_endpoint_and_body_check() -> None: asked: list[str] = [] def handler(request: httpx.Request) -> httpx.Response: asked.append(str(request.url)) if request.url.host == "api.eu.lever.co": return httpx.Response(200, json=[]) if request.url.host == "api.ashbyhq.com": return httpx.Response(200, content=b'{"jobs":[]}') return httpx.Response(404) with httpx.Client(transport=httpx.MockTransport(handler)) as client: assert mod.probe(client, "lever", "eu/acme") assert not mod.probe(client, "lever", "acme") assert not mod.probe(client, "ashby", "acme") # an empty listing is no board assert asked == ["https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json", "https://api.lever.co/v0/postings/acme?limit=1&mode=json", "https://api.ashbyhq.com/posting-api/job-board/acme"] assert "workable" not in mod._PROBES and "lever" in mod._PROBES def test_a_site_that_never_answers_gives_an_empty_discovery() -> None: site = Site({}) assert run(site) == Discovery() assert len(site.fetched) <= mod.MAX_PAGES + 1 def test_a_domain_is_folded_to_its_site() -> None: site = Site({"https://acme.com/": 'Jobs'}) with httpx.Client() as client: found = discover("Acme", "https://www.acme.com/some/page", fetch=site.fetch, probe_slug=site.probe, client=client) assert found.board == "lever:acme" assert discover("Acme", "", fetch=site.fetch, probe_slug=site.probe) == Discovery() # ---------------------------------------------------------------- the seed @pytest.fixture def store(tmp_path: Path) -> Store: return Store(tmp_path / "t.db") def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Path) -> None: seed = tmp_path / "companies.json" seed.write_text(json.dumps({"companies": [ {"name": "Pleo", "domain": "pleo.io", "country": "DK", "city": "Copenhagen", "board": "ashby:pleo"}, {"name": "Novo Nordisk", "domain": "novonordisk.com", "country": "DK", "city": "Bagsværd", "ats": "taleo", "careers_url": "https://www.novonordisk.com/careers.html"}, {"name": "Odd", "board": "nosuch:odd"}, ]})) result = seed_companies(store, seed) assert result == {"added": 3, "boards": 1, "skipped": 0, "total": 3, "problems": ["odd: 'nosuch:odd' is not a known board; kept without it"]} pleo = store.company("pleo") assert pleo is not None assert (pleo.ats, pleo.ats_slug, pleo.country, pleo.city, pleo.careers_url) == ( "ashby", "pleo", "DK", "Copenhagen", "https://jobs.ashbyhq.com/pleo") [sub] = store.subscriptions(source="ats") assert (sub.target, sub.company) == ("ashby:pleo", "Pleo") novo = store.company("novo-nordisk") assert novo is not None assert (novo.ats, novo.careers_url) == ("taleo", "https://www.novonordisk.com/careers.html") odd = store.company("odd") assert odd is not None and odd.ats == "" # Re-running adds nothing and edits nothing. assert seed_companies(store, seed)["skipped"] == 3