Files
jobboard/tests/test_discover.py
WolfandClaude Fable 5.1 531c8bc337 Delete the _UNREADABLE entries of hosts that have a module
Ten entries were shadowed by their modules. A test now holds that no entry
names a reader, which replaces the per-host checks.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-30 15:51:48 +02:00

384 lines
17 KiB
Python

"""Board discovery: which ATS a careers site names, and in what order the
site's pages are read."""
from __future__ import annotations
import json
import re
import types
from collections.abc import Set
from pathlib import Path
from typing import Any
import httpx
import pytest
from app.jobs import discover as mod
from app.jobs.discover import Discovery, boards_in, careers_links, discover, slug_guesses
from app.jobs.sources import seed_companies
from app.storage import Store
from plugins import readers
# ------------------------------------------------------------------ parsing
def test_boards_in_names_readable_hosts_with_their_slug() -> None:
page = """
<a href="https://job-boards.greenhouse.io/pleo">Open roles</a>
<script src="https://boards.greenhouse.io/embed/job_board/js?for=pleo"></script>
<a href="https://jobs.lever.co/spotify/abc-123">One role</a>
<iframe src="https://jobs.ashbyhq.com/DeepL/embed"></iframe>
<a href="https://apply.workable.com/acme/">Workable</a>
<a href="https://jobs.smartrecruiters.com/Bosch/123">SR</a>
<a href="https://acme.recruitee.com/o/backend">Recruitee</a>
<a href="https://acme-gmbh.jobs.personio.de/job/1">Personio</a>
"""
assert [(h, s) for h, s, _url in boards_in(page)] == [
("greenhouse", "pleo"), ("lever", "spotify"), ("ashby", "deepl"),
("workable", "acme"), ("smartrecruiters", "bosch"), ("recruitee", "acme"),
("personio", "acme-gmbh")]
def test_boards_in_keeps_hosts_the_app_cannot_read() -> None:
page = """
<a href="https://novo.taleo.net/careersection/2/jobsearch.ftl">Taleo</a>
<a href="https://acme.teamtailor.com/jobs">TT</a>
"""
assert boards_in(page) == [
("taleo", "", ".taleo.net"),
("teamtailor", "acme", "https://acme.teamtailor.com")]
def test_boards_in_names_a_lever_eu_board_by_its_region() -> None:
page = ('<a href="https://jobs.eu.lever.co/tomtom/abc-123">EU</a>'
'<script src="https://api.eu.lever.co/v0/postings/tomtom?mode=json"></script>'
'<a href="https://jobs.lever.co/whereby">US</a>')
assert [(h, s) for h, s, _ in boards_in(page)] == [("lever", "eu/tomtom"),
("lever", "whereby")]
def test_boards_in_lowercases_a_slug_unless_the_host_keeps_its_case(
monkeypatch: pytest.MonkeyPatch) -> None:
"""A Workday-shaped host with two groups and `KEEP_CASE`: the site keeps
its case, and a site called `jobs` is a site, since only group 1 is
checked against the hosts' own path words."""
workday = re.compile(r"https?://([a-z0-9-]+\.wd\d+)\.myworkdayjobs\.com/"
r"(?:[a-z]{2}-[A-Z]{2}/)?([A-Za-z0-9_-]+)", re.IGNORECASE)
monkeypatch.setattr(mod, "HOSTS", (("workday", workday),
*(h for h in mod.HOSTS if h[0] != "workday")))
monkeypatch.setattr(mod, "_KEEP_CASE", frozenset({"workday"}))
page = ('<a href="https://lego.wd103.myworkdayjobs.com/LEGO_External">a</a>'
'<a href="https://redhat.wd5.myworkdayjobs.com/en-US/jobs">b</a>'
'<a href="https://jobs.smartrecruiters.com/BoschGroup">c</a>'
'<a href="https://jobs.eu.lever.co/TomTom">d</a>')
assert [(h, s) for h, s, _ in boards_in(page)] == [
("workday", "lego.wd103/LEGO_External"), ("workday", "redhat.wd5/jobs"),
("smartrecruiters", "boschgroup"), ("lever", "eu/tomtom")]
# A host's own path word in group 1 still names no board.
assert boards_in('<a href="https://jobs.smartrecruiters.com/api">x</a>') == []
def test_no_module_of_the_eight_keeps_its_case() -> None:
"""SmartRecruiters and Ashby ignore case (checked live, 2026-09-29), so a
slug from a page stays lowercased as it always was. A host added since
may keep it, as Workday does for its site names."""
assert not mod._KEEP_CASE & {"greenhouse", "lever", "ashby", "workable", "smartrecruiters",
"recruitee", "personio", "teamtailor"}
def test_a_host_that_gains_a_module_replaces_its_unreadable_pattern() -> None:
"""Wave by wave, a host named here becomes readable by its module alone."""
taleo: Any = types.ModuleType("taleo")
taleo.PATTERNS = (r"https?://([a-z0-9-]+)\.taleo\.net",)
hosts = mod._hosts({"taleo": taleo}, mod._UNREADABLE)
assert [pattern.pattern for host, pattern in hosts if host == "taleo"] == [
r"https?://([a-z0-9-]+)\.taleo\.net"]
assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE
def test_no_unreadable_entry_names_a_host_with_a_module() -> None:
"""A new module deletes its host's `_UNREADABLE` entry: its own patterns
replace it."""
assert not {host for host, _ in mod._UNREADABLE} & set(readers.HOSTS)
def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None:
page = ('<a href="https://boards.greenhouse.io/embed/job_board?for=acme">a</a>'
'<a href="https://boards-api.greenhouse.io/v1/boards/acme/jobs">b</a>')
assert [(h, s) for h, s, _ in boards_in(page)] == [("greenhouse", "acme")]
assert boards_in("nothing here") == []
def test_careers_links_keeps_on_site_links_that_read_as_careers() -> None:
html = """
<a href="/about">About</a>
<a href="/careers">Careers</a>
<a href="https://jobs.acme.com/">We are hiring</a>
<a href="https://other.example/jobs">Partner jobs</a>
<a href="/en/karriere#top"><span>Karriere</span></a>
<a href="mailto:jobs@acme.com">Mail</a>
<a href="/careers">Careers again</a>
"""
assert careers_links(html, "https://www.acme.com/") == [
"https://www.acme.com/careers", "https://jobs.acme.com/",
"https://www.acme.com/en/karriere"]
def test_slug_guesses_come_from_the_key_and_the_domain() -> None:
assert slug_guesses("Trade Republic Bank GmbH", "www.traderepublic.com") == [
"trade-republic-bank", "traderepublicbank", "traderepublic"]
assert slug_guesses("Pleo", "pleo.io") == ["pleo"]
# ------------------------------------------------------------------- discover
class Site:
"""A fake web: `pages` maps a URL to `(final URL, html)`; anything else
is a page that did not answer. Remembers the order of fetches."""
def __init__(self, pages: dict[str, tuple[str, str] | str],
boards: Set[tuple[str, str]] = frozenset()) -> None:
self.pages = {k: (v if isinstance(v, tuple) else (k, v)) for k, v in pages.items()}
self.boards = boards
self.fetched: list[str] = []
self.probed: list[tuple[str, str]] = []
def fetch(self, client: httpx.Client, url: str) -> tuple[str, str]:
self.fetched.append(url)
return self.pages.get(url, ("", ""))
def probe(self, client: httpx.Client, host: str, slug: str) -> bool:
self.probed.append((host, slug))
return (host, slug) in self.boards
def run(site: Site, name: str = "Acme", domain: str = "acme.com") -> Discovery:
with httpx.Client() as client:
return discover(name, domain, fetch=site.fetch, probe_slug=site.probe, client=client)
def test_a_board_named_on_the_home_page_ends_discovery_there() -> None:
site = Site({"https://acme.com/": '<a href="https://jobs.lever.co/acme">Jobs</a>'})
assert run(site) == Discovery(board="lever:acme", ats="lever", ats_slug="acme",
careers_url="https://acme.com/",
evidence="jobs.lever.co/acme")
assert site.fetched == ["https://acme.com/"]
assert site.probed == []
def test_the_home_pages_own_careers_link_is_read_before_the_usual_paths() -> None:
site = Site({
"https://acme.com/": '<a href="/company/join-us">Join us</a>',
"https://acme.com/company/join-us": ('https://acme.com/company/join-us',
'<a href="https://acme.recruitee.com/">Open</a>'),
"https://acme.com/careers": '<a href="https://jobs.lever.co/wrong">no</a>',
})
found = run(site)
assert found.board == "recruitee:acme"
assert found.careers_url == "https://acme.com/company/join-us"
assert site.fetched == ["https://acme.com/", "https://acme.com/company/join-us"]
def test_a_careers_pages_own_link_is_followed_one_level_down() -> None:
site = Site({
"https://acme.com/": "<p>no links</p>",
"https://acme.com/careers": '<a href="/careers/open-positions">See open positions</a>',
"https://acme.com/careers/open-positions":
'<iframe src="https://jobs.ashbyhq.com/acme"></iframe>',
})
assert run(site).board == "ashby:acme"
assert site.fetched[:3] == ["https://acme.com/", "https://acme.com/careers",
"https://acme.com/careers/open-positions"]
def test_an_unreadable_host_named_by_a_page_is_the_answer_and_nothing_is_probed() -> None:
site = Site({
"https://acme.com/": "<p>welcome</p>",
"https://acme.com/careers": '<a href="https://acme.taleo.net/careersection/x">Jobs</a>',
}, boards={("greenhouse", "acme")})
found = run(site)
assert found == Discovery(ats="taleo", ats_slug="",
careers_url="https://acme.com/careers",
evidence=".taleo.net")
assert not found.readable
# The page named the host: a probe could only find another tenant's
# board of the same name, like the greenhouse:acme this fake web has.
assert site.probed == []
def test_a_readable_board_on_a_later_page_still_beats_an_unreadable_host() -> None:
site = Site({
"https://acme.com/": '<script src="https://cdn.teamtailor.com/x.js"></script>'
'<a href="/careers">Careers</a>',
"https://acme.com/careers": '<a href="https://jobs.lever.co/acme">Jobs</a>',
})
assert run(site).board == "lever:acme"
assert site.probed == []
def test_a_teamtailor_link_is_a_board_even_after_the_cdn_named_it_first() -> None:
"""The CDN names teamtailor before any link does; the slug comes from the
first link that carries one, the acme of acme.teamtailor.com."""
site = Site({
"https://acme.com/": '<script src="https://cdn.teamtailor.com/x.js"></script>',
"https://acme.com/careers": '<a href="https://acme-jobs.teamtailor.com/jobs">Jobs</a>',
})
found = run(site)
assert (found.ats, found.ats_slug, found.board) == ("teamtailor", "acme-jobs",
"teamtailor:acme-jobs")
assert found.readable
assert found.evidence == "https://acme-jobs.teamtailor.com"
assert found.careers_url == "https://acme.com/careers"
def test_a_teamtailor_site_on_its_own_domain_is_read_off_its_host() -> None:
"""Only the CDN is named, so the site's host is the slug, once that host
serves the feed."""
site = Site({
"https://acme.com/": '<a href="https://careers.acme.com/">Careers</a>',
"https://careers.acme.com/": '<script src="https://cdn.teamtailor.com/x.js"></script>',
"https://careers.acme.com/jobs.rss":
'<rss xmlns:tt="https://teamtailor.com/locations"><channel/></rss>',
})
found = run(site)
assert found == Discovery(board="teamtailor:careers.acme.com", ats="teamtailor",
ats_slug="careers.acme.com",
careers_url="https://careers.acme.com/",
evidence="https://careers.acme.com/jobs.rss")
assert site.probed == []
def test_a_page_that_only_loads_the_cdn_is_not_a_board_without_the_feed() -> None:
site = Site({"https://acme.com/": '<script src="https://cdn.teamtailor.com/x.js"></script>'})
found = run(site)
assert (found.ats, found.ats_slug, found.board) == ("teamtailor", "", "")
assert not found.readable
assert "https://acme.com/jobs.rss" in site.fetched
def test_the_admin_app_and_cdn_hosts_are_no_teamtailor_slug() -> None:
page = '<a href="https://app.teamtailor.com/x">a</a><img src="https://cdn.teamtailor.com/y">'
assert [(h, s) for h, s, _ in boards_in(page)] == [("teamtailor", "")]
def test_a_teamtailor_board_is_found_by_its_slug_probe() -> None:
site = Site({"https://acme.com/": "<p>a script draws the jobs</p>"},
boards={("teamtailor", "acme")})
found = run(site)
assert found.board == "teamtailor:acme"
assert found.evidence == "https://acme.teamtailor.com/jobs.rss"
def test_slug_probes_find_a_board_the_pages_never_link() -> None:
site = Site({"https://acme.com/": "<p>a script draws the jobs</p>",
"https://acme.com/jobs": "<div id=app></div>"},
boards={("ashby", "acme")})
found = run(site)
assert found.board == "ashby:acme"
assert found.careers_url == "https://acme.com/jobs"
assert found.evidence == "https://api.ashbyhq.com/posting-api/job-board/acme"
@pytest.mark.parametrize(("boards", "found"), [
({("greenhouse", "acme"), ("ashby", "acme")}, "greenhouse:acme"),
({("lever", "acme"), ("ashby", "acme")}, "lever:acme"),
({("smartrecruiters", "acme"), ("personio", "acme")}, "smartrecruiters:acme"),
({("lever", "eu/acme"), ("personio", "acme")}, "lever:eu/acme"),
])
def test_a_slug_on_two_hosts_goes_to_the_one_probed_first(
boards: set[tuple[str, str]], found: str) -> None:
site = Site({}, boards=boards)
assert run(site).board == found
def test_the_probe_order_is_the_one_before_modules() -> None:
assert list(mod._PROBES)[:7] == ["greenhouse", "lever", "ashby", "smartrecruiters",
"recruitee", "personio", "teamtailor"]
site = Site({})
run(site)
assert site.probed[:8] == [("greenhouse", "acme"), ("lever", "acme"), ("lever", "eu/acme"),
("ashby", "acme"), ("smartrecruiters", "acme"),
("recruitee", "acme"), ("personio", "acme"),
("teamtailor", "acme")]
def test_a_lever_eu_board_is_found_by_its_probe() -> None:
site = Site({"https://acme.com/": "<p>a script draws the jobs</p>"},
boards={("lever", "eu/acme")})
found = run(site)
assert found.board == "lever:eu/acme"
assert found.evidence == "https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json"
assert site.probed.index(("lever", "acme")) + 1 == site.probed.index(("lever", "eu/acme"))
def test_probe_asks_each_hosts_own_endpoint_and_body_check() -> None:
asked: list[str] = []
def handler(request: httpx.Request) -> httpx.Response:
asked.append(str(request.url))
if request.url.host == "api.eu.lever.co":
return httpx.Response(200, json=[])
if request.url.host == "api.ashbyhq.com":
return httpx.Response(200, content=b'{"jobs":[]}')
return httpx.Response(404)
with httpx.Client(transport=httpx.MockTransport(handler)) as client:
assert mod.probe(client, "lever", "eu/acme")
assert not mod.probe(client, "lever", "acme")
assert not mod.probe(client, "ashby", "acme") # an empty listing is no board
assert asked == ["https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json",
"https://api.lever.co/v0/postings/acme?limit=1&mode=json",
"https://api.ashbyhq.com/posting-api/job-board/acme"]
assert "workable" not in mod._PROBES and "lever" in mod._PROBES
def test_a_site_that_never_answers_gives_an_empty_discovery() -> None:
site = Site({})
assert run(site) == Discovery()
assert len(site.fetched) <= mod.MAX_PAGES + 1
def test_a_domain_is_folded_to_its_site() -> None:
site = Site({"https://acme.com/": '<a href="https://jobs.lever.co/acme">Jobs</a>'})
with httpx.Client() as client:
found = discover("Acme", "https://www.acme.com/some/page", fetch=site.fetch,
probe_slug=site.probe, client=client)
assert found.board == "lever:acme"
assert discover("Acme", "", fetch=site.fetch, probe_slug=site.probe) == Discovery()
# ---------------------------------------------------------------- the seed
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "t.db")
def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Path) -> None:
seed = tmp_path / "companies.json"
seed.write_text(json.dumps({"companies": [
{"name": "Pleo", "domain": "pleo.io", "country": "DK", "city": "Copenhagen",
"board": "ashby:pleo"},
{"name": "Novo Nordisk", "domain": "novonordisk.com", "country": "DK",
"city": "Bagsværd", "ats": "taleo",
"careers_url": "https://www.novonordisk.com/careers.html"},
{"name": "Odd", "board": "nosuch:odd"},
]}))
result = seed_companies(store, seed)
assert result == {"added": 3, "boards": 1, "skipped": 0, "total": 3,
"problems": ["odd: 'nosuch:odd' is not a known board; kept without it"]}
pleo = store.company("pleo")
assert pleo is not None
assert (pleo.ats, pleo.ats_slug, pleo.country, pleo.city, pleo.careers_url) == (
"ashby", "pleo", "DK", "Copenhagen", "https://jobs.ashbyhq.com/pleo")
[sub] = store.subscriptions(source="ats")
assert (sub.target, sub.company) == ("ashby:pleo", "Pleo")
novo = store.company("novo-nordisk")
assert novo is not None
assert (novo.ats, novo.careers_url) == ("taleo",
"https://www.novonordisk.com/careers.html")
odd = store.company("odd")
assert odd is not None and odd.ats == ""
# Re-running adds nothing and edits nothing.
assert seed_companies(store, seed)["skipped"] == 3