373 lines
17 KiB
Python
373 lines
17 KiB
Python
"""Board discovery: which ATS a careers site names, and in what order the
|
|
site's pages are read."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import types
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from app.jobs import discover as mod
|
|
from app.jobs.discover import Discovery, boards_in, careers_links, discover, slug_guesses
|
|
from app.jobs.sources import seed_companies
|
|
from app.storage import Store
|
|
|
|
|
|
# ------------------------------------------------------------------ parsing
|
|
|
|
def test_boards_in_names_readable_hosts_with_their_slug() -> None:
|
|
page = """
|
|
<a href="https://job-boards.greenhouse.io/pleo">Open roles</a>
|
|
<script src="https://boards.greenhouse.io/embed/job_board/js?for=pleo"></script>
|
|
<a href="https://jobs.lever.co/spotify/abc-123">One role</a>
|
|
<iframe src="https://jobs.ashbyhq.com/DeepL/embed"></iframe>
|
|
<a href="https://apply.workable.com/acme/">Workable</a>
|
|
<a href="https://jobs.smartrecruiters.com/Bosch/123">SR</a>
|
|
<a href="https://acme.recruitee.com/o/backend">Recruitee</a>
|
|
<a href="https://acme-gmbh.jobs.personio.de/job/1">Personio</a>
|
|
"""
|
|
assert [(h, s) for h, s, _url in boards_in(page)] == [
|
|
("greenhouse", "pleo"), ("lever", "spotify"), ("ashby", "deepl"),
|
|
("workable", "acme"), ("smartrecruiters", "bosch"), ("recruitee", "acme"),
|
|
("personio", "acme-gmbh")]
|
|
|
|
|
|
def test_boards_in_keeps_hosts_the_app_cannot_read() -> None:
|
|
page = """
|
|
<a href="https://novo.taleo.net/careersection/2/jobsearch.ftl">Taleo</a>
|
|
<a href="https://acme.teamtailor.com/jobs">TT</a>
|
|
"""
|
|
assert boards_in(page) == [
|
|
("taleo", "", ".taleo.net"),
|
|
("teamtailor", "acme", "https://acme.teamtailor.com")]
|
|
|
|
|
|
def test_boards_in_names_a_lever_eu_board_by_its_region() -> None:
|
|
page = ('<a href="https://jobs.eu.lever.co/tomtom/abc-123">EU</a>'
|
|
'<script src="https://api.eu.lever.co/v0/postings/tomtom?mode=json"></script>'
|
|
'<a href="https://jobs.lever.co/whereby">US</a>')
|
|
assert [(h, s) for h, s, _ in boards_in(page)] == [("lever", "eu/tomtom"),
|
|
("lever", "whereby")]
|
|
|
|
|
|
def test_boards_in_lowercases_a_slug_unless_the_host_keeps_its_case(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A Workday-shaped host with two groups and `KEEP_CASE`: the site keeps
|
|
its case, and a site called `jobs` is a site, since only group 1 is
|
|
checked against the hosts' own path words."""
|
|
workday = re.compile(r"https?://([a-z0-9-]+\.wd\d+)\.myworkdayjobs\.com/"
|
|
r"(?:[a-z]{2}-[A-Z]{2}/)?([A-Za-z0-9_-]+)", re.IGNORECASE)
|
|
monkeypatch.setattr(mod, "HOSTS", (("workday", workday),
|
|
*(h for h in mod.HOSTS if h[0] != "workday")))
|
|
monkeypatch.setattr(mod, "_KEEP_CASE", frozenset({"workday"}))
|
|
page = ('<a href="https://lego.wd103.myworkdayjobs.com/LEGO_External">a</a>'
|
|
'<a href="https://redhat.wd5.myworkdayjobs.com/en-US/jobs">b</a>'
|
|
'<a href="https://jobs.smartrecruiters.com/BoschGroup">c</a>'
|
|
'<a href="https://jobs.eu.lever.co/TomTom">d</a>')
|
|
assert [(h, s) for h, s, _ in boards_in(page)] == [
|
|
("workday", "lego.wd103/LEGO_External"), ("workday", "redhat.wd5/jobs"),
|
|
("smartrecruiters", "boschgroup"), ("lever", "eu/tomtom")]
|
|
# A host's own path word in group 1 still names no board.
|
|
assert boards_in('<a href="https://jobs.smartrecruiters.com/api">x</a>') == []
|
|
|
|
|
|
def test_no_module_of_the_eight_keeps_its_case() -> None:
|
|
"""SmartRecruiters and Ashby ignore case (checked live, 2026-09-29), so a
|
|
slug from a page stays lowercased as it always was."""
|
|
assert not mod._KEEP_CASE
|
|
|
|
|
|
def test_a_host_that_gains_a_module_replaces_its_unreadable_pattern() -> None:
|
|
"""Wave by wave, a host named here becomes readable by its module alone."""
|
|
taleo = types.ModuleType("taleo")
|
|
taleo.PATTERNS = (r"https?://([a-z0-9-]+)\.taleo\.net",)
|
|
hosts = mod._hosts({"taleo": taleo}, mod._UNREADABLE)
|
|
assert [pattern.pattern for host, pattern in hosts if host == "taleo"] == [
|
|
r"https?://([a-z0-9-]+)\.taleo\.net"]
|
|
assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE
|
|
|
|
|
|
def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None:
|
|
page = ('<a href="https://boards.greenhouse.io/embed/job_board?for=acme">a</a>'
|
|
'<a href="https://boards-api.greenhouse.io/v1/boards/acme/jobs">b</a>')
|
|
assert [(h, s) for h, s, _ in boards_in(page)] == [("greenhouse", "acme")]
|
|
assert boards_in("nothing here") == []
|
|
|
|
|
|
def test_careers_links_keeps_on_site_links_that_read_as_careers() -> None:
|
|
html = """
|
|
<a href="/about">About</a>
|
|
<a href="/careers">Careers</a>
|
|
<a href="https://jobs.acme.com/">We are hiring</a>
|
|
<a href="https://other.example/jobs">Partner jobs</a>
|
|
<a href="/en/karriere#top"><span>Karriere</span></a>
|
|
<a href="mailto:jobs@acme.com">Mail</a>
|
|
<a href="/careers">Careers again</a>
|
|
"""
|
|
assert careers_links(html, "https://www.acme.com/") == [
|
|
"https://www.acme.com/careers", "https://jobs.acme.com/",
|
|
"https://www.acme.com/en/karriere"]
|
|
|
|
|
|
def test_slug_guesses_come_from_the_key_and_the_domain() -> None:
|
|
assert slug_guesses("Trade Republic Bank GmbH", "www.traderepublic.com") == [
|
|
"trade-republic-bank", "traderepublicbank", "traderepublic"]
|
|
assert slug_guesses("Pleo", "pleo.io") == ["pleo"]
|
|
|
|
|
|
# ------------------------------------------------------------------- discover
|
|
|
|
class Site:
|
|
"""A fake web: `pages` maps a URL to `(final URL, html)`; anything else
|
|
is a page that did not answer. Remembers the order of fetches."""
|
|
|
|
def __init__(self, pages: dict[str, tuple[str, str] | str],
|
|
boards: set[tuple[str, str]] = frozenset()) -> None:
|
|
self.pages = {k: (v if isinstance(v, tuple) else (k, v)) for k, v in pages.items()}
|
|
self.boards = boards
|
|
self.fetched: list[str] = []
|
|
self.probed: list[tuple[str, str]] = []
|
|
|
|
def fetch(self, client: httpx.Client, url: str) -> tuple[str, str]:
|
|
self.fetched.append(url)
|
|
return self.pages.get(url, ("", ""))
|
|
|
|
def probe(self, client: httpx.Client, host: str, slug: str) -> bool:
|
|
self.probed.append((host, slug))
|
|
return (host, slug) in self.boards
|
|
|
|
|
|
def run(site: Site, name: str = "Acme", domain: str = "acme.com") -> Discovery:
|
|
with httpx.Client() as client:
|
|
return discover(name, domain, fetch=site.fetch, probe_slug=site.probe, client=client)
|
|
|
|
|
|
def test_a_board_named_on_the_home_page_ends_discovery_there() -> None:
|
|
site = Site({"https://acme.com/": '<a href="https://jobs.lever.co/acme">Jobs</a>'})
|
|
assert run(site) == Discovery(board="lever:acme", ats="lever", ats_slug="acme",
|
|
careers_url="https://acme.com/",
|
|
evidence="jobs.lever.co/acme")
|
|
assert site.fetched == ["https://acme.com/"]
|
|
assert site.probed == []
|
|
|
|
|
|
def test_the_home_pages_own_careers_link_is_read_before_the_usual_paths() -> None:
|
|
site = Site({
|
|
"https://acme.com/": '<a href="/company/join-us">Join us</a>',
|
|
"https://acme.com/company/join-us": ('https://acme.com/company/join-us',
|
|
'<a href="https://acme.recruitee.com/">Open</a>'),
|
|
"https://acme.com/careers": '<a href="https://jobs.lever.co/wrong">no</a>',
|
|
})
|
|
found = run(site)
|
|
assert found.board == "recruitee:acme"
|
|
assert found.careers_url == "https://acme.com/company/join-us"
|
|
assert site.fetched == ["https://acme.com/", "https://acme.com/company/join-us"]
|
|
|
|
|
|
def test_a_careers_pages_own_link_is_followed_one_level_down() -> None:
|
|
site = Site({
|
|
"https://acme.com/": "<p>no links</p>",
|
|
"https://acme.com/careers": '<a href="/careers/open-positions">See open positions</a>',
|
|
"https://acme.com/careers/open-positions":
|
|
'<iframe src="https://jobs.ashbyhq.com/acme"></iframe>',
|
|
})
|
|
assert run(site).board == "ashby:acme"
|
|
assert site.fetched[:3] == ["https://acme.com/", "https://acme.com/careers",
|
|
"https://acme.com/careers/open-positions"]
|
|
|
|
|
|
def test_an_unreadable_host_named_by_a_page_is_the_answer_and_nothing_is_probed() -> None:
|
|
site = Site({
|
|
"https://acme.com/": "<p>welcome</p>",
|
|
"https://acme.com/careers": '<a href="https://acme.taleo.net/careersection/x">Jobs</a>',
|
|
}, boards={("greenhouse", "acme")})
|
|
found = run(site)
|
|
assert found == Discovery(ats="taleo", ats_slug="",
|
|
careers_url="https://acme.com/careers",
|
|
evidence=".taleo.net")
|
|
assert not found.readable
|
|
# The page named the host: a probe could only find another tenant's
|
|
# board of the same name, like the greenhouse:acme this fake web has.
|
|
assert site.probed == []
|
|
|
|
|
|
def test_a_readable_board_on_a_later_page_still_beats_an_unreadable_host() -> None:
|
|
site = Site({
|
|
"https://acme.com/": '<script src="https://cdn.teamtailor.com/x.js"></script>'
|
|
'<a href="/careers">Careers</a>',
|
|
"https://acme.com/careers": '<a href="https://jobs.lever.co/acme">Jobs</a>',
|
|
})
|
|
assert run(site).board == "lever:acme"
|
|
assert site.probed == []
|
|
|
|
|
|
def test_a_teamtailor_link_is_a_board_even_after_the_cdn_named_it_first() -> None:
|
|
"""The CDN names teamtailor before any link does; the slug comes from the
|
|
first link that carries one, the acme of acme.teamtailor.com."""
|
|
site = Site({
|
|
"https://acme.com/": '<script src="https://cdn.teamtailor.com/x.js"></script>',
|
|
"https://acme.com/careers": '<a href="https://acme-jobs.teamtailor.com/jobs">Jobs</a>',
|
|
})
|
|
found = run(site)
|
|
assert (found.ats, found.ats_slug, found.board) == ("teamtailor", "acme-jobs",
|
|
"teamtailor:acme-jobs")
|
|
assert found.readable
|
|
assert found.evidence == "https://acme-jobs.teamtailor.com"
|
|
assert found.careers_url == "https://acme.com/careers"
|
|
|
|
|
|
def test_a_teamtailor_site_on_its_own_domain_is_read_off_its_host() -> None:
|
|
"""Only the CDN is named, so the site's host is the slug, once that host
|
|
serves the feed."""
|
|
site = Site({
|
|
"https://acme.com/": '<a href="https://careers.acme.com/">Careers</a>',
|
|
"https://careers.acme.com/": '<script src="https://cdn.teamtailor.com/x.js"></script>',
|
|
"https://careers.acme.com/jobs.rss":
|
|
'<rss xmlns:tt="https://teamtailor.com/locations"><channel/></rss>',
|
|
})
|
|
found = run(site)
|
|
assert found == Discovery(board="teamtailor:careers.acme.com", ats="teamtailor",
|
|
ats_slug="careers.acme.com",
|
|
careers_url="https://careers.acme.com/",
|
|
evidence="https://careers.acme.com/jobs.rss")
|
|
assert site.probed == []
|
|
|
|
|
|
def test_a_page_that_only_loads_the_cdn_is_not_a_board_without_the_feed() -> None:
|
|
site = Site({"https://acme.com/": '<script src="https://cdn.teamtailor.com/x.js"></script>'})
|
|
found = run(site)
|
|
assert (found.ats, found.ats_slug, found.board) == ("teamtailor", "", "")
|
|
assert not found.readable
|
|
assert "https://acme.com/jobs.rss" in site.fetched
|
|
|
|
|
|
def test_the_admin_app_and_cdn_hosts_are_no_teamtailor_slug() -> None:
|
|
page = '<a href="https://app.teamtailor.com/x">a</a><img src="https://cdn.teamtailor.com/y">'
|
|
assert [(h, s) for h, s, _ in boards_in(page)] == [("teamtailor", "")]
|
|
|
|
|
|
def test_a_teamtailor_board_is_found_by_its_slug_probe() -> None:
|
|
site = Site({"https://acme.com/": "<p>a script draws the jobs</p>"},
|
|
boards={("teamtailor", "acme")})
|
|
found = run(site)
|
|
assert found.board == "teamtailor:acme"
|
|
assert found.evidence == "https://acme.teamtailor.com/jobs.rss"
|
|
|
|
|
|
def test_slug_probes_find_a_board_the_pages_never_link() -> None:
|
|
site = Site({"https://acme.com/": "<p>a script draws the jobs</p>",
|
|
"https://acme.com/jobs": "<div id=app></div>"},
|
|
boards={("ashby", "acme")})
|
|
found = run(site)
|
|
assert found.board == "ashby:acme"
|
|
assert found.careers_url == "https://acme.com/jobs"
|
|
assert found.evidence == "https://api.ashbyhq.com/posting-api/job-board/acme"
|
|
|
|
|
|
@pytest.mark.parametrize(("boards", "found"), [
|
|
({("greenhouse", "acme"), ("ashby", "acme")}, "greenhouse:acme"),
|
|
({("lever", "acme"), ("ashby", "acme")}, "lever:acme"),
|
|
({("smartrecruiters", "acme"), ("personio", "acme")}, "smartrecruiters:acme"),
|
|
({("lever", "eu/acme"), ("personio", "acme")}, "lever:eu/acme"),
|
|
])
|
|
def test_a_slug_on_two_hosts_goes_to_the_one_probed_first(
|
|
boards: set[tuple[str, str]], found: str) -> None:
|
|
site = Site({}, boards=boards)
|
|
assert run(site).board == found
|
|
|
|
|
|
def test_the_probe_order_is_the_one_before_modules() -> None:
|
|
assert list(mod._PROBES)[:7] == ["greenhouse", "lever", "ashby", "smartrecruiters",
|
|
"recruitee", "personio", "teamtailor"]
|
|
site = Site({})
|
|
run(site)
|
|
assert site.probed[:8] == [("greenhouse", "acme"), ("lever", "acme"), ("lever", "eu/acme"),
|
|
("ashby", "acme"), ("smartrecruiters", "acme"),
|
|
("recruitee", "acme"), ("personio", "acme"),
|
|
("teamtailor", "acme")]
|
|
|
|
|
|
def test_a_lever_eu_board_is_found_by_its_probe() -> None:
|
|
site = Site({"https://acme.com/": "<p>a script draws the jobs</p>"},
|
|
boards={("lever", "eu/acme")})
|
|
found = run(site)
|
|
assert found.board == "lever:eu/acme"
|
|
assert found.evidence == "https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json"
|
|
assert site.probed.index(("lever", "acme")) + 1 == site.probed.index(("lever", "eu/acme"))
|
|
|
|
|
|
def test_probe_asks_each_hosts_own_endpoint_and_body_check() -> None:
|
|
asked: list[str] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
asked.append(str(request.url))
|
|
if request.url.host == "api.eu.lever.co":
|
|
return httpx.Response(200, json=[])
|
|
if request.url.host == "api.ashbyhq.com":
|
|
return httpx.Response(200, content=b'{"jobs":[]}')
|
|
return httpx.Response(404)
|
|
|
|
with httpx.Client(transport=httpx.MockTransport(handler)) as client:
|
|
assert mod.probe(client, "lever", "eu/acme")
|
|
assert not mod.probe(client, "lever", "acme")
|
|
assert not mod.probe(client, "ashby", "acme") # an empty listing is no board
|
|
assert asked == ["https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json",
|
|
"https://api.lever.co/v0/postings/acme?limit=1&mode=json",
|
|
"https://api.ashbyhq.com/posting-api/job-board/acme"]
|
|
assert "workable" not in mod._PROBES and "lever" in mod._PROBES
|
|
|
|
|
|
def test_a_site_that_never_answers_gives_an_empty_discovery() -> None:
|
|
site = Site({})
|
|
assert run(site) == Discovery()
|
|
assert len(site.fetched) <= mod.MAX_PAGES + 1
|
|
|
|
|
|
def test_a_domain_is_folded_to_its_site() -> None:
|
|
site = Site({"https://acme.com/": '<a href="https://jobs.lever.co/acme">Jobs</a>'})
|
|
with httpx.Client() as client:
|
|
found = discover("Acme", "https://www.acme.com/some/page", fetch=site.fetch,
|
|
probe_slug=site.probe, client=client)
|
|
assert found.board == "lever:acme"
|
|
assert discover("Acme", "", fetch=site.fetch, probe_slug=site.probe) == Discovery()
|
|
|
|
|
|
# ---------------------------------------------------------------- the seed
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path) -> Store:
|
|
return Store(tmp_path / "t.db")
|
|
|
|
|
|
def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Path) -> None:
|
|
seed = tmp_path / "companies.json"
|
|
seed.write_text(json.dumps({"companies": [
|
|
{"name": "Pleo", "domain": "pleo.io", "country": "DK", "city": "Copenhagen",
|
|
"board": "ashby:pleo"},
|
|
{"name": "Novo Nordisk", "domain": "novonordisk.com", "country": "DK",
|
|
"city": "Bagsværd", "ats": "taleo",
|
|
"careers_url": "https://www.novonordisk.com/careers.html"},
|
|
{"name": "Odd", "board": "nosuch:odd"},
|
|
]}))
|
|
result = seed_companies(store, seed)
|
|
assert result == {"added": 3, "boards": 1, "skipped": 0, "total": 3,
|
|
"problems": ["odd: 'nosuch:odd' is not a known board; kept without it"]}
|
|
pleo = store.company("pleo")
|
|
assert pleo is not None
|
|
assert (pleo.ats, pleo.ats_slug, pleo.country, pleo.city, pleo.careers_url) == (
|
|
"ashby", "pleo", "DK", "Copenhagen", "https://jobs.ashbyhq.com/pleo")
|
|
[sub] = store.subscriptions(source="ats")
|
|
assert (sub.target, sub.company) == ("ashby:pleo", "Pleo")
|
|
novo = store.company("novo-nordisk")
|
|
assert novo is not None
|
|
assert (novo.ats, novo.careers_url) == ("taleo",
|
|
"https://www.novonordisk.com/careers.html")
|
|
odd = store.company("odd")
|
|
assert odd is not None and odd.ats == ""
|
|
# Re-running adds nothing and edits nothing.
|
|
assert seed_companies(store, seed)["skipped"] == 3
|