From a30b1ff09ff7dc291e959086f3ef9cb10264fce0 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 17:07:34 +0200 Subject: [PATCH] Take discovery's readable hosts, patterns and probes from the host modules, and find Lever boards on its EU instance Co-Authored-By: Claude Opus 5.5 --- app/jobs/discover.py | 172 ++++++++++++++++++++--------------------- tests/test_discover.py | 65 +++++++++++++--- 2 files changed, 141 insertions(+), 96 deletions(-) diff --git a/app/jobs/discover.py b/app/jobs/discover.py index 2e35765..b75405d 100644 --- a/app/jobs/discover.py +++ b/app/jobs/discover.py @@ -5,10 +5,12 @@ The seed list and the LinkedIn flow both start from a company name and a website, not from a board. This module gets from the one to the other: it fetches the site's careers page, or the pages the home page links to as careers, and looks for a link into one of the hosts in `HOSTS`. A readable -host (one the `ats` plugin parses) becomes a `board` target the pull can -read; any other host is recorded as `ats`, with its slug where the URL -carries one, so the company is kept as one the app cannot read yet rather -than dropped. +host (one with a module in `plugins/atshosts/`, which the `ats` plugin +parses) becomes a `board` target the pull can read; any other host is +recorded as `ats`, with its slug where the URL carries one, so the company is +kept as one the app cannot read yet rather than dropped. The patterns and +probes of a readable host are its module's; the hosts only named are in +`_UNREADABLE` here, and one of them that gains a module is read from then on. A Teamtailor site on the company's own domain names no slug: the page only loads Teamtailor's CDN. The board there is the site's own host, and it is @@ -16,7 +18,8 @@ taken only once that host answers `/jobs.rss` with a Teamtailor feed. Only when no page names a host at all are the hosts with a cheap per-slug endpoint (`_PROBES`) probed with the company's own slug, because a site that -draws its jobs from an iframe or a script leaves no link to find. A page that +draws its jobs from an iframe or a script leaves no link to find. Lever is +probed on its EU instance as well, as `lever:eu/`. A page that names an unreadable host is the answer: a probe on a common name can hit another tenant's board. @@ -37,13 +40,15 @@ from __future__ import annotations import logging import re -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass, replace +from types import ModuleType from urllib.parse import urljoin import httpx from desk.sources import USER_AGENT +from plugins import atshosts log = logging.getLogger(__name__) @@ -53,55 +58,53 @@ MAX_BYTES = 2 << 20 MAX_PAGES = 6 #: The hosts the `ats` plugin parses; a match on one of these is a `board`. -READABLE = ("greenhouse", "lever", "ashby", "workable", "smartrecruiters", - "recruitee", "personio", "teamtailor") +READABLE: tuple[str, ...] = tuple(atshosts.HOSTS) -#: host -> pattern over a URL. Group 1, where there is one, is the slug the -#: host names the company by; a host with no group is recognised but its -#: board has no slug the app could use. Order matters only for readability: -#: the readable hosts come first so the first match on a page is the one the -#: app can read. -HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = tuple( - (host, re.compile(pattern, re.IGNORECASE)) for host, pattern in ( - ("greenhouse", r"(?:boards|job-boards|api)\.(?:eu\.)?greenhouse\.io/(?:v1/boards/|embed/job_board\?[^\"'\s]*for=)?([A-Za-z0-9_-]+)"), - ("greenhouse", r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)"), - ("lever", r"jobs\.(?:eu\.)?lever\.co/([A-Za-z0-9_-]+)"), - ("lever", r"api\.(?:eu\.)?lever\.co/v0/postings/([A-Za-z0-9_-]+)"), - ("ashby", r"jobs\.ashbyhq\.com/([A-Za-z0-9_.-]+)"), - ("ashby", r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)"), - ("workable", r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_-]+)"), - ("workable", r"https?://(?!apply\.|www\.)([a-z0-9-]+)\.workable\.com"), - ("smartrecruiters", r"(?:jobs|careers|api)\.smartrecruiters\.com/(?:v1/companies/)?([A-Za-z0-9_-]+)"), - ("recruitee", r"https?://([a-z0-9-]+)\.recruitee\.com"), - ("personio", r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)"), - ("teamtailor", r"https?://([a-z0-9-]+)\.teamtailor\.com"), - # No slug: a career site on the company's own domain, which only loads - # Teamtailor's CDN. `_discover` reads its board off the page's host. - ("teamtailor", r"cdn\.teamtailor\.com|teamtailor-cdn|teamtailor\.com/assets"), - # Not readable. The slug, where the URL has one, is kept for the day - # a parser exists. - ("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"), - ("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"), - ("icims", r"https?://([a-z0-9-]+)\.icims\.com"), - ("taleo", r"\.taleo\.net"), - ("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"), - ("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"), - ("join", r"join\.com/companies/([a-z0-9-]+)"), - ("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"), - ("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"), - ("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"), - ("gem", r"jobs\.gem\.com/([a-z0-9-]+)"), - ("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"), - ("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"), - ("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"), - ("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"), - ("eightfold", r"\.eightfold\.ai"), - ("phenom", r"phenompeople\.com|phenom\.com/careers"), - ("avature", r"\.avature\.net"), - ("oracle", r"oraclecloud\.com/hcmUI"), - ("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"), - ("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"), - )) +#: host -> pattern over a URL, for the hosts discovery names but the app +#: cannot read. Group 1, where there is one, is the slug the host names the +#: company by, kept for the day a parser exists; a host with no group is +#: recognised but its board has no slug the app could use. +_UNREADABLE: tuple[tuple[str, str], ...] = ( + ("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"), + ("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"), + ("icims", r"https?://([a-z0-9-]+)\.icims\.com"), + ("taleo", r"\.taleo\.net"), + ("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"), + ("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"), + ("join", r"join\.com/companies/([a-z0-9-]+)"), + ("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"), + ("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"), + ("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"), + ("gem", r"jobs\.gem\.com/([a-z0-9-]+)"), + ("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"), + ("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"), + ("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"), + ("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"), + ("eightfold", r"\.eightfold\.ai"), + ("phenom", r"phenompeople\.com|phenom\.com/careers"), + ("avature", r"\.avature\.net"), + ("oracle", r"oraclecloud\.com/hcmUI"), + ("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"), + ("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"), +) + + +def _hosts(modules: Mapping[str, ModuleType], + unreadable: Iterable[tuple[str, str]]) -> tuple[tuple[str, re.Pattern[str]], ...]: + """Every host's patterns, compiled: the readable hosts' from their + modules first, so the first match at one place on a page is the one the + app can read, then the unreadable ones. An unreadable host that has + gained a module is left out here, since its module's patterns replace + it.""" + out = [(name, pattern) for name, module in modules.items() for pattern in module.PATTERNS] + out += [(host, pattern) for host, pattern in unreadable if host not in modules] + return tuple((host, re.compile(pattern, re.IGNORECASE)) for host, pattern in out) + + +#: host -> pattern over a URL, every host discovery can name. The slug is +#: group 1, or, for a pattern with more groups, the groups that matched +#: joined by "/" (Lever's `eu/`). +HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = _hosts(atshosts.HOSTS, _UNREADABLE) #: What a Teamtailor feed carries and a page that only looks like one does not: #: the namespace of `tt:locations`, declared on `` even when no role is open. @@ -161,9 +164,10 @@ def boards_in(text: str) -> list[Found]: hits: list[tuple[int, str, str, str]] = [] for host, pattern in HOSTS: for m in pattern.finditer(text): - slug = (m.group(1) if m.groups() and m.group(1) else "").lower() - if slug in _NOT_SLUGS: + parts = [part.lower() for part in m.groups() if part] + if any(part in _NOT_SLUGS for part in parts): continue + slug = "/".join(parts) hits.append((m.start(), host, slug, m.group(0))) for _pos, host, slug, url in sorted(hits): if (host, slug) in seen: @@ -237,25 +241,28 @@ def fetch_page(client: httpx.Client, url: str) -> tuple[str, str]: return "", "" -#: host -> a cheap URL that answers 2xx only when the slug is a board there. -#: Workable is left out: its API sits behind a rate limit that a run of -#: guesses trips, and a Workable site links its board plainly anyway. -_PROBES = { - "greenhouse": "https://boards-api.greenhouse.io/v1/boards/{slug}", - "lever": "https://api.lever.co/v0/postings/{slug}?limit=1&mode=json", - "ashby": "https://api.ashbyhq.com/posting-api/job-board/{slug}", - "smartrecruiters": "https://api.smartrecruiters.com/v1/companies/{slug}/postings?limit=1", - "recruitee": "https://{slug}.recruitee.com/api/offers/", - "personio": "https://{slug}.jobs.personio.de/xml", - "teamtailor": "https://{slug}.teamtailor.com/jobs.rss", -} +#: host -> a cheap URL that answers 2xx only when the slug is a board there, +#: from each module's `probe_url`. Workable has none: its API sits behind a +#: rate limit that a run of guesses trips, and a Workable site links its +#: board plainly anyway. +_PROBES: dict[str, Callable[[str], str]] = { + name: module.probe_url for name, module in atshosts.HOSTS.items() + if hasattr(module, "probe_url")} + + +def _probe_slugs(host: str, guess: str) -> tuple[str, ...]: + """The slugs a guessed name is probed as on `host`: the name itself, and + for Lever the name on its EU instance too (the module's `probe_slugs`).""" + module = atshosts.HOSTS[host] + return module.probe_slugs(guess) if hasattr(module, "probe_slugs") else (guess,) def probe(client: httpx.Client, host: str, slug: str) -> bool: """Whether `host` has a board called `slug`, by its cheapest endpoint. - Ashby and SmartRecruiters answer an empty listing for an unknown slug, - so there a listing without jobs counts as no board.""" - url = _PROBES[host].format(slug=slug) + A host that answers an unknown slug with an empty listing (Ashby, + SmartRecruiters, ...) says in its module's `probe_ok` what a listing + with a board looks like.""" + url = _PROBES[host](slug) try: response = client.get(url, timeout=TIMEOUT, follow_redirects=False, headers={"User-Agent": USER_AGENT}) @@ -263,18 +270,8 @@ def probe(client: httpx.Client, host: str, slug: str) -> bool: return False if not response.is_success: return False - body = response.content - if host == "ashby": - return b'"jobs":[' in body and b'"jobs":[]' not in body - if host == "smartrecruiters": - return b'"totalFound":' in body and b'"totalFound":0' not in body - if host == "recruitee": - return b'"offers":[' in body and b'"offers":[]' not in body - if host == "personio": - return b"" in body - if host == "teamtailor": - return b"" in body - return True + module = atshosts.HOSTS[host] + return module.probe_ok(response.content) if hasattr(module, "probe_ok") else True Fetch = Callable[[httpx.Client, str], tuple[str, str]] @@ -377,9 +374,10 @@ def _discover(name: str, site: str, client: httpx.Client, fetch: Fetch, # probe could only add another tenant's board of the same name. if unreadable is not None: return unreadable - for slug in slug_guesses(name, site): + for guess in slug_guesses(name, site): for host in _PROBES: - if probe_slug(client, host, slug): - return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug, - careers_url=careers_page, evidence=_PROBES[host].format(slug=slug)) + for slug in _probe_slugs(host, guess): + if probe_slug(client, host, slug): + return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug, + careers_url=careers_page, evidence=_PROBES[host](slug)) return Discovery(careers_url=careers_page) diff --git a/tests/test_discover.py b/tests/test_discover.py index 5c1b581..e92e64b 100644 --- a/tests/test_discover.py +++ b/tests/test_discover.py @@ -4,6 +4,7 @@ site's pages are read.""" from __future__ import annotations import json +import types from pathlib import Path import httpx @@ -36,16 +37,32 @@ def test_boards_in_names_readable_hosts_with_their_slug() -> None: def test_boards_in_keeps_hosts_the_app_cannot_read() -> None: page = """ - Workday - SF + Taleo TT """ assert boards_in(page) == [ - ("workday", "novo", "https://novo.wd3.myworkdayjobs.com"), - ("successfactors", "", "performancemanager.successfactors.eu"), + ("taleo", "", ".taleo.net"), ("teamtailor", "acme", "https://acme.teamtailor.com")] +def test_boards_in_names_a_lever_eu_board_by_its_region() -> None: + page = ('EU' + '' + 'US') + assert [(h, s) for h, s, _ in boards_in(page)] == [("lever", "eu/tomtom"), + ("lever", "whereby")] + + +def test_a_host_that_gains_a_module_replaces_its_unreadable_pattern() -> None: + """Wave by wave, a host named here becomes readable by its module alone.""" + taleo = types.ModuleType("taleo") + taleo.PATTERNS = (r"https?://([a-z0-9-]+)\.taleo\.net",) + hosts = mod._hosts({"taleo": taleo}, mod._UNREADABLE) + assert [pattern.pattern for host, pattern in hosts if host == "taleo"] == [ + r"https?://([a-z0-9-]+)\.taleo\.net"] + assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE + + def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None: page = ('a' 'b') @@ -138,12 +155,12 @@ def test_a_careers_pages_own_link_is_followed_one_level_down() -> None: def test_an_unreadable_host_named_by_a_page_is_the_answer_and_nothing_is_probed() -> None: site = Site({ "https://acme.com/": "

welcome

", - "https://acme.com/careers": 'Jobs', + "https://acme.com/careers": 'Jobs', }, boards={("greenhouse", "acme")}) found = run(site) - assert found == Discovery(ats="workday", ats_slug="acme", + assert found == Discovery(ats="taleo", ats_slug="", careers_url="https://acme.com/careers", - evidence="https://acme.wd3.myworkdayjobs.com") + evidence=".taleo.net") assert not found.readable # The page named the host: a probe could only find another tenant's # board of the same name, like the greenhouse:acme this fake web has. @@ -223,6 +240,36 @@ def test_slug_probes_find_a_board_the_pages_never_link() -> None: assert found.evidence == "https://api.ashbyhq.com/posting-api/job-board/acme" +def test_a_lever_eu_board_is_found_by_its_probe() -> None: + site = Site({"https://acme.com/": "

a script draws the jobs

"}, + boards={("lever", "eu/acme")}) + found = run(site) + assert found.board == "lever:eu/acme" + assert found.evidence == "https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json" + assert site.probed.index(("lever", "acme")) + 1 == site.probed.index(("lever", "eu/acme")) + + +def test_probe_asks_each_hosts_own_endpoint_and_body_check() -> None: + asked: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + asked.append(str(request.url)) + if request.url.host == "api.eu.lever.co": + return httpx.Response(200, json=[]) + if request.url.host == "api.ashbyhq.com": + return httpx.Response(200, content=b'{"jobs":[]}') + return httpx.Response(404) + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + assert mod.probe(client, "lever", "eu/acme") + assert not mod.probe(client, "lever", "acme") + assert not mod.probe(client, "ashby", "acme") # an empty listing is no board + assert asked == ["https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json", + "https://api.lever.co/v0/postings/acme?limit=1&mode=json", + "https://api.ashbyhq.com/posting-api/job-board/acme"] + assert "workable" not in mod._PROBES and "lever" in mod._PROBES + + def test_a_site_that_never_answers_gives_an_empty_discovery() -> None: site = Site({}) assert run(site) == Discovery() @@ -251,7 +298,7 @@ def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Pa {"name": "Pleo", "domain": "pleo.io", "country": "DK", "city": "Copenhagen", "board": "ashby:pleo"}, {"name": "Novo Nordisk", "domain": "novonordisk.com", "country": "DK", - "city": "Bagsværd", "ats": "successfactors", + "city": "Bagsværd", "ats": "taleo", "careers_url": "https://www.novonordisk.com/careers.html"}, {"name": "Odd", "board": "nosuch:odd"}, ]})) @@ -266,7 +313,7 @@ def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Pa assert (sub.target, sub.company) == ("ashby:pleo", "Pleo") novo = store.company("novo-nordisk") assert novo is not None - assert (novo.ats, novo.careers_url) == ("successfactors", + assert (novo.ats, novo.careers_url) == ("taleo", "https://www.novonordisk.com/careers.html") odd = store.company("odd") assert odd is not None and odd.ats == ""