Take discovery's readable hosts, patterns and probes from the host modules, and find Lever boards on its EU instance

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-29 17:07:34 +02:00
1 parent 8b14d8575b
commit a30b1ff09f
2 files changed
+118 -73

No files matched your search

+62 -64
View File
@@ -5,10 +5,12 @@ The seed list and the LinkedIn flow both start from a company name and a
website, not from a board. This module gets from the one to the other: it website, not from a board. This module gets from the one to the other: it
fetches the site's careers page, or the pages the home page links to as fetches the site's careers page, or the pages the home page links to as
careers, and looks for a link into one of the hosts in `HOSTS`. A readable careers, and looks for a link into one of the hosts in `HOSTS`. A readable
host (one the `ats` plugin parses) becomes a `board` target the pull can host (one with a module in `plugins/atshosts/`, which the `ats` plugin
read; any other host is recorded as `ats`, with its slug where the URL parses) becomes a `board` target the pull can read; any other host is
carries one, so the company is kept as one the app cannot read yet rather recorded as `ats`, with its slug where the URL carries one, so the company is
than dropped. kept as one the app cannot read yet rather than dropped. The patterns and
probes of a readable host are its module's; the hosts only named are in
`_UNREADABLE` here, and one of them that gains a module is read from then on.
A Teamtailor site on the company's own domain names no slug: the page only A Teamtailor site on the company's own domain names no slug: the page only
loads Teamtailor's CDN. The board there is the site's own host, and it is loads Teamtailor's CDN. The board there is the site's own host, and it is
@@ -16,7 +18,8 @@ taken only once that host answers `/jobs.rss` with a Teamtailor feed.
Only when no page names a host at all are the hosts with a cheap per-slug Only when no page names a host at all are the hosts with a cheap per-slug
endpoint (`_PROBES`) probed with the company's own slug, because a site that endpoint (`_PROBES`) probed with the company's own slug, because a site that
draws its jobs from an iframe or a script leaves no link to find. A page that draws its jobs from an iframe or a script leaves no link to find. Lever is
probed on its EU instance as well, as `lever:eu/<slug>`. A page that
names an unreadable host is the answer: a probe on a common name can hit names an unreadable host is the answer: a probe on a common name can hit
another tenant's board. another tenant's board.
@@ -37,13 +40,15 @@ from __future__ import annotations
import logging import logging
import re import re
from collections.abc import Callable, Iterable from collections.abc import Callable, Iterable, Mapping
from dataclasses import dataclass, replace from dataclasses import dataclass, replace
from types import ModuleType
from urllib.parse import urljoin from urllib.parse import urljoin
import httpx import httpx
from desk.sources import USER_AGENT from desk.sources import USER_AGENT
from plugins import atshosts
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -53,33 +58,13 @@ MAX_BYTES = 2 << 20
MAX_PAGES = 6 MAX_PAGES = 6
#: The hosts the `ats` plugin parses; a match on one of these is a `board`. #: The hosts the `ats` plugin parses; a match on one of these is a `board`.
READABLE = ("greenhouse", "lever", "ashby", "workable", "smartrecruiters", READABLE: tuple[str, ...] = tuple(atshosts.HOSTS)
"recruitee", "personio", "teamtailor")
#: host -> pattern over a URL. Group 1, where there is one, is the slug the #: host -> pattern over a URL, for the hosts discovery names but the app
#: host names the company by; a host with no group is recognised but its #: cannot read. Group 1, where there is one, is the slug the host names the
#: board has no slug the app could use. Order matters only for readability: #: company by, kept for the day a parser exists; a host with no group is
#: the readable hosts come first so the first match on a page is the one the #: recognised but its board has no slug the app could use.
#: app can read. _UNREADABLE: tuple[tuple[str, str], ...] = (
HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = tuple(
(host, re.compile(pattern, re.IGNORECASE)) for host, pattern in (
("greenhouse", r"(?:boards|job-boards|api)\.(?:eu\.)?greenhouse\.io/(?:v1/boards/|embed/job_board\?[^\"'\s]*for=)?([A-Za-z0-9_-]+)"),
("greenhouse", r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)"),
("lever", r"jobs\.(?:eu\.)?lever\.co/([A-Za-z0-9_-]+)"),
("lever", r"api\.(?:eu\.)?lever\.co/v0/postings/([A-Za-z0-9_-]+)"),
("ashby", r"jobs\.ashbyhq\.com/([A-Za-z0-9_.-]+)"),
("ashby", r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)"),
("workable", r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_-]+)"),
("workable", r"https?://(?!apply\.|www\.)([a-z0-9-]+)\.workable\.com"),
("smartrecruiters", r"(?:jobs|careers|api)\.smartrecruiters\.com/(?:v1/companies/)?([A-Za-z0-9_-]+)"),
("recruitee", r"https?://([a-z0-9-]+)\.recruitee\.com"),
("personio", r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)"),
("teamtailor", r"https?://([a-z0-9-]+)\.teamtailor\.com"),
# No slug: a career site on the company's own domain, which only loads
# Teamtailor's CDN. `_discover` reads its board off the page's host.
("teamtailor", r"cdn\.teamtailor\.com|teamtailor-cdn|teamtailor\.com/assets"),
# Not readable. The slug, where the URL has one, is kept for the day
# a parser exists.
("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"), ("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"),
("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"), ("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"),
("icims", r"https?://([a-z0-9-]+)\.icims\.com"), ("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
@@ -101,7 +86,25 @@ HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = tuple(
("oracle", r"oraclecloud\.com/hcmUI"), ("oracle", r"oraclecloud\.com/hcmUI"),
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"), ("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"), ("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"),
)) )
def _hosts(modules: Mapping[str, ModuleType],
unreadable: Iterable[tuple[str, str]]) -> tuple[tuple[str, re.Pattern[str]], ...]:
"""Every host's patterns, compiled: the readable hosts' from their
modules first, so the first match at one place on a page is the one the
app can read, then the unreadable ones. An unreadable host that has
gained a module is left out here, since its module's patterns replace
it."""
out = [(name, pattern) for name, module in modules.items() for pattern in module.PATTERNS]
out += [(host, pattern) for host, pattern in unreadable if host not in modules]
return tuple((host, re.compile(pattern, re.IGNORECASE)) for host, pattern in out)
#: host -> pattern over a URL, every host discovery can name. The slug is
#: group 1, or, for a pattern with more groups, the groups that matched
#: joined by "/" (Lever's `eu/<site>`).
HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = _hosts(atshosts.HOSTS, _UNREADABLE)
#: What a Teamtailor feed carries and a page that only looks like one does not: #: What a Teamtailor feed carries and a page that only looks like one does not:
#: the namespace of `tt:locations`, declared on `<rss>` even when no role is open. #: the namespace of `tt:locations`, declared on `<rss>` even when no role is open.
@@ -161,9 +164,10 @@ def boards_in(text: str) -> list[Found]:
hits: list[tuple[int, str, str, str]] = [] hits: list[tuple[int, str, str, str]] = []
for host, pattern in HOSTS: for host, pattern in HOSTS:
for m in pattern.finditer(text): for m in pattern.finditer(text):
slug = (m.group(1) if m.groups() and m.group(1) else "").lower() parts = [part.lower() for part in m.groups() if part]
if slug in _NOT_SLUGS: if any(part in _NOT_SLUGS for part in parts):
continue continue
slug = "/".join(parts)
hits.append((m.start(), host, slug, m.group(0))) hits.append((m.start(), host, slug, m.group(0)))
for _pos, host, slug, url in sorted(hits): for _pos, host, slug, url in sorted(hits):
if (host, slug) in seen: if (host, slug) in seen:
@@ -237,25 +241,28 @@ def fetch_page(client: httpx.Client, url: str) -> tuple[str, str]:
return "", "" return "", ""
#: host -> a cheap URL that answers 2xx only when the slug is a board there. #: host -> a cheap URL that answers 2xx only when the slug is a board there,
#: Workable is left out: its API sits behind a rate limit that a run of #: from each module's `probe_url`. Workable has none: its API sits behind a
#: guesses trips, and a Workable site links its board plainly anyway. #: rate limit that a run of guesses trips, and a Workable site links its
_PROBES = { #: board plainly anyway.
"greenhouse": "https://boards-api.greenhouse.io/v1/boards/{slug}", _PROBES: dict[str, Callable[[str], str]] = {
"lever": "https://api.lever.co/v0/postings/{slug}?limit=1&mode=json", name: module.probe_url for name, module in atshosts.HOSTS.items()
"ashby": "https://api.ashbyhq.com/posting-api/job-board/{slug}", if hasattr(module, "probe_url")}
"smartrecruiters": "https://api.smartrecruiters.com/v1/companies/{slug}/postings?limit=1",
"recruitee": "https://{slug}.recruitee.com/api/offers/",
"personio": "https://{slug}.jobs.personio.de/xml", def _probe_slugs(host: str, guess: str) -> tuple[str, ...]:
"teamtailor": "https://{slug}.teamtailor.com/jobs.rss", """The slugs a guessed name is probed as on `host`: the name itself, and
} for Lever the name on its EU instance too (the module's `probe_slugs`)."""
module = atshosts.HOSTS[host]
return module.probe_slugs(guess) if hasattr(module, "probe_slugs") else (guess,)
def probe(client: httpx.Client, host: str, slug: str) -> bool: def probe(client: httpx.Client, host: str, slug: str) -> bool:
"""Whether `host` has a board called `slug`, by its cheapest endpoint. """Whether `host` has a board called `slug`, by its cheapest endpoint.
Ashby and SmartRecruiters answer an empty listing for an unknown slug, A host that answers an unknown slug with an empty listing (Ashby,
so there a listing without jobs counts as no board.""" SmartRecruiters, ...) says in its module's `probe_ok` what a listing
url = _PROBES[host].format(slug=slug) with a board looks like."""
url = _PROBES[host](slug)
try: try:
response = client.get(url, timeout=TIMEOUT, follow_redirects=False, response = client.get(url, timeout=TIMEOUT, follow_redirects=False,
headers={"User-Agent": USER_AGENT}) headers={"User-Agent": USER_AGENT})
@@ -263,18 +270,8 @@ def probe(client: httpx.Client, host: str, slug: str) -> bool:
return False return False
if not response.is_success: if not response.is_success:
return False return False
body = response.content module = atshosts.HOSTS[host]
if host == "ashby": return module.probe_ok(response.content) if hasattr(module, "probe_ok") else True
return b'"jobs":[' in body and b'"jobs":[]' not in body
if host == "smartrecruiters":
return b'"totalFound":' in body and b'"totalFound":0' not in body
if host == "recruitee":
return b'"offers":[' in body and b'"offers":[]' not in body
if host == "personio":
return b"<position>" in body
if host == "teamtailor":
return b"<item>" in body
return True
Fetch = Callable[[httpx.Client, str], tuple[str, str]] Fetch = Callable[[httpx.Client, str], tuple[str, str]]
@@ -377,9 +374,10 @@ def _discover(name: str, site: str, client: httpx.Client, fetch: Fetch,
# probe could only add another tenant's board of the same name. # probe could only add another tenant's board of the same name.
if unreadable is not None: if unreadable is not None:
return unreadable return unreadable
for slug in slug_guesses(name, site): for guess in slug_guesses(name, site):
for host in _PROBES: for host in _PROBES:
for slug in _probe_slugs(host, guess):
if probe_slug(client, host, slug): if probe_slug(client, host, slug):
return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug, return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug,
careers_url=careers_page, evidence=_PROBES[host].format(slug=slug)) careers_url=careers_page, evidence=_PROBES[host](slug))
return Discovery(careers_url=careers_page) return Discovery(careers_url=careers_page)
+56 -9
View File
@@ -4,6 +4,7 @@ site's pages are read."""
from __future__ import annotations from __future__ import annotations
import json import json
import types
from pathlib import Path from pathlib import Path
import httpx import httpx
@@ -36,16 +37,32 @@ def test_boards_in_names_readable_hosts_with_their_slug() -> None:
def test_boards_in_keeps_hosts_the_app_cannot_read() -> None: def test_boards_in_keeps_hosts_the_app_cannot_read() -> None:
page = """ page = """
<a href="https://novo.wd3.myworkdayjobs.com/en-US/careers">Workday</a> <a href="https://novo.taleo.net/careersection/2/jobsearch.ftl">Taleo</a>
<a href="https://performancemanager.successfactors.eu/career?company=x">SF</a>
<a href="https://acme.teamtailor.com/jobs">TT</a> <a href="https://acme.teamtailor.com/jobs">TT</a>
""" """
assert boards_in(page) == [ assert boards_in(page) == [
("workday", "novo", "https://novo.wd3.myworkdayjobs.com"), ("taleo", "", ".taleo.net"),
("successfactors", "", "performancemanager.successfactors.eu"),
("teamtailor", "acme", "https://acme.teamtailor.com")] ("teamtailor", "acme", "https://acme.teamtailor.com")]
def test_boards_in_names_a_lever_eu_board_by_its_region() -> None:
page = ('<a href="https://jobs.eu.lever.co/tomtom/abc-123">EU</a>'
'<script src="https://api.eu.lever.co/v0/postings/tomtom?mode=json"></script>'
'<a href="https://jobs.lever.co/whereby">US</a>')
assert [(h, s) for h, s, _ in boards_in(page)] == [("lever", "eu/tomtom"),
("lever", "whereby")]
def test_a_host_that_gains_a_module_replaces_its_unreadable_pattern() -> None:
"""Wave by wave, a host named here becomes readable by its module alone."""
taleo = types.ModuleType("taleo")
taleo.PATTERNS = (r"https?://([a-z0-9-]+)\.taleo\.net",)
hosts = mod._hosts({"taleo": taleo}, mod._UNREADABLE)
assert [pattern.pattern for host, pattern in hosts if host == "taleo"] == [
r"https?://([a-z0-9-]+)\.taleo\.net"]
assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE
def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None: def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None:
page = ('<a href="https://boards.greenhouse.io/embed/job_board?for=acme">a</a>' page = ('<a href="https://boards.greenhouse.io/embed/job_board?for=acme">a</a>'
'<a href="https://boards-api.greenhouse.io/v1/boards/acme/jobs">b</a>') '<a href="https://boards-api.greenhouse.io/v1/boards/acme/jobs">b</a>')
@@ -138,12 +155,12 @@ def test_a_careers_pages_own_link_is_followed_one_level_down() -> None:
def test_an_unreadable_host_named_by_a_page_is_the_answer_and_nothing_is_probed() -> None: def test_an_unreadable_host_named_by_a_page_is_the_answer_and_nothing_is_probed() -> None:
site = Site({ site = Site({
"https://acme.com/": "<p>welcome</p>", "https://acme.com/": "<p>welcome</p>",
"https://acme.com/careers": '<a href="https://acme.wd3.myworkdayjobs.com/x">Jobs</a>', "https://acme.com/careers": '<a href="https://acme.taleo.net/careersection/x">Jobs</a>',
}, boards={("greenhouse", "acme")}) }, boards={("greenhouse", "acme")})
found = run(site) found = run(site)
assert found == Discovery(ats="workday", ats_slug="acme", assert found == Discovery(ats="taleo", ats_slug="",
careers_url="https://acme.com/careers", careers_url="https://acme.com/careers",
evidence="https://acme.wd3.myworkdayjobs.com") evidence=".taleo.net")
assert not found.readable assert not found.readable
# The page named the host: a probe could only find another tenant's # The page named the host: a probe could only find another tenant's
# board of the same name, like the greenhouse:acme this fake web has. # board of the same name, like the greenhouse:acme this fake web has.
@@ -223,6 +240,36 @@ def test_slug_probes_find_a_board_the_pages_never_link() -> None:
assert found.evidence == "https://api.ashbyhq.com/posting-api/job-board/acme" assert found.evidence == "https://api.ashbyhq.com/posting-api/job-board/acme"
def test_a_lever_eu_board_is_found_by_its_probe() -> None:
site = Site({"https://acme.com/": "<p>a script draws the jobs</p>"},
boards={("lever", "eu/acme")})
found = run(site)
assert found.board == "lever:eu/acme"
assert found.evidence == "https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json"
assert site.probed.index(("lever", "acme")) + 1 == site.probed.index(("lever", "eu/acme"))
def test_probe_asks_each_hosts_own_endpoint_and_body_check() -> None:
asked: list[str] = []
def handler(request: httpx.Request) -> httpx.Response:
asked.append(str(request.url))
if request.url.host == "api.eu.lever.co":
return httpx.Response(200, json=[])
if request.url.host == "api.ashbyhq.com":
return httpx.Response(200, content=b'{"jobs":[]}')
return httpx.Response(404)
with httpx.Client(transport=httpx.MockTransport(handler)) as client:
assert mod.probe(client, "lever", "eu/acme")
assert not mod.probe(client, "lever", "acme")
assert not mod.probe(client, "ashby", "acme") # an empty listing is no board
assert asked == ["https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json",
"https://api.lever.co/v0/postings/acme?limit=1&mode=json",
"https://api.ashbyhq.com/posting-api/job-board/acme"]
assert "workable" not in mod._PROBES and "lever" in mod._PROBES
def test_a_site_that_never_answers_gives_an_empty_discovery() -> None: def test_a_site_that_never_answers_gives_an_empty_discovery() -> None:
site = Site({}) site = Site({})
assert run(site) == Discovery() assert run(site) == Discovery()
@@ -251,7 +298,7 @@ def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Pa
{"name": "Pleo", "domain": "pleo.io", "country": "DK", "city": "Copenhagen", {"name": "Pleo", "domain": "pleo.io", "country": "DK", "city": "Copenhagen",
"board": "ashby:pleo"}, "board": "ashby:pleo"},
{"name": "Novo Nordisk", "domain": "novonordisk.com", "country": "DK", {"name": "Novo Nordisk", "domain": "novonordisk.com", "country": "DK",
"city": "Bagsværd", "ats": "successfactors", "city": "Bagsværd", "ats": "taleo",
"careers_url": "https://www.novonordisk.com/careers.html"}, "careers_url": "https://www.novonordisk.com/careers.html"},
{"name": "Odd", "board": "nosuch:odd"}, {"name": "Odd", "board": "nosuch:odd"},
]})) ]}))
@@ -266,7 +313,7 @@ def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Pa
assert (sub.target, sub.company) == ("ashby:pleo", "Pleo") assert (sub.target, sub.company) == ("ashby:pleo", "Pleo")
novo = store.company("novo-nordisk") novo = store.company("novo-nordisk")
assert novo is not None assert novo is not None
assert (novo.ats, novo.careers_url) == ("successfactors", assert (novo.ats, novo.careers_url) == ("taleo",
"https://www.novonordisk.com/careers.html") "https://www.novonordisk.com/careers.html")
odd = store.company("odd") odd = store.company("odd")
assert odd is not None and odd.ats == "" assert odd is not None and odd.ats == ""