Take discovery's readable hosts, patterns and probes from the host modules, and find Lever boards on its EU instance

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-29 17:07:34 +02:00
1 parent 8b14d8575b
commit a30b1ff09f
2 files changed
+141 -96

No files matched your search

+85 -87
View File
@@ -5,10 +5,12 @@ The seed list and the LinkedIn flow both start from a company name and a
website, not from a board. This module gets from the one to the other: it
fetches the site's careers page, or the pages the home page links to as
careers, and looks for a link into one of the hosts in `HOSTS`. A readable
host (one the `ats` plugin parses) becomes a `board` target the pull can
read; any other host is recorded as `ats`, with its slug where the URL
carries one, so the company is kept as one the app cannot read yet rather
than dropped.
host (one with a module in `plugins/atshosts/`, which the `ats` plugin
parses) becomes a `board` target the pull can read; any other host is
recorded as `ats`, with its slug where the URL carries one, so the company is
kept as one the app cannot read yet rather than dropped. The patterns and
probes of a readable host are its module's; the hosts only named are in
`_UNREADABLE` here, and one of them that gains a module is read from then on.
A Teamtailor site on the company's own domain names no slug: the page only
loads Teamtailor's CDN. The board there is the site's own host, and it is
@@ -16,7 +18,8 @@ taken only once that host answers `/jobs.rss` with a Teamtailor feed.
Only when no page names a host at all are the hosts with a cheap per-slug
endpoint (`_PROBES`) probed with the company's own slug, because a site that
draws its jobs from an iframe or a script leaves no link to find. A page that
draws its jobs from an iframe or a script leaves no link to find. Lever is
probed on its EU instance as well, as `lever:eu/<slug>`. A page that
names an unreadable host is the answer: a probe on a common name can hit
another tenant's board.
@@ -37,13 +40,15 @@ from __future__ import annotations
import logging
import re
from collections.abc import Callable, Iterable
from collections.abc import Callable, Iterable, Mapping
from dataclasses import dataclass, replace
from types import ModuleType
from urllib.parse import urljoin
import httpx
from desk.sources import USER_AGENT
from plugins import atshosts
log = logging.getLogger(__name__)
@@ -53,55 +58,53 @@ MAX_BYTES = 2 << 20
MAX_PAGES = 6
#: The hosts the `ats` plugin parses; a match on one of these is a `board`.
READABLE = ("greenhouse", "lever", "ashby", "workable", "smartrecruiters",
"recruitee", "personio", "teamtailor")
READABLE: tuple[str, ...] = tuple(atshosts.HOSTS)
#: host -> pattern over a URL. Group 1, where there is one, is the slug the
#: host names the company by; a host with no group is recognised but its
#: board has no slug the app could use. Order matters only for readability:
#: the readable hosts come first so the first match on a page is the one the
#: app can read.
HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = tuple(
(host, re.compile(pattern, re.IGNORECASE)) for host, pattern in (
("greenhouse", r"(?:boards|job-boards|api)\.(?:eu\.)?greenhouse\.io/(?:v1/boards/|embed/job_board\?[^\"'\s]*for=)?([A-Za-z0-9_-]+)"),
("greenhouse", r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)"),
("lever", r"jobs\.(?:eu\.)?lever\.co/([A-Za-z0-9_-]+)"),
("lever", r"api\.(?:eu\.)?lever\.co/v0/postings/([A-Za-z0-9_-]+)"),
("ashby", r"jobs\.ashbyhq\.com/([A-Za-z0-9_.-]+)"),
("ashby", r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)"),
("workable", r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_-]+)"),
("workable", r"https?://(?!apply\.|www\.)([a-z0-9-]+)\.workable\.com"),
("smartrecruiters", r"(?:jobs|careers|api)\.smartrecruiters\.com/(?:v1/companies/)?([A-Za-z0-9_-]+)"),
("recruitee", r"https?://([a-z0-9-]+)\.recruitee\.com"),
("personio", r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)"),
("teamtailor", r"https?://([a-z0-9-]+)\.teamtailor\.com"),
# No slug: a career site on the company's own domain, which only loads
# Teamtailor's CDN. `_discover` reads its board off the page's host.
("teamtailor", r"cdn\.teamtailor\.com|teamtailor-cdn|teamtailor\.com/assets"),
# Not readable. The slug, where the URL has one, is kept for the day
# a parser exists.
("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"),
("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"),
("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
("taleo", r"\.taleo\.net"),
("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"),
("join", r"join\.com/companies/([a-z0-9-]+)"),
("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"),
("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"),
("gem", r"jobs\.gem\.com/([a-z0-9-]+)"),
("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
("eightfold", r"\.eightfold\.ai"),
("phenom", r"phenompeople\.com|phenom\.com/careers"),
("avature", r"\.avature\.net"),
("oracle", r"oraclecloud\.com/hcmUI"),
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"),
))
#: host -> pattern over a URL, for the hosts discovery names but the app
#: cannot read. Group 1, where there is one, is the slug the host names the
#: company by, kept for the day a parser exists; a host with no group is
#: recognised but its board has no slug the app could use.
_UNREADABLE: tuple[tuple[str, str], ...] = (
("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"),
("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"),
("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
("taleo", r"\.taleo\.net"),
("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"),
("join", r"join\.com/companies/([a-z0-9-]+)"),
("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"),
("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"),
("gem", r"jobs\.gem\.com/([a-z0-9-]+)"),
("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
("eightfold", r"\.eightfold\.ai"),
("phenom", r"phenompeople\.com|phenom\.com/careers"),
("avature", r"\.avature\.net"),
("oracle", r"oraclecloud\.com/hcmUI"),
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"),
)
def _hosts(modules: Mapping[str, ModuleType],
unreadable: Iterable[tuple[str, str]]) -> tuple[tuple[str, re.Pattern[str]], ...]:
"""Every host's patterns, compiled: the readable hosts' from their
modules first, so the first match at one place on a page is the one the
app can read, then the unreadable ones. An unreadable host that has
gained a module is left out here, since its module's patterns replace
it."""
out = [(name, pattern) for name, module in modules.items() for pattern in module.PATTERNS]
out += [(host, pattern) for host, pattern in unreadable if host not in modules]
return tuple((host, re.compile(pattern, re.IGNORECASE)) for host, pattern in out)
#: host -> pattern over a URL, every host discovery can name. The slug is
#: group 1, or, for a pattern with more groups, the groups that matched
#: joined by "/" (Lever's `eu/<site>`).
HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = _hosts(atshosts.HOSTS, _UNREADABLE)
#: What a Teamtailor feed carries and a page that only looks like one does not:
#: the namespace of `tt:locations`, declared on `<rss>` even when no role is open.
@@ -161,9 +164,10 @@ def boards_in(text: str) -> list[Found]:
hits: list[tuple[int, str, str, str]] = []
for host, pattern in HOSTS:
for m in pattern.finditer(text):
slug = (m.group(1) if m.groups() and m.group(1) else "").lower()
if slug in _NOT_SLUGS:
parts = [part.lower() for part in m.groups() if part]
if any(part in _NOT_SLUGS for part in parts):
continue
slug = "/".join(parts)
hits.append((m.start(), host, slug, m.group(0)))
for _pos, host, slug, url in sorted(hits):
if (host, slug) in seen:
@@ -237,25 +241,28 @@ def fetch_page(client: httpx.Client, url: str) -> tuple[str, str]:
return "", ""
#: host -> a cheap URL that answers 2xx only when the slug is a board there.
#: Workable is left out: its API sits behind a rate limit that a run of
#: guesses trips, and a Workable site links its board plainly anyway.
_PROBES = {
"greenhouse": "https://boards-api.greenhouse.io/v1/boards/{slug}",
"lever": "https://api.lever.co/v0/postings/{slug}?limit=1&mode=json",
"ashby": "https://api.ashbyhq.com/posting-api/job-board/{slug}",
"smartrecruiters": "https://api.smartrecruiters.com/v1/companies/{slug}/postings?limit=1",
"recruitee": "https://{slug}.recruitee.com/api/offers/",
"personio": "https://{slug}.jobs.personio.de/xml",
"teamtailor": "https://{slug}.teamtailor.com/jobs.rss",
}
#: host -> a cheap URL that answers 2xx only when the slug is a board there,
#: from each module's `probe_url`. Workable has none: its API sits behind a
#: rate limit that a run of guesses trips, and a Workable site links its
#: board plainly anyway.
_PROBES: dict[str, Callable[[str], str]] = {
name: module.probe_url for name, module in atshosts.HOSTS.items()
if hasattr(module, "probe_url")}
def _probe_slugs(host: str, guess: str) -> tuple[str, ...]:
"""The slugs a guessed name is probed as on `host`: the name itself, and
for Lever the name on its EU instance too (the module's `probe_slugs`)."""
module = atshosts.HOSTS[host]
return module.probe_slugs(guess) if hasattr(module, "probe_slugs") else (guess,)
def probe(client: httpx.Client, host: str, slug: str) -> bool:
"""Whether `host` has a board called `slug`, by its cheapest endpoint.
Ashby and SmartRecruiters answer an empty listing for an unknown slug,
so there a listing without jobs counts as no board."""
url = _PROBES[host].format(slug=slug)
A host that answers an unknown slug with an empty listing (Ashby,
SmartRecruiters, ...) says in its module's `probe_ok` what a listing
with a board looks like."""
url = _PROBES[host](slug)
try:
response = client.get(url, timeout=TIMEOUT, follow_redirects=False,
headers={"User-Agent": USER_AGENT})
@@ -263,18 +270,8 @@ def probe(client: httpx.Client, host: str, slug: str) -> bool:
return False
if not response.is_success:
return False
body = response.content
if host == "ashby":
return b'"jobs":[' in body and b'"jobs":[]' not in body
if host == "smartrecruiters":
return b'"totalFound":' in body and b'"totalFound":0' not in body
if host == "recruitee":
return b'"offers":[' in body and b'"offers":[]' not in body
if host == "personio":
return b"<position>" in body
if host == "teamtailor":
return b"<item>" in body
return True
module = atshosts.HOSTS[host]
return module.probe_ok(response.content) if hasattr(module, "probe_ok") else True
Fetch = Callable[[httpx.Client, str], tuple[str, str]]
@@ -377,9 +374,10 @@ def _discover(name: str, site: str, client: httpx.Client, fetch: Fetch,
# probe could only add another tenant's board of the same name.
if unreadable is not None:
return unreadable
for slug in slug_guesses(name, site):
for guess in slug_guesses(name, site):
for host in _PROBES:
if probe_slug(client, host, slug):
return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug,
careers_url=careers_page, evidence=_PROBES[host].format(slug=slug))
for slug in _probe_slugs(host, guess):
if probe_slug(client, host, slug):
return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug,
careers_url=careers_page, evidence=_PROBES[host](slug))
return Discovery(careers_url=careers_page)