Take discovery's readable hosts, patterns and probes from the host modules, and find Lever boards on its EU instance
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8b14d8575b
commit
a30b1ff09f
2 files changed
+141
-96
No files matched your search
+85
-87
@@ -5,10 +5,12 @@ The seed list and the LinkedIn flow both start from a company name and a
|
||||
website, not from a board. This module gets from the one to the other: it
|
||||
fetches the site's careers page, or the pages the home page links to as
|
||||
careers, and looks for a link into one of the hosts in `HOSTS`. A readable
|
||||
host (one the `ats` plugin parses) becomes a `board` target the pull can
|
||||
read; any other host is recorded as `ats`, with its slug where the URL
|
||||
carries one, so the company is kept as one the app cannot read yet rather
|
||||
than dropped.
|
||||
host (one with a module in `plugins/atshosts/`, which the `ats` plugin
|
||||
parses) becomes a `board` target the pull can read; any other host is
|
||||
recorded as `ats`, with its slug where the URL carries one, so the company is
|
||||
kept as one the app cannot read yet rather than dropped. The patterns and
|
||||
probes of a readable host are its module's; the hosts only named are in
|
||||
`_UNREADABLE` here, and one of them that gains a module is read from then on.
|
||||
|
||||
A Teamtailor site on the company's own domain names no slug: the page only
|
||||
loads Teamtailor's CDN. The board there is the site's own host, and it is
|
||||
@@ -16,7 +18,8 @@ taken only once that host answers `/jobs.rss` with a Teamtailor feed.
|
||||
|
||||
Only when no page names a host at all are the hosts with a cheap per-slug
|
||||
endpoint (`_PROBES`) probed with the company's own slug, because a site that
|
||||
draws its jobs from an iframe or a script leaves no link to find. A page that
|
||||
draws its jobs from an iframe or a script leaves no link to find. Lever is
|
||||
probed on its EU instance as well, as `lever:eu/<slug>`. A page that
|
||||
names an unreadable host is the answer: a probe on a common name can hit
|
||||
another tenant's board.
|
||||
|
||||
@@ -37,13 +40,15 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from types import ModuleType
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import httpx
|
||||
|
||||
from desk.sources import USER_AGENT
|
||||
from plugins import atshosts
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -53,55 +58,53 @@ MAX_BYTES = 2 << 20
|
||||
MAX_PAGES = 6
|
||||
|
||||
#: The hosts the `ats` plugin parses; a match on one of these is a `board`.
|
||||
READABLE = ("greenhouse", "lever", "ashby", "workable", "smartrecruiters",
|
||||
"recruitee", "personio", "teamtailor")
|
||||
READABLE: tuple[str, ...] = tuple(atshosts.HOSTS)
|
||||
|
||||
#: host -> pattern over a URL. Group 1, where there is one, is the slug the
|
||||
#: host names the company by; a host with no group is recognised but its
|
||||
#: board has no slug the app could use. Order matters only for readability:
|
||||
#: the readable hosts come first so the first match on a page is the one the
|
||||
#: app can read.
|
||||
HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = tuple(
|
||||
(host, re.compile(pattern, re.IGNORECASE)) for host, pattern in (
|
||||
("greenhouse", r"(?:boards|job-boards|api)\.(?:eu\.)?greenhouse\.io/(?:v1/boards/|embed/job_board\?[^\"'\s]*for=)?([A-Za-z0-9_-]+)"),
|
||||
("greenhouse", r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)"),
|
||||
("lever", r"jobs\.(?:eu\.)?lever\.co/([A-Za-z0-9_-]+)"),
|
||||
("lever", r"api\.(?:eu\.)?lever\.co/v0/postings/([A-Za-z0-9_-]+)"),
|
||||
("ashby", r"jobs\.ashbyhq\.com/([A-Za-z0-9_.-]+)"),
|
||||
("ashby", r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)"),
|
||||
("workable", r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_-]+)"),
|
||||
("workable", r"https?://(?!apply\.|www\.)([a-z0-9-]+)\.workable\.com"),
|
||||
("smartrecruiters", r"(?:jobs|careers|api)\.smartrecruiters\.com/(?:v1/companies/)?([A-Za-z0-9_-]+)"),
|
||||
("recruitee", r"https?://([a-z0-9-]+)\.recruitee\.com"),
|
||||
("personio", r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)"),
|
||||
("teamtailor", r"https?://([a-z0-9-]+)\.teamtailor\.com"),
|
||||
# No slug: a career site on the company's own domain, which only loads
|
||||
# Teamtailor's CDN. `_discover` reads its board off the page's host.
|
||||
("teamtailor", r"cdn\.teamtailor\.com|teamtailor-cdn|teamtailor\.com/assets"),
|
||||
# Not readable. The slug, where the URL has one, is kept for the day
|
||||
# a parser exists.
|
||||
("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"),
|
||||
("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"),
|
||||
("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
|
||||
("taleo", r"\.taleo\.net"),
|
||||
("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
|
||||
("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"),
|
||||
("join", r"join\.com/companies/([a-z0-9-]+)"),
|
||||
("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
|
||||
("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"),
|
||||
("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"),
|
||||
("gem", r"jobs\.gem\.com/([a-z0-9-]+)"),
|
||||
("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
|
||||
("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
|
||||
("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
|
||||
("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
|
||||
("eightfold", r"\.eightfold\.ai"),
|
||||
("phenom", r"phenompeople\.com|phenom\.com/careers"),
|
||||
("avature", r"\.avature\.net"),
|
||||
("oracle", r"oraclecloud\.com/hcmUI"),
|
||||
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
|
||||
("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"),
|
||||
))
|
||||
#: host -> pattern over a URL, for the hosts discovery names but the app
|
||||
#: cannot read. Group 1, where there is one, is the slug the host names the
|
||||
#: company by, kept for the day a parser exists; a host with no group is
|
||||
#: recognised but its board has no slug the app could use.
|
||||
_UNREADABLE: tuple[tuple[str, str], ...] = (
|
||||
("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"),
|
||||
("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"),
|
||||
("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
|
||||
("taleo", r"\.taleo\.net"),
|
||||
("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
|
||||
("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"),
|
||||
("join", r"join\.com/companies/([a-z0-9-]+)"),
|
||||
("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
|
||||
("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"),
|
||||
("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"),
|
||||
("gem", r"jobs\.gem\.com/([a-z0-9-]+)"),
|
||||
("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
|
||||
("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
|
||||
("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
|
||||
("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
|
||||
("eightfold", r"\.eightfold\.ai"),
|
||||
("phenom", r"phenompeople\.com|phenom\.com/careers"),
|
||||
("avature", r"\.avature\.net"),
|
||||
("oracle", r"oraclecloud\.com/hcmUI"),
|
||||
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
|
||||
("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"),
|
||||
)
|
||||
|
||||
|
||||
def _hosts(modules: Mapping[str, ModuleType],
|
||||
unreadable: Iterable[tuple[str, str]]) -> tuple[tuple[str, re.Pattern[str]], ...]:
|
||||
"""Every host's patterns, compiled: the readable hosts' from their
|
||||
modules first, so the first match at one place on a page is the one the
|
||||
app can read, then the unreadable ones. An unreadable host that has
|
||||
gained a module is left out here, since its module's patterns replace
|
||||
it."""
|
||||
out = [(name, pattern) for name, module in modules.items() for pattern in module.PATTERNS]
|
||||
out += [(host, pattern) for host, pattern in unreadable if host not in modules]
|
||||
return tuple((host, re.compile(pattern, re.IGNORECASE)) for host, pattern in out)
|
||||
|
||||
|
||||
#: host -> pattern over a URL, every host discovery can name. The slug is
|
||||
#: group 1, or, for a pattern with more groups, the groups that matched
|
||||
#: joined by "/" (Lever's `eu/<site>`).
|
||||
HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = _hosts(atshosts.HOSTS, _UNREADABLE)
|
||||
|
||||
#: What a Teamtailor feed carries and a page that only looks like one does not:
|
||||
#: the namespace of `tt:locations`, declared on `<rss>` even when no role is open.
|
||||
@@ -161,9 +164,10 @@ def boards_in(text: str) -> list[Found]:
|
||||
hits: list[tuple[int, str, str, str]] = []
|
||||
for host, pattern in HOSTS:
|
||||
for m in pattern.finditer(text):
|
||||
slug = (m.group(1) if m.groups() and m.group(1) else "").lower()
|
||||
if slug in _NOT_SLUGS:
|
||||
parts = [part.lower() for part in m.groups() if part]
|
||||
if any(part in _NOT_SLUGS for part in parts):
|
||||
continue
|
||||
slug = "/".join(parts)
|
||||
hits.append((m.start(), host, slug, m.group(0)))
|
||||
for _pos, host, slug, url in sorted(hits):
|
||||
if (host, slug) in seen:
|
||||
@@ -237,25 +241,28 @@ def fetch_page(client: httpx.Client, url: str) -> tuple[str, str]:
|
||||
return "", ""
|
||||
|
||||
|
||||
#: host -> a cheap URL that answers 2xx only when the slug is a board there.
|
||||
#: Workable is left out: its API sits behind a rate limit that a run of
|
||||
#: guesses trips, and a Workable site links its board plainly anyway.
|
||||
_PROBES = {
|
||||
"greenhouse": "https://boards-api.greenhouse.io/v1/boards/{slug}",
|
||||
"lever": "https://api.lever.co/v0/postings/{slug}?limit=1&mode=json",
|
||||
"ashby": "https://api.ashbyhq.com/posting-api/job-board/{slug}",
|
||||
"smartrecruiters": "https://api.smartrecruiters.com/v1/companies/{slug}/postings?limit=1",
|
||||
"recruitee": "https://{slug}.recruitee.com/api/offers/",
|
||||
"personio": "https://{slug}.jobs.personio.de/xml",
|
||||
"teamtailor": "https://{slug}.teamtailor.com/jobs.rss",
|
||||
}
|
||||
#: host -> a cheap URL that answers 2xx only when the slug is a board there,
|
||||
#: from each module's `probe_url`. Workable has none: its API sits behind a
|
||||
#: rate limit that a run of guesses trips, and a Workable site links its
|
||||
#: board plainly anyway.
|
||||
_PROBES: dict[str, Callable[[str], str]] = {
|
||||
name: module.probe_url for name, module in atshosts.HOSTS.items()
|
||||
if hasattr(module, "probe_url")}
|
||||
|
||||
|
||||
def _probe_slugs(host: str, guess: str) -> tuple[str, ...]:
|
||||
"""The slugs a guessed name is probed as on `host`: the name itself, and
|
||||
for Lever the name on its EU instance too (the module's `probe_slugs`)."""
|
||||
module = atshosts.HOSTS[host]
|
||||
return module.probe_slugs(guess) if hasattr(module, "probe_slugs") else (guess,)
|
||||
|
||||
|
||||
def probe(client: httpx.Client, host: str, slug: str) -> bool:
|
||||
"""Whether `host` has a board called `slug`, by its cheapest endpoint.
|
||||
Ashby and SmartRecruiters answer an empty listing for an unknown slug,
|
||||
so there a listing without jobs counts as no board."""
|
||||
url = _PROBES[host].format(slug=slug)
|
||||
A host that answers an unknown slug with an empty listing (Ashby,
|
||||
SmartRecruiters, ...) says in its module's `probe_ok` what a listing
|
||||
with a board looks like."""
|
||||
url = _PROBES[host](slug)
|
||||
try:
|
||||
response = client.get(url, timeout=TIMEOUT, follow_redirects=False,
|
||||
headers={"User-Agent": USER_AGENT})
|
||||
@@ -263,18 +270,8 @@ def probe(client: httpx.Client, host: str, slug: str) -> bool:
|
||||
return False
|
||||
if not response.is_success:
|
||||
return False
|
||||
body = response.content
|
||||
if host == "ashby":
|
||||
return b'"jobs":[' in body and b'"jobs":[]' not in body
|
||||
if host == "smartrecruiters":
|
||||
return b'"totalFound":' in body and b'"totalFound":0' not in body
|
||||
if host == "recruitee":
|
||||
return b'"offers":[' in body and b'"offers":[]' not in body
|
||||
if host == "personio":
|
||||
return b"<position>" in body
|
||||
if host == "teamtailor":
|
||||
return b"<item>" in body
|
||||
return True
|
||||
module = atshosts.HOSTS[host]
|
||||
return module.probe_ok(response.content) if hasattr(module, "probe_ok") else True
|
||||
|
||||
|
||||
Fetch = Callable[[httpx.Client, str], tuple[str, str]]
|
||||
@@ -377,9 +374,10 @@ def _discover(name: str, site: str, client: httpx.Client, fetch: Fetch,
|
||||
# probe could only add another tenant's board of the same name.
|
||||
if unreadable is not None:
|
||||
return unreadable
|
||||
for slug in slug_guesses(name, site):
|
||||
for guess in slug_guesses(name, site):
|
||||
for host in _PROBES:
|
||||
if probe_slug(client, host, slug):
|
||||
return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug,
|
||||
careers_url=careers_page, evidence=_PROBES[host].format(slug=slug))
|
||||
for slug in _probe_slugs(host, guess):
|
||||
if probe_slug(client, host, slug):
|
||||
return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug,
|
||||
careers_url=careers_page, evidence=_PROBES[host](slug))
|
||||
return Discovery(careers_url=careers_page)
|
||||
+56
-9
@@ -4,6 +4,7 @@ site's pages are read."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
@@ -36,16 +37,32 @@ def test_boards_in_names_readable_hosts_with_their_slug() -> None:
|
||||
|
||||
def test_boards_in_keeps_hosts_the_app_cannot_read() -> None:
|
||||
page = """
|
||||
<a href="https://novo.wd3.myworkdayjobs.com/en-US/careers">Workday</a>
|
||||
<a href="https://performancemanager.successfactors.eu/career?company=x">SF</a>
|
||||
<a href="https://novo.taleo.net/careersection/2/jobsearch.ftl">Taleo</a>
|
||||
<a href="https://acme.teamtailor.com/jobs">TT</a>
|
||||
"""
|
||||
assert boards_in(page) == [
|
||||
("workday", "novo", "https://novo.wd3.myworkdayjobs.com"),
|
||||
("successfactors", "", "performancemanager.successfactors.eu"),
|
||||
("taleo", "", ".taleo.net"),
|
||||
("teamtailor", "acme", "https://acme.teamtailor.com")]
|
||||
|
||||
|
||||
def test_boards_in_names_a_lever_eu_board_by_its_region() -> None:
|
||||
page = ('<a href="https://jobs.eu.lever.co/tomtom/abc-123">EU</a>'
|
||||
'<script src="https://api.eu.lever.co/v0/postings/tomtom?mode=json"></script>'
|
||||
'<a href="https://jobs.lever.co/whereby">US</a>')
|
||||
assert [(h, s) for h, s, _ in boards_in(page)] == [("lever", "eu/tomtom"),
|
||||
("lever", "whereby")]
|
||||
|
||||
|
||||
def test_a_host_that_gains_a_module_replaces_its_unreadable_pattern() -> None:
|
||||
"""Wave by wave, a host named here becomes readable by its module alone."""
|
||||
taleo = types.ModuleType("taleo")
|
||||
taleo.PATTERNS = (r"https?://([a-z0-9-]+)\.taleo\.net",)
|
||||
hosts = mod._hosts({"taleo": taleo}, mod._UNREADABLE)
|
||||
assert [pattern.pattern for host, pattern in hosts if host == "taleo"] == [
|
||||
r"https?://([a-z0-9-]+)\.taleo\.net"]
|
||||
assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE
|
||||
|
||||
|
||||
def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None:
|
||||
page = ('<a href="https://boards.greenhouse.io/embed/job_board?for=acme">a</a>'
|
||||
'<a href="https://boards-api.greenhouse.io/v1/boards/acme/jobs">b</a>')
|
||||
@@ -138,12 +155,12 @@ def test_a_careers_pages_own_link_is_followed_one_level_down() -> None:
|
||||
def test_an_unreadable_host_named_by_a_page_is_the_answer_and_nothing_is_probed() -> None:
|
||||
site = Site({
|
||||
"https://acme.com/": "<p>welcome</p>",
|
||||
"https://acme.com/careers": '<a href="https://acme.wd3.myworkdayjobs.com/x">Jobs</a>',
|
||||
"https://acme.com/careers": '<a href="https://acme.taleo.net/careersection/x">Jobs</a>',
|
||||
}, boards={("greenhouse", "acme")})
|
||||
found = run(site)
|
||||
assert found == Discovery(ats="workday", ats_slug="acme",
|
||||
assert found == Discovery(ats="taleo", ats_slug="",
|
||||
careers_url="https://acme.com/careers",
|
||||
evidence="https://acme.wd3.myworkdayjobs.com")
|
||||
evidence=".taleo.net")
|
||||
assert not found.readable
|
||||
# The page named the host: a probe could only find another tenant's
|
||||
# board of the same name, like the greenhouse:acme this fake web has.
|
||||
@@ -223,6 +240,36 @@ def test_slug_probes_find_a_board_the_pages_never_link() -> None:
|
||||
assert found.evidence == "https://api.ashbyhq.com/posting-api/job-board/acme"
|
||||
|
||||
|
||||
def test_a_lever_eu_board_is_found_by_its_probe() -> None:
|
||||
site = Site({"https://acme.com/": "<p>a script draws the jobs</p>"},
|
||||
boards={("lever", "eu/acme")})
|
||||
found = run(site)
|
||||
assert found.board == "lever:eu/acme"
|
||||
assert found.evidence == "https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json"
|
||||
assert site.probed.index(("lever", "acme")) + 1 == site.probed.index(("lever", "eu/acme"))
|
||||
|
||||
|
||||
def test_probe_asks_each_hosts_own_endpoint_and_body_check() -> None:
|
||||
asked: list[str] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
asked.append(str(request.url))
|
||||
if request.url.host == "api.eu.lever.co":
|
||||
return httpx.Response(200, json=[])
|
||||
if request.url.host == "api.ashbyhq.com":
|
||||
return httpx.Response(200, content=b'{"jobs":[]}')
|
||||
return httpx.Response(404)
|
||||
|
||||
with httpx.Client(transport=httpx.MockTransport(handler)) as client:
|
||||
assert mod.probe(client, "lever", "eu/acme")
|
||||
assert not mod.probe(client, "lever", "acme")
|
||||
assert not mod.probe(client, "ashby", "acme") # an empty listing is no board
|
||||
assert asked == ["https://api.eu.lever.co/v0/postings/acme?limit=1&mode=json",
|
||||
"https://api.lever.co/v0/postings/acme?limit=1&mode=json",
|
||||
"https://api.ashbyhq.com/posting-api/job-board/acme"]
|
||||
assert "workable" not in mod._PROBES and "lever" in mod._PROBES
|
||||
|
||||
|
||||
def test_a_site_that_never_answers_gives_an_empty_discovery() -> None:
|
||||
site = Site({})
|
||||
assert run(site) == Discovery()
|
||||
@@ -251,7 +298,7 @@ def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Pa
|
||||
{"name": "Pleo", "domain": "pleo.io", "country": "DK", "city": "Copenhagen",
|
||||
"board": "ashby:pleo"},
|
||||
{"name": "Novo Nordisk", "domain": "novonordisk.com", "country": "DK",
|
||||
"city": "Bagsværd", "ats": "successfactors",
|
||||
"city": "Bagsværd", "ats": "taleo",
|
||||
"careers_url": "https://www.novonordisk.com/careers.html"},
|
||||
{"name": "Odd", "board": "nosuch:odd"},
|
||||
]}))
|
||||
@@ -266,7 +313,7 @@ def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Pa
|
||||
assert (sub.target, sub.company) == ("ashby:pleo", "Pleo")
|
||||
novo = store.company("novo-nordisk")
|
||||
assert novo is not None
|
||||
assert (novo.ats, novo.careers_url) == ("successfactors",
|
||||
assert (novo.ats, novo.careers_url) == ("taleo",
|
||||
"https://www.novonordisk.com/careers.html")
|
||||
odd = store.company("odd")
|
||||
assert odd is not None and odd.ats == ""
|
||||
|
||||
Reference in new issue
Block a user