Ten entries were shadowed by their modules. A test now holds that no entry names a reader, which replaces the per-host checks. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
386 lines
16 KiB
Python
386 lines
16 KiB
Python
"""Board discovery: which applicant tracking system a company's careers site
|
|
runs on, read off its own pages.
|
|
|
|
The seed list and the LinkedIn flow both start from a company name and a
|
|
website, not from a board. This module gets from the one to the other: it
|
|
fetches the site's careers page, or the pages the home page links to as
|
|
careers, and looks for a link into one of the hosts in `HOSTS`. A readable
|
|
host (one with a module in `plugins/readers/`, which the `ats` plugin
|
|
parses) becomes a `board` target the pull can read; any other host is
|
|
recorded as `ats`, with its slug where the URL carries one, so the company is
|
|
kept as one the app cannot read yet rather than dropped. The patterns and
|
|
probes of a readable host are its module's; the hosts only named are in
|
|
`_UNREADABLE` here, and a host that gains a module loses its entry there
|
|
(`tests/test_discover.py` holds that no entry names a reader).
|
|
|
|
A Teamtailor site on the company's own domain names no slug: the page only
|
|
loads Teamtailor's CDN. The board there is the site's own host, and it is
|
|
taken only once that host answers `/jobs.rss` with a Teamtailor feed.
|
|
|
|
Only when no page names a host at all are the hosts with a cheap per-slug
|
|
endpoint (`_PROBES`) probed with the company's own slug, because a site that
|
|
draws its jobs from an iframe or a script leaves no link to find. Lever is
|
|
probed on its EU instance as well, as `lever:eu/<slug>`. A page that
|
|
names an unreadable host is the answer: a probe on a common name can hit
|
|
another tenant's board.
|
|
|
|
found = discover("Pleo", "pleo.io")
|
|
found.board # "greenhouse:pleo", or "" when nothing readable
|
|
found.ats # "greenhouse", "workday", ... or "" when unknown
|
|
found.ats_slug # the name that host knows the company by, or ""
|
|
found.careers_url # the page that named the host, or the careers page
|
|
found.evidence # the URL that named it
|
|
|
|
Only https is fetched, redirects are followed at most five hops, a body is
|
|
cut at `MAX_BYTES`, and every failure is one page skipped. A company's site
|
|
is content the app was pointed at by its owner, not an ATS API, so nothing
|
|
here trusts what it finds beyond a host name matching a fixed pattern.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import re
|
|
from collections.abc import Callable, Iterable, Mapping
|
|
from dataclasses import dataclass, replace
|
|
from types import ModuleType
|
|
from urllib.parse import urljoin
|
|
|
|
import httpx
|
|
|
|
from desk.sources import USER_AGENT
|
|
from plugins import readers
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
TIMEOUT = 10.0
|
|
MAX_BYTES = 2 << 20
|
|
#: How many pages of one site are read before giving up on links.
|
|
MAX_PAGES = 6
|
|
|
|
#: The hosts the `ats` plugin parses; a match on one of these is a `board`.
|
|
READABLE: tuple[str, ...] = tuple(readers.HOSTS)
|
|
|
|
#: host -> pattern over a URL, for the hosts discovery names but the app
|
|
#: cannot read. Group 1, where there is one, is the slug the host names the
|
|
#: company by, kept for the day a parser exists; a host with no group is
|
|
#: recognised but its board has no slug the app could use.
|
|
_UNREADABLE: tuple[tuple[str, str], ...] = (
|
|
("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
|
|
("taleo", r"\.taleo\.net"),
|
|
("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
|
|
("join", r"join\.com/companies/([a-z0-9-]+)"),
|
|
("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
|
|
("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
|
|
("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
|
|
("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
|
|
("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
|
|
("avature", r"\.avature\.net"),
|
|
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
|
|
)
|
|
|
|
|
|
def _hosts(modules: Mapping[str, ModuleType],
|
|
unreadable: Iterable[tuple[str, str]]) -> tuple[tuple[str, re.Pattern[str]], ...]:
|
|
"""Every host's patterns, compiled: the readable hosts' from their
|
|
modules first, so the first match at one place on a page is the one the
|
|
app can read, then the unreadable ones. An unreadable host that has
|
|
a module is left out, so its module's patterns win even before its
|
|
entry is deleted."""
|
|
out = [(name, pattern) for name, module in modules.items() for pattern in module.PATTERNS]
|
|
out += [(host, pattern) for host, pattern in unreadable if host not in modules]
|
|
return tuple((host, re.compile(pattern, re.IGNORECASE)) for host, pattern in out)
|
|
|
|
|
|
#: host -> pattern over a URL, every host discovery can name. The slug is
|
|
#: group 1, or, for a pattern with more groups, the groups that matched
|
|
#: joined by "/" (Lever's `eu/<site>`).
|
|
HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = _hosts(readers.HOSTS, _UNREADABLE)
|
|
|
|
#: The hosts whose slugs keep the case the URL wrote them in, by their
|
|
#: module's `KEEP_CASE`: a Workday site is `LEGO_External`, and both its
|
|
#: listing and its robots.txt paths are case-sensitive. Every other slug is
|
|
#: lowercased, which is how a page that writes `BoschGroup` and one that
|
|
#: writes `boschgroup` name one board.
|
|
_KEEP_CASE = frozenset(name for name, module in readers.HOSTS.items()
|
|
if getattr(module, "KEEP_CASE", False))
|
|
|
|
#: What a Teamtailor feed carries and a page that only looks like one does not:
|
|
#: the namespace of `tt:locations`, declared on `<rss>` even when no role is open.
|
|
TEAMTAILOR_FEED = "https://teamtailor.com/locations"
|
|
|
|
#: Paths a careers page usually lives at, tried after the home page's own
|
|
#: links so the site's word for it wins over a guess.
|
|
CAREERS_PATHS = ("/careers", "/careers/", "/jobs", "/en/careers", "/company/careers",
|
|
"/about/careers", "/join-us", "/careers/jobs", "/en/jobs")
|
|
#: Subdomains careers pages move to.
|
|
CAREERS_HOSTS = ("careers.", "jobs.", "career.")
|
|
|
|
#: A link text or href that points at a careers page.
|
|
_CAREERS_LINK = re.compile(
|
|
r"career|jobs|join[-_ ]?us|work[-_ ]?with[-_ ]?us|open[-_ ]?positions|vacanc|"
|
|
r"we[-_ ]?are[-_ ]?hiring|ledige[-_ ]?stillinger|stellen|karriere|lediga[-_ ]?jobb",
|
|
re.IGNORECASE)
|
|
_HREF = re.compile(r"""(?:href|src|action|content)\s*=\s*["']([^"'#][^"']*)["']""",
|
|
re.IGNORECASE)
|
|
_ANCHOR = re.compile(r"<a\b[^>]*href\s*=\s*[\"']([^\"'#][^\"']*)[\"'][^>]*>(.*?)</a>",
|
|
re.IGNORECASE | re.DOTALL)
|
|
_TAGS = re.compile(r"<[^>]+>")
|
|
|
|
#: Words a pattern's group 1 can catch that name no board: the hosts'
|
|
#: own path segments, the analytics subdomain Recruitee sites embed, and
|
|
#: the CDN and the admin app Teamtailor sites load from (cdn.teamtailor.com,
|
|
#: app.teamtailor.com).
|
|
_NOT_SLUGS = frozenset({"embed", "boards", "v1", "api", "www", "jobs", "careers", "apply",
|
|
"careers-analytics", "cdn", "app"})
|
|
|
|
Found = tuple[str, str, str] # (host, slug, the URL that named it)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Discovery:
|
|
"""What discovery learned about one company; every field may be empty."""
|
|
|
|
board: str = ""
|
|
ats: str = ""
|
|
ats_slug: str = ""
|
|
careers_url: str = ""
|
|
evidence: str = ""
|
|
|
|
@property
|
|
def readable(self) -> bool:
|
|
return bool(self.board)
|
|
|
|
|
|
# ------------------------------------------------------------------ parsing
|
|
|
|
def boards_in(text: str) -> list[Found]:
|
|
"""Every ATS host named in `text` (a page, a script, a mail), in order of
|
|
first appearance, once each. `slug` is "" for a host whose URL carries
|
|
none."""
|
|
seen: set[tuple[str, str]] = set()
|
|
out: list[Found] = []
|
|
hits: list[tuple[int, str, str, str]] = []
|
|
for host, pattern in HOSTS:
|
|
for m in pattern.finditer(text):
|
|
# Only group 1 names a board or a host's own path word; a later
|
|
# group may be anything, a Workday site called `jobs` included.
|
|
if m.re.groups and (m.group(1) or "").lower() in _NOT_SLUGS:
|
|
continue
|
|
slug = "/".join(part for part in m.groups() if part)
|
|
if host not in _KEEP_CASE:
|
|
slug = slug.lower()
|
|
hits.append((m.start(), host, slug, m.group(0)))
|
|
for _pos, host, slug, url in sorted(hits):
|
|
if (host, slug) in seen:
|
|
continue
|
|
seen.add((host, slug))
|
|
out.append((host, slug, url))
|
|
return out
|
|
|
|
|
|
def careers_links(html: str, base: str) -> list[str]:
|
|
"""The links on a page that read as its careers page, absolute, in page
|
|
order, once each. Nothing off-site is followed: a careers page on another
|
|
domain is what `boards_in` finds by its host."""
|
|
out: list[str] = []
|
|
site = httpx.URL(base).host or ""
|
|
for href, label in _ANCHOR.findall(html):
|
|
text = _TAGS.sub(" ", label)
|
|
if not (_CAREERS_LINK.search(href) or _CAREERS_LINK.search(text)):
|
|
continue
|
|
url = urljoin(base, href.strip())
|
|
parsed = httpx.URL(url)
|
|
if parsed.scheme not in ("http", "https"):
|
|
continue
|
|
host = parsed.host or ""
|
|
if host != site and not host.endswith("." + _apex(site)):
|
|
continue
|
|
url = str(parsed.copy_with(scheme="https", fragment=None))
|
|
if url not in out:
|
|
out.append(url)
|
|
return out
|
|
|
|
|
|
def _apex(host: str) -> str:
|
|
parts = host.split(".")
|
|
return ".".join(parts[-2:]) if len(parts) > 2 else host
|
|
|
|
|
|
def slug_guesses(name: str, domain: str) -> list[str]:
|
|
"""The slugs a host most likely knows the company by: its key, the key
|
|
without hyphens, and the domain's own label."""
|
|
from app.models import company_key
|
|
|
|
key = company_key(name)
|
|
label = domain.lower().split("/")[0]
|
|
for prefix in ("www.",):
|
|
label = label.removeprefix(prefix)
|
|
label = label.split(".")[0]
|
|
out = []
|
|
for guess in (key, key.replace("-", ""), label, label.replace("-", "")):
|
|
if guess and guess not in out:
|
|
out.append(guess)
|
|
return out
|
|
|
|
|
|
# ----------------------------------------------------------------- fetching
|
|
|
|
def fetch_page(client: httpx.Client, url: str) -> tuple[str, str]:
|
|
"""`(final URL, text)` for one page, or `("", "")` when it did not answer
|
|
with a 2xx text body within the limits. Never raises."""
|
|
try:
|
|
response = client.get(
|
|
url, timeout=TIMEOUT, follow_redirects=True,
|
|
headers={"User-Agent": USER_AGENT,
|
|
"Accept": "text/html,application/xhtml+xml,*/*;q=0.5"})
|
|
if not response.is_success:
|
|
return "", ""
|
|
body = response.content[:MAX_BYTES]
|
|
return str(response.url), body.decode(response.encoding or "utf-8", "replace")
|
|
except Exception as exc: # one page skipped, whatever went wrong
|
|
log.debug("discover: %s: %s", url, exc)
|
|
return "", ""
|
|
|
|
|
|
#: host -> a cheap URL that answers 2xx only when the slug is a board there,
|
|
#: from each module's `probe_url`. Workable has none: its API sits behind a
|
|
#: rate limit that a run of guesses trips, and a Workable site links its
|
|
#: board plainly anyway.
|
|
_PROBES: dict[str, Callable[[str], str]] = {
|
|
name: module.probe_url for name, module in readers.HOSTS.items()
|
|
if hasattr(module, "probe_url")}
|
|
|
|
|
|
def _probe_slugs(host: str, guess: str) -> tuple[str, ...]:
|
|
"""The slugs a guessed name is probed as on `host`: the name itself, and
|
|
for Lever the name on its EU instance too (the module's `probe_slugs`)."""
|
|
module = readers.HOSTS[host]
|
|
return module.probe_slugs(guess) if hasattr(module, "probe_slugs") else (guess,)
|
|
|
|
|
|
def probe(client: httpx.Client, host: str, slug: str) -> bool:
|
|
"""Whether `host` has a board called `slug`, by its cheapest endpoint.
|
|
A host that answers an unknown slug with an empty listing (Ashby,
|
|
SmartRecruiters, ...) says in its module's `probe_ok` what a listing
|
|
with a board looks like."""
|
|
url = _PROBES[host](slug)
|
|
try:
|
|
response = client.get(url, timeout=TIMEOUT, follow_redirects=False,
|
|
headers={"User-Agent": USER_AGENT})
|
|
except Exception:
|
|
return False
|
|
if not response.is_success:
|
|
return False
|
|
module = readers.HOSTS[host]
|
|
return module.probe_ok(response.content) if hasattr(module, "probe_ok") else True
|
|
|
|
|
|
Fetch = Callable[[httpx.Client, str], tuple[str, str]]
|
|
Probe = Callable[[httpx.Client, str, str], bool]
|
|
|
|
|
|
def _site(domain: str) -> str:
|
|
domain = domain.strip().lower()
|
|
domain = re.sub(r"^https?://", "", domain).split("/")[0]
|
|
return domain.removeprefix("www.")
|
|
|
|
|
|
def _pages(domain: str, home_links: Iterable[str]) -> list[str]:
|
|
site = _site(domain)
|
|
urls = list(home_links)
|
|
urls += [f"https://{site}{path}" for path in CAREERS_PATHS]
|
|
urls += [f"https://{sub}{site}/" for sub in CAREERS_HOSTS]
|
|
out: list[str] = []
|
|
for url in urls:
|
|
if url not in out:
|
|
out.append(url)
|
|
return out
|
|
|
|
|
|
def discover(name: str, domain: str, *, fetch: Fetch = fetch_page,
|
|
probe_slug: Probe = probe, client: httpx.Client | None = None) -> Discovery:
|
|
"""Where `name`'s careers site draws its jobs from; see the module
|
|
docstring for the order pages are read in."""
|
|
site = _site(domain)
|
|
if not site:
|
|
return Discovery()
|
|
own = client or httpx.Client()
|
|
try:
|
|
return _discover(name, site, own, fetch, probe_slug)
|
|
finally:
|
|
if client is None:
|
|
own.close()
|
|
|
|
|
|
def _discover(name: str, site: str, client: httpx.Client, fetch: Fetch,
|
|
probe_slug: Probe) -> Discovery:
|
|
home_url, home = fetch(client, f"https://{site}/")
|
|
unreadable: Discovery | None = None
|
|
careers_page = ""
|
|
own_sites: list[tuple[str, str]] = [] # (host, page) that loaded Teamtailor's CDN
|
|
|
|
def first_readable(found: list[Found], page: str) -> Discovery | None:
|
|
"""The first readable board on a page. Otherwise the first host named
|
|
is remembered, with the first slug any page gives that host: a page
|
|
may name teamtailor by its CDN before it links acme.teamtailor.com."""
|
|
nonlocal unreadable
|
|
for host, slug, url in found:
|
|
site_host = httpx.URL(page).host if page else ""
|
|
if host == "teamtailor" and not slug and site_host and (site_host, page) not in own_sites:
|
|
own_sites.append((site_host, page))
|
|
if host in READABLE and slug:
|
|
return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug,
|
|
careers_url=page, evidence=url)
|
|
if unreadable is None and found:
|
|
host, slug, url = found[0]
|
|
unreadable = Discovery(ats=host, ats_slug=slug, careers_url=page, evidence=url)
|
|
if unreadable is not None and not unreadable.ats_slug:
|
|
for host, slug, url in found:
|
|
if host == unreadable.ats and slug:
|
|
unreadable = replace(unreadable, ats_slug=slug, evidence=url)
|
|
break
|
|
return None
|
|
|
|
if home and (hit := first_readable(boards_in(home), home_url)):
|
|
return hit
|
|
# The home page's own careers links, then the usual paths; a careers page's
|
|
# own links (a "See open positions" one level down) go to the front of the
|
|
# queue. `MAX_PAGES` fetches in all, whatever the queue holds.
|
|
queue = _pages(site, careers_links(home, home_url) if home else [])
|
|
seen = {home_url, f"https://{site}/"}
|
|
budget = MAX_PAGES
|
|
while queue and budget > 0:
|
|
url = queue.pop(0)
|
|
if url in seen:
|
|
continue
|
|
seen.add(url)
|
|
budget -= 1
|
|
final, text = fetch(client, url)
|
|
if not text:
|
|
continue
|
|
seen.add(final)
|
|
careers_page = careers_page or final
|
|
if hit := first_readable(boards_in(text), final):
|
|
return hit
|
|
queue[:0] = [link for link in careers_links(text, final)[:2] if link not in seen]
|
|
# A page loaded Teamtailor's CDN and linked no *.teamtailor.com: the site
|
|
# itself is the board, if it serves the feed.
|
|
for site_host, page in own_sites:
|
|
feed = f"https://{site_host}/jobs.rss"
|
|
_final, text = fetch(client, feed)
|
|
if TEAMTAILOR_FEED in text:
|
|
return Discovery(board=f"teamtailor:{site_host}", ats="teamtailor",
|
|
ats_slug=site_host, careers_url=page, evidence=feed)
|
|
# A page named a host the app cannot read: that is the answer, and a
|
|
# probe could only add another tenant's board of the same name.
|
|
if unreadable is not None:
|
|
return unreadable
|
|
for guess in slug_guesses(name, site):
|
|
for host in _PROBES:
|
|
for slug in _probe_slugs(host, guess):
|
|
if probe_slug(client, host, slug):
|
|
return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug,
|
|
careers_url=careers_page, evidence=_PROBES[host](slug))
|
|
return Discovery(careers_url=careers_page)
|