"""Board discovery: which applicant tracking system a company's careers site runs on, read off its own pages. The seed list and the LinkedIn flow both start from a company name and a website, not from a board. This module gets from the one to the other: it fetches the site's careers page, or the pages the home page links to as careers, and looks for a link into one of the hosts in `HOSTS`. A readable host (one the `ats` plugin parses) becomes a `board` target the pull can read; any other host is recorded as `ats`, with its slug where the URL carries one, so the company is kept as one the app cannot read yet rather than dropped. Only when no page names a host at all are the hosts with a cheap per-slug endpoint (`_PROBES`) probed with the company's own slug, because a site that draws its jobs from an iframe or a script leaves no link to find. A page that names an unreadable host is the answer: a probe on a common name can hit another tenant's board. found = discover("Pleo", "pleo.io") found.board # "greenhouse:pleo", or "" when nothing readable found.ats # "greenhouse", "workday", ... or "" when unknown found.ats_slug # the name that host knows the company by, or "" found.careers_url # the page that named the host, or the careers page found.evidence # the URL that named it Only https is fetched, redirects are followed at most five hops, a body is cut at `MAX_BYTES`, and every failure is one page skipped. A company's site is content the app was pointed at by its owner, not an ATS API, so nothing here trusts what it finds beyond a host name matching a fixed pattern. """ from __future__ import annotations import logging import re from collections.abc import Callable, Iterable from dataclasses import dataclass, replace from urllib.parse import urljoin import httpx from desk.sources import USER_AGENT log = logging.getLogger(__name__) TIMEOUT = 10.0 MAX_BYTES = 2 << 20 #: How many pages of one site are read before giving up on links. MAX_PAGES = 6 #: The hosts the `ats` plugin parses; a match on one of these is a `board`. READABLE = ("greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee", "personio") #: host -> pattern over a URL. Group 1, where there is one, is the slug the #: host names the company by; a host with no group is recognised but its #: board has no slug the app could use. Order matters only for readability: #: the readable hosts come first so the first match on a page is the one the #: app can read. HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = tuple( (host, re.compile(pattern, re.IGNORECASE)) for host, pattern in ( ("greenhouse", r"(?:boards|job-boards|api)\.(?:eu\.)?greenhouse\.io/(?:v1/boards/|embed/job_board\?[^\"'\s]*for=)?([A-Za-z0-9_-]+)"), ("greenhouse", r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)"), ("lever", r"jobs\.(?:eu\.)?lever\.co/([A-Za-z0-9_-]+)"), ("lever", r"api\.(?:eu\.)?lever\.co/v0/postings/([A-Za-z0-9_-]+)"), ("ashby", r"jobs\.ashbyhq\.com/([A-Za-z0-9_.-]+)"), ("ashby", r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)"), ("workable", r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_-]+)"), ("workable", r"https?://(?!apply\.|www\.)([a-z0-9-]+)\.workable\.com"), ("smartrecruiters", r"(?:jobs|careers|api)\.smartrecruiters\.com/(?:v1/companies/)?([A-Za-z0-9_-]+)"), ("recruitee", r"https?://([a-z0-9-]+)\.recruitee\.com"), ("personio", r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)"), # Not readable. The slug, where the URL has one, is kept for the day # a parser exists. ("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"), ("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"), ("teamtailor", r"https?://([a-z0-9-]+)\.teamtailor\.com"), ("teamtailor", r"cdn\.teamtailor\.com|teamtailor-cdn|teamtailor\.com/assets"), ("icims", r"https?://([a-z0-9-]+)\.icims\.com"), ("taleo", r"\.taleo\.net"), ("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"), ("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"), ("join", r"join\.com/companies/([a-z0-9-]+)"), ("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"), ("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"), ("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"), ("gem", r"jobs\.gem\.com/([a-z0-9-]+)"), ("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"), ("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"), ("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"), ("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"), ("eightfold", r"\.eightfold\.ai"), ("phenom", r"phenompeople\.com|phenom\.com/careers"), ("avature", r"\.avature\.net"), ("oracle", r"oraclecloud\.com/hcmUI"), ("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"), ("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"), )) #: Paths a careers page usually lives at, tried after the home page's own #: links so the site's word for it wins over a guess. CAREERS_PATHS = ("/careers", "/careers/", "/jobs", "/en/careers", "/company/careers", "/about/careers", "/join-us", "/careers/jobs", "/en/jobs") #: Subdomains careers pages move to. CAREERS_HOSTS = ("careers.", "jobs.", "career.") #: A link text or href that points at a careers page. _CAREERS_LINK = re.compile( r"career|jobs|join[-_ ]?us|work[-_ ]?with[-_ ]?us|open[-_ ]?positions|vacanc|" r"we[-_ ]?are[-_ ]?hiring|ledige[-_ ]?stillinger|stellen|karriere|lediga[-_ ]?jobb", re.IGNORECASE) _HREF = re.compile(r"""(?:href|src|action|content)\s*=\s*["']([^"'#][^"']*)["']""", re.IGNORECASE) _ANCHOR = re.compile(r"]*href\s*=\s*[\"']([^\"'#][^\"']*)[\"'][^>]*>(.*?)", re.IGNORECASE | re.DOTALL) _TAGS = re.compile(r"<[^>]+>") #: Words a pattern's slug group can catch that name no board: the hosts' #: own path segments, the analytics subdomain Recruitee sites embed, and #: the CDN Teamtailor sites load from (cdn.teamtailor.com). _NOT_SLUGS = frozenset({"embed", "boards", "v1", "api", "www", "jobs", "careers", "apply", "careers-analytics", "cdn"}) Found = tuple[str, str, str] # (host, slug, the URL that named it) @dataclass(frozen=True) class Discovery: """What discovery learned about one company; every field may be empty.""" board: str = "" ats: str = "" ats_slug: str = "" careers_url: str = "" evidence: str = "" @property def readable(self) -> bool: return bool(self.board) # ------------------------------------------------------------------ parsing def boards_in(text: str) -> list[Found]: """Every ATS host named in `text` (a page, a script, a mail), in order of first appearance, once each. `slug` is "" for a host whose URL carries none.""" seen: set[tuple[str, str]] = set() out: list[Found] = [] hits: list[tuple[int, str, str, str]] = [] for host, pattern in HOSTS: for m in pattern.finditer(text): slug = (m.group(1) if m.groups() and m.group(1) else "").lower() if slug in _NOT_SLUGS: continue hits.append((m.start(), host, slug, m.group(0))) for _pos, host, slug, url in sorted(hits): if (host, slug) in seen: continue seen.add((host, slug)) out.append((host, slug, url)) return out def careers_links(html: str, base: str) -> list[str]: """The links on a page that read as its careers page, absolute, in page order, once each. Nothing off-site is followed: a careers page on another domain is what `boards_in` finds by its host.""" out: list[str] = [] site = httpx.URL(base).host or "" for href, label in _ANCHOR.findall(html): text = _TAGS.sub(" ", label) if not (_CAREERS_LINK.search(href) or _CAREERS_LINK.search(text)): continue url = urljoin(base, href.strip()) parsed = httpx.URL(url) if parsed.scheme not in ("http", "https"): continue host = parsed.host or "" if host != site and not host.endswith("." + _apex(site)): continue url = str(parsed.copy_with(scheme="https", fragment=None)) if url not in out: out.append(url) return out def _apex(host: str) -> str: parts = host.split(".") return ".".join(parts[-2:]) if len(parts) > 2 else host def slug_guesses(name: str, domain: str) -> list[str]: """The slugs a host most likely knows the company by: its key, the key without hyphens, and the domain's own label.""" from app.models import company_key key = company_key(name) label = domain.lower().split("/")[0] for prefix in ("www.",): label = label.removeprefix(prefix) label = label.split(".")[0] out = [] for guess in (key, key.replace("-", ""), label, label.replace("-", "")): if guess and guess not in out: out.append(guess) return out # ----------------------------------------------------------------- fetching def fetch_page(client: httpx.Client, url: str) -> tuple[str, str]: """`(final URL, text)` for one page, or `("", "")` when it did not answer with a 2xx text body within the limits. Never raises.""" try: response = client.get( url, timeout=TIMEOUT, follow_redirects=True, headers={"User-Agent": USER_AGENT, "Accept": "text/html,application/xhtml+xml,*/*;q=0.5"}) if not response.is_success: return "", "" body = response.content[:MAX_BYTES] return str(response.url), body.decode(response.encoding or "utf-8", "replace") except Exception as exc: # one page skipped, whatever went wrong log.debug("discover: %s: %s", url, exc) return "", "" #: host -> a cheap URL that answers 2xx only when the slug is a board there. #: Workable is left out: its API sits behind a rate limit that a run of #: guesses trips, and a Workable site links its board plainly anyway. _PROBES = { "greenhouse": "https://boards-api.greenhouse.io/v1/boards/{slug}", "lever": "https://api.lever.co/v0/postings/{slug}?limit=1&mode=json", "ashby": "https://api.ashbyhq.com/posting-api/job-board/{slug}", "smartrecruiters": "https://api.smartrecruiters.com/v1/companies/{slug}/postings?limit=1", "recruitee": "https://{slug}.recruitee.com/api/offers/", "personio": "https://{slug}.jobs.personio.de/xml", } def probe(client: httpx.Client, host: str, slug: str) -> bool: """Whether `host` has a board called `slug`, by its cheapest endpoint. Ashby and SmartRecruiters answer an empty listing for an unknown slug, so there a listing without jobs counts as no board.""" url = _PROBES[host].format(slug=slug) try: response = client.get(url, timeout=TIMEOUT, follow_redirects=False, headers={"User-Agent": USER_AGENT}) except Exception: return False if not response.is_success: return False body = response.content if host == "ashby": return b'"jobs":[' in body and b'"jobs":[]' not in body if host == "smartrecruiters": return b'"totalFound":' in body and b'"totalFound":0' not in body if host == "recruitee": return b'"offers":[' in body and b'"offers":[]' not in body if host == "personio": return b"" in body return True Fetch = Callable[[httpx.Client, str], tuple[str, str]] Probe = Callable[[httpx.Client, str, str], bool] def _site(domain: str) -> str: domain = domain.strip().lower() domain = re.sub(r"^https?://", "", domain).split("/")[0] return domain.removeprefix("www.") def _pages(domain: str, home_links: Iterable[str]) -> list[str]: site = _site(domain) urls = list(home_links) urls += [f"https://{site}{path}" for path in CAREERS_PATHS] urls += [f"https://{sub}{site}/" for sub in CAREERS_HOSTS] out: list[str] = [] for url in urls: if url not in out: out.append(url) return out def discover(name: str, domain: str, *, fetch: Fetch = fetch_page, probe_slug: Probe = probe, client: httpx.Client | None = None) -> Discovery: """Where `name`'s careers site draws its jobs from; see the module docstring for the order pages are read in.""" site = _site(domain) if not site: return Discovery() own = client or httpx.Client() try: return _discover(name, site, own, fetch, probe_slug) finally: if client is None: own.close() def _discover(name: str, site: str, client: httpx.Client, fetch: Fetch, probe_slug: Probe) -> Discovery: home_url, home = fetch(client, f"https://{site}/") unreadable: Discovery | None = None careers_page = "" def first_readable(found: list[Found], page: str) -> Discovery | None: """The first readable board on a page. Otherwise the first host named is remembered, with the first slug any page gives that host: a page may name teamtailor by its CDN before it links acme.teamtailor.com.""" nonlocal unreadable for host, slug, url in found: if host in READABLE and slug: return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug, careers_url=page, evidence=url) if unreadable is None and found: host, slug, url = found[0] unreadable = Discovery(ats=host, ats_slug=slug, careers_url=page, evidence=url) if unreadable is not None and not unreadable.ats_slug: for host, slug, url in found: if host == unreadable.ats and slug: unreadable = replace(unreadable, ats_slug=slug, evidence=url) break return None if home and (hit := first_readable(boards_in(home), home_url)): return hit # The home page's own careers links, then the usual paths; a careers page's # own links (a "See open positions" one level down) go to the front of the # queue. `MAX_PAGES` fetches in all, whatever the queue holds. queue = _pages(site, careers_links(home, home_url) if home else []) seen = {home_url, f"https://{site}/"} budget = MAX_PAGES while queue and budget > 0: url = queue.pop(0) if url in seen: continue seen.add(url) budget -= 1 final, text = fetch(client, url) if not text: continue seen.add(final) careers_page = careers_page or final if hit := first_readable(boards_in(text), final): return hit queue[:0] = [link for link in careers_links(text, final)[:2] if link not in seen] # A page named a host the app cannot read: that is the answer, and a # probe could only add another tenant's board of the same name. if unreadable is not None: return unreadable for slug in slug_guesses(name, site): for host in _PROBES: if probe_slug(client, host, slug): return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug, careers_url=careers_page, evidence=_PROBES[host].format(slug=slug)) return Discovery(careers_url=careers_page)