"""Board discovery: which applicant tracking system a company's careers site runs on, read off its own pages. The seed list and the LinkedIn flow both start from a company name and a website, not from a board. This module gets from the one to the other: it fetches the site's careers page, or the pages the home page links to as careers, and looks for a link into one of the hosts in `HOSTS`. A readable host (one with a module in `plugins/readers/`, which the `ats` plugin parses) becomes a `board` target the pull can read; any other host is recorded as `ats`, with its slug where the URL carries one, so the company is kept as one the app cannot read yet rather than dropped. The patterns and probes of a readable host are its module's; the hosts only named are in `_UNREADABLE` here, and one of them that gains a module is read from then on. A Teamtailor site on the company's own domain names no slug: the page only loads Teamtailor's CDN. The board there is the site's own host, and it is taken only once that host answers `/jobs.rss` with a Teamtailor feed. Only when no page names a host at all are the hosts with a cheap per-slug endpoint (`_PROBES`) probed with the company's own slug, because a site that draws its jobs from an iframe or a script leaves no link to find. Lever is probed on its EU instance as well, as `lever:eu/`. A page that names an unreadable host is the answer: a probe on a common name can hit another tenant's board. found = discover("Pleo", "pleo.io") found.board # "greenhouse:pleo", or "" when nothing readable found.ats # "greenhouse", "workday", ... or "" when unknown found.ats_slug # the name that host knows the company by, or "" found.careers_url # the page that named the host, or the careers page found.evidence # the URL that named it Only https is fetched, redirects are followed at most five hops, a body is cut at `MAX_BYTES`, and every failure is one page skipped. A company's site is content the app was pointed at by its owner, not an ATS API, so nothing here trusts what it finds beyond a host name matching a fixed pattern. """ from __future__ import annotations import logging import re from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass, replace from types import ModuleType from urllib.parse import urljoin import httpx from desk.sources import USER_AGENT from plugins import readers log = logging.getLogger(__name__) TIMEOUT = 10.0 MAX_BYTES = 2 << 20 #: How many pages of one site are read before giving up on links. MAX_PAGES = 6 #: The hosts the `ats` plugin parses; a match on one of these is a `board`. READABLE: tuple[str, ...] = tuple(readers.HOSTS) #: host -> pattern over a URL, for the hosts discovery names but the app #: cannot read. Group 1, where there is one, is the slug the host names the #: company by, kept for the day a parser exists; a host with no group is #: recognised but its board has no slug the app could use. _UNREADABLE: tuple[tuple[str, str], ...] = ( ("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"), ("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"), ("icims", r"https?://([a-z0-9-]+)\.icims\.com"), ("taleo", r"\.taleo\.net"), ("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"), ("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"), ("join", r"join\.com/companies/([a-z0-9-]+)"), ("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"), ("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"), ("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"), ("gem", r"jobs\.gem\.com/([a-z0-9-]+)"), ("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"), ("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"), ("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"), ("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"), ("eightfold", r"\.eightfold\.ai"), ("phenom", r"phenompeople\.com|phenom\.com/careers"), ("avature", r"\.avature\.net"), ("oracle", r"oraclecloud\.com/hcmUI"), ("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"), ("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"), ) def _hosts(modules: Mapping[str, ModuleType], unreadable: Iterable[tuple[str, str]]) -> tuple[tuple[str, re.Pattern[str]], ...]: """Every host's patterns, compiled: the readable hosts' from their modules first, so the first match at one place on a page is the one the app can read, then the unreadable ones. An unreadable host that has gained a module is left out here, since its module's patterns replace it.""" out = [(name, pattern) for name, module in modules.items() for pattern in module.PATTERNS] out += [(host, pattern) for host, pattern in unreadable if host not in modules] return tuple((host, re.compile(pattern, re.IGNORECASE)) for host, pattern in out) #: host -> pattern over a URL, every host discovery can name. The slug is #: group 1, or, for a pattern with more groups, the groups that matched #: joined by "/" (Lever's `eu/`). HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = _hosts(readers.HOSTS, _UNREADABLE) #: The hosts whose slugs keep the case the URL wrote them in, by their #: module's `KEEP_CASE`: a Workday site is `LEGO_External`, and both its #: listing and its robots.txt paths are case-sensitive. Every other slug is #: lowercased, which is how a page that writes `BoschGroup` and one that #: writes `boschgroup` name one board. _KEEP_CASE = frozenset(name for name, module in readers.HOSTS.items() if getattr(module, "KEEP_CASE", False)) #: What a Teamtailor feed carries and a page that only looks like one does not: #: the namespace of `tt:locations`, declared on `` even when no role is open. TEAMTAILOR_FEED = "https://teamtailor.com/locations" #: Paths a careers page usually lives at, tried after the home page's own #: links so the site's word for it wins over a guess. CAREERS_PATHS = ("/careers", "/careers/", "/jobs", "/en/careers", "/company/careers", "/about/careers", "/join-us", "/careers/jobs", "/en/jobs") #: Subdomains careers pages move to. CAREERS_HOSTS = ("careers.", "jobs.", "career.") #: A link text or href that points at a careers page. _CAREERS_LINK = re.compile( r"career|jobs|join[-_ ]?us|work[-_ ]?with[-_ ]?us|open[-_ ]?positions|vacanc|" r"we[-_ ]?are[-_ ]?hiring|ledige[-_ ]?stillinger|stellen|karriere|lediga[-_ ]?jobb", re.IGNORECASE) _HREF = re.compile(r"""(?:href|src|action|content)\s*=\s*["']([^"'#][^"']*)["']""", re.IGNORECASE) _ANCHOR = re.compile(r"]*href\s*=\s*[\"']([^\"'#][^\"']*)[\"'][^>]*>(.*?)", re.IGNORECASE | re.DOTALL) _TAGS = re.compile(r"<[^>]+>") #: Words a pattern's group 1 can catch that name no board: the hosts' #: own path segments, the analytics subdomain Recruitee sites embed, and #: the CDN and the admin app Teamtailor sites load from (cdn.teamtailor.com, #: app.teamtailor.com). _NOT_SLUGS = frozenset({"embed", "boards", "v1", "api", "www", "jobs", "careers", "apply", "careers-analytics", "cdn", "app"}) Found = tuple[str, str, str] # (host, slug, the URL that named it) @dataclass(frozen=True) class Discovery: """What discovery learned about one company; every field may be empty.""" board: str = "" ats: str = "" ats_slug: str = "" careers_url: str = "" evidence: str = "" @property def readable(self) -> bool: return bool(self.board) # ------------------------------------------------------------------ parsing def boards_in(text: str) -> list[Found]: """Every ATS host named in `text` (a page, a script, a mail), in order of first appearance, once each. `slug` is "" for a host whose URL carries none.""" seen: set[tuple[str, str]] = set() out: list[Found] = [] hits: list[tuple[int, str, str, str]] = [] for host, pattern in HOSTS: for m in pattern.finditer(text): # Only group 1 names a board or a host's own path word; a later # group may be anything, a Workday site called `jobs` included. if m.re.groups and (m.group(1) or "").lower() in _NOT_SLUGS: continue slug = "/".join(part for part in m.groups() if part) if host not in _KEEP_CASE: slug = slug.lower() hits.append((m.start(), host, slug, m.group(0))) for _pos, host, slug, url in sorted(hits): if (host, slug) in seen: continue seen.add((host, slug)) out.append((host, slug, url)) return out def careers_links(html: str, base: str) -> list[str]: """The links on a page that read as its careers page, absolute, in page order, once each. Nothing off-site is followed: a careers page on another domain is what `boards_in` finds by its host.""" out: list[str] = [] site = httpx.URL(base).host or "" for href, label in _ANCHOR.findall(html): text = _TAGS.sub(" ", label) if not (_CAREERS_LINK.search(href) or _CAREERS_LINK.search(text)): continue url = urljoin(base, href.strip()) parsed = httpx.URL(url) if parsed.scheme not in ("http", "https"): continue host = parsed.host or "" if host != site and not host.endswith("." + _apex(site)): continue url = str(parsed.copy_with(scheme="https", fragment=None)) if url not in out: out.append(url) return out def _apex(host: str) -> str: parts = host.split(".") return ".".join(parts[-2:]) if len(parts) > 2 else host def slug_guesses(name: str, domain: str) -> list[str]: """The slugs a host most likely knows the company by: its key, the key without hyphens, and the domain's own label.""" from app.models import company_key key = company_key(name) label = domain.lower().split("/")[0] for prefix in ("www.",): label = label.removeprefix(prefix) label = label.split(".")[0] out = [] for guess in (key, key.replace("-", ""), label, label.replace("-", "")): if guess and guess not in out: out.append(guess) return out # ----------------------------------------------------------------- fetching def fetch_page(client: httpx.Client, url: str) -> tuple[str, str]: """`(final URL, text)` for one page, or `("", "")` when it did not answer with a 2xx text body within the limits. Never raises.""" try: response = client.get( url, timeout=TIMEOUT, follow_redirects=True, headers={"User-Agent": USER_AGENT, "Accept": "text/html,application/xhtml+xml,*/*;q=0.5"}) if not response.is_success: return "", "" body = response.content[:MAX_BYTES] return str(response.url), body.decode(response.encoding or "utf-8", "replace") except Exception as exc: # one page skipped, whatever went wrong log.debug("discover: %s: %s", url, exc) return "", "" #: host -> a cheap URL that answers 2xx only when the slug is a board there, #: from each module's `probe_url`. Workable has none: its API sits behind a #: rate limit that a run of guesses trips, and a Workable site links its #: board plainly anyway. _PROBES: dict[str, Callable[[str], str]] = { name: module.probe_url for name, module in readers.HOSTS.items() if hasattr(module, "probe_url")} def _probe_slugs(host: str, guess: str) -> tuple[str, ...]: """The slugs a guessed name is probed as on `host`: the name itself, and for Lever the name on its EU instance too (the module's `probe_slugs`).""" module = readers.HOSTS[host] return module.probe_slugs(guess) if hasattr(module, "probe_slugs") else (guess,) def probe(client: httpx.Client, host: str, slug: str) -> bool: """Whether `host` has a board called `slug`, by its cheapest endpoint. A host that answers an unknown slug with an empty listing (Ashby, SmartRecruiters, ...) says in its module's `probe_ok` what a listing with a board looks like.""" url = _PROBES[host](slug) try: response = client.get(url, timeout=TIMEOUT, follow_redirects=False, headers={"User-Agent": USER_AGENT}) except Exception: return False if not response.is_success: return False module = readers.HOSTS[host] return module.probe_ok(response.content) if hasattr(module, "probe_ok") else True Fetch = Callable[[httpx.Client, str], tuple[str, str]] Probe = Callable[[httpx.Client, str, str], bool] def _site(domain: str) -> str: domain = domain.strip().lower() domain = re.sub(r"^https?://", "", domain).split("/")[0] return domain.removeprefix("www.") def _pages(domain: str, home_links: Iterable[str]) -> list[str]: site = _site(domain) urls = list(home_links) urls += [f"https://{site}{path}" for path in CAREERS_PATHS] urls += [f"https://{sub}{site}/" for sub in CAREERS_HOSTS] out: list[str] = [] for url in urls: if url not in out: out.append(url) return out def discover(name: str, domain: str, *, fetch: Fetch = fetch_page, probe_slug: Probe = probe, client: httpx.Client | None = None) -> Discovery: """Where `name`'s careers site draws its jobs from; see the module docstring for the order pages are read in.""" site = _site(domain) if not site: return Discovery() own = client or httpx.Client() try: return _discover(name, site, own, fetch, probe_slug) finally: if client is None: own.close() def _discover(name: str, site: str, client: httpx.Client, fetch: Fetch, probe_slug: Probe) -> Discovery: home_url, home = fetch(client, f"https://{site}/") unreadable: Discovery | None = None careers_page = "" own_sites: list[tuple[str, str]] = [] # (host, page) that loaded Teamtailor's CDN def first_readable(found: list[Found], page: str) -> Discovery | None: """The first readable board on a page. Otherwise the first host named is remembered, with the first slug any page gives that host: a page may name teamtailor by its CDN before it links acme.teamtailor.com.""" nonlocal unreadable for host, slug, url in found: site_host = httpx.URL(page).host if page else "" if host == "teamtailor" and not slug and site_host and (site_host, page) not in own_sites: own_sites.append((site_host, page)) if host in READABLE and slug: return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug, careers_url=page, evidence=url) if unreadable is None and found: host, slug, url = found[0] unreadable = Discovery(ats=host, ats_slug=slug, careers_url=page, evidence=url) if unreadable is not None and not unreadable.ats_slug: for host, slug, url in found: if host == unreadable.ats and slug: unreadable = replace(unreadable, ats_slug=slug, evidence=url) break return None if home and (hit := first_readable(boards_in(home), home_url)): return hit # The home page's own careers links, then the usual paths; a careers page's # own links (a "See open positions" one level down) go to the front of the # queue. `MAX_PAGES` fetches in all, whatever the queue holds. queue = _pages(site, careers_links(home, home_url) if home else []) seen = {home_url, f"https://{site}/"} budget = MAX_PAGES while queue and budget > 0: url = queue.pop(0) if url in seen: continue seen.add(url) budget -= 1 final, text = fetch(client, url) if not text: continue seen.add(final) careers_page = careers_page or final if hit := first_readable(boards_in(text), final): return hit queue[:0] = [link for link in careers_links(text, final)[:2] if link not in seen] # A page loaded Teamtailor's CDN and linked no *.teamtailor.com: the site # itself is the board, if it serves the feed. for site_host, page in own_sites: feed = f"https://{site_host}/jobs.rss" _final, text = fetch(client, feed) if TEAMTAILOR_FEED in text: return Discovery(board=f"teamtailor:{site_host}", ats="teamtailor", ats_slug=site_host, careers_url=page, evidence=feed) # A page named a host the app cannot read: that is the answer, and a # probe could only add another tenant's board of the same name. if unreadable is not None: return unreadable for guess in slug_guesses(name, site): for host in _PROBES: for slug in _probe_slugs(host, guess): if probe_slug(client, host, slug): return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug, careers_url=careers_page, evidence=_PROBES[host](slug)) return Discovery(careers_url=careers_page)