Files
jobboard/app/jobs/discover.py
T

395 lines
17 KiB
Python

"""Board discovery: which applicant tracking system a company's careers site
runs on, read off its own pages.
The seed list and the LinkedIn flow both start from a company name and a
website, not from a board. This module gets from the one to the other: it
fetches the site's careers page, or the pages the home page links to as
careers, and looks for a link into one of the hosts in `HOSTS`. A readable
host (one with a module in `plugins/atshosts/`, which the `ats` plugin
parses) becomes a `board` target the pull can read; any other host is
recorded as `ats`, with its slug where the URL carries one, so the company is
kept as one the app cannot read yet rather than dropped. The patterns and
probes of a readable host are its module's; the hosts only named are in
`_UNREADABLE` here, and one of them that gains a module is read from then on.
A Teamtailor site on the company's own domain names no slug: the page only
loads Teamtailor's CDN. The board there is the site's own host, and it is
taken only once that host answers `/jobs.rss` with a Teamtailor feed.
Only when no page names a host at all are the hosts with a cheap per-slug
endpoint (`_PROBES`) probed with the company's own slug, because a site that
draws its jobs from an iframe or a script leaves no link to find. Lever is
probed on its EU instance as well, as `lever:eu/<slug>`. A page that
names an unreadable host is the answer: a probe on a common name can hit
another tenant's board.
found = discover("Pleo", "pleo.io")
found.board # "greenhouse:pleo", or "" when nothing readable
found.ats # "greenhouse", "workday", ... or "" when unknown
found.ats_slug # the name that host knows the company by, or ""
found.careers_url # the page that named the host, or the careers page
found.evidence # the URL that named it
Only https is fetched, redirects are followed at most five hops, a body is
cut at `MAX_BYTES`, and every failure is one page skipped. A company's site
is content the app was pointed at by its owner, not an ATS API, so nothing
here trusts what it finds beyond a host name matching a fixed pattern.
"""
from __future__ import annotations
import logging
import re
from collections.abc import Callable, Iterable, Mapping
from dataclasses import dataclass, replace
from types import ModuleType
from urllib.parse import urljoin
import httpx
from desk.sources import USER_AGENT
from plugins import atshosts
log = logging.getLogger(__name__)
TIMEOUT = 10.0
MAX_BYTES = 2 << 20
#: How many pages of one site are read before giving up on links.
MAX_PAGES = 6
#: The hosts the `ats` plugin parses; a match on one of these is a `board`.
READABLE: tuple[str, ...] = tuple(atshosts.HOSTS)
#: host -> pattern over a URL, for the hosts discovery names but the app
#: cannot read. Group 1, where there is one, is the slug the host names the
#: company by, kept for the day a parser exists; a host with no group is
#: recognised but its board has no slug the app could use.
_UNREADABLE: tuple[tuple[str, str], ...] = (
("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"),
("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"),
("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
("taleo", r"\.taleo\.net"),
("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"),
("join", r"join\.com/companies/([a-z0-9-]+)"),
("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"),
("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"),
("gem", r"jobs\.gem\.com/([a-z0-9-]+)"),
("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
("eightfold", r"\.eightfold\.ai"),
("phenom", r"phenompeople\.com|phenom\.com/careers"),
("avature", r"\.avature\.net"),
("oracle", r"oraclecloud\.com/hcmUI"),
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"),
)
def _hosts(modules: Mapping[str, ModuleType],
unreadable: Iterable[tuple[str, str]]) -> tuple[tuple[str, re.Pattern[str]], ...]:
"""Every host's patterns, compiled: the readable hosts' from their
modules first, so the first match at one place on a page is the one the
app can read, then the unreadable ones. An unreadable host that has
gained a module is left out here, since its module's patterns replace
it."""
out = [(name, pattern) for name, module in modules.items() for pattern in module.PATTERNS]
out += [(host, pattern) for host, pattern in unreadable if host not in modules]
return tuple((host, re.compile(pattern, re.IGNORECASE)) for host, pattern in out)
#: host -> pattern over a URL, every host discovery can name. The slug is
#: group 1, or, for a pattern with more groups, the groups that matched
#: joined by "/" (Lever's `eu/<site>`).
HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = _hosts(atshosts.HOSTS, _UNREADABLE)
#: The hosts whose slugs keep the case the URL wrote them in, by their
#: module's `KEEP_CASE`: a Workday site is `LEGO_External`, and both its
#: listing and its robots.txt paths are case-sensitive. Every other slug is
#: lowercased, which is how a page that writes `BoschGroup` and one that
#: writes `boschgroup` name one board.
_KEEP_CASE = frozenset(name for name, module in atshosts.HOSTS.items()
if getattr(module, "KEEP_CASE", False))
#: What a Teamtailor feed carries and a page that only looks like one does not:
#: the namespace of `tt:locations`, declared on `<rss>` even when no role is open.
TEAMTAILOR_FEED = "https://teamtailor.com/locations"
#: Paths a careers page usually lives at, tried after the home page's own
#: links so the site's word for it wins over a guess.
CAREERS_PATHS = ("/careers", "/careers/", "/jobs", "/en/careers", "/company/careers",
"/about/careers", "/join-us", "/careers/jobs", "/en/jobs")
#: Subdomains careers pages move to.
CAREERS_HOSTS = ("careers.", "jobs.", "career.")
#: A link text or href that points at a careers page.
_CAREERS_LINK = re.compile(
r"career|jobs|join[-_ ]?us|work[-_ ]?with[-_ ]?us|open[-_ ]?positions|vacanc|"
r"we[-_ ]?are[-_ ]?hiring|ledige[-_ ]?stillinger|stellen|karriere|lediga[-_ ]?jobb",
re.IGNORECASE)
_HREF = re.compile(r"""(?:href|src|action|content)\s*=\s*["']([^"'#][^"']*)["']""",
re.IGNORECASE)
_ANCHOR = re.compile(r"<a\b[^>]*href\s*=\s*[\"']([^\"'#][^\"']*)[\"'][^>]*>(.*?)</a>",
re.IGNORECASE | re.DOTALL)
_TAGS = re.compile(r"<[^>]+>")
#: Words a pattern's group 1 can catch that name no board: the hosts'
#: own path segments, the analytics subdomain Recruitee sites embed, and
#: the CDN and the admin app Teamtailor sites load from (cdn.teamtailor.com,
#: app.teamtailor.com).
_NOT_SLUGS = frozenset({"embed", "boards", "v1", "api", "www", "jobs", "careers", "apply",
"careers-analytics", "cdn", "app"})
Found = tuple[str, str, str] # (host, slug, the URL that named it)
@dataclass(frozen=True)
class Discovery:
"""What discovery learned about one company; every field may be empty."""
board: str = ""
ats: str = ""
ats_slug: str = ""
careers_url: str = ""
evidence: str = ""
@property
def readable(self) -> bool:
return bool(self.board)
# ------------------------------------------------------------------ parsing
def boards_in(text: str) -> list[Found]:
"""Every ATS host named in `text` (a page, a script, a mail), in order of
first appearance, once each. `slug` is "" for a host whose URL carries
none."""
seen: set[tuple[str, str]] = set()
out: list[Found] = []
hits: list[tuple[int, str, str, str]] = []
for host, pattern in HOSTS:
for m in pattern.finditer(text):
# Only group 1 names a board or a host's own path word; a later
# group may be anything, a Workday site called `jobs` included.
if m.re.groups and (m.group(1) or "").lower() in _NOT_SLUGS:
continue
slug = "/".join(part for part in m.groups() if part)
if host not in _KEEP_CASE:
slug = slug.lower()
hits.append((m.start(), host, slug, m.group(0)))
for _pos, host, slug, url in sorted(hits):
if (host, slug) in seen:
continue
seen.add((host, slug))
out.append((host, slug, url))
return out
def careers_links(html: str, base: str) -> list[str]:
"""The links on a page that read as its careers page, absolute, in page
order, once each. Nothing off-site is followed: a careers page on another
domain is what `boards_in` finds by its host."""
out: list[str] = []
site = httpx.URL(base).host or ""
for href, label in _ANCHOR.findall(html):
text = _TAGS.sub(" ", label)
if not (_CAREERS_LINK.search(href) or _CAREERS_LINK.search(text)):
continue
url = urljoin(base, href.strip())
parsed = httpx.URL(url)
if parsed.scheme not in ("http", "https"):
continue
host = parsed.host or ""
if host != site and not host.endswith("." + _apex(site)):
continue
url = str(parsed.copy_with(scheme="https", fragment=None))
if url not in out:
out.append(url)
return out
def _apex(host: str) -> str:
parts = host.split(".")
return ".".join(parts[-2:]) if len(parts) > 2 else host
def slug_guesses(name: str, domain: str) -> list[str]:
"""The slugs a host most likely knows the company by: its key, the key
without hyphens, and the domain's own label."""
from app.models import company_key
key = company_key(name)
label = domain.lower().split("/")[0]
for prefix in ("www.",):
label = label.removeprefix(prefix)
label = label.split(".")[0]
out = []
for guess in (key, key.replace("-", ""), label, label.replace("-", "")):
if guess and guess not in out:
out.append(guess)
return out
# ----------------------------------------------------------------- fetching
def fetch_page(client: httpx.Client, url: str) -> tuple[str, str]:
"""`(final URL, text)` for one page, or `("", "")` when it did not answer
with a 2xx text body within the limits. Never raises."""
try:
response = client.get(
url, timeout=TIMEOUT, follow_redirects=True,
headers={"User-Agent": USER_AGENT,
"Accept": "text/html,application/xhtml+xml,*/*;q=0.5"})
if not response.is_success:
return "", ""
body = response.content[:MAX_BYTES]
return str(response.url), body.decode(response.encoding or "utf-8", "replace")
except Exception as exc: # one page skipped, whatever went wrong
log.debug("discover: %s: %s", url, exc)
return "", ""
#: host -> a cheap URL that answers 2xx only when the slug is a board there,
#: from each module's `probe_url`. Workable has none: its API sits behind a
#: rate limit that a run of guesses trips, and a Workable site links its
#: board plainly anyway.
_PROBES: dict[str, Callable[[str], str]] = {
name: module.probe_url for name, module in atshosts.HOSTS.items()
if hasattr(module, "probe_url")}
def _probe_slugs(host: str, guess: str) -> tuple[str, ...]:
"""The slugs a guessed name is probed as on `host`: the name itself, and
for Lever the name on its EU instance too (the module's `probe_slugs`)."""
module = atshosts.HOSTS[host]
return module.probe_slugs(guess) if hasattr(module, "probe_slugs") else (guess,)
def probe(client: httpx.Client, host: str, slug: str) -> bool:
"""Whether `host` has a board called `slug`, by its cheapest endpoint.
A host that answers an unknown slug with an empty listing (Ashby,
SmartRecruiters, ...) says in its module's `probe_ok` what a listing
with a board looks like."""
url = _PROBES[host](slug)
try:
response = client.get(url, timeout=TIMEOUT, follow_redirects=False,
headers={"User-Agent": USER_AGENT})
except Exception:
return False
if not response.is_success:
return False
module = atshosts.HOSTS[host]
return module.probe_ok(response.content) if hasattr(module, "probe_ok") else True
Fetch = Callable[[httpx.Client, str], tuple[str, str]]
Probe = Callable[[httpx.Client, str, str], bool]
def _site(domain: str) -> str:
domain = domain.strip().lower()
domain = re.sub(r"^https?://", "", domain).split("/")[0]
return domain.removeprefix("www.")
def _pages(domain: str, home_links: Iterable[str]) -> list[str]:
site = _site(domain)
urls = list(home_links)
urls += [f"https://{site}{path}" for path in CAREERS_PATHS]
urls += [f"https://{sub}{site}/" for sub in CAREERS_HOSTS]
out: list[str] = []
for url in urls:
if url not in out:
out.append(url)
return out
def discover(name: str, domain: str, *, fetch: Fetch = fetch_page,
probe_slug: Probe = probe, client: httpx.Client | None = None) -> Discovery:
"""Where `name`'s careers site draws its jobs from; see the module
docstring for the order pages are read in."""
site = _site(domain)
if not site:
return Discovery()
own = client or httpx.Client()
try:
return _discover(name, site, own, fetch, probe_slug)
finally:
if client is None:
own.close()
def _discover(name: str, site: str, client: httpx.Client, fetch: Fetch,
probe_slug: Probe) -> Discovery:
home_url, home = fetch(client, f"https://{site}/")
unreadable: Discovery | None = None
careers_page = ""
own_sites: list[tuple[str, str]] = [] # (host, page) that loaded Teamtailor's CDN
def first_readable(found: list[Found], page: str) -> Discovery | None:
"""The first readable board on a page. Otherwise the first host named
is remembered, with the first slug any page gives that host: a page
may name teamtailor by its CDN before it links acme.teamtailor.com."""
nonlocal unreadable
for host, slug, url in found:
site_host = httpx.URL(page).host if page else ""
if host == "teamtailor" and not slug and site_host and (site_host, page) not in own_sites:
own_sites.append((site_host, page))
if host in READABLE and slug:
return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug,
careers_url=page, evidence=url)
if unreadable is None and found:
host, slug, url = found[0]
unreadable = Discovery(ats=host, ats_slug=slug, careers_url=page, evidence=url)
if unreadable is not None and not unreadable.ats_slug:
for host, slug, url in found:
if host == unreadable.ats and slug:
unreadable = replace(unreadable, ats_slug=slug, evidence=url)
break
return None
if home and (hit := first_readable(boards_in(home), home_url)):
return hit
# The home page's own careers links, then the usual paths; a careers page's
# own links (a "See open positions" one level down) go to the front of the
# queue. `MAX_PAGES` fetches in all, whatever the queue holds.
queue = _pages(site, careers_links(home, home_url) if home else [])
seen = {home_url, f"https://{site}/"}
budget = MAX_PAGES
while queue and budget > 0:
url = queue.pop(0)
if url in seen:
continue
seen.add(url)
budget -= 1
final, text = fetch(client, url)
if not text:
continue
seen.add(final)
careers_page = careers_page or final
if hit := first_readable(boards_in(text), final):
return hit
queue[:0] = [link for link in careers_links(text, final)[:2] if link not in seen]
# A page loaded Teamtailor's CDN and linked no *.teamtailor.com: the site
# itself is the board, if it serves the feed.
for site_host, page in own_sites:
feed = f"https://{site_host}/jobs.rss"
_final, text = fetch(client, feed)
if TEAMTAILOR_FEED in text:
return Discovery(board=f"teamtailor:{site_host}", ats="teamtailor",
ats_slug=site_host, careers_url=page, evidence=feed)
# A page named a host the app cannot read: that is the answer, and a
# probe could only add another tenant's board of the same name.
if unreadable is not None:
return unreadable
for guess in slug_guesses(name, site):
for host in _PROBES:
for slug in _probe_slugs(host, guess):
if probe_slug(client, host, slug):
return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug,
careers_url=careers_page, evidence=_PROBES[host](slug))
return Discovery(careers_url=careers_page)