Merge branch 'seed-europe'
# Conflicts: # README.md
This commit is contained in:
commit
3aee48997c
19 files changed
+3362
-1394
No files matched your search
@@ -10,13 +10,14 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
|
||||
offers, the `pull` job a schedule fires, and that schedule
|
||||
sources.py what a pull reads, changed by hand: boards and searches
|
||||
added, switched on or off, pruned by region
|
||||
discover.py which ATS a company's careers site runs on, from its pages
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.jobs import cascade, embed, extract, pull, score, sources, summarise
|
||||
from app.jobs import cascade, discover, embed, extract, pull, score, sources, summarise
|
||||
from app.jobs.pull import search_target, seed_searches
|
||||
from app.jobs.score import seed_agents
|
||||
|
||||
@@ -35,5 +36,5 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]:
|
||||
return config, Store(config.database)
|
||||
|
||||
|
||||
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "embed", "extract", "pull", "score",
|
||||
"search_target", "seed_agents", "seed_searches", "sources", "summarise"]
|
||||
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "pull",
|
||||
"score", "search_target", "seed_agents", "seed_searches", "sources", "summarise"]
|
||||
@@ -0,0 +1,343 @@
|
||||
"""Board discovery: which applicant tracking system a company's careers site
|
||||
runs on, read off its own pages.
|
||||
|
||||
The seed list and the LinkedIn flow both start from a company name and a
|
||||
website, not from a board. This module gets from the one to the other: it
|
||||
fetches the site's careers page, or the pages the home page links to as
|
||||
careers, and looks for a link into one of the hosts in `HOSTS`. A readable
|
||||
host (one the `ats` plugin parses) becomes a `board` target the pull can
|
||||
read; any other host is recorded as `ats`, so the company is kept as one the
|
||||
app cannot read yet rather than dropped.
|
||||
|
||||
When no page names a host, the three hosts with a cheap per-slug endpoint
|
||||
(Greenhouse, Lever, Ashby) are probed with the company's own slug, because a
|
||||
site that draws its jobs from an iframe or a script leaves no link to find.
|
||||
|
||||
found = discover("Pleo", "pleo.io")
|
||||
found.board # "greenhouse:pleo", or "" when nothing readable
|
||||
found.ats # "greenhouse", "workday", ... or "" when unknown
|
||||
found.careers_url # the page that named the host, or the careers page
|
||||
found.evidence # the URL that named it
|
||||
|
||||
Only https is fetched, redirects are followed at most five hops, a body is
|
||||
cut at `MAX_BYTES`, and every failure is one page skipped. A company's site
|
||||
is content the app was pointed at by its owner, not an ATS API, so nothing
|
||||
here trusts what it finds beyond a host name matching a fixed pattern.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import httpx
|
||||
|
||||
from desk.sources import USER_AGENT
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
TIMEOUT = 10.0
|
||||
MAX_BYTES = 2 << 20
|
||||
#: How many pages of one site are read before giving up on links.
|
||||
MAX_PAGES = 6
|
||||
|
||||
#: The hosts the `ats` plugin parses; a match on one of these is a `board`.
|
||||
READABLE = ("greenhouse", "lever", "ashby", "workable", "smartrecruiters",
|
||||
"recruitee", "personio")
|
||||
|
||||
#: host -> pattern over a URL. Group 1, where there is one, is the slug the
|
||||
#: host names the company by; a host with no group is recognised but its
|
||||
#: board has no slug the app could use. Order matters only for readability:
|
||||
#: the readable hosts come first so the first match on a page is the one the
|
||||
#: app can read.
|
||||
HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = tuple(
|
||||
(host, re.compile(pattern, re.IGNORECASE)) for host, pattern in (
|
||||
("greenhouse", r"(?:boards|job-boards|api)\.(?:eu\.)?greenhouse\.io/(?:v1/boards/|embed/job_board\?[^\"'\s]*for=)?([A-Za-z0-9_-]+)"),
|
||||
("greenhouse", r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)"),
|
||||
("lever", r"jobs\.(?:eu\.)?lever\.co/([A-Za-z0-9_-]+)"),
|
||||
("lever", r"api\.(?:eu\.)?lever\.co/v0/postings/([A-Za-z0-9_-]+)"),
|
||||
("ashby", r"jobs\.ashbyhq\.com/([A-Za-z0-9_.-]+)"),
|
||||
("ashby", r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)"),
|
||||
("workable", r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_-]+)"),
|
||||
("workable", r"https?://(?!apply\.|www\.)([a-z0-9-]+)\.workable\.com"),
|
||||
("smartrecruiters", r"(?:jobs|careers|api)\.smartrecruiters\.com/(?:v1/companies/)?([A-Za-z0-9_-]+)"),
|
||||
("recruitee", r"https?://([a-z0-9-]+)\.recruitee\.com"),
|
||||
("personio", r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)"),
|
||||
# Not readable. The slug, where the URL has one, is kept for the day
|
||||
# a parser exists.
|
||||
("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"),
|
||||
("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"),
|
||||
("teamtailor", r"https?://([a-z0-9-]+)\.teamtailor\.com"),
|
||||
("teamtailor", r"cdn\.teamtailor\.com|teamtailor-cdn|teamtailor\.com/assets"),
|
||||
("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
|
||||
("taleo", r"\.taleo\.net"),
|
||||
("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
|
||||
("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"),
|
||||
("join", r"join\.com/companies/([a-z0-9-]+)"),
|
||||
("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
|
||||
("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"),
|
||||
("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"),
|
||||
("gem", r"jobs\.gem\.com/([a-z0-9-]+)"),
|
||||
("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
|
||||
("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
|
||||
("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
|
||||
("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
|
||||
("eightfold", r"\.eightfold\.ai"),
|
||||
("phenom", r"phenompeople\.com|phenom\.com/careers"),
|
||||
("avature", r"\.avature\.net"),
|
||||
("oracle", r"oraclecloud\.com/hcmUI"),
|
||||
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
|
||||
("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"),
|
||||
))
|
||||
|
||||
#: Paths a careers page usually lives at, tried after the home page's own
|
||||
#: links so the site's word for it wins over a guess.
|
||||
CAREERS_PATHS = ("/careers", "/careers/", "/jobs", "/en/careers", "/company/careers",
|
||||
"/about/careers", "/join-us", "/careers/jobs", "/en/jobs")
|
||||
#: Subdomains careers pages move to.
|
||||
CAREERS_HOSTS = ("careers.", "jobs.", "career.")
|
||||
|
||||
#: A link text or href that points at a careers page.
|
||||
_CAREERS_LINK = re.compile(
|
||||
r"career|jobs|join[-_ ]?us|work[-_ ]?with[-_ ]?us|open[-_ ]?positions|vacanc|"
|
||||
r"we[-_ ]?are[-_ ]?hiring|ledige[-_ ]?stillinger|stellen|karriere|lediga[-_ ]?jobb",
|
||||
re.IGNORECASE)
|
||||
_HREF = re.compile(r"""(?:href|src|action|content)\s*=\s*["']([^"'#][^"']*)["']""",
|
||||
re.IGNORECASE)
|
||||
_ANCHOR = re.compile(r"<a\b[^>]*href\s*=\s*[\"']([^\"'#][^\"']*)[\"'][^>]*>(.*?)</a>",
|
||||
re.IGNORECASE | re.DOTALL)
|
||||
_TAGS = re.compile(r"<[^>]+>")
|
||||
|
||||
#: Words a pattern's slug group can catch that name no board: the hosts'
|
||||
#: own path segments, and the analytics subdomain Recruitee sites embed.
|
||||
_NOT_SLUGS = frozenset({"embed", "boards", "v1", "api", "www", "jobs", "careers", "apply",
|
||||
"careers-analytics"})
|
||||
|
||||
Found = tuple[str, str, str] # (host, slug, the URL that named it)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Discovery:
|
||||
"""What discovery learned about one company; every field may be empty."""
|
||||
|
||||
board: str = ""
|
||||
ats: str = ""
|
||||
careers_url: str = ""
|
||||
evidence: str = ""
|
||||
|
||||
@property
|
||||
def readable(self) -> bool:
|
||||
return bool(self.board)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ parsing
|
||||
|
||||
def boards_in(text: str) -> list[Found]:
|
||||
"""Every ATS host named in `text` (a page, a script, a mail), in order of
|
||||
first appearance, once each. `slug` is "" for a host whose URL carries
|
||||
none."""
|
||||
seen: set[tuple[str, str]] = set()
|
||||
out: list[Found] = []
|
||||
hits: list[tuple[int, str, str, str]] = []
|
||||
for host, pattern in HOSTS:
|
||||
for m in pattern.finditer(text):
|
||||
slug = (m.group(1) if m.groups() and m.group(1) else "").lower()
|
||||
if slug in _NOT_SLUGS:
|
||||
continue
|
||||
hits.append((m.start(), host, slug, m.group(0)))
|
||||
for _pos, host, slug, url in sorted(hits):
|
||||
if (host, slug) in seen:
|
||||
continue
|
||||
seen.add((host, slug))
|
||||
out.append((host, slug, url))
|
||||
return out
|
||||
|
||||
|
||||
def careers_links(html: str, base: str) -> list[str]:
|
||||
"""The links on a page that read as its careers page, absolute, in page
|
||||
order, once each. Nothing off-site is followed: a careers page on another
|
||||
domain is what `boards_in` finds by its host."""
|
||||
out: list[str] = []
|
||||
site = httpx.URL(base).host or ""
|
||||
for href, label in _ANCHOR.findall(html):
|
||||
text = _TAGS.sub(" ", label)
|
||||
if not (_CAREERS_LINK.search(href) or _CAREERS_LINK.search(text)):
|
||||
continue
|
||||
url = urljoin(base, href.strip())
|
||||
parsed = httpx.URL(url)
|
||||
if parsed.scheme not in ("http", "https"):
|
||||
continue
|
||||
host = parsed.host or ""
|
||||
if host != site and not host.endswith("." + _apex(site)):
|
||||
continue
|
||||
url = str(parsed.copy_with(scheme="https", fragment=None))
|
||||
if url not in out:
|
||||
out.append(url)
|
||||
return out
|
||||
|
||||
|
||||
def _apex(host: str) -> str:
|
||||
parts = host.split(".")
|
||||
return ".".join(parts[-2:]) if len(parts) > 2 else host
|
||||
|
||||
|
||||
def slug_guesses(name: str, domain: str) -> list[str]:
|
||||
"""The slugs a host most likely knows the company by: its key, the key
|
||||
without hyphens, and the domain's own label."""
|
||||
from app.models import company_key
|
||||
|
||||
key = company_key(name)
|
||||
label = domain.lower().split("/")[0]
|
||||
for prefix in ("www.",):
|
||||
label = label.removeprefix(prefix)
|
||||
label = label.split(".")[0]
|
||||
out = []
|
||||
for guess in (key, key.replace("-", ""), label, label.replace("-", "")):
|
||||
if guess and guess not in out:
|
||||
out.append(guess)
|
||||
return out
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- fetching
|
||||
|
||||
def fetch_page(client: httpx.Client, url: str) -> tuple[str, str]:
|
||||
"""`(final URL, text)` for one page, or `("", "")` when it did not answer
|
||||
with a 2xx text body within the limits. Never raises."""
|
||||
try:
|
||||
response = client.get(
|
||||
url, timeout=TIMEOUT, follow_redirects=True,
|
||||
headers={"User-Agent": USER_AGENT,
|
||||
"Accept": "text/html,application/xhtml+xml,*/*;q=0.5"})
|
||||
if not response.is_success:
|
||||
return "", ""
|
||||
body = response.content[:MAX_BYTES]
|
||||
return str(response.url), body.decode(response.encoding or "utf-8", "replace")
|
||||
except Exception as exc: # one page skipped, whatever went wrong
|
||||
log.debug("discover: %s: %s", url, exc)
|
||||
return "", ""
|
||||
|
||||
|
||||
#: host -> a cheap URL that answers 2xx only when the slug is a board there.
|
||||
#: Workable is left out: its API sits behind a rate limit that a run of
|
||||
#: guesses trips, and a Workable site links its board plainly anyway.
|
||||
_PROBES = {
|
||||
"greenhouse": "https://boards-api.greenhouse.io/v1/boards/{slug}",
|
||||
"lever": "https://api.lever.co/v0/postings/{slug}?limit=1&mode=json",
|
||||
"ashby": "https://api.ashbyhq.com/posting-api/job-board/{slug}",
|
||||
"smartrecruiters": "https://api.smartrecruiters.com/v1/companies/{slug}/postings?limit=1",
|
||||
"recruitee": "https://{slug}.recruitee.com/api/offers/",
|
||||
"personio": "https://{slug}.jobs.personio.de/xml",
|
||||
}
|
||||
|
||||
|
||||
def probe(client: httpx.Client, host: str, slug: str) -> bool:
|
||||
"""Whether `host` has a board called `slug`, by its cheapest endpoint.
|
||||
Ashby and SmartRecruiters answer an empty listing for an unknown slug,
|
||||
so there a listing without jobs counts as no board."""
|
||||
url = _PROBES[host].format(slug=slug)
|
||||
try:
|
||||
response = client.get(url, timeout=TIMEOUT, follow_redirects=False,
|
||||
headers={"User-Agent": USER_AGENT})
|
||||
except Exception:
|
||||
return False
|
||||
if not response.is_success:
|
||||
return False
|
||||
body = response.content
|
||||
if host == "ashby":
|
||||
return b'"jobs":[' in body and b'"jobs":[]' not in body
|
||||
if host == "smartrecruiters":
|
||||
return b'"totalFound":' in body and b'"totalFound":0' not in body
|
||||
if host == "recruitee":
|
||||
return b'"offers":[' in body and b'"offers":[]' not in body
|
||||
if host == "personio":
|
||||
return b"<position>" in body
|
||||
return True
|
||||
|
||||
|
||||
Fetch = Callable[[httpx.Client, str], tuple[str, str]]
|
||||
Probe = Callable[[httpx.Client, str, str], bool]
|
||||
|
||||
|
||||
def _site(domain: str) -> str:
|
||||
domain = domain.strip().lower()
|
||||
domain = re.sub(r"^https?://", "", domain).split("/")[0]
|
||||
return domain.removeprefix("www.")
|
||||
|
||||
|
||||
def _pages(domain: str, home_links: Iterable[str]) -> list[str]:
|
||||
site = _site(domain)
|
||||
urls = list(home_links)
|
||||
urls += [f"https://{site}{path}" for path in CAREERS_PATHS]
|
||||
urls += [f"https://{sub}{site}/" for sub in CAREERS_HOSTS]
|
||||
out: list[str] = []
|
||||
for url in urls:
|
||||
if url not in out:
|
||||
out.append(url)
|
||||
return out
|
||||
|
||||
|
||||
def discover(name: str, domain: str, *, fetch: Fetch = fetch_page,
|
||||
probe_slug: Probe = probe, client: httpx.Client | None = None) -> Discovery:
|
||||
"""Where `name`'s careers site draws its jobs from; see the module
|
||||
docstring for the order pages are read in."""
|
||||
site = _site(domain)
|
||||
if not site:
|
||||
return Discovery()
|
||||
own = client or httpx.Client()
|
||||
try:
|
||||
return _discover(name, site, own, fetch, probe_slug)
|
||||
finally:
|
||||
if client is None:
|
||||
own.close()
|
||||
|
||||
|
||||
def _discover(name: str, site: str, client: httpx.Client, fetch: Fetch,
|
||||
probe_slug: Probe) -> Discovery:
|
||||
home_url, home = fetch(client, f"https://{site}/")
|
||||
unreadable: Discovery | None = None
|
||||
careers_page = ""
|
||||
|
||||
def first_readable(found: list[Found], page: str) -> Discovery | None:
|
||||
nonlocal unreadable
|
||||
for host, slug, url in found:
|
||||
if host in READABLE and slug:
|
||||
return Discovery(board=f"{host}:{slug}", ats=host, careers_url=page,
|
||||
evidence=url)
|
||||
if unreadable is None and found:
|
||||
host, slug, url = found[0]
|
||||
unreadable = Discovery(ats=host, careers_url=page, evidence=url)
|
||||
return None
|
||||
|
||||
if home and (hit := first_readable(boards_in(home), home_url)):
|
||||
return hit
|
||||
# The home page's own careers links, then the usual paths; a careers page's
|
||||
# own links (a "See open positions" one level down) go to the front of the
|
||||
# queue. `MAX_PAGES` fetches in all, whatever the queue holds.
|
||||
queue = _pages(site, careers_links(home, home_url) if home else [])
|
||||
seen = {home_url, f"https://{site}/"}
|
||||
budget = MAX_PAGES
|
||||
while queue and budget > 0:
|
||||
url = queue.pop(0)
|
||||
if url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
budget -= 1
|
||||
final, text = fetch(client, url)
|
||||
if not text:
|
||||
continue
|
||||
seen.add(final)
|
||||
careers_page = careers_page or final
|
||||
if hit := first_readable(boards_in(text), final):
|
||||
return hit
|
||||
queue[:0] = [link for link in careers_links(text, final)[:2] if link not in seen]
|
||||
for slug in slug_guesses(name, site):
|
||||
for host in _PROBES:
|
||||
if probe_slug(client, host, slug):
|
||||
return Discovery(board=f"{host}:{slug}", ats=host,
|
||||
careers_url=careers_page, evidence=_PROBES[host].format(slug=slug))
|
||||
if unreadable is not None:
|
||||
return unreadable
|
||||
return Discovery(careers_url=careers_page)
|
||||
+26
-1
@@ -11,13 +11,16 @@ roles; switched back on, the next pull reopens every role it still lists.
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from app.config import Config
|
||||
from app.models import EUROPE, Subscription, company_key
|
||||
from app.storage import Store
|
||||
from app.jobs.pull import search_target
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.jobs.discover import Discovery
|
||||
|
||||
#: The providers `plugins.search` runs, and which of them needs a key.
|
||||
PROVIDERS = ("jobindex", "adzuna")
|
||||
|
||||
@@ -91,6 +94,28 @@ def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
|
||||
return sub, len(rows)
|
||||
|
||||
|
||||
def add_discovered(store: Store, name: str, domain: str,
|
||||
found: Discovery) -> tuple[str, int]:
|
||||
"""Follow a company from what `app.jobs.discover` found: through
|
||||
`add_board` when the board is readable, else as a row with its `ats` host
|
||||
and careers page, so the company is kept as one the app cannot read yet.
|
||||
Returns the company key and how many roles its board lists (0 with no
|
||||
board). Refuses a company that exists, as `add_board` does."""
|
||||
name = name.strip()
|
||||
if not name:
|
||||
raise ValueError("a company needs a name")
|
||||
key = company_key(name)
|
||||
if found.board:
|
||||
_sub, roles = add_board(store, found.board, name)
|
||||
store.update_company(key, domain=domain, ats=found.ats)
|
||||
return key, roles
|
||||
if store.company(key) is not None:
|
||||
raise ValueError(f"{name} is already followed")
|
||||
store.add_company(name, key=key, domain=domain, ats=found.ats,
|
||||
careers_url=found.careers_url)
|
||||
return key, 0
|
||||
|
||||
|
||||
def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
|
||||
where: str = "", pages: int = 0) -> dict[str, Any]:
|
||||
"""The `params` a search subscription stores, with only the keys its
|
||||
|
||||
Reference in new issue
Block a user