Merge branch 'seed-europe'

# Conflicts:
#	README.md
This commit is contained in:
Wolf committed 2026-09-28 15:27:48 +02:00
commit 3aee48997c
19 files changed
+3362 -1394

No files matched your search

+4 -3
View File
@@ -10,13 +10,14 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
offers, the `pull` job a schedule fires, and that schedule
sources.py what a pull reads, changed by hand: boards and searches
added, switched on or off, pruned by region
discover.py which ATS a company's careers site runs on, from its pages
"""
from __future__ import annotations
from typing import Any
from app.jobs import cascade, embed, extract, pull, score, sources, summarise
from app.jobs import cascade, discover, embed, extract, pull, score, sources, summarise
from app.jobs.pull import search_target, seed_searches
from app.jobs.score import seed_agents
@@ -35,5 +36,5 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]:
return config, Store(config.database)
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "embed", "extract", "pull", "score",
"search_target", "seed_agents", "seed_searches", "sources", "summarise"]
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "pull",
"score", "search_target", "seed_agents", "seed_searches", "sources", "summarise"]
+343
View File
@@ -0,0 +1,343 @@
"""Board discovery: which applicant tracking system a company's careers site
runs on, read off its own pages.
The seed list and the LinkedIn flow both start from a company name and a
website, not from a board. This module gets from the one to the other: it
fetches the site's careers page, or the pages the home page links to as
careers, and looks for a link into one of the hosts in `HOSTS`. A readable
host (one the `ats` plugin parses) becomes a `board` target the pull can
read; any other host is recorded as `ats`, so the company is kept as one the
app cannot read yet rather than dropped.
When no page names a host, the three hosts with a cheap per-slug endpoint
(Greenhouse, Lever, Ashby) are probed with the company's own slug, because a
site that draws its jobs from an iframe or a script leaves no link to find.
found = discover("Pleo", "pleo.io")
found.board # "greenhouse:pleo", or "" when nothing readable
found.ats # "greenhouse", "workday", ... or "" when unknown
found.careers_url # the page that named the host, or the careers page
found.evidence # the URL that named it
Only https is fetched, redirects are followed at most five hops, a body is
cut at `MAX_BYTES`, and every failure is one page skipped. A company's site
is content the app was pointed at by its owner, not an ATS API, so nothing
here trusts what it finds beyond a host name matching a fixed pattern.
"""
from __future__ import annotations
import logging
import re
from collections.abc import Callable, Iterable
from dataclasses import dataclass
from urllib.parse import urljoin
import httpx
from desk.sources import USER_AGENT
log = logging.getLogger(__name__)
TIMEOUT = 10.0
MAX_BYTES = 2 << 20
#: How many pages of one site are read before giving up on links.
MAX_PAGES = 6
#: The hosts the `ats` plugin parses; a match on one of these is a `board`.
READABLE = ("greenhouse", "lever", "ashby", "workable", "smartrecruiters",
"recruitee", "personio")
#: host -> pattern over a URL. Group 1, where there is one, is the slug the
#: host names the company by; a host with no group is recognised but its
#: board has no slug the app could use. Order matters only for readability:
#: the readable hosts come first so the first match on a page is the one the
#: app can read.
HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = tuple(
(host, re.compile(pattern, re.IGNORECASE)) for host, pattern in (
("greenhouse", r"(?:boards|job-boards|api)\.(?:eu\.)?greenhouse\.io/(?:v1/boards/|embed/job_board\?[^\"'\s]*for=)?([A-Za-z0-9_-]+)"),
("greenhouse", r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)"),
("lever", r"jobs\.(?:eu\.)?lever\.co/([A-Za-z0-9_-]+)"),
("lever", r"api\.(?:eu\.)?lever\.co/v0/postings/([A-Za-z0-9_-]+)"),
("ashby", r"jobs\.ashbyhq\.com/([A-Za-z0-9_.-]+)"),
("ashby", r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)"),
("workable", r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_-]+)"),
("workable", r"https?://(?!apply\.|www\.)([a-z0-9-]+)\.workable\.com"),
("smartrecruiters", r"(?:jobs|careers|api)\.smartrecruiters\.com/(?:v1/companies/)?([A-Za-z0-9_-]+)"),
("recruitee", r"https?://([a-z0-9-]+)\.recruitee\.com"),
("personio", r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)"),
# Not readable. The slug, where the URL has one, is kept for the day
# a parser exists.
("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"),
("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"),
("teamtailor", r"https?://([a-z0-9-]+)\.teamtailor\.com"),
("teamtailor", r"cdn\.teamtailor\.com|teamtailor-cdn|teamtailor\.com/assets"),
("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
("taleo", r"\.taleo\.net"),
("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"),
("join", r"join\.com/companies/([a-z0-9-]+)"),
("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"),
("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"),
("gem", r"jobs\.gem\.com/([a-z0-9-]+)"),
("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
("eightfold", r"\.eightfold\.ai"),
("phenom", r"phenompeople\.com|phenom\.com/careers"),
("avature", r"\.avature\.net"),
("oracle", r"oraclecloud\.com/hcmUI"),
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"),
))
#: Paths a careers page usually lives at, tried after the home page's own
#: links so the site's word for it wins over a guess.
CAREERS_PATHS = ("/careers", "/careers/", "/jobs", "/en/careers", "/company/careers",
"/about/careers", "/join-us", "/careers/jobs", "/en/jobs")
#: Subdomains careers pages move to.
CAREERS_HOSTS = ("careers.", "jobs.", "career.")
#: A link text or href that points at a careers page.
_CAREERS_LINK = re.compile(
r"career|jobs|join[-_ ]?us|work[-_ ]?with[-_ ]?us|open[-_ ]?positions|vacanc|"
r"we[-_ ]?are[-_ ]?hiring|ledige[-_ ]?stillinger|stellen|karriere|lediga[-_ ]?jobb",
re.IGNORECASE)
_HREF = re.compile(r"""(?:href|src|action|content)\s*=\s*["']([^"'#][^"']*)["']""",
re.IGNORECASE)
_ANCHOR = re.compile(r"<a\b[^>]*href\s*=\s*[\"']([^\"'#][^\"']*)[\"'][^>]*>(.*?)</a>",
re.IGNORECASE | re.DOTALL)
_TAGS = re.compile(r"<[^>]+>")
#: Words a pattern's slug group can catch that name no board: the hosts'
#: own path segments, and the analytics subdomain Recruitee sites embed.
_NOT_SLUGS = frozenset({"embed", "boards", "v1", "api", "www", "jobs", "careers", "apply",
"careers-analytics"})
Found = tuple[str, str, str] # (host, slug, the URL that named it)
@dataclass(frozen=True)
class Discovery:
"""What discovery learned about one company; every field may be empty."""
board: str = ""
ats: str = ""
careers_url: str = ""
evidence: str = ""
@property
def readable(self) -> bool:
return bool(self.board)
# ------------------------------------------------------------------ parsing
def boards_in(text: str) -> list[Found]:
"""Every ATS host named in `text` (a page, a script, a mail), in order of
first appearance, once each. `slug` is "" for a host whose URL carries
none."""
seen: set[tuple[str, str]] = set()
out: list[Found] = []
hits: list[tuple[int, str, str, str]] = []
for host, pattern in HOSTS:
for m in pattern.finditer(text):
slug = (m.group(1) if m.groups() and m.group(1) else "").lower()
if slug in _NOT_SLUGS:
continue
hits.append((m.start(), host, slug, m.group(0)))
for _pos, host, slug, url in sorted(hits):
if (host, slug) in seen:
continue
seen.add((host, slug))
out.append((host, slug, url))
return out
def careers_links(html: str, base: str) -> list[str]:
"""The links on a page that read as its careers page, absolute, in page
order, once each. Nothing off-site is followed: a careers page on another
domain is what `boards_in` finds by its host."""
out: list[str] = []
site = httpx.URL(base).host or ""
for href, label in _ANCHOR.findall(html):
text = _TAGS.sub(" ", label)
if not (_CAREERS_LINK.search(href) or _CAREERS_LINK.search(text)):
continue
url = urljoin(base, href.strip())
parsed = httpx.URL(url)
if parsed.scheme not in ("http", "https"):
continue
host = parsed.host or ""
if host != site and not host.endswith("." + _apex(site)):
continue
url = str(parsed.copy_with(scheme="https", fragment=None))
if url not in out:
out.append(url)
return out
def _apex(host: str) -> str:
parts = host.split(".")
return ".".join(parts[-2:]) if len(parts) > 2 else host
def slug_guesses(name: str, domain: str) -> list[str]:
"""The slugs a host most likely knows the company by: its key, the key
without hyphens, and the domain's own label."""
from app.models import company_key
key = company_key(name)
label = domain.lower().split("/")[0]
for prefix in ("www.",):
label = label.removeprefix(prefix)
label = label.split(".")[0]
out = []
for guess in (key, key.replace("-", ""), label, label.replace("-", "")):
if guess and guess not in out:
out.append(guess)
return out
# ----------------------------------------------------------------- fetching
def fetch_page(client: httpx.Client, url: str) -> tuple[str, str]:
"""`(final URL, text)` for one page, or `("", "")` when it did not answer
with a 2xx text body within the limits. Never raises."""
try:
response = client.get(
url, timeout=TIMEOUT, follow_redirects=True,
headers={"User-Agent": USER_AGENT,
"Accept": "text/html,application/xhtml+xml,*/*;q=0.5"})
if not response.is_success:
return "", ""
body = response.content[:MAX_BYTES]
return str(response.url), body.decode(response.encoding or "utf-8", "replace")
except Exception as exc: # one page skipped, whatever went wrong
log.debug("discover: %s: %s", url, exc)
return "", ""
#: host -> a cheap URL that answers 2xx only when the slug is a board there.
#: Workable is left out: its API sits behind a rate limit that a run of
#: guesses trips, and a Workable site links its board plainly anyway.
_PROBES = {
"greenhouse": "https://boards-api.greenhouse.io/v1/boards/{slug}",
"lever": "https://api.lever.co/v0/postings/{slug}?limit=1&mode=json",
"ashby": "https://api.ashbyhq.com/posting-api/job-board/{slug}",
"smartrecruiters": "https://api.smartrecruiters.com/v1/companies/{slug}/postings?limit=1",
"recruitee": "https://{slug}.recruitee.com/api/offers/",
"personio": "https://{slug}.jobs.personio.de/xml",
}
def probe(client: httpx.Client, host: str, slug: str) -> bool:
"""Whether `host` has a board called `slug`, by its cheapest endpoint.
Ashby and SmartRecruiters answer an empty listing for an unknown slug,
so there a listing without jobs counts as no board."""
url = _PROBES[host].format(slug=slug)
try:
response = client.get(url, timeout=TIMEOUT, follow_redirects=False,
headers={"User-Agent": USER_AGENT})
except Exception:
return False
if not response.is_success:
return False
body = response.content
if host == "ashby":
return b'"jobs":[' in body and b'"jobs":[]' not in body
if host == "smartrecruiters":
return b'"totalFound":' in body and b'"totalFound":0' not in body
if host == "recruitee":
return b'"offers":[' in body and b'"offers":[]' not in body
if host == "personio":
return b"<position>" in body
return True
Fetch = Callable[[httpx.Client, str], tuple[str, str]]
Probe = Callable[[httpx.Client, str, str], bool]
def _site(domain: str) -> str:
domain = domain.strip().lower()
domain = re.sub(r"^https?://", "", domain).split("/")[0]
return domain.removeprefix("www.")
def _pages(domain: str, home_links: Iterable[str]) -> list[str]:
site = _site(domain)
urls = list(home_links)
urls += [f"https://{site}{path}" for path in CAREERS_PATHS]
urls += [f"https://{sub}{site}/" for sub in CAREERS_HOSTS]
out: list[str] = []
for url in urls:
if url not in out:
out.append(url)
return out
def discover(name: str, domain: str, *, fetch: Fetch = fetch_page,
probe_slug: Probe = probe, client: httpx.Client | None = None) -> Discovery:
"""Where `name`'s careers site draws its jobs from; see the module
docstring for the order pages are read in."""
site = _site(domain)
if not site:
return Discovery()
own = client or httpx.Client()
try:
return _discover(name, site, own, fetch, probe_slug)
finally:
if client is None:
own.close()
def _discover(name: str, site: str, client: httpx.Client, fetch: Fetch,
probe_slug: Probe) -> Discovery:
home_url, home = fetch(client, f"https://{site}/")
unreadable: Discovery | None = None
careers_page = ""
def first_readable(found: list[Found], page: str) -> Discovery | None:
nonlocal unreadable
for host, slug, url in found:
if host in READABLE and slug:
return Discovery(board=f"{host}:{slug}", ats=host, careers_url=page,
evidence=url)
if unreadable is None and found:
host, slug, url = found[0]
unreadable = Discovery(ats=host, careers_url=page, evidence=url)
return None
if home and (hit := first_readable(boards_in(home), home_url)):
return hit
# The home page's own careers links, then the usual paths; a careers page's
# own links (a "See open positions" one level down) go to the front of the
# queue. `MAX_PAGES` fetches in all, whatever the queue holds.
queue = _pages(site, careers_links(home, home_url) if home else [])
seen = {home_url, f"https://{site}/"}
budget = MAX_PAGES
while queue and budget > 0:
url = queue.pop(0)
if url in seen:
continue
seen.add(url)
budget -= 1
final, text = fetch(client, url)
if not text:
continue
seen.add(final)
careers_page = careers_page or final
if hit := first_readable(boards_in(text), final):
return hit
queue[:0] = [link for link in careers_links(text, final)[:2] if link not in seen]
for slug in slug_guesses(name, site):
for host in _PROBES:
if probe_slug(client, host, slug):
return Discovery(board=f"{host}:{slug}", ats=host,
careers_url=careers_page, evidence=_PROBES[host].format(slug=slug))
if unreadable is not None:
return unreadable
return Discovery(careers_url=careers_page)
+26 -1
View File
@@ -11,13 +11,16 @@ roles; switched back on, the next pull reopens every role it still lists.
from __future__ import annotations
import os
from typing import Any
from typing import TYPE_CHECKING, Any
from app.config import Config
from app.models import EUROPE, Subscription, company_key
from app.storage import Store
from app.jobs.pull import search_target
if TYPE_CHECKING:
from app.jobs.discover import Discovery
#: The providers `plugins.search` runs, and which of them needs a key.
PROVIDERS = ("jobindex", "adzuna")
@@ -91,6 +94,28 @@ def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
return sub, len(rows)
def add_discovered(store: Store, name: str, domain: str,
found: Discovery) -> tuple[str, int]:
"""Follow a company from what `app.jobs.discover` found: through
`add_board` when the board is readable, else as a row with its `ats` host
and careers page, so the company is kept as one the app cannot read yet.
Returns the company key and how many roles its board lists (0 with no
board). Refuses a company that exists, as `add_board` does."""
name = name.strip()
if not name:
raise ValueError("a company needs a name")
key = company_key(name)
if found.board:
_sub, roles = add_board(store, found.board, name)
store.update_company(key, domain=domain, ats=found.ats)
return key, roles
if store.company(key) is not None:
raise ValueError(f"{name} is already followed")
store.add_company(name, key=key, domain=domain, ats=found.ats,
careers_url=found.careers_url)
return key, 0
def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
where: str = "", pages: int = 0) -> dict[str, Any]:
"""The `params` a search subscription stores, with only the keys its