Part A: Ashby (?includeCompensation=true), Greenhouse (pay_transparency=true, already free on the list request), Lever (salaryRange), Recruitee (salary) and Personio (salaryInformation) all give structured pay; Workable and SmartRecruiters do not (checked live). Every host converts to a yearly salary_min/salary_max/currency, multiplying a monthly figure by 12 and skipping an hourly or daily one rather than annualising a number nobody stated. A range with no currency is ignored, and Greenhouse's own multi-zone ranges are skipped outright since nothing says which zone the role is at. Part B: classify() matched substrings, so " it" matched inside "Italy", filing European sales roles as engineering; it now matches on word boundaries (full boundary for the handful of short tokens that are themselves the start of an unrelated word, prefix matching kept for the stems). Title is checked before department (a hand-checked sample found title-first right about twice as often), nothing matching returns '' rather than "other", and a modest set of German/Dutch/Danish/Norwegian/Swedish/ French keywords covers the seed's other languages. function is not a board field, so it gets the opposite rule from every other raw column: extract's answer overwrites the classifier's guess outright, and once extracted_at is set no later pull may put the guess back. Migration 9 recomputes function for every not-yet-extracted role from its stored title and department. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
763 lines
35 KiB
Python
763 lines
35 KiB
Python
"""ATS boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio.
|
|
|
|
Every one of these hosts is a company's own careers page speaking public,
|
|
unauthenticated JSON (Personio: XML) — first-party, no scraping, and a role
|
|
that drops off the board is *closed*, which is the cheapest staleness check
|
|
there is.
|
|
|
|
The field map, the redirect allow-list and the 10 s timeout are ported from
|
|
`openings.go` and `cmd/seedgen/main.go` in
|
|
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including
|
|
`classify`'s nine functions and their keywords. Workable, SmartRecruiters,
|
|
Recruitee and Personio are new: jobhunt-hq only read Greenhouse, Lever and
|
|
Ashby, so those four are written from one real board of each host, fetched
|
|
while this file was written (2026-09-23; Workable on 2026-09-28, when the
|
|
first board with roles was seen), rather than from the Go source.
|
|
|
|
The body cap is not a straight port — see `MAX_BYTES` below.
|
|
|
|
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
|
`lever:acme`. Every board is fetched independently; one that fails or times
|
|
out is logged and skipped, and the rest continue.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import html
|
|
import json
|
|
import logging
|
|
import re
|
|
import xml.etree.ElementTree as ET
|
|
from collections.abc import Callable, Iterator
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from desk.sources import (
|
|
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
|
|
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, strip_html,
|
|
)
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: Matches openings.go: a board answers in 10s or it is skipped. openings.go's
|
|
#: 8 MB body cap does not: it was written against arbitrary URLs, and every
|
|
#: URL fetched here is `?content=true` against a fixed allow-list of seven ATS
|
|
#: API hosts. The real run of 2026-09-23 hit it on the biggest employers —
|
|
#: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and
|
|
#: Airwallex between 9 and 19 MB — and skipped their boards outright. Raised
|
|
#: to 96 MB so those boards fit; a board that still exceeds it gets one retry
|
|
#: (Greenhouse only, see `_fetch_rows`) without `?content=true`, so at least
|
|
#: the titles land.
|
|
TIMEOUT = 10.0
|
|
MAX_BYTES = 96 << 20
|
|
|
|
#: host -> (feed URL template, careers URL template). `{slug}` is filled in;
|
|
#: recruitee and personio put the slug in the hostname instead of the path.
|
|
_BOARDS: dict[str, tuple[str, str]] = {
|
|
"greenhouse": ("https://boards-api.greenhouse.io/v1/boards/{slug}/jobs"
|
|
"?content=true&pay_transparency=true",
|
|
"https://job-boards.greenhouse.io/{slug}"),
|
|
"lever": ("https://api.lever.co/v0/postings/{slug}?mode=json",
|
|
"https://jobs.lever.co/{slug}"),
|
|
"ashby": ("https://api.ashbyhq.com/posting-api/job-board/{slug}"
|
|
"?includeCompensation=true",
|
|
"https://jobs.ashbyhq.com/{slug}"),
|
|
"workable": ("https://apply.workable.com/api/v3/accounts/{slug}/jobs",
|
|
"https://apply.workable.com/{slug}"),
|
|
"smartrecruiters": ("https://api.smartrecruiters.com/v1/companies/{slug}/postings",
|
|
"https://jobs.smartrecruiters.com/{slug}"),
|
|
"recruitee": ("https://{slug}.recruitee.com/api/offers/",
|
|
"https://{slug}.recruitee.com"),
|
|
"personio": ("https://{slug}.jobs.personio.de/xml",
|
|
"https://{slug}.jobs.personio.de"),
|
|
}
|
|
|
|
#: The hosts a request or a redirect may land on for a given board host — a
|
|
#: fixed API host for five of them, and the company's own subdomain for the
|
|
#: two that put the slug in the hostname. Anything else is refused, at the
|
|
#: first hop and at every redirect: a feed is external content, and a board
|
|
#: row must never be able to point this fetch at an internal address.
|
|
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
|
fixed = {
|
|
"greenhouse": "boards-api.greenhouse.io",
|
|
"lever": "api.lever.co",
|
|
"ashby": "api.ashbyhq.com",
|
|
"workable": "apply.workable.com",
|
|
"smartrecruiters": "api.smartrecruiters.com",
|
|
}
|
|
if host in fixed:
|
|
want = fixed[host]
|
|
return lambda h: h == want
|
|
if host == "recruitee":
|
|
want = f"{slug}.recruitee.com"
|
|
return lambda h: h == want
|
|
if host == "personio":
|
|
want = f"{slug}.jobs.personio.de"
|
|
return lambda h: h == want
|
|
return lambda h: False
|
|
|
|
|
|
def _is_web_url(url: str) -> bool:
|
|
"""A trimmed http(s) URL, or refuse — a feed is external content and a
|
|
`javascript:` or `data:` link must never reach an href. Port of
|
|
openings.go's `webURL`."""
|
|
url = (url or "").strip()
|
|
if not url:
|
|
return False
|
|
parsed = httpx.URL(url)
|
|
return parsed.scheme in ("http", "https") and bool(parsed.host)
|
|
|
|
|
|
class _Refused(Exception):
|
|
"""A board answered somewhere its allow-list does not cover."""
|
|
|
|
|
|
class _TooLarge(ValueError):
|
|
"""A response exceeded `MAX_BYTES`. A `ValueError` so a caller that only
|
|
knows the old shape still catches it; `_fetch_rows` catches it by type to
|
|
retry Greenhouse without `?content=true`."""
|
|
|
|
|
|
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
|
method: str = "GET") -> bytes:
|
|
"""Fetch one board feed: https only, the host's own allow-list, at most
|
|
three redirect hops each checked the same way, and the body cut off at
|
|
`MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`."""
|
|
allow = _allowed_host(host, slug)
|
|
current = url
|
|
for _hop in range(4):
|
|
parsed = httpx.URL(current)
|
|
if parsed.scheme != "https" or not allow(parsed.host):
|
|
raise _Refused(f"{current} is not an allowed host for {host}")
|
|
response = client.request(
|
|
method, current, timeout=TIMEOUT, follow_redirects=False,
|
|
headers={"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"},
|
|
json={} if method == "POST" else None,
|
|
)
|
|
if response.is_redirect:
|
|
location = response.headers.get("location") or ""
|
|
if not location:
|
|
raise ValueError("redirect with no location")
|
|
current = str(httpx.URL(current).join(location))
|
|
continue
|
|
response.raise_for_status()
|
|
body = response.content
|
|
if len(body) > MAX_BYTES:
|
|
raise _TooLarge(f"response too large ({len(body)} bytes)")
|
|
return body
|
|
raise ValueError("too many redirects")
|
|
|
|
|
|
def _parse_dt(value: Any) -> datetime | None:
|
|
"""An aware datetime from whatever a board sends: ISO-8601 with an
|
|
offset, `Z`, a bare date, epoch milliseconds, or "YYYY-MM-DD HH:MM:SS
|
|
UTC". None for anything else, which `Item` reads as "now"."""
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, int | float):
|
|
if value <= 0:
|
|
return None
|
|
seconds = value / 1000 if value > 10**12 else value
|
|
return datetime.fromtimestamp(seconds, tz=UTC)
|
|
text = str(value).strip()
|
|
if not text:
|
|
return None
|
|
try:
|
|
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
match = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})", text)
|
|
if not match:
|
|
return None
|
|
parsed = datetime.fromisoformat(f"{match.group(1)}T{match.group(2)}+00:00")
|
|
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
|
|
|
|
|
# ------------------------------------------------------------- the parsers
|
|
|
|
#: What every parser turns a feed into: one dict per row, with these keys.
|
|
Row = dict[str, Any]
|
|
|
|
|
|
# ---------------------------------------------------------- structured pay
|
|
#
|
|
# A board's own salary range, mapped to yearly `salary_min`/`salary_max`
|
|
# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks
|
|
# `raw` into, per docs/decisions.md's "raw is transport, columns are
|
|
# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give
|
|
# structured pay on their list endpoint; Workable and SmartRecruiters do not
|
|
# (checked live, 2026-09-29), so their parsers are untouched and `row.get`
|
|
# below reads a plain '' or None for them.
|
|
|
|
|
|
def _num(value: Any) -> float | None:
|
|
"""A positive number from whatever a board sends (int, float, or a
|
|
numeral string), or None for anything else, including zero and a
|
|
negative — a board's placeholder 0 (an unfilled field, not a real
|
|
figure) must not become a salary floor of zero."""
|
|
try:
|
|
number = float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return number if number > 0 else None
|
|
|
|
|
|
def _yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any,
|
|
factors: dict[str, int]) -> tuple[int | None, int | None, str]:
|
|
"""`(salary_min, salary_max, currency)` as yearly integers, or `(None,
|
|
None, "")` when there is no currency, no bound, or `interval` is not one
|
|
of `factors`' own keys — which is how an hourly or a daily rate is left
|
|
out rather than annualised into a number nobody stated. `factors` maps
|
|
the board's own interval spelling to the multiplier that makes it
|
|
yearly: 1 for a figure already yearly, 12 for a monthly one."""
|
|
code = str(currency or "").strip().upper()
|
|
factor = factors.get(str(interval or "").strip())
|
|
if not code or not factor:
|
|
return None, None, ""
|
|
lo, hi = _num(min_value), _num(max_value)
|
|
if lo is None and hi is None:
|
|
return None, None, ""
|
|
return (int(lo * factor) if lo is not None else None,
|
|
int(hi * factor) if hi is not None else None, code)
|
|
|
|
|
|
def _greenhouse_salary(ranges: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
|
|
"""The yearly range from `pay_input_ranges` (needs `pay_transparency=true`
|
|
on the feed URL, see `_BOARDS`), which the list endpoint already carries
|
|
in the same request — no per-role fetch. Only when there is exactly one
|
|
range: a board with several ("Zone 1 Pay Range", "US Tier 2 Locations",
|
|
one per office) gives no way to tell which office the role in hand is
|
|
at, and picking one would misrepresent the columns. Greenhouse's own
|
|
ranges are cents, and yearly by convention except an hourly or a daily
|
|
one, named as such in `title` with no separate field to check — those
|
|
are skipped rather than annualised into a number nobody stated."""
|
|
if len(ranges) != 1:
|
|
return None, None, ""
|
|
entry = ranges[0]
|
|
title = str(entry.get("title") or "").lower()
|
|
if any(word in title for word in ("hour", "/hr", "daily", "per day", "day rate")):
|
|
return None, None, ""
|
|
code = str(entry.get("currency_type") or "").strip().upper()
|
|
if not code:
|
|
return None, None, ""
|
|
lo, hi = _num(entry.get("min_cents")), _num(entry.get("max_cents"))
|
|
if lo is None and hi is None:
|
|
return None, None, ""
|
|
return (int(lo / 100) if lo is not None else None,
|
|
int(hi / 100) if hi is not None else None, code)
|
|
|
|
|
|
def _ashby_salary(components: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
|
|
"""The yearly range from `compensation.summaryComponents` (needs
|
|
`?includeCompensation=true` on the feed URL, see `_BOARDS`): the first
|
|
`Salary` component, yearly or monthly (a monthly one multiplied by 12);
|
|
an hourly one is left out rather than annualised."""
|
|
for component in components:
|
|
if component.get("compensationType") != "Salary":
|
|
continue
|
|
return _yearly_amount(component.get("minValue"), component.get("maxValue"),
|
|
component.get("currencyCode"), component.get("interval"),
|
|
{"1 YEAR": 1, "1 MONTH": 12})
|
|
return None, None, ""
|
|
|
|
|
|
def _parse_greenhouse(body: bytes) -> list[Row]:
|
|
data = json.loads(body)
|
|
rows = []
|
|
for j in data.get("jobs") or []:
|
|
title = (j.get("title") or "").strip()
|
|
url = (j.get("absolute_url") or "").strip()
|
|
if not title or not _is_web_url(url):
|
|
continue
|
|
departments = j.get("departments") or []
|
|
# Greenhouse ships the description HTML-entity-escaped inside the
|
|
# JSON string (`<p>`), so it needs unescaping before the tags
|
|
# themselves can be stripped.
|
|
content = html.unescape(j.get("content") or "")
|
|
salary_min, salary_max, currency = _greenhouse_salary(j.get("pay_input_ranges") or [])
|
|
rows.append({
|
|
"source_id": str(j.get("id") or ""),
|
|
"title": title,
|
|
"location": ((j.get("location") or {}).get("name") or "").strip(),
|
|
"url": url,
|
|
"posted": _parse_dt(j.get("first_published") or j.get("updated_at")),
|
|
"department": (departments[0].get("name") if departments else "") or "",
|
|
"description": strip_html(content),
|
|
"remote": None, # Greenhouse's feed does not say
|
|
"salary": "",
|
|
"salary_min": salary_min,
|
|
"salary_max": salary_max,
|
|
"currency": currency,
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_lever(body: bytes) -> list[Row]:
|
|
data = json.loads(body)
|
|
if not isinstance(data, list):
|
|
return []
|
|
rows = []
|
|
for j in data:
|
|
title = (j.get("text") or "").strip()
|
|
url = (j.get("hostedUrl") or "").strip()
|
|
if not title or not _is_web_url(url):
|
|
continue
|
|
categories = j.get("categories") or {}
|
|
description = j.get("description") or j.get("descriptionPlain") or ""
|
|
salary_range = j.get("salaryRange") or {}
|
|
salary_min, salary_max, currency = _yearly_amount(
|
|
salary_range.get("min"), salary_range.get("max"), salary_range.get("currency"),
|
|
salary_range.get("interval"), {"per-year-salary": 1, "per-month-salary": 12})
|
|
rows.append({
|
|
"source_id": str(j.get("id") or ""),
|
|
"title": title,
|
|
"location": (categories.get("location") or "").strip(),
|
|
"url": url,
|
|
"posted": _parse_dt(j.get("createdAt")),
|
|
"department": (categories.get("department") or categories.get("team") or "").strip(),
|
|
"description": strip_html(description),
|
|
"remote": (j.get("workplaceType") or "").strip() or None,
|
|
"salary": "",
|
|
"salary_min": salary_min,
|
|
"salary_max": salary_max,
|
|
"currency": currency,
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_ashby(body: bytes) -> list[Row]:
|
|
data = json.loads(body)
|
|
rows = []
|
|
for j in data.get("jobs") or []:
|
|
title = (j.get("title") or "").strip()
|
|
url = (j.get("jobUrl") or j.get("applyUrl") or "").strip()
|
|
if not title or not _is_web_url(url):
|
|
continue
|
|
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
|
|
compensation = j.get("compensation") or {}
|
|
salary = (compensation.get("compensationTierSummary")
|
|
or compensation.get("scrapeableCompensationSalarySummary") or "")
|
|
salary_min, salary_max, currency = _ashby_salary(compensation.get("summaryComponents") or [])
|
|
rows.append({
|
|
"source_id": str(j.get("id") or ""),
|
|
"title": title,
|
|
"location": (j.get("location") or "").strip(),
|
|
"url": url,
|
|
"posted": _parse_dt(j.get("publishedAt")),
|
|
"department": (j.get("department") or j.get("team") or "").strip(),
|
|
"description": strip_html(description),
|
|
"remote": j.get("isRemote"),
|
|
"salary": salary,
|
|
"salary_min": salary_min,
|
|
"salary_max": salary_max,
|
|
"currency": currency,
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_workable(body: bytes, slug: str) -> list[Row]:
|
|
"""POST https://apply.workable.com/api/v3/accounts/{slug}/jobs with `{}` ->
|
|
`{"total": int, "results": [...]}`, as Hugging Face's board answered on
|
|
2026-09-28. A result carries no URL and no text: the job page is
|
|
`https://apply.workable.com/{slug}/j/{shortcode}/`, and the description
|
|
is only on the per-job endpoint, one request per role, so `description`
|
|
stays empty here as it does for SmartRecruiters. `location` is an object
|
|
and `department` a list; `remote` is a flag beside `workplace`
|
|
(remote, hybrid, on_site)."""
|
|
data = json.loads(body)
|
|
rows = []
|
|
for j in data.get("results") or []:
|
|
title = (j.get("title") or "").strip()
|
|
shortcode = (j.get("shortcode") or "").strip()
|
|
if not title or not shortcode:
|
|
continue
|
|
loc = j.get("location") or {}
|
|
location = ", ".join(p for p in (loc.get("city"), loc.get("region"), loc.get("country")) if p)
|
|
dept = j.get("department") or ""
|
|
if isinstance(dept, list):
|
|
dept = ", ".join(d for d in dept if d)
|
|
rows.append({
|
|
"source_id": str(j.get("id") or shortcode),
|
|
"title": title,
|
|
"location": location,
|
|
"url": f"https://apply.workable.com/{slug}/j/{shortcode}/",
|
|
"posted": _parse_dt(j.get("published")),
|
|
"department": dept.strip(),
|
|
"description": "",
|
|
"remote": bool(j.get("remote")) or j.get("workplace") == "remote",
|
|
"salary": "",
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_smartrecruiters(body: bytes, slug: str) -> list[Row]:
|
|
"""https://api.smartrecruiters.com/v1/companies/{slug}/postings ->
|
|
`{"content": [...]}`. The list omits the job description (only the
|
|
per-posting detail endpoint has it, and fetching one per row would be
|
|
hundreds of extra requests per board), so `description` stays empty here."""
|
|
data = json.loads(body)
|
|
rows = []
|
|
for j in data.get("content") or []:
|
|
title = (j.get("name") or "").strip()
|
|
job_id = str(j.get("id") or "")
|
|
if not title or not job_id:
|
|
continue
|
|
location = j.get("location") or {}
|
|
loc_text = (location.get("fullLocation")
|
|
or ", ".join(p for p in (location.get("city"), location.get("country")) if p))
|
|
rows.append({
|
|
"source_id": job_id,
|
|
"title": title,
|
|
"location": (loc_text or "").strip(),
|
|
"url": f"https://jobs.smartrecruiters.com/{slug}/{job_id}",
|
|
"posted": _parse_dt(j.get("releasedDate")),
|
|
"department": ((j.get("department") or {}).get("label") or "").strip(),
|
|
"description": "",
|
|
"remote": location.get("remote"),
|
|
"salary": "",
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_recruitee(body: bytes) -> list[Row]:
|
|
data = json.loads(body)
|
|
rows = []
|
|
for j in data.get("offers") or []:
|
|
title = (j.get("title") or "").strip()
|
|
url = (j.get("careers_url") or "").strip()
|
|
if not title or not _is_web_url(url):
|
|
continue
|
|
salary = j.get("salary") or {}
|
|
salary_text = ""
|
|
if isinstance(salary, dict) and (salary.get("min") or salary.get("max")):
|
|
bounds = "-".join(str(v) for v in (salary.get("min"), salary.get("max")) if v)
|
|
salary_text = f"{bounds} {salary.get('currency') or ''}".strip()
|
|
salary_min, salary_max, currency = _yearly_amount(
|
|
salary.get("min"), salary.get("max"), salary.get("currency"),
|
|
salary.get("period"), {"year": 1, "month": 12})
|
|
rows.append({
|
|
"source_id": str(j.get("id") or ""),
|
|
"title": title,
|
|
"location": (j.get("location") or j.get("city") or "").strip(),
|
|
"url": url,
|
|
"posted": _parse_dt(j.get("published_at") or j.get("created_at")),
|
|
"department": (j.get("department") or "").strip(),
|
|
"description": strip_html(j.get("description") or ""),
|
|
"remote": j.get("remote"),
|
|
"salary": salary_text,
|
|
"salary_min": salary_min,
|
|
"salary_max": salary_max,
|
|
"currency": currency,
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_personio(body: bytes, slug: str) -> list[Row]:
|
|
"""`{slug}.jobs.personio.de/xml` -> `<workzag-jobs><position>...`. No job
|
|
URL in the feed itself; Personio's own page is `/job/{id}`, confirmed
|
|
live against a real posting (2026-09-23)."""
|
|
root = ET.fromstring(body)
|
|
rows = []
|
|
for position in root.findall("position"):
|
|
job_id = (position.findtext("id") or "").strip()
|
|
title = (position.findtext("name") or "").strip()
|
|
if not job_id or not title:
|
|
continue
|
|
description = " ".join(
|
|
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
|
|
salary_info = position.find("salaryInformation")
|
|
salary_min = salary_max = None
|
|
currency = ""
|
|
if salary_info is not None:
|
|
salary_min, salary_max, currency = _yearly_amount(
|
|
salary_info.findtext("min"), salary_info.findtext("max"),
|
|
salary_info.findtext("currencyCode"), salary_info.findtext("type"),
|
|
{"yearly": 1, "monthly": 12})
|
|
rows.append({
|
|
"source_id": job_id,
|
|
"title": title,
|
|
"location": (position.findtext("office") or "").strip(),
|
|
"url": f"https://{slug}.jobs.personio.de/job/{job_id}",
|
|
"posted": _parse_dt(position.findtext("createdAt")),
|
|
"department": (position.findtext("department") or "").strip(),
|
|
"description": strip_html(description),
|
|
"remote": "remote" in (position.findtext("office") or "").lower() or None,
|
|
"salary": "",
|
|
"salary_min": salary_min,
|
|
"salary_max": salary_max,
|
|
"currency": currency,
|
|
})
|
|
return rows
|
|
|
|
|
|
#: host -> parser. Every one takes the raw response body, plus the slug for
|
|
#: the two hosts whose feed does not carry the URL or the company itself.
|
|
_PARSERS: dict[str, Callable[..., list[Row]]] = {
|
|
"greenhouse": lambda body, slug: _parse_greenhouse(body),
|
|
"lever": lambda body, slug: _parse_lever(body),
|
|
"ashby": lambda body, slug: _parse_ashby(body),
|
|
"workable": _parse_workable,
|
|
"smartrecruiters": _parse_smartrecruiters,
|
|
"recruitee": lambda body, slug: _parse_recruitee(body),
|
|
"personio": _parse_personio,
|
|
}
|
|
|
|
|
|
# --------------------------------------------------------- the classifier
|
|
|
|
#: The fixed set of functions, and the keyword rules that map a title
|
|
#: (preferred) or a department to one of them. Ported from `cmd/seedgen/
|
|
#: main.go`'s `rules` — first match wins, and "product" is genuinely listed
|
|
#: twice: the first, narrower entry is checked before
|
|
#: recruiting/design/marketing/support/sales/data, and the second, which also
|
|
#: matches the bare word "product", is checked only after all of those, so a
|
|
#: title like "Data Product Manager" keeps classifying as data/product by the
|
|
#: same rule order the Go tool used.
|
|
#:
|
|
#: Not a straight port any more: the Go tool matched substrings, which let
|
|
#: " it" (meant to catch "IT" as a department) match inside "Italy" and
|
|
#: "Italia" — a European department like "Commercial Italy" filed as
|
|
#: engineering. `_classify_text` now matches on word boundaries. A handful of
|
|
#: short tokens (`_WHOLE_WORD`) need a boundary on *both* sides, because they
|
|
#: are themselves the start of an unrelated word ("it" of "Italy", "seo" of
|
|
#: "Seoul", "qa" of "Qatar"); every other keyword keeps a leading boundary
|
|
#: only; most of them are deliberate stems ("manag", "engineer", "recruit")
|
|
#: meant to also catch "management", "engineering", "recruiting", and a
|
|
#: trailing boundary on those would stop them matching the plural and
|
|
#: gerund forms real postings use ("Software Developers", "Account
|
|
#: Managers").
|
|
FUNCTIONS = ("engineering", "data", "product", "design", "sales", "marketing",
|
|
"customer success/support", "recruiting/people", "finance/legal/ops")
|
|
|
|
_RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
|
("product", ("product manag", "product management", "product operations",
|
|
"program manag", "technical program")),
|
|
("recruiting/people", ("recruit", "talent", "people ops", "people &",
|
|
"people team", "people operations", "human resources",
|
|
"hr", "hrbp")),
|
|
("design", ("design", "ux", "user experience", "creative")),
|
|
("marketing", ("marketing", "growth", "content", "communications", "brand",
|
|
"seo", "social media", "demand gen", "events")),
|
|
("customer success/support", ("customer success", "customer support",
|
|
"customer experience", "customer service",
|
|
"support", "success", "implementation",
|
|
"onboarding", "professional services",
|
|
"solutions consult", "technical account",
|
|
"customer engineer", "deployment")),
|
|
("sales", ("sales", "account executive", "account manager", "account director",
|
|
"business development", "bdr", "sdr", "revenue", "partnership",
|
|
"go-to-market", "gtm", "solutions engineer", "solution engineer",
|
|
"pre-sales", "presales", "channel",
|
|
# "Commercial" is what French, Italian and Spanish org charts
|
|
# call the sales function ("Commercial Italy" is a sales
|
|
# department; see the docstring above), and the rest are the
|
|
# sales words of the seed's other languages that "sales"
|
|
# itself does not already cover (German, Dutch, Danish,
|
|
# Norwegian, Swedish, French) — a modest set, not a full port;
|
|
# `extract` is the rest.
|
|
"commercial", "vertrieb", "verkauf", "verkoop", "salg",
|
|
"försäljning", "vente")),
|
|
("data", ("data", "analytics", "machine learning", "ml",
|
|
"applied science", "research scientist", "statistic")),
|
|
("product", ("product manag", "product management", "program manag",
|
|
"product operations", "technical program", "product")),
|
|
("finance/legal/ops", ("finance", "financ", "accounting", "legal", "counsel",
|
|
"compliance", "operations", "ops", "administrat",
|
|
"office", "procurement", "strategy", "corporate",
|
|
"facilities", "risk", "audit", "tax", "treasury",
|
|
"payroll", "executive assistant", "chief of staff",
|
|
"workplace", "supply chain", "logistics", "fraud",
|
|
"trust & safety", "trust and safety", "policy")),
|
|
("engineering", ("engineer", "software", "developer", "infrastructure",
|
|
"platform", "devops", "sre", "site reliability", "security",
|
|
"information technology", "it", "technology",
|
|
"technical staff", "hardware", "qa", "quality assurance",
|
|
"architect", "research", "science", "scientist", "systems",
|
|
"network", "cloud", "mobile", "frontend", "backend",
|
|
"full stack", "fullstack", "web",
|
|
# The developer/engineer words of the seed's other
|
|
# languages, same reasoning as sales' above.
|
|
"entwickler", "ingenieur", "ontwikkelaar", "udvikler",
|
|
"utvikler", "utvecklare", "ingenjör", "ingeniør",
|
|
"développeur", "developpeur", "ingénieur")),
|
|
)
|
|
|
|
#: Keywords that must match a whole word, not just its start: each one is
|
|
#: itself the first letters of an unrelated word that would otherwise false-
|
|
#: match ("it" of "Italy", "seo" of "Seoul", "qa" of "Qatar"). Every keyword
|
|
#: not listed here keeps the old prefix behaviour on purpose — see the
|
|
#: `_RULES` docstring above.
|
|
_WHOLE_WORD = frozenset({
|
|
"it", "hr", "hrbp", "ux", "ml", "qa", "seo", "web", "tax", "ops",
|
|
"sre", "bdr", "sdr", "gtm",
|
|
})
|
|
|
|
|
|
def _keyword_pattern(keyword: str) -> re.Pattern[str]:
|
|
trailing = r"\b" if keyword in _WHOLE_WORD else ""
|
|
return re.compile(r"\b" + re.escape(keyword) + trailing, re.IGNORECASE)
|
|
|
|
|
|
#: `_RULES`, compiled once: `(function, patterns)` in the same order.
|
|
_COMPILED_RULES: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = tuple(
|
|
(function, tuple(_keyword_pattern(keyword) for keyword in keywords))
|
|
for function, keywords in _RULES)
|
|
|
|
|
|
def _classify_text(text: str) -> str:
|
|
stripped = (text or "").strip()
|
|
if not stripped:
|
|
return ""
|
|
for function, patterns in _COMPILED_RULES:
|
|
if any(pattern.search(stripped) for pattern in patterns):
|
|
return function
|
|
return ""
|
|
|
|
|
|
def classify(department: str, title: str) -> str:
|
|
"""A title (preferred) or a department, to one of `FUNCTIONS`, or `''`
|
|
when neither names one — `''` and not `"other"`, which would claim a
|
|
knowledge the classifier does not have (`extract` may still fill it from
|
|
reading the posting; see `Store.set_extracted`).
|
|
|
|
Title before department: a department is often a generic umbrella
|
|
("Operations", "Engineering & Product") that says less about the role
|
|
than its title does. A hand-checked sample of 60 roles where the two
|
|
orders disagreed (docs/decisions.md) found title-first right about twice
|
|
as often as department-first, mostly because a specific title beats a
|
|
broad department name, so title now wins; department is the fallback
|
|
when the title matches nothing."""
|
|
return _classify_text(title or "") or _classify_text(department or "") or ""
|
|
|
|
|
|
def is_known_target(target: str) -> bool:
|
|
"""`host:slug`, with `host` one of `_BOARDS` and a non-empty slug. What
|
|
the target validator refuses: `app companies add` and the pull job
|
|
both call this before touching the network."""
|
|
host, sep, slug = target.partition(":")
|
|
return bool(sep) and host in _BOARDS and bool(slug.strip())
|
|
|
|
|
|
def careers_url(target: str) -> str:
|
|
"""The company's own careers page for `host:slug`, or "" for an unknown
|
|
host. What `app companies add` stores alongside the board."""
|
|
host, _, slug = target.partition(":")
|
|
if host not in _BOARDS:
|
|
return ""
|
|
return _BOARDS[host][1].format(slug=slug)
|
|
|
|
|
|
def _fetch_rows(client: httpx.Client, target: str) -> list[Row]:
|
|
"""One board, fetched and parsed. Raises on any failure — refused host,
|
|
timeout, a feed shape none of the parsers understand.
|
|
|
|
A Greenhouse board whose `?content=true` body still exceeds `MAX_BYTES`
|
|
gets one retry without it: the description is lost, but the titles land
|
|
rather than the whole board being skipped."""
|
|
if not is_known_target(target):
|
|
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
|
host, _, slug = target.partition(":")
|
|
url = _BOARDS[host][0].format(slug=slug)
|
|
try:
|
|
body = _get(client, url, host, slug, method="POST" if host == "workable" else "GET")
|
|
except _TooLarge:
|
|
if host != "greenhouse" or "?" not in url:
|
|
raise
|
|
log.warning("ats: %s: body exceeded %d bytes with content=true; "
|
|
"retrying without it", target, MAX_BYTES)
|
|
url = url.split("?", 1)[0]
|
|
body = _get(client, url, host, slug, method="GET")
|
|
return _PARSERS[host](body, slug)
|
|
|
|
|
|
def fetch_board(target: str) -> list[Row]:
|
|
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
|
|
"does this board answer, and what does it list". For `app companies
|
|
add|check`, which ask that question about one board at a time outside any
|
|
pull. Raises the same way `_fetch_rows` does."""
|
|
with httpx.Client() as client:
|
|
return _fetch_rows(client, target)
|
|
|
|
|
|
# -------------------------------------------------------------- the plugin
|
|
|
|
class ATSSource(Source):
|
|
name = "ats"
|
|
description = "Company job boards: Greenhouse, Lever, Ashby, Workable, " \
|
|
"SmartRecruiters, Recruitee, Personio."
|
|
#: The raw entries under `sources.ats.boards` are `{"url": target, "label":
|
|
#: company}` — `url` carries the `host:slug` target and `label` the
|
|
#: company name, which is how `Source.targets()`'s generic dict reading
|
|
#: hands both back without a plugin-specific override.
|
|
subscription_field = "boards"
|
|
|
|
spec = PluginSpec(
|
|
api_version=API_VERSION, id=name, version="1", label="ATS boards",
|
|
targets=TargetSpec(mode="editable", kind="board", defaults=[]),
|
|
profiles={"default": Profile(
|
|
kind="role",
|
|
tasks=Tasks(
|
|
summary=SummaryPolicy(enabled=False),
|
|
tagging=TaggingPolicy(enabled=False),
|
|
),
|
|
presentation=ARTICLE_READER,
|
|
)},
|
|
)
|
|
|
|
def fetch(self) -> Iterator[Item]:
|
|
with httpx.Client() as client:
|
|
# Written by the pull job before any board is fetched (see
|
|
# `app/jobs/pull.py`); a plugin run on its own — a test, a dry
|
|
# run — has no run to stamp itself against, so it makes one up.
|
|
run_marker = self.state.get("ats:run", "") or datetime.now(UTC).isoformat(
|
|
timespec="seconds")
|
|
|
|
for target, label in self.targets():
|
|
host, _, slug = target.partition(":")
|
|
try:
|
|
rows = _fetch_rows(client, target)
|
|
except Exception as exc:
|
|
log.warning("ats: %s failed (%s)", target, exc)
|
|
continue
|
|
|
|
seen_ids = []
|
|
for row in rows:
|
|
# The compound id, matching `roles.source_id` — what
|
|
# `Store.mark_closed` compares against, not the board's own
|
|
# bare job id.
|
|
compound_id = f"{host}:{slug}:{row['source_id']}"
|
|
seen_ids.append(compound_id)
|
|
yield Item(
|
|
source_id=compound_id,
|
|
location=row["url"],
|
|
title=row["title"],
|
|
excerpt=(row["description"] or "")[:300],
|
|
body=row["description"],
|
|
published_at=row["posted"],
|
|
subscription=label,
|
|
kind="role",
|
|
raw={
|
|
"company": label,
|
|
"place": row["location"],
|
|
"department": row["department"],
|
|
"function": classify(row["department"], row["title"]),
|
|
"remote": row["remote"],
|
|
"salary": row["salary"],
|
|
# Only Ashby, Greenhouse, Lever, Recruitee and
|
|
# Personio's parsers set these (see "structured
|
|
# pay" above); the rest leave them at row.get's
|
|
# default of no figure.
|
|
"salary_min": row.get("salary_min"),
|
|
"salary_max": row.get("salary_max"),
|
|
"currency": row.get("currency", ""),
|
|
"board": host,
|
|
"slug": slug,
|
|
},
|
|
)
|
|
# Stamped only once the board has parsed without raising, so a
|
|
# board that blew up half way through is neither marked ok nor
|
|
# has a partial id list treated as the whole of it.
|
|
self.state[f"ats:ok:{target}"] = run_marker
|
|
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|