Unpack structured pay from five ATS hosts; fix the function classifier

Part A: Ashby (?includeCompensation=true), Greenhouse (pay_transparency=true,
already free on the list request), Lever (salaryRange), Recruitee (salary)
and Personio (salaryInformation) all give structured pay; Workable and
SmartRecruiters do not (checked live). Every host converts to a yearly
salary_min/salary_max/currency, multiplying a monthly figure by 12 and
skipping an hourly or daily one rather than annualising a number nobody
stated. A range with no currency is ignored, and Greenhouse's own multi-zone
ranges are skipped outright since nothing says which zone the role is at.

Part B: classify() matched substrings, so " it" matched inside "Italy",
filing European sales roles as engineering; it now matches on word
boundaries (full boundary for the handful of short tokens that are
themselves the start of an unrelated word, prefix matching kept for the
stems). Title is checked before department (a hand-checked sample found
title-first right about twice as often), nothing matching returns '' rather
than "other", and a modest set of German/Dutch/Danish/Norwegian/Swedish/
French keywords covers the seed's other languages.

function is not a board field, so it gets the opposite rule from every
other raw column: extract's answer overwrites the classifier's guess
outright, and once extracted_at is set no later pull may put the guess
back. Migration 9 recomputes function for every not-yet-extracted role from
its stored title and department.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet 5 committed 2026-09-29 14:29:12 +02:00
1 parent 0558ef19ed
commit 44ff7c0131
10 files changed
+458 -35

No files matched your search

+199 -18
View File
@@ -56,11 +56,13 @@ MAX_BYTES = 96 << 20
#: host -> (feed URL template, careers URL template). `{slug}` is filled in;
#: recruitee and personio put the slug in the hostname instead of the path.
_BOARDS: dict[str, tuple[str, str]] = {
"greenhouse": ("https://boards-api.greenhouse.io/v1/boards/{slug}/jobs?content=true",
"greenhouse": ("https://boards-api.greenhouse.io/v1/boards/{slug}/jobs"
"?content=true&pay_transparency=true",
"https://job-boards.greenhouse.io/{slug}"),
"lever": ("https://api.lever.co/v0/postings/{slug}?mode=json",
"https://jobs.lever.co/{slug}"),
"ashby": ("https://api.ashbyhq.com/posting-api/job-board/{slug}",
"ashby": ("https://api.ashbyhq.com/posting-api/job-board/{slug}"
"?includeCompensation=true",
"https://jobs.ashbyhq.com/{slug}"),
"workable": ("https://apply.workable.com/api/v3/accounts/{slug}/jobs",
"https://apply.workable.com/{slug}"),
@@ -178,6 +180,88 @@ def _parse_dt(value: Any) -> datetime | None:
Row = dict[str, Any]
# ---------------------------------------------------------- structured pay
#
# A board's own salary range, mapped to yearly `salary_min`/`salary_max`
# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks
# `raw` into, per docs/decisions.md's "raw is transport, columns are
# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give
# structured pay on their list endpoint; Workable and SmartRecruiters do not
# (checked live, 2026-09-29), so their parsers are untouched and `row.get`
# below reads a plain '' or None for them.
def _num(value: Any) -> float | None:
"""A positive number from whatever a board sends (int, float, or a
numeral string), or None for anything else, including zero and a
negative — a board's placeholder 0 (an unfilled field, not a real
figure) must not become a salary floor of zero."""
try:
number = float(value)
except (TypeError, ValueError):
return None
return number if number > 0 else None
def _yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any,
factors: dict[str, int]) -> tuple[int | None, int | None, str]:
"""`(salary_min, salary_max, currency)` as yearly integers, or `(None,
None, "")` when there is no currency, no bound, or `interval` is not one
of `factors`' own keys — which is how an hourly or a daily rate is left
out rather than annualised into a number nobody stated. `factors` maps
the board's own interval spelling to the multiplier that makes it
yearly: 1 for a figure already yearly, 12 for a monthly one."""
code = str(currency or "").strip().upper()
factor = factors.get(str(interval or "").strip())
if not code or not factor:
return None, None, ""
lo, hi = _num(min_value), _num(max_value)
if lo is None and hi is None:
return None, None, ""
return (int(lo * factor) if lo is not None else None,
int(hi * factor) if hi is not None else None, code)
def _greenhouse_salary(ranges: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
"""The yearly range from `pay_input_ranges` (needs `pay_transparency=true`
on the feed URL, see `_BOARDS`), which the list endpoint already carries
in the same request — no per-role fetch. Only when there is exactly one
range: a board with several ("Zone 1 Pay Range", "US Tier 2 Locations",
one per office) gives no way to tell which office the role in hand is
at, and picking one would misrepresent the columns. Greenhouse's own
ranges are cents, and yearly by convention except an hourly or a daily
one, named as such in `title` with no separate field to check — those
are skipped rather than annualised into a number nobody stated."""
if len(ranges) != 1:
return None, None, ""
entry = ranges[0]
title = str(entry.get("title") or "").lower()
if any(word in title for word in ("hour", "/hr", "daily", "per day", "day rate")):
return None, None, ""
code = str(entry.get("currency_type") or "").strip().upper()
if not code:
return None, None, ""
lo, hi = _num(entry.get("min_cents")), _num(entry.get("max_cents"))
if lo is None and hi is None:
return None, None, ""
return (int(lo / 100) if lo is not None else None,
int(hi / 100) if hi is not None else None, code)
def _ashby_salary(components: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
"""The yearly range from `compensation.summaryComponents` (needs
`?includeCompensation=true` on the feed URL, see `_BOARDS`): the first
`Salary` component, yearly or monthly (a monthly one multiplied by 12);
an hourly one is left out rather than annualised."""
for component in components:
if component.get("compensationType") != "Salary":
continue
return _yearly_amount(component.get("minValue"), component.get("maxValue"),
component.get("currencyCode"), component.get("interval"),
{"1 YEAR": 1, "1 MONTH": 12})
return None, None, ""
def _parse_greenhouse(body: bytes) -> list[Row]:
data = json.loads(body)
rows = []
@@ -191,6 +275,7 @@ def _parse_greenhouse(body: bytes) -> list[Row]:
# JSON string (`&lt;p&gt;`), so it needs unescaping before the tags
# themselves can be stripped.
content = html.unescape(j.get("content") or "")
salary_min, salary_max, currency = _greenhouse_salary(j.get("pay_input_ranges") or [])
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
@@ -201,6 +286,9 @@ def _parse_greenhouse(body: bytes) -> list[Row]:
"description": strip_html(content),
"remote": None, # Greenhouse's feed does not say
"salary": "",
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
@@ -217,6 +305,10 @@ def _parse_lever(body: bytes) -> list[Row]:
continue
categories = j.get("categories") or {}
description = j.get("description") or j.get("descriptionPlain") or ""
salary_range = j.get("salaryRange") or {}
salary_min, salary_max, currency = _yearly_amount(
salary_range.get("min"), salary_range.get("max"), salary_range.get("currency"),
salary_range.get("interval"), {"per-year-salary": 1, "per-month-salary": 12})
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
@@ -227,6 +319,9 @@ def _parse_lever(body: bytes) -> list[Row]:
"description": strip_html(description),
"remote": (j.get("workplaceType") or "").strip() or None,
"salary": "",
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
@@ -243,6 +338,7 @@ def _parse_ashby(body: bytes) -> list[Row]:
compensation = j.get("compensation") or {}
salary = (compensation.get("compensationTierSummary")
or compensation.get("scrapeableCompensationSalarySummary") or "")
salary_min, salary_max, currency = _ashby_salary(compensation.get("summaryComponents") or [])
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
@@ -253,6 +349,9 @@ def _parse_ashby(body: bytes) -> list[Row]:
"description": strip_html(description),
"remote": j.get("isRemote"),
"salary": salary,
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
@@ -334,6 +433,9 @@ def _parse_recruitee(body: bytes) -> list[Row]:
if isinstance(salary, dict) and (salary.get("min") or salary.get("max")):
bounds = "-".join(str(v) for v in (salary.get("min"), salary.get("max")) if v)
salary_text = f"{bounds} {salary.get('currency') or ''}".strip()
salary_min, salary_max, currency = _yearly_amount(
salary.get("min"), salary.get("max"), salary.get("currency"),
salary.get("period"), {"year": 1, "month": 12})
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
@@ -344,6 +446,9 @@ def _parse_recruitee(body: bytes) -> list[Row]:
"description": strip_html(j.get("description") or ""),
"remote": j.get("remote"),
"salary": salary_text,
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
@@ -361,6 +466,14 @@ def _parse_personio(body: bytes, slug: str) -> list[Row]:
continue
description = " ".join(
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
salary_info = position.find("salaryInformation")
salary_min = salary_max = None
currency = ""
if salary_info is not None:
salary_min, salary_max, currency = _yearly_amount(
salary_info.findtext("min"), salary_info.findtext("max"),
salary_info.findtext("currencyCode"), salary_info.findtext("type"),
{"yearly": 1, "monthly": 12})
rows.append({
"source_id": job_id,
"title": title,
@@ -371,6 +484,9 @@ def _parse_personio(body: bytes, slug: str) -> list[Row]:
"description": strip_html(description),
"remote": "remote" in (position.findtext("office") or "").lower() or None,
"salary": "",
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
@@ -390,14 +506,27 @@ _PARSERS: dict[str, Callable[..., list[Row]]] = {
# --------------------------------------------------------- the classifier
#: The fixed set of functions, and the keyword rules that map a department
#: (preferred) or a title to one of them. Ported verbatim, in order, from
#: `cmd/seedgen/main.go`'s `rules` — first match wins, and "product" is
#: genuinely listed twice: the first, narrower entry is checked before
#: The fixed set of functions, and the keyword rules that map a title
#: (preferred) or a department to one of them. Ported from `cmd/seedgen/
#: main.go`'s `rules` — first match wins, and "product" is genuinely listed
#: twice: the first, narrower entry is checked before
#: recruiting/design/marketing/support/sales/data, and the second, which also
#: matches the bare word "product", is checked only after all of those, so a
#: title like "Data Product Manager" keeps classifying as data/product by the
#: same rule order the Go tool used.
#:
#: Not a straight port any more: the Go tool matched substrings, which let
#: " it" (meant to catch "IT" as a department) match inside "Italy" and
#: "Italia" — a European department like "Commercial Italy" filed as
#: engineering. `_classify_text` now matches on word boundaries. A handful of
#: short tokens (`_WHOLE_WORD`) need a boundary on *both* sides, because they
#: are themselves the start of an unrelated word ("it" of "Italy", "seo" of
#: "Seoul", "qa" of "Qatar"); every other keyword keeps a leading boundary
#: only; most of them are deliberate stems ("manag", "engineer", "recruit")
#: meant to also catch "management", "engineering", "recruiting", and a
#: trailing boundary on those would stop them matching the plural and
#: gerund forms real postings use ("Software Developers", "Account
#: Managers").
FUNCTIONS = ("engineering", "data", "product", "design", "sales", "marketing",
"customer success/support", "recruiting/people", "finance/legal/ops")
@@ -406,7 +535,7 @@ _RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
"program manag", "technical program")),
("recruiting/people", ("recruit", "talent", "people ops", "people &",
"people team", "people operations", "human resources",
" hr ", "hr ", "hrbp")),
"hr", "hrbp")),
("design", ("design", "ux", "user experience", "creative")),
("marketing", ("marketing", "growth", "content", "communications", "brand",
"seo", "social media", "demand gen", "events")),
@@ -419,8 +548,17 @@ _RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
("sales", ("sales", "account executive", "account manager", "account director",
"business development", "bdr", "sdr", "revenue", "partnership",
"go-to-market", "gtm", "solutions engineer", "solution engineer",
"pre-sales", "presales", "channel")),
("data", ("data", "analytics", "machine learning", "ml ", " ml",
"pre-sales", "presales", "channel",
# "Commercial" is what French, Italian and Spanish org charts
# call the sales function ("Commercial Italy" is a sales
# department; see the docstring above), and the rest are the
# sales words of the seed's other languages that "sales"
# itself does not already cover (German, Dutch, Danish,
# Norwegian, Swedish, French) — a modest set, not a full port;
# `extract` is the rest.
"commercial", "vertrieb", "verkauf", "verkoop", "salg",
"försäljning", "vente")),
("data", ("data", "analytics", "machine learning", "ml",
"applied science", "research scientist", "statistic")),
("product", ("product manag", "product management", "program manag",
"product operations", "technical program", "product")),
@@ -433,28 +571,64 @@ _RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
"trust & safety", "trust and safety", "policy")),
("engineering", ("engineer", "software", "developer", "infrastructure",
"platform", "devops", "sre", "site reliability", "security",
"information technology", " it", "it ", "technology",
"information technology", "it", "technology",
"technical staff", "hardware", "qa", "quality assurance",
"architect", "research", "science", "scientist", "systems",
"network", "cloud", "mobile", "frontend", "backend",
"full stack", "fullstack", "web")),
"full stack", "fullstack", "web",
# The developer/engineer words of the seed's other
# languages, same reasoning as sales' above.
"entwickler", "ingenieur", "ontwikkelaar", "udvikler",
"utvikler", "utvecklare", "ingenjör", "ingeniør",
"développeur", "developpeur", "ingénieur")),
)
#: Keywords that must match a whole word, not just its start: each one is
#: itself the first letters of an unrelated word that would otherwise false-
#: match ("it" of "Italy", "seo" of "Seoul", "qa" of "Qatar"). Every keyword
#: not listed here keeps the old prefix behaviour on purpose — see the
#: `_RULES` docstring above.
_WHOLE_WORD = frozenset({
"it", "hr", "hrbp", "ux", "ml", "qa", "seo", "web", "tax", "ops",
"sre", "bdr", "sdr", "gtm",
})
def _keyword_pattern(keyword: str) -> re.Pattern[str]:
trailing = r"\b" if keyword in _WHOLE_WORD else ""
return re.compile(r"\b" + re.escape(keyword) + trailing, re.IGNORECASE)
#: `_RULES`, compiled once: `(function, patterns)` in the same order.
_COMPILED_RULES: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = tuple(
(function, tuple(_keyword_pattern(keyword) for keyword in keywords))
for function, keywords in _RULES)
def _classify_text(text: str) -> str:
padded = f" {text.strip().lower()} "
if not padded.strip():
stripped = (text or "").strip()
if not stripped:
return ""
for function, keywords in _RULES:
if any(keyword in padded for keyword in keywords):
for function, patterns in _COMPILED_RULES:
if any(pattern.search(stripped) for pattern in patterns):
return function
return ""
def classify(department: str, title: str) -> str:
"""A department (preferred) or a title, to one of `FUNCTIONS`, or
`"other"`. Port of seedgen's `classify`/`classifyText`."""
return _classify_text(department or "") or _classify_text(title or "") or "other"
"""A title (preferred) or a department, to one of `FUNCTIONS`, or `''`
when neither names one — `''` and not `"other"`, which would claim a
knowledge the classifier does not have (`extract` may still fill it from
reading the posting; see `Store.set_extracted`).
Title before department: a department is often a generic umbrella
("Operations", "Engineering & Product") that says less about the role
than its title does. A hand-checked sample of 60 roles where the two
orders disagreed (docs/decisions.md) found title-first right about twice
as often as department-first, mostly because a specific title beats a
broad department name, so title now wins; department is the fallback
when the title matches nothing."""
return _classify_text(title or "") or _classify_text(department or "") or ""
def is_known_target(target: str) -> bool:
@@ -570,6 +744,13 @@ class ATSSource(Source):
"function": classify(row["department"], row["title"]),
"remote": row["remote"],
"salary": row["salary"],
# Only Ashby, Greenhouse, Lever, Recruitee and
# Personio's parsers set these (see "structured
# pay" above); the rest leave them at row.get's
# default of no figure.
"salary_min": row.get("salary_min"),
"salary_max": row.get("salary_max"),
"currency": row.get("currency", ""),
"board": host,
"slug": slug,
},