Unpack structured pay from five ATS hosts; fix the function classifier

Part A: Ashby (?includeCompensation=true), Greenhouse (pay_transparency=true,
already free on the list request), Lever (salaryRange), Recruitee (salary)
and Personio (salaryInformation) all give structured pay; Workable and
SmartRecruiters do not (checked live). Every host converts to a yearly
salary_min/salary_max/currency, multiplying a monthly figure by 12 and
skipping an hourly or daily one rather than annualising a number nobody
stated. A range with no currency is ignored, and Greenhouse's own multi-zone
ranges are skipped outright since nothing says which zone the role is at.

Part B: classify() matched substrings, so " it" matched inside "Italy",
filing European sales roles as engineering; it now matches on word
boundaries (full boundary for the handful of short tokens that are
themselves the start of an unrelated word, prefix matching kept for the
stems). Title is checked before department (a hand-checked sample found
title-first right about twice as often), nothing matching returns '' rather
than "other", and a modest set of German/Dutch/Danish/Norwegian/Swedish/
French keywords covers the seed's other languages.

function is not a board field, so it gets the opposite rule from every
other raw column: extract's answer overwrites the classifier's guess
outright, and once extracted_at is set no later pull may put the guess
back. Migration 9 recomputes function for every not-yet-extracted role from
its stored title and department.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet 5 committed 2026-09-29 14:29:12 +02:00
1 parent 0558ef19ed
commit 44ff7c0131
10 files changed
+458 -35

No files matched your search

+47
View File
@@ -12,6 +12,7 @@ shipped.
from __future__ import annotations
import json
import logging
import sqlite3
from collections.abc import Callable
@@ -338,12 +339,58 @@ ALTER TABLE companies ADD COLUMN city TEXT NOT NULL DEFAULT '';
(8, "agents.limits: how deep into its ranking the model stages read is each agent's own", """
ALTER TABLE agents ADD COLUMN limits TEXT NOT NULL DEFAULT '{}';
"""),
# A Python step, not SQL: classify() lives in plugins/ats.py, and running
# its current rules is the whole point (see _recompute_function below).
(9, "roles.function recomputed for every not-yet-extracted role, word-boundary "
"classifier fix", lambda conn: _recompute_function(conn)),
)
#: What a file created from `SCHEMA` today is already at.
LATEST = MIGRATIONS[-1][0]
def _recompute_function(conn: sqlite3.Connection) -> None:
"""Migration 9's step. `roles.function` used to be filled by a keyword
classifier that matched substrings (" it" inside "Italy"), never
returned '' (only "other", outside the declared vocabulary) and checked
department before title; `plugins.ats.classify` no longer does any of
that (see its own docstring). A role `extract` has already read keeps
whatever it decided — `function` is extract's from then on, see
`Store.set_extracted` — so only roles with no `extracted_at` are
recomputed here, from their stored title and department (department
lives in `raw`, not its own column: docs/decisions.md, "raw is
transport, columns are storage").
A Python step because the classifier is Python, not SQL; `plugins` is a
top-level package outside `app`'s own tree, so it is reached the same
way `app/jobs/extract.py` reaches `FUNCTIONS`: inserted onto `sys.path`
rather than imported as an installed dependency, since `plugins` is not
in `pyproject.toml`'s packages."""
import sys
from pathlib import Path
root = Path(__file__).resolve().parents[2]
if str(root) not in sys.path:
sys.path.insert(0, str(root))
from plugins.ats import classify
rows = conn.execute(
"SELECT uid, title, raw, function FROM roles WHERE extracted_at IS NULL").fetchall()
updates = []
for uid, title, raw, old in rows:
try:
department = json.loads(raw or "{}").get("department") or ""
except (TypeError, ValueError):
department = ""
new = classify(department, title)
if new != old:
updates.append((new, uid))
if updates:
conn.executemany("UPDATE roles SET function = ? WHERE uid = ?", updates)
log.info("migration 9: recomputed function for %d of %d not-yet-extracted roles",
len(updates), len(rows))
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])
+35 -6
View File
@@ -29,15 +29,16 @@ from app.storage import schema
#: Text columns a plugin may fill from `Item.raw` under the same name. An
#: empty value never overwrites a filled column, so an upsert from a board that
#: says less than `extract` found keeps what `extract` found.
_RAW_TEXT = ("place", "country", "currency", "seniority", "function")
#: says less than `extract` found keeps what `extract` found. `function` is
#: not here: it is not a board field (see the note by `keep_filled` below).
_RAW_TEXT = ("place", "country", "currency", "seniority")
_RAW_INT = ("salary_min", "salary_max")
#: What `extract` may fill. Only an empty column is written: a board's own
#: structured field wins over what a model read out of the prose, and the next
#: upsert from that board would put it back anyway.
_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "function",
"permit")
#: upsert from that board would put it back anyway. `function` is not here
#: either, for the opposite reason — see `set_extracted`.
_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "permit")
#: How many replies may leave a role unanswered before `extract` stops
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
@@ -262,6 +263,15 @@ class Store:
from `item.raw` under their own names, with `company` falling back to
the subscription's and `country` to what `place` names; an empty value
never overwrites a filled column.
`function` is the one gate column no board actually sends: the `ats`
plugin's own keyword guess rides in `raw` under that name because
`raw` is the only channel a plugin has, but a guess is not first-party
the way a board's own structured field is (docs/decisions.md, "a
guess is not a board field"). So once a role has been through
`extract` (`extracted_at` is set), this upsert leaves `function`
alone for good, whatever the plugin's raw guess now says — see
`set_extracted` for the other half of the rule.
"""
if not item.source:
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
@@ -293,6 +303,7 @@ class Store:
row["country"] = row["country"] or country_of(place)
row["remote"] = row["remote"] or remote_of(place)
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
row["function"] = str(raw.get("function") or "")
keep_filled = ("company", "company_key", "place_key", "remote", "description",
"feed_text", "original_url", *_RAW_TEXT)
@@ -301,6 +312,15 @@ class Store:
for c in (*_RAW_INT, "subscription_id")]
updates += [f"{c}=excluded.{c}"
for c in ("title", "title_key", "published_at", "raw", "last_seen")]
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
# pull's classifier guess overwrite what extract decided, which is
# the flip the "board wins" rule exists to prevent for an actual
# board field — see the docstring above. Once extracted_at is set,
# function is extract's to own; before that, it is the guess, same
# as any other raw field.
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
"ELSE roles.function END")
updates.append("closed_at=NULL")
columns = ", ".join(row)
marks = ", ".join(f":{c}" for c in row)
@@ -685,7 +705,13 @@ class Store:
field is never replaced and nothing is ever blanked. `place` brings
`place_key` with it. Returns how many columns were filled, or None
for an unknown uid. Values are trusted to be in vocabulary; the
job's `apply` is where a reply is checked."""
job's `apply` is where a reply is checked.
`function` is the one exception to "only into empty columns": it is
the `ats` classifier's keyword guess, not a board field, so a
non-empty answer overwrites it outright rather than only filling an
empty column — see `upsert_role`'s docstring for the other half of
the rule that then keeps a later pull from putting the guess back."""
with self.transaction() as conn:
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
if row is None:
@@ -695,6 +721,9 @@ class Store:
value = str(fields.get(name) or "")
if value and not row[name]:
sets[name] = value
function = str(fields.get("function") or "")
if function:
sets["function"] = function
for name in _RAW_INT:
number = _int_or_none(fields.get(name))
if number is not None and row[name] is None:
+6
View File
@@ -24,6 +24,12 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**`extract` fills, the board wins.** `set_extracted` writes a model's value only into an empty column. A board's structured field is first-party, and `upsert_role` would put it back on the next pull anyway, so letting the model replace it would make a column flip between two answers. `place` brings `place_key` with it, which only ever moves a role from the empty place key to a real one.
**`function` is the one column where the model wins, because the board never filled it.** `classify(department, title)` in `plugins/ats.py` is `ats`'s own keyword guess, not a board field, so "the board wins" does not apply to it: `set_extracted` writes a non-empty `function` over the guess outright, rather than only into an empty column, and `upsert_role` then leaves `function` alone for any role with `extracted_at` set, whatever the classifier says on the next pull. A role not yet extracted still gets a fresh guess every pull, the same as any other raw field. One rule, so `function` cannot flip between the classifier's answer and the model's on alternating pulls, which is exactly what "the board wins" above was written to prevent for a real board field.
**`classify` matches whole words, not substrings, and checks the title before the department.** The ported version matched substrings, so `" it"` (meant to catch an "IT" department) matched inside "Italy" and "Italia", filing European sales roles ("Commercial Italy") as engineering; it now requires a word boundary, doubled on both sides for the handful of short tokens ("it", "seo", "qa", "ux", ...) that are themselves the start of an unrelated word ("Seoul", "Qatar"), while the rest keep a leading boundary only, since most are deliberate stems ("manag", "engineer", "recruit") meant to also match "management", "engineering", "recruiting". A department is often a generic umbrella ("Operations", "Engineering & Product") that says less than the title does, so the title is checked first and the department is the fallback; a hand-checked sample of 60 roles where the two orders disagreed found title-first right about twice as often. Nothing matching returns `''`, not `"other"`, which claimed a knowledge the classifier does not have. A modest set of German, Dutch, Danish, Norwegian, Swedish and French keywords (developer/engineer, sales) covers the seed's other languages; it is deliberately not complete, since `extract` is the rest.
**Structured pay is unpacked the same as any other board field, except that Greenhouse and Ashby need an extra query parameter to get it.** Ashby sends it with `?includeCompensation=true`, Lever's `salaryRange` and Recruitee's `salary` are already on the plain list response, and Greenhouse needs `pay_transparency=true` on the same request it already makes (no per-role fetch); Personio's XML rarely carries a `salaryInformation` block, but reads the same way when it does. Workable and SmartRecruiters give neither, checked live against their real boards (2026-09-29). Every host converts to yearly `salary_min`/`salary_max`/`currency`: a monthly figure is multiplied by 12, an hourly or a daily one is left out rather than annualised into a number nobody stated, and a range with no currency is ignored. Greenhouse can give several ranges for one role (one per office or pay zone); with more than one there is no way to tell which office the role in hand is at, so all of them are skipped rather than one being picked at random.
**`extract` reads the board's location too.** The model gets title, the board's `place` text, and the first 3000 characters of the description. Greenhouse's location line ("Berlin, Germany") is often the only place a country is stated, and it is part of the posting, so the "only what the posting states" contract still holds.
**`country` is filled from `place` by a lookup, before `extract`.** `country_of` maps the folded location line to an ISO code from a table of European countries and cities, plus the non-European places that fill the seed boards, and returns nothing when a line names two countries. It resolves about 90% of the seed boards' roles for free, where `extract` would be a paid call per role for the same answer. `EU`, ISO's reserved code, stands for "Remote - Europe" and "EMEA", so a Europe gate is a list of codes that includes it. A US state after a comma counts only when no name matched, because DE and IN are countries too.
+199 -18
View File
@@ -56,11 +56,13 @@ MAX_BYTES = 96 << 20
#: host -> (feed URL template, careers URL template). `{slug}` is filled in;
#: recruitee and personio put the slug in the hostname instead of the path.
_BOARDS: dict[str, tuple[str, str]] = {
"greenhouse": ("https://boards-api.greenhouse.io/v1/boards/{slug}/jobs?content=true",
"greenhouse": ("https://boards-api.greenhouse.io/v1/boards/{slug}/jobs"
"?content=true&pay_transparency=true",
"https://job-boards.greenhouse.io/{slug}"),
"lever": ("https://api.lever.co/v0/postings/{slug}?mode=json",
"https://jobs.lever.co/{slug}"),
"ashby": ("https://api.ashbyhq.com/posting-api/job-board/{slug}",
"ashby": ("https://api.ashbyhq.com/posting-api/job-board/{slug}"
"?includeCompensation=true",
"https://jobs.ashbyhq.com/{slug}"),
"workable": ("https://apply.workable.com/api/v3/accounts/{slug}/jobs",
"https://apply.workable.com/{slug}"),
@@ -178,6 +180,88 @@ def _parse_dt(value: Any) -> datetime | None:
Row = dict[str, Any]
# ---------------------------------------------------------- structured pay
#
# A board's own salary range, mapped to yearly `salary_min`/`salary_max`
# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks
# `raw` into, per docs/decisions.md's "raw is transport, columns are
# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give
# structured pay on their list endpoint; Workable and SmartRecruiters do not
# (checked live, 2026-09-29), so their parsers are untouched and `row.get`
# below reads a plain '' or None for them.
def _num(value: Any) -> float | None:
"""A positive number from whatever a board sends (int, float, or a
numeral string), or None for anything else, including zero and a
negative — a board's placeholder 0 (an unfilled field, not a real
figure) must not become a salary floor of zero."""
try:
number = float(value)
except (TypeError, ValueError):
return None
return number if number > 0 else None
def _yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any,
factors: dict[str, int]) -> tuple[int | None, int | None, str]:
"""`(salary_min, salary_max, currency)` as yearly integers, or `(None,
None, "")` when there is no currency, no bound, or `interval` is not one
of `factors`' own keys — which is how an hourly or a daily rate is left
out rather than annualised into a number nobody stated. `factors` maps
the board's own interval spelling to the multiplier that makes it
yearly: 1 for a figure already yearly, 12 for a monthly one."""
code = str(currency or "").strip().upper()
factor = factors.get(str(interval or "").strip())
if not code or not factor:
return None, None, ""
lo, hi = _num(min_value), _num(max_value)
if lo is None and hi is None:
return None, None, ""
return (int(lo * factor) if lo is not None else None,
int(hi * factor) if hi is not None else None, code)
def _greenhouse_salary(ranges: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
"""The yearly range from `pay_input_ranges` (needs `pay_transparency=true`
on the feed URL, see `_BOARDS`), which the list endpoint already carries
in the same request — no per-role fetch. Only when there is exactly one
range: a board with several ("Zone 1 Pay Range", "US Tier 2 Locations",
one per office) gives no way to tell which office the role in hand is
at, and picking one would misrepresent the columns. Greenhouse's own
ranges are cents, and yearly by convention except an hourly or a daily
one, named as such in `title` with no separate field to check — those
are skipped rather than annualised into a number nobody stated."""
if len(ranges) != 1:
return None, None, ""
entry = ranges[0]
title = str(entry.get("title") or "").lower()
if any(word in title for word in ("hour", "/hr", "daily", "per day", "day rate")):
return None, None, ""
code = str(entry.get("currency_type") or "").strip().upper()
if not code:
return None, None, ""
lo, hi = _num(entry.get("min_cents")), _num(entry.get("max_cents"))
if lo is None and hi is None:
return None, None, ""
return (int(lo / 100) if lo is not None else None,
int(hi / 100) if hi is not None else None, code)
def _ashby_salary(components: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
"""The yearly range from `compensation.summaryComponents` (needs
`?includeCompensation=true` on the feed URL, see `_BOARDS`): the first
`Salary` component, yearly or monthly (a monthly one multiplied by 12);
an hourly one is left out rather than annualised."""
for component in components:
if component.get("compensationType") != "Salary":
continue
return _yearly_amount(component.get("minValue"), component.get("maxValue"),
component.get("currencyCode"), component.get("interval"),
{"1 YEAR": 1, "1 MONTH": 12})
return None, None, ""
def _parse_greenhouse(body: bytes) -> list[Row]:
data = json.loads(body)
rows = []
@@ -191,6 +275,7 @@ def _parse_greenhouse(body: bytes) -> list[Row]:
# JSON string (`&lt;p&gt;`), so it needs unescaping before the tags
# themselves can be stripped.
content = html.unescape(j.get("content") or "")
salary_min, salary_max, currency = _greenhouse_salary(j.get("pay_input_ranges") or [])
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
@@ -201,6 +286,9 @@ def _parse_greenhouse(body: bytes) -> list[Row]:
"description": strip_html(content),
"remote": None, # Greenhouse's feed does not say
"salary": "",
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
@@ -217,6 +305,10 @@ def _parse_lever(body: bytes) -> list[Row]:
continue
categories = j.get("categories") or {}
description = j.get("description") or j.get("descriptionPlain") or ""
salary_range = j.get("salaryRange") or {}
salary_min, salary_max, currency = _yearly_amount(
salary_range.get("min"), salary_range.get("max"), salary_range.get("currency"),
salary_range.get("interval"), {"per-year-salary": 1, "per-month-salary": 12})
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
@@ -227,6 +319,9 @@ def _parse_lever(body: bytes) -> list[Row]:
"description": strip_html(description),
"remote": (j.get("workplaceType") or "").strip() or None,
"salary": "",
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
@@ -243,6 +338,7 @@ def _parse_ashby(body: bytes) -> list[Row]:
compensation = j.get("compensation") or {}
salary = (compensation.get("compensationTierSummary")
or compensation.get("scrapeableCompensationSalarySummary") or "")
salary_min, salary_max, currency = _ashby_salary(compensation.get("summaryComponents") or [])
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
@@ -253,6 +349,9 @@ def _parse_ashby(body: bytes) -> list[Row]:
"description": strip_html(description),
"remote": j.get("isRemote"),
"salary": salary,
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
@@ -334,6 +433,9 @@ def _parse_recruitee(body: bytes) -> list[Row]:
if isinstance(salary, dict) and (salary.get("min") or salary.get("max")):
bounds = "-".join(str(v) for v in (salary.get("min"), salary.get("max")) if v)
salary_text = f"{bounds} {salary.get('currency') or ''}".strip()
salary_min, salary_max, currency = _yearly_amount(
salary.get("min"), salary.get("max"), salary.get("currency"),
salary.get("period"), {"year": 1, "month": 12})
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
@@ -344,6 +446,9 @@ def _parse_recruitee(body: bytes) -> list[Row]:
"description": strip_html(j.get("description") or ""),
"remote": j.get("remote"),
"salary": salary_text,
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
@@ -361,6 +466,14 @@ def _parse_personio(body: bytes, slug: str) -> list[Row]:
continue
description = " ".join(
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
salary_info = position.find("salaryInformation")
salary_min = salary_max = None
currency = ""
if salary_info is not None:
salary_min, salary_max, currency = _yearly_amount(
salary_info.findtext("min"), salary_info.findtext("max"),
salary_info.findtext("currencyCode"), salary_info.findtext("type"),
{"yearly": 1, "monthly": 12})
rows.append({
"source_id": job_id,
"title": title,
@@ -371,6 +484,9 @@ def _parse_personio(body: bytes, slug: str) -> list[Row]:
"description": strip_html(description),
"remote": "remote" in (position.findtext("office") or "").lower() or None,
"salary": "",
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
return rows
@@ -390,14 +506,27 @@ _PARSERS: dict[str, Callable[..., list[Row]]] = {
# --------------------------------------------------------- the classifier
#: The fixed set of functions, and the keyword rules that map a department
#: (preferred) or a title to one of them. Ported verbatim, in order, from
#: `cmd/seedgen/main.go`'s `rules` — first match wins, and "product" is
#: genuinely listed twice: the first, narrower entry is checked before
#: The fixed set of functions, and the keyword rules that map a title
#: (preferred) or a department to one of them. Ported from `cmd/seedgen/
#: main.go`'s `rules` — first match wins, and "product" is genuinely listed
#: twice: the first, narrower entry is checked before
#: recruiting/design/marketing/support/sales/data, and the second, which also
#: matches the bare word "product", is checked only after all of those, so a
#: title like "Data Product Manager" keeps classifying as data/product by the
#: same rule order the Go tool used.
#:
#: Not a straight port any more: the Go tool matched substrings, which let
#: " it" (meant to catch "IT" as a department) match inside "Italy" and
#: "Italia" — a European department like "Commercial Italy" filed as
#: engineering. `_classify_text` now matches on word boundaries. A handful of
#: short tokens (`_WHOLE_WORD`) need a boundary on *both* sides, because they
#: are themselves the start of an unrelated word ("it" of "Italy", "seo" of
#: "Seoul", "qa" of "Qatar"); every other keyword keeps a leading boundary
#: only; most of them are deliberate stems ("manag", "engineer", "recruit")
#: meant to also catch "management", "engineering", "recruiting", and a
#: trailing boundary on those would stop them matching the plural and
#: gerund forms real postings use ("Software Developers", "Account
#: Managers").
FUNCTIONS = ("engineering", "data", "product", "design", "sales", "marketing",
"customer success/support", "recruiting/people", "finance/legal/ops")
@@ -406,7 +535,7 @@ _RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
"program manag", "technical program")),
("recruiting/people", ("recruit", "talent", "people ops", "people &",
"people team", "people operations", "human resources",
" hr ", "hr ", "hrbp")),
"hr", "hrbp")),
("design", ("design", "ux", "user experience", "creative")),
("marketing", ("marketing", "growth", "content", "communications", "brand",
"seo", "social media", "demand gen", "events")),
@@ -419,8 +548,17 @@ _RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
("sales", ("sales", "account executive", "account manager", "account director",
"business development", "bdr", "sdr", "revenue", "partnership",
"go-to-market", "gtm", "solutions engineer", "solution engineer",
"pre-sales", "presales", "channel")),
("data", ("data", "analytics", "machine learning", "ml ", " ml",
"pre-sales", "presales", "channel",
# "Commercial" is what French, Italian and Spanish org charts
# call the sales function ("Commercial Italy" is a sales
# department; see the docstring above), and the rest are the
# sales words of the seed's other languages that "sales"
# itself does not already cover (German, Dutch, Danish,
# Norwegian, Swedish, French) — a modest set, not a full port;
# `extract` is the rest.
"commercial", "vertrieb", "verkauf", "verkoop", "salg",
"försäljning", "vente")),
("data", ("data", "analytics", "machine learning", "ml",
"applied science", "research scientist", "statistic")),
("product", ("product manag", "product management", "program manag",
"product operations", "technical program", "product")),
@@ -433,28 +571,64 @@ _RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
"trust & safety", "trust and safety", "policy")),
("engineering", ("engineer", "software", "developer", "infrastructure",
"platform", "devops", "sre", "site reliability", "security",
"information technology", " it", "it ", "technology",
"information technology", "it", "technology",
"technical staff", "hardware", "qa", "quality assurance",
"architect", "research", "science", "scientist", "systems",
"network", "cloud", "mobile", "frontend", "backend",
"full stack", "fullstack", "web")),
"full stack", "fullstack", "web",
# The developer/engineer words of the seed's other
# languages, same reasoning as sales' above.
"entwickler", "ingenieur", "ontwikkelaar", "udvikler",
"utvikler", "utvecklare", "ingenjör", "ingeniør",
"développeur", "developpeur", "ingénieur")),
)
#: Keywords that must match a whole word, not just its start: each one is
#: itself the first letters of an unrelated word that would otherwise false-
#: match ("it" of "Italy", "seo" of "Seoul", "qa" of "Qatar"). Every keyword
#: not listed here keeps the old prefix behaviour on purpose — see the
#: `_RULES` docstring above.
_WHOLE_WORD = frozenset({
"it", "hr", "hrbp", "ux", "ml", "qa", "seo", "web", "tax", "ops",
"sre", "bdr", "sdr", "gtm",
})
def _keyword_pattern(keyword: str) -> re.Pattern[str]:
trailing = r"\b" if keyword in _WHOLE_WORD else ""
return re.compile(r"\b" + re.escape(keyword) + trailing, re.IGNORECASE)
#: `_RULES`, compiled once: `(function, patterns)` in the same order.
_COMPILED_RULES: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = tuple(
(function, tuple(_keyword_pattern(keyword) for keyword in keywords))
for function, keywords in _RULES)
def _classify_text(text: str) -> str:
padded = f" {text.strip().lower()} "
if not padded.strip():
stripped = (text or "").strip()
if not stripped:
return ""
for function, keywords in _RULES:
if any(keyword in padded for keyword in keywords):
for function, patterns in _COMPILED_RULES:
if any(pattern.search(stripped) for pattern in patterns):
return function
return ""
def classify(department: str, title: str) -> str:
"""A department (preferred) or a title, to one of `FUNCTIONS`, or
`"other"`. Port of seedgen's `classify`/`classifyText`."""
return _classify_text(department or "") or _classify_text(title or "") or "other"
"""A title (preferred) or a department, to one of `FUNCTIONS`, or `''`
when neither names one — `''` and not `"other"`, which would claim a
knowledge the classifier does not have (`extract` may still fill it from
reading the posting; see `Store.set_extracted`).
Title before department: a department is often a generic umbrella
("Operations", "Engineering & Product") that says less about the role
than its title does. A hand-checked sample of 60 roles where the two
orders disagreed (docs/decisions.md) found title-first right about twice
as often as department-first, mostly because a specific title beats a
broad department name, so title now wins; department is the fallback
when the title matches nothing."""
return _classify_text(title or "") or _classify_text(department or "") or ""
def is_known_target(target: str) -> bool:
@@ -570,6 +744,13 @@ class ATSSource(Source):
"function": classify(row["department"], row["title"]),
"remote": row["remote"],
"salary": row["salary"],
# Only Ashby, Greenhouse, Lever, Recruitee and
# Personio's parsers set these (see "structured
# pay" above); the rest leave them at row.get's
# default of no figure.
"salary_min": row.get("salary_min"),
"salary_max": row.get("salary_max"),
"currency": row.get("currency", ""),
"board": host,
"slug": slug,
},
+7 -1
View File
@@ -11,7 +11,13 @@
"jobUrl": "https://ashby.example/1",
"applyUrl": "https://ashby.example/1/apply",
"descriptionHtml": "<p>Design our <strong>onboarding</strong> flow.</p>",
"compensation": {"compensationTierSummary": "70,000-90,000 EUR"}
"compensation": {
"compensationTierSummary": "70,000-90,000 EUR",
"summaryComponents": [
{"compensationType": "Salary", "interval": "1 YEAR", "currencyCode": "EUR",
"minValue": 70000, "maxValue": 90000}
]
}
},
{
"id": "3002",
+10 -2
View File
@@ -8,7 +8,11 @@
"updated_at": "2026-09-10T12:00:00-04:00",
"first_published": "2026-09-09T08:00:00-04:00",
"departments": [{"id": 1, "name": "Engineering"}],
"content": "&lt;p&gt;Build our &lt;strong&gt;core platform&lt;/strong&gt;.&lt;/p&gt;"
"content": "&lt;p&gt;Build our &lt;strong&gt;core platform&lt;/strong&gt;.&lt;/p&gt;",
"pay_input_ranges": [
{"min_cents": 6500000, "max_cents": 8500000, "currency_type": "EUR",
"title": "Base salary range:"}
]
},
{
"id": 1002,
@@ -17,7 +21,11 @@
"absolute_url": "https://gh.example/2",
"updated_at": "2026-09-11T12:00:00-04:00",
"departments": [{"id": 2, "name": "Sales"}],
"content": ""
"content": "",
"pay_input_ranges": [
{"min_cents": 3000, "max_cents": 4500, "currency_type": "USD",
"title": "SF Bay Area Hourly Rate"}
]
},
{
"id": 1003,
+4 -2
View File
@@ -6,7 +6,8 @@
"createdAt": 1789430400000,
"categories": {"location": "Remote", "department": "Engineering", "team": "Platform"},
"workplaceType": "remote",
"description": "<p>Own our <b>platform</b> reliability.</p>"
"description": "<p>Own our <b>platform</b> reliability.</p>",
"salaryRange": {"min": 90000, "max": 120000, "currency": "EUR", "interval": "per-year-salary"}
},
{
"id": "2002",
@@ -15,6 +16,7 @@
"createdAt": 0,
"categories": {"location": "New York", "department": "Customer Success"},
"workplaceType": "on-site",
"description": ""
"description": "",
"salaryRange": {"min": 25, "max": 35, "currency": "USD", "interval": "per-hour-wage"}
}
]
+7
View File
@@ -19,6 +19,13 @@
<seniority>entry-level</seniority>
<schedule>full-time</schedule>
<createdAt>2026-08-19T10:04:22+00:00</createdAt>
<salaryInformation>
<min>42000.00</min>
<max>55000.00</max>
<currencySymbol>€</currencySymbol>
<currencyCode>EUR</currencyCode>
<type>yearly</type>
</salaryInformation>
</position>
<position>
+101 -5
View File
@@ -40,6 +40,16 @@ def test_greenhouse_parses_titles_locations_and_html_description() -> None:
assert first["remote"] is None
def test_greenhouse_reads_pay_input_ranges_and_skips_an_hourly_one() -> None:
rows = _rows("greenhouse")
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
(65000, 85000, "EUR")
# "SF Bay Area Hourly Rate" is skipped: a yearly figure invented from an
# hourly one would be wrong.
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
(None, None, "")
def test_lever_parses_categories_and_workplace_type() -> None:
rows = _rows("lever")
assert [r["source_id"] for r in rows] == ["2001", "2002"]
@@ -51,6 +61,14 @@ def test_lever_parses_categories_and_workplace_type() -> None:
assert rows[1]["posted"] is None
def test_lever_reads_salary_range_and_skips_an_hourly_wage() -> None:
rows = _rows("lever")
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
(90000, 120000, "EUR")
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
(None, None, "")
def test_ashby_parses_is_remote_and_compensation() -> None:
rows = _rows("ashby")
assert [r["source_id"] for r in rows] == ["3001", "3002"]
@@ -59,6 +77,16 @@ def test_ashby_parses_is_remote_and_compensation() -> None:
assert rows[1]["remote"] is True
def test_ashby_reads_summary_components_only_with_includecompensation() -> None:
# summaryComponents only appears with ?includeCompensation=true on the
# feed URL (see _BOARDS); the second job has no compensation key at all.
rows = _rows("ashby")
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
(70000, 90000, "EUR")
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
(None, None, "")
def test_workable_builds_the_job_url_and_reads_location_and_remote() -> None:
rows = _rows("workable", slug="huggingface")
assert [r["source_id"] for r in rows] == ["6125026", "6125041"]
@@ -85,6 +113,10 @@ def test_recruitee_parses_salary_bounds_and_careers_url() -> None:
assert rows[0]["url"] == "https://acme.example/o/player-support-agent-german-1"
assert rows[0]["salary"] == "35000-45000 EUR"
assert rows[1]["salary"] == ""
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
(35000, 45000, "EUR")
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
(None, None, "")
def test_personio_builds_the_job_url_and_reads_cdata_description() -> None:
@@ -96,16 +128,80 @@ def test_personio_builds_the_job_url_and_reads_cdata_description() -> None:
assert rows[1]["remote"] is None # "Munich"
def test_personio_reads_salary_information_when_present() -> None:
rows = _rows("personio", slug="acme")
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
(42000, 55000, "EUR")
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
(None, None, "")
# ------------------------------------------------------------ salary math
def test_yearly_amount_converts_a_monthly_range_by_12() -> None:
assert ats._yearly_amount(3000, 4000, "eur", "month", {"year": 1, "month": 12}) == \
(36000, 48000, "EUR")
def test_yearly_amount_ignores_a_range_with_no_currency() -> None:
assert ats._yearly_amount(3000, 4000, "", "year", {"year": 1, "month": 12}) == \
(None, None, "")
def test_yearly_amount_skips_an_interval_outside_the_factor_table() -> None:
# An hourly or a daily rate: a yearly figure invented from it would be
# wrong, so it is left out rather than multiplied by a guess.
assert ats._yearly_amount(20, 30, "usd", "hour", {"year": 1, "month": 12}) == \
(None, None, "")
def test_greenhouse_salary_skips_more_than_one_range() -> None:
# Several ranges (one per office/zone) give no way to tell which one the
# role in hand is at, so all of them are left out rather than guessed.
ranges = [
{"min_cents": 100000, "max_cents": 200000, "currency_type": "EUR", "title": "Zone 1"},
{"min_cents": 90000, "max_cents": 180000, "currency_type": "EUR", "title": "Zone 2"},
]
assert ats._greenhouse_salary(ranges) == (None, None, "")
# --------------------------------------------------------------- classify
def test_classify_matches_department_before_title() -> None:
assert ats.classify("Engineering", "Marketing Lead") == "engineering"
assert ats.classify("", "Senior Backend Engineer") == "engineering"
def test_classify_matches_title_before_department() -> None:
# A specific title beats a generic department (see classify's docstring
# for the hand-checked sample this is based on).
assert ats.classify("Engineering", "Marketing Lead") == "marketing"
assert ats.classify("Operations", "Senior Backend Engineer") == "engineering"
assert ats.classify("", "Product Manager, Payments") == "product"
assert ats.classify("", "Data Product Manager") == "product"
assert ats.classify("", "Machine Learning Engineer") == "data"
assert ats.classify("", "Executive Assistant to the CEO") == "finance/legal/ops"
assert ats.classify("", "") == "other"
# The department is still the fallback when the title matches nothing.
assert ats.classify("Engineering", "") == "engineering"
# Nothing matches either: '', not "other", which would claim a
# knowledge the classifier does not have.
assert ats.classify("", "") == ""
assert ats.classify("", "Chief Bottle Washer") == ""
def test_classify_matches_whole_words_not_substrings() -> None:
# "it" used to match inside "Italy"/"Italia" ("IT" meant the department,
# not the country), filing a sales role as engineering.
assert ats.classify("Commercial Italy", "Account Manager - Liguria") == "sales"
assert ats.classify("IT", "") == "engineering"
assert ats.classify("", "Marketing Lead, Seoul") == "marketing"
# A stem still matches its suffixed forms: "Engineering" the department,
# not just "engineer" the exact word.
assert ats.classify("Engineering", "") == "engineering"
assert ats.classify("", "Software Developers") == "engineering"
def test_classify_understands_a_modest_set_of_other_languages() -> None:
assert ats.classify("", "Senior Entwickler (m/w/d)") == "engineering"
assert ats.classify("", "Vertriebsmitarbeiter (m/w/d)") == "sales"
assert ats.classify("", "Ontwikkelaar") == "engineering"
assert ats.classify("", "Udvikler") == "engineering"
assert ats.classify("", "Ingénieur logiciel") == "engineering"
# ------------------------------------------------------------- allow-list
@@ -163,7 +259,7 @@ def test_fetch_rows_retries_greenhouse_without_content_on_a_too_large_body(
rows = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type]
assert calls == [
"https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true",
"https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true&pay_transparency=true",
"https://boards-api.greenhouse.io/v1/boards/acme/jobs",
]
assert [r["source_id"] for r in rows] == ["1001", "1002"]
+42 -1
View File
@@ -62,7 +62,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 8
assert LATEST == 9
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -259,6 +259,47 @@ def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
assert (role.place, role.remote, role.salary_min) == ("Berlin", "remote", 80000)
def test_function_is_the_classifiers_guess_not_a_board_field(store: Store) -> None:
"""`function` behaves the opposite of every other raw field: a pull may
only set it before extract has run, and once extract has answered — even
with a different function than the pull's guess — no later pull may put
the guess back. The no-flip property across pull, extract, pull."""
store.upsert_role(posting(function="engineering"))
uid = posting().uid
assert store.role(uid).function == "engineering" # type: ignore[union-attr]
store.set_extracted(uid, {"function": "sales"})
assert store.role(uid).function == "sales" # type: ignore[union-attr]
# A later pull with the classifier's old (now wrong) guess must not flip
# it back — the "board wins" COALESCE that every other raw field gets
# would otherwise put "engineering" straight back on every pull.
store.upsert_role(posting(function="engineering"))
assert store.role(uid).function == "sales" # type: ignore[union-attr]
def test_function_is_frozen_once_extracted_even_with_no_answer(store: Store) -> None:
store.upsert_role(posting(function="engineering"))
uid = posting().uid
store.set_extracted(uid, {}) # extract answered this role, but not function
assert store.role(uid).extracted_at is not None # type: ignore[union-attr]
# extracted_at alone freezes it, whatever a later pull's guess says.
store.upsert_role(posting(function="sales"))
assert store.role(uid).function == "engineering" # type: ignore[union-attr]
def test_function_is_set_from_raw_before_extract_like_any_other_field(store: Store) -> None:
store.upsert_role(posting(function="engineering"))
role = store.role(posting().uid)
assert role is not None and role.function == "engineering"
# Before extract, a fresh guess still overwrites the old one, the same
# "board wins" behaviour as place, country and the rest.
store.upsert_role(posting(function="sales"))
role = store.role(posting().uid)
assert role is not None and role.function == "sales"
def test_company_falls_back_to_the_subscription(store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="Greenhouse")
store.upsert_role(posting(), sub)