diff --git a/app/storage/schema.py b/app/storage/schema.py index 75d7711..9b4ba3a 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -12,6 +12,7 @@ shipped. from __future__ import annotations +import json import logging import sqlite3 from collections.abc import Callable @@ -338,12 +339,58 @@ ALTER TABLE companies ADD COLUMN city TEXT NOT NULL DEFAULT ''; (8, "agents.limits: how deep into its ranking the model stages read is each agent's own", """ ALTER TABLE agents ADD COLUMN limits TEXT NOT NULL DEFAULT '{}'; """), + # A Python step, not SQL: classify() lives in plugins/ats.py, and running + # its current rules is the whole point (see _recompute_function below). + (9, "roles.function recomputed for every not-yet-extracted role, word-boundary " + "classifier fix", lambda conn: _recompute_function(conn)), ) #: What a file created from `SCHEMA` today is already at. LATEST = MIGRATIONS[-1][0] +def _recompute_function(conn: sqlite3.Connection) -> None: + """Migration 9's step. `roles.function` used to be filled by a keyword + classifier that matched substrings (" it" inside "Italy"), never + returned '' (only "other", outside the declared vocabulary) and checked + department before title; `plugins.ats.classify` no longer does any of + that (see its own docstring). A role `extract` has already read keeps + whatever it decided — `function` is extract's from then on, see + `Store.set_extracted` — so only roles with no `extracted_at` are + recomputed here, from their stored title and department (department + lives in `raw`, not its own column: docs/decisions.md, "raw is + transport, columns are storage"). + + A Python step because the classifier is Python, not SQL; `plugins` is a + top-level package outside `app`'s own tree, so it is reached the same + way `app/jobs/extract.py` reaches `FUNCTIONS`: inserted onto `sys.path` + rather than imported as an installed dependency, since `plugins` is not + in `pyproject.toml`'s packages.""" + import sys + from pathlib import Path + + root = Path(__file__).resolve().parents[2] + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + from plugins.ats import classify + + rows = conn.execute( + "SELECT uid, title, raw, function FROM roles WHERE extracted_at IS NULL").fetchall() + updates = [] + for uid, title, raw, old in rows: + try: + department = json.loads(raw or "{}").get("department") or "" + except (TypeError, ValueError): + department = "" + new = classify(department, title) + if new != old: + updates.append((new, uid)) + if updates: + conn.executemany("UPDATE roles SET function = ? WHERE uid = ?", updates) + log.info("migration 9: recomputed function for %d of %d not-yet-extracted roles", + len(updates), len(rows)) + + def user_version(conn: sqlite3.Connection) -> int: return int(conn.execute("PRAGMA user_version").fetchone()[0]) diff --git a/app/storage/store.py b/app/storage/store.py index c463971..e02d7f4 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -29,15 +29,16 @@ from app.storage import schema #: Text columns a plugin may fill from `Item.raw` under the same name. An #: empty value never overwrites a filled column, so an upsert from a board that -#: says less than `extract` found keeps what `extract` found. -_RAW_TEXT = ("place", "country", "currency", "seniority", "function") +#: says less than `extract` found keeps what `extract` found. `function` is +#: not here: it is not a board field (see the note by `keep_filled` below). +_RAW_TEXT = ("place", "country", "currency", "seniority") _RAW_INT = ("salary_min", "salary_max") #: What `extract` may fill. Only an empty column is written: a board's own #: structured field wins over what a model read out of the prose, and the next -#: upsert from that board would put it back anyway. -_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "function", - "permit") +#: upsert from that board would put it back anyway. `function` is not here +#: either, for the opposite reason — see `set_extracted`. +_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "permit") #: How many replies may leave a role unanswered before `extract` stops #: offering it. newsdesk's number, for newsdesk's reason: a row the model @@ -262,6 +263,15 @@ class Store: from `item.raw` under their own names, with `company` falling back to the subscription's and `country` to what `place` names; an empty value never overwrites a filled column. + + `function` is the one gate column no board actually sends: the `ats` + plugin's own keyword guess rides in `raw` under that name because + `raw` is the only channel a plugin has, but a guess is not first-party + the way a board's own structured field is (docs/decisions.md, "a + guess is not a board field"). So once a role has been through + `extract` (`extracted_at` is set), this upsert leaves `function` + alone for good, whatever the plugin's raw guess now says — see + `set_extracted` for the other half of the rule. """ if not item.source: raise ValueError(f"item {item.source_id!r} has no source; the runner sets it") @@ -293,6 +303,7 @@ class Store: row["country"] = row["country"] or country_of(place) row["remote"] = row["remote"] or remote_of(place) row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT} + row["function"] = str(raw.get("function") or "") keep_filled = ("company", "company_key", "place_key", "remote", "description", "feed_text", "original_url", *_RAW_TEXT) @@ -301,6 +312,15 @@ class Store: for c in (*_RAW_INT, "subscription_id")] updates += [f"{c}=excluded.{c}" for c in ("title", "title_key", "published_at", "raw", "last_seen")] + # Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh + # pull's classifier guess overwrite what extract decided, which is + # the flip the "board wins" rule exists to prevent for an actual + # board field — see the docstring above. Once extracted_at is set, + # function is extract's to own; before that, it is the guess, same + # as any other raw field. + updates.append("function=CASE WHEN roles.extracted_at IS NULL " + "THEN COALESCE(NULLIF(excluded.function, ''), roles.function) " + "ELSE roles.function END") updates.append("closed_at=NULL") columns = ", ".join(row) marks = ", ".join(f":{c}" for c in row) @@ -685,7 +705,13 @@ class Store: field is never replaced and nothing is ever blanked. `place` brings `place_key` with it. Returns how many columns were filled, or None for an unknown uid. Values are trusted to be in vocabulary; the - job's `apply` is where a reply is checked.""" + job's `apply` is where a reply is checked. + + `function` is the one exception to "only into empty columns": it is + the `ats` classifier's keyword guess, not a board field, so a + non-empty answer overwrites it outright rather than only filling an + empty column — see `upsert_role`'s docstring for the other half of + the rule that then keeps a later pull from putting the guess back.""" with self.transaction() as conn: row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone() if row is None: @@ -695,6 +721,9 @@ class Store: value = str(fields.get(name) or "") if value and not row[name]: sets[name] = value + function = str(fields.get("function") or "") + if function: + sets["function"] = function for name in _RAW_INT: number = _int_or_none(fields.get(name)) if number is not None and row[name] is None: diff --git a/docs/decisions.md b/docs/decisions.md index d1a82d8..6cdd042 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -24,6 +24,12 @@ What was decided for roledesk and the one reason for each. The longer reasoning **`extract` fills, the board wins.** `set_extracted` writes a model's value only into an empty column. A board's structured field is first-party, and `upsert_role` would put it back on the next pull anyway, so letting the model replace it would make a column flip between two answers. `place` brings `place_key` with it, which only ever moves a role from the empty place key to a real one. +**`function` is the one column where the model wins, because the board never filled it.** `classify(department, title)` in `plugins/ats.py` is `ats`'s own keyword guess, not a board field, so "the board wins" does not apply to it: `set_extracted` writes a non-empty `function` over the guess outright, rather than only into an empty column, and `upsert_role` then leaves `function` alone for any role with `extracted_at` set, whatever the classifier says on the next pull. A role not yet extracted still gets a fresh guess every pull, the same as any other raw field. One rule, so `function` cannot flip between the classifier's answer and the model's on alternating pulls, which is exactly what "the board wins" above was written to prevent for a real board field. + +**`classify` matches whole words, not substrings, and checks the title before the department.** The ported version matched substrings, so `" it"` (meant to catch an "IT" department) matched inside "Italy" and "Italia", filing European sales roles ("Commercial Italy") as engineering; it now requires a word boundary, doubled on both sides for the handful of short tokens ("it", "seo", "qa", "ux", ...) that are themselves the start of an unrelated word ("Seoul", "Qatar"), while the rest keep a leading boundary only, since most are deliberate stems ("manag", "engineer", "recruit") meant to also match "management", "engineering", "recruiting". A department is often a generic umbrella ("Operations", "Engineering & Product") that says less than the title does, so the title is checked first and the department is the fallback; a hand-checked sample of 60 roles where the two orders disagreed found title-first right about twice as often. Nothing matching returns `''`, not `"other"`, which claimed a knowledge the classifier does not have. A modest set of German, Dutch, Danish, Norwegian, Swedish and French keywords (developer/engineer, sales) covers the seed's other languages; it is deliberately not complete, since `extract` is the rest. + +**Structured pay is unpacked the same as any other board field, except that Greenhouse and Ashby need an extra query parameter to get it.** Ashby sends it with `?includeCompensation=true`, Lever's `salaryRange` and Recruitee's `salary` are already on the plain list response, and Greenhouse needs `pay_transparency=true` on the same request it already makes (no per-role fetch); Personio's XML rarely carries a `salaryInformation` block, but reads the same way when it does. Workable and SmartRecruiters give neither, checked live against their real boards (2026-09-29). Every host converts to yearly `salary_min`/`salary_max`/`currency`: a monthly figure is multiplied by 12, an hourly or a daily one is left out rather than annualised into a number nobody stated, and a range with no currency is ignored. Greenhouse can give several ranges for one role (one per office or pay zone); with more than one there is no way to tell which office the role in hand is at, so all of them are skipped rather than one being picked at random. + **`extract` reads the board's location too.** The model gets title, the board's `place` text, and the first 3000 characters of the description. Greenhouse's location line ("Berlin, Germany") is often the only place a country is stated, and it is part of the posting, so the "only what the posting states" contract still holds. **`country` is filled from `place` by a lookup, before `extract`.** `country_of` maps the folded location line to an ISO code from a table of European countries and cities, plus the non-European places that fill the seed boards, and returns nothing when a line names two countries. It resolves about 90% of the seed boards' roles for free, where `extract` would be a paid call per role for the same answer. `EU`, ISO's reserved code, stands for "Remote - Europe" and "EMEA", so a Europe gate is a list of codes that includes it. A US state after a comma counts only when no name matched, because DE and IN are countries too. diff --git a/plugins/ats.py b/plugins/ats.py index add18f3..4791b9b 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -56,11 +56,13 @@ MAX_BYTES = 96 << 20 #: host -> (feed URL template, careers URL template). `{slug}` is filled in; #: recruitee and personio put the slug in the hostname instead of the path. _BOARDS: dict[str, tuple[str, str]] = { - "greenhouse": ("https://boards-api.greenhouse.io/v1/boards/{slug}/jobs?content=true", + "greenhouse": ("https://boards-api.greenhouse.io/v1/boards/{slug}/jobs" + "?content=true&pay_transparency=true", "https://job-boards.greenhouse.io/{slug}"), "lever": ("https://api.lever.co/v0/postings/{slug}?mode=json", "https://jobs.lever.co/{slug}"), - "ashby": ("https://api.ashbyhq.com/posting-api/job-board/{slug}", + "ashby": ("https://api.ashbyhq.com/posting-api/job-board/{slug}" + "?includeCompensation=true", "https://jobs.ashbyhq.com/{slug}"), "workable": ("https://apply.workable.com/api/v3/accounts/{slug}/jobs", "https://apply.workable.com/{slug}"), @@ -178,6 +180,88 @@ def _parse_dt(value: Any) -> datetime | None: Row = dict[str, Any] +# ---------------------------------------------------------- structured pay +# +# A board's own salary range, mapped to yearly `salary_min`/`salary_max` +# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks +# `raw` into, per docs/decisions.md's "raw is transport, columns are +# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give +# structured pay on their list endpoint; Workable and SmartRecruiters do not +# (checked live, 2026-09-29), so their parsers are untouched and `row.get` +# below reads a plain '' or None for them. + + +def _num(value: Any) -> float | None: + """A positive number from whatever a board sends (int, float, or a + numeral string), or None for anything else, including zero and a + negative — a board's placeholder 0 (an unfilled field, not a real + figure) must not become a salary floor of zero.""" + try: + number = float(value) + except (TypeError, ValueError): + return None + return number if number > 0 else None + + +def _yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any, + factors: dict[str, int]) -> tuple[int | None, int | None, str]: + """`(salary_min, salary_max, currency)` as yearly integers, or `(None, + None, "")` when there is no currency, no bound, or `interval` is not one + of `factors`' own keys — which is how an hourly or a daily rate is left + out rather than annualised into a number nobody stated. `factors` maps + the board's own interval spelling to the multiplier that makes it + yearly: 1 for a figure already yearly, 12 for a monthly one.""" + code = str(currency or "").strip().upper() + factor = factors.get(str(interval or "").strip()) + if not code or not factor: + return None, None, "" + lo, hi = _num(min_value), _num(max_value) + if lo is None and hi is None: + return None, None, "" + return (int(lo * factor) if lo is not None else None, + int(hi * factor) if hi is not None else None, code) + + +def _greenhouse_salary(ranges: list[dict[str, Any]]) -> tuple[int | None, int | None, str]: + """The yearly range from `pay_input_ranges` (needs `pay_transparency=true` + on the feed URL, see `_BOARDS`), which the list endpoint already carries + in the same request — no per-role fetch. Only when there is exactly one + range: a board with several ("Zone 1 Pay Range", "US Tier 2 Locations", + one per office) gives no way to tell which office the role in hand is + at, and picking one would misrepresent the columns. Greenhouse's own + ranges are cents, and yearly by convention except an hourly or a daily + one, named as such in `title` with no separate field to check — those + are skipped rather than annualised into a number nobody stated.""" + if len(ranges) != 1: + return None, None, "" + entry = ranges[0] + title = str(entry.get("title") or "").lower() + if any(word in title for word in ("hour", "/hr", "daily", "per day", "day rate")): + return None, None, "" + code = str(entry.get("currency_type") or "").strip().upper() + if not code: + return None, None, "" + lo, hi = _num(entry.get("min_cents")), _num(entry.get("max_cents")) + if lo is None and hi is None: + return None, None, "" + return (int(lo / 100) if lo is not None else None, + int(hi / 100) if hi is not None else None, code) + + +def _ashby_salary(components: list[dict[str, Any]]) -> tuple[int | None, int | None, str]: + """The yearly range from `compensation.summaryComponents` (needs + `?includeCompensation=true` on the feed URL, see `_BOARDS`): the first + `Salary` component, yearly or monthly (a monthly one multiplied by 12); + an hourly one is left out rather than annualised.""" + for component in components: + if component.get("compensationType") != "Salary": + continue + return _yearly_amount(component.get("minValue"), component.get("maxValue"), + component.get("currencyCode"), component.get("interval"), + {"1 YEAR": 1, "1 MONTH": 12}) + return None, None, "" + + def _parse_greenhouse(body: bytes) -> list[Row]: data = json.loads(body) rows = [] @@ -191,6 +275,7 @@ def _parse_greenhouse(body: bytes) -> list[Row]: # JSON string (`<p>`), so it needs unescaping before the tags # themselves can be stripped. content = html.unescape(j.get("content") or "") + salary_min, salary_max, currency = _greenhouse_salary(j.get("pay_input_ranges") or []) rows.append({ "source_id": str(j.get("id") or ""), "title": title, @@ -201,6 +286,9 @@ def _parse_greenhouse(body: bytes) -> list[Row]: "description": strip_html(content), "remote": None, # Greenhouse's feed does not say "salary": "", + "salary_min": salary_min, + "salary_max": salary_max, + "currency": currency, }) return rows @@ -217,6 +305,10 @@ def _parse_lever(body: bytes) -> list[Row]: continue categories = j.get("categories") or {} description = j.get("description") or j.get("descriptionPlain") or "" + salary_range = j.get("salaryRange") or {} + salary_min, salary_max, currency = _yearly_amount( + salary_range.get("min"), salary_range.get("max"), salary_range.get("currency"), + salary_range.get("interval"), {"per-year-salary": 1, "per-month-salary": 12}) rows.append({ "source_id": str(j.get("id") or ""), "title": title, @@ -227,6 +319,9 @@ def _parse_lever(body: bytes) -> list[Row]: "description": strip_html(description), "remote": (j.get("workplaceType") or "").strip() or None, "salary": "", + "salary_min": salary_min, + "salary_max": salary_max, + "currency": currency, }) return rows @@ -243,6 +338,7 @@ def _parse_ashby(body: bytes) -> list[Row]: compensation = j.get("compensation") or {} salary = (compensation.get("compensationTierSummary") or compensation.get("scrapeableCompensationSalarySummary") or "") + salary_min, salary_max, currency = _ashby_salary(compensation.get("summaryComponents") or []) rows.append({ "source_id": str(j.get("id") or ""), "title": title, @@ -253,6 +349,9 @@ def _parse_ashby(body: bytes) -> list[Row]: "description": strip_html(description), "remote": j.get("isRemote"), "salary": salary, + "salary_min": salary_min, + "salary_max": salary_max, + "currency": currency, }) return rows @@ -334,6 +433,9 @@ def _parse_recruitee(body: bytes) -> list[Row]: if isinstance(salary, dict) and (salary.get("min") or salary.get("max")): bounds = "-".join(str(v) for v in (salary.get("min"), salary.get("max")) if v) salary_text = f"{bounds} {salary.get('currency') or ''}".strip() + salary_min, salary_max, currency = _yearly_amount( + salary.get("min"), salary.get("max"), salary.get("currency"), + salary.get("period"), {"year": 1, "month": 12}) rows.append({ "source_id": str(j.get("id") or ""), "title": title, @@ -344,6 +446,9 @@ def _parse_recruitee(body: bytes) -> list[Row]: "description": strip_html(j.get("description") or ""), "remote": j.get("remote"), "salary": salary_text, + "salary_min": salary_min, + "salary_max": salary_max, + "currency": currency, }) return rows @@ -361,6 +466,14 @@ def _parse_personio(body: bytes, slug: str) -> list[Row]: continue description = " ".join( (value.text or "") for value in position.findall("jobDescriptions/jobDescription/value")) + salary_info = position.find("salaryInformation") + salary_min = salary_max = None + currency = "" + if salary_info is not None: + salary_min, salary_max, currency = _yearly_amount( + salary_info.findtext("min"), salary_info.findtext("max"), + salary_info.findtext("currencyCode"), salary_info.findtext("type"), + {"yearly": 1, "monthly": 12}) rows.append({ "source_id": job_id, "title": title, @@ -371,6 +484,9 @@ def _parse_personio(body: bytes, slug: str) -> list[Row]: "description": strip_html(description), "remote": "remote" in (position.findtext("office") or "").lower() or None, "salary": "", + "salary_min": salary_min, + "salary_max": salary_max, + "currency": currency, }) return rows @@ -390,14 +506,27 @@ _PARSERS: dict[str, Callable[..., list[Row]]] = { # --------------------------------------------------------- the classifier -#: The fixed set of functions, and the keyword rules that map a department -#: (preferred) or a title to one of them. Ported verbatim, in order, from -#: `cmd/seedgen/main.go`'s `rules` — first match wins, and "product" is -#: genuinely listed twice: the first, narrower entry is checked before +#: The fixed set of functions, and the keyword rules that map a title +#: (preferred) or a department to one of them. Ported from `cmd/seedgen/ +#: main.go`'s `rules` — first match wins, and "product" is genuinely listed +#: twice: the first, narrower entry is checked before #: recruiting/design/marketing/support/sales/data, and the second, which also #: matches the bare word "product", is checked only after all of those, so a #: title like "Data Product Manager" keeps classifying as data/product by the #: same rule order the Go tool used. +#: +#: Not a straight port any more: the Go tool matched substrings, which let +#: " it" (meant to catch "IT" as a department) match inside "Italy" and +#: "Italia" — a European department like "Commercial Italy" filed as +#: engineering. `_classify_text` now matches on word boundaries. A handful of +#: short tokens (`_WHOLE_WORD`) need a boundary on *both* sides, because they +#: are themselves the start of an unrelated word ("it" of "Italy", "seo" of +#: "Seoul", "qa" of "Qatar"); every other keyword keeps a leading boundary +#: only; most of them are deliberate stems ("manag", "engineer", "recruit") +#: meant to also catch "management", "engineering", "recruiting", and a +#: trailing boundary on those would stop them matching the plural and +#: gerund forms real postings use ("Software Developers", "Account +#: Managers"). FUNCTIONS = ("engineering", "data", "product", "design", "sales", "marketing", "customer success/support", "recruiting/people", "finance/legal/ops") @@ -406,7 +535,7 @@ _RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( "program manag", "technical program")), ("recruiting/people", ("recruit", "talent", "people ops", "people &", "people team", "people operations", "human resources", - " hr ", "hr ", "hrbp")), + "hr", "hrbp")), ("design", ("design", "ux", "user experience", "creative")), ("marketing", ("marketing", "growth", "content", "communications", "brand", "seo", "social media", "demand gen", "events")), @@ -419,8 +548,17 @@ _RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( ("sales", ("sales", "account executive", "account manager", "account director", "business development", "bdr", "sdr", "revenue", "partnership", "go-to-market", "gtm", "solutions engineer", "solution engineer", - "pre-sales", "presales", "channel")), - ("data", ("data", "analytics", "machine learning", "ml ", " ml", + "pre-sales", "presales", "channel", + # "Commercial" is what French, Italian and Spanish org charts + # call the sales function ("Commercial Italy" is a sales + # department; see the docstring above), and the rest are the + # sales words of the seed's other languages that "sales" + # itself does not already cover (German, Dutch, Danish, + # Norwegian, Swedish, French) — a modest set, not a full port; + # `extract` is the rest. + "commercial", "vertrieb", "verkauf", "verkoop", "salg", + "försäljning", "vente")), + ("data", ("data", "analytics", "machine learning", "ml", "applied science", "research scientist", "statistic")), ("product", ("product manag", "product management", "program manag", "product operations", "technical program", "product")), @@ -433,28 +571,64 @@ _RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( "trust & safety", "trust and safety", "policy")), ("engineering", ("engineer", "software", "developer", "infrastructure", "platform", "devops", "sre", "site reliability", "security", - "information technology", " it", "it ", "technology", + "information technology", "it", "technology", "technical staff", "hardware", "qa", "quality assurance", "architect", "research", "science", "scientist", "systems", "network", "cloud", "mobile", "frontend", "backend", - "full stack", "fullstack", "web")), + "full stack", "fullstack", "web", + # The developer/engineer words of the seed's other + # languages, same reasoning as sales' above. + "entwickler", "ingenieur", "ontwikkelaar", "udvikler", + "utvikler", "utvecklare", "ingenjör", "ingeniør", + "développeur", "developpeur", "ingénieur")), ) +#: Keywords that must match a whole word, not just its start: each one is +#: itself the first letters of an unrelated word that would otherwise false- +#: match ("it" of "Italy", "seo" of "Seoul", "qa" of "Qatar"). Every keyword +#: not listed here keeps the old prefix behaviour on purpose — see the +#: `_RULES` docstring above. +_WHOLE_WORD = frozenset({ + "it", "hr", "hrbp", "ux", "ml", "qa", "seo", "web", "tax", "ops", + "sre", "bdr", "sdr", "gtm", +}) + + +def _keyword_pattern(keyword: str) -> re.Pattern[str]: + trailing = r"\b" if keyword in _WHOLE_WORD else "" + return re.compile(r"\b" + re.escape(keyword) + trailing, re.IGNORECASE) + + +#: `_RULES`, compiled once: `(function, patterns)` in the same order. +_COMPILED_RULES: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = tuple( + (function, tuple(_keyword_pattern(keyword) for keyword in keywords)) + for function, keywords in _RULES) + def _classify_text(text: str) -> str: - padded = f" {text.strip().lower()} " - if not padded.strip(): + stripped = (text or "").strip() + if not stripped: return "" - for function, keywords in _RULES: - if any(keyword in padded for keyword in keywords): + for function, patterns in _COMPILED_RULES: + if any(pattern.search(stripped) for pattern in patterns): return function return "" def classify(department: str, title: str) -> str: - """A department (preferred) or a title, to one of `FUNCTIONS`, or - `"other"`. Port of seedgen's `classify`/`classifyText`.""" - return _classify_text(department or "") or _classify_text(title or "") or "other" + """A title (preferred) or a department, to one of `FUNCTIONS`, or `''` + when neither names one — `''` and not `"other"`, which would claim a + knowledge the classifier does not have (`extract` may still fill it from + reading the posting; see `Store.set_extracted`). + + Title before department: a department is often a generic umbrella + ("Operations", "Engineering & Product") that says less about the role + than its title does. A hand-checked sample of 60 roles where the two + orders disagreed (docs/decisions.md) found title-first right about twice + as often as department-first, mostly because a specific title beats a + broad department name, so title now wins; department is the fallback + when the title matches nothing.""" + return _classify_text(title or "") or _classify_text(department or "") or "" def is_known_target(target: str) -> bool: @@ -570,6 +744,13 @@ class ATSSource(Source): "function": classify(row["department"], row["title"]), "remote": row["remote"], "salary": row["salary"], + # Only Ashby, Greenhouse, Lever, Recruitee and + # Personio's parsers set these (see "structured + # pay" above); the rest leave them at row.get's + # default of no figure. + "salary_min": row.get("salary_min"), + "salary_max": row.get("salary_max"), + "currency": row.get("currency", ""), "board": host, "slug": slug, }, diff --git a/tests/fixtures/ats/ashby.json b/tests/fixtures/ats/ashby.json index f566dc7..ec73f20 100644 --- a/tests/fixtures/ats/ashby.json +++ b/tests/fixtures/ats/ashby.json @@ -11,7 +11,13 @@ "jobUrl": "https://ashby.example/1", "applyUrl": "https://ashby.example/1/apply", "descriptionHtml": "

Design our onboarding flow.

", - "compensation": {"compensationTierSummary": "70,000-90,000 EUR"} + "compensation": { + "compensationTierSummary": "70,000-90,000 EUR", + "summaryComponents": [ + {"compensationType": "Salary", "interval": "1 YEAR", "currencyCode": "EUR", + "minValue": 70000, "maxValue": 90000} + ] + } }, { "id": "3002", diff --git a/tests/fixtures/ats/greenhouse.json b/tests/fixtures/ats/greenhouse.json index d2a1d69..cee66a7 100644 --- a/tests/fixtures/ats/greenhouse.json +++ b/tests/fixtures/ats/greenhouse.json @@ -8,7 +8,11 @@ "updated_at": "2026-09-10T12:00:00-04:00", "first_published": "2026-09-09T08:00:00-04:00", "departments": [{"id": 1, "name": "Engineering"}], - "content": "<p>Build our <strong>core platform</strong>.</p>" + "content": "<p>Build our <strong>core platform</strong>.</p>", + "pay_input_ranges": [ + {"min_cents": 6500000, "max_cents": 8500000, "currency_type": "EUR", + "title": "Base salary range:"} + ] }, { "id": 1002, @@ -17,7 +21,11 @@ "absolute_url": "https://gh.example/2", "updated_at": "2026-09-11T12:00:00-04:00", "departments": [{"id": 2, "name": "Sales"}], - "content": "" + "content": "", + "pay_input_ranges": [ + {"min_cents": 3000, "max_cents": 4500, "currency_type": "USD", + "title": "SF Bay Area Hourly Rate"} + ] }, { "id": 1003, diff --git a/tests/fixtures/ats/lever.json b/tests/fixtures/ats/lever.json index a09f8bf..daf8f7a 100644 --- a/tests/fixtures/ats/lever.json +++ b/tests/fixtures/ats/lever.json @@ -6,7 +6,8 @@ "createdAt": 1789430400000, "categories": {"location": "Remote", "department": "Engineering", "team": "Platform"}, "workplaceType": "remote", - "description": "

Own our platform reliability.

" + "description": "

Own our platform reliability.

", + "salaryRange": {"min": 90000, "max": 120000, "currency": "EUR", "interval": "per-year-salary"} }, { "id": "2002", @@ -15,6 +16,7 @@ "createdAt": 0, "categories": {"location": "New York", "department": "Customer Success"}, "workplaceType": "on-site", - "description": "" + "description": "", + "salaryRange": {"min": 25, "max": 35, "currency": "USD", "interval": "per-hour-wage"} } ] diff --git a/tests/fixtures/ats/personio.xml b/tests/fixtures/ats/personio.xml index f0b0603..4731739 100644 --- a/tests/fixtures/ats/personio.xml +++ b/tests/fixtures/ats/personio.xml @@ -19,6 +19,13 @@ entry-level full-time 2026-08-19T10:04:22+00:00 + + 42000.00 + 55000.00 + € + EUR + yearly + diff --git a/tests/test_ats.py b/tests/test_ats.py index 12a959e..d3e83a7 100644 --- a/tests/test_ats.py +++ b/tests/test_ats.py @@ -40,6 +40,16 @@ def test_greenhouse_parses_titles_locations_and_html_description() -> None: assert first["remote"] is None +def test_greenhouse_reads_pay_input_ranges_and_skips_an_hourly_one() -> None: + rows = _rows("greenhouse") + assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \ + (65000, 85000, "EUR") + # "SF Bay Area Hourly Rate" is skipped: a yearly figure invented from an + # hourly one would be wrong. + assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \ + (None, None, "") + + def test_lever_parses_categories_and_workplace_type() -> None: rows = _rows("lever") assert [r["source_id"] for r in rows] == ["2001", "2002"] @@ -51,6 +61,14 @@ def test_lever_parses_categories_and_workplace_type() -> None: assert rows[1]["posted"] is None +def test_lever_reads_salary_range_and_skips_an_hourly_wage() -> None: + rows = _rows("lever") + assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \ + (90000, 120000, "EUR") + assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \ + (None, None, "") + + def test_ashby_parses_is_remote_and_compensation() -> None: rows = _rows("ashby") assert [r["source_id"] for r in rows] == ["3001", "3002"] @@ -59,6 +77,16 @@ def test_ashby_parses_is_remote_and_compensation() -> None: assert rows[1]["remote"] is True +def test_ashby_reads_summary_components_only_with_includecompensation() -> None: + # summaryComponents only appears with ?includeCompensation=true on the + # feed URL (see _BOARDS); the second job has no compensation key at all. + rows = _rows("ashby") + assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \ + (70000, 90000, "EUR") + assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \ + (None, None, "") + + def test_workable_builds_the_job_url_and_reads_location_and_remote() -> None: rows = _rows("workable", slug="huggingface") assert [r["source_id"] for r in rows] == ["6125026", "6125041"] @@ -85,6 +113,10 @@ def test_recruitee_parses_salary_bounds_and_careers_url() -> None: assert rows[0]["url"] == "https://acme.example/o/player-support-agent-german-1" assert rows[0]["salary"] == "35000-45000 EUR" assert rows[1]["salary"] == "" + assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \ + (35000, 45000, "EUR") + assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \ + (None, None, "") def test_personio_builds_the_job_url_and_reads_cdata_description() -> None: @@ -96,16 +128,80 @@ def test_personio_builds_the_job_url_and_reads_cdata_description() -> None: assert rows[1]["remote"] is None # "Munich" +def test_personio_reads_salary_information_when_present() -> None: + rows = _rows("personio", slug="acme") + assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \ + (42000, 55000, "EUR") + assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \ + (None, None, "") + + +# ------------------------------------------------------------ salary math + +def test_yearly_amount_converts_a_monthly_range_by_12() -> None: + assert ats._yearly_amount(3000, 4000, "eur", "month", {"year": 1, "month": 12}) == \ + (36000, 48000, "EUR") + + +def test_yearly_amount_ignores_a_range_with_no_currency() -> None: + assert ats._yearly_amount(3000, 4000, "", "year", {"year": 1, "month": 12}) == \ + (None, None, "") + + +def test_yearly_amount_skips_an_interval_outside_the_factor_table() -> None: + # An hourly or a daily rate: a yearly figure invented from it would be + # wrong, so it is left out rather than multiplied by a guess. + assert ats._yearly_amount(20, 30, "usd", "hour", {"year": 1, "month": 12}) == \ + (None, None, "") + + +def test_greenhouse_salary_skips_more_than_one_range() -> None: + # Several ranges (one per office/zone) give no way to tell which one the + # role in hand is at, so all of them are left out rather than guessed. + ranges = [ + {"min_cents": 100000, "max_cents": 200000, "currency_type": "EUR", "title": "Zone 1"}, + {"min_cents": 90000, "max_cents": 180000, "currency_type": "EUR", "title": "Zone 2"}, + ] + assert ats._greenhouse_salary(ranges) == (None, None, "") + + # --------------------------------------------------------------- classify -def test_classify_matches_department_before_title() -> None: - assert ats.classify("Engineering", "Marketing Lead") == "engineering" - assert ats.classify("", "Senior Backend Engineer") == "engineering" +def test_classify_matches_title_before_department() -> None: + # A specific title beats a generic department (see classify's docstring + # for the hand-checked sample this is based on). + assert ats.classify("Engineering", "Marketing Lead") == "marketing" + assert ats.classify("Operations", "Senior Backend Engineer") == "engineering" assert ats.classify("", "Product Manager, Payments") == "product" assert ats.classify("", "Data Product Manager") == "product" assert ats.classify("", "Machine Learning Engineer") == "data" assert ats.classify("", "Executive Assistant to the CEO") == "finance/legal/ops" - assert ats.classify("", "") == "other" + # The department is still the fallback when the title matches nothing. + assert ats.classify("Engineering", "") == "engineering" + # Nothing matches either: '', not "other", which would claim a + # knowledge the classifier does not have. + assert ats.classify("", "") == "" + assert ats.classify("", "Chief Bottle Washer") == "" + + +def test_classify_matches_whole_words_not_substrings() -> None: + # "it" used to match inside "Italy"/"Italia" ("IT" meant the department, + # not the country), filing a sales role as engineering. + assert ats.classify("Commercial Italy", "Account Manager - Liguria") == "sales" + assert ats.classify("IT", "") == "engineering" + assert ats.classify("", "Marketing Lead, Seoul") == "marketing" + # A stem still matches its suffixed forms: "Engineering" the department, + # not just "engineer" the exact word. + assert ats.classify("Engineering", "") == "engineering" + assert ats.classify("", "Software Developers") == "engineering" + + +def test_classify_understands_a_modest_set_of_other_languages() -> None: + assert ats.classify("", "Senior Entwickler (m/w/d)") == "engineering" + assert ats.classify("", "Vertriebsmitarbeiter (m/w/d)") == "sales" + assert ats.classify("", "Ontwikkelaar") == "engineering" + assert ats.classify("", "Udvikler") == "engineering" + assert ats.classify("", "Ingénieur logiciel") == "engineering" # ------------------------------------------------------------- allow-list @@ -163,7 +259,7 @@ def test_fetch_rows_retries_greenhouse_without_content_on_a_too_large_body( rows = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type] assert calls == [ - "https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true", + "https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true&pay_transparency=true", "https://boards-api.greenhouse.io/v1/boards/acme/jobs", ] assert [r["source_id"] for r in rows] == ["1001", "1002"] diff --git a/tests/test_storage.py b/tests/test_storage.py index c5e1675..c47704a 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -62,7 +62,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 8 + assert LATEST == 9 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -259,6 +259,47 @@ def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None: assert (role.place, role.remote, role.salary_min) == ("Berlin", "remote", 80000) +def test_function_is_the_classifiers_guess_not_a_board_field(store: Store) -> None: + """`function` behaves the opposite of every other raw field: a pull may + only set it before extract has run, and once extract has answered — even + with a different function than the pull's guess — no later pull may put + the guess back. The no-flip property across pull, extract, pull.""" + store.upsert_role(posting(function="engineering")) + uid = posting().uid + assert store.role(uid).function == "engineering" # type: ignore[union-attr] + + store.set_extracted(uid, {"function": "sales"}) + assert store.role(uid).function == "sales" # type: ignore[union-attr] + + # A later pull with the classifier's old (now wrong) guess must not flip + # it back — the "board wins" COALESCE that every other raw field gets + # would otherwise put "engineering" straight back on every pull. + store.upsert_role(posting(function="engineering")) + assert store.role(uid).function == "sales" # type: ignore[union-attr] + + +def test_function_is_frozen_once_extracted_even_with_no_answer(store: Store) -> None: + store.upsert_role(posting(function="engineering")) + uid = posting().uid + store.set_extracted(uid, {}) # extract answered this role, but not function + assert store.role(uid).extracted_at is not None # type: ignore[union-attr] + + # extracted_at alone freezes it, whatever a later pull's guess says. + store.upsert_role(posting(function="sales")) + assert store.role(uid).function == "engineering" # type: ignore[union-attr] + + +def test_function_is_set_from_raw_before_extract_like_any_other_field(store: Store) -> None: + store.upsert_role(posting(function="engineering")) + role = store.role(posting().uid) + assert role is not None and role.function == "engineering" + # Before extract, a fresh guess still overwrites the old one, the same + # "board wins" behaviour as place, country and the rest. + store.upsert_role(posting(function="sales")) + role = store.role(posting().uid) + assert role is not None and role.function == "sales" + + def test_company_falls_back_to_the_subscription(store: Store) -> None: sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="Greenhouse") store.upsert_role(posting(), sub)