Unpack structured pay from five ATS hosts; fix the function classifier
Part A: Ashby (?includeCompensation=true), Greenhouse (pay_transparency=true, already free on the list request), Lever (salaryRange), Recruitee (salary) and Personio (salaryInformation) all give structured pay; Workable and SmartRecruiters do not (checked live). Every host converts to a yearly salary_min/salary_max/currency, multiplying a monthly figure by 12 and skipping an hourly or daily one rather than annualising a number nobody stated. A range with no currency is ignored, and Greenhouse's own multi-zone ranges are skipped outright since nothing says which zone the role is at. Part B: classify() matched substrings, so " it" matched inside "Italy", filing European sales roles as engineering; it now matches on word boundaries (full boundary for the handful of short tokens that are themselves the start of an unrelated word, prefix matching kept for the stems). Title is checked before department (a hand-checked sample found title-first right about twice as often), nothing matching returns '' rather than "other", and a modest set of German/Dutch/Danish/Norwegian/Swedish/ French keywords covers the seed's other languages. function is not a board field, so it gets the opposite rule from every other raw column: extract's answer overwrites the classifier's guess outright, and once extracted_at is set no later pull may put the guess back. Migration 9 recomputes function for every not-yet-extracted role from its stored title and department. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0558ef19ed
commit
44ff7c0131
10 files changed
+458
-35
No files matched your search
@@ -12,6 +12,7 @@ shipped.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections.abc import Callable
|
||||
@@ -338,12 +339,58 @@ ALTER TABLE companies ADD COLUMN city TEXT NOT NULL DEFAULT '';
|
||||
(8, "agents.limits: how deep into its ranking the model stages read is each agent's own", """
|
||||
ALTER TABLE agents ADD COLUMN limits TEXT NOT NULL DEFAULT '{}';
|
||||
"""),
|
||||
# A Python step, not SQL: classify() lives in plugins/ats.py, and running
|
||||
# its current rules is the whole point (see _recompute_function below).
|
||||
(9, "roles.function recomputed for every not-yet-extracted role, word-boundary "
|
||||
"classifier fix", lambda conn: _recompute_function(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
LATEST = MIGRATIONS[-1][0]
|
||||
|
||||
|
||||
def _recompute_function(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 9's step. `roles.function` used to be filled by a keyword
|
||||
classifier that matched substrings (" it" inside "Italy"), never
|
||||
returned '' (only "other", outside the declared vocabulary) and checked
|
||||
department before title; `plugins.ats.classify` no longer does any of
|
||||
that (see its own docstring). A role `extract` has already read keeps
|
||||
whatever it decided — `function` is extract's from then on, see
|
||||
`Store.set_extracted` — so only roles with no `extracted_at` are
|
||||
recomputed here, from their stored title and department (department
|
||||
lives in `raw`, not its own column: docs/decisions.md, "raw is
|
||||
transport, columns are storage").
|
||||
|
||||
A Python step because the classifier is Python, not SQL; `plugins` is a
|
||||
top-level package outside `app`'s own tree, so it is reached the same
|
||||
way `app/jobs/extract.py` reaches `FUNCTIONS`: inserted onto `sys.path`
|
||||
rather than imported as an installed dependency, since `plugins` is not
|
||||
in `pyproject.toml`'s packages."""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
if str(root) not in sys.path:
|
||||
sys.path.insert(0, str(root))
|
||||
from plugins.ats import classify
|
||||
|
||||
rows = conn.execute(
|
||||
"SELECT uid, title, raw, function FROM roles WHERE extracted_at IS NULL").fetchall()
|
||||
updates = []
|
||||
for uid, title, raw, old in rows:
|
||||
try:
|
||||
department = json.loads(raw or "{}").get("department") or ""
|
||||
except (TypeError, ValueError):
|
||||
department = ""
|
||||
new = classify(department, title)
|
||||
if new != old:
|
||||
updates.append((new, uid))
|
||||
if updates:
|
||||
conn.executemany("UPDATE roles SET function = ? WHERE uid = ?", updates)
|
||||
log.info("migration 9: recomputed function for %d of %d not-yet-extracted roles",
|
||||
len(updates), len(rows))
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
+35
-6
@@ -29,15 +29,16 @@ from app.storage import schema
|
||||
|
||||
#: Text columns a plugin may fill from `Item.raw` under the same name. An
|
||||
#: empty value never overwrites a filled column, so an upsert from a board that
|
||||
#: says less than `extract` found keeps what `extract` found.
|
||||
_RAW_TEXT = ("place", "country", "currency", "seniority", "function")
|
||||
#: says less than `extract` found keeps what `extract` found. `function` is
|
||||
#: not here: it is not a board field (see the note by `keep_filled` below).
|
||||
_RAW_TEXT = ("place", "country", "currency", "seniority")
|
||||
_RAW_INT = ("salary_min", "salary_max")
|
||||
|
||||
#: What `extract` may fill. Only an empty column is written: a board's own
|
||||
#: structured field wins over what a model read out of the prose, and the next
|
||||
#: upsert from that board would put it back anyway.
|
||||
_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "function",
|
||||
"permit")
|
||||
#: upsert from that board would put it back anyway. `function` is not here
|
||||
#: either, for the opposite reason — see `set_extracted`.
|
||||
_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "permit")
|
||||
|
||||
#: How many replies may leave a role unanswered before `extract` stops
|
||||
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
|
||||
@@ -262,6 +263,15 @@ class Store:
|
||||
from `item.raw` under their own names, with `company` falling back to
|
||||
the subscription's and `country` to what `place` names; an empty value
|
||||
never overwrites a filled column.
|
||||
|
||||
`function` is the one gate column no board actually sends: the `ats`
|
||||
plugin's own keyword guess rides in `raw` under that name because
|
||||
`raw` is the only channel a plugin has, but a guess is not first-party
|
||||
the way a board's own structured field is (docs/decisions.md, "a
|
||||
guess is not a board field"). So once a role has been through
|
||||
`extract` (`extracted_at` is set), this upsert leaves `function`
|
||||
alone for good, whatever the plugin's raw guess now says — see
|
||||
`set_extracted` for the other half of the rule.
|
||||
"""
|
||||
if not item.source:
|
||||
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
|
||||
@@ -293,6 +303,7 @@ class Store:
|
||||
row["country"] = row["country"] or country_of(place)
|
||||
row["remote"] = row["remote"] or remote_of(place)
|
||||
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
||||
row["function"] = str(raw.get("function") or "")
|
||||
|
||||
keep_filled = ("company", "company_key", "place_key", "remote", "description",
|
||||
"feed_text", "original_url", *_RAW_TEXT)
|
||||
@@ -301,6 +312,15 @@ class Store:
|
||||
for c in (*_RAW_INT, "subscription_id")]
|
||||
updates += [f"{c}=excluded.{c}"
|
||||
for c in ("title", "title_key", "published_at", "raw", "last_seen")]
|
||||
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
|
||||
# pull's classifier guess overwrite what extract decided, which is
|
||||
# the flip the "board wins" rule exists to prevent for an actual
|
||||
# board field — see the docstring above. Once extracted_at is set,
|
||||
# function is extract's to own; before that, it is the guess, same
|
||||
# as any other raw field.
|
||||
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
|
||||
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
|
||||
"ELSE roles.function END")
|
||||
updates.append("closed_at=NULL")
|
||||
columns = ", ".join(row)
|
||||
marks = ", ".join(f":{c}" for c in row)
|
||||
@@ -685,7 +705,13 @@ class Store:
|
||||
field is never replaced and nothing is ever blanked. `place` brings
|
||||
`place_key` with it. Returns how many columns were filled, or None
|
||||
for an unknown uid. Values are trusted to be in vocabulary; the
|
||||
job's `apply` is where a reply is checked."""
|
||||
job's `apply` is where a reply is checked.
|
||||
|
||||
`function` is the one exception to "only into empty columns": it is
|
||||
the `ats` classifier's keyword guess, not a board field, so a
|
||||
non-empty answer overwrites it outright rather than only filling an
|
||||
empty column — see `upsert_role`'s docstring for the other half of
|
||||
the rule that then keeps a later pull from putting the guess back."""
|
||||
with self.transaction() as conn:
|
||||
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
|
||||
if row is None:
|
||||
@@ -695,6 +721,9 @@ class Store:
|
||||
value = str(fields.get(name) or "")
|
||||
if value and not row[name]:
|
||||
sets[name] = value
|
||||
function = str(fields.get("function") or "")
|
||||
if function:
|
||||
sets["function"] = function
|
||||
for name in _RAW_INT:
|
||||
number = _int_or_none(fields.get(name))
|
||||
if number is not None and row[name] is None:
|
||||
|
||||
Reference in new issue
Block a user