Merge branch 'worktree-agent-afad54f12aef2cccf'
# Conflicts: # app/storage/schema.py # app/storage/store.py # tests/test_storage.py
This commit is contained in:
commit
706b127bc0
10 files changed
+459
-37
No files matched your search
@@ -377,12 +377,58 @@ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT '';
|
||||
"""),
|
||||
(10, "roles.posted_at: recency and the newest feed count from when a role "
|
||||
"was posted, not just when the app found it", _migrate_posted_at),
|
||||
# A Python step, not SQL: classify() lives in plugins/ats.py, and running
|
||||
# its current rules is the whole point (see _recompute_function below).
|
||||
(11, "roles.function recomputed for every not-yet-extracted role, word-boundary "
|
||||
"classifier fix", lambda conn: _recompute_function(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
LATEST = MIGRATIONS[-1][0]
|
||||
|
||||
|
||||
def _recompute_function(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 11's step. `roles.function` used to be filled by a keyword
|
||||
classifier that matched substrings (" it" inside "Italy"), never
|
||||
returned '' (only "other", outside the declared vocabulary) and checked
|
||||
department before title; `plugins.ats.classify` no longer does any of
|
||||
that (see its own docstring). A role `extract` has already read keeps
|
||||
whatever it decided — `function` is extract's from then on, see
|
||||
`Store.set_extracted` — so only roles with no `extracted_at` are
|
||||
recomputed here, from their stored title and department (department
|
||||
lives in `raw`, not its own column: docs/decisions.md, "raw is
|
||||
transport, columns are storage").
|
||||
|
||||
A Python step because the classifier is Python, not SQL; `plugins` is a
|
||||
top-level package outside `app`'s own tree, so it is reached the same
|
||||
way `app/jobs/extract.py` reaches `FUNCTIONS`: inserted onto `sys.path`
|
||||
rather than imported as an installed dependency, since `plugins` is not
|
||||
in `pyproject.toml`'s packages."""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
if str(root) not in sys.path:
|
||||
sys.path.insert(0, str(root))
|
||||
from plugins.ats import classify
|
||||
|
||||
rows = conn.execute(
|
||||
"SELECT uid, title, raw, function FROM roles WHERE extracted_at IS NULL").fetchall()
|
||||
updates = []
|
||||
for uid, title, raw, old in rows:
|
||||
try:
|
||||
department = json.loads(raw or "{}").get("department") or ""
|
||||
except (TypeError, ValueError):
|
||||
department = ""
|
||||
new = classify(department, title)
|
||||
if new != old:
|
||||
updates.append((new, uid))
|
||||
if updates:
|
||||
conn.executemany("UPDATE roles SET function = ? WHERE uid = ?", updates)
|
||||
log.info("migration 11: recomputed function for %d of %d not-yet-extracted roles",
|
||||
len(updates), len(rows))
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
+35
-6
@@ -29,15 +29,16 @@ from app.storage import schema
|
||||
|
||||
#: Text columns a plugin may fill from `Item.raw` under the same name. An
|
||||
#: empty value never overwrites a filled column, so an upsert from a board that
|
||||
#: says less than `extract` found keeps what `extract` found.
|
||||
_RAW_TEXT = ("place", "country", "currency", "seniority", "function")
|
||||
#: says less than `extract` found keeps what `extract` found. `function` is
|
||||
#: not here: it is not a board field (see the note by `keep_filled` below).
|
||||
_RAW_TEXT = ("place", "country", "currency", "seniority")
|
||||
_RAW_INT = ("salary_min", "salary_max")
|
||||
|
||||
#: What `extract` may fill. Only an empty column is written: a board's own
|
||||
#: structured field wins over what a model read out of the prose, and the next
|
||||
#: upsert from that board would put it back anyway.
|
||||
_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "function",
|
||||
"permit")
|
||||
#: upsert from that board would put it back anyway. `function` is not here
|
||||
#: either, for the opposite reason — see `set_extracted`.
|
||||
_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "permit")
|
||||
|
||||
#: How many replies may leave a role unanswered before `extract` stops
|
||||
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
|
||||
@@ -269,6 +270,15 @@ class Store:
|
||||
keeps the earlier of that and what is already stored, so an edit on
|
||||
the board — which can only ever move `published_at` later, never
|
||||
earlier — cannot move `posted_at` later either.
|
||||
|
||||
`function` is the one gate column no board actually sends: the `ats`
|
||||
plugin's own keyword guess rides in `raw` under that name because
|
||||
`raw` is the only channel a plugin has, but a guess is not first-party
|
||||
the way a board's own structured field is (docs/decisions.md, "a
|
||||
guess is not a board field"). So once a role has been through
|
||||
`extract` (`extracted_at` is set), this upsert leaves `function`
|
||||
alone for good, whatever the plugin's raw guess now says — see
|
||||
`set_extracted` for the other half of the rule.
|
||||
"""
|
||||
if not item.source:
|
||||
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
|
||||
@@ -302,6 +312,7 @@ class Store:
|
||||
row["country"] = row["country"] or country_of(place)
|
||||
row["remote"] = row["remote"] or remote_of(place)
|
||||
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
||||
row["function"] = str(raw.get("function") or "")
|
||||
|
||||
keep_filled = ("company", "company_key", "place_key", "remote", "description",
|
||||
"feed_text", "original_url", *_RAW_TEXT)
|
||||
@@ -311,6 +322,15 @@ class Store:
|
||||
updates += [f"{c}=excluded.{c}"
|
||||
for c in ("title", "title_key", "published_at", "raw", "last_seen")]
|
||||
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
|
||||
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
|
||||
# pull's classifier guess overwrite what extract decided, which is
|
||||
# the flip the "board wins" rule exists to prevent for an actual
|
||||
# board field — see the docstring above. Once extracted_at is set,
|
||||
# function is extract's to own; before that, it is the guess, same
|
||||
# as any other raw field.
|
||||
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
|
||||
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
|
||||
"ELSE roles.function END")
|
||||
updates.append("closed_at=NULL")
|
||||
columns = ", ".join(row)
|
||||
marks = ", ".join(f":{c}" for c in row)
|
||||
@@ -705,7 +725,13 @@ class Store:
|
||||
field is never replaced and nothing is ever blanked. `place` brings
|
||||
`place_key` with it. Returns how many columns were filled, or None
|
||||
for an unknown uid. Values are trusted to be in vocabulary; the
|
||||
job's `apply` is where a reply is checked."""
|
||||
job's `apply` is where a reply is checked.
|
||||
|
||||
`function` is the one exception to "only into empty columns": it is
|
||||
the `ats` classifier's keyword guess, not a board field, so a
|
||||
non-empty answer overwrites it outright rather than only filling an
|
||||
empty column — see `upsert_role`'s docstring for the other half of
|
||||
the rule that then keeps a later pull from putting the guess back."""
|
||||
with self.transaction() as conn:
|
||||
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
|
||||
if row is None:
|
||||
@@ -715,6 +741,9 @@ class Store:
|
||||
value = str(fields.get(name) or "")
|
||||
if value and not row[name]:
|
||||
sets[name] = value
|
||||
function = str(fields.get("function") or "")
|
||||
if function:
|
||||
sets["function"] = function
|
||||
for name in _RAW_INT:
|
||||
number = _int_or_none(fields.get(name))
|
||||
if number is not None and row[name] is None:
|
||||
|
||||
Reference in new issue
Block a user