Prep stage: host words stored raw, mapped centrally in app/prep after every pull
upsert_role stores the host's own words (roles.*_host), the place texts unresolved when they change, and the company key unresolved on insert, and marks the role prep_due. app.prep maps seniority, employment, remote and country through the host tables in app/prep/hosts, resolves places and company aliases, lists unknown words in `unmapped`, remaps every open role when the tables' fingerprint changes, and runs rebuild_dupes, which moves out of the pull's settle. Migration 22 adds the columns and the table and fills the host words from raw. Cascade: pull, prep, embed, score, enrich, prep, embed, score, extract, score, summarise. `app prep [unmapped] [--remap]`, /api/prep/unmapped and a fold on the Sources page. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a66aac95cf
commit
d04ceae7e0
31 files changed
+1468
-330
No files matched your search
+82
-2
@@ -18,7 +18,9 @@ import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
from app import places
|
||||
from app.models import SENIORITIES, company_fold, company_key, country_of, posted_at, posted_host
|
||||
from app.models import (
|
||||
SENIORITIES, company_fold, company_key, country_of, host_word, posted_at, posted_host,
|
||||
)
|
||||
from app.seniority import from_title
|
||||
from app.storage import gazetteer
|
||||
|
||||
@@ -107,8 +109,19 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
teaser TEXT NOT NULL DEFAULT '',
|
||||
seniority_model TEXT NOT NULL DEFAULT '',
|
||||
seniority_stated TEXT NOT NULL DEFAULT '',
|
||||
seniority_title TEXT NOT NULL DEFAULT ''
|
||||
seniority_title TEXT NOT NULL DEFAULT '',
|
||||
seniority_host TEXT NOT NULL DEFAULT '',
|
||||
employment_host TEXT NOT NULL DEFAULT '',
|
||||
remote_host TEXT NOT NULL DEFAULT '',
|
||||
country_host TEXT NOT NULL DEFAULT '',
|
||||
employment TEXT NOT NULL DEFAULT '',
|
||||
prep_due INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
-- The *_host columns hold the host's own words as a reader handed them on
|
||||
-- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps
|
||||
-- them into seniority_stated, employment, remote and country. prep_due is above 0
|
||||
-- on a role whose words, company, title or places changed since prep last
|
||||
-- mapped it (Store.upsert_role), which is what the next prep reads.
|
||||
-- roles.teaser is the short text a list carries, kept until the full text
|
||||
-- arrives and embedded while there is none; a later empty teaser does not
|
||||
-- wipe it. seniority_model, seniority_stated and seniority_title are the
|
||||
@@ -352,6 +365,19 @@ CREATE TABLE IF NOT EXISTS vectors (
|
||||
embedded_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- A host word no table of app/prep/hosts maps, per field (seniority,
|
||||
-- employment, remote, country) and host: how many open roles carry it and
|
||||
-- when prep first met it. Rewritten by every prep; a word added to a table
|
||||
-- leaves at the next one.
|
||||
CREATE TABLE IF NOT EXISTS unmapped (
|
||||
field TEXT NOT NULL,
|
||||
host TEXT NOT NULL,
|
||||
word TEXT NOT NULL,
|
||||
roles INTEGER NOT NULL DEFAULT 0,
|
||||
first_seen TEXT NOT NULL,
|
||||
PRIMARY KEY (field, host, word)
|
||||
);
|
||||
|
||||
-- The string map a desk plugin is handed as self.state. One namespace shared
|
||||
-- by every plugin, so a plugin prefixes its keys.
|
||||
CREATE TABLE IF NOT EXISTS state (
|
||||
@@ -389,6 +415,8 @@ CREATE INDEX IF NOT EXISTS idx_roles_dup_of
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
|
||||
-- The roles the next prep maps. Partial: after a prep there are none.
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0;
|
||||
"""
|
||||
|
||||
#: The table whose absence marks a file that has never been a roledesk
|
||||
@@ -567,6 +595,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
|
||||
(21, "companies.all_roles: a company tag that reads its boards with the agents' wide "
|
||||
"reach, given to every favourite and taking over a board's {\"reach\": \"all\"}",
|
||||
lambda conn: _add_all_roles(conn)),
|
||||
# A Python step: the host words are read out of raw's JSON.
|
||||
(22, "roles.*_host, roles.employment, roles.prep_due and unmapped: the host's own "
|
||||
"words are stored and app.prep maps them", lambda conn: _add_host_words(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -833,6 +864,55 @@ def _add_all_roles(conn: sqlite3.Connection) -> None:
|
||||
"to tag; it reads the agents' reach now", target, key)
|
||||
|
||||
|
||||
_V22_COLUMNS = ("seniority_host", "employment_host", "remote_host", "country_host", "employment")
|
||||
|
||||
_V22 = """
|
||||
CREATE TABLE IF NOT EXISTS unmapped (
|
||||
field TEXT NOT NULL,
|
||||
host TEXT NOT NULL,
|
||||
word TEXT NOT NULL,
|
||||
roles INTEGER NOT NULL DEFAULT 0,
|
||||
first_seen TEXT NOT NULL,
|
||||
PRIMARY KEY (field, host, word)
|
||||
);
|
||||
DELETE FROM state WHERE key = 'prep:tables';
|
||||
"""
|
||||
|
||||
|
||||
def _add_host_words(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 22's step. The columns and `unmapped`, and the host words
|
||||
of every role filled from its `raw` where it has them. Before this,
|
||||
readers mapped `seniority` and `employment` in their own tables, so what
|
||||
`raw` holds there is mostly a value of the vocabulary already, which
|
||||
`app.prep` takes as it is. `remote` and `country` are the host's as
|
||||
given. The `prep:tables` fingerprint goes, so the first prep maps every
|
||||
open role from these words. Runs twice without harm."""
|
||||
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
||||
for column in _V22_COLUMNS:
|
||||
if column not in columns:
|
||||
conn.execute(f"ALTER TABLE roles ADD COLUMN {column} TEXT NOT NULL DEFAULT ''")
|
||||
if "prep_due" not in columns:
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN prep_due INTEGER NOT NULL DEFAULT 0")
|
||||
for statement in _statements(_V22):
|
||||
conn.execute(statement)
|
||||
updates = []
|
||||
for uid, raw in conn.execute(
|
||||
"SELECT uid, raw FROM roles WHERE json_extract(raw, '$.seniority') IS NOT NULL "
|
||||
"OR json_extract(raw, '$.employment') IS NOT NULL "
|
||||
"OR json_extract(raw, '$.remote') IS NOT NULL "
|
||||
"OR json_extract(raw, '$.country') IS NOT NULL").fetchall():
|
||||
try:
|
||||
given = json.loads(raw or "{}")
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(given, dict):
|
||||
updates.append((*(host_word(given.get(k)) for k in (
|
||||
"seniority", "employment", "remote", "country")), uid))
|
||||
conn.executemany("UPDATE roles SET seniority_host = ?, employment_host = ?, "
|
||||
"remote_host = ?, country_host = ? WHERE uid = ?", updates)
|
||||
log.info("migration 22: host words from raw for %d roles", len(updates))
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
Reference in new issue
Block a user