The CV is cut into chunks inside e5's window, embedded like a prompt and cached per agent under a hash of the model, chunking and text. A role's cv_fit (scores.cv_fit, migration 31) is its best chunk cosine, scaled with its own calibration band: fit's clipped 88 of agent 2's ungated roles to 1. It is no term of the Agent match score. judge_cv_top (default 50) sits beside judge_top and judge_min on the agents page and card. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1308 lines
64 KiB
Python
1308 lines
64 KiB
Python
"""The tables, and how a database file gets from an old shape to this one.
|
|
|
|
`SCHEMA` is the current shape. A new file is created from it and stamped with
|
|
`LATEST`, so it never runs a migration. `MIGRATIONS` is how an existing file
|
|
catches up: ordered, numbered, applied once each, tracked in SQLite's own
|
|
`PRAGMA user_version`.
|
|
|
|
Adding a column means editing both: the DDL in `SCHEMA` for new files, and an
|
|
appended migration for old ones. Never renumber or edit a migration that has
|
|
shipped.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import sqlite3
|
|
from collections.abc import Callable
|
|
|
|
from app import places
|
|
from app.models import (
|
|
SENIORITIES, company_fold, company_key, country_of, host_word, iso_day, posted_at,
|
|
posted_host,
|
|
)
|
|
from app.seniority import from_title
|
|
from app.storage import gazetteer
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
SCHEMA = """
|
|
-- One posting on one board. `uid` is sha1(source, source_id), the identity desk
|
|
-- gives every Item. The gate fields are real columns, unpacked from the item's
|
|
-- raw on upsert or filled later by extract: raw is transport, the columns are
|
|
-- what gets queried.
|
|
CREATE TABLE IF NOT EXISTS roles (
|
|
uid TEXT PRIMARY KEY,
|
|
source TEXT NOT NULL,
|
|
source_id TEXT NOT NULL,
|
|
subscription_id INTEGER REFERENCES subscriptions(id) ON DELETE SET NULL,
|
|
-- The cluster key is (company_key, title_key, place_key). No foreign key
|
|
-- to companies: a role may arrive before anyone has made a company row.
|
|
company_key TEXT NOT NULL DEFAULT '',
|
|
company TEXT NOT NULL DEFAULT '',
|
|
title TEXT NOT NULL DEFAULT '',
|
|
title_key TEXT NOT NULL DEFAULT '',
|
|
-- Geography. place and not location, which is where an Item's original
|
|
-- lives. remote and country are separate so a gate on remote can also
|
|
-- require the country: "UK | Remote" is not remote from Berlin.
|
|
place TEXT NOT NULL DEFAULT '',
|
|
place_key TEXT NOT NULL DEFAULT '',
|
|
-- places.fold(place): what a place: token the gazetteer does not know is
|
|
-- matched against as whole words, through place_match (app/places.py).
|
|
-- The filter and the gates otherwise read role_places.
|
|
place_fold TEXT NOT NULL DEFAULT '',
|
|
country TEXT NOT NULL DEFAULT '',
|
|
remote TEXT NOT NULL DEFAULT ''
|
|
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
|
|
salary_min INTEGER,
|
|
salary_max INTEGER,
|
|
currency TEXT NOT NULL DEFAULT '',
|
|
-- The level shown and filtered on: the first filled of seniority_model,
|
|
-- seniority_stated and seniority_title (the last columns), written with them.
|
|
seniority TEXT NOT NULL DEFAULT '',
|
|
function TEXT NOT NULL DEFAULT '',
|
|
-- Whether the posting says it sponsors a work permit, and the languages
|
|
-- (ISO 639-1, a JSON array) it says are required. Only extract fills these.
|
|
permit TEXT NOT NULL DEFAULT ''
|
|
CHECK (permit IN ('', 'sponsors', 'no_sponsorship')),
|
|
languages TEXT NOT NULL DEFAULT '[]',
|
|
description TEXT NOT NULL DEFAULT '', -- the posting's body text
|
|
feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over
|
|
original_url TEXT NOT NULL DEFAULT '', -- the posting page
|
|
-- Where a search host's own page leads: Jobindex's "Se jobbet" link, found
|
|
-- when the posting text is fetched (app.jobs.posting). Never written by an upsert.
|
|
posting_url TEXT NOT NULL DEFAULT '',
|
|
published_at TEXT NOT NULL, -- what the source claims, UTC
|
|
-- What storing postings buys. first_seen is when the app itself first
|
|
-- stored the role; posted_at (app.models.posted_at) is the earlier of
|
|
-- first_seen and published_at, trusted only for a host known to report a
|
|
-- creation date rather than one that moves on an edit
|
|
-- (app.models.trusted_posted) — recency and the feed's newest order
|
|
-- read posted_at, first_seen stays "when the app found it". closed_at is
|
|
-- set when a board fetched in full no longer lists the role, and cleared
|
|
-- if it returns.
|
|
posted_at TEXT NOT NULL DEFAULT '',
|
|
first_seen TEXT NOT NULL,
|
|
last_seen TEXT NOT NULL,
|
|
closed_at TEXT,
|
|
raw TEXT NOT NULL DEFAULT '{}',
|
|
extracted_at TEXT, -- when extract last filled columns
|
|
-- Replies from extract that left this role unanswered. At GIVE_UP_AFTER it
|
|
-- is no longer offered. A column, not raw, because every upsert rewrites raw.
|
|
extract_tries INTEGER NOT NULL DEFAULT 0,
|
|
-- NULL unread, else when the feed first opened it or judged it: a vote
|
|
-- stamps this too, see Store.set_vote, because voting is reading.
|
|
read_at TEXT,
|
|
-- About 60 words from summarise, and when it wrote them.
|
|
summary TEXT NOT NULL DEFAULT '',
|
|
summarised_at TEXT,
|
|
-- The posting's own page as plain text, fetched on demand for a role
|
|
-- whose description is only a teaser.
|
|
article_text TEXT NOT NULL DEFAULT '',
|
|
article_at TEXT,
|
|
-- The uid of the kept role when this open role is a copy of one posting
|
|
-- (app/dupes.py), NULL on the kept role, on a role without copies and on
|
|
-- every closed role. Written only by Store.rebuild_dupes, at the end of
|
|
-- every pull; the feed, the counts and the model stages skip a copy.
|
|
-- Not the last column: SQLite cannot drop a last column that a comment
|
|
-- precedes, which the migration tests do.
|
|
dup_of TEXT,
|
|
created_at TEXT NOT NULL,
|
|
salary_text TEXT NOT NULL DEFAULT '',
|
|
teaser TEXT NOT NULL DEFAULT '',
|
|
seniority_model TEXT NOT NULL DEFAULT '',
|
|
seniority_stated TEXT NOT NULL DEFAULT '',
|
|
seniority_title TEXT NOT NULL DEFAULT '',
|
|
seniority_host TEXT NOT NULL DEFAULT '',
|
|
employment_host TEXT NOT NULL DEFAULT '',
|
|
remote_host TEXT NOT NULL DEFAULT '',
|
|
country_host TEXT NOT NULL DEFAULT '',
|
|
employment TEXT NOT NULL DEFAULT '',
|
|
prep_due INTEGER NOT NULL DEFAULT 0,
|
|
remote_model TEXT NOT NULL DEFAULT '',
|
|
remote_where_model TEXT,
|
|
languages_required TEXT,
|
|
deadline_host TEXT NOT NULL DEFAULT '',
|
|
deadline_model TEXT NOT NULL DEFAULT '',
|
|
deadline TEXT
|
|
);
|
|
-- The *_host columns hold the host's own words as a reader handed them on
|
|
-- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps
|
|
-- them into seniority_stated, employment, remote and country. prep_due is above 0
|
|
-- on a role whose words, company, title or places changed since prep last
|
|
-- mapped it (Store.upsert_role), which is what the next prep reads.
|
|
-- roles.teaser is the short text a list carries, kept until the full text
|
|
-- arrives and embedded while there is none; a later empty teaser does not
|
|
-- wipe it. seniority_model, seniority_stated and seniority_title are the
|
|
-- three levels behind roles.seniority: the model's (extract), the host's own
|
|
-- word mapped to app.seniority's vocabulary, and the title's. remote_model is
|
|
-- extract's remote, which wins over the host's word and the place line, and
|
|
-- remote_where_model the names it read for where a remote hire may live (a
|
|
-- JSON list, NULL never asked, [] not stated). languages_required is every
|
|
-- language extract read in the posting, a JSON list of {language, level,
|
|
-- kind}, NULL never asked; languages keeps the required codes for the
|
|
-- languages gate. deadline_host is the host's own last day to apply, as a
|
|
-- pull hands it on, deadline_model the one extract read, both YYYY-MM-DD or
|
|
-- '', and deadline the first of them, NULL for none, written by prep. No
|
|
-- comment sits among the last columns: SQLite cannot drop a column a comment
|
|
-- precedes, which the migration tests do.
|
|
|
|
-- The gazetteer: a place is a continent, a country (key the ISO code), a
|
|
-- region or a city, in the place parent_id names. key is unique per kind:
|
|
-- a region's key (app.places.REGIONS), a town's group. Seeded from the
|
|
-- dictionaries in app/models.py and app/places.py (app.places.seed).
|
|
CREATE TABLE IF NOT EXISTS places (
|
|
id INTEGER PRIMARY KEY,
|
|
kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')),
|
|
key TEXT NOT NULL,
|
|
name TEXT NOT NULL,
|
|
parent_id INTEGER REFERENCES places(id),
|
|
UNIQUE (kind, key)
|
|
);
|
|
|
|
-- Every folded spelling of a place (app.places.fold): kobenhavn, kopenhamn
|
|
-- and copenhagen are Copenhagen. One fold may name several places
|
|
-- (Luxembourg the town and the country).
|
|
CREATE TABLE IF NOT EXISTS place_aliases (
|
|
fold TEXT NOT NULL,
|
|
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
|
UNIQUE (fold, place_id)
|
|
);
|
|
|
|
-- What a place holds beyond its children: Øresund holds the Capital Region
|
|
-- of Denmark and Skåne, which lie in two countries.
|
|
CREATE TABLE IF NOT EXISTS place_includes (
|
|
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
|
part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
|
PRIMARY KEY (place_id, part_id)
|
|
);
|
|
|
|
-- One row per place of a role (app/storage/gazetteer.py): the host's text in
|
|
-- the host's order (position), its fold, and the place it resolves to, NULL
|
|
-- while the gazetteer does not know it. A text naming several places is a
|
|
-- row for each. url is the link of a copy folded into this role, '' for
|
|
-- the role's own places.
|
|
CREATE TABLE IF NOT EXISTS role_places (
|
|
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
|
position INTEGER NOT NULL,
|
|
text TEXT NOT NULL,
|
|
fold TEXT NOT NULL,
|
|
place_id INTEGER REFERENCES places(id),
|
|
url TEXT NOT NULL DEFAULT ''
|
|
);
|
|
|
|
-- A remote role's area, the places a remote hire may live in (app/prep/area.py),
|
|
-- in the order found. Anywhere (places.ANYWHERE) is one of them. A role with
|
|
-- no row is not remote, or remote with its area not stated.
|
|
CREATE TABLE IF NOT EXISTS role_remote (
|
|
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
|
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
|
PRIMARY KEY (uid, place_id)
|
|
);
|
|
|
|
-- The open vocabulary summarise reads off a posting: the role family, the
|
|
-- technologies and skills it names, and the domain. What the feed's filter
|
|
-- row is built from; see Store.set_tags and Store.tag_counts.
|
|
CREATE TABLE IF NOT EXISTS role_tags (
|
|
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
|
tag TEXT NOT NULL,
|
|
PRIMARY KEY (uid, tag)
|
|
);
|
|
|
|
-- A company as a target, whether or not any board of it is read. key is
|
|
-- models.company_key(name), the slug roles.company_key holds.
|
|
-- ats is the host the careers site runs on, readable by the ats plugin or
|
|
-- not (workday, successfactors, ...): what discovery found, '' unknown. A
|
|
-- company with an ats and no board subscription is one the app cannot read
|
|
-- yet. country is an ISO code and city the head office's. ats_slug is the
|
|
-- name that host knows the company by (the acme of acme.teamtailor.com),
|
|
-- '' where its URL carries none, kept for the parser written later.
|
|
-- watch is the Watchlist's rating, global and not an agent's: NULL unrated, 1
|
|
-- worth checking by hand, 0 not; careers_opened_at when its careers page was
|
|
-- last opened from there, which is also when a person last looked at it.
|
|
-- hand_reason is why a person reads the careers page (models.HAND_REASONS),
|
|
-- '' when nobody has to. The two fetch overrides, global since fetching is
|
|
-- shared (app/jobs/reach.py): all_countries 1 reads every big board of the
|
|
-- company with the agents' wide reach, Europe today, rather than their
|
|
-- places; all_roles 1 reads them with no role-type narrowing, no fetch terms
|
|
-- today. Until migration 24, all_roles meant what all_countries means now.
|
|
CREATE TABLE IF NOT EXISTS companies (
|
|
key TEXT PRIMARY KEY,
|
|
name TEXT NOT NULL,
|
|
domain TEXT NOT NULL DEFAULT '',
|
|
status TEXT NOT NULL DEFAULT 'researching',
|
|
careers_url TEXT NOT NULL DEFAULT '',
|
|
notes TEXT NOT NULL DEFAULT '',
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
ats TEXT NOT NULL DEFAULT '',
|
|
country TEXT NOT NULL DEFAULT '',
|
|
city TEXT NOT NULL DEFAULT '',
|
|
ats_slug TEXT NOT NULL DEFAULT '',
|
|
watch INTEGER CHECK (watch IN (0, 1)),
|
|
careers_opened_at TEXT NOT NULL DEFAULT '',
|
|
hand_reason TEXT NOT NULL DEFAULT '',
|
|
sector TEXT NOT NULL DEFAULT '',
|
|
stage TEXT NOT NULL DEFAULT '',
|
|
size_band TEXT NOT NULL DEFAULT '',
|
|
founded INTEGER,
|
|
last_funding INTEGER,
|
|
last_funding_currency TEXT NOT NULL DEFAULT '',
|
|
facts_source TEXT NOT NULL DEFAULT '',
|
|
facts_at TEXT NOT NULL DEFAULT '',
|
|
all_roles INTEGER NOT NULL DEFAULT 0,
|
|
all_countries INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
|
|
-- Every folded spelling of a company (models.company_fold): orsted is Ørsted
|
|
-- whichever way a board writes it, and a person can add another name
|
|
-- ("Dong Energy"). One fold may name several companies until a person merges
|
|
-- them (Store.merge_companies). The enrichment columns above are empty until
|
|
-- a reader states them; facts_source and facts_at say where and when.
|
|
CREATE TABLE IF NOT EXISTS company_aliases (
|
|
fold TEXT NOT NULL,
|
|
company_key TEXT NOT NULL,
|
|
UNIQUE (fold, company_key)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_company_aliases_key ON company_aliases(company_key);
|
|
|
|
-- The targets desk's runner hands to plugins. newsdesk's shape, with its two
|
|
-- labels named company and board from the first migration.
|
|
CREATE TABLE IF NOT EXISTS subscriptions (
|
|
id INTEGER PRIMARY KEY,
|
|
source TEXT NOT NULL, -- plugin name
|
|
target TEXT NOT NULL, -- e.g. greenhouse:acme
|
|
label TEXT NOT NULL, -- what Item.subscription names
|
|
company TEXT NOT NULL DEFAULT '',
|
|
board TEXT NOT NULL DEFAULT '',
|
|
params TEXT NOT NULL DEFAULT '{}', -- what a query target asks with
|
|
subscribed INTEGER NOT NULL DEFAULT 1,
|
|
fetch INTEGER NOT NULL DEFAULT 1,
|
|
show INTEGER NOT NULL DEFAULT 1,
|
|
added_at TEXT NOT NULL,
|
|
target_id TEXT NOT NULL DEFAULT '', -- the plugin's durable name for it
|
|
UNIQUE (source, target),
|
|
UNIQUE (source, label)
|
|
);
|
|
|
|
-- The tracker, in jobhunt-hq's shape (MIT). role_uid is NULL for a role that
|
|
-- was never posted, so an unsolicited approach is tracked like any other.
|
|
CREATE TABLE IF NOT EXISTS applications (
|
|
id TEXT PRIMARY KEY,
|
|
role_uid TEXT REFERENCES roles(uid) ON DELETE SET NULL,
|
|
company_key TEXT NOT NULL DEFAULT '',
|
|
role_title TEXT NOT NULL DEFAULT '',
|
|
url TEXT NOT NULL DEFAULT '',
|
|
status TEXT NOT NULL DEFAULT 'saved'
|
|
CHECK (status IN ('saved', 'applied', 'screen', 'technical',
|
|
'onsite', 'offer', 'rejected', 'withdrawn')),
|
|
source TEXT NOT NULL DEFAULT '',
|
|
applied_at TEXT,
|
|
next_action TEXT NOT NULL DEFAULT '',
|
|
next_date TEXT,
|
|
contact TEXT NOT NULL DEFAULT '',
|
|
notes TEXT NOT NULL DEFAULT '',
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
|
|
-- One row per status an application entered, written by the store and never
|
|
-- updated: what days-to-first-screen and conversion by source are read from.
|
|
CREATE TABLE IF NOT EXISTS application_events (
|
|
id INTEGER PRIMARY KEY,
|
|
application_id TEXT NOT NULL REFERENCES applications(id) ON DELETE CASCADE,
|
|
status TEXT NOT NULL,
|
|
at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TRIGGER IF NOT EXISTS application_events_append_only
|
|
BEFORE UPDATE ON application_events
|
|
BEGIN
|
|
SELECT RAISE(ABORT, 'application_events is append-only');
|
|
END;
|
|
|
|
-- A search agent: a named saved search with its own prompts, gates and
|
|
-- weights. gates is a list of {field, values, mode: require|prefer, weight}.
|
|
-- limits is how deep into this agent's ranking extract and summarise read,
|
|
-- as JSON: a key it lacks means the `Limits` default. fetch_terms is a JSON
|
|
-- list of words that narrow what big boards send (app/jobs/reach.py); empty
|
|
-- means no narrowing. cv_text, cv_languages (a JSON list of {language,
|
|
-- level}), like_text (what they like: the work, the workplace, situations),
|
|
-- avoid_text (what they do not like) and must_text (must-haves and
|
|
-- deal-breakers, set through the API only) are what judge reads a role
|
|
-- against; an agent with no CV is not judged.
|
|
CREATE TABLE IF NOT EXISTS agents (
|
|
id INTEGER PRIMARY KEY,
|
|
name TEXT NOT NULL UNIQUE,
|
|
gates TEXT NOT NULL DEFAULT '[]',
|
|
weights TEXT NOT NULL DEFAULT '{}',
|
|
enabled INTEGER NOT NULL DEFAULT 1,
|
|
created_at TEXT NOT NULL,
|
|
limits TEXT NOT NULL DEFAULT '{}',
|
|
fetch_terms TEXT NOT NULL DEFAULT '[]',
|
|
cv_text TEXT NOT NULL DEFAULT '',
|
|
cv_languages TEXT NOT NULL DEFAULT '[]',
|
|
avoid_text TEXT NOT NULL DEFAULT '',
|
|
must_text TEXT NOT NULL DEFAULT '',
|
|
like_text TEXT NOT NULL DEFAULT ''
|
|
);
|
|
|
|
-- What an agent looks for, as several named texts in order; a role's fit is
|
|
-- its best prompt's. vec caches the text's vector and vec_key is sha1 of the
|
|
-- embedding model and the text, so editing one prompt re-embeds that one.
|
|
CREATE TABLE IF NOT EXISTS agent_prompts (
|
|
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
|
position INTEGER NOT NULL,
|
|
name TEXT NOT NULL,
|
|
text TEXT NOT NULL DEFAULT '',
|
|
vec_key TEXT NOT NULL DEFAULT '',
|
|
vec BLOB,
|
|
PRIMARY KEY (agent_id, position),
|
|
UNIQUE (agent_id, name COLLATE NOCASE)
|
|
);
|
|
|
|
-- One agent's stance on a company: -1 hidden, 0 neutral, 1 liked, 2
|
|
-- favourite. No row is 0, so the table holds only what somebody set. Per
|
|
-- agent, because a company that is noise for one search is the target of
|
|
-- another.
|
|
CREATE TABLE IF NOT EXISTS agent_companies (
|
|
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
|
company_key TEXT NOT NULL,
|
|
affinity INTEGER NOT NULL CHECK (affinity BETWEEN -1 AND 2),
|
|
updated_at TEXT NOT NULL,
|
|
PRIMARY KEY (agent_id, company_key)
|
|
);
|
|
|
|
-- One agent's judgement of a role: -1 pass, 0 neutral, 1 like. No row is
|
|
-- unjudged. A vote moves the rank at read time, never hides a role, and its
|
|
-- vectors shape the agent's taste direction in `score`.
|
|
CREATE TABLE IF NOT EXISTS role_votes (
|
|
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
|
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
|
vote INTEGER NOT NULL CHECK (vote IN (-1, 0, 1)),
|
|
voted_at TEXT NOT NULL,
|
|
PRIMARY KEY (agent_id, uid)
|
|
);
|
|
|
|
-- A role is scored per agent. A gated role keeps its row, with the reason.
|
|
-- prompt names the agent's prompt that gave the fit, '' when fit did not count.
|
|
-- cv_fit is how close the role is to the agent's CV, 0..1, NULL without a CV
|
|
-- or a vector and for a gated role; it is no term of score.
|
|
CREATE TABLE IF NOT EXISTS scores (
|
|
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
|
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
|
score REAL NOT NULL DEFAULT 0,
|
|
gated_reason TEXT NOT NULL DEFAULT '',
|
|
scored_at TEXT NOT NULL,
|
|
prompt TEXT NOT NULL DEFAULT '',
|
|
cv_fit REAL,
|
|
PRIMARY KEY (uid, agent_id)
|
|
);
|
|
|
|
-- The vectors of an agent's CV, one per chunk in order, for scores.cv_fit.
|
|
-- vec_key is sha1 of the embedding model, the chunking and the CV text, the
|
|
-- same on every row of an agent, so an unchanged CV is not embedded again.
|
|
CREATE TABLE IF NOT EXISTS agent_cv_vectors (
|
|
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
|
position INTEGER NOT NULL,
|
|
vec_key TEXT NOT NULL,
|
|
vec BLOB NOT NULL,
|
|
PRIMARY KEY (agent_id, position)
|
|
);
|
|
|
|
-- One agent's judge reading of a role against its CV. Its own table, since
|
|
-- replace_scores rewrites scores every run and a judgement outlives that
|
|
-- and the role closing. skills, experience and career are 0 to 100,
|
|
-- cv_score their weighted total, strengths and gaps JSON lists. input_key
|
|
-- is a hash of the CV, prompts, avoid text and the role's text it read: a
|
|
-- different one is read again. musts is a JSON list of {id, text, verdict},
|
|
-- the model's verdict on each of the agent's must-haves (one per line of its
|
|
-- must text), broken how many are "broken": career counts as at most 39 then
|
|
-- (app.models.BROKEN_CAP), applied when the role is served. tries counts
|
|
-- replies that left the role without one since; at 3 it is given up until the agent is saved again. A
|
|
-- row with judged_at NULL holds only tries.
|
|
CREATE TABLE IF NOT EXISTS role_judgements (
|
|
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
|
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
|
skills INTEGER,
|
|
experience INTEGER,
|
|
career INTEGER,
|
|
cv_score REAL,
|
|
strengths TEXT NOT NULL DEFAULT '[]',
|
|
gaps TEXT NOT NULL DEFAULT '[]',
|
|
musts TEXT NOT NULL DEFAULT '[]',
|
|
broken INTEGER NOT NULL DEFAULT 0,
|
|
input_key TEXT NOT NULL DEFAULT '',
|
|
tries INTEGER NOT NULL DEFAULT 0,
|
|
judged_at TEXT,
|
|
PRIMARY KEY (uid, agent_id)
|
|
);
|
|
|
|
-- One vector per role, little-endian float32. Only comparable to another from
|
|
-- the same model, so every read is scoped by it.
|
|
CREATE TABLE IF NOT EXISTS vectors (
|
|
uid TEXT PRIMARY KEY REFERENCES roles(uid) ON DELETE CASCADE,
|
|
model TEXT NOT NULL,
|
|
dim INTEGER NOT NULL,
|
|
vec BLOB NOT NULL,
|
|
embedded_at TEXT NOT NULL
|
|
);
|
|
|
|
-- A host word no table of app/prep/hosts maps, per field (seniority,
|
|
-- employment, remote, country) and host: how many open roles carry it and
|
|
-- when prep first met it. Rewritten by every prep; a word added to a table
|
|
-- leaves at the next one.
|
|
CREATE TABLE IF NOT EXISTS unmapped (
|
|
field TEXT NOT NULL,
|
|
host TEXT NOT NULL,
|
|
word TEXT NOT NULL,
|
|
roles INTEGER NOT NULL DEFAULT 0,
|
|
first_seen TEXT NOT NULL,
|
|
PRIMARY KEY (field, host, word)
|
|
);
|
|
|
|
-- The string map a desk plugin is handed as self.state. One namespace shared
|
|
-- by every plugin, so a plugin prefixes its keys.
|
|
CREATE TABLE IF NOT EXISTS state (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT NOT NULL
|
|
);
|
|
"""
|
|
|
|
#: Applied after `SCHEMA` or the migrations, every open. An index may name a
|
|
#: column only a migration adds, which is why these are kept apart.
|
|
INDEXES = """
|
|
CREATE INDEX IF NOT EXISTS idx_roles_company ON roles(company_key);
|
|
CREATE INDEX IF NOT EXISTS idx_roles_place ON roles(place_key);
|
|
CREATE INDEX IF NOT EXISTS idx_roles_first_seen ON roles(first_seen DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_roles_closed ON roles(closed_at);
|
|
-- Serves the newest feed (open_only, ordered by posted_at DESC), which is
|
|
-- most of what `Store.roles` runs: partial, since only an open role's
|
|
-- position in that order is ever asked for.
|
|
CREATE INDEX IF NOT EXISTS idx_roles_posted
|
|
ON roles(posted_at DESC, uid) WHERE closed_at IS NULL;
|
|
CREATE INDEX IF NOT EXISTS idx_roles_cluster ON roles(company_key, title_key, place_key);
|
|
CREATE INDEX IF NOT EXISTS idx_roles_board ON roles(source, subscription_id);
|
|
CREATE INDEX IF NOT EXISTS idx_application_events
|
|
ON application_events(application_id, at);
|
|
CREATE INDEX IF NOT EXISTS idx_scores_agent ON scores(agent_id, score DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_role_votes_agent ON role_votes(agent_id, vote);
|
|
CREATE INDEX IF NOT EXISTS idx_vectors_model ON vectors(model);
|
|
CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag);
|
|
-- The copies of a kept role, for `Store.role` and the feed's copy counts.
|
|
-- Partial: all but about 3% of roles are no copy.
|
|
CREATE INDEX IF NOT EXISTS idx_roles_dup_of
|
|
ON roles(dup_of) WHERE dup_of IS NOT NULL;
|
|
-- A role's places, and the roles in a set of places (the Place filter and
|
|
-- the country and place gates, `uid IN (... WHERE place_id IN ...)`).
|
|
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
|
|
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
|
|
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
|
|
-- The remote roles whose area holds a set of places (a remote: token).
|
|
CREATE INDEX IF NOT EXISTS idx_role_remote_place ON role_remote(place_id, uid);
|
|
-- The roles the next prep maps. Partial: after a prep there are none.
|
|
CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0;
|
|
-- The feed's Closing soon filter. Partial: most roles state no deadline.
|
|
CREATE INDEX IF NOT EXISTS idx_roles_deadline ON roles(deadline) WHERE deadline IS NOT NULL;
|
|
-- An agent's judgements, for its feed under CV match.
|
|
CREATE INDEX IF NOT EXISTS idx_role_judgements_agent ON role_judgements(agent_id, cv_score DESC);
|
|
"""
|
|
|
|
#: The table whose absence marks a file that has never been a roledesk
|
|
#: database, and is therefore created from `SCHEMA` rather than migrated.
|
|
MARKER = "roles"
|
|
|
|
Step = str | Callable[[sqlite3.Connection], None]
|
|
|
|
|
|
def _migrate_posted_at(conn: sqlite3.Connection) -> None:
|
|
"""v10's step: add `posted_at`, then backfill it for every row already
|
|
stored, by the same rule `Store.upsert_role` applies to a new one (see
|
|
`app.models.posted_at`). A callable, not SQL, because the rule reads
|
|
`raw` for which host a role came from. `'' NOT NULL` is a placeholder
|
|
only until the UPDATE below runs, in the same transaction as the ALTER
|
|
(`_apply` wraps every migration in one); every row gets a real value."""
|
|
conn.execute("ALTER TABLE roles ADD COLUMN posted_at TEXT NOT NULL DEFAULT ''")
|
|
rows = conn.execute("SELECT uid, published_at, first_seen, raw FROM roles").fetchall()
|
|
conn.executemany(
|
|
"UPDATE roles SET posted_at = ? WHERE uid = ?",
|
|
[(posted_at(row["published_at"], row["first_seen"],
|
|
posted_host(json.loads(row["raw"] or "{}"))), row["uid"])
|
|
for row in rows])
|
|
|
|
|
|
MIGRATIONS: tuple[tuple[int, str, Step], ...] = (
|
|
# Never run as a migration: a file without the marker table is created
|
|
# from `SCHEMA` and stamped. It exists so the first real migration is 2.
|
|
(1, "roles, companies, subscriptions, applications, agents, scores, state", ""),
|
|
(2, "roles.permit, roles.languages and roles.extract_tries, for extract", """
|
|
ALTER TABLE roles ADD COLUMN permit TEXT NOT NULL DEFAULT ''
|
|
CHECK (permit IN ('', 'sponsors', 'no_sponsorship'));
|
|
ALTER TABLE roles ADD COLUMN languages TEXT NOT NULL DEFAULT '[]';
|
|
ALTER TABLE roles ADD COLUMN extract_tries INTEGER NOT NULL DEFAULT 0;
|
|
"""),
|
|
(3, "vectors, and the agent's cached intent vector, for score", """
|
|
CREATE TABLE vectors (
|
|
uid TEXT PRIMARY KEY REFERENCES roles(uid) ON DELETE CASCADE,
|
|
model TEXT NOT NULL,
|
|
dim INTEGER NOT NULL,
|
|
vec BLOB NOT NULL,
|
|
embedded_at TEXT NOT NULL
|
|
);
|
|
ALTER TABLE agents ADD COLUMN intent_key TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE agents ADD COLUMN intent_vec BLOB;
|
|
"""),
|
|
(4, "roles.vote, roles.summary and roles.article_text, for the phone UI", """
|
|
ALTER TABLE roles ADD COLUMN vote INTEGER CHECK (vote IN (-1, 0, 1));
|
|
ALTER TABLE roles ADD COLUMN voted_at TEXT;
|
|
ALTER TABLE roles ADD COLUMN summary TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE roles ADD COLUMN summarised_at TEXT;
|
|
ALTER TABLE roles ADD COLUMN article_text TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE roles ADD COLUMN article_at TEXT;
|
|
"""),
|
|
(5, "roles.read_at and role_tags, for the feed's read state and tags", """
|
|
ALTER TABLE roles ADD COLUMN read_at TEXT;
|
|
CREATE TABLE role_tags (
|
|
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
|
tag TEXT NOT NULL,
|
|
PRIMARY KEY (uid, tag)
|
|
);
|
|
"""),
|
|
# Votes and affinities move from the role and company rows to one row per
|
|
# agent. What was global goes to the default agent, the enabled one with
|
|
# the lowest id (`Store.default_agent`); a file with something to move and
|
|
# no agent gets one first, so nothing is lost. The columns are then
|
|
# dropped, which SQLite has allowed since 3.35.
|
|
(6, "agent_companies and role_votes: affinity and vote become per agent", """
|
|
CREATE TABLE agent_companies (
|
|
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
|
company_key TEXT NOT NULL,
|
|
affinity INTEGER NOT NULL CHECK (affinity BETWEEN -1 AND 2),
|
|
updated_at TEXT NOT NULL,
|
|
PRIMARY KEY (agent_id, company_key)
|
|
);
|
|
CREATE TABLE role_votes (
|
|
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
|
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
|
vote INTEGER NOT NULL CHECK (vote IN (-1, 0, 1)),
|
|
voted_at TEXT NOT NULL,
|
|
PRIMARY KEY (agent_id, uid)
|
|
);
|
|
INSERT INTO agents (name, intent, gates, weights, enabled, created_at)
|
|
SELECT 'My ranking', '', '[]', '{}', 1, strftime('%Y-%m-%dT%H:%M:%S+00:00', 'now')
|
|
WHERE NOT EXISTS (SELECT 1 FROM agents)
|
|
AND (EXISTS (SELECT 1 FROM roles WHERE vote IS NOT NULL)
|
|
OR EXISTS (SELECT 1 FROM companies WHERE affinity <> 0));
|
|
INSERT INTO role_votes (agent_id, uid, vote, voted_at)
|
|
SELECT (SELECT id FROM agents ORDER BY enabled DESC, id LIMIT 1), uid, vote,
|
|
COALESCE(voted_at, strftime('%Y-%m-%dT%H:%M:%S+00:00', 'now'))
|
|
FROM roles WHERE vote IS NOT NULL;
|
|
INSERT INTO agent_companies (agent_id, company_key, affinity, updated_at)
|
|
SELECT (SELECT id FROM agents ORDER BY enabled DESC, id LIMIT 1), key, affinity, updated_at
|
|
FROM companies WHERE affinity <> 0;
|
|
ALTER TABLE roles DROP COLUMN vote;
|
|
ALTER TABLE roles DROP COLUMN voted_at;
|
|
ALTER TABLE companies DROP COLUMN affinity;
|
|
"""),
|
|
(7, "companies.ats, country and city: a company the app cannot read yet is still a row", """
|
|
ALTER TABLE companies ADD COLUMN ats TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE companies ADD COLUMN country TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE companies ADD COLUMN city TEXT NOT NULL DEFAULT '';
|
|
"""),
|
|
(8, "agents.limits: how deep into its ranking the model stages read is each agent's own", """
|
|
ALTER TABLE agents ADD COLUMN limits TEXT NOT NULL DEFAULT '{}';
|
|
"""),
|
|
(9, "companies.ats_slug: the slug of a host the app cannot read yet is kept", """
|
|
ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT '';
|
|
"""),
|
|
(10, "roles.posted_at: recency and the newest feed count from when a role "
|
|
"was posted, not just when the app found it", _migrate_posted_at),
|
|
# A Python step, not SQL: classify() lives in plugins/ats.py, and running
|
|
# its current rules is the whole point (see _recompute_function below).
|
|
(11, "roles.function recomputed for every not-yet-extracted role, word-boundary "
|
|
"classifier fix", lambda conn: _recompute_function(conn)),
|
|
(12, "roles.salary_text: the pay a board stated as one line, kept even when it "
|
|
"could not be normalised into salary_min/salary_max/currency", """
|
|
ALTER TABLE roles ADD COLUMN salary_text TEXT NOT NULL DEFAULT '';
|
|
"""),
|
|
(13, "roles.country recomputed from place for every not-yet-extracted role whose "
|
|
"board gave none, country lookup fix", lambda conn: _recompute_country(conn)),
|
|
# An agent's one intent becomes its first prompt, named after the agent,
|
|
# with its cached vector: the key hashes the model and the text, which
|
|
# are unchanged, so nothing is embedded again. An empty intent becomes no
|
|
# prompt. The three columns are then dropped, as v6 dropped the ones it
|
|
# moved.
|
|
(14, "agent_prompts and scores.prompt: an agent holds several prompts, and a "
|
|
"score names the one that gave the fit", """
|
|
CREATE TABLE agent_prompts (
|
|
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
|
position INTEGER NOT NULL,
|
|
name TEXT NOT NULL,
|
|
text TEXT NOT NULL DEFAULT '',
|
|
vec_key TEXT NOT NULL DEFAULT '',
|
|
vec BLOB,
|
|
PRIMARY KEY (agent_id, position),
|
|
UNIQUE (agent_id, name COLLATE NOCASE)
|
|
);
|
|
INSERT INTO agent_prompts (agent_id, position, name, text, vec_key, vec)
|
|
SELECT id, 0, name, intent, intent_key, intent_vec FROM agents
|
|
WHERE TRIM(intent, ' ' || char(9, 10, 13)) <> '';
|
|
ALTER TABLE agents DROP COLUMN intent;
|
|
ALTER TABLE agents DROP COLUMN intent_key;
|
|
ALTER TABLE agents DROP COLUMN intent_vec;
|
|
ALTER TABLE scores ADD COLUMN prompt TEXT NOT NULL DEFAULT '';
|
|
"""),
|
|
# The Watchlist's own rating, global and apart from an agent's affinity:
|
|
# NULL not yet rated, 1 worth checking by hand, 0 not. Every company
|
|
# starts unrated.
|
|
(15, "companies.watch and careers_opened_at: the Watchlist's rating and "
|
|
"when its careers page was last opened", """
|
|
ALTER TABLE companies ADD COLUMN watch INTEGER CHECK (watch IN (0, 1));
|
|
ALTER TABLE companies ADD COLUMN careers_opened_at TEXT NOT NULL DEFAULT '';
|
|
"""),
|
|
# Why a person reads the company's careers page by hand, one of
|
|
# models.HAND_REASONS; '' when nobody has to. When it was last looked at
|
|
# is careers_opened_at.
|
|
(16, "companies.hand_reason: why a company is read by hand", """
|
|
ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
|
|
"""),
|
|
# A Python step: the fold and the title classifier are Python, as in 11.
|
|
(17, "roles.place_fold for the whole-word place filter, and roles.seniority from "
|
|
"the title where extract has not read the role", lambda conn: _fold_places(conn)),
|
|
# A Python step: the first rebuild runs the rule in app/dupes.py.
|
|
(18, "roles.dup_of: one posting listed several times is one role, the others "
|
|
"copies of it", lambda conn: _add_dup_of(conn)),
|
|
# A Python step: the seed and the resolver are Python.
|
|
(19, "places, place_aliases, place_includes and role_places: a role has a list of "
|
|
"places, resolved against a gazetteer; roles.teaser; the three seniority "
|
|
"sources", lambda conn: _add_places(conn)),
|
|
# A Python step: the seed folds every name.
|
|
(20, "company_aliases and the empty enrichment columns of companies",
|
|
lambda conn: _add_company_aliases(conn)),
|
|
# A Python step: params is JSON, the company is found by company_key and
|
|
# each board that read everywhere is logged.
|
|
(21, "companies.all_roles: a company tag that reads its boards with the agents' wide "
|
|
"reach, given to every favourite and taking over a board's {\"reach\": \"all\"}",
|
|
lambda conn: _add_all_roles(conn)),
|
|
# A Python step: the host words are read out of raw's JSON.
|
|
(22, "roles.*_host, roles.employment, roles.prep_due and unmapped: the host's own "
|
|
"words are stored and app.prep maps them", lambda conn: _add_host_words(conn)),
|
|
# A Python step: it adds the column only where it is missing.
|
|
(23, "agents.fetch_terms: words that narrow what big boards read with the wide reach "
|
|
"send", lambda conn: _add_fetch_terms(conn)),
|
|
# A Python step: params is JSON and the company is found by company_key.
|
|
(24, "companies.all_countries takes over the All roles tag, and companies.all_roles "
|
|
"means no role narrowing; a board's {\"terms\": false} becomes its company's "
|
|
"all_roles", lambda conn: _split_all_roles(conn)),
|
|
# Seniority became a ladder of two tracks (models.LADDER): staff, lead,
|
|
# manager and director each split, so a model level in one of them is
|
|
# emptied for extract to read the role again. The level shown is then
|
|
# the first filled of the three, as Store.write_prep writes it; the
|
|
# title's is mapped again by the next prep (app.prep.stage.VERSION 2).
|
|
(25, "roles.seniority_model cleared where it is staff, lead, manager or director, "
|
|
"which the ladder splits; roles.seniority from the model, stated and title levels", """
|
|
UPDATE roles SET seniority_model = ''
|
|
WHERE seniority_model IN ('staff', 'lead', 'manager', 'director');
|
|
UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
|
|
NULLIF(seniority_stated, ''), seniority_title);
|
|
"""),
|
|
# A Python step: it adds the column only where it is missing.
|
|
(26, "roles.posting_url: where a Jobindex ad's own page leads, found when its text is "
|
|
"fetched", lambda conn: _add_posting_url(conn)),
|
|
# A Python step: it adds the columns only where they are missing, and seeds
|
|
# Anywhere. The areas are written by the next prep (app.prep.stage.VERSION 3).
|
|
(27, "roles.remote_model and roles.remote_where_model, role_remote and the Anywhere "
|
|
"place: where a remote hire may live", lambda conn: _add_remote_area(conn)),
|
|
# A Python step: it adds the columns only where they are missing. The
|
|
# deadline is composed by the next prep (app.prep.stage.VERSION 4).
|
|
(28, "agents.cv_text, cv_languages, avoid_text and must_text, roles.languages_required and the "
|
|
"deadline columns, and role_judgements: the CV match, the language flag and the "
|
|
"deadline", lambda conn: _add_cv_match(conn)),
|
|
# A Python step: it adds the columns only where they are missing. Rows
|
|
# keep their numbers; the judge reads them again (judge.FORMAT).
|
|
(29, "role_judgements.musts and broken: the verdict on each must-have, and how many are "
|
|
"broken", lambda conn: _add_musts(conn)),
|
|
# A Python step: it adds the column only where it is missing. The judge
|
|
# reads every role again (judge.FORMAT).
|
|
(30, "agents.like_text: what the person likes, beside what they do not like",
|
|
lambda conn: _add_like_text(conn)),
|
|
# A Python step: it adds the column and the table only where they are
|
|
# missing. The next score fills them.
|
|
(31, "scores.cv_fit and agent_cv_vectors: how close each role is to the agent's CV",
|
|
lambda conn: _add_cv_fit(conn)),
|
|
)
|
|
|
|
#: What a file created from `SCHEMA` today is already at.
|
|
LATEST = MIGRATIONS[-1][0]
|
|
|
|
|
|
def _recompute_function(conn: sqlite3.Connection) -> None:
|
|
"""Migration 11's step. `roles.function` used to be filled by a keyword
|
|
classifier that matched substrings (" it" inside "Italy"), never
|
|
returned '' (only "other", outside the declared vocabulary) and checked
|
|
department before title; `plugins.ats.classify` no longer does any of
|
|
that (see its own docstring). A role `extract` has already read keeps
|
|
whatever it decided — `function` is extract's from then on, see
|
|
`Store.set_extracted` — so only roles with no `extracted_at` are
|
|
recomputed here, from their stored title and department (department
|
|
lives in `raw`, not its own column: docs/decisions.md, "raw is
|
|
transport, columns are storage").
|
|
|
|
A Python step because the classifier is Python, not SQL; `plugins` is a
|
|
top-level package outside `app`'s own tree, so it is reached the same
|
|
way `app/jobs/extract.py` reaches `FUNCTIONS`: inserted onto `sys.path`
|
|
rather than imported as an installed dependency, since `plugins` is not
|
|
in `pyproject.toml`'s packages."""
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
root = Path(__file__).resolve().parents[2]
|
|
if str(root) not in sys.path:
|
|
sys.path.insert(0, str(root))
|
|
from plugins.ats import classify
|
|
|
|
rows = conn.execute(
|
|
"SELECT uid, title, raw, function FROM roles WHERE extracted_at IS NULL").fetchall()
|
|
updates = []
|
|
for uid, title, raw, old in rows:
|
|
try:
|
|
department = json.loads(raw or "{}").get("department") or ""
|
|
except (TypeError, ValueError):
|
|
department = ""
|
|
new = classify(department, title)
|
|
if new != old:
|
|
updates.append((new, uid))
|
|
if updates:
|
|
conn.executemany("UPDATE roles SET function = ? WHERE uid = ?", updates)
|
|
log.info("migration 11: recomputed function for %d of %d not-yet-extracted roles",
|
|
len(updates), len(rows))
|
|
|
|
|
|
def _recompute_country(conn: sqlite3.Connection) -> None:
|
|
"""Migration 13's step. `app.models.country_of` used to read a US state
|
|
code over a country's ("Bogota, CO" was US), return '' for a line naming
|
|
several European countries, and miss most countries outside Europe;
|
|
`Store.backfill_from_place` fills only an empty column, so a wrong code
|
|
would stay. Recomputed from `place` for every role `extract` has not
|
|
read (its answer only ever filled an empty column, but a role it read is
|
|
left as it is) and whose board sent no `country` of its own in `raw`
|
|
(the `search` plugin's Jobindex and Adzuna rows do), which is what
|
|
`Store.upsert_role` would write for that role today."""
|
|
rows = conn.execute(
|
|
"SELECT uid, place, raw, country FROM roles WHERE extracted_at IS NULL").fetchall()
|
|
updates = []
|
|
for uid, place, raw, old in rows:
|
|
try:
|
|
given = json.loads(raw or "{}").get("country") or ""
|
|
except (TypeError, ValueError, AttributeError):
|
|
given = ""
|
|
if given:
|
|
continue
|
|
new = country_of(place or "")
|
|
if new != old:
|
|
updates.append((new, uid))
|
|
if updates:
|
|
conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", updates)
|
|
log.info("migration 13: recomputed country for %d of %d not-yet-extracted roles",
|
|
len(updates), len(rows))
|
|
|
|
|
|
def _fold_places(conn: sqlite3.Connection) -> None:
|
|
"""Migration 17's step. Adds `roles.place_fold` and fills it with
|
|
`places.fold(place)` for every role, as `Store.upsert_role` writes it for
|
|
a new one, and fills an empty `seniority` from `seniority.from_title` on
|
|
every role `extract` has not read (it overwrites the title's level when it
|
|
does, see `Store.set_extracted`).
|
|
|
|
Each distinct place and title is folded or classified once: 34,538 roles
|
|
hold about 5,600 places. The rows are written with one `executemany` per
|
|
column inside `_apply`'s transaction, and only those with something to
|
|
write."""
|
|
conn.execute("ALTER TABLE roles ADD COLUMN place_fold TEXT NOT NULL DEFAULT ''")
|
|
rows = conn.execute(
|
|
"SELECT uid, place, title, seniority, extracted_at FROM roles").fetchall()
|
|
folds: dict[str, str] = {}
|
|
levels: dict[str, str] = {}
|
|
place_updates, level_updates = [], []
|
|
for uid, place, title, level, extracted_at in rows:
|
|
if place:
|
|
if place not in folds:
|
|
folds[place] = places.fold(place)
|
|
if folds[place]:
|
|
place_updates.append((folds[place], uid))
|
|
if not level and extracted_at is None and title:
|
|
if title not in levels:
|
|
levels[title] = from_title(title)
|
|
if levels[title]:
|
|
level_updates.append((levels[title], uid))
|
|
conn.executemany("UPDATE roles SET place_fold = ? WHERE uid = ?", place_updates)
|
|
conn.executemany("UPDATE roles SET seniority = ? WHERE uid = ?", level_updates)
|
|
log.info("migration 17: folded %d places over %d roles, a level from the title for %d",
|
|
len(folds), len(place_updates), len(level_updates))
|
|
|
|
|
|
def _add_dup_of(conn: sqlite3.Connection) -> None:
|
|
"""Migration 18's step. Adds `roles.dup_of` and runs the first rebuild
|
|
(`store.rebuild_dupes`) inside `_apply`'s transaction, so a migrated file
|
|
hides its copies from the first open rather than from the next pull.
|
|
Imported here, not at the top: `store` imports this module."""
|
|
from app.storage.store import rebuild_dupes
|
|
|
|
conn.execute("ALTER TABLE roles ADD COLUMN dup_of TEXT")
|
|
counts = rebuild_dupes(conn)
|
|
log.info("migration 18: %d copies of %d kept roles", counts.copies, counts.groups)
|
|
|
|
|
|
_V19 = """
|
|
ALTER TABLE roles ADD COLUMN teaser TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE roles ADD COLUMN seniority_model TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE roles ADD COLUMN seniority_stated TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE roles ADD COLUMN seniority_title TEXT NOT NULL DEFAULT '';
|
|
CREATE TABLE places (
|
|
id INTEGER PRIMARY KEY,
|
|
kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')),
|
|
key TEXT NOT NULL,
|
|
name TEXT NOT NULL,
|
|
parent_id INTEGER REFERENCES places(id),
|
|
UNIQUE (kind, key)
|
|
);
|
|
CREATE TABLE place_aliases (
|
|
fold TEXT NOT NULL,
|
|
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
|
UNIQUE (fold, place_id)
|
|
);
|
|
CREATE TABLE place_includes (
|
|
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
|
part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
|
PRIMARY KEY (place_id, part_id)
|
|
);
|
|
CREATE TABLE role_places (
|
|
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
|
position INTEGER NOT NULL,
|
|
text TEXT NOT NULL,
|
|
fold TEXT NOT NULL,
|
|
place_id INTEGER REFERENCES places(id),
|
|
url TEXT NOT NULL DEFAULT ''
|
|
);
|
|
"""
|
|
|
|
|
|
def _add_places(conn: sqlite3.Connection) -> None:
|
|
"""Migration 19's step. The tables and columns, the gazetteer seeded, and
|
|
every role's places: the `places` of its `raw` where a reader stated
|
|
them, else its `place` column cut at `;` (not `raw`'s, which a pull that
|
|
read the role's own page sends empty), each resolved once per distinct
|
|
text.
|
|
|
|
The level splits into its sources without changing: on a role extract
|
|
has read, `seniority` is the model's; on any other, it is the stated
|
|
level when `raw` states one in the vocabulary, else the title's."""
|
|
for statement in _statements(_V19):
|
|
conn.execute(statement)
|
|
gazetteer.seed(conn)
|
|
gaz = gazetteer.load(conn)
|
|
rows = conn.execute(
|
|
"SELECT uid, raw, place, title, seniority, extracted_at FROM roles").fetchall()
|
|
by_uid: list[tuple[str, list[str]]] = []
|
|
levels = []
|
|
for uid, raw, place, title, level, extracted_at in rows:
|
|
try:
|
|
given = json.loads(raw or "{}")
|
|
except ValueError:
|
|
given = {}
|
|
given = given if isinstance(given, dict) else {}
|
|
if found := gazetteer.texts({"places": given.get("places"), "place": place}):
|
|
by_uid.append((uid, found))
|
|
stated = str(given.get("seniority") or "")
|
|
stated = stated if stated in SENIORITIES else ""
|
|
if extracted_at is not None:
|
|
levels.append((level, stated, from_title(title) if level else "", uid))
|
|
elif stated:
|
|
levels.append(("", stated, from_title(title), uid))
|
|
elif level:
|
|
levels.append(("", "", level, uid))
|
|
written = gazetteer.write_many(conn, gaz, by_uid)
|
|
conn.executemany("UPDATE roles SET seniority_model = ?, seniority_stated = ?, "
|
|
"seniority_title = ? WHERE uid = ?", levels)
|
|
log.info("migration 19: %d places seeded, %d place rows for %d roles, levels split on %d",
|
|
len(gaz.places), written, len(by_uid), len(levels))
|
|
|
|
|
|
_V20 = """
|
|
ALTER TABLE companies ADD COLUMN sector TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE companies ADD COLUMN stage TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE companies ADD COLUMN size_band TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE companies ADD COLUMN founded INTEGER;
|
|
ALTER TABLE companies ADD COLUMN last_funding INTEGER;
|
|
ALTER TABLE companies ADD COLUMN last_funding_currency TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE companies ADD COLUMN facts_source TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE companies ADD COLUMN facts_at TEXT NOT NULL DEFAULT '';
|
|
CREATE TABLE company_aliases (
|
|
fold TEXT NOT NULL,
|
|
company_key TEXT NOT NULL,
|
|
UNIQUE (fold, company_key)
|
|
);
|
|
CREATE INDEX idx_company_aliases_key ON company_aliases(company_key);
|
|
"""
|
|
|
|
|
|
def _add_company_aliases(conn: sqlite3.Connection) -> None:
|
|
"""Migration 20's step. The columns and the table, and every company's
|
|
name folded into it. Companies whose folds collide are logged and left
|
|
alone: merging is a person's call (`Store.merge_companies`)."""
|
|
for statement in _statements(_V20):
|
|
conn.execute(statement)
|
|
rows = conn.execute("SELECT key, name FROM companies").fetchall()
|
|
conn.executemany("INSERT OR IGNORE INTO company_aliases (fold, company_key) VALUES (?, ?)",
|
|
[(fold, key) for key, name in rows if (fold := company_fold(name))])
|
|
clashes = conn.execute("SELECT fold, group_concat(company_key, ', ') FROM company_aliases "
|
|
"GROUP BY fold HAVING COUNT(*) > 1").fetchall()
|
|
for fold, keys in clashes:
|
|
log.warning("migration 20: %r is one fold of %s", fold, keys)
|
|
log.info("migration 20: %d companies folded, %d folds shared", len(rows), len(clashes))
|
|
|
|
|
|
def _add_fetch_terms(conn: sqlite3.Connection) -> None:
|
|
"""Migration 23's step: `agents.fetch_terms`, empty for every agent, so
|
|
no board is narrowed by words until an agent is given some. Runs twice
|
|
without harm."""
|
|
columns = {r[1] for r in conn.execute("PRAGMA table_info(agents)")}
|
|
if "fetch_terms" not in columns:
|
|
conn.execute("ALTER TABLE agents ADD COLUMN fetch_terms TEXT NOT NULL DEFAULT '[]'")
|
|
|
|
|
|
def _add_posting_url(conn: sqlite3.Connection) -> None:
|
|
"""Migration 26's step: `roles.posting_url`, empty for every role. Runs
|
|
twice without harm."""
|
|
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
|
if "posting_url" not in columns:
|
|
conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''")
|
|
|
|
|
|
def _add_remote_area(conn: sqlite3.Connection) -> None:
|
|
"""Migration 27's step: `roles.remote_model` empty and
|
|
`roles.remote_where_model` NULL for every role, so `extract` reads the
|
|
roles it has read once more for the area; `role_remote`, empty until
|
|
the next prep; and Anywhere, seeded with the rest of the gazetteer.
|
|
|
|
Prep now writes `remote` outright from the model, the host's word and
|
|
the place line, where it used to keep a filled column. A board role's
|
|
`remote` without a host word becomes its `remote_host`, which a pull
|
|
keeps until the host states another word: on live 1,280 of the 3,880
|
|
open ones are given by nothing prep reads now, mostly Greenhouse, Oracle
|
|
and Workday roles an older reader mapped. A search role's `remote_host`
|
|
its reader no longer sends is an older reading of its title ("...
|
|
Remote Sensing ...", 41 roles), and is cleared for prep to read the
|
|
title again. Runs twice without harm."""
|
|
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
|
if "remote_model" not in columns:
|
|
conn.execute("ALTER TABLE roles ADD COLUMN remote_model TEXT NOT NULL DEFAULT ''")
|
|
if "remote_where_model" not in columns:
|
|
conn.execute("ALTER TABLE roles ADD COLUMN remote_where_model TEXT")
|
|
kept = conn.execute("UPDATE roles SET remote_host = remote WHERE remote_host = '' "
|
|
"AND remote != '' AND source != 'search'").rowcount
|
|
cleared = conn.execute("UPDATE roles SET remote_host = '' WHERE source = 'search' "
|
|
"AND remote_host != '' AND json_extract(raw, '$.remote') IS NULL"
|
|
).rowcount
|
|
log.info("migration 27: remote kept as the host's word on %d roles, cleared on %d",
|
|
kept, cleared)
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS role_remote (
|
|
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
|
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
|
PRIMARY KEY (uid, place_id))""")
|
|
gazetteer.seed(conn)
|
|
|
|
|
|
def _add_cv_match(conn: sqlite3.Connection) -> None:
|
|
"""Migration 28's step: an agent's CV, languages, avoid text and
|
|
must-haves, all empty; `roles.languages_required` NULL for every role, so `extract`
|
|
reads the roles it has read once more for the languages and the
|
|
deadline; `roles.deadline_host` from the date a reader already keeps in
|
|
`raw` (`applicationDeadline`, Jobnet and Jobbank), `deadline_model`
|
|
empty and `deadline` NULL until the next prep; and `role_judgements`,
|
|
empty. Runs twice without harm."""
|
|
agents = {r[1] for r in conn.execute("PRAGMA table_info(agents)")}
|
|
for column, ddl in (("cv_text", "TEXT NOT NULL DEFAULT ''"),
|
|
("cv_languages", "TEXT NOT NULL DEFAULT '[]'"),
|
|
("avoid_text", "TEXT NOT NULL DEFAULT ''"),
|
|
("must_text", "TEXT NOT NULL DEFAULT ''")):
|
|
if column not in agents:
|
|
conn.execute(f"ALTER TABLE agents ADD COLUMN {column} {ddl}")
|
|
roles = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
|
for column, ddl in (("languages_required", "TEXT"),
|
|
("deadline_host", "TEXT NOT NULL DEFAULT ''"),
|
|
("deadline_model", "TEXT NOT NULL DEFAULT ''"),
|
|
("deadline", "TEXT")):
|
|
if column not in roles:
|
|
conn.execute(f"ALTER TABLE roles ADD COLUMN {column} {ddl}")
|
|
dated = conn.execute(
|
|
"SELECT uid, json_extract(raw, '$.applicationDeadline') FROM roles "
|
|
"WHERE deadline_host = '' AND json_extract(raw, '$.applicationDeadline') IS NOT NULL"
|
|
).fetchall()
|
|
found = [(day, uid) for uid, value in dated if (day := iso_day(value))]
|
|
conn.executemany("UPDATE roles SET deadline_host = ? WHERE uid = ?", found)
|
|
log.info("migration 28: a host deadline on %d roles", len(found))
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS role_judgements (
|
|
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
|
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
|
skills INTEGER,
|
|
experience INTEGER,
|
|
career INTEGER,
|
|
cv_score REAL,
|
|
strengths TEXT NOT NULL DEFAULT '[]',
|
|
gaps TEXT NOT NULL DEFAULT '[]',
|
|
input_key TEXT NOT NULL DEFAULT '',
|
|
tries INTEGER NOT NULL DEFAULT 0,
|
|
judged_at TEXT,
|
|
PRIMARY KEY (uid, agent_id))""")
|
|
|
|
|
|
def _add_musts(conn: sqlite3.Connection) -> None:
|
|
"""Migration 29's step: `role_judgements.musts`, a JSON list, and
|
|
`broken`, the count of must-haves it holds as broken, empty and 0 on
|
|
every row. `judge.input_key` changed with it, so each judgement is read
|
|
again; until then it serves with no must-haves and no cap. Runs twice
|
|
without harm."""
|
|
columns = {r[1] for r in conn.execute("PRAGMA table_info(role_judgements)")}
|
|
for column, ddl in (("musts", "TEXT NOT NULL DEFAULT '[]'"),
|
|
("broken", "INTEGER NOT NULL DEFAULT 0")):
|
|
if column not in columns:
|
|
conn.execute(f"ALTER TABLE role_judgements ADD COLUMN {column} {ddl}")
|
|
|
|
|
|
def _add_like_text(conn: sqlite3.Connection) -> None:
|
|
"""Migration 30's step: `agents.like_text`, empty on every agent. Runs
|
|
twice without harm."""
|
|
columns = {r[1] for r in conn.execute("PRAGMA table_info(agents)")}
|
|
if "like_text" not in columns:
|
|
conn.execute("ALTER TABLE agents ADD COLUMN like_text TEXT NOT NULL DEFAULT ''")
|
|
|
|
|
|
def _add_cv_fit(conn: sqlite3.Connection) -> None:
|
|
"""Migration 31's step: `scores.cv_fit`, NULL on every row until the
|
|
next score, and `agent_cv_vectors`, empty until then. Runs twice without
|
|
harm."""
|
|
columns = {r[1] for r in conn.execute("PRAGMA table_info(scores)")}
|
|
if "cv_fit" not in columns:
|
|
conn.execute("ALTER TABLE scores ADD COLUMN cv_fit REAL")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS agent_cv_vectors (
|
|
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
|
position INTEGER NOT NULL,
|
|
vec_key TEXT NOT NULL,
|
|
vec BLOB NOT NULL,
|
|
PRIMARY KEY (agent_id, position))""")
|
|
|
|
|
|
def _add_all_roles(conn: sqlite3.Connection) -> None:
|
|
"""Migration 21's step. The column; the tag for every company that is a
|
|
favourite of any agent, since favourite no longer widens what a pull
|
|
fetches and the tag does, so nothing read today is lost; and the tag for
|
|
each company whose board's `params` held `{"reach": "all"}`, which the
|
|
step removes from them. Such a board read everywhere and reads the wide
|
|
reach from now on, so each is logged. Runs twice without harm."""
|
|
columns = {r[1] for r in conn.execute("PRAGMA table_info(companies)")}
|
|
if "all_roles" not in columns:
|
|
conn.execute("ALTER TABLE companies ADD COLUMN all_roles INTEGER NOT NULL DEFAULT 0")
|
|
starred = conn.execute(
|
|
"UPDATE companies SET all_roles=1 WHERE all_roles=0 AND key IN "
|
|
"(SELECT company_key FROM agent_companies WHERE affinity=2)").rowcount
|
|
log.info("migration 21: %d favourite companies tagged All roles", starred)
|
|
for sub_id, target, company, label, raw in conn.execute(
|
|
"SELECT id, target, company, label, params FROM subscriptions "
|
|
"WHERE source = 'ats'").fetchall():
|
|
try:
|
|
params = json.loads(raw or "{}")
|
|
except ValueError:
|
|
continue
|
|
if not isinstance(params, dict) or params.get("reach") != "all":
|
|
continue
|
|
del params["reach"]
|
|
conn.execute("UPDATE subscriptions SET params=? WHERE id=?", (json.dumps(params), sub_id))
|
|
key = company_key(company or label)
|
|
if conn.execute("UPDATE companies SET all_roles=1 WHERE key=?", (key,)).rowcount:
|
|
log.warning("migration 21: %s read all roles everywhere; %r is tagged All roles, "
|
|
"which reads the agents' wide reach", target, key)
|
|
else:
|
|
log.warning("migration 21: %s read all roles everywhere, but %r has no company row "
|
|
"to tag; it reads the agents' reach now", target, key)
|
|
|
|
|
|
def _split_all_roles(conn: sqlite3.Connection) -> None:
|
|
"""Migration 24's step. Fetching has two filters, countries and role
|
|
types, and each gets its own company override. `all_countries` is the
|
|
old All roles tag, the wide reach, copied from `all_roles` when the
|
|
column is added. `all_roles` keeps its column and now means no role-type
|
|
narrowing: a company that had the tag keeps it too, since the tag was
|
|
given to the favourites whose every role the owner wanted. A board whose
|
|
`params` hold `{"terms": false}` gives its company `all_roles`, and the
|
|
param goes. Runs twice without harm: the copy happens only with the new
|
|
column."""
|
|
columns = {r[1] for r in conn.execute("PRAGMA table_info(companies)")}
|
|
if "all_countries" not in columns:
|
|
conn.execute("ALTER TABLE companies ADD COLUMN all_countries INTEGER NOT NULL DEFAULT 0")
|
|
copied = conn.execute("UPDATE companies SET all_countries=1 WHERE all_roles=1").rowcount
|
|
log.info("migration 24: %d companies tagged All roles are All countries and All roles",
|
|
copied)
|
|
moved = 0
|
|
for sub_id, target, company, label, raw in conn.execute(
|
|
"SELECT id, target, company, label, params FROM subscriptions "
|
|
"WHERE source = 'ats'").fetchall():
|
|
try:
|
|
params = json.loads(raw or "{}")
|
|
except ValueError:
|
|
continue
|
|
if not isinstance(params, dict) or "terms" not in params:
|
|
continue
|
|
off = params.pop("terms") is False
|
|
conn.execute("UPDATE subscriptions SET params=? WHERE id=?", (json.dumps(params), sub_id))
|
|
key = company_key(company or label)
|
|
if not off:
|
|
continue
|
|
if conn.execute("UPDATE companies SET all_roles=1 WHERE key=?", (key,)).rowcount:
|
|
moved += 1
|
|
else:
|
|
log.warning("migration 24: %s read without terms, but %r has no company row "
|
|
"to mark All roles; it takes the agents' terms now", target, key)
|
|
log.info("migration 24: %d boards' {\"terms\": false} became All roles", moved)
|
|
|
|
|
|
_V22_COLUMNS = ("seniority_host", "employment_host", "remote_host", "country_host", "employment")
|
|
|
|
_V22 = """
|
|
CREATE TABLE IF NOT EXISTS unmapped (
|
|
field TEXT NOT NULL,
|
|
host TEXT NOT NULL,
|
|
word TEXT NOT NULL,
|
|
roles INTEGER NOT NULL DEFAULT 0,
|
|
first_seen TEXT NOT NULL,
|
|
PRIMARY KEY (field, host, word)
|
|
);
|
|
DELETE FROM state WHERE key = 'prep:tables';
|
|
"""
|
|
|
|
|
|
def _add_host_words(conn: sqlite3.Connection) -> None:
|
|
"""Migration 22's step. The columns and `unmapped`, and the host words
|
|
of every role filled from its `raw` where it has them. Before this,
|
|
readers mapped `seniority` and `employment` in their own tables, so what
|
|
`raw` holds there is mostly a value of the vocabulary already, which
|
|
`app.prep` takes as it is. `remote` and `country` are the host's as
|
|
given. The `prep:tables` fingerprint goes, so the first prep maps every
|
|
open role from these words. Runs twice without harm."""
|
|
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
|
for column in _V22_COLUMNS:
|
|
if column not in columns:
|
|
conn.execute(f"ALTER TABLE roles ADD COLUMN {column} TEXT NOT NULL DEFAULT ''")
|
|
if "prep_due" not in columns:
|
|
conn.execute("ALTER TABLE roles ADD COLUMN prep_due INTEGER NOT NULL DEFAULT 0")
|
|
for statement in _statements(_V22):
|
|
conn.execute(statement)
|
|
updates = []
|
|
for uid, raw in conn.execute(
|
|
"SELECT uid, raw FROM roles WHERE json_extract(raw, '$.seniority') IS NOT NULL "
|
|
"OR json_extract(raw, '$.employment') IS NOT NULL "
|
|
"OR json_extract(raw, '$.remote') IS NOT NULL "
|
|
"OR json_extract(raw, '$.country') IS NOT NULL").fetchall():
|
|
try:
|
|
given = json.loads(raw or "{}")
|
|
except ValueError:
|
|
continue
|
|
if isinstance(given, dict):
|
|
updates.append((*(host_word(given.get(k)) for k in (
|
|
"seniority", "employment", "remote", "country")), uid))
|
|
conn.executemany("UPDATE roles SET seniority_host = ?, employment_host = ?, "
|
|
"remote_host = ?, country_host = ? WHERE uid = ?", updates)
|
|
log.info("migration 22: host words from raw for %d roles", len(updates))
|
|
|
|
|
|
def user_version(conn: sqlite3.Connection) -> int:
|
|
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
|
|
|
|
|
def _stamp(conn: sqlite3.Connection, version: int) -> None:
|
|
# PRAGMA takes no bound parameters; the value is an int from MIGRATIONS.
|
|
conn.execute(f"PRAGMA user_version = {int(version)}")
|
|
|
|
|
|
def _statements(script: str) -> list[str]:
|
|
"""A script split into statements, cut where `sqlite3.complete_statement`
|
|
says one ends, which knows about comments and trigger bodies. Not
|
|
`executescript`, which commits before it starts and autocommits each
|
|
statement, so a failing migration would be half applied."""
|
|
statements, buffer = [], ""
|
|
for line in script.splitlines(keepends=True):
|
|
buffer += line
|
|
if sqlite3.complete_statement(buffer):
|
|
statements.append(buffer.strip())
|
|
buffer = ""
|
|
if buffer.strip():
|
|
statements.append(buffer.strip())
|
|
return statements
|
|
|
|
|
|
def migrate(conn: sqlite3.Connection) -> list[int]:
|
|
"""Bring an existing database up to `LATEST`. Returns what it applied."""
|
|
current = user_version(conn)
|
|
if current > LATEST:
|
|
raise RuntimeError(
|
|
f"this database is at schema v{current}, but this roledesk only knows "
|
|
f"up to v{LATEST}. Upgrade roledesk rather than let older code write to it.")
|
|
applied = []
|
|
for version, description, step in MIGRATIONS:
|
|
if version <= current:
|
|
continue
|
|
log.info("migrating database to v%d: %s", version, description)
|
|
_apply(conn, version, step)
|
|
applied.append(version)
|
|
return applied
|
|
|
|
|
|
def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
|
|
"""One migration and its version stamp, in one explicit transaction.
|
|
|
|
Explicit because sqlite3 only opens a transaction implicitly before DML: an
|
|
`ALTER` would otherwise be durable at once while the stamp stayed behind,
|
|
and the next open would replay it into a "duplicate column" forever.
|
|
"""
|
|
started = not conn.in_transaction
|
|
if started:
|
|
conn.execute("BEGIN")
|
|
try:
|
|
if callable(step):
|
|
step(conn)
|
|
else:
|
|
for statement in _statements(step):
|
|
conn.execute(statement)
|
|
_stamp(conn, version)
|
|
except Exception:
|
|
if started:
|
|
conn.execute("ROLLBACK")
|
|
raise
|
|
if started:
|
|
conn.execute("COMMIT")
|
|
|
|
|
|
def _create(conn: sqlite3.Connection) -> None:
|
|
"""A new file: `SCHEMA`, and the gazetteer seeded."""
|
|
for statement in _statements(SCHEMA):
|
|
conn.execute(statement)
|
|
gazetteer.seed(conn)
|
|
|
|
|
|
def initialise(conn: sqlite3.Connection) -> list[int]:
|
|
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
|
|
fresh = conn.execute(
|
|
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (MARKER,)
|
|
).fetchone() is None
|
|
applied: list[int] = []
|
|
if fresh:
|
|
# Atomic like a migration: a file half created would otherwise carry
|
|
# the marker table and be taken for one to migrate.
|
|
_apply(conn, LATEST, _create)
|
|
else:
|
|
applied = migrate(conn)
|
|
for statement in _statements(INDEXES):
|
|
conn.execute(statement)
|
|
return applied
|