514 lines
23 KiB
Python
514 lines
23 KiB
Python
"""The tables, and how a database file gets from an old shape to this one.
|
|
|
|
`SCHEMA` is the current shape. A new file is created from it and stamped with
|
|
`LATEST`, so it never runs a migration. `MIGRATIONS` is how an existing file
|
|
catches up: ordered, numbered, applied once each, tracked in SQLite's own
|
|
`PRAGMA user_version`.
|
|
|
|
Adding a column means editing both: the DDL in `SCHEMA` for new files, and an
|
|
appended migration for old ones. Never renumber or edit a migration that has
|
|
shipped.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import sqlite3
|
|
from collections.abc import Callable
|
|
|
|
from app.models import posted_at, posted_host
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
SCHEMA = """
|
|
-- One posting on one board. `uid` is sha1(source, source_id), the identity desk
|
|
-- gives every Item. The gate fields are real columns, unpacked from the item's
|
|
-- raw on upsert or filled later by extract: raw is transport, the columns are
|
|
-- what gets queried.
|
|
CREATE TABLE IF NOT EXISTS roles (
|
|
uid TEXT PRIMARY KEY,
|
|
source TEXT NOT NULL,
|
|
source_id TEXT NOT NULL,
|
|
subscription_id INTEGER REFERENCES subscriptions(id) ON DELETE SET NULL,
|
|
-- The cluster key is (company_key, title_key, place_key). No foreign key
|
|
-- to companies: a role may arrive before anyone has made a company row.
|
|
company_key TEXT NOT NULL DEFAULT '',
|
|
company TEXT NOT NULL DEFAULT '',
|
|
title TEXT NOT NULL DEFAULT '',
|
|
title_key TEXT NOT NULL DEFAULT '',
|
|
-- Geography. place and not location, which is where an Item's original
|
|
-- lives. remote and country are separate so a gate on remote can also
|
|
-- require the country: "UK | Remote" is not remote from Berlin.
|
|
place TEXT NOT NULL DEFAULT '',
|
|
place_key TEXT NOT NULL DEFAULT '',
|
|
country TEXT NOT NULL DEFAULT '',
|
|
remote TEXT NOT NULL DEFAULT ''
|
|
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
|
|
salary_min INTEGER,
|
|
salary_max INTEGER,
|
|
currency TEXT NOT NULL DEFAULT '',
|
|
seniority TEXT NOT NULL DEFAULT '',
|
|
function TEXT NOT NULL DEFAULT '',
|
|
-- Whether the posting says it sponsors a work permit, and the languages
|
|
-- (ISO 639-1, a JSON array) it says are required. Only extract fills these.
|
|
permit TEXT NOT NULL DEFAULT ''
|
|
CHECK (permit IN ('', 'sponsors', 'no_sponsorship')),
|
|
languages TEXT NOT NULL DEFAULT '[]',
|
|
description TEXT NOT NULL DEFAULT '', -- the posting's body text
|
|
feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over
|
|
original_url TEXT NOT NULL DEFAULT '', -- the posting page
|
|
published_at TEXT NOT NULL, -- what the source claims, UTC
|
|
-- What storing postings buys. first_seen is when the app itself first
|
|
-- stored the role; posted_at (app.models.posted_at) is the earlier of
|
|
-- first_seen and published_at, trusted only for a host known to report a
|
|
-- creation date rather than one that moves on an edit
|
|
-- (app.models.TRUSTED_POSTED_HOSTS) — recency and the feed's newest order
|
|
-- read posted_at, first_seen stays "when the app found it". closed_at is
|
|
-- set when a board fetched in full no longer lists the role, and cleared
|
|
-- if it returns.
|
|
posted_at TEXT NOT NULL DEFAULT '',
|
|
first_seen TEXT NOT NULL,
|
|
last_seen TEXT NOT NULL,
|
|
closed_at TEXT,
|
|
raw TEXT NOT NULL DEFAULT '{}',
|
|
extracted_at TEXT, -- when extract last filled columns
|
|
-- Replies from extract that left this role unanswered. At GIVE_UP_AFTER it
|
|
-- is no longer offered. A column, not raw, because every upsert rewrites raw.
|
|
extract_tries INTEGER NOT NULL DEFAULT 0,
|
|
-- NULL unread, else when the feed first opened it or judged it: a vote
|
|
-- stamps this too, see Store.set_vote, because voting is reading.
|
|
read_at TEXT,
|
|
-- About 60 words from summarise, and when it wrote them.
|
|
summary TEXT NOT NULL DEFAULT '',
|
|
summarised_at TEXT,
|
|
-- The posting's own page as plain text, fetched on demand for a role
|
|
-- whose description is only a teaser.
|
|
article_text TEXT NOT NULL DEFAULT '',
|
|
article_at TEXT,
|
|
created_at TEXT NOT NULL
|
|
);
|
|
|
|
-- The open vocabulary summarise reads off a posting: the role family, the
|
|
-- technologies and skills it names, and the domain. What the feed's filter
|
|
-- row is built from; see Store.set_tags and Store.tag_counts.
|
|
CREATE TABLE IF NOT EXISTS role_tags (
|
|
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
|
tag TEXT NOT NULL,
|
|
PRIMARY KEY (uid, tag)
|
|
);
|
|
|
|
-- A company as a target, whether or not any board of it is read. key is
|
|
-- models.company_key(name), the slug roles.company_key holds.
|
|
-- ats is the host the careers site runs on, readable by the ats plugin or
|
|
-- not (workday, successfactors, ...): what discovery found, '' unknown. A
|
|
-- company with an ats and no board subscription is one the app cannot read
|
|
-- yet. country is an ISO code and city the head office's. ats_slug is the
|
|
-- name that host knows the company by (the acme of acme.teamtailor.com),
|
|
-- '' where its URL carries none, kept for the parser written later.
|
|
CREATE TABLE IF NOT EXISTS companies (
|
|
key TEXT PRIMARY KEY,
|
|
name TEXT NOT NULL,
|
|
domain TEXT NOT NULL DEFAULT '',
|
|
status TEXT NOT NULL DEFAULT 'researching',
|
|
careers_url TEXT NOT NULL DEFAULT '',
|
|
notes TEXT NOT NULL DEFAULT '',
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
ats TEXT NOT NULL DEFAULT '',
|
|
country TEXT NOT NULL DEFAULT '',
|
|
city TEXT NOT NULL DEFAULT '',
|
|
ats_slug TEXT NOT NULL DEFAULT ''
|
|
);
|
|
|
|
-- The targets desk's runner hands to plugins. newsdesk's shape, with its two
|
|
-- labels named company and board from the first migration.
|
|
CREATE TABLE IF NOT EXISTS subscriptions (
|
|
id INTEGER PRIMARY KEY,
|
|
source TEXT NOT NULL, -- plugin name
|
|
target TEXT NOT NULL, -- e.g. greenhouse:acme
|
|
label TEXT NOT NULL, -- what Item.subscription names
|
|
company TEXT NOT NULL DEFAULT '',
|
|
board TEXT NOT NULL DEFAULT '',
|
|
params TEXT NOT NULL DEFAULT '{}', -- what a query target asks with
|
|
subscribed INTEGER NOT NULL DEFAULT 1,
|
|
fetch INTEGER NOT NULL DEFAULT 1,
|
|
show INTEGER NOT NULL DEFAULT 1,
|
|
added_at TEXT NOT NULL,
|
|
target_id TEXT NOT NULL DEFAULT '', -- the plugin's durable name for it
|
|
UNIQUE (source, target),
|
|
UNIQUE (source, label)
|
|
);
|
|
|
|
-- The tracker, in jobhunt-hq's shape (MIT). role_uid is NULL for a role that
|
|
-- was never posted, so an unsolicited approach is tracked like any other.
|
|
CREATE TABLE IF NOT EXISTS applications (
|
|
id TEXT PRIMARY KEY,
|
|
role_uid TEXT REFERENCES roles(uid) ON DELETE SET NULL,
|
|
company_key TEXT NOT NULL DEFAULT '',
|
|
role_title TEXT NOT NULL DEFAULT '',
|
|
url TEXT NOT NULL DEFAULT '',
|
|
status TEXT NOT NULL DEFAULT 'saved'
|
|
CHECK (status IN ('saved', 'applied', 'screen', 'technical',
|
|
'onsite', 'offer', 'rejected', 'withdrawn')),
|
|
source TEXT NOT NULL DEFAULT '',
|
|
applied_at TEXT,
|
|
next_action TEXT NOT NULL DEFAULT '',
|
|
next_date TEXT,
|
|
contact TEXT NOT NULL DEFAULT '',
|
|
notes TEXT NOT NULL DEFAULT '',
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
|
|
-- One row per status an application entered, written by the store and never
|
|
-- updated: what days-to-first-screen and conversion by source are read from.
|
|
CREATE TABLE IF NOT EXISTS application_events (
|
|
id INTEGER PRIMARY KEY,
|
|
application_id TEXT NOT NULL REFERENCES applications(id) ON DELETE CASCADE,
|
|
status TEXT NOT NULL,
|
|
at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TRIGGER IF NOT EXISTS application_events_append_only
|
|
BEFORE UPDATE ON application_events
|
|
BEGIN
|
|
SELECT RAISE(ABORT, 'application_events is append-only');
|
|
END;
|
|
|
|
-- A search agent: a named saved search with its own intent text, gates and
|
|
-- weights. gates is a list of {field, values, mode: require|prefer, weight}.
|
|
-- intent_vec caches the intent's vector; intent_key is sha1 of the embedding
|
|
-- model and the intent text, so editing either makes it stale. limits is how
|
|
-- deep into this agent's ranking extract and summarise read, as JSON: a key
|
|
-- it lacks means the `Limits` default.
|
|
CREATE TABLE IF NOT EXISTS agents (
|
|
id INTEGER PRIMARY KEY,
|
|
name TEXT NOT NULL UNIQUE,
|
|
intent TEXT NOT NULL DEFAULT '',
|
|
gates TEXT NOT NULL DEFAULT '[]',
|
|
weights TEXT NOT NULL DEFAULT '{}',
|
|
enabled INTEGER NOT NULL DEFAULT 1,
|
|
created_at TEXT NOT NULL,
|
|
intent_key TEXT NOT NULL DEFAULT '',
|
|
intent_vec BLOB,
|
|
limits TEXT NOT NULL DEFAULT '{}'
|
|
);
|
|
|
|
-- One agent's stance on a company: -1 hidden, 0 neutral, 1 liked, 2
|
|
-- favourite. No row is 0, so the table holds only what somebody set. Per
|
|
-- agent, because a company that is noise for one search is the target of
|
|
-- another.
|
|
CREATE TABLE IF NOT EXISTS agent_companies (
|
|
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
|
company_key TEXT NOT NULL,
|
|
affinity INTEGER NOT NULL CHECK (affinity BETWEEN -1 AND 2),
|
|
updated_at TEXT NOT NULL,
|
|
PRIMARY KEY (agent_id, company_key)
|
|
);
|
|
|
|
-- One agent's judgement of a role: -1 pass, 0 neutral, 1 like. No row is
|
|
-- unjudged. A vote moves the rank at read time, never hides a role, and its
|
|
-- vectors shape the agent's taste direction in `score`.
|
|
CREATE TABLE IF NOT EXISTS role_votes (
|
|
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
|
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
|
vote INTEGER NOT NULL CHECK (vote IN (-1, 0, 1)),
|
|
voted_at TEXT NOT NULL,
|
|
PRIMARY KEY (agent_id, uid)
|
|
);
|
|
|
|
-- A role is scored per agent. A gated role keeps its row, with the reason.
|
|
CREATE TABLE IF NOT EXISTS scores (
|
|
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
|
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
|
score REAL NOT NULL DEFAULT 0,
|
|
gated_reason TEXT NOT NULL DEFAULT '',
|
|
scored_at TEXT NOT NULL,
|
|
PRIMARY KEY (uid, agent_id)
|
|
);
|
|
|
|
-- One vector per role, little-endian float32. Only comparable to another from
|
|
-- the same model, so every read is scoped by it.
|
|
CREATE TABLE IF NOT EXISTS vectors (
|
|
uid TEXT PRIMARY KEY REFERENCES roles(uid) ON DELETE CASCADE,
|
|
model TEXT NOT NULL,
|
|
dim INTEGER NOT NULL,
|
|
vec BLOB NOT NULL,
|
|
embedded_at TEXT NOT NULL
|
|
);
|
|
|
|
-- The string map a desk plugin is handed as self.state. One namespace shared
|
|
-- by every plugin, so a plugin prefixes its keys.
|
|
CREATE TABLE IF NOT EXISTS state (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT NOT NULL
|
|
);
|
|
"""
|
|
|
|
#: Applied after `SCHEMA` or the migrations, every open. An index may name a
|
|
#: column only a migration adds, which is why these are kept apart.
|
|
INDEXES = """
|
|
CREATE INDEX IF NOT EXISTS idx_roles_company ON roles(company_key);
|
|
CREATE INDEX IF NOT EXISTS idx_roles_place ON roles(place_key);
|
|
CREATE INDEX IF NOT EXISTS idx_roles_first_seen ON roles(first_seen DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_roles_closed ON roles(closed_at);
|
|
-- Serves the newest feed (open_only, ordered by posted_at DESC), which is
|
|
-- most of what `Store.roles` runs: partial, since only an open role's
|
|
-- position in that order is ever asked for.
|
|
CREATE INDEX IF NOT EXISTS idx_roles_posted
|
|
ON roles(posted_at DESC, uid) WHERE closed_at IS NULL;
|
|
CREATE INDEX IF NOT EXISTS idx_roles_cluster ON roles(company_key, title_key, place_key);
|
|
CREATE INDEX IF NOT EXISTS idx_roles_board ON roles(source, subscription_id);
|
|
CREATE INDEX IF NOT EXISTS idx_application_events
|
|
ON application_events(application_id, at);
|
|
CREATE INDEX IF NOT EXISTS idx_scores_agent ON scores(agent_id, score DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_role_votes_agent ON role_votes(agent_id, vote);
|
|
CREATE INDEX IF NOT EXISTS idx_vectors_model ON vectors(model);
|
|
CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag);
|
|
"""
|
|
|
|
#: The table whose absence marks a file that has never been a roledesk
|
|
#: database, and is therefore created from `SCHEMA` rather than migrated.
|
|
MARKER = "roles"
|
|
|
|
Step = str | Callable[[sqlite3.Connection], None]
|
|
|
|
|
|
def _migrate_posted_at(conn: sqlite3.Connection) -> None:
|
|
"""v10's step: add `posted_at`, then backfill it for every row already
|
|
stored, by the same rule `Store.upsert_role` applies to a new one (see
|
|
`app.models.posted_at`). A callable, not SQL, because the rule reads
|
|
`raw` for which host a role came from. `'' NOT NULL` is a placeholder
|
|
only until the UPDATE below runs, in the same transaction as the ALTER
|
|
(`_apply` wraps every migration in one); every row gets a real value."""
|
|
conn.execute("ALTER TABLE roles ADD COLUMN posted_at TEXT NOT NULL DEFAULT ''")
|
|
rows = conn.execute("SELECT uid, published_at, first_seen, raw FROM roles").fetchall()
|
|
conn.executemany(
|
|
"UPDATE roles SET posted_at = ? WHERE uid = ?",
|
|
[(posted_at(row["published_at"], row["first_seen"],
|
|
posted_host(json.loads(row["raw"] or "{}"))), row["uid"])
|
|
for row in rows])
|
|
|
|
|
|
MIGRATIONS: tuple[tuple[int, str, Step], ...] = (
|
|
# Never run as a migration: a file without the marker table is created
|
|
# from `SCHEMA` and stamped. It exists so the first real migration is 2.
|
|
(1, "roles, companies, subscriptions, applications, agents, scores, state", ""),
|
|
(2, "roles.permit, roles.languages and roles.extract_tries, for extract", """
|
|
ALTER TABLE roles ADD COLUMN permit TEXT NOT NULL DEFAULT ''
|
|
CHECK (permit IN ('', 'sponsors', 'no_sponsorship'));
|
|
ALTER TABLE roles ADD COLUMN languages TEXT NOT NULL DEFAULT '[]';
|
|
ALTER TABLE roles ADD COLUMN extract_tries INTEGER NOT NULL DEFAULT 0;
|
|
"""),
|
|
(3, "vectors, and the agent's cached intent vector, for score", """
|
|
CREATE TABLE vectors (
|
|
uid TEXT PRIMARY KEY REFERENCES roles(uid) ON DELETE CASCADE,
|
|
model TEXT NOT NULL,
|
|
dim INTEGER NOT NULL,
|
|
vec BLOB NOT NULL,
|
|
embedded_at TEXT NOT NULL
|
|
);
|
|
ALTER TABLE agents ADD COLUMN intent_key TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE agents ADD COLUMN intent_vec BLOB;
|
|
"""),
|
|
(4, "roles.vote, roles.summary and roles.article_text, for the phone UI", """
|
|
ALTER TABLE roles ADD COLUMN vote INTEGER CHECK (vote IN (-1, 0, 1));
|
|
ALTER TABLE roles ADD COLUMN voted_at TEXT;
|
|
ALTER TABLE roles ADD COLUMN summary TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE roles ADD COLUMN summarised_at TEXT;
|
|
ALTER TABLE roles ADD COLUMN article_text TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE roles ADD COLUMN article_at TEXT;
|
|
"""),
|
|
(5, "roles.read_at and role_tags, for the feed's read state and tags", """
|
|
ALTER TABLE roles ADD COLUMN read_at TEXT;
|
|
CREATE TABLE role_tags (
|
|
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
|
tag TEXT NOT NULL,
|
|
PRIMARY KEY (uid, tag)
|
|
);
|
|
"""),
|
|
# Votes and affinities move from the role and company rows to one row per
|
|
# agent. What was global goes to the default agent, the enabled one with
|
|
# the lowest id (`Store.default_agent`); a file with something to move and
|
|
# no agent gets one first, so nothing is lost. The columns are then
|
|
# dropped, which SQLite has allowed since 3.35.
|
|
(6, "agent_companies and role_votes: affinity and vote become per agent", """
|
|
CREATE TABLE agent_companies (
|
|
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
|
company_key TEXT NOT NULL,
|
|
affinity INTEGER NOT NULL CHECK (affinity BETWEEN -1 AND 2),
|
|
updated_at TEXT NOT NULL,
|
|
PRIMARY KEY (agent_id, company_key)
|
|
);
|
|
CREATE TABLE role_votes (
|
|
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
|
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
|
vote INTEGER NOT NULL CHECK (vote IN (-1, 0, 1)),
|
|
voted_at TEXT NOT NULL,
|
|
PRIMARY KEY (agent_id, uid)
|
|
);
|
|
INSERT INTO agents (name, intent, gates, weights, enabled, created_at)
|
|
SELECT 'My ranking', '', '[]', '{}', 1, strftime('%Y-%m-%dT%H:%M:%S+00:00', 'now')
|
|
WHERE NOT EXISTS (SELECT 1 FROM agents)
|
|
AND (EXISTS (SELECT 1 FROM roles WHERE vote IS NOT NULL)
|
|
OR EXISTS (SELECT 1 FROM companies WHERE affinity <> 0));
|
|
INSERT INTO role_votes (agent_id, uid, vote, voted_at)
|
|
SELECT (SELECT id FROM agents ORDER BY enabled DESC, id LIMIT 1), uid, vote,
|
|
COALESCE(voted_at, strftime('%Y-%m-%dT%H:%M:%S+00:00', 'now'))
|
|
FROM roles WHERE vote IS NOT NULL;
|
|
INSERT INTO agent_companies (agent_id, company_key, affinity, updated_at)
|
|
SELECT (SELECT id FROM agents ORDER BY enabled DESC, id LIMIT 1), key, affinity, updated_at
|
|
FROM companies WHERE affinity <> 0;
|
|
ALTER TABLE roles DROP COLUMN vote;
|
|
ALTER TABLE roles DROP COLUMN voted_at;
|
|
ALTER TABLE companies DROP COLUMN affinity;
|
|
"""),
|
|
(7, "companies.ats, country and city: a company the app cannot read yet is still a row", """
|
|
ALTER TABLE companies ADD COLUMN ats TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE companies ADD COLUMN country TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE companies ADD COLUMN city TEXT NOT NULL DEFAULT '';
|
|
"""),
|
|
(8, "agents.limits: how deep into its ranking the model stages read is each agent's own", """
|
|
ALTER TABLE agents ADD COLUMN limits TEXT NOT NULL DEFAULT '{}';
|
|
"""),
|
|
(9, "companies.ats_slug: the slug of a host the app cannot read yet is kept", """
|
|
ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT '';
|
|
"""),
|
|
(10, "roles.posted_at: recency and the newest feed count from when a role "
|
|
"was posted, not just when the app found it", _migrate_posted_at),
|
|
# A Python step, not SQL: classify() lives in plugins/ats.py, and running
|
|
# its current rules is the whole point (see _recompute_function below).
|
|
(11, "roles.function recomputed for every not-yet-extracted role, word-boundary "
|
|
"classifier fix", lambda conn: _recompute_function(conn)),
|
|
)
|
|
|
|
#: What a file created from `SCHEMA` today is already at.
|
|
LATEST = MIGRATIONS[-1][0]
|
|
|
|
|
|
def _recompute_function(conn: sqlite3.Connection) -> None:
|
|
"""Migration 11's step. `roles.function` used to be filled by a keyword
|
|
classifier that matched substrings (" it" inside "Italy"), never
|
|
returned '' (only "other", outside the declared vocabulary) and checked
|
|
department before title; `plugins.ats.classify` no longer does any of
|
|
that (see its own docstring). A role `extract` has already read keeps
|
|
whatever it decided — `function` is extract's from then on, see
|
|
`Store.set_extracted` — so only roles with no `extracted_at` are
|
|
recomputed here, from their stored title and department (department
|
|
lives in `raw`, not its own column: docs/decisions.md, "raw is
|
|
transport, columns are storage").
|
|
|
|
A Python step because the classifier is Python, not SQL; `plugins` is a
|
|
top-level package outside `app`'s own tree, so it is reached the same
|
|
way `app/jobs/extract.py` reaches `FUNCTIONS`: inserted onto `sys.path`
|
|
rather than imported as an installed dependency, since `plugins` is not
|
|
in `pyproject.toml`'s packages."""
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
root = Path(__file__).resolve().parents[2]
|
|
if str(root) not in sys.path:
|
|
sys.path.insert(0, str(root))
|
|
from plugins.ats import classify
|
|
|
|
rows = conn.execute(
|
|
"SELECT uid, title, raw, function FROM roles WHERE extracted_at IS NULL").fetchall()
|
|
updates = []
|
|
for uid, title, raw, old in rows:
|
|
try:
|
|
department = json.loads(raw or "{}").get("department") or ""
|
|
except (TypeError, ValueError):
|
|
department = ""
|
|
new = classify(department, title)
|
|
if new != old:
|
|
updates.append((new, uid))
|
|
if updates:
|
|
conn.executemany("UPDATE roles SET function = ? WHERE uid = ?", updates)
|
|
log.info("migration 11: recomputed function for %d of %d not-yet-extracted roles",
|
|
len(updates), len(rows))
|
|
|
|
|
|
def user_version(conn: sqlite3.Connection) -> int:
|
|
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
|
|
|
|
|
def _stamp(conn: sqlite3.Connection, version: int) -> None:
|
|
# PRAGMA takes no bound parameters; the value is an int from MIGRATIONS.
|
|
conn.execute(f"PRAGMA user_version = {int(version)}")
|
|
|
|
|
|
def _statements(script: str) -> list[str]:
|
|
"""A script split into statements, cut where `sqlite3.complete_statement`
|
|
says one ends, which knows about comments and trigger bodies. Not
|
|
`executescript`, which commits before it starts and autocommits each
|
|
statement, so a failing migration would be half applied."""
|
|
statements, buffer = [], ""
|
|
for line in script.splitlines(keepends=True):
|
|
buffer += line
|
|
if sqlite3.complete_statement(buffer):
|
|
statements.append(buffer.strip())
|
|
buffer = ""
|
|
if buffer.strip():
|
|
statements.append(buffer.strip())
|
|
return statements
|
|
|
|
|
|
def migrate(conn: sqlite3.Connection) -> list[int]:
|
|
"""Bring an existing database up to `LATEST`. Returns what it applied."""
|
|
current = user_version(conn)
|
|
if current > LATEST:
|
|
raise RuntimeError(
|
|
f"this database is at schema v{current}, but this roledesk only knows "
|
|
f"up to v{LATEST}. Upgrade roledesk rather than let older code write to it.")
|
|
applied = []
|
|
for version, description, step in MIGRATIONS:
|
|
if version <= current:
|
|
continue
|
|
log.info("migrating database to v%d: %s", version, description)
|
|
_apply(conn, version, step)
|
|
applied.append(version)
|
|
return applied
|
|
|
|
|
|
def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
|
|
"""One migration and its version stamp, in one explicit transaction.
|
|
|
|
Explicit because sqlite3 only opens a transaction implicitly before DML: an
|
|
`ALTER` would otherwise be durable at once while the stamp stayed behind,
|
|
and the next open would replay it into a "duplicate column" forever.
|
|
"""
|
|
started = not conn.in_transaction
|
|
if started:
|
|
conn.execute("BEGIN")
|
|
try:
|
|
if callable(step):
|
|
step(conn)
|
|
else:
|
|
for statement in _statements(step):
|
|
conn.execute(statement)
|
|
_stamp(conn, version)
|
|
except Exception:
|
|
if started:
|
|
conn.execute("ROLLBACK")
|
|
raise
|
|
if started:
|
|
conn.execute("COMMIT")
|
|
|
|
|
|
def initialise(conn: sqlite3.Connection) -> list[int]:
|
|
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
|
|
fresh = conn.execute(
|
|
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (MARKER,)
|
|
).fetchone() is None
|
|
applied: list[int] = []
|
|
if fresh:
|
|
# Atomic like a migration: a file half created would otherwise carry
|
|
# the marker table and be taken for one to migrate.
|
|
_apply(conn, LATEST, SCHEMA)
|
|
else:
|
|
applied = migrate(conn)
|
|
for statement in _statements(INDEXES):
|
|
conn.execute(statement)
|
|
return applied
|