"""The tables, and how a database file gets from an old shape to this one. `SCHEMA` is the current shape. A new file is created from it and stamped with `LATEST`, so it never runs a migration. `MIGRATIONS` is how an existing file catches up: ordered, numbered, applied once each, tracked in SQLite's own `PRAGMA user_version`. Adding a column means editing both: the DDL in `SCHEMA` for new files, and an appended migration for old ones. Never renumber or edit a migration that has shipped. """ from __future__ import annotations import json import logging import sqlite3 from collections.abc import Callable from app.models import posted_at, posted_host log = logging.getLogger(__name__) SCHEMA = """ -- One posting on one board. `uid` is sha1(source, source_id), the identity desk -- gives every Item. The gate fields are real columns, unpacked from the item's -- raw on upsert or filled later by extract: raw is transport, the columns are -- what gets queried. CREATE TABLE IF NOT EXISTS roles ( uid TEXT PRIMARY KEY, source TEXT NOT NULL, source_id TEXT NOT NULL, subscription_id INTEGER REFERENCES subscriptions(id) ON DELETE SET NULL, -- The cluster key is (company_key, title_key, place_key). No foreign key -- to companies: a role may arrive before anyone has made a company row. company_key TEXT NOT NULL DEFAULT '', company TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '', title_key TEXT NOT NULL DEFAULT '', -- Geography. place and not location, which is where an Item's original -- lives. remote and country are separate so a gate on remote can also -- require the country: "UK | Remote" is not remote from Berlin. place TEXT NOT NULL DEFAULT '', place_key TEXT NOT NULL DEFAULT '', country TEXT NOT NULL DEFAULT '', remote TEXT NOT NULL DEFAULT '' CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')), salary_min INTEGER, salary_max INTEGER, currency TEXT NOT NULL DEFAULT '', seniority TEXT NOT NULL DEFAULT '', function TEXT NOT NULL DEFAULT '', -- Whether the posting says it sponsors a work permit, and the languages -- (ISO 639-1, a JSON array) it says are required. Only extract fills these. permit TEXT NOT NULL DEFAULT '' CHECK (permit IN ('', 'sponsors', 'no_sponsorship')), languages TEXT NOT NULL DEFAULT '[]', description TEXT NOT NULL DEFAULT '', -- the posting's body text feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over original_url TEXT NOT NULL DEFAULT '', -- the posting page published_at TEXT NOT NULL, -- what the source claims, UTC -- What storing postings buys. first_seen is when the app itself first -- stored the role; posted_at (app.models.posted_at) is the earlier of -- first_seen and published_at, trusted only for a host known to report a -- creation date rather than one that moves on an edit -- (app.models.TRUSTED_POSTED_HOSTS) — recency and the feed's newest order -- read posted_at, first_seen stays "when the app found it". closed_at is -- set when a board fetched in full no longer lists the role, and cleared -- if it returns. posted_at TEXT NOT NULL DEFAULT '', first_seen TEXT NOT NULL, last_seen TEXT NOT NULL, closed_at TEXT, raw TEXT NOT NULL DEFAULT '{}', extracted_at TEXT, -- when extract last filled columns -- Replies from extract that left this role unanswered. At GIVE_UP_AFTER it -- is no longer offered. A column, not raw, because every upsert rewrites raw. extract_tries INTEGER NOT NULL DEFAULT 0, -- NULL unread, else when the feed first opened it or judged it: a vote -- stamps this too, see Store.set_vote, because voting is reading. read_at TEXT, -- About 60 words from summarise, and when it wrote them. summary TEXT NOT NULL DEFAULT '', summarised_at TEXT, -- The posting's own page as plain text, fetched on demand for a role -- whose description is only a teaser. article_text TEXT NOT NULL DEFAULT '', article_at TEXT, created_at TEXT NOT NULL ); -- The open vocabulary summarise reads off a posting: the role family, the -- technologies and skills it names, and the domain. What the feed's filter -- row is built from; see Store.set_tags and Store.tag_counts. CREATE TABLE IF NOT EXISTS role_tags ( uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, tag TEXT NOT NULL, PRIMARY KEY (uid, tag) ); -- A company as a target, whether or not any board of it is read. key is -- models.company_key(name), the slug roles.company_key holds. -- ats is the host the careers site runs on, readable by the ats plugin or -- not (workday, successfactors, ...): what discovery found, '' unknown. A -- company with an ats and no board subscription is one the app cannot read -- yet. country is an ISO code and city the head office's. ats_slug is the -- name that host knows the company by (the acme of acme.teamtailor.com), -- '' where its URL carries none, kept for the parser written later. CREATE TABLE IF NOT EXISTS companies ( key TEXT PRIMARY KEY, name TEXT NOT NULL, domain TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'researching', careers_url TEXT NOT NULL DEFAULT '', notes TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL, updated_at TEXT NOT NULL, ats TEXT NOT NULL DEFAULT '', country TEXT NOT NULL DEFAULT '', city TEXT NOT NULL DEFAULT '', ats_slug TEXT NOT NULL DEFAULT '' ); -- The targets desk's runner hands to plugins. newsdesk's shape, with its two -- labels named company and board from the first migration. CREATE TABLE IF NOT EXISTS subscriptions ( id INTEGER PRIMARY KEY, source TEXT NOT NULL, -- plugin name target TEXT NOT NULL, -- e.g. greenhouse:acme label TEXT NOT NULL, -- what Item.subscription names company TEXT NOT NULL DEFAULT '', board TEXT NOT NULL DEFAULT '', params TEXT NOT NULL DEFAULT '{}', -- what a query target asks with subscribed INTEGER NOT NULL DEFAULT 1, fetch INTEGER NOT NULL DEFAULT 1, show INTEGER NOT NULL DEFAULT 1, added_at TEXT NOT NULL, target_id TEXT NOT NULL DEFAULT '', -- the plugin's durable name for it UNIQUE (source, target), UNIQUE (source, label) ); -- The tracker, in jobhunt-hq's shape (MIT). role_uid is NULL for a role that -- was never posted, so an unsolicited approach is tracked like any other. CREATE TABLE IF NOT EXISTS applications ( id TEXT PRIMARY KEY, role_uid TEXT REFERENCES roles(uid) ON DELETE SET NULL, company_key TEXT NOT NULL DEFAULT '', role_title TEXT NOT NULL DEFAULT '', url TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'saved' CHECK (status IN ('saved', 'applied', 'screen', 'technical', 'onsite', 'offer', 'rejected', 'withdrawn')), source TEXT NOT NULL DEFAULT '', applied_at TEXT, next_action TEXT NOT NULL DEFAULT '', next_date TEXT, contact TEXT NOT NULL DEFAULT '', notes TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL, updated_at TEXT NOT NULL ); -- One row per status an application entered, written by the store and never -- updated: what days-to-first-screen and conversion by source are read from. CREATE TABLE IF NOT EXISTS application_events ( id INTEGER PRIMARY KEY, application_id TEXT NOT NULL REFERENCES applications(id) ON DELETE CASCADE, status TEXT NOT NULL, at TEXT NOT NULL ); CREATE TRIGGER IF NOT EXISTS application_events_append_only BEFORE UPDATE ON application_events BEGIN SELECT RAISE(ABORT, 'application_events is append-only'); END; -- A search agent: a named saved search with its own intent text, gates and -- weights. gates is a list of {field, values, mode: require|prefer, weight}. -- intent_vec caches the intent's vector; intent_key is sha1 of the embedding -- model and the intent text, so editing either makes it stale. limits is how -- deep into this agent's ranking extract and summarise read, as JSON: a key -- it lacks means the `Limits` default. CREATE TABLE IF NOT EXISTS agents ( id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, intent TEXT NOT NULL DEFAULT '', gates TEXT NOT NULL DEFAULT '[]', weights TEXT NOT NULL DEFAULT '{}', enabled INTEGER NOT NULL DEFAULT 1, created_at TEXT NOT NULL, intent_key TEXT NOT NULL DEFAULT '', intent_vec BLOB, limits TEXT NOT NULL DEFAULT '{}' ); -- One agent's stance on a company: -1 hidden, 0 neutral, 1 liked, 2 -- favourite. No row is 0, so the table holds only what somebody set. Per -- agent, because a company that is noise for one search is the target of -- another. CREATE TABLE IF NOT EXISTS agent_companies ( agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE, company_key TEXT NOT NULL, affinity INTEGER NOT NULL CHECK (affinity BETWEEN -1 AND 2), updated_at TEXT NOT NULL, PRIMARY KEY (agent_id, company_key) ); -- One agent's judgement of a role: -1 pass, 0 neutral, 1 like. No row is -- unjudged. A vote moves the rank at read time, never hides a role, and its -- vectors shape the agent's taste direction in `score`. CREATE TABLE IF NOT EXISTS role_votes ( agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE, uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, vote INTEGER NOT NULL CHECK (vote IN (-1, 0, 1)), voted_at TEXT NOT NULL, PRIMARY KEY (agent_id, uid) ); -- A role is scored per agent. A gated role keeps its row, with the reason. CREATE TABLE IF NOT EXISTS scores ( uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE, score REAL NOT NULL DEFAULT 0, gated_reason TEXT NOT NULL DEFAULT '', scored_at TEXT NOT NULL, PRIMARY KEY (uid, agent_id) ); -- One vector per role, little-endian float32. Only comparable to another from -- the same model, so every read is scoped by it. CREATE TABLE IF NOT EXISTS vectors ( uid TEXT PRIMARY KEY REFERENCES roles(uid) ON DELETE CASCADE, model TEXT NOT NULL, dim INTEGER NOT NULL, vec BLOB NOT NULL, embedded_at TEXT NOT NULL ); -- The string map a desk plugin is handed as self.state. One namespace shared -- by every plugin, so a plugin prefixes its keys. CREATE TABLE IF NOT EXISTS state ( key TEXT PRIMARY KEY, value TEXT NOT NULL ); """ #: Applied after `SCHEMA` or the migrations, every open. An index may name a #: column only a migration adds, which is why these are kept apart. INDEXES = """ CREATE INDEX IF NOT EXISTS idx_roles_company ON roles(company_key); CREATE INDEX IF NOT EXISTS idx_roles_place ON roles(place_key); CREATE INDEX IF NOT EXISTS idx_roles_first_seen ON roles(first_seen DESC); CREATE INDEX IF NOT EXISTS idx_roles_closed ON roles(closed_at); -- Serves the newest feed (open_only, ordered by posted_at DESC), which is -- most of what `Store.roles` runs: partial, since only an open role's -- position in that order is ever asked for. CREATE INDEX IF NOT EXISTS idx_roles_posted ON roles(posted_at DESC, uid) WHERE closed_at IS NULL; CREATE INDEX IF NOT EXISTS idx_roles_cluster ON roles(company_key, title_key, place_key); CREATE INDEX IF NOT EXISTS idx_roles_board ON roles(source, subscription_id); CREATE INDEX IF NOT EXISTS idx_application_events ON application_events(application_id, at); CREATE INDEX IF NOT EXISTS idx_scores_agent ON scores(agent_id, score DESC); CREATE INDEX IF NOT EXISTS idx_role_votes_agent ON role_votes(agent_id, vote); CREATE INDEX IF NOT EXISTS idx_vectors_model ON vectors(model); CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag); """ #: The table whose absence marks a file that has never been a roledesk #: database, and is therefore created from `SCHEMA` rather than migrated. MARKER = "roles" Step = str | Callable[[sqlite3.Connection], None] def _migrate_posted_at(conn: sqlite3.Connection) -> None: """v10's step: add `posted_at`, then backfill it for every row already stored, by the same rule `Store.upsert_role` applies to a new one (see `app.models.posted_at`). A callable, not SQL, because the rule reads `raw` for which host a role came from. `'' NOT NULL` is a placeholder only until the UPDATE below runs, in the same transaction as the ALTER (`_apply` wraps every migration in one); every row gets a real value.""" conn.execute("ALTER TABLE roles ADD COLUMN posted_at TEXT NOT NULL DEFAULT ''") rows = conn.execute("SELECT uid, published_at, first_seen, raw FROM roles").fetchall() conn.executemany( "UPDATE roles SET posted_at = ? WHERE uid = ?", [(posted_at(row["published_at"], row["first_seen"], posted_host(json.loads(row["raw"] or "{}"))), row["uid"]) for row in rows]) MIGRATIONS: tuple[tuple[int, str, Step], ...] = ( # Never run as a migration: a file without the marker table is created # from `SCHEMA` and stamped. It exists so the first real migration is 2. (1, "roles, companies, subscriptions, applications, agents, scores, state", ""), (2, "roles.permit, roles.languages and roles.extract_tries, for extract", """ ALTER TABLE roles ADD COLUMN permit TEXT NOT NULL DEFAULT '' CHECK (permit IN ('', 'sponsors', 'no_sponsorship')); ALTER TABLE roles ADD COLUMN languages TEXT NOT NULL DEFAULT '[]'; ALTER TABLE roles ADD COLUMN extract_tries INTEGER NOT NULL DEFAULT 0; """), (3, "vectors, and the agent's cached intent vector, for score", """ CREATE TABLE vectors ( uid TEXT PRIMARY KEY REFERENCES roles(uid) ON DELETE CASCADE, model TEXT NOT NULL, dim INTEGER NOT NULL, vec BLOB NOT NULL, embedded_at TEXT NOT NULL ); ALTER TABLE agents ADD COLUMN intent_key TEXT NOT NULL DEFAULT ''; ALTER TABLE agents ADD COLUMN intent_vec BLOB; """), (4, "roles.vote, roles.summary and roles.article_text, for the phone UI", """ ALTER TABLE roles ADD COLUMN vote INTEGER CHECK (vote IN (-1, 0, 1)); ALTER TABLE roles ADD COLUMN voted_at TEXT; ALTER TABLE roles ADD COLUMN summary TEXT NOT NULL DEFAULT ''; ALTER TABLE roles ADD COLUMN summarised_at TEXT; ALTER TABLE roles ADD COLUMN article_text TEXT NOT NULL DEFAULT ''; ALTER TABLE roles ADD COLUMN article_at TEXT; """), (5, "roles.read_at and role_tags, for the feed's read state and tags", """ ALTER TABLE roles ADD COLUMN read_at TEXT; CREATE TABLE role_tags ( uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, tag TEXT NOT NULL, PRIMARY KEY (uid, tag) ); """), # Votes and affinities move from the role and company rows to one row per # agent. What was global goes to the default agent, the enabled one with # the lowest id (`Store.default_agent`); a file with something to move and # no agent gets one first, so nothing is lost. The columns are then # dropped, which SQLite has allowed since 3.35. (6, "agent_companies and role_votes: affinity and vote become per agent", """ CREATE TABLE agent_companies ( agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE, company_key TEXT NOT NULL, affinity INTEGER NOT NULL CHECK (affinity BETWEEN -1 AND 2), updated_at TEXT NOT NULL, PRIMARY KEY (agent_id, company_key) ); CREATE TABLE role_votes ( agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE, uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, vote INTEGER NOT NULL CHECK (vote IN (-1, 0, 1)), voted_at TEXT NOT NULL, PRIMARY KEY (agent_id, uid) ); INSERT INTO agents (name, intent, gates, weights, enabled, created_at) SELECT 'My ranking', '', '[]', '{}', 1, strftime('%Y-%m-%dT%H:%M:%S+00:00', 'now') WHERE NOT EXISTS (SELECT 1 FROM agents) AND (EXISTS (SELECT 1 FROM roles WHERE vote IS NOT NULL) OR EXISTS (SELECT 1 FROM companies WHERE affinity <> 0)); INSERT INTO role_votes (agent_id, uid, vote, voted_at) SELECT (SELECT id FROM agents ORDER BY enabled DESC, id LIMIT 1), uid, vote, COALESCE(voted_at, strftime('%Y-%m-%dT%H:%M:%S+00:00', 'now')) FROM roles WHERE vote IS NOT NULL; INSERT INTO agent_companies (agent_id, company_key, affinity, updated_at) SELECT (SELECT id FROM agents ORDER BY enabled DESC, id LIMIT 1), key, affinity, updated_at FROM companies WHERE affinity <> 0; ALTER TABLE roles DROP COLUMN vote; ALTER TABLE roles DROP COLUMN voted_at; ALTER TABLE companies DROP COLUMN affinity; """), (7, "companies.ats, country and city: a company the app cannot read yet is still a row", """ ALTER TABLE companies ADD COLUMN ats TEXT NOT NULL DEFAULT ''; ALTER TABLE companies ADD COLUMN country TEXT NOT NULL DEFAULT ''; ALTER TABLE companies ADD COLUMN city TEXT NOT NULL DEFAULT ''; """), (8, "agents.limits: how deep into its ranking the model stages read is each agent's own", """ ALTER TABLE agents ADD COLUMN limits TEXT NOT NULL DEFAULT '{}'; """), (9, "companies.ats_slug: the slug of a host the app cannot read yet is kept", """ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT ''; """), (10, "roles.posted_at: recency and the newest feed count from when a role " "was posted, not just when the app found it", _migrate_posted_at), ) #: What a file created from `SCHEMA` today is already at. LATEST = MIGRATIONS[-1][0] def user_version(conn: sqlite3.Connection) -> int: return int(conn.execute("PRAGMA user_version").fetchone()[0]) def _stamp(conn: sqlite3.Connection, version: int) -> None: # PRAGMA takes no bound parameters; the value is an int from MIGRATIONS. conn.execute(f"PRAGMA user_version = {int(version)}") def _statements(script: str) -> list[str]: """A script split into statements, cut where `sqlite3.complete_statement` says one ends, which knows about comments and trigger bodies. Not `executescript`, which commits before it starts and autocommits each statement, so a failing migration would be half applied.""" statements, buffer = [], "" for line in script.splitlines(keepends=True): buffer += line if sqlite3.complete_statement(buffer): statements.append(buffer.strip()) buffer = "" if buffer.strip(): statements.append(buffer.strip()) return statements def migrate(conn: sqlite3.Connection) -> list[int]: """Bring an existing database up to `LATEST`. Returns what it applied.""" current = user_version(conn) if current > LATEST: raise RuntimeError( f"this database is at schema v{current}, but this roledesk only knows " f"up to v{LATEST}. Upgrade roledesk rather than let older code write to it.") applied = [] for version, description, step in MIGRATIONS: if version <= current: continue log.info("migrating database to v%d: %s", version, description) _apply(conn, version, step) applied.append(version) return applied def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None: """One migration and its version stamp, in one explicit transaction. Explicit because sqlite3 only opens a transaction implicitly before DML: an `ALTER` would otherwise be durable at once while the stamp stayed behind, and the next open would replay it into a "duplicate column" forever. """ started = not conn.in_transaction if started: conn.execute("BEGIN") try: if callable(step): step(conn) else: for statement in _statements(step): conn.execute(statement) _stamp(conn, version) except Exception: if started: conn.execute("ROLLBACK") raise if started: conn.execute("COMMIT") def initialise(conn: sqlite3.Connection) -> list[int]: """Create or upgrade, whichever this file needs. Returns migrations applied.""" fresh = conn.execute( "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (MARKER,) ).fetchone() is None applied: list[int] = [] if fresh: # Atomic like a migration: a file half created would otherwise carry # the marker table and be taken for one to migrate. _apply(conn, LATEST, SCHEMA) else: applied = migrate(conn) for statement in _statements(INDEXES): conn.execute(statement) return applied