"""The tables, and how a database file gets from an old shape to this one. `SCHEMA` is the current shape. A new file is created from it and stamped with `LATEST`, so it never runs a migration. `MIGRATIONS` is how an existing file catches up: ordered, numbered, applied once each, tracked in SQLite's own `PRAGMA user_version`. Adding a column means editing both: the DDL in `SCHEMA` for new files, and an appended migration for old ones. Never renumber or edit a migration that has shipped. """ from __future__ import annotations import json import logging import sqlite3 from collections.abc import Callable from app import places from app.models import ( SENIORITIES, company_fold, company_key, country_of, host_word, posted_at, posted_host, ) from app.seniority import from_title from app.storage import gazetteer log = logging.getLogger(__name__) SCHEMA = """ -- One posting on one board. `uid` is sha1(source, source_id), the identity desk -- gives every Item. The gate fields are real columns, unpacked from the item's -- raw on upsert or filled later by extract: raw is transport, the columns are -- what gets queried. CREATE TABLE IF NOT EXISTS roles ( uid TEXT PRIMARY KEY, source TEXT NOT NULL, source_id TEXT NOT NULL, subscription_id INTEGER REFERENCES subscriptions(id) ON DELETE SET NULL, -- The cluster key is (company_key, title_key, place_key). No foreign key -- to companies: a role may arrive before anyone has made a company row. company_key TEXT NOT NULL DEFAULT '', company TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '', title_key TEXT NOT NULL DEFAULT '', -- Geography. place and not location, which is where an Item's original -- lives. remote and country are separate so a gate on remote can also -- require the country: "UK | Remote" is not remote from Berlin. place TEXT NOT NULL DEFAULT '', place_key TEXT NOT NULL DEFAULT '', -- places.fold(place): what a place: token the gazetteer does not know is -- matched against as whole words, through place_match (app/places.py). -- The filter and the gates otherwise read role_places. place_fold TEXT NOT NULL DEFAULT '', country TEXT NOT NULL DEFAULT '', remote TEXT NOT NULL DEFAULT '' CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')), salary_min INTEGER, salary_max INTEGER, currency TEXT NOT NULL DEFAULT '', -- The level shown and filtered on: the first filled of seniority_model, -- seniority_stated and seniority_title (the last columns), written with them. seniority TEXT NOT NULL DEFAULT '', function TEXT NOT NULL DEFAULT '', -- Whether the posting says it sponsors a work permit, and the languages -- (ISO 639-1, a JSON array) it says are required. Only extract fills these. permit TEXT NOT NULL DEFAULT '' CHECK (permit IN ('', 'sponsors', 'no_sponsorship')), languages TEXT NOT NULL DEFAULT '[]', description TEXT NOT NULL DEFAULT '', -- the posting's body text feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over original_url TEXT NOT NULL DEFAULT '', -- the posting page published_at TEXT NOT NULL, -- what the source claims, UTC -- What storing postings buys. first_seen is when the app itself first -- stored the role; posted_at (app.models.posted_at) is the earlier of -- first_seen and published_at, trusted only for a host known to report a -- creation date rather than one that moves on an edit -- (app.models.trusted_posted) — recency and the feed's newest order -- read posted_at, first_seen stays "when the app found it". closed_at is -- set when a board fetched in full no longer lists the role, and cleared -- if it returns. posted_at TEXT NOT NULL DEFAULT '', first_seen TEXT NOT NULL, last_seen TEXT NOT NULL, closed_at TEXT, raw TEXT NOT NULL DEFAULT '{}', extracted_at TEXT, -- when extract last filled columns -- Replies from extract that left this role unanswered. At GIVE_UP_AFTER it -- is no longer offered. A column, not raw, because every upsert rewrites raw. extract_tries INTEGER NOT NULL DEFAULT 0, -- NULL unread, else when the feed first opened it or judged it: a vote -- stamps this too, see Store.set_vote, because voting is reading. read_at TEXT, -- About 60 words from summarise, and when it wrote them. summary TEXT NOT NULL DEFAULT '', summarised_at TEXT, -- The posting's own page as plain text, fetched on demand for a role -- whose description is only a teaser. article_text TEXT NOT NULL DEFAULT '', article_at TEXT, -- The uid of the kept role when this open role is a copy of one posting -- (app/dupes.py), NULL on the kept role, on a role without copies and on -- every closed role. Written only by Store.rebuild_dupes, at the end of -- every pull; the feed, the counts and the model stages skip a copy. -- Not the last column: SQLite cannot drop a last column that a comment -- precedes, which the migration tests do. dup_of TEXT, created_at TEXT NOT NULL, salary_text TEXT NOT NULL DEFAULT '', teaser TEXT NOT NULL DEFAULT '', seniority_model TEXT NOT NULL DEFAULT '', seniority_stated TEXT NOT NULL DEFAULT '', seniority_title TEXT NOT NULL DEFAULT '', seniority_host TEXT NOT NULL DEFAULT '', employment_host TEXT NOT NULL DEFAULT '', remote_host TEXT NOT NULL DEFAULT '', country_host TEXT NOT NULL DEFAULT '', employment TEXT NOT NULL DEFAULT '', prep_due INTEGER NOT NULL DEFAULT 0 ); -- The *_host columns hold the host's own words as a reader handed them on -- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps -- them into seniority_stated, employment, remote and country. prep_due is above 0 -- on a role whose words, company, title or places changed since prep last -- mapped it (Store.upsert_role), which is what the next prep reads. -- roles.teaser is the short text a list carries, kept until the full text -- arrives and embedded while there is none; a later empty teaser does not -- wipe it. seniority_model, seniority_stated and seniority_title are the -- three levels behind roles.seniority: the model's (extract), the host's own -- word mapped to app.seniority's vocabulary, and the title's. No comment sits -- among the last columns: SQLite cannot drop a column a comment precedes, -- which the migration tests do. -- The gazetteer: a place is a continent, a country (key the ISO code), a -- region or a city, in the place parent_id names. key is unique per kind: -- a region's key (app.places.REGIONS), a town's group. Seeded from the -- dictionaries in app/models.py and app/places.py (app.places.seed). CREATE TABLE IF NOT EXISTS places ( id INTEGER PRIMARY KEY, kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')), key TEXT NOT NULL, name TEXT NOT NULL, parent_id INTEGER REFERENCES places(id), UNIQUE (kind, key) ); -- Every folded spelling of a place (app.places.fold): kobenhavn, kopenhamn -- and copenhagen are Copenhagen. One fold may name several places -- (Luxembourg the town and the country). CREATE TABLE IF NOT EXISTS place_aliases ( fold TEXT NOT NULL, place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE, UNIQUE (fold, place_id) ); -- What a place holds beyond its children: Øresund holds the Capital Region -- of Denmark and Skåne, which lie in two countries. CREATE TABLE IF NOT EXISTS place_includes ( place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE, part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE, PRIMARY KEY (place_id, part_id) ); -- One row per place of a role (app/storage/gazetteer.py): the host's text in -- the host's order (position), its fold, and the place it resolves to, NULL -- while the gazetteer does not know it. A text naming several places is a -- row for each. url is the link of a copy folded into this role, '' for -- the role's own places. CREATE TABLE IF NOT EXISTS role_places ( uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, position INTEGER NOT NULL, text TEXT NOT NULL, fold TEXT NOT NULL, place_id INTEGER REFERENCES places(id), url TEXT NOT NULL DEFAULT '' ); -- The open vocabulary summarise reads off a posting: the role family, the -- technologies and skills it names, and the domain. What the feed's filter -- row is built from; see Store.set_tags and Store.tag_counts. CREATE TABLE IF NOT EXISTS role_tags ( uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, tag TEXT NOT NULL, PRIMARY KEY (uid, tag) ); -- A company as a target, whether or not any board of it is read. key is -- models.company_key(name), the slug roles.company_key holds. -- ats is the host the careers site runs on, readable by the ats plugin or -- not (workday, successfactors, ...): what discovery found, '' unknown. A -- company with an ats and no board subscription is one the app cannot read -- yet. country is an ISO code and city the head office's. ats_slug is the -- name that host knows the company by (the acme of acme.teamtailor.com), -- '' where its URL carries none, kept for the parser written later. -- watch is the Watchlist's rating, global and not an agent's: NULL unrated, 1 -- worth checking by hand, 0 not; careers_opened_at when its careers page was -- last opened from there, which is also when a person last looked at it. -- hand_reason is why a person reads the careers page (models.HAND_REASONS), -- '' when nobody has to. all_roles 1 reads every board of the company with -- the agents' wide reach, Europe today (app/jobs/reach.py); global, since -- fetching is shared. CREATE TABLE IF NOT EXISTS companies ( key TEXT PRIMARY KEY, name TEXT NOT NULL, domain TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'researching', careers_url TEXT NOT NULL DEFAULT '', notes TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL, updated_at TEXT NOT NULL, ats TEXT NOT NULL DEFAULT '', country TEXT NOT NULL DEFAULT '', city TEXT NOT NULL DEFAULT '', ats_slug TEXT NOT NULL DEFAULT '', watch INTEGER CHECK (watch IN (0, 1)), careers_opened_at TEXT NOT NULL DEFAULT '', hand_reason TEXT NOT NULL DEFAULT '', sector TEXT NOT NULL DEFAULT '', stage TEXT NOT NULL DEFAULT '', size_band TEXT NOT NULL DEFAULT '', founded INTEGER, last_funding INTEGER, last_funding_currency TEXT NOT NULL DEFAULT '', facts_source TEXT NOT NULL DEFAULT '', facts_at TEXT NOT NULL DEFAULT '', all_roles INTEGER NOT NULL DEFAULT 0 ); -- Every folded spelling of a company (models.company_fold): orsted is Ørsted -- whichever way a board writes it, and a person can add another name -- ("Dong Energy"). One fold may name several companies until a person merges -- them (Store.merge_companies). The enrichment columns above are empty until -- a reader states them; facts_source and facts_at say where and when. CREATE TABLE IF NOT EXISTS company_aliases ( fold TEXT NOT NULL, company_key TEXT NOT NULL, UNIQUE (fold, company_key) ); CREATE INDEX IF NOT EXISTS idx_company_aliases_key ON company_aliases(company_key); -- The targets desk's runner hands to plugins. newsdesk's shape, with its two -- labels named company and board from the first migration. CREATE TABLE IF NOT EXISTS subscriptions ( id INTEGER PRIMARY KEY, source TEXT NOT NULL, -- plugin name target TEXT NOT NULL, -- e.g. greenhouse:acme label TEXT NOT NULL, -- what Item.subscription names company TEXT NOT NULL DEFAULT '', board TEXT NOT NULL DEFAULT '', params TEXT NOT NULL DEFAULT '{}', -- what a query target asks with subscribed INTEGER NOT NULL DEFAULT 1, fetch INTEGER NOT NULL DEFAULT 1, show INTEGER NOT NULL DEFAULT 1, added_at TEXT NOT NULL, target_id TEXT NOT NULL DEFAULT '', -- the plugin's durable name for it UNIQUE (source, target), UNIQUE (source, label) ); -- The tracker, in jobhunt-hq's shape (MIT). role_uid is NULL for a role that -- was never posted, so an unsolicited approach is tracked like any other. CREATE TABLE IF NOT EXISTS applications ( id TEXT PRIMARY KEY, role_uid TEXT REFERENCES roles(uid) ON DELETE SET NULL, company_key TEXT NOT NULL DEFAULT '', role_title TEXT NOT NULL DEFAULT '', url TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'saved' CHECK (status IN ('saved', 'applied', 'screen', 'technical', 'onsite', 'offer', 'rejected', 'withdrawn')), source TEXT NOT NULL DEFAULT '', applied_at TEXT, next_action TEXT NOT NULL DEFAULT '', next_date TEXT, contact TEXT NOT NULL DEFAULT '', notes TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL, updated_at TEXT NOT NULL ); -- One row per status an application entered, written by the store and never -- updated: what days-to-first-screen and conversion by source are read from. CREATE TABLE IF NOT EXISTS application_events ( id INTEGER PRIMARY KEY, application_id TEXT NOT NULL REFERENCES applications(id) ON DELETE CASCADE, status TEXT NOT NULL, at TEXT NOT NULL ); CREATE TRIGGER IF NOT EXISTS application_events_append_only BEFORE UPDATE ON application_events BEGIN SELECT RAISE(ABORT, 'application_events is append-only'); END; -- A search agent: a named saved search with its own prompts, gates and -- weights. gates is a list of {field, values, mode: require|prefer, weight}. -- limits is how deep into this agent's ranking extract and summarise read, -- as JSON: a key it lacks means the `Limits` default. CREATE TABLE IF NOT EXISTS agents ( id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, gates TEXT NOT NULL DEFAULT '[]', weights TEXT NOT NULL DEFAULT '{}', enabled INTEGER NOT NULL DEFAULT 1, created_at TEXT NOT NULL, limits TEXT NOT NULL DEFAULT '{}' ); -- What an agent looks for, as several named texts in order; a role's fit is -- its best prompt's. vec caches the text's vector and vec_key is sha1 of the -- embedding model and the text, so editing one prompt re-embeds that one. CREATE TABLE IF NOT EXISTS agent_prompts ( agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE, position INTEGER NOT NULL, name TEXT NOT NULL, text TEXT NOT NULL DEFAULT '', vec_key TEXT NOT NULL DEFAULT '', vec BLOB, PRIMARY KEY (agent_id, position), UNIQUE (agent_id, name COLLATE NOCASE) ); -- One agent's stance on a company: -1 hidden, 0 neutral, 1 liked, 2 -- favourite. No row is 0, so the table holds only what somebody set. Per -- agent, because a company that is noise for one search is the target of -- another. CREATE TABLE IF NOT EXISTS agent_companies ( agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE, company_key TEXT NOT NULL, affinity INTEGER NOT NULL CHECK (affinity BETWEEN -1 AND 2), updated_at TEXT NOT NULL, PRIMARY KEY (agent_id, company_key) ); -- One agent's judgement of a role: -1 pass, 0 neutral, 1 like. No row is -- unjudged. A vote moves the rank at read time, never hides a role, and its -- vectors shape the agent's taste direction in `score`. CREATE TABLE IF NOT EXISTS role_votes ( agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE, uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, vote INTEGER NOT NULL CHECK (vote IN (-1, 0, 1)), voted_at TEXT NOT NULL, PRIMARY KEY (agent_id, uid) ); -- A role is scored per agent. A gated role keeps its row, with the reason. -- prompt names the agent's prompt that gave the fit, '' when fit did not count. CREATE TABLE IF NOT EXISTS scores ( uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE, score REAL NOT NULL DEFAULT 0, gated_reason TEXT NOT NULL DEFAULT '', scored_at TEXT NOT NULL, prompt TEXT NOT NULL DEFAULT '', PRIMARY KEY (uid, agent_id) ); -- One vector per role, little-endian float32. Only comparable to another from -- the same model, so every read is scoped by it. CREATE TABLE IF NOT EXISTS vectors ( uid TEXT PRIMARY KEY REFERENCES roles(uid) ON DELETE CASCADE, model TEXT NOT NULL, dim INTEGER NOT NULL, vec BLOB NOT NULL, embedded_at TEXT NOT NULL ); -- A host word no table of app/prep/hosts maps, per field (seniority, -- employment, remote, country) and host: how many open roles carry it and -- when prep first met it. Rewritten by every prep; a word added to a table -- leaves at the next one. CREATE TABLE IF NOT EXISTS unmapped ( field TEXT NOT NULL, host TEXT NOT NULL, word TEXT NOT NULL, roles INTEGER NOT NULL DEFAULT 0, first_seen TEXT NOT NULL, PRIMARY KEY (field, host, word) ); -- The string map a desk plugin is handed as self.state. One namespace shared -- by every plugin, so a plugin prefixes its keys. CREATE TABLE IF NOT EXISTS state ( key TEXT PRIMARY KEY, value TEXT NOT NULL ); """ #: Applied after `SCHEMA` or the migrations, every open. An index may name a #: column only a migration adds, which is why these are kept apart. INDEXES = """ CREATE INDEX IF NOT EXISTS idx_roles_company ON roles(company_key); CREATE INDEX IF NOT EXISTS idx_roles_place ON roles(place_key); CREATE INDEX IF NOT EXISTS idx_roles_first_seen ON roles(first_seen DESC); CREATE INDEX IF NOT EXISTS idx_roles_closed ON roles(closed_at); -- Serves the newest feed (open_only, ordered by posted_at DESC), which is -- most of what `Store.roles` runs: partial, since only an open role's -- position in that order is ever asked for. CREATE INDEX IF NOT EXISTS idx_roles_posted ON roles(posted_at DESC, uid) WHERE closed_at IS NULL; CREATE INDEX IF NOT EXISTS idx_roles_cluster ON roles(company_key, title_key, place_key); CREATE INDEX IF NOT EXISTS idx_roles_board ON roles(source, subscription_id); CREATE INDEX IF NOT EXISTS idx_application_events ON application_events(application_id, at); CREATE INDEX IF NOT EXISTS idx_scores_agent ON scores(agent_id, score DESC); CREATE INDEX IF NOT EXISTS idx_role_votes_agent ON role_votes(agent_id, vote); CREATE INDEX IF NOT EXISTS idx_vectors_model ON vectors(model); CREATE INDEX IF NOT EXISTS idx_role_tags_tag ON role_tags(tag); -- The copies of a kept role, for `Store.role` and the feed's copy counts. -- Partial: all but about 3% of roles are no copy. CREATE INDEX IF NOT EXISTS idx_roles_dup_of ON roles(dup_of) WHERE dup_of IS NOT NULL; -- A role's places, and the roles in a set of places (the Place filter and -- the country and place gates, `uid IN (... WHERE place_id IN ...)`). CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid); CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid); CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id); -- The roles the next prep maps. Partial: after a prep there are none. CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0; """ #: The table whose absence marks a file that has never been a roledesk #: database, and is therefore created from `SCHEMA` rather than migrated. MARKER = "roles" Step = str | Callable[[sqlite3.Connection], None] def _migrate_posted_at(conn: sqlite3.Connection) -> None: """v10's step: add `posted_at`, then backfill it for every row already stored, by the same rule `Store.upsert_role` applies to a new one (see `app.models.posted_at`). A callable, not SQL, because the rule reads `raw` for which host a role came from. `'' NOT NULL` is a placeholder only until the UPDATE below runs, in the same transaction as the ALTER (`_apply` wraps every migration in one); every row gets a real value.""" conn.execute("ALTER TABLE roles ADD COLUMN posted_at TEXT NOT NULL DEFAULT ''") rows = conn.execute("SELECT uid, published_at, first_seen, raw FROM roles").fetchall() conn.executemany( "UPDATE roles SET posted_at = ? WHERE uid = ?", [(posted_at(row["published_at"], row["first_seen"], posted_host(json.loads(row["raw"] or "{}"))), row["uid"]) for row in rows]) MIGRATIONS: tuple[tuple[int, str, Step], ...] = ( # Never run as a migration: a file without the marker table is created # from `SCHEMA` and stamped. It exists so the first real migration is 2. (1, "roles, companies, subscriptions, applications, agents, scores, state", ""), (2, "roles.permit, roles.languages and roles.extract_tries, for extract", """ ALTER TABLE roles ADD COLUMN permit TEXT NOT NULL DEFAULT '' CHECK (permit IN ('', 'sponsors', 'no_sponsorship')); ALTER TABLE roles ADD COLUMN languages TEXT NOT NULL DEFAULT '[]'; ALTER TABLE roles ADD COLUMN extract_tries INTEGER NOT NULL DEFAULT 0; """), (3, "vectors, and the agent's cached intent vector, for score", """ CREATE TABLE vectors ( uid TEXT PRIMARY KEY REFERENCES roles(uid) ON DELETE CASCADE, model TEXT NOT NULL, dim INTEGER NOT NULL, vec BLOB NOT NULL, embedded_at TEXT NOT NULL ); ALTER TABLE agents ADD COLUMN intent_key TEXT NOT NULL DEFAULT ''; ALTER TABLE agents ADD COLUMN intent_vec BLOB; """), (4, "roles.vote, roles.summary and roles.article_text, for the phone UI", """ ALTER TABLE roles ADD COLUMN vote INTEGER CHECK (vote IN (-1, 0, 1)); ALTER TABLE roles ADD COLUMN voted_at TEXT; ALTER TABLE roles ADD COLUMN summary TEXT NOT NULL DEFAULT ''; ALTER TABLE roles ADD COLUMN summarised_at TEXT; ALTER TABLE roles ADD COLUMN article_text TEXT NOT NULL DEFAULT ''; ALTER TABLE roles ADD COLUMN article_at TEXT; """), (5, "roles.read_at and role_tags, for the feed's read state and tags", """ ALTER TABLE roles ADD COLUMN read_at TEXT; CREATE TABLE role_tags ( uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, tag TEXT NOT NULL, PRIMARY KEY (uid, tag) ); """), # Votes and affinities move from the role and company rows to one row per # agent. What was global goes to the default agent, the enabled one with # the lowest id (`Store.default_agent`); a file with something to move and # no agent gets one first, so nothing is lost. The columns are then # dropped, which SQLite has allowed since 3.35. (6, "agent_companies and role_votes: affinity and vote become per agent", """ CREATE TABLE agent_companies ( agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE, company_key TEXT NOT NULL, affinity INTEGER NOT NULL CHECK (affinity BETWEEN -1 AND 2), updated_at TEXT NOT NULL, PRIMARY KEY (agent_id, company_key) ); CREATE TABLE role_votes ( agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE, uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, vote INTEGER NOT NULL CHECK (vote IN (-1, 0, 1)), voted_at TEXT NOT NULL, PRIMARY KEY (agent_id, uid) ); INSERT INTO agents (name, intent, gates, weights, enabled, created_at) SELECT 'My ranking', '', '[]', '{}', 1, strftime('%Y-%m-%dT%H:%M:%S+00:00', 'now') WHERE NOT EXISTS (SELECT 1 FROM agents) AND (EXISTS (SELECT 1 FROM roles WHERE vote IS NOT NULL) OR EXISTS (SELECT 1 FROM companies WHERE affinity <> 0)); INSERT INTO role_votes (agent_id, uid, vote, voted_at) SELECT (SELECT id FROM agents ORDER BY enabled DESC, id LIMIT 1), uid, vote, COALESCE(voted_at, strftime('%Y-%m-%dT%H:%M:%S+00:00', 'now')) FROM roles WHERE vote IS NOT NULL; INSERT INTO agent_companies (agent_id, company_key, affinity, updated_at) SELECT (SELECT id FROM agents ORDER BY enabled DESC, id LIMIT 1), key, affinity, updated_at FROM companies WHERE affinity <> 0; ALTER TABLE roles DROP COLUMN vote; ALTER TABLE roles DROP COLUMN voted_at; ALTER TABLE companies DROP COLUMN affinity; """), (7, "companies.ats, country and city: a company the app cannot read yet is still a row", """ ALTER TABLE companies ADD COLUMN ats TEXT NOT NULL DEFAULT ''; ALTER TABLE companies ADD COLUMN country TEXT NOT NULL DEFAULT ''; ALTER TABLE companies ADD COLUMN city TEXT NOT NULL DEFAULT ''; """), (8, "agents.limits: how deep into its ranking the model stages read is each agent's own", """ ALTER TABLE agents ADD COLUMN limits TEXT NOT NULL DEFAULT '{}'; """), (9, "companies.ats_slug: the slug of a host the app cannot read yet is kept", """ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT ''; """), (10, "roles.posted_at: recency and the newest feed count from when a role " "was posted, not just when the app found it", _migrate_posted_at), # A Python step, not SQL: classify() lives in plugins/ats.py, and running # its current rules is the whole point (see _recompute_function below). (11, "roles.function recomputed for every not-yet-extracted role, word-boundary " "classifier fix", lambda conn: _recompute_function(conn)), (12, "roles.salary_text: the pay a board stated as one line, kept even when it " "could not be normalised into salary_min/salary_max/currency", """ ALTER TABLE roles ADD COLUMN salary_text TEXT NOT NULL DEFAULT ''; """), (13, "roles.country recomputed from place for every not-yet-extracted role whose " "board gave none, country lookup fix", lambda conn: _recompute_country(conn)), # An agent's one intent becomes its first prompt, named after the agent, # with its cached vector: the key hashes the model and the text, which # are unchanged, so nothing is embedded again. An empty intent becomes no # prompt. The three columns are then dropped, as v6 dropped the ones it # moved. (14, "agent_prompts and scores.prompt: an agent holds several prompts, and a " "score names the one that gave the fit", """ CREATE TABLE agent_prompts ( agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE, position INTEGER NOT NULL, name TEXT NOT NULL, text TEXT NOT NULL DEFAULT '', vec_key TEXT NOT NULL DEFAULT '', vec BLOB, PRIMARY KEY (agent_id, position), UNIQUE (agent_id, name COLLATE NOCASE) ); INSERT INTO agent_prompts (agent_id, position, name, text, vec_key, vec) SELECT id, 0, name, intent, intent_key, intent_vec FROM agents WHERE TRIM(intent, ' ' || char(9, 10, 13)) <> ''; ALTER TABLE agents DROP COLUMN intent; ALTER TABLE agents DROP COLUMN intent_key; ALTER TABLE agents DROP COLUMN intent_vec; ALTER TABLE scores ADD COLUMN prompt TEXT NOT NULL DEFAULT ''; """), # The Watchlist's own rating, global and apart from an agent's affinity: # NULL not yet rated, 1 worth checking by hand, 0 not. Every company # starts unrated. (15, "companies.watch and careers_opened_at: the Watchlist's rating and " "when its careers page was last opened", """ ALTER TABLE companies ADD COLUMN watch INTEGER CHECK (watch IN (0, 1)); ALTER TABLE companies ADD COLUMN careers_opened_at TEXT NOT NULL DEFAULT ''; """), # Why a person reads the company's careers page by hand, one of # models.HAND_REASONS; '' when nobody has to. When it was last looked at # is careers_opened_at. (16, "companies.hand_reason: why a company is read by hand", """ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT ''; """), # A Python step: the fold and the title classifier are Python, as in 11. (17, "roles.place_fold for the whole-word place filter, and roles.seniority from " "the title where extract has not read the role", lambda conn: _fold_places(conn)), # A Python step: the first rebuild runs the rule in app/dupes.py. (18, "roles.dup_of: one posting listed several times is one role, the others " "copies of it", lambda conn: _add_dup_of(conn)), # A Python step: the seed and the resolver are Python. (19, "places, place_aliases, place_includes and role_places: a role has a list of " "places, resolved against a gazetteer; roles.teaser; the three seniority " "sources", lambda conn: _add_places(conn)), # A Python step: the seed folds every name. (20, "company_aliases and the empty enrichment columns of companies", lambda conn: _add_company_aliases(conn)), # A Python step: params is JSON, the company is found by company_key and # each board that read everywhere is logged. (21, "companies.all_roles: a company tag that reads its boards with the agents' wide " "reach, given to every favourite and taking over a board's {\"reach\": \"all\"}", lambda conn: _add_all_roles(conn)), # A Python step: the host words are read out of raw's JSON. (22, "roles.*_host, roles.employment, roles.prep_due and unmapped: the host's own " "words are stored and app.prep maps them", lambda conn: _add_host_words(conn)), ) #: What a file created from `SCHEMA` today is already at. LATEST = MIGRATIONS[-1][0] def _recompute_function(conn: sqlite3.Connection) -> None: """Migration 11's step. `roles.function` used to be filled by a keyword classifier that matched substrings (" it" inside "Italy"), never returned '' (only "other", outside the declared vocabulary) and checked department before title; `plugins.ats.classify` no longer does any of that (see its own docstring). A role `extract` has already read keeps whatever it decided — `function` is extract's from then on, see `Store.set_extracted` — so only roles with no `extracted_at` are recomputed here, from their stored title and department (department lives in `raw`, not its own column: docs/decisions.md, "raw is transport, columns are storage"). A Python step because the classifier is Python, not SQL; `plugins` is a top-level package outside `app`'s own tree, so it is reached the same way `app/jobs/extract.py` reaches `FUNCTIONS`: inserted onto `sys.path` rather than imported as an installed dependency, since `plugins` is not in `pyproject.toml`'s packages.""" import sys from pathlib import Path root = Path(__file__).resolve().parents[2] if str(root) not in sys.path: sys.path.insert(0, str(root)) from plugins.ats import classify rows = conn.execute( "SELECT uid, title, raw, function FROM roles WHERE extracted_at IS NULL").fetchall() updates = [] for uid, title, raw, old in rows: try: department = json.loads(raw or "{}").get("department") or "" except (TypeError, ValueError): department = "" new = classify(department, title) if new != old: updates.append((new, uid)) if updates: conn.executemany("UPDATE roles SET function = ? WHERE uid = ?", updates) log.info("migration 11: recomputed function for %d of %d not-yet-extracted roles", len(updates), len(rows)) def _recompute_country(conn: sqlite3.Connection) -> None: """Migration 13's step. `app.models.country_of` used to read a US state code over a country's ("Bogota, CO" was US), return '' for a line naming several European countries, and miss most countries outside Europe; `Store.backfill_from_place` fills only an empty column, so a wrong code would stay. Recomputed from `place` for every role `extract` has not read (its answer only ever filled an empty column, but a role it read is left as it is) and whose board sent no `country` of its own in `raw` (the `search` plugin's Jobindex and Adzuna rows do), which is what `Store.upsert_role` would write for that role today.""" rows = conn.execute( "SELECT uid, place, raw, country FROM roles WHERE extracted_at IS NULL").fetchall() updates = [] for uid, place, raw, old in rows: try: given = json.loads(raw or "{}").get("country") or "" except (TypeError, ValueError, AttributeError): given = "" if given: continue new = country_of(place or "") if new != old: updates.append((new, uid)) if updates: conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", updates) log.info("migration 13: recomputed country for %d of %d not-yet-extracted roles", len(updates), len(rows)) def _fold_places(conn: sqlite3.Connection) -> None: """Migration 17's step. Adds `roles.place_fold` and fills it with `places.fold(place)` for every role, as `Store.upsert_role` writes it for a new one, and fills an empty `seniority` from `seniority.from_title` on every role `extract` has not read (it overwrites the title's level when it does, see `Store.set_extracted`). Each distinct place and title is folded or classified once: 34,538 roles hold about 5,600 places. The rows are written with one `executemany` per column inside `_apply`'s transaction, and only those with something to write.""" conn.execute("ALTER TABLE roles ADD COLUMN place_fold TEXT NOT NULL DEFAULT ''") rows = conn.execute( "SELECT uid, place, title, seniority, extracted_at FROM roles").fetchall() folds: dict[str, str] = {} levels: dict[str, str] = {} place_updates, level_updates = [], [] for uid, place, title, level, extracted_at in rows: if place: if place not in folds: folds[place] = places.fold(place) if folds[place]: place_updates.append((folds[place], uid)) if not level and extracted_at is None and title: if title not in levels: levels[title] = from_title(title) if levels[title]: level_updates.append((levels[title], uid)) conn.executemany("UPDATE roles SET place_fold = ? WHERE uid = ?", place_updates) conn.executemany("UPDATE roles SET seniority = ? WHERE uid = ?", level_updates) log.info("migration 17: folded %d places over %d roles, a level from the title for %d", len(folds), len(place_updates), len(level_updates)) def _add_dup_of(conn: sqlite3.Connection) -> None: """Migration 18's step. Adds `roles.dup_of` and runs the first rebuild (`store.rebuild_dupes`) inside `_apply`'s transaction, so a migrated file hides its copies from the first open rather than from the next pull. Imported here, not at the top: `store` imports this module.""" from app.storage.store import rebuild_dupes conn.execute("ALTER TABLE roles ADD COLUMN dup_of TEXT") counts = rebuild_dupes(conn) log.info("migration 18: %d copies of %d kept roles", counts.copies, counts.groups) _V19 = """ ALTER TABLE roles ADD COLUMN teaser TEXT NOT NULL DEFAULT ''; ALTER TABLE roles ADD COLUMN seniority_model TEXT NOT NULL DEFAULT ''; ALTER TABLE roles ADD COLUMN seniority_stated TEXT NOT NULL DEFAULT ''; ALTER TABLE roles ADD COLUMN seniority_title TEXT NOT NULL DEFAULT ''; CREATE TABLE places ( id INTEGER PRIMARY KEY, kind TEXT NOT NULL CHECK (kind IN ('continent', 'country', 'region', 'city')), key TEXT NOT NULL, name TEXT NOT NULL, parent_id INTEGER REFERENCES places(id), UNIQUE (kind, key) ); CREATE TABLE place_aliases ( fold TEXT NOT NULL, place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE, UNIQUE (fold, place_id) ); CREATE TABLE place_includes ( place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE, part_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE, PRIMARY KEY (place_id, part_id) ); CREATE TABLE role_places ( uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, position INTEGER NOT NULL, text TEXT NOT NULL, fold TEXT NOT NULL, place_id INTEGER REFERENCES places(id), url TEXT NOT NULL DEFAULT '' ); """ def _add_places(conn: sqlite3.Connection) -> None: """Migration 19's step. The tables and columns, the gazetteer seeded, and every role's places: the `places` of its `raw` where a reader stated them, else its `place` column cut at `;` (not `raw`'s, which a pull that read the role's own page sends empty), each resolved once per distinct text. The level splits into its sources without changing: on a role extract has read, `seniority` is the model's; on any other, it is the stated level when `raw` states one in the vocabulary, else the title's.""" for statement in _statements(_V19): conn.execute(statement) gazetteer.seed(conn) gaz = gazetteer.load(conn) rows = conn.execute( "SELECT uid, raw, place, title, seniority, extracted_at FROM roles").fetchall() by_uid: list[tuple[str, list[str]]] = [] levels = [] for uid, raw, place, title, level, extracted_at in rows: try: given = json.loads(raw or "{}") except ValueError: given = {} given = given if isinstance(given, dict) else {} if found := gazetteer.texts({"places": given.get("places"), "place": place}): by_uid.append((uid, found)) stated = str(given.get("seniority") or "") stated = stated if stated in SENIORITIES else "" if extracted_at is not None: levels.append((level, stated, from_title(title) if level else "", uid)) elif stated: levels.append(("", stated, from_title(title), uid)) elif level: levels.append(("", "", level, uid)) written = gazetteer.write_many(conn, gaz, by_uid) conn.executemany("UPDATE roles SET seniority_model = ?, seniority_stated = ?, " "seniority_title = ? WHERE uid = ?", levels) log.info("migration 19: %d places seeded, %d place rows for %d roles, levels split on %d", len(gaz.places), written, len(by_uid), len(levels)) _V20 = """ ALTER TABLE companies ADD COLUMN sector TEXT NOT NULL DEFAULT ''; ALTER TABLE companies ADD COLUMN stage TEXT NOT NULL DEFAULT ''; ALTER TABLE companies ADD COLUMN size_band TEXT NOT NULL DEFAULT ''; ALTER TABLE companies ADD COLUMN founded INTEGER; ALTER TABLE companies ADD COLUMN last_funding INTEGER; ALTER TABLE companies ADD COLUMN last_funding_currency TEXT NOT NULL DEFAULT ''; ALTER TABLE companies ADD COLUMN facts_source TEXT NOT NULL DEFAULT ''; ALTER TABLE companies ADD COLUMN facts_at TEXT NOT NULL DEFAULT ''; CREATE TABLE company_aliases ( fold TEXT NOT NULL, company_key TEXT NOT NULL, UNIQUE (fold, company_key) ); CREATE INDEX idx_company_aliases_key ON company_aliases(company_key); """ def _add_company_aliases(conn: sqlite3.Connection) -> None: """Migration 20's step. The columns and the table, and every company's name folded into it. Companies whose folds collide are logged and left alone: merging is a person's call (`Store.merge_companies`).""" for statement in _statements(_V20): conn.execute(statement) rows = conn.execute("SELECT key, name FROM companies").fetchall() conn.executemany("INSERT OR IGNORE INTO company_aliases (fold, company_key) VALUES (?, ?)", [(fold, key) for key, name in rows if (fold := company_fold(name))]) clashes = conn.execute("SELECT fold, group_concat(company_key, ', ') FROM company_aliases " "GROUP BY fold HAVING COUNT(*) > 1").fetchall() for fold, keys in clashes: log.warning("migration 20: %r is one fold of %s", fold, keys) log.info("migration 20: %d companies folded, %d folds shared", len(rows), len(clashes)) def _add_all_roles(conn: sqlite3.Connection) -> None: """Migration 21's step. The column; the tag for every company that is a favourite of any agent, since favourite no longer widens what a pull fetches and the tag does, so nothing read today is lost; and the tag for each company whose board's `params` held `{"reach": "all"}`, which the step removes from them. Such a board read everywhere and reads the wide reach from now on, so each is logged. Runs twice without harm.""" columns = {r[1] for r in conn.execute("PRAGMA table_info(companies)")} if "all_roles" not in columns: conn.execute("ALTER TABLE companies ADD COLUMN all_roles INTEGER NOT NULL DEFAULT 0") starred = conn.execute( "UPDATE companies SET all_roles=1 WHERE all_roles=0 AND key IN " "(SELECT company_key FROM agent_companies WHERE affinity=2)").rowcount log.info("migration 21: %d favourite companies tagged All roles", starred) for sub_id, target, company, label, raw in conn.execute( "SELECT id, target, company, label, params FROM subscriptions " "WHERE source = 'ats'").fetchall(): try: params = json.loads(raw or "{}") except ValueError: continue if not isinstance(params, dict) or params.get("reach") != "all": continue del params["reach"] conn.execute("UPDATE subscriptions SET params=? WHERE id=?", (json.dumps(params), sub_id)) key = company_key(company or label) if conn.execute("UPDATE companies SET all_roles=1 WHERE key=?", (key,)).rowcount: log.warning("migration 21: %s read all roles everywhere; %r is tagged All roles, " "which reads the agents' wide reach", target, key) else: log.warning("migration 21: %s read all roles everywhere, but %r has no company row " "to tag; it reads the agents' reach now", target, key) _V22_COLUMNS = ("seniority_host", "employment_host", "remote_host", "country_host", "employment") _V22 = """ CREATE TABLE IF NOT EXISTS unmapped ( field TEXT NOT NULL, host TEXT NOT NULL, word TEXT NOT NULL, roles INTEGER NOT NULL DEFAULT 0, first_seen TEXT NOT NULL, PRIMARY KEY (field, host, word) ); DELETE FROM state WHERE key = 'prep:tables'; """ def _add_host_words(conn: sqlite3.Connection) -> None: """Migration 22's step. The columns and `unmapped`, and the host words of every role filled from its `raw` where it has them. Before this, readers mapped `seniority` and `employment` in their own tables, so what `raw` holds there is mostly a value of the vocabulary already, which `app.prep` takes as it is. `remote` and `country` are the host's as given. The `prep:tables` fingerprint goes, so the first prep maps every open role from these words. Runs twice without harm.""" columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")} for column in _V22_COLUMNS: if column not in columns: conn.execute(f"ALTER TABLE roles ADD COLUMN {column} TEXT NOT NULL DEFAULT ''") if "prep_due" not in columns: conn.execute("ALTER TABLE roles ADD COLUMN prep_due INTEGER NOT NULL DEFAULT 0") for statement in _statements(_V22): conn.execute(statement) updates = [] for uid, raw in conn.execute( "SELECT uid, raw FROM roles WHERE json_extract(raw, '$.seniority') IS NOT NULL " "OR json_extract(raw, '$.employment') IS NOT NULL " "OR json_extract(raw, '$.remote') IS NOT NULL " "OR json_extract(raw, '$.country') IS NOT NULL").fetchall(): try: given = json.loads(raw or "{}") except ValueError: continue if isinstance(given, dict): updates.append((*(host_word(given.get(k)) for k in ( "seniority", "employment", "remote", "country")), uid)) conn.executemany("UPDATE roles SET seniority_host = ?, employment_host = ?, " "remote_host = ?, country_host = ? WHERE uid = ?", updates) log.info("migration 22: host words from raw for %d roles", len(updates)) def user_version(conn: sqlite3.Connection) -> int: return int(conn.execute("PRAGMA user_version").fetchone()[0]) def _stamp(conn: sqlite3.Connection, version: int) -> None: # PRAGMA takes no bound parameters; the value is an int from MIGRATIONS. conn.execute(f"PRAGMA user_version = {int(version)}") def _statements(script: str) -> list[str]: """A script split into statements, cut where `sqlite3.complete_statement` says one ends, which knows about comments and trigger bodies. Not `executescript`, which commits before it starts and autocommits each statement, so a failing migration would be half applied.""" statements, buffer = [], "" for line in script.splitlines(keepends=True): buffer += line if sqlite3.complete_statement(buffer): statements.append(buffer.strip()) buffer = "" if buffer.strip(): statements.append(buffer.strip()) return statements def migrate(conn: sqlite3.Connection) -> list[int]: """Bring an existing database up to `LATEST`. Returns what it applied.""" current = user_version(conn) if current > LATEST: raise RuntimeError( f"this database is at schema v{current}, but this roledesk only knows " f"up to v{LATEST}. Upgrade roledesk rather than let older code write to it.") applied = [] for version, description, step in MIGRATIONS: if version <= current: continue log.info("migrating database to v%d: %s", version, description) _apply(conn, version, step) applied.append(version) return applied def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None: """One migration and its version stamp, in one explicit transaction. Explicit because sqlite3 only opens a transaction implicitly before DML: an `ALTER` would otherwise be durable at once while the stamp stayed behind, and the next open would replay it into a "duplicate column" forever. """ started = not conn.in_transaction if started: conn.execute("BEGIN") try: if callable(step): step(conn) else: for statement in _statements(step): conn.execute(statement) _stamp(conn, version) except Exception: if started: conn.execute("ROLLBACK") raise if started: conn.execute("COMMIT") def _create(conn: sqlite3.Connection) -> None: """A new file: `SCHEMA`, and the gazetteer seeded.""" for statement in _statements(SCHEMA): conn.execute(statement) gazetteer.seed(conn) def initialise(conn: sqlite3.Connection) -> list[int]: """Create or upgrade, whichever this file needs. Returns migrations applied.""" fresh = conn.execute( "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (MARKER,) ).fetchone() is None applied: list[int] = [] if fresh: # Atomic like a migration: a file half created would otherwise carry # the marker table and be taken for one to migrate. _apply(conn, LATEST, _create) else: applied = migrate(conn) for statement in _statements(INDEXES): conn.execute(statement) return applied