Files
jobboard/roledesk/storage/schema.py
T
WolfandClaude Opus 5.5 bc60324fab Score roles per search agent, and run pull into extract, embed and score
embed stores one vector per open role; score rates every open role for
every enabled agent from require and prefer gates, fit against the
agent's intent, recency and company affinity, keeping gated rows with a
reason. Schema v3 adds vectors and the cached intent vector. The page
gets an agent order, the score on each row, and an Agents tab. Two
starter agents in roles.yaml are imported once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:14:21 +02:00

317 lines
13 KiB
Python

"""The tables, and how a database file gets from an old shape to this one.
`SCHEMA` is the current shape. A new file is created from it and stamped with
`LATEST`, so it never runs a migration. `MIGRATIONS` is how an existing file
catches up: ordered, numbered, applied once each, tracked in SQLite's own
`PRAGMA user_version`.
Adding a column means editing both: the DDL in `SCHEMA` for new files, and an
appended migration for old ones. Never renumber or edit a migration that has
shipped.
"""
from __future__ import annotations
import logging
import sqlite3
from collections.abc import Callable
log = logging.getLogger(__name__)
SCHEMA = """
-- One posting on one board. `uid` is sha1(source, source_id), the identity desk
-- gives every Item. The gate fields are real columns, unpacked from the item's
-- raw on upsert or filled later by extract: raw is transport, the columns are
-- what gets queried.
CREATE TABLE IF NOT EXISTS roles (
uid TEXT PRIMARY KEY,
source TEXT NOT NULL,
source_id TEXT NOT NULL,
subscription_id INTEGER REFERENCES subscriptions(id) ON DELETE SET NULL,
-- The cluster key is (company_key, title_key, place_key). No foreign key
-- to companies: a role may arrive before anyone has made a company row.
company_key TEXT NOT NULL DEFAULT '',
company TEXT NOT NULL DEFAULT '',
title TEXT NOT NULL DEFAULT '',
title_key TEXT NOT NULL DEFAULT '',
-- Geography. place and not location, which is where an Item's original
-- lives. remote and country are separate so a gate on remote can also
-- require the country: "UK | Remote" is not remote from Berlin.
place TEXT NOT NULL DEFAULT '',
place_key TEXT NOT NULL DEFAULT '',
country TEXT NOT NULL DEFAULT '',
remote TEXT NOT NULL DEFAULT ''
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
salary_min INTEGER,
salary_max INTEGER,
currency TEXT NOT NULL DEFAULT '',
seniority TEXT NOT NULL DEFAULT '',
function TEXT NOT NULL DEFAULT '',
-- Whether the posting says it sponsors a work permit, and the languages
-- (ISO 639-1, a JSON array) it says are required. Only extract fills these.
permit TEXT NOT NULL DEFAULT ''
CHECK (permit IN ('', 'sponsors', 'no_sponsorship')),
languages TEXT NOT NULL DEFAULT '[]',
description TEXT NOT NULL DEFAULT '', -- the posting's body text
feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over
original_url TEXT NOT NULL DEFAULT '', -- the posting page
published_at TEXT NOT NULL, -- what the source claims, UTC
-- What storing postings buys. first_seen is the only honest posted date,
-- since Greenhouse resets updated_at on every edit. closed_at is set when a
-- board fetched in full no longer lists the role, and cleared if it returns.
first_seen TEXT NOT NULL,
last_seen TEXT NOT NULL,
closed_at TEXT,
raw TEXT NOT NULL DEFAULT '{}',
extracted_at TEXT, -- when extract last filled columns
-- Replies from extract that left this role unanswered. At GIVE_UP_AFTER it
-- is no longer offered. A column, not raw, because every upsert rewrites raw.
extract_tries INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL
);
-- A company as a target, whether or not any board of it is read. key is
-- models.company_key(name), the slug roles.company_key holds.
CREATE TABLE IF NOT EXISTS companies (
key TEXT PRIMARY KEY,
name TEXT NOT NULL,
domain TEXT NOT NULL DEFAULT '',
-- -1 mute, 0 neutral, 1 interested, 2 favourite.
affinity INTEGER NOT NULL DEFAULT 0 CHECK (affinity BETWEEN -1 AND 2),
status TEXT NOT NULL DEFAULT 'researching',
careers_url TEXT NOT NULL DEFAULT '',
notes TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
-- The targets desk's runner hands to plugins. newsdesk's shape, with its two
-- labels named company and board from the first migration.
CREATE TABLE IF NOT EXISTS subscriptions (
id INTEGER PRIMARY KEY,
source TEXT NOT NULL, -- plugin name
target TEXT NOT NULL, -- e.g. greenhouse:acme
label TEXT NOT NULL, -- what Item.subscription names
company TEXT NOT NULL DEFAULT '',
board TEXT NOT NULL DEFAULT '',
params TEXT NOT NULL DEFAULT '{}', -- what a query target asks with
subscribed INTEGER NOT NULL DEFAULT 1,
fetch INTEGER NOT NULL DEFAULT 1,
show INTEGER NOT NULL DEFAULT 1,
added_at TEXT NOT NULL,
target_id TEXT NOT NULL DEFAULT '', -- the plugin's durable name for it
UNIQUE (source, target),
UNIQUE (source, label)
);
-- The tracker, in jobhunt-hq's shape (MIT). role_uid is NULL for a role that
-- was never posted, so an unsolicited approach is tracked like any other.
CREATE TABLE IF NOT EXISTS applications (
id TEXT PRIMARY KEY,
role_uid TEXT REFERENCES roles(uid) ON DELETE SET NULL,
company_key TEXT NOT NULL DEFAULT '',
role_title TEXT NOT NULL DEFAULT '',
url TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL DEFAULT 'saved'
CHECK (status IN ('saved', 'applied', 'screen', 'technical',
'onsite', 'offer', 'rejected', 'withdrawn')),
source TEXT NOT NULL DEFAULT '',
applied_at TEXT,
next_action TEXT NOT NULL DEFAULT '',
next_date TEXT,
contact TEXT NOT NULL DEFAULT '',
notes TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
-- One row per status an application entered, written by the store and never
-- updated: what days-to-first-screen and conversion by source are read from.
CREATE TABLE IF NOT EXISTS application_events (
id INTEGER PRIMARY KEY,
application_id TEXT NOT NULL REFERENCES applications(id) ON DELETE CASCADE,
status TEXT NOT NULL,
at TEXT NOT NULL
);
CREATE TRIGGER IF NOT EXISTS application_events_append_only
BEFORE UPDATE ON application_events
BEGIN
SELECT RAISE(ABORT, 'application_events is append-only');
END;
-- A search agent: a named saved search with its own intent text, gates and
-- weights. gates is a list of {field, values, mode: require|prefer, weight}.
-- intent_vec caches the intent's vector; intent_key is sha1 of the embedding
-- model and the intent text, so editing either makes it stale.
CREATE TABLE IF NOT EXISTS agents (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
intent TEXT NOT NULL DEFAULT '',
gates TEXT NOT NULL DEFAULT '[]',
weights TEXT NOT NULL DEFAULT '{}',
enabled INTEGER NOT NULL DEFAULT 1,
created_at TEXT NOT NULL,
intent_key TEXT NOT NULL DEFAULT '',
intent_vec BLOB
);
-- A role is scored per agent. A gated role keeps its row, with the reason.
CREATE TABLE IF NOT EXISTS scores (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
score REAL NOT NULL DEFAULT 0,
gated_reason TEXT NOT NULL DEFAULT '',
scored_at TEXT NOT NULL,
PRIMARY KEY (uid, agent_id)
);
-- One vector per role, little-endian float32. Only comparable to another from
-- the same model, so every read is scoped by it.
CREATE TABLE IF NOT EXISTS vectors (
uid TEXT PRIMARY KEY REFERENCES roles(uid) ON DELETE CASCADE,
model TEXT NOT NULL,
dim INTEGER NOT NULL,
vec BLOB NOT NULL,
embedded_at TEXT NOT NULL
);
-- The string map a desk plugin is handed as self.state. One namespace shared
-- by every plugin, so a plugin prefixes its keys.
CREATE TABLE IF NOT EXISTS state (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
"""
#: Applied after `SCHEMA` or the migrations, every open. An index may name a
#: column only a migration adds, which is why these are kept apart.
INDEXES = """
CREATE INDEX IF NOT EXISTS idx_roles_company ON roles(company_key);
CREATE INDEX IF NOT EXISTS idx_roles_place ON roles(place_key);
CREATE INDEX IF NOT EXISTS idx_roles_first_seen ON roles(first_seen DESC);
CREATE INDEX IF NOT EXISTS idx_roles_closed ON roles(closed_at);
CREATE INDEX IF NOT EXISTS idx_roles_cluster ON roles(company_key, title_key, place_key);
CREATE INDEX IF NOT EXISTS idx_roles_board ON roles(source, subscription_id);
CREATE INDEX IF NOT EXISTS idx_application_events
ON application_events(application_id, at);
CREATE INDEX IF NOT EXISTS idx_scores_agent ON scores(agent_id, score DESC);
CREATE INDEX IF NOT EXISTS idx_vectors_model ON vectors(model);
"""
#: The table whose absence marks a file that has never been a roledesk
#: database, and is therefore created from `SCHEMA` rather than migrated.
MARKER = "roles"
Step = str | Callable[[sqlite3.Connection], None]
MIGRATIONS: tuple[tuple[int, str, Step], ...] = (
# Never run as a migration: a file without the marker table is created
# from `SCHEMA` and stamped. It exists so the first real migration is 2.
(1, "roles, companies, subscriptions, applications, agents, scores, state", ""),
(2, "roles.permit, roles.languages and roles.extract_tries, for extract", """
ALTER TABLE roles ADD COLUMN permit TEXT NOT NULL DEFAULT ''
CHECK (permit IN ('', 'sponsors', 'no_sponsorship'));
ALTER TABLE roles ADD COLUMN languages TEXT NOT NULL DEFAULT '[]';
ALTER TABLE roles ADD COLUMN extract_tries INTEGER NOT NULL DEFAULT 0;
"""),
(3, "vectors, and the agent's cached intent vector, for score", """
CREATE TABLE vectors (
uid TEXT PRIMARY KEY REFERENCES roles(uid) ON DELETE CASCADE,
model TEXT NOT NULL,
dim INTEGER NOT NULL,
vec BLOB NOT NULL,
embedded_at TEXT NOT NULL
);
ALTER TABLE agents ADD COLUMN intent_key TEXT NOT NULL DEFAULT '';
ALTER TABLE agents ADD COLUMN intent_vec BLOB;
"""),
)
#: What a file created from `SCHEMA` today is already at.
LATEST = MIGRATIONS[-1][0]
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])
def _stamp(conn: sqlite3.Connection, version: int) -> None:
# PRAGMA takes no bound parameters; the value is an int from MIGRATIONS.
conn.execute(f"PRAGMA user_version = {int(version)}")
def _statements(script: str) -> list[str]:
"""A script split into statements, cut where `sqlite3.complete_statement`
says one ends, which knows about comments and trigger bodies. Not
`executescript`, which commits before it starts and autocommits each
statement, so a failing migration would be half applied."""
statements, buffer = [], ""
for line in script.splitlines(keepends=True):
buffer += line
if sqlite3.complete_statement(buffer):
statements.append(buffer.strip())
buffer = ""
if buffer.strip():
statements.append(buffer.strip())
return statements
def migrate(conn: sqlite3.Connection) -> list[int]:
"""Bring an existing database up to `LATEST`. Returns what it applied."""
current = user_version(conn)
if current > LATEST:
raise RuntimeError(
f"this database is at schema v{current}, but this roledesk only knows "
f"up to v{LATEST}. Upgrade roledesk rather than let older code write to it.")
applied = []
for version, description, step in MIGRATIONS:
if version <= current:
continue
log.info("migrating database to v%d: %s", version, description)
_apply(conn, version, step)
applied.append(version)
return applied
def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
"""One migration and its version stamp, in one explicit transaction.
Explicit because sqlite3 only opens a transaction implicitly before DML: an
`ALTER` would otherwise be durable at once while the stamp stayed behind,
and the next open would replay it into a "duplicate column" forever.
"""
started = not conn.in_transaction
if started:
conn.execute("BEGIN")
try:
if callable(step):
step(conn)
else:
for statement in _statements(step):
conn.execute(statement)
_stamp(conn, version)
except Exception:
if started:
conn.execute("ROLLBACK")
raise
if started:
conn.execute("COMMIT")
def initialise(conn: sqlite3.Connection) -> list[int]:
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
fresh = conn.execute(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (MARKER,)
).fetchone() is None
applied: list[int] = []
if fresh:
# Atomic like a migration: a file half created would otherwise carry
# the marker table and be taken for one to migrate.
_apply(conn, LATEST, SCHEMA)
else:
applied = migrate(conn)
for statement in _statements(INDEXES):
conn.execute(statement)
return applied