Rename the package roledesk to app
The boilerplate's rule: the deploy kit's Dockerfile, the APP_CONFIG, APP_PASSWORD and APP_DATA variables and its commands all assume a package called app, so a project renames its display title and nothing else. The console script is `app`, and the launch entry runs `uv run app serve --port 8790`. The domain noun stays `role`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
248ea1636d
commit
7a63311714
32 files changed
+102
-98
No files matched your search
@@ -0,0 +1,316 @@
|
||||
"""The tables, and how a database file gets from an old shape to this one.
|
||||
|
||||
`SCHEMA` is the current shape. A new file is created from it and stamped with
|
||||
`LATEST`, so it never runs a migration. `MIGRATIONS` is how an existing file
|
||||
catches up: ordered, numbered, applied once each, tracked in SQLite's own
|
||||
`PRAGMA user_version`.
|
||||
|
||||
Adding a column means editing both: the DDL in `SCHEMA` for new files, and an
|
||||
appended migration for old ones. Never renumber or edit a migration that has
|
||||
shipped.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SCHEMA = """
|
||||
-- One posting on one board. `uid` is sha1(source, source_id), the identity desk
|
||||
-- gives every Item. The gate fields are real columns, unpacked from the item's
|
||||
-- raw on upsert or filled later by extract: raw is transport, the columns are
|
||||
-- what gets queried.
|
||||
CREATE TABLE IF NOT EXISTS roles (
|
||||
uid TEXT PRIMARY KEY,
|
||||
source TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL,
|
||||
subscription_id INTEGER REFERENCES subscriptions(id) ON DELETE SET NULL,
|
||||
-- The cluster key is (company_key, title_key, place_key). No foreign key
|
||||
-- to companies: a role may arrive before anyone has made a company row.
|
||||
company_key TEXT NOT NULL DEFAULT '',
|
||||
company TEXT NOT NULL DEFAULT '',
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
title_key TEXT NOT NULL DEFAULT '',
|
||||
-- Geography. place and not location, which is where an Item's original
|
||||
-- lives. remote and country are separate so a gate on remote can also
|
||||
-- require the country: "UK | Remote" is not remote from Berlin.
|
||||
place TEXT NOT NULL DEFAULT '',
|
||||
place_key TEXT NOT NULL DEFAULT '',
|
||||
country TEXT NOT NULL DEFAULT '',
|
||||
remote TEXT NOT NULL DEFAULT ''
|
||||
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
|
||||
salary_min INTEGER,
|
||||
salary_max INTEGER,
|
||||
currency TEXT NOT NULL DEFAULT '',
|
||||
seniority TEXT NOT NULL DEFAULT '',
|
||||
function TEXT NOT NULL DEFAULT '',
|
||||
-- Whether the posting says it sponsors a work permit, and the languages
|
||||
-- (ISO 639-1, a JSON array) it says are required. Only extract fills these.
|
||||
permit TEXT NOT NULL DEFAULT ''
|
||||
CHECK (permit IN ('', 'sponsors', 'no_sponsorship')),
|
||||
languages TEXT NOT NULL DEFAULT '[]',
|
||||
description TEXT NOT NULL DEFAULT '', -- the posting's body text
|
||||
feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over
|
||||
original_url TEXT NOT NULL DEFAULT '', -- the posting page
|
||||
published_at TEXT NOT NULL, -- what the source claims, UTC
|
||||
-- What storing postings buys. first_seen is the only honest posted date,
|
||||
-- since Greenhouse resets updated_at on every edit. closed_at is set when a
|
||||
-- board fetched in full no longer lists the role, and cleared if it returns.
|
||||
first_seen TEXT NOT NULL,
|
||||
last_seen TEXT NOT NULL,
|
||||
closed_at TEXT,
|
||||
raw TEXT NOT NULL DEFAULT '{}',
|
||||
extracted_at TEXT, -- when extract last filled columns
|
||||
-- Replies from extract that left this role unanswered. At GIVE_UP_AFTER it
|
||||
-- is no longer offered. A column, not raw, because every upsert rewrites raw.
|
||||
extract_tries INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- A company as a target, whether or not any board of it is read. key is
|
||||
-- models.company_key(name), the slug roles.company_key holds.
|
||||
CREATE TABLE IF NOT EXISTS companies (
|
||||
key TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
domain TEXT NOT NULL DEFAULT '',
|
||||
-- -1 mute, 0 neutral, 1 interested, 2 favourite.
|
||||
affinity INTEGER NOT NULL DEFAULT 0 CHECK (affinity BETWEEN -1 AND 2),
|
||||
status TEXT NOT NULL DEFAULT 'researching',
|
||||
careers_url TEXT NOT NULL DEFAULT '',
|
||||
notes TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- The targets desk's runner hands to plugins. newsdesk's shape, with its two
|
||||
-- labels named company and board from the first migration.
|
||||
CREATE TABLE IF NOT EXISTS subscriptions (
|
||||
id INTEGER PRIMARY KEY,
|
||||
source TEXT NOT NULL, -- plugin name
|
||||
target TEXT NOT NULL, -- e.g. greenhouse:acme
|
||||
label TEXT NOT NULL, -- what Item.subscription names
|
||||
company TEXT NOT NULL DEFAULT '',
|
||||
board TEXT NOT NULL DEFAULT '',
|
||||
params TEXT NOT NULL DEFAULT '{}', -- what a query target asks with
|
||||
subscribed INTEGER NOT NULL DEFAULT 1,
|
||||
fetch INTEGER NOT NULL DEFAULT 1,
|
||||
show INTEGER NOT NULL DEFAULT 1,
|
||||
added_at TEXT NOT NULL,
|
||||
target_id TEXT NOT NULL DEFAULT '', -- the plugin's durable name for it
|
||||
UNIQUE (source, target),
|
||||
UNIQUE (source, label)
|
||||
);
|
||||
|
||||
-- The tracker, in jobhunt-hq's shape (MIT). role_uid is NULL for a role that
|
||||
-- was never posted, so an unsolicited approach is tracked like any other.
|
||||
CREATE TABLE IF NOT EXISTS applications (
|
||||
id TEXT PRIMARY KEY,
|
||||
role_uid TEXT REFERENCES roles(uid) ON DELETE SET NULL,
|
||||
company_key TEXT NOT NULL DEFAULT '',
|
||||
role_title TEXT NOT NULL DEFAULT '',
|
||||
url TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'saved'
|
||||
CHECK (status IN ('saved', 'applied', 'screen', 'technical',
|
||||
'onsite', 'offer', 'rejected', 'withdrawn')),
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
applied_at TEXT,
|
||||
next_action TEXT NOT NULL DEFAULT '',
|
||||
next_date TEXT,
|
||||
contact TEXT NOT NULL DEFAULT '',
|
||||
notes TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- One row per status an application entered, written by the store and never
|
||||
-- updated: what days-to-first-screen and conversion by source are read from.
|
||||
CREATE TABLE IF NOT EXISTS application_events (
|
||||
id INTEGER PRIMARY KEY,
|
||||
application_id TEXT NOT NULL REFERENCES applications(id) ON DELETE CASCADE,
|
||||
status TEXT NOT NULL,
|
||||
at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS application_events_append_only
|
||||
BEFORE UPDATE ON application_events
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'application_events is append-only');
|
||||
END;
|
||||
|
||||
-- A search agent: a named saved search with its own intent text, gates and
|
||||
-- weights. gates is a list of {field, values, mode: require|prefer, weight}.
|
||||
-- intent_vec caches the intent's vector; intent_key is sha1 of the embedding
|
||||
-- model and the intent text, so editing either makes it stale.
|
||||
CREATE TABLE IF NOT EXISTS agents (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
intent TEXT NOT NULL DEFAULT '',
|
||||
gates TEXT NOT NULL DEFAULT '[]',
|
||||
weights TEXT NOT NULL DEFAULT '{}',
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TEXT NOT NULL,
|
||||
intent_key TEXT NOT NULL DEFAULT '',
|
||||
intent_vec BLOB
|
||||
);
|
||||
|
||||
-- A role is scored per agent. A gated role keeps its row, with the reason.
|
||||
CREATE TABLE IF NOT EXISTS scores (
|
||||
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
||||
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
||||
score REAL NOT NULL DEFAULT 0,
|
||||
gated_reason TEXT NOT NULL DEFAULT '',
|
||||
scored_at TEXT NOT NULL,
|
||||
PRIMARY KEY (uid, agent_id)
|
||||
);
|
||||
|
||||
-- One vector per role, little-endian float32. Only comparable to another from
|
||||
-- the same model, so every read is scoped by it.
|
||||
CREATE TABLE IF NOT EXISTS vectors (
|
||||
uid TEXT PRIMARY KEY REFERENCES roles(uid) ON DELETE CASCADE,
|
||||
model TEXT NOT NULL,
|
||||
dim INTEGER NOT NULL,
|
||||
vec BLOB NOT NULL,
|
||||
embedded_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- The string map a desk plugin is handed as self.state. One namespace shared
|
||||
-- by every plugin, so a plugin prefixes its keys.
|
||||
CREATE TABLE IF NOT EXISTS state (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
"""
|
||||
|
||||
#: Applied after `SCHEMA` or the migrations, every open. An index may name a
|
||||
#: column only a migration adds, which is why these are kept apart.
|
||||
INDEXES = """
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_company ON roles(company_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_place ON roles(place_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_first_seen ON roles(first_seen DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_closed ON roles(closed_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_cluster ON roles(company_key, title_key, place_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_board ON roles(source, subscription_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_application_events
|
||||
ON application_events(application_id, at);
|
||||
CREATE INDEX IF NOT EXISTS idx_scores_agent ON scores(agent_id, score DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_vectors_model ON vectors(model);
|
||||
"""
|
||||
|
||||
#: The table whose absence marks a file that has never been a roledesk
|
||||
#: database, and is therefore created from `SCHEMA` rather than migrated.
|
||||
MARKER = "roles"
|
||||
|
||||
Step = str | Callable[[sqlite3.Connection], None]
|
||||
|
||||
MIGRATIONS: tuple[tuple[int, str, Step], ...] = (
|
||||
# Never run as a migration: a file without the marker table is created
|
||||
# from `SCHEMA` and stamped. It exists so the first real migration is 2.
|
||||
(1, "roles, companies, subscriptions, applications, agents, scores, state", ""),
|
||||
(2, "roles.permit, roles.languages and roles.extract_tries, for extract", """
|
||||
ALTER TABLE roles ADD COLUMN permit TEXT NOT NULL DEFAULT ''
|
||||
CHECK (permit IN ('', 'sponsors', 'no_sponsorship'));
|
||||
ALTER TABLE roles ADD COLUMN languages TEXT NOT NULL DEFAULT '[]';
|
||||
ALTER TABLE roles ADD COLUMN extract_tries INTEGER NOT NULL DEFAULT 0;
|
||||
"""),
|
||||
(3, "vectors, and the agent's cached intent vector, for score", """
|
||||
CREATE TABLE vectors (
|
||||
uid TEXT PRIMARY KEY REFERENCES roles(uid) ON DELETE CASCADE,
|
||||
model TEXT NOT NULL,
|
||||
dim INTEGER NOT NULL,
|
||||
vec BLOB NOT NULL,
|
||||
embedded_at TEXT NOT NULL
|
||||
);
|
||||
ALTER TABLE agents ADD COLUMN intent_key TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE agents ADD COLUMN intent_vec BLOB;
|
||||
"""),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
LATEST = MIGRATIONS[-1][0]
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
def _stamp(conn: sqlite3.Connection, version: int) -> None:
|
||||
# PRAGMA takes no bound parameters; the value is an int from MIGRATIONS.
|
||||
conn.execute(f"PRAGMA user_version = {int(version)}")
|
||||
|
||||
|
||||
def _statements(script: str) -> list[str]:
|
||||
"""A script split into statements, cut where `sqlite3.complete_statement`
|
||||
says one ends, which knows about comments and trigger bodies. Not
|
||||
`executescript`, which commits before it starts and autocommits each
|
||||
statement, so a failing migration would be half applied."""
|
||||
statements, buffer = [], ""
|
||||
for line in script.splitlines(keepends=True):
|
||||
buffer += line
|
||||
if sqlite3.complete_statement(buffer):
|
||||
statements.append(buffer.strip())
|
||||
buffer = ""
|
||||
if buffer.strip():
|
||||
statements.append(buffer.strip())
|
||||
return statements
|
||||
|
||||
|
||||
def migrate(conn: sqlite3.Connection) -> list[int]:
|
||||
"""Bring an existing database up to `LATEST`. Returns what it applied."""
|
||||
current = user_version(conn)
|
||||
if current > LATEST:
|
||||
raise RuntimeError(
|
||||
f"this database is at schema v{current}, but this roledesk only knows "
|
||||
f"up to v{LATEST}. Upgrade roledesk rather than let older code write to it.")
|
||||
applied = []
|
||||
for version, description, step in MIGRATIONS:
|
||||
if version <= current:
|
||||
continue
|
||||
log.info("migrating database to v%d: %s", version, description)
|
||||
_apply(conn, version, step)
|
||||
applied.append(version)
|
||||
return applied
|
||||
|
||||
|
||||
def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
|
||||
"""One migration and its version stamp, in one explicit transaction.
|
||||
|
||||
Explicit because sqlite3 only opens a transaction implicitly before DML: an
|
||||
`ALTER` would otherwise be durable at once while the stamp stayed behind,
|
||||
and the next open would replay it into a "duplicate column" forever.
|
||||
"""
|
||||
started = not conn.in_transaction
|
||||
if started:
|
||||
conn.execute("BEGIN")
|
||||
try:
|
||||
if callable(step):
|
||||
step(conn)
|
||||
else:
|
||||
for statement in _statements(step):
|
||||
conn.execute(statement)
|
||||
_stamp(conn, version)
|
||||
except Exception:
|
||||
if started:
|
||||
conn.execute("ROLLBACK")
|
||||
raise
|
||||
if started:
|
||||
conn.execute("COMMIT")
|
||||
|
||||
|
||||
def initialise(conn: sqlite3.Connection) -> list[int]:
|
||||
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
|
||||
fresh = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (MARKER,)
|
||||
).fetchone() is None
|
||||
applied: list[int] = []
|
||||
if fresh:
|
||||
# Atomic like a migration: a file half created would otherwise carry
|
||||
# the marker table and be taken for one to migrate.
|
||||
_apply(conn, LATEST, SCHEMA)
|
||||
else:
|
||||
applied = migrate(conn)
|
||||
for statement in _statements(INDEXES):
|
||||
conn.execute(statement)
|
||||
return applied
|
||||
Reference in new issue
Block a user