Rename the package roledesk to app

The boilerplate's rule: the deploy kit's Dockerfile, the APP_CONFIG,
APP_PASSWORD and APP_DATA variables and its commands all assume a package
called app, so a project renames its display title and nothing else. The
console script is `app`, and the launch entry runs `uv run app serve --port
8790`. The domain noun stays `role`.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-23 12:30:10 +02:00
1 parent 248ea1636d
commit 7a63311714
32 files changed
+102 -98

No files matched your search

+16
View File
@@ -0,0 +1,16 @@
"""Where things are kept. One door: `from app.storage import Store`.
schema.py the tables, and the numbered migrations that bring an older
file to the current shape
store.py `Store`, every SQL statement in the application
Written for roledesk rather than imported from newsdesk: `desk` has no storage
package on purpose, and the connection plumbing here is a deliberate copy.
"""
from __future__ import annotations
from app.storage.schema import LATEST
from app.storage.store import Store
__all__ = ["LATEST", "Store"]
+316
View File
@@ -0,0 +1,316 @@
"""The tables, and how a database file gets from an old shape to this one.
`SCHEMA` is the current shape. A new file is created from it and stamped with
`LATEST`, so it never runs a migration. `MIGRATIONS` is how an existing file
catches up: ordered, numbered, applied once each, tracked in SQLite's own
`PRAGMA user_version`.
Adding a column means editing both: the DDL in `SCHEMA` for new files, and an
appended migration for old ones. Never renumber or edit a migration that has
shipped.
"""
from __future__ import annotations
import logging
import sqlite3
from collections.abc import Callable
log = logging.getLogger(__name__)
SCHEMA = """
-- One posting on one board. `uid` is sha1(source, source_id), the identity desk
-- gives every Item. The gate fields are real columns, unpacked from the item's
-- raw on upsert or filled later by extract: raw is transport, the columns are
-- what gets queried.
CREATE TABLE IF NOT EXISTS roles (
uid TEXT PRIMARY KEY,
source TEXT NOT NULL,
source_id TEXT NOT NULL,
subscription_id INTEGER REFERENCES subscriptions(id) ON DELETE SET NULL,
-- The cluster key is (company_key, title_key, place_key). No foreign key
-- to companies: a role may arrive before anyone has made a company row.
company_key TEXT NOT NULL DEFAULT '',
company TEXT NOT NULL DEFAULT '',
title TEXT NOT NULL DEFAULT '',
title_key TEXT NOT NULL DEFAULT '',
-- Geography. place and not location, which is where an Item's original
-- lives. remote and country are separate so a gate on remote can also
-- require the country: "UK | Remote" is not remote from Berlin.
place TEXT NOT NULL DEFAULT '',
place_key TEXT NOT NULL DEFAULT '',
country TEXT NOT NULL DEFAULT '',
remote TEXT NOT NULL DEFAULT ''
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
salary_min INTEGER,
salary_max INTEGER,
currency TEXT NOT NULL DEFAULT '',
seniority TEXT NOT NULL DEFAULT '',
function TEXT NOT NULL DEFAULT '',
-- Whether the posting says it sponsors a work permit, and the languages
-- (ISO 639-1, a JSON array) it says are required. Only extract fills these.
permit TEXT NOT NULL DEFAULT ''
CHECK (permit IN ('', 'sponsors', 'no_sponsorship')),
languages TEXT NOT NULL DEFAULT '[]',
description TEXT NOT NULL DEFAULT '', -- the posting's body text
feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over
original_url TEXT NOT NULL DEFAULT '', -- the posting page
published_at TEXT NOT NULL, -- what the source claims, UTC
-- What storing postings buys. first_seen is the only honest posted date,
-- since Greenhouse resets updated_at on every edit. closed_at is set when a
-- board fetched in full no longer lists the role, and cleared if it returns.
first_seen TEXT NOT NULL,
last_seen TEXT NOT NULL,
closed_at TEXT,
raw TEXT NOT NULL DEFAULT '{}',
extracted_at TEXT, -- when extract last filled columns
-- Replies from extract that left this role unanswered. At GIVE_UP_AFTER it
-- is no longer offered. A column, not raw, because every upsert rewrites raw.
extract_tries INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL
);
-- A company as a target, whether or not any board of it is read. key is
-- models.company_key(name), the slug roles.company_key holds.
CREATE TABLE IF NOT EXISTS companies (
key TEXT PRIMARY KEY,
name TEXT NOT NULL,
domain TEXT NOT NULL DEFAULT '',
-- -1 mute, 0 neutral, 1 interested, 2 favourite.
affinity INTEGER NOT NULL DEFAULT 0 CHECK (affinity BETWEEN -1 AND 2),
status TEXT NOT NULL DEFAULT 'researching',
careers_url TEXT NOT NULL DEFAULT '',
notes TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
-- The targets desk's runner hands to plugins. newsdesk's shape, with its two
-- labels named company and board from the first migration.
CREATE TABLE IF NOT EXISTS subscriptions (
id INTEGER PRIMARY KEY,
source TEXT NOT NULL, -- plugin name
target TEXT NOT NULL, -- e.g. greenhouse:acme
label TEXT NOT NULL, -- what Item.subscription names
company TEXT NOT NULL DEFAULT '',
board TEXT NOT NULL DEFAULT '',
params TEXT NOT NULL DEFAULT '{}', -- what a query target asks with
subscribed INTEGER NOT NULL DEFAULT 1,
fetch INTEGER NOT NULL DEFAULT 1,
show INTEGER NOT NULL DEFAULT 1,
added_at TEXT NOT NULL,
target_id TEXT NOT NULL DEFAULT '', -- the plugin's durable name for it
UNIQUE (source, target),
UNIQUE (source, label)
);
-- The tracker, in jobhunt-hq's shape (MIT). role_uid is NULL for a role that
-- was never posted, so an unsolicited approach is tracked like any other.
CREATE TABLE IF NOT EXISTS applications (
id TEXT PRIMARY KEY,
role_uid TEXT REFERENCES roles(uid) ON DELETE SET NULL,
company_key TEXT NOT NULL DEFAULT '',
role_title TEXT NOT NULL DEFAULT '',
url TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL DEFAULT 'saved'
CHECK (status IN ('saved', 'applied', 'screen', 'technical',
'onsite', 'offer', 'rejected', 'withdrawn')),
source TEXT NOT NULL DEFAULT '',
applied_at TEXT,
next_action TEXT NOT NULL DEFAULT '',
next_date TEXT,
contact TEXT NOT NULL DEFAULT '',
notes TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
-- One row per status an application entered, written by the store and never
-- updated: what days-to-first-screen and conversion by source are read from.
CREATE TABLE IF NOT EXISTS application_events (
id INTEGER PRIMARY KEY,
application_id TEXT NOT NULL REFERENCES applications(id) ON DELETE CASCADE,
status TEXT NOT NULL,
at TEXT NOT NULL
);
CREATE TRIGGER IF NOT EXISTS application_events_append_only
BEFORE UPDATE ON application_events
BEGIN
SELECT RAISE(ABORT, 'application_events is append-only');
END;
-- A search agent: a named saved search with its own intent text, gates and
-- weights. gates is a list of {field, values, mode: require|prefer, weight}.
-- intent_vec caches the intent's vector; intent_key is sha1 of the embedding
-- model and the intent text, so editing either makes it stale.
CREATE TABLE IF NOT EXISTS agents (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
intent TEXT NOT NULL DEFAULT '',
gates TEXT NOT NULL DEFAULT '[]',
weights TEXT NOT NULL DEFAULT '{}',
enabled INTEGER NOT NULL DEFAULT 1,
created_at TEXT NOT NULL,
intent_key TEXT NOT NULL DEFAULT '',
intent_vec BLOB
);
-- A role is scored per agent. A gated role keeps its row, with the reason.
CREATE TABLE IF NOT EXISTS scores (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
score REAL NOT NULL DEFAULT 0,
gated_reason TEXT NOT NULL DEFAULT '',
scored_at TEXT NOT NULL,
PRIMARY KEY (uid, agent_id)
);
-- One vector per role, little-endian float32. Only comparable to another from
-- the same model, so every read is scoped by it.
CREATE TABLE IF NOT EXISTS vectors (
uid TEXT PRIMARY KEY REFERENCES roles(uid) ON DELETE CASCADE,
model TEXT NOT NULL,
dim INTEGER NOT NULL,
vec BLOB NOT NULL,
embedded_at TEXT NOT NULL
);
-- The string map a desk plugin is handed as self.state. One namespace shared
-- by every plugin, so a plugin prefixes its keys.
CREATE TABLE IF NOT EXISTS state (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
"""
#: Applied after `SCHEMA` or the migrations, every open. An index may name a
#: column only a migration adds, which is why these are kept apart.
INDEXES = """
CREATE INDEX IF NOT EXISTS idx_roles_company ON roles(company_key);
CREATE INDEX IF NOT EXISTS idx_roles_place ON roles(place_key);
CREATE INDEX IF NOT EXISTS idx_roles_first_seen ON roles(first_seen DESC);
CREATE INDEX IF NOT EXISTS idx_roles_closed ON roles(closed_at);
CREATE INDEX IF NOT EXISTS idx_roles_cluster ON roles(company_key, title_key, place_key);
CREATE INDEX IF NOT EXISTS idx_roles_board ON roles(source, subscription_id);
CREATE INDEX IF NOT EXISTS idx_application_events
ON application_events(application_id, at);
CREATE INDEX IF NOT EXISTS idx_scores_agent ON scores(agent_id, score DESC);
CREATE INDEX IF NOT EXISTS idx_vectors_model ON vectors(model);
"""
#: The table whose absence marks a file that has never been a roledesk
#: database, and is therefore created from `SCHEMA` rather than migrated.
MARKER = "roles"
Step = str | Callable[[sqlite3.Connection], None]
MIGRATIONS: tuple[tuple[int, str, Step], ...] = (
# Never run as a migration: a file without the marker table is created
# from `SCHEMA` and stamped. It exists so the first real migration is 2.
(1, "roles, companies, subscriptions, applications, agents, scores, state", ""),
(2, "roles.permit, roles.languages and roles.extract_tries, for extract", """
ALTER TABLE roles ADD COLUMN permit TEXT NOT NULL DEFAULT ''
CHECK (permit IN ('', 'sponsors', 'no_sponsorship'));
ALTER TABLE roles ADD COLUMN languages TEXT NOT NULL DEFAULT '[]';
ALTER TABLE roles ADD COLUMN extract_tries INTEGER NOT NULL DEFAULT 0;
"""),
(3, "vectors, and the agent's cached intent vector, for score", """
CREATE TABLE vectors (
uid TEXT PRIMARY KEY REFERENCES roles(uid) ON DELETE CASCADE,
model TEXT NOT NULL,
dim INTEGER NOT NULL,
vec BLOB NOT NULL,
embedded_at TEXT NOT NULL
);
ALTER TABLE agents ADD COLUMN intent_key TEXT NOT NULL DEFAULT '';
ALTER TABLE agents ADD COLUMN intent_vec BLOB;
"""),
)
#: What a file created from `SCHEMA` today is already at.
LATEST = MIGRATIONS[-1][0]
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])
def _stamp(conn: sqlite3.Connection, version: int) -> None:
# PRAGMA takes no bound parameters; the value is an int from MIGRATIONS.
conn.execute(f"PRAGMA user_version = {int(version)}")
def _statements(script: str) -> list[str]:
"""A script split into statements, cut where `sqlite3.complete_statement`
says one ends, which knows about comments and trigger bodies. Not
`executescript`, which commits before it starts and autocommits each
statement, so a failing migration would be half applied."""
statements, buffer = [], ""
for line in script.splitlines(keepends=True):
buffer += line
if sqlite3.complete_statement(buffer):
statements.append(buffer.strip())
buffer = ""
if buffer.strip():
statements.append(buffer.strip())
return statements
def migrate(conn: sqlite3.Connection) -> list[int]:
"""Bring an existing database up to `LATEST`. Returns what it applied."""
current = user_version(conn)
if current > LATEST:
raise RuntimeError(
f"this database is at schema v{current}, but this roledesk only knows "
f"up to v{LATEST}. Upgrade roledesk rather than let older code write to it.")
applied = []
for version, description, step in MIGRATIONS:
if version <= current:
continue
log.info("migrating database to v%d: %s", version, description)
_apply(conn, version, step)
applied.append(version)
return applied
def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
"""One migration and its version stamp, in one explicit transaction.
Explicit because sqlite3 only opens a transaction implicitly before DML: an
`ALTER` would otherwise be durable at once while the stamp stayed behind,
and the next open would replay it into a "duplicate column" forever.
"""
started = not conn.in_transaction
if started:
conn.execute("BEGIN")
try:
if callable(step):
step(conn)
else:
for statement in _statements(step):
conn.execute(statement)
_stamp(conn, version)
except Exception:
if started:
conn.execute("ROLLBACK")
raise
if started:
conn.execute("COMMIT")
def initialise(conn: sqlite3.Connection) -> list[int]:
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
fresh = conn.execute(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (MARKER,)
).fetchone() is None
applied: list[int] = []
if fresh:
# Atomic like a migration: a file half created would otherwise carry
# the marker table and be taken for one to migrate.
_apply(conn, LATEST, SCHEMA)
else:
applied = migrate(conn)
for statement in _statements(INDEXES):
conn.execute(statement)
return applied
+783
View File
@@ -0,0 +1,783 @@
"""`Store`: every SQL statement in the application.
The connection plumbing is a copy of newsdesk's `Store`, on purpose: `desk`
has no storage package. A connection is opened per call and closed again, so
one caller's commit can never land inside another's transaction; in WAL mode
that costs a file handle.
"""
from __future__ import annotations
import json
import sqlite3
from collections.abc import Callable, Iterator, MutableMapping, Sequence
from contextlib import closing, contextmanager
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from uuid import uuid4
import numpy as np
from desk.models import Item
from app.models import (
AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate,
Company, Gate, Role, ScoredRole, Subscription, Weights, company_key, place_key,
title_key,
)
from app.storage import schema
#: Text columns a plugin may fill from `Item.raw` under the same name. An
#: empty value never overwrites a filled column, so an upsert from a board that
#: says less than `extract` found keeps what `extract` found.
_RAW_TEXT = ("place", "country", "currency", "seniority", "function")
_RAW_INT = ("salary_min", "salary_max")
#: What `extract` may fill. Only an empty column is written: a board's own
#: structured field wins over what a model read out of the prose, and the next
#: upsert from that board would put it back anyway.
_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "function",
"permit")
#: How many replies may leave a role unanswered before `extract` stops
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
#: never answers is otherwise paid for on every pull for as long as it exists.
GIVE_UP_AFTER = 3
#: A role `extract` has yet to read: never answered, still open, not given up.
_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL "
f"AND extract_tries < {GIVE_UP_AFTER}")
_COMPANY_FIELDS = ("name", "domain", "affinity", "status", "careers_url", "notes")
_SUBSCRIPTION_FIELDS = ("label", "company", "board", "params", "subscribed", "fetch",
"show", "target_id")
def _iso(moment: datetime) -> str:
"""UTC, to the second. Every timestamp column is compared as text, so every
writer has to spell it the same way."""
return moment.astimezone(UTC).isoformat(timespec="seconds")
def _remote(value: Any) -> str:
"""A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything
unrecognised is '' (unknown), which `extract` may fill later."""
if value is True:
return "remote"
text = str(value or "").strip().lower().replace("-", "").replace(" ", "")
text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text)
return text if text in REMOTE else ""
def _like(text: str) -> str:
"""A `LIKE` pattern for a case-insensitive substring match, with SQLite's
own wildcards escaped so a search for "50%" or "a_b" stays literal."""
escaped = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
return f"%{escaped}%"
#: Vectors are stored little-endian float32, fixed here so a file written on
#: one machine reads on another. newsdesk's encoding, written again.
_DTYPE = np.dtype("<f4")
def pack(vec: np.ndarray) -> bytes:
return np.asarray(vec, dtype=_DTYPE).tobytes()
def unpack(blob: bytes) -> np.ndarray:
return np.frombuffer(blob, dtype=_DTYPE)
def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
"""One gate as `(empty, match, params)`: SQL that is true when the role's
column is empty, and SQL that is true when it matches, over `roles r`.
`params` belong to `match`; `empty` takes none."""
column = f"r.{gate.field}"
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
marks = ", ".join("?" for _ in values) or "NULL"
if gate.field == "salary_min":
floor = min(_int_or_none(v) or 0 for v in gate.values)
return f"{column} IS NULL", f"{column} >= ?", [floor]
if gate.field == "place":
likes = " OR ".join("LOWER(r.place) LIKE ? ESCAPE '\\'" for _ in values) or "0"
return "r.place = ''", f"({likes})", [_like(v) for v in values]
if gate.field == "company":
keys = [company_key(v) for v in values]
return ("r.company_key = ''",
f"r.company_key IN ({', '.join('?' for _ in keys) or 'NULL'})", keys)
if gate.field == "languages":
return ("r.languages IN ('', '[]')",
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
f"WHERE LOWER(value) NOT IN ({marks}))", values)
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
def _int_or_none(value: Any) -> int | None:
if value is None or value == "" or isinstance(value, bool):
return None
try:
return int(float(value))
except (TypeError, ValueError):
return None
class _StateMap(MutableMapping[str, str]):
"""`Store.state`: the key/value table, read and written one key at a time."""
def __init__(self, store: Store) -> None:
self._store = store
def __getitem__(self, key: str) -> str:
row = self._store._row("SELECT value FROM state WHERE key=?", (key,))
if row is None:
raise KeyError(key)
return str(row["value"])
def __setitem__(self, key: str, value: str) -> None:
self._store._write(
"""INSERT INTO state (key, value) VALUES (?,?)
ON CONFLICT(key) DO UPDATE SET value=excluded.value""",
(key, value))
def __delitem__(self, key: str) -> None:
if not self._store._write("DELETE FROM state WHERE key=?", (key,)):
raise KeyError(key)
def __iter__(self) -> Iterator[str]:
return iter([r["key"] for r in self._store._rows("SELECT key FROM state ORDER BY key")])
def __len__(self) -> int:
row = self._store._row("SELECT COUNT(*) AS n FROM state")
return int(row["n"]) if row else 0
class Store:
def __init__(self, path: str | Path) -> None:
self.path = Path(path).expanduser()
self.path.parent.mkdir(parents=True, exist_ok=True)
#: What "now" is. Replaced by tests that need two writes a second apart.
self.clock: Callable[[], datetime] = lambda: datetime.now(UTC)
with self.transaction() as conn:
# Persistent: recorded in the file, inherited by every connection.
conn.execute("PRAGMA journal_mode=WAL")
#: Migration versions this open applied; empty when already current.
self.migrations_applied: list[int] = schema.initialise(conn)
# ------------------------------------------------------------ connections
def _connect(self) -> sqlite3.Connection:
"""A fresh connection. `foreign_keys` is off by default per connection;
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
`synchronous=NORMAL` is the standard WAL companion."""
conn = sqlite3.connect(self.path)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA foreign_keys=ON")
conn.execute("PRAGMA busy_timeout=30000")
conn.execute("PRAGMA synchronous=NORMAL")
return conn
@contextmanager
def transaction(self) -> Iterator[sqlite3.Connection]:
"""One connection, one transaction, closed on the way out."""
conn = self._connect()
try:
with conn:
yield conn
finally:
conn.close()
def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]:
with closing(self._connect()) as conn:
return conn.execute(sql, params).fetchall()
def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None:
with closing(self._connect()) as conn:
row: sqlite3.Row | None = conn.execute(sql, params).fetchone()
return row
def _write(self, sql: str, params: Any = ()) -> int:
"""One statement in its own transaction. Returns the rowcount."""
with self.transaction() as conn:
return conn.execute(sql, params).rowcount
def now(self) -> str:
return _iso(self.clock())
# ---------------------------------------------------------------- roles
def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool:
"""Store one posting. Returns True when it was new.
`first_seen` and `created_at` are written on insert only; `last_seen` on
every call, and a role seen again is open again. The gate columns come
from `item.raw` under their own names, with `company` falling back to
the subscription's; an empty value never overwrites a filled column.
"""
if not item.source:
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
raw = item.raw or {}
company = str(raw.get("company") or (subscription.company if subscription else ""))
place = str(raw.get("place") or "")
stamp = self.now()
row: dict[str, Any] = {
"uid": item.uid,
"source": item.source,
"source_id": item.source_id,
"subscription_id": subscription.id if subscription else None,
"company_key": company_key(company),
"company": company,
"title": item.title,
"title_key": title_key(item.title),
"place_key": place_key(place),
"remote": _remote(raw.get("remote")),
"description": item.body,
"feed_text": item.excerpt,
"original_url": item.location,
"published_at": _iso(item.published_at),
"first_seen": stamp,
"last_seen": stamp,
"raw": json.dumps(raw, default=str),
"created_at": stamp,
}
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
keep_filled = ("company", "company_key", "place_key", "remote", "description",
"feed_text", "original_url", *_RAW_TEXT)
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
for c in (*_RAW_INT, "subscription_id")]
updates += [f"{c}=excluded.{c}"
for c in ("title", "title_key", "published_at", "raw", "last_seen")]
updates.append("closed_at=NULL")
columns = ", ".join(row)
marks = ", ".join(f":{c}" for c in row)
with self.transaction() as conn:
existed = conn.execute("SELECT 1 FROM roles WHERE uid=?",
(row["uid"],)).fetchone() is not None
conn.execute(
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
return not existed
def role(self, uid: str) -> Role | None:
row = self._row("SELECT * FROM roles WHERE uid=?", (uid,))
return Role.from_row(row) if row else None
def roles(self, limit: int = 100, company: str | None = None,
place: str | None = None, open_only: bool = True,
q: str | None = None, function: str | None = None,
remote: str | None = None, days: int | None = None,
hide_muted: bool = False, agent: int | None = None,
gated: bool = False) -> list[ScoredRole]:
"""Newest first by `first_seen`. `company` and `place` are
case-insensitive substrings of the columns a board wrote — not the
key, so a search does not have to know the exact slug a company folds
to. `q` is matched word by word against `title`; a role must match
every word. `function` and `remote` match a gate column exactly.
`days` keeps only roles first seen within that many days of now.
`hide_muted` excludes a muted company's roles before `limit` is
applied, in SQL rather than after the fact: a company that has
published a lot, all at once — a fresh board added, a re-pull that
finally reached it — can otherwise fill the whole page with roles
that then all turn out to be from the one company someone muted,
leaving nothing to show for a default query that would otherwise
have plenty.
`agent` orders by that agent's score instead, highest first, and
carries `score` and `gated_reason` on each row. Only roles it has
scored are listed, and a gated one only when `gated` is set."""
where, params = [], []
if company:
where.append("company LIKE ? ESCAPE '\\'")
params.append(_like(company))
if place:
where.append("place LIKE ? ESCAPE '\\'")
params.append(_like(place))
if function:
where.append("function = ?")
params.append(function)
if remote:
where.append("remote = ?")
params.append(remote)
if days is not None:
where.append("first_seen >= ?")
params.append(_iso(self.clock() - timedelta(days=days)))
for word in (q or "").split():
where.append("title LIKE ? ESCAPE '\\'")
params.append(_like(word))
if open_only:
where.append("closed_at IS NULL")
if hide_muted:
where.append("company_key NOT IN "
"(SELECT key FROM companies WHERE affinity = -1)")
joins: list[Any] = []
if agent is None:
sql = "SELECT roles.* FROM roles"
order = "roles.first_seen DESC, roles.uid"
else:
sql = ("SELECT roles.*, s.score, s.gated_reason FROM roles "
"JOIN scores s ON s.uid = roles.uid AND s.agent_id = ?")
joins.append(agent)
if not gated:
where.append("s.gated_reason = ''")
order = "s.score DESC, roles.first_seen DESC, roles.uid"
if where:
sql += " WHERE " + " AND ".join(where)
sql += f" ORDER BY {order} LIMIT ?"
return [ScoredRole.from_row(r)
for r in self._rows(sql, [*joins, *params, limit])]
def mark_closed(self, source: str, seen_ids: Sequence[str],
subscription_id: int) -> int:
"""Close every open role of one board that a complete fetch of it did not
list. Returns how many closed. Only call it after a fetch that finished:
a failed one lists nothing, and would close the whole board."""
return self._write(
"""UPDATE roles SET closed_at = ?
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
AND source_id NOT IN (SELECT value FROM json_each(?))""",
(self.now(), source, subscription_id, json.dumps(list(seen_ids))))
# -------------------------------------------------------------- extract
def extract_pending(self) -> int:
row = self._row(f"SELECT COUNT(*) AS n FROM roles WHERE {_EXTRACT_WAITING}")
return int(row["n"]) if row else 0
def extract_waiting(self, limit: int,
uids: Sequence[str] | None = None) -> list[Role]:
"""Roles `extract` has yet to read, newest first. Given `uids`, only
those of them still waiting: a batch asks again when it runs, since a
role may have been answered since the task shared the work out."""
sql, params = f"SELECT * FROM roles WHERE {_EXTRACT_WAITING}", []
if uids is not None:
sql += " AND uid IN (SELECT value FROM json_each(?))"
params.append(json.dumps(list(uids)))
sql += " ORDER BY first_seen DESC, uid LIMIT ?"
return [Role.from_row(r) for r in self._rows(sql, [*params, limit])]
def set_extracted(self, uid: str, fields: dict[str, Any]) -> int | None:
"""Write what `extract` read for one role and stamp `extracted_at`.
Only into empty columns, and only a non-empty value: a board's own
field is never replaced and nothing is ever blanked. `place` brings
`place_key` with it. Returns how many columns were filled, or None
for an unknown uid. Values are trusted to be in vocabulary; the
job's `apply` is where a reply is checked."""
with self.transaction() as conn:
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
if row is None:
return None
sets: dict[str, Any] = {}
for name in _EXTRACT_TEXT:
value = str(fields.get(name) or "")
if value and not row[name]:
sets[name] = value
for name in _RAW_INT:
number = _int_or_none(fields.get(name))
if number is not None and row[name] is None:
sets[name] = number
languages = [str(code) for code in fields.get("languages") or []]
if languages and json.loads(row["languages"] or "[]") == []:
sets["languages"] = json.dumps(languages)
filled = len(sets)
if "place" in sets:
sets["place_key"] = place_key(sets["place"])
sets["extracted_at"] = self.now()
conn.execute(
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
(*sets.values(), uid))
return filled
def extract_missed(self, uids: Sequence[str], why: str = "") -> int:
"""One reply that left these roles unanswered. Counted only against
those still waiting, so a role the reply did answer is not charged.
Returns how many this pushed to `GIVE_UP_AFTER`. `why` is accepted
for `ModelTask.missed` and not kept: there is no Given up list yet."""
if not uids:
return 0
where = f"uid IN (SELECT value FROM json_each(?)) AND {_EXTRACT_WAITING}"
wanted = json.dumps(list(uids))
with self.transaction() as conn:
crossed = conn.execute(
f"SELECT COUNT(*) FROM roles WHERE {where} AND extract_tries + 1 >= ?",
(wanted, GIVE_UP_AFTER)).fetchone()[0]
conn.execute(f"UPDATE roles SET extract_tries = extract_tries + 1 "
f"WHERE {where}", (wanted,))
return int(crossed)
# -------------------------------------------------------------- vectors
def roles_to_embed(self, model: str, limit: int) -> list[Role]:
"""Open roles with no vector from `model`, newest first, so a backlog
embedded from the top improves what is on screen first."""
return [Role.from_row(r) for r in self._rows(
"""SELECT r.* FROM roles r
LEFT JOIN vectors v ON v.uid = r.uid AND v.model = ?
WHERE v.uid IS NULL AND r.closed_at IS NULL
ORDER BY r.first_seen DESC, r.uid LIMIT ?""", (model, limit))]
def put_vectors(self, model: str, pairs: Sequence[tuple[str, np.ndarray]]) -> int:
stamp = self.now()
with self.transaction() as conn:
return conn.executemany(
"""INSERT INTO vectors (uid, model, dim, vec, embedded_at)
VALUES (?,?,?,?,?)
ON CONFLICT(uid) DO UPDATE SET model=excluded.model,
dim=excluded.dim, vec=excluded.vec,
embedded_at=excluded.embedded_at""",
[(uid, model, int(vec.shape[-1]), pack(vec), stamp)
for uid, vec in pairs]).rowcount
def vectors(self, model: str) -> dict[str, np.ndarray]:
"""Every vector from `model`, by uid. The whole table: at ~30k roles of
384 floats that is ~45 MB, and one read beats one per agent."""
return {r["uid"]: unpack(r["vec"]) for r in self._rows(
"SELECT uid, vec FROM vectors WHERE model = ?", (model,))}
def vector_counts(self) -> dict[str, int]:
return {r["model"]: int(r["n"]) for r in self._rows(
"SELECT model, COUNT(*) AS n FROM vectors GROUP BY model")}
def drop_vectors(self, keep: str) -> int:
"""Delete vectors from every model but `keep`: another model's are a
different geometry, never compared, and only make counts lie."""
return self._write("DELETE FROM vectors WHERE model <> ?", (keep,))
# --------------------------------------------------------------- agents
def agents(self, enabled_only: bool = False) -> list[Agent]:
sql = "SELECT * FROM agents"
if enabled_only:
sql += " WHERE enabled = 1"
return [Agent.from_row(r)
for r in self._rows(sql + " ORDER BY name COLLATE NOCASE")]
def agent(self, agent_id: int) -> Agent | None:
row = self._row("SELECT * FROM agents WHERE id=?", (agent_id,))
return Agent.from_row(row) if row else None
def save_agent(self, name: str, intent: str = "", gates: Sequence[Gate] = (),
weights: Weights | None = None, enabled: bool = True,
agent_id: int | None = None) -> Agent:
"""Create an agent, or with `agent_id` replace that one's fields.
Raises ValueError on an empty or taken name, KeyError on an unknown id."""
name = name.strip()
if not name:
raise ValueError("an agent needs a name")
fields = (name, intent, json.dumps([g.model_dump() for g in gates]),
(weights or Weights()).model_dump_json(), int(enabled))
try:
with self.transaction() as conn:
if agent_id is None:
agent_id = conn.execute(
"""INSERT INTO agents (name, intent, gates, weights, enabled,
created_at) VALUES (?,?,?,?,?,?)""",
(*fields, self.now())).lastrowid
elif not conn.execute(
"""UPDATE agents SET name=?, intent=?, gates=?, weights=?,
enabled=? WHERE id=?""",
(*fields, agent_id)).rowcount:
raise KeyError(agent_id)
except sqlite3.IntegrityError as exc:
raise ValueError(f"an agent named {name!r} already exists") from exc
assert agent_id is not None
found = self.agent(agent_id)
assert found is not None
return found
def delete_agent(self, agent_id: int) -> bool:
"""Its scores go with it, by the foreign key."""
return self._write("DELETE FROM agents WHERE id=?", (agent_id,)) > 0
def intent_vector(self, agent_id: int, key: str) -> np.ndarray | None:
"""The agent's cached intent vector, if it was made under `key`."""
row = self._row("SELECT intent_key, intent_vec FROM agents WHERE id=?",
(agent_id,))
if row is None or row["intent_key"] != key or row["intent_vec"] is None:
return None
return unpack(row["intent_vec"])
def set_intent_vector(self, agent_id: int, key: str, vec: np.ndarray) -> None:
self._write("UPDATE agents SET intent_key=?, intent_vec=? WHERE id=?",
(key, pack(vec), agent_id))
# --------------------------------------------------------------- scores
def score_candidates(self, gates: Sequence[Gate]) -> list[Candidate]:
"""Every open role of a company that is not muted, with how it meets
each gate: 1 match, 0 no match, -1 the column is empty. The gates are
evaluated here, in SQL, so a `require` gate is the `WHERE` it reads
as, written as a column so a gated role keeps its row and reason."""
columns: list[str] = []
params: list[Any] = []
for n, gate in enumerate(gates):
empty, match, values = _gate_sql(gate)
columns.append(
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
params += values
select = ", ".join(["r.uid", "r.first_seen",
"COALESCE(c.affinity, 0) AS affinity", *columns])
rows = self._rows(
f"""SELECT {select} FROM roles r
LEFT JOIN companies c ON c.key = r.company_key
WHERE r.closed_at IS NULL AND COALESCE(c.affinity, 0) <> -1""",
params)
return [Candidate(uid=r["uid"], first_seen=r["first_seen"],
affinity=int(r["affinity"]),
gates=[int(r[f"g{n}"]) for n in range(len(gates))])
for r in rows]
def replace_scores(self, agent_id: int,
rows: Sequence[tuple[str, float, str]]) -> int:
"""`(uid, score, gated_reason)` for every role this agent scored,
replacing whatever it had: a role closed or muted since the last run
drops out rather than keeping a stale score. One transaction, so a
reader sees the old list or the new one."""
stamp = self.now()
with self.transaction() as conn:
conn.execute("DELETE FROM scores WHERE agent_id=?", (agent_id,))
conn.executemany(
"""INSERT INTO scores (uid, agent_id, score, gated_reason, scored_at)
VALUES (?,?,?,?,?)""",
[(uid, agent_id, score, reason, stamp) for uid, score, reason in rows])
return len(rows)
def scores(self, agent_id: int) -> dict[str, tuple[float, str]]:
"""`uid -> (score, gated_reason)` for one agent."""
return {r["uid"]: (float(r["score"]), str(r["gated_reason"]))
for r in self._rows(
"SELECT uid, score, gated_reason FROM scores WHERE agent_id=?",
(agent_id,))}
# ------------------------------------------------------------ companies
def add_company(self, name: str, *, key: str = "", domain: str = "",
affinity: int = 0, status: str = "researching",
careers_url: str = "", notes: str = "") -> Company:
"""Raises ValueError on a key already taken or an affinity out of range."""
name = name.strip()
key = key or company_key(name)
if not key:
raise ValueError("a company needs a name")
_check_affinity(affinity)
stamp = self.now()
try:
self._write(
"""INSERT INTO companies (key, name, domain, affinity, status,
careers_url, notes, created_at, updated_at)
VALUES (?,?,?,?,?,?,?,?,?)""",
(key, name, domain, affinity, status, careers_url, notes, stamp, stamp))
except sqlite3.IntegrityError as exc:
raise ValueError(f"company {key!r} already exists") from exc
return self._company(key)
def company(self, key: str) -> Company | None:
row = self._row("SELECT * FROM companies WHERE key=?", (key,))
return Company.from_row(row) if row else None
def companies(self) -> list[Company]:
return [Company.from_row(r)
for r in self._rows("SELECT * FROM companies ORDER BY name COLLATE NOCASE")]
def update_company(self, key: str, **fields: Any) -> Company:
"""Change any of name, domain, affinity, status, careers_url, notes."""
unknown = set(fields) - set(_COMPANY_FIELDS)
if unknown:
raise ValueError(f"not a company field: {', '.join(sorted(unknown))}")
if "affinity" in fields:
_check_affinity(fields["affinity"])
if fields:
sets = ", ".join(f"{f}=?" for f in fields)
changed = self._write(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?",
(*fields.values(), self.now(), key))
if not changed:
raise KeyError(key)
return self._company(key)
def remove_company(self, key: str) -> bool:
return self._write("DELETE FROM companies WHERE key=?", (key,)) > 0
def _company(self, key: str) -> Company:
found = self.company(key)
if found is None:
raise KeyError(key)
return found
def open_role_counts(self) -> dict[str, int]:
"""`company_key -> open role count`, for the company list's row
count. A key with no open roles is simply absent."""
return {str(r["company_key"]): int(r["n"]) for r in self._rows(
"SELECT company_key, COUNT(*) AS n FROM roles "
"WHERE closed_at IS NULL GROUP BY company_key")}
def company_boards(self) -> dict[str, str]:
"""`company_key -> ats host`, read off each company's own
subscription. `Subscription.company` is the display name it and the
company row were both added with, so `company_key` of one finds the
other; a company added by hand with no subscription is simply
absent."""
boards: dict[str, str] = {}
for sub in self.subscriptions(source="ats"):
boards.setdefault(company_key(sub.company), sub.board)
return boards
# -------------------------------------------------------- subscriptions
def subscriptions(self, source: str | None = None,
fetchable: bool = False) -> list[Subscription]:
"""`fetchable` is what a pull reads: subscribed and with fetch on."""
where, params = [], []
if source:
where.append("source = ?")
params.append(source)
if fetchable:
where.append("fetch = 1 AND subscribed = 1")
sql = "SELECT * FROM subscriptions"
if where:
sql += " WHERE " + " AND ".join(where)
sql += " ORDER BY company, board, label"
return [Subscription.from_row(r) for r in self._rows(sql, params)]
def subscription(self, sub_id: int) -> Subscription | None:
row = self._row("SELECT * FROM subscriptions WHERE id=?", (sub_id,))
return Subscription.from_row(row) if row else None
def subscription_for(self, source: str, label: str) -> Subscription | None:
"""The row an item names: `Item.subscription` is the label."""
row = self._row("SELECT * FROM subscriptions WHERE source=? AND label=?",
(source, label))
return Subscription.from_row(row) if row else None
def add_subscription(self, source: str, target: str, label: str = "",
company: str = "", board: str = "",
params: dict[str, Any] | None = None,
target_id: str = "") -> Subscription:
"""Raises ValueError on a duplicate target or label within one plugin."""
target = target.strip()
if not target:
raise ValueError("a subscription needs a target")
label = label.strip() or target
try:
with self.transaction() as conn:
new_id = conn.execute(
"""INSERT INTO subscriptions (source, target, label, company, board,
params, added_at, target_id)
VALUES (?,?,?,?,?,?,?,?)""",
(source, target, label, company, board, json.dumps(params or {}),
self.now(), target_id)).lastrowid
except sqlite3.IntegrityError as exc:
raise ValueError(f"{source} already has {target!r} or the label {label!r}") from exc
assert new_id is not None
return self._subscription(new_id)
def update_subscription(self, sub_id: int, **fields: Any) -> Subscription:
"""Change labels, params, target_id or the three switches."""
unknown = set(fields) - set(_SUBSCRIPTION_FIELDS)
if unknown:
raise ValueError(f"not a subscription field: {', '.join(sorted(unknown))}")
if "params" in fields:
fields["params"] = json.dumps(fields["params"] or {})
for switch in ("subscribed", "fetch", "show"):
if switch in fields:
fields[switch] = int(bool(fields[switch]))
if fields:
sets = ", ".join(f"{f}=?" for f in fields)
try:
changed = self._write(f"UPDATE subscriptions SET {sets} WHERE id=?",
(*fields.values(), sub_id))
except sqlite3.IntegrityError as exc:
raise ValueError(f"label {fields.get('label')!r} is already in use") from exc
if not changed:
raise KeyError(sub_id)
return self._subscription(sub_id)
def unsubscribe(self, sub_id: int) -> Subscription:
"""Stop following it and hide it. The row stays, and so do its roles."""
return self.update_subscription(sub_id, subscribed=0, fetch=0, show=0)
def _subscription(self, sub_id: int) -> Subscription:
found = self.subscription(sub_id)
if found is None:
raise KeyError(sub_id)
return found
# --------------------------------------------------------- applications
def save_application(self, application: Application) -> Application:
"""Insert or update one application. An empty `id` is minted. Entering a
status, on insert or by change, appends an `application_events` row in
the same transaction."""
_check_status(application.status)
app_id = application.id or uuid4().hex
stamp = self.now()
fields = application.model_dump(exclude={"id", "created_at", "updated_at"})
with self.transaction() as conn:
before = conn.execute("SELECT status FROM applications WHERE id=?",
(app_id,)).fetchone()
if before is None:
columns = ["id", *fields, "created_at", "updated_at"]
conn.execute(
f"INSERT INTO applications ({', '.join(columns)}) "
f"VALUES ({', '.join('?' for _ in columns)})",
(app_id, *fields.values(), stamp, stamp))
else:
sets = ", ".join(f"{f}=?" for f in fields)
conn.execute(f"UPDATE applications SET {sets}, updated_at=? WHERE id=?",
(*fields.values(), stamp, app_id))
if before is None or before["status"] != application.status:
conn.execute("INSERT INTO application_events (application_id, status, at) "
"VALUES (?,?,?)", (app_id, application.status, stamp))
return self._application(app_id)
def set_application_status(self, app_id: str, status: str) -> Application:
"""Move one application to `status`. Raises ValueError for a status
outside `STATUSES` and KeyError for an unknown id."""
_check_status(status)
current = self._application(app_id)
return self.save_application(current.model_copy(update={"status": status}))
def application(self, app_id: str) -> Application | None:
row = self._row("SELECT * FROM applications WHERE id=?", (app_id,))
return Application.from_row(row) if row else None
def applications(self, status: str | None = None) -> list[Application]:
sql, params = "SELECT * FROM applications", []
if status:
sql += " WHERE status = ?"
params.append(status)
sql += " ORDER BY updated_at DESC, id"
return [Application.from_row(r) for r in self._rows(sql, params)]
def application_events(self, app_id: str) -> list[ApplicationEvent]:
return [ApplicationEvent.from_row(r) for r in self._rows(
"SELECT * FROM application_events WHERE application_id=? ORDER BY id",
(app_id,))]
def _application(self, app_id: str) -> Application:
found = self.application(app_id)
if found is None:
raise KeyError(app_id)
return found
# ---------------------------------------------------------------- state
@property
def state(self) -> MutableMapping[str, str]:
"""The `state` table as a mapping. What the runner hands a plugin as
`self.state`: it can keep a counter without being able to read roles."""
return _StateMap(self)
def _check_status(status: str) -> None:
if status not in STATUSES:
raise ValueError(f"{status!r} is not an application status "
f"({', '.join(STATUSES)})")
def _check_affinity(affinity: Any) -> None:
if not isinstance(affinity, int) or affinity not in AFFINITY:
raise ValueError(f"affinity is -1, 0, 1 or 2, not {affinity!r}")