Rename the package roledesk to app
The boilerplate's rule: the deploy kit's Dockerfile, the APP_CONFIG, APP_PASSWORD and APP_DATA variables and its commands all assume a package called app, so a project renames its display title and nothing else. The console script is `app`, and the launch entry runs `uv run app serve --port 8790`. The domain noun stays `role`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
248ea1636d
commit
7a63311714
32 files changed
+102
-98
No files matched your search
@@ -0,0 +1,16 @@
|
||||
"""Where things are kept. One door: `from app.storage import Store`.
|
||||
|
||||
schema.py the tables, and the numbered migrations that bring an older
|
||||
file to the current shape
|
||||
store.py `Store`, every SQL statement in the application
|
||||
|
||||
Written for roledesk rather than imported from newsdesk: `desk` has no storage
|
||||
package on purpose, and the connection plumbing here is a deliberate copy.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.storage.schema import LATEST
|
||||
from app.storage.store import Store
|
||||
|
||||
__all__ = ["LATEST", "Store"]
|
||||
@@ -0,0 +1,316 @@
|
||||
"""The tables, and how a database file gets from an old shape to this one.
|
||||
|
||||
`SCHEMA` is the current shape. A new file is created from it and stamped with
|
||||
`LATEST`, so it never runs a migration. `MIGRATIONS` is how an existing file
|
||||
catches up: ordered, numbered, applied once each, tracked in SQLite's own
|
||||
`PRAGMA user_version`.
|
||||
|
||||
Adding a column means editing both: the DDL in `SCHEMA` for new files, and an
|
||||
appended migration for old ones. Never renumber or edit a migration that has
|
||||
shipped.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SCHEMA = """
|
||||
-- One posting on one board. `uid` is sha1(source, source_id), the identity desk
|
||||
-- gives every Item. The gate fields are real columns, unpacked from the item's
|
||||
-- raw on upsert or filled later by extract: raw is transport, the columns are
|
||||
-- what gets queried.
|
||||
CREATE TABLE IF NOT EXISTS roles (
|
||||
uid TEXT PRIMARY KEY,
|
||||
source TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL,
|
||||
subscription_id INTEGER REFERENCES subscriptions(id) ON DELETE SET NULL,
|
||||
-- The cluster key is (company_key, title_key, place_key). No foreign key
|
||||
-- to companies: a role may arrive before anyone has made a company row.
|
||||
company_key TEXT NOT NULL DEFAULT '',
|
||||
company TEXT NOT NULL DEFAULT '',
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
title_key TEXT NOT NULL DEFAULT '',
|
||||
-- Geography. place and not location, which is where an Item's original
|
||||
-- lives. remote and country are separate so a gate on remote can also
|
||||
-- require the country: "UK | Remote" is not remote from Berlin.
|
||||
place TEXT NOT NULL DEFAULT '',
|
||||
place_key TEXT NOT NULL DEFAULT '',
|
||||
country TEXT NOT NULL DEFAULT '',
|
||||
remote TEXT NOT NULL DEFAULT ''
|
||||
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
|
||||
salary_min INTEGER,
|
||||
salary_max INTEGER,
|
||||
currency TEXT NOT NULL DEFAULT '',
|
||||
seniority TEXT NOT NULL DEFAULT '',
|
||||
function TEXT NOT NULL DEFAULT '',
|
||||
-- Whether the posting says it sponsors a work permit, and the languages
|
||||
-- (ISO 639-1, a JSON array) it says are required. Only extract fills these.
|
||||
permit TEXT NOT NULL DEFAULT ''
|
||||
CHECK (permit IN ('', 'sponsors', 'no_sponsorship')),
|
||||
languages TEXT NOT NULL DEFAULT '[]',
|
||||
description TEXT NOT NULL DEFAULT '', -- the posting's body text
|
||||
feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over
|
||||
original_url TEXT NOT NULL DEFAULT '', -- the posting page
|
||||
published_at TEXT NOT NULL, -- what the source claims, UTC
|
||||
-- What storing postings buys. first_seen is the only honest posted date,
|
||||
-- since Greenhouse resets updated_at on every edit. closed_at is set when a
|
||||
-- board fetched in full no longer lists the role, and cleared if it returns.
|
||||
first_seen TEXT NOT NULL,
|
||||
last_seen TEXT NOT NULL,
|
||||
closed_at TEXT,
|
||||
raw TEXT NOT NULL DEFAULT '{}',
|
||||
extracted_at TEXT, -- when extract last filled columns
|
||||
-- Replies from extract that left this role unanswered. At GIVE_UP_AFTER it
|
||||
-- is no longer offered. A column, not raw, because every upsert rewrites raw.
|
||||
extract_tries INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- A company as a target, whether or not any board of it is read. key is
|
||||
-- models.company_key(name), the slug roles.company_key holds.
|
||||
CREATE TABLE IF NOT EXISTS companies (
|
||||
key TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
domain TEXT NOT NULL DEFAULT '',
|
||||
-- -1 mute, 0 neutral, 1 interested, 2 favourite.
|
||||
affinity INTEGER NOT NULL DEFAULT 0 CHECK (affinity BETWEEN -1 AND 2),
|
||||
status TEXT NOT NULL DEFAULT 'researching',
|
||||
careers_url TEXT NOT NULL DEFAULT '',
|
||||
notes TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- The targets desk's runner hands to plugins. newsdesk's shape, with its two
|
||||
-- labels named company and board from the first migration.
|
||||
CREATE TABLE IF NOT EXISTS subscriptions (
|
||||
id INTEGER PRIMARY KEY,
|
||||
source TEXT NOT NULL, -- plugin name
|
||||
target TEXT NOT NULL, -- e.g. greenhouse:acme
|
||||
label TEXT NOT NULL, -- what Item.subscription names
|
||||
company TEXT NOT NULL DEFAULT '',
|
||||
board TEXT NOT NULL DEFAULT '',
|
||||
params TEXT NOT NULL DEFAULT '{}', -- what a query target asks with
|
||||
subscribed INTEGER NOT NULL DEFAULT 1,
|
||||
fetch INTEGER NOT NULL DEFAULT 1,
|
||||
show INTEGER NOT NULL DEFAULT 1,
|
||||
added_at TEXT NOT NULL,
|
||||
target_id TEXT NOT NULL DEFAULT '', -- the plugin's durable name for it
|
||||
UNIQUE (source, target),
|
||||
UNIQUE (source, label)
|
||||
);
|
||||
|
||||
-- The tracker, in jobhunt-hq's shape (MIT). role_uid is NULL for a role that
|
||||
-- was never posted, so an unsolicited approach is tracked like any other.
|
||||
CREATE TABLE IF NOT EXISTS applications (
|
||||
id TEXT PRIMARY KEY,
|
||||
role_uid TEXT REFERENCES roles(uid) ON DELETE SET NULL,
|
||||
company_key TEXT NOT NULL DEFAULT '',
|
||||
role_title TEXT NOT NULL DEFAULT '',
|
||||
url TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'saved'
|
||||
CHECK (status IN ('saved', 'applied', 'screen', 'technical',
|
||||
'onsite', 'offer', 'rejected', 'withdrawn')),
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
applied_at TEXT,
|
||||
next_action TEXT NOT NULL DEFAULT '',
|
||||
next_date TEXT,
|
||||
contact TEXT NOT NULL DEFAULT '',
|
||||
notes TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- One row per status an application entered, written by the store and never
|
||||
-- updated: what days-to-first-screen and conversion by source are read from.
|
||||
CREATE TABLE IF NOT EXISTS application_events (
|
||||
id INTEGER PRIMARY KEY,
|
||||
application_id TEXT NOT NULL REFERENCES applications(id) ON DELETE CASCADE,
|
||||
status TEXT NOT NULL,
|
||||
at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS application_events_append_only
|
||||
BEFORE UPDATE ON application_events
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'application_events is append-only');
|
||||
END;
|
||||
|
||||
-- A search agent: a named saved search with its own intent text, gates and
|
||||
-- weights. gates is a list of {field, values, mode: require|prefer, weight}.
|
||||
-- intent_vec caches the intent's vector; intent_key is sha1 of the embedding
|
||||
-- model and the intent text, so editing either makes it stale.
|
||||
CREATE TABLE IF NOT EXISTS agents (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
intent TEXT NOT NULL DEFAULT '',
|
||||
gates TEXT NOT NULL DEFAULT '[]',
|
||||
weights TEXT NOT NULL DEFAULT '{}',
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TEXT NOT NULL,
|
||||
intent_key TEXT NOT NULL DEFAULT '',
|
||||
intent_vec BLOB
|
||||
);
|
||||
|
||||
-- A role is scored per agent. A gated role keeps its row, with the reason.
|
||||
CREATE TABLE IF NOT EXISTS scores (
|
||||
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
||||
agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
|
||||
score REAL NOT NULL DEFAULT 0,
|
||||
gated_reason TEXT NOT NULL DEFAULT '',
|
||||
scored_at TEXT NOT NULL,
|
||||
PRIMARY KEY (uid, agent_id)
|
||||
);
|
||||
|
||||
-- One vector per role, little-endian float32. Only comparable to another from
|
||||
-- the same model, so every read is scoped by it.
|
||||
CREATE TABLE IF NOT EXISTS vectors (
|
||||
uid TEXT PRIMARY KEY REFERENCES roles(uid) ON DELETE CASCADE,
|
||||
model TEXT NOT NULL,
|
||||
dim INTEGER NOT NULL,
|
||||
vec BLOB NOT NULL,
|
||||
embedded_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- The string map a desk plugin is handed as self.state. One namespace shared
|
||||
-- by every plugin, so a plugin prefixes its keys.
|
||||
CREATE TABLE IF NOT EXISTS state (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
"""
|
||||
|
||||
#: Applied after `SCHEMA` or the migrations, every open. An index may name a
|
||||
#: column only a migration adds, which is why these are kept apart.
|
||||
INDEXES = """
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_company ON roles(company_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_place ON roles(place_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_first_seen ON roles(first_seen DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_closed ON roles(closed_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_cluster ON roles(company_key, title_key, place_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_board ON roles(source, subscription_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_application_events
|
||||
ON application_events(application_id, at);
|
||||
CREATE INDEX IF NOT EXISTS idx_scores_agent ON scores(agent_id, score DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_vectors_model ON vectors(model);
|
||||
"""
|
||||
|
||||
#: The table whose absence marks a file that has never been a roledesk
|
||||
#: database, and is therefore created from `SCHEMA` rather than migrated.
|
||||
MARKER = "roles"
|
||||
|
||||
Step = str | Callable[[sqlite3.Connection], None]
|
||||
|
||||
MIGRATIONS: tuple[tuple[int, str, Step], ...] = (
|
||||
# Never run as a migration: a file without the marker table is created
|
||||
# from `SCHEMA` and stamped. It exists so the first real migration is 2.
|
||||
(1, "roles, companies, subscriptions, applications, agents, scores, state", ""),
|
||||
(2, "roles.permit, roles.languages and roles.extract_tries, for extract", """
|
||||
ALTER TABLE roles ADD COLUMN permit TEXT NOT NULL DEFAULT ''
|
||||
CHECK (permit IN ('', 'sponsors', 'no_sponsorship'));
|
||||
ALTER TABLE roles ADD COLUMN languages TEXT NOT NULL DEFAULT '[]';
|
||||
ALTER TABLE roles ADD COLUMN extract_tries INTEGER NOT NULL DEFAULT 0;
|
||||
"""),
|
||||
(3, "vectors, and the agent's cached intent vector, for score", """
|
||||
CREATE TABLE vectors (
|
||||
uid TEXT PRIMARY KEY REFERENCES roles(uid) ON DELETE CASCADE,
|
||||
model TEXT NOT NULL,
|
||||
dim INTEGER NOT NULL,
|
||||
vec BLOB NOT NULL,
|
||||
embedded_at TEXT NOT NULL
|
||||
);
|
||||
ALTER TABLE agents ADD COLUMN intent_key TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE agents ADD COLUMN intent_vec BLOB;
|
||||
"""),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
LATEST = MIGRATIONS[-1][0]
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
def _stamp(conn: sqlite3.Connection, version: int) -> None:
|
||||
# PRAGMA takes no bound parameters; the value is an int from MIGRATIONS.
|
||||
conn.execute(f"PRAGMA user_version = {int(version)}")
|
||||
|
||||
|
||||
def _statements(script: str) -> list[str]:
|
||||
"""A script split into statements, cut where `sqlite3.complete_statement`
|
||||
says one ends, which knows about comments and trigger bodies. Not
|
||||
`executescript`, which commits before it starts and autocommits each
|
||||
statement, so a failing migration would be half applied."""
|
||||
statements, buffer = [], ""
|
||||
for line in script.splitlines(keepends=True):
|
||||
buffer += line
|
||||
if sqlite3.complete_statement(buffer):
|
||||
statements.append(buffer.strip())
|
||||
buffer = ""
|
||||
if buffer.strip():
|
||||
statements.append(buffer.strip())
|
||||
return statements
|
||||
|
||||
|
||||
def migrate(conn: sqlite3.Connection) -> list[int]:
|
||||
"""Bring an existing database up to `LATEST`. Returns what it applied."""
|
||||
current = user_version(conn)
|
||||
if current > LATEST:
|
||||
raise RuntimeError(
|
||||
f"this database is at schema v{current}, but this roledesk only knows "
|
||||
f"up to v{LATEST}. Upgrade roledesk rather than let older code write to it.")
|
||||
applied = []
|
||||
for version, description, step in MIGRATIONS:
|
||||
if version <= current:
|
||||
continue
|
||||
log.info("migrating database to v%d: %s", version, description)
|
||||
_apply(conn, version, step)
|
||||
applied.append(version)
|
||||
return applied
|
||||
|
||||
|
||||
def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
|
||||
"""One migration and its version stamp, in one explicit transaction.
|
||||
|
||||
Explicit because sqlite3 only opens a transaction implicitly before DML: an
|
||||
`ALTER` would otherwise be durable at once while the stamp stayed behind,
|
||||
and the next open would replay it into a "duplicate column" forever.
|
||||
"""
|
||||
started = not conn.in_transaction
|
||||
if started:
|
||||
conn.execute("BEGIN")
|
||||
try:
|
||||
if callable(step):
|
||||
step(conn)
|
||||
else:
|
||||
for statement in _statements(step):
|
||||
conn.execute(statement)
|
||||
_stamp(conn, version)
|
||||
except Exception:
|
||||
if started:
|
||||
conn.execute("ROLLBACK")
|
||||
raise
|
||||
if started:
|
||||
conn.execute("COMMIT")
|
||||
|
||||
|
||||
def initialise(conn: sqlite3.Connection) -> list[int]:
|
||||
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
|
||||
fresh = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (MARKER,)
|
||||
).fetchone() is None
|
||||
applied: list[int] = []
|
||||
if fresh:
|
||||
# Atomic like a migration: a file half created would otherwise carry
|
||||
# the marker table and be taken for one to migrate.
|
||||
_apply(conn, LATEST, SCHEMA)
|
||||
else:
|
||||
applied = migrate(conn)
|
||||
for statement in _statements(INDEXES):
|
||||
conn.execute(statement)
|
||||
return applied
|
||||
@@ -0,0 +1,783 @@
|
||||
"""`Store`: every SQL statement in the application.
|
||||
|
||||
The connection plumbing is a copy of newsdesk's `Store`, on purpose: `desk`
|
||||
has no storage package. A connection is opened per call and closed again, so
|
||||
one caller's commit can never land inside another's transaction; in WAL mode
|
||||
that costs a file handle.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from collections.abc import Callable, Iterator, MutableMapping, Sequence
|
||||
from contextlib import closing, contextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import numpy as np
|
||||
|
||||
from desk.models import Item
|
||||
from app.models import (
|
||||
AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate,
|
||||
Company, Gate, Role, ScoredRole, Subscription, Weights, company_key, place_key,
|
||||
title_key,
|
||||
)
|
||||
from app.storage import schema
|
||||
|
||||
#: Text columns a plugin may fill from `Item.raw` under the same name. An
|
||||
#: empty value never overwrites a filled column, so an upsert from a board that
|
||||
#: says less than `extract` found keeps what `extract` found.
|
||||
_RAW_TEXT = ("place", "country", "currency", "seniority", "function")
|
||||
_RAW_INT = ("salary_min", "salary_max")
|
||||
|
||||
#: What `extract` may fill. Only an empty column is written: a board's own
|
||||
#: structured field wins over what a model read out of the prose, and the next
|
||||
#: upsert from that board would put it back anyway.
|
||||
_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "function",
|
||||
"permit")
|
||||
|
||||
#: How many replies may leave a role unanswered before `extract` stops
|
||||
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
|
||||
#: never answers is otherwise paid for on every pull for as long as it exists.
|
||||
GIVE_UP_AFTER = 3
|
||||
|
||||
#: A role `extract` has yet to read: never answered, still open, not given up.
|
||||
_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL "
|
||||
f"AND extract_tries < {GIVE_UP_AFTER}")
|
||||
|
||||
_COMPANY_FIELDS = ("name", "domain", "affinity", "status", "careers_url", "notes")
|
||||
_SUBSCRIPTION_FIELDS = ("label", "company", "board", "params", "subscribed", "fetch",
|
||||
"show", "target_id")
|
||||
|
||||
|
||||
def _iso(moment: datetime) -> str:
|
||||
"""UTC, to the second. Every timestamp column is compared as text, so every
|
||||
writer has to spell it the same way."""
|
||||
return moment.astimezone(UTC).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def _remote(value: Any) -> str:
|
||||
"""A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything
|
||||
unrecognised is '' (unknown), which `extract` may fill later."""
|
||||
if value is True:
|
||||
return "remote"
|
||||
text = str(value or "").strip().lower().replace("-", "").replace(" ", "")
|
||||
text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text)
|
||||
return text if text in REMOTE else ""
|
||||
|
||||
|
||||
def _like(text: str) -> str:
|
||||
"""A `LIKE` pattern for a case-insensitive substring match, with SQLite's
|
||||
own wildcards escaped so a search for "50%" or "a_b" stays literal."""
|
||||
escaped = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
return f"%{escaped}%"
|
||||
|
||||
|
||||
#: Vectors are stored little-endian float32, fixed here so a file written on
|
||||
#: one machine reads on another. newsdesk's encoding, written again.
|
||||
_DTYPE = np.dtype("<f4")
|
||||
|
||||
|
||||
def pack(vec: np.ndarray) -> bytes:
|
||||
return np.asarray(vec, dtype=_DTYPE).tobytes()
|
||||
|
||||
|
||||
def unpack(blob: bytes) -> np.ndarray:
|
||||
return np.frombuffer(blob, dtype=_DTYPE)
|
||||
|
||||
|
||||
def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
|
||||
"""One gate as `(empty, match, params)`: SQL that is true when the role's
|
||||
column is empty, and SQL that is true when it matches, over `roles r`.
|
||||
`params` belong to `match`; `empty` takes none."""
|
||||
column = f"r.{gate.field}"
|
||||
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
|
||||
marks = ", ".join("?" for _ in values) or "NULL"
|
||||
if gate.field == "salary_min":
|
||||
floor = min(_int_or_none(v) or 0 for v in gate.values)
|
||||
return f"{column} IS NULL", f"{column} >= ?", [floor]
|
||||
if gate.field == "place":
|
||||
likes = " OR ".join("LOWER(r.place) LIKE ? ESCAPE '\\'" for _ in values) or "0"
|
||||
return "r.place = ''", f"({likes})", [_like(v) for v in values]
|
||||
if gate.field == "company":
|
||||
keys = [company_key(v) for v in values]
|
||||
return ("r.company_key = ''",
|
||||
f"r.company_key IN ({', '.join('?' for _ in keys) or 'NULL'})", keys)
|
||||
if gate.field == "languages":
|
||||
return ("r.languages IN ('', '[]')",
|
||||
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
|
||||
f"WHERE LOWER(value) NOT IN ({marks}))", values)
|
||||
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
|
||||
|
||||
|
||||
def _int_or_none(value: Any) -> int | None:
|
||||
if value is None or value == "" or isinstance(value, bool):
|
||||
return None
|
||||
try:
|
||||
return int(float(value))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
class _StateMap(MutableMapping[str, str]):
|
||||
"""`Store.state`: the key/value table, read and written one key at a time."""
|
||||
|
||||
def __init__(self, store: Store) -> None:
|
||||
self._store = store
|
||||
|
||||
def __getitem__(self, key: str) -> str:
|
||||
row = self._store._row("SELECT value FROM state WHERE key=?", (key,))
|
||||
if row is None:
|
||||
raise KeyError(key)
|
||||
return str(row["value"])
|
||||
|
||||
def __setitem__(self, key: str, value: str) -> None:
|
||||
self._store._write(
|
||||
"""INSERT INTO state (key, value) VALUES (?,?)
|
||||
ON CONFLICT(key) DO UPDATE SET value=excluded.value""",
|
||||
(key, value))
|
||||
|
||||
def __delitem__(self, key: str) -> None:
|
||||
if not self._store._write("DELETE FROM state WHERE key=?", (key,)):
|
||||
raise KeyError(key)
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
return iter([r["key"] for r in self._store._rows("SELECT key FROM state ORDER BY key")])
|
||||
|
||||
def __len__(self) -> int:
|
||||
row = self._store._row("SELECT COUNT(*) AS n FROM state")
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, path: str | Path) -> None:
|
||||
self.path = Path(path).expanduser()
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
#: What "now" is. Replaced by tests that need two writes a second apart.
|
||||
self.clock: Callable[[], datetime] = lambda: datetime.now(UTC)
|
||||
with self.transaction() as conn:
|
||||
# Persistent: recorded in the file, inherited by every connection.
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
#: Migration versions this open applied; empty when already current.
|
||||
self.migrations_applied: list[int] = schema.initialise(conn)
|
||||
|
||||
# ------------------------------------------------------------ connections
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
"""A fresh connection. `foreign_keys` is off by default per connection;
|
||||
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
|
||||
`synchronous=NORMAL` is the standard WAL companion."""
|
||||
conn = sqlite3.connect(self.path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
conn.execute("PRAGMA busy_timeout=30000")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
return conn
|
||||
|
||||
@contextmanager
|
||||
def transaction(self) -> Iterator[sqlite3.Connection]:
|
||||
"""One connection, one transaction, closed on the way out."""
|
||||
conn = self._connect()
|
||||
try:
|
||||
with conn:
|
||||
yield conn
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]:
|
||||
with closing(self._connect()) as conn:
|
||||
return conn.execute(sql, params).fetchall()
|
||||
|
||||
def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None:
|
||||
with closing(self._connect()) as conn:
|
||||
row: sqlite3.Row | None = conn.execute(sql, params).fetchone()
|
||||
return row
|
||||
|
||||
def _write(self, sql: str, params: Any = ()) -> int:
|
||||
"""One statement in its own transaction. Returns the rowcount."""
|
||||
with self.transaction() as conn:
|
||||
return conn.execute(sql, params).rowcount
|
||||
|
||||
def now(self) -> str:
|
||||
return _iso(self.clock())
|
||||
|
||||
# ---------------------------------------------------------------- roles
|
||||
|
||||
def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool:
|
||||
"""Store one posting. Returns True when it was new.
|
||||
|
||||
`first_seen` and `created_at` are written on insert only; `last_seen` on
|
||||
every call, and a role seen again is open again. The gate columns come
|
||||
from `item.raw` under their own names, with `company` falling back to
|
||||
the subscription's; an empty value never overwrites a filled column.
|
||||
"""
|
||||
if not item.source:
|
||||
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
|
||||
raw = item.raw or {}
|
||||
company = str(raw.get("company") or (subscription.company if subscription else ""))
|
||||
place = str(raw.get("place") or "")
|
||||
stamp = self.now()
|
||||
row: dict[str, Any] = {
|
||||
"uid": item.uid,
|
||||
"source": item.source,
|
||||
"source_id": item.source_id,
|
||||
"subscription_id": subscription.id if subscription else None,
|
||||
"company_key": company_key(company),
|
||||
"company": company,
|
||||
"title": item.title,
|
||||
"title_key": title_key(item.title),
|
||||
"place_key": place_key(place),
|
||||
"remote": _remote(raw.get("remote")),
|
||||
"description": item.body,
|
||||
"feed_text": item.excerpt,
|
||||
"original_url": item.location,
|
||||
"published_at": _iso(item.published_at),
|
||||
"first_seen": stamp,
|
||||
"last_seen": stamp,
|
||||
"raw": json.dumps(raw, default=str),
|
||||
"created_at": stamp,
|
||||
}
|
||||
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
|
||||
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
||||
|
||||
keep_filled = ("company", "company_key", "place_key", "remote", "description",
|
||||
"feed_text", "original_url", *_RAW_TEXT)
|
||||
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
|
||||
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
|
||||
for c in (*_RAW_INT, "subscription_id")]
|
||||
updates += [f"{c}=excluded.{c}"
|
||||
for c in ("title", "title_key", "published_at", "raw", "last_seen")]
|
||||
updates.append("closed_at=NULL")
|
||||
columns = ", ".join(row)
|
||||
marks = ", ".join(f":{c}" for c in row)
|
||||
with self.transaction() as conn:
|
||||
existed = conn.execute("SELECT 1 FROM roles WHERE uid=?",
|
||||
(row["uid"],)).fetchone() is not None
|
||||
conn.execute(
|
||||
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
|
||||
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
|
||||
return not existed
|
||||
|
||||
def role(self, uid: str) -> Role | None:
|
||||
row = self._row("SELECT * FROM roles WHERE uid=?", (uid,))
|
||||
return Role.from_row(row) if row else None
|
||||
|
||||
def roles(self, limit: int = 100, company: str | None = None,
|
||||
place: str | None = None, open_only: bool = True,
|
||||
q: str | None = None, function: str | None = None,
|
||||
remote: str | None = None, days: int | None = None,
|
||||
hide_muted: bool = False, agent: int | None = None,
|
||||
gated: bool = False) -> list[ScoredRole]:
|
||||
"""Newest first by `first_seen`. `company` and `place` are
|
||||
case-insensitive substrings of the columns a board wrote — not the
|
||||
key, so a search does not have to know the exact slug a company folds
|
||||
to. `q` is matched word by word against `title`; a role must match
|
||||
every word. `function` and `remote` match a gate column exactly.
|
||||
`days` keeps only roles first seen within that many days of now.
|
||||
|
||||
`hide_muted` excludes a muted company's roles before `limit` is
|
||||
applied, in SQL rather than after the fact: a company that has
|
||||
published a lot, all at once — a fresh board added, a re-pull that
|
||||
finally reached it — can otherwise fill the whole page with roles
|
||||
that then all turn out to be from the one company someone muted,
|
||||
leaving nothing to show for a default query that would otherwise
|
||||
have plenty.
|
||||
|
||||
`agent` orders by that agent's score instead, highest first, and
|
||||
carries `score` and `gated_reason` on each row. Only roles it has
|
||||
scored are listed, and a gated one only when `gated` is set."""
|
||||
where, params = [], []
|
||||
if company:
|
||||
where.append("company LIKE ? ESCAPE '\\'")
|
||||
params.append(_like(company))
|
||||
if place:
|
||||
where.append("place LIKE ? ESCAPE '\\'")
|
||||
params.append(_like(place))
|
||||
if function:
|
||||
where.append("function = ?")
|
||||
params.append(function)
|
||||
if remote:
|
||||
where.append("remote = ?")
|
||||
params.append(remote)
|
||||
if days is not None:
|
||||
where.append("first_seen >= ?")
|
||||
params.append(_iso(self.clock() - timedelta(days=days)))
|
||||
for word in (q or "").split():
|
||||
where.append("title LIKE ? ESCAPE '\\'")
|
||||
params.append(_like(word))
|
||||
if open_only:
|
||||
where.append("closed_at IS NULL")
|
||||
if hide_muted:
|
||||
where.append("company_key NOT IN "
|
||||
"(SELECT key FROM companies WHERE affinity = -1)")
|
||||
joins: list[Any] = []
|
||||
if agent is None:
|
||||
sql = "SELECT roles.* FROM roles"
|
||||
order = "roles.first_seen DESC, roles.uid"
|
||||
else:
|
||||
sql = ("SELECT roles.*, s.score, s.gated_reason FROM roles "
|
||||
"JOIN scores s ON s.uid = roles.uid AND s.agent_id = ?")
|
||||
joins.append(agent)
|
||||
if not gated:
|
||||
where.append("s.gated_reason = ''")
|
||||
order = "s.score DESC, roles.first_seen DESC, roles.uid"
|
||||
if where:
|
||||
sql += " WHERE " + " AND ".join(where)
|
||||
sql += f" ORDER BY {order} LIMIT ?"
|
||||
return [ScoredRole.from_row(r)
|
||||
for r in self._rows(sql, [*joins, *params, limit])]
|
||||
|
||||
def mark_closed(self, source: str, seen_ids: Sequence[str],
|
||||
subscription_id: int) -> int:
|
||||
"""Close every open role of one board that a complete fetch of it did not
|
||||
list. Returns how many closed. Only call it after a fetch that finished:
|
||||
a failed one lists nothing, and would close the whole board."""
|
||||
return self._write(
|
||||
"""UPDATE roles SET closed_at = ?
|
||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||
AND source_id NOT IN (SELECT value FROM json_each(?))""",
|
||||
(self.now(), source, subscription_id, json.dumps(list(seen_ids))))
|
||||
|
||||
# -------------------------------------------------------------- extract
|
||||
|
||||
def extract_pending(self) -> int:
|
||||
row = self._row(f"SELECT COUNT(*) AS n FROM roles WHERE {_EXTRACT_WAITING}")
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
def extract_waiting(self, limit: int,
|
||||
uids: Sequence[str] | None = None) -> list[Role]:
|
||||
"""Roles `extract` has yet to read, newest first. Given `uids`, only
|
||||
those of them still waiting: a batch asks again when it runs, since a
|
||||
role may have been answered since the task shared the work out."""
|
||||
sql, params = f"SELECT * FROM roles WHERE {_EXTRACT_WAITING}", []
|
||||
if uids is not None:
|
||||
sql += " AND uid IN (SELECT value FROM json_each(?))"
|
||||
params.append(json.dumps(list(uids)))
|
||||
sql += " ORDER BY first_seen DESC, uid LIMIT ?"
|
||||
return [Role.from_row(r) for r in self._rows(sql, [*params, limit])]
|
||||
|
||||
def set_extracted(self, uid: str, fields: dict[str, Any]) -> int | None:
|
||||
"""Write what `extract` read for one role and stamp `extracted_at`.
|
||||
|
||||
Only into empty columns, and only a non-empty value: a board's own
|
||||
field is never replaced and nothing is ever blanked. `place` brings
|
||||
`place_key` with it. Returns how many columns were filled, or None
|
||||
for an unknown uid. Values are trusted to be in vocabulary; the
|
||||
job's `apply` is where a reply is checked."""
|
||||
with self.transaction() as conn:
|
||||
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
sets: dict[str, Any] = {}
|
||||
for name in _EXTRACT_TEXT:
|
||||
value = str(fields.get(name) or "")
|
||||
if value and not row[name]:
|
||||
sets[name] = value
|
||||
for name in _RAW_INT:
|
||||
number = _int_or_none(fields.get(name))
|
||||
if number is not None and row[name] is None:
|
||||
sets[name] = number
|
||||
languages = [str(code) for code in fields.get("languages") or []]
|
||||
if languages and json.loads(row["languages"] or "[]") == []:
|
||||
sets["languages"] = json.dumps(languages)
|
||||
filled = len(sets)
|
||||
if "place" in sets:
|
||||
sets["place_key"] = place_key(sets["place"])
|
||||
sets["extracted_at"] = self.now()
|
||||
conn.execute(
|
||||
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
|
||||
(*sets.values(), uid))
|
||||
return filled
|
||||
|
||||
def extract_missed(self, uids: Sequence[str], why: str = "") -> int:
|
||||
"""One reply that left these roles unanswered. Counted only against
|
||||
those still waiting, so a role the reply did answer is not charged.
|
||||
Returns how many this pushed to `GIVE_UP_AFTER`. `why` is accepted
|
||||
for `ModelTask.missed` and not kept: there is no Given up list yet."""
|
||||
if not uids:
|
||||
return 0
|
||||
where = f"uid IN (SELECT value FROM json_each(?)) AND {_EXTRACT_WAITING}"
|
||||
wanted = json.dumps(list(uids))
|
||||
with self.transaction() as conn:
|
||||
crossed = conn.execute(
|
||||
f"SELECT COUNT(*) FROM roles WHERE {where} AND extract_tries + 1 >= ?",
|
||||
(wanted, GIVE_UP_AFTER)).fetchone()[0]
|
||||
conn.execute(f"UPDATE roles SET extract_tries = extract_tries + 1 "
|
||||
f"WHERE {where}", (wanted,))
|
||||
return int(crossed)
|
||||
|
||||
# -------------------------------------------------------------- vectors
|
||||
|
||||
def roles_to_embed(self, model: str, limit: int) -> list[Role]:
|
||||
"""Open roles with no vector from `model`, newest first, so a backlog
|
||||
embedded from the top improves what is on screen first."""
|
||||
return [Role.from_row(r) for r in self._rows(
|
||||
"""SELECT r.* FROM roles r
|
||||
LEFT JOIN vectors v ON v.uid = r.uid AND v.model = ?
|
||||
WHERE v.uid IS NULL AND r.closed_at IS NULL
|
||||
ORDER BY r.first_seen DESC, r.uid LIMIT ?""", (model, limit))]
|
||||
|
||||
def put_vectors(self, model: str, pairs: Sequence[tuple[str, np.ndarray]]) -> int:
|
||||
stamp = self.now()
|
||||
with self.transaction() as conn:
|
||||
return conn.executemany(
|
||||
"""INSERT INTO vectors (uid, model, dim, vec, embedded_at)
|
||||
VALUES (?,?,?,?,?)
|
||||
ON CONFLICT(uid) DO UPDATE SET model=excluded.model,
|
||||
dim=excluded.dim, vec=excluded.vec,
|
||||
embedded_at=excluded.embedded_at""",
|
||||
[(uid, model, int(vec.shape[-1]), pack(vec), stamp)
|
||||
for uid, vec in pairs]).rowcount
|
||||
|
||||
def vectors(self, model: str) -> dict[str, np.ndarray]:
|
||||
"""Every vector from `model`, by uid. The whole table: at ~30k roles of
|
||||
384 floats that is ~45 MB, and one read beats one per agent."""
|
||||
return {r["uid"]: unpack(r["vec"]) for r in self._rows(
|
||||
"SELECT uid, vec FROM vectors WHERE model = ?", (model,))}
|
||||
|
||||
def vector_counts(self) -> dict[str, int]:
|
||||
return {r["model"]: int(r["n"]) for r in self._rows(
|
||||
"SELECT model, COUNT(*) AS n FROM vectors GROUP BY model")}
|
||||
|
||||
def drop_vectors(self, keep: str) -> int:
|
||||
"""Delete vectors from every model but `keep`: another model's are a
|
||||
different geometry, never compared, and only make counts lie."""
|
||||
return self._write("DELETE FROM vectors WHERE model <> ?", (keep,))
|
||||
|
||||
# --------------------------------------------------------------- agents
|
||||
|
||||
def agents(self, enabled_only: bool = False) -> list[Agent]:
|
||||
sql = "SELECT * FROM agents"
|
||||
if enabled_only:
|
||||
sql += " WHERE enabled = 1"
|
||||
return [Agent.from_row(r)
|
||||
for r in self._rows(sql + " ORDER BY name COLLATE NOCASE")]
|
||||
|
||||
def agent(self, agent_id: int) -> Agent | None:
|
||||
row = self._row("SELECT * FROM agents WHERE id=?", (agent_id,))
|
||||
return Agent.from_row(row) if row else None
|
||||
|
||||
def save_agent(self, name: str, intent: str = "", gates: Sequence[Gate] = (),
|
||||
weights: Weights | None = None, enabled: bool = True,
|
||||
agent_id: int | None = None) -> Agent:
|
||||
"""Create an agent, or with `agent_id` replace that one's fields.
|
||||
Raises ValueError on an empty or taken name, KeyError on an unknown id."""
|
||||
name = name.strip()
|
||||
if not name:
|
||||
raise ValueError("an agent needs a name")
|
||||
fields = (name, intent, json.dumps([g.model_dump() for g in gates]),
|
||||
(weights or Weights()).model_dump_json(), int(enabled))
|
||||
try:
|
||||
with self.transaction() as conn:
|
||||
if agent_id is None:
|
||||
agent_id = conn.execute(
|
||||
"""INSERT INTO agents (name, intent, gates, weights, enabled,
|
||||
created_at) VALUES (?,?,?,?,?,?)""",
|
||||
(*fields, self.now())).lastrowid
|
||||
elif not conn.execute(
|
||||
"""UPDATE agents SET name=?, intent=?, gates=?, weights=?,
|
||||
enabled=? WHERE id=?""",
|
||||
(*fields, agent_id)).rowcount:
|
||||
raise KeyError(agent_id)
|
||||
except sqlite3.IntegrityError as exc:
|
||||
raise ValueError(f"an agent named {name!r} already exists") from exc
|
||||
assert agent_id is not None
|
||||
found = self.agent(agent_id)
|
||||
assert found is not None
|
||||
return found
|
||||
|
||||
def delete_agent(self, agent_id: int) -> bool:
|
||||
"""Its scores go with it, by the foreign key."""
|
||||
return self._write("DELETE FROM agents WHERE id=?", (agent_id,)) > 0
|
||||
|
||||
def intent_vector(self, agent_id: int, key: str) -> np.ndarray | None:
|
||||
"""The agent's cached intent vector, if it was made under `key`."""
|
||||
row = self._row("SELECT intent_key, intent_vec FROM agents WHERE id=?",
|
||||
(agent_id,))
|
||||
if row is None or row["intent_key"] != key or row["intent_vec"] is None:
|
||||
return None
|
||||
return unpack(row["intent_vec"])
|
||||
|
||||
def set_intent_vector(self, agent_id: int, key: str, vec: np.ndarray) -> None:
|
||||
self._write("UPDATE agents SET intent_key=?, intent_vec=? WHERE id=?",
|
||||
(key, pack(vec), agent_id))
|
||||
|
||||
# --------------------------------------------------------------- scores
|
||||
|
||||
def score_candidates(self, gates: Sequence[Gate]) -> list[Candidate]:
|
||||
"""Every open role of a company that is not muted, with how it meets
|
||||
each gate: 1 match, 0 no match, -1 the column is empty. The gates are
|
||||
evaluated here, in SQL, so a `require` gate is the `WHERE` it reads
|
||||
as, written as a column so a gated role keeps its row and reason."""
|
||||
columns: list[str] = []
|
||||
params: list[Any] = []
|
||||
for n, gate in enumerate(gates):
|
||||
empty, match, values = _gate_sql(gate)
|
||||
columns.append(
|
||||
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
|
||||
params += values
|
||||
select = ", ".join(["r.uid", "r.first_seen",
|
||||
"COALESCE(c.affinity, 0) AS affinity", *columns])
|
||||
rows = self._rows(
|
||||
f"""SELECT {select} FROM roles r
|
||||
LEFT JOIN companies c ON c.key = r.company_key
|
||||
WHERE r.closed_at IS NULL AND COALESCE(c.affinity, 0) <> -1""",
|
||||
params)
|
||||
return [Candidate(uid=r["uid"], first_seen=r["first_seen"],
|
||||
affinity=int(r["affinity"]),
|
||||
gates=[int(r[f"g{n}"]) for n in range(len(gates))])
|
||||
for r in rows]
|
||||
|
||||
def replace_scores(self, agent_id: int,
|
||||
rows: Sequence[tuple[str, float, str]]) -> int:
|
||||
"""`(uid, score, gated_reason)` for every role this agent scored,
|
||||
replacing whatever it had: a role closed or muted since the last run
|
||||
drops out rather than keeping a stale score. One transaction, so a
|
||||
reader sees the old list or the new one."""
|
||||
stamp = self.now()
|
||||
with self.transaction() as conn:
|
||||
conn.execute("DELETE FROM scores WHERE agent_id=?", (agent_id,))
|
||||
conn.executemany(
|
||||
"""INSERT INTO scores (uid, agent_id, score, gated_reason, scored_at)
|
||||
VALUES (?,?,?,?,?)""",
|
||||
[(uid, agent_id, score, reason, stamp) for uid, score, reason in rows])
|
||||
return len(rows)
|
||||
|
||||
def scores(self, agent_id: int) -> dict[str, tuple[float, str]]:
|
||||
"""`uid -> (score, gated_reason)` for one agent."""
|
||||
return {r["uid"]: (float(r["score"]), str(r["gated_reason"]))
|
||||
for r in self._rows(
|
||||
"SELECT uid, score, gated_reason FROM scores WHERE agent_id=?",
|
||||
(agent_id,))}
|
||||
|
||||
# ------------------------------------------------------------ companies
|
||||
|
||||
def add_company(self, name: str, *, key: str = "", domain: str = "",
|
||||
affinity: int = 0, status: str = "researching",
|
||||
careers_url: str = "", notes: str = "") -> Company:
|
||||
"""Raises ValueError on a key already taken or an affinity out of range."""
|
||||
name = name.strip()
|
||||
key = key or company_key(name)
|
||||
if not key:
|
||||
raise ValueError("a company needs a name")
|
||||
_check_affinity(affinity)
|
||||
stamp = self.now()
|
||||
try:
|
||||
self._write(
|
||||
"""INSERT INTO companies (key, name, domain, affinity, status,
|
||||
careers_url, notes, created_at, updated_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?)""",
|
||||
(key, name, domain, affinity, status, careers_url, notes, stamp, stamp))
|
||||
except sqlite3.IntegrityError as exc:
|
||||
raise ValueError(f"company {key!r} already exists") from exc
|
||||
return self._company(key)
|
||||
|
||||
def company(self, key: str) -> Company | None:
|
||||
row = self._row("SELECT * FROM companies WHERE key=?", (key,))
|
||||
return Company.from_row(row) if row else None
|
||||
|
||||
def companies(self) -> list[Company]:
|
||||
return [Company.from_row(r)
|
||||
for r in self._rows("SELECT * FROM companies ORDER BY name COLLATE NOCASE")]
|
||||
|
||||
def update_company(self, key: str, **fields: Any) -> Company:
|
||||
"""Change any of name, domain, affinity, status, careers_url, notes."""
|
||||
unknown = set(fields) - set(_COMPANY_FIELDS)
|
||||
if unknown:
|
||||
raise ValueError(f"not a company field: {', '.join(sorted(unknown))}")
|
||||
if "affinity" in fields:
|
||||
_check_affinity(fields["affinity"])
|
||||
if fields:
|
||||
sets = ", ".join(f"{f}=?" for f in fields)
|
||||
changed = self._write(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?",
|
||||
(*fields.values(), self.now(), key))
|
||||
if not changed:
|
||||
raise KeyError(key)
|
||||
return self._company(key)
|
||||
|
||||
def remove_company(self, key: str) -> bool:
|
||||
return self._write("DELETE FROM companies WHERE key=?", (key,)) > 0
|
||||
|
||||
def _company(self, key: str) -> Company:
|
||||
found = self.company(key)
|
||||
if found is None:
|
||||
raise KeyError(key)
|
||||
return found
|
||||
|
||||
def open_role_counts(self) -> dict[str, int]:
|
||||
"""`company_key -> open role count`, for the company list's row
|
||||
count. A key with no open roles is simply absent."""
|
||||
return {str(r["company_key"]): int(r["n"]) for r in self._rows(
|
||||
"SELECT company_key, COUNT(*) AS n FROM roles "
|
||||
"WHERE closed_at IS NULL GROUP BY company_key")}
|
||||
|
||||
def company_boards(self) -> dict[str, str]:
|
||||
"""`company_key -> ats host`, read off each company's own
|
||||
subscription. `Subscription.company` is the display name it and the
|
||||
company row were both added with, so `company_key` of one finds the
|
||||
other; a company added by hand with no subscription is simply
|
||||
absent."""
|
||||
boards: dict[str, str] = {}
|
||||
for sub in self.subscriptions(source="ats"):
|
||||
boards.setdefault(company_key(sub.company), sub.board)
|
||||
return boards
|
||||
|
||||
# -------------------------------------------------------- subscriptions
|
||||
|
||||
def subscriptions(self, source: str | None = None,
|
||||
fetchable: bool = False) -> list[Subscription]:
|
||||
"""`fetchable` is what a pull reads: subscribed and with fetch on."""
|
||||
where, params = [], []
|
||||
if source:
|
||||
where.append("source = ?")
|
||||
params.append(source)
|
||||
if fetchable:
|
||||
where.append("fetch = 1 AND subscribed = 1")
|
||||
sql = "SELECT * FROM subscriptions"
|
||||
if where:
|
||||
sql += " WHERE " + " AND ".join(where)
|
||||
sql += " ORDER BY company, board, label"
|
||||
return [Subscription.from_row(r) for r in self._rows(sql, params)]
|
||||
|
||||
def subscription(self, sub_id: int) -> Subscription | None:
|
||||
row = self._row("SELECT * FROM subscriptions WHERE id=?", (sub_id,))
|
||||
return Subscription.from_row(row) if row else None
|
||||
|
||||
def subscription_for(self, source: str, label: str) -> Subscription | None:
|
||||
"""The row an item names: `Item.subscription` is the label."""
|
||||
row = self._row("SELECT * FROM subscriptions WHERE source=? AND label=?",
|
||||
(source, label))
|
||||
return Subscription.from_row(row) if row else None
|
||||
|
||||
def add_subscription(self, source: str, target: str, label: str = "",
|
||||
company: str = "", board: str = "",
|
||||
params: dict[str, Any] | None = None,
|
||||
target_id: str = "") -> Subscription:
|
||||
"""Raises ValueError on a duplicate target or label within one plugin."""
|
||||
target = target.strip()
|
||||
if not target:
|
||||
raise ValueError("a subscription needs a target")
|
||||
label = label.strip() or target
|
||||
try:
|
||||
with self.transaction() as conn:
|
||||
new_id = conn.execute(
|
||||
"""INSERT INTO subscriptions (source, target, label, company, board,
|
||||
params, added_at, target_id)
|
||||
VALUES (?,?,?,?,?,?,?,?)""",
|
||||
(source, target, label, company, board, json.dumps(params or {}),
|
||||
self.now(), target_id)).lastrowid
|
||||
except sqlite3.IntegrityError as exc:
|
||||
raise ValueError(f"{source} already has {target!r} or the label {label!r}") from exc
|
||||
assert new_id is not None
|
||||
return self._subscription(new_id)
|
||||
|
||||
def update_subscription(self, sub_id: int, **fields: Any) -> Subscription:
|
||||
"""Change labels, params, target_id or the three switches."""
|
||||
unknown = set(fields) - set(_SUBSCRIPTION_FIELDS)
|
||||
if unknown:
|
||||
raise ValueError(f"not a subscription field: {', '.join(sorted(unknown))}")
|
||||
if "params" in fields:
|
||||
fields["params"] = json.dumps(fields["params"] or {})
|
||||
for switch in ("subscribed", "fetch", "show"):
|
||||
if switch in fields:
|
||||
fields[switch] = int(bool(fields[switch]))
|
||||
if fields:
|
||||
sets = ", ".join(f"{f}=?" for f in fields)
|
||||
try:
|
||||
changed = self._write(f"UPDATE subscriptions SET {sets} WHERE id=?",
|
||||
(*fields.values(), sub_id))
|
||||
except sqlite3.IntegrityError as exc:
|
||||
raise ValueError(f"label {fields.get('label')!r} is already in use") from exc
|
||||
if not changed:
|
||||
raise KeyError(sub_id)
|
||||
return self._subscription(sub_id)
|
||||
|
||||
def unsubscribe(self, sub_id: int) -> Subscription:
|
||||
"""Stop following it and hide it. The row stays, and so do its roles."""
|
||||
return self.update_subscription(sub_id, subscribed=0, fetch=0, show=0)
|
||||
|
||||
def _subscription(self, sub_id: int) -> Subscription:
|
||||
found = self.subscription(sub_id)
|
||||
if found is None:
|
||||
raise KeyError(sub_id)
|
||||
return found
|
||||
|
||||
# --------------------------------------------------------- applications
|
||||
|
||||
def save_application(self, application: Application) -> Application:
|
||||
"""Insert or update one application. An empty `id` is minted. Entering a
|
||||
status, on insert or by change, appends an `application_events` row in
|
||||
the same transaction."""
|
||||
_check_status(application.status)
|
||||
app_id = application.id or uuid4().hex
|
||||
stamp = self.now()
|
||||
fields = application.model_dump(exclude={"id", "created_at", "updated_at"})
|
||||
with self.transaction() as conn:
|
||||
before = conn.execute("SELECT status FROM applications WHERE id=?",
|
||||
(app_id,)).fetchone()
|
||||
if before is None:
|
||||
columns = ["id", *fields, "created_at", "updated_at"]
|
||||
conn.execute(
|
||||
f"INSERT INTO applications ({', '.join(columns)}) "
|
||||
f"VALUES ({', '.join('?' for _ in columns)})",
|
||||
(app_id, *fields.values(), stamp, stamp))
|
||||
else:
|
||||
sets = ", ".join(f"{f}=?" for f in fields)
|
||||
conn.execute(f"UPDATE applications SET {sets}, updated_at=? WHERE id=?",
|
||||
(*fields.values(), stamp, app_id))
|
||||
if before is None or before["status"] != application.status:
|
||||
conn.execute("INSERT INTO application_events (application_id, status, at) "
|
||||
"VALUES (?,?,?)", (app_id, application.status, stamp))
|
||||
return self._application(app_id)
|
||||
|
||||
def set_application_status(self, app_id: str, status: str) -> Application:
|
||||
"""Move one application to `status`. Raises ValueError for a status
|
||||
outside `STATUSES` and KeyError for an unknown id."""
|
||||
_check_status(status)
|
||||
current = self._application(app_id)
|
||||
return self.save_application(current.model_copy(update={"status": status}))
|
||||
|
||||
def application(self, app_id: str) -> Application | None:
|
||||
row = self._row("SELECT * FROM applications WHERE id=?", (app_id,))
|
||||
return Application.from_row(row) if row else None
|
||||
|
||||
def applications(self, status: str | None = None) -> list[Application]:
|
||||
sql, params = "SELECT * FROM applications", []
|
||||
if status:
|
||||
sql += " WHERE status = ?"
|
||||
params.append(status)
|
||||
sql += " ORDER BY updated_at DESC, id"
|
||||
return [Application.from_row(r) for r in self._rows(sql, params)]
|
||||
|
||||
def application_events(self, app_id: str) -> list[ApplicationEvent]:
|
||||
return [ApplicationEvent.from_row(r) for r in self._rows(
|
||||
"SELECT * FROM application_events WHERE application_id=? ORDER BY id",
|
||||
(app_id,))]
|
||||
|
||||
def _application(self, app_id: str) -> Application:
|
||||
found = self.application(app_id)
|
||||
if found is None:
|
||||
raise KeyError(app_id)
|
||||
return found
|
||||
|
||||
# ---------------------------------------------------------------- state
|
||||
|
||||
@property
|
||||
def state(self) -> MutableMapping[str, str]:
|
||||
"""The `state` table as a mapping. What the runner hands a plugin as
|
||||
`self.state`: it can keep a counter without being able to read roles."""
|
||||
return _StateMap(self)
|
||||
|
||||
|
||||
def _check_status(status: str) -> None:
|
||||
if status not in STATUSES:
|
||||
raise ValueError(f"{status!r} is not an application status "
|
||||
f"({', '.join(STATUSES)})")
|
||||
|
||||
|
||||
def _check_affinity(affinity: Any) -> None:
|
||||
if not isinstance(affinity, int) or affinity not in AFFINITY:
|
||||
raise ValueError(f"affinity is -1, 0, 1 or 2, not {affinity!r}")
|
||||
Reference in new issue
Block a user