Files
jobboard/roledesk/storage/store.py
T
WolfandClaude Opus 5.5 ac5eca874a Storage: roles, companies, subscriptions, applications, agents
roledesk/storage/schema.py creates the tables at v1 with WAL and the
versioned-migration machinery from newsdesk; store.py adds upsert_role
(unpacks raw into gate columns, first_seen on insert, last_seen every
time), roles(), mark_closed, companies and subscriptions CRUD, and
save_application writing an append-only application_events row on every
status entered. Key helpers company_key, title_key, place_key live in
models.py. docs/decisions.md records the key decisions.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 11:03:51 +02:00

446 lines
20 KiB
Python

"""`Store`: every SQL statement in the application.
The connection plumbing is a copy of newsdesk's `Store`, on purpose: `desk`
has no storage package. A connection is opened per call and closed again, so
one caller's commit can never land inside another's transaction; in WAL mode
that costs a file handle.
"""
from __future__ import annotations
import json
import sqlite3
from collections.abc import Callable, Iterator, MutableMapping, Sequence
from contextlib import closing, contextmanager
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from uuid import uuid4
from desk.models import Item
from roledesk.models import (
AFFINITY, REMOTE, STATUSES, Application, ApplicationEvent, Company, Role,
Subscription, company_key, place_key, title_key,
)
from roledesk.storage import schema
#: Text columns a plugin may fill from `Item.raw` under the same name. An
#: empty value never overwrites a filled column, so an upsert from a board that
#: says less than `extract` found keeps what `extract` found.
_RAW_TEXT = ("place", "country", "currency", "seniority", "function")
_RAW_INT = ("salary_min", "salary_max")
_COMPANY_FIELDS = ("name", "domain", "affinity", "status", "careers_url", "notes")
_SUBSCRIPTION_FIELDS = ("label", "company", "board", "params", "subscribed", "fetch",
"show", "target_id")
def _iso(moment: datetime) -> str:
"""UTC, to the second. Every timestamp column is compared as text, so every
writer has to spell it the same way."""
return moment.astimezone(UTC).isoformat(timespec="seconds")
def _remote(value: Any) -> str:
"""A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything
unrecognised is '' (unknown), which `extract` may fill later."""
if value is True:
return "remote"
text = str(value or "").strip().lower().replace("-", "").replace(" ", "")
text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text)
return text if text in REMOTE else ""
def _int_or_none(value: Any) -> int | None:
if value is None or value == "" or isinstance(value, bool):
return None
try:
return int(float(value))
except (TypeError, ValueError):
return None
class _StateMap(MutableMapping[str, str]):
"""`Store.state`: the key/value table, read and written one key at a time."""
def __init__(self, store: Store) -> None:
self._store = store
def __getitem__(self, key: str) -> str:
row = self._store._row("SELECT value FROM state WHERE key=?", (key,))
if row is None:
raise KeyError(key)
return str(row["value"])
def __setitem__(self, key: str, value: str) -> None:
self._store._write(
"""INSERT INTO state (key, value) VALUES (?,?)
ON CONFLICT(key) DO UPDATE SET value=excluded.value""",
(key, value))
def __delitem__(self, key: str) -> None:
if not self._store._write("DELETE FROM state WHERE key=?", (key,)):
raise KeyError(key)
def __iter__(self) -> Iterator[str]:
return iter([r["key"] for r in self._store._rows("SELECT key FROM state ORDER BY key")])
def __len__(self) -> int:
row = self._store._row("SELECT COUNT(*) AS n FROM state")
return int(row["n"]) if row else 0
class Store:
def __init__(self, path: str | Path) -> None:
self.path = Path(path).expanduser()
self.path.parent.mkdir(parents=True, exist_ok=True)
#: What "now" is. Replaced by tests that need two writes a second apart.
self.clock: Callable[[], datetime] = lambda: datetime.now(UTC)
with self.transaction() as conn:
# Persistent: recorded in the file, inherited by every connection.
conn.execute("PRAGMA journal_mode=WAL")
#: Migration versions this open applied; empty when already current.
self.migrations_applied: list[int] = schema.initialise(conn)
# ------------------------------------------------------------ connections
def _connect(self) -> sqlite3.Connection:
"""A fresh connection. `foreign_keys` is off by default per connection;
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
`synchronous=NORMAL` is the standard WAL companion."""
conn = sqlite3.connect(self.path)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA foreign_keys=ON")
conn.execute("PRAGMA busy_timeout=30000")
conn.execute("PRAGMA synchronous=NORMAL")
return conn
@contextmanager
def transaction(self) -> Iterator[sqlite3.Connection]:
"""One connection, one transaction, closed on the way out."""
conn = self._connect()
try:
with conn:
yield conn
finally:
conn.close()
def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]:
with closing(self._connect()) as conn:
return conn.execute(sql, params).fetchall()
def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None:
with closing(self._connect()) as conn:
row: sqlite3.Row | None = conn.execute(sql, params).fetchone()
return row
def _write(self, sql: str, params: Any = ()) -> int:
"""One statement in its own transaction. Returns the rowcount."""
with self.transaction() as conn:
return conn.execute(sql, params).rowcount
def now(self) -> str:
return _iso(self.clock())
# ---------------------------------------------------------------- roles
def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool:
"""Store one posting. Returns True when it was new.
`first_seen` and `created_at` are written on insert only; `last_seen` on
every call, and a role seen again is open again. The gate columns come
from `item.raw` under their own names, with `company` falling back to
the subscription's; an empty value never overwrites a filled column.
"""
if not item.source:
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
raw = item.raw or {}
company = str(raw.get("company") or (subscription.company if subscription else ""))
place = str(raw.get("place") or "")
stamp = self.now()
row: dict[str, Any] = {
"uid": item.uid,
"source": item.source,
"source_id": item.source_id,
"subscription_id": subscription.id if subscription else None,
"company_key": company_key(company),
"company": company,
"title": item.title,
"title_key": title_key(item.title),
"place_key": place_key(place),
"remote": _remote(raw.get("remote")),
"description": item.body,
"feed_text": item.excerpt,
"original_url": item.location,
"published_at": _iso(item.published_at),
"first_seen": stamp,
"last_seen": stamp,
"raw": json.dumps(raw, default=str),
"created_at": stamp,
}
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
keep_filled = ("company", "company_key", "place_key", "remote", "description",
"feed_text", "original_url", *_RAW_TEXT)
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
for c in (*_RAW_INT, "subscription_id")]
updates += [f"{c}=excluded.{c}"
for c in ("title", "title_key", "published_at", "raw", "last_seen")]
updates.append("closed_at=NULL")
columns = ", ".join(row)
marks = ", ".join(f":{c}" for c in row)
with self.transaction() as conn:
existed = conn.execute("SELECT 1 FROM roles WHERE uid=?",
(row["uid"],)).fetchone() is not None
conn.execute(
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
return not existed
def role(self, uid: str) -> Role | None:
row = self._row("SELECT * FROM roles WHERE uid=?", (uid,))
return Role.from_row(row) if row else None
def roles(self, limit: int = 100, company: str | None = None,
place: str | None = None, open_only: bool = True) -> list[Role]:
"""Newest first by `first_seen`. `company` and `place` are matched by
key, so any spelling that keys the same finds the same rows."""
where, params = [], []
if company:
where.append("company_key = ?")
params.append(company_key(company))
if place:
where.append("place_key = ?")
params.append(place_key(place))
if open_only:
where.append("closed_at IS NULL")
sql = "SELECT * FROM roles"
if where:
sql += " WHERE " + " AND ".join(where)
sql += " ORDER BY first_seen DESC, uid LIMIT ?"
return [Role.from_row(r) for r in self._rows(sql, [*params, limit])]
def mark_closed(self, source: str, seen_ids: Sequence[str],
subscription_id: int) -> int:
"""Close every open role of one board that a complete fetch of it did not
list. Returns how many closed. Only call it after a fetch that finished:
a failed one lists nothing, and would close the whole board."""
return self._write(
"""UPDATE roles SET closed_at = ?
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
AND source_id NOT IN (SELECT value FROM json_each(?))""",
(self.now(), source, subscription_id, json.dumps(list(seen_ids))))
# ------------------------------------------------------------ companies
def add_company(self, name: str, *, key: str = "", domain: str = "",
affinity: int = 0, status: str = "researching",
careers_url: str = "", notes: str = "") -> Company:
"""Raises ValueError on a key already taken or an affinity out of range."""
name = name.strip()
key = key or company_key(name)
if not key:
raise ValueError("a company needs a name")
_check_affinity(affinity)
stamp = self.now()
try:
self._write(
"""INSERT INTO companies (key, name, domain, affinity, status,
careers_url, notes, created_at, updated_at)
VALUES (?,?,?,?,?,?,?,?,?)""",
(key, name, domain, affinity, status, careers_url, notes, stamp, stamp))
except sqlite3.IntegrityError as exc:
raise ValueError(f"company {key!r} already exists") from exc
return self._company(key)
def company(self, key: str) -> Company | None:
row = self._row("SELECT * FROM companies WHERE key=?", (key,))
return Company.from_row(row) if row else None
def companies(self) -> list[Company]:
return [Company.from_row(r)
for r in self._rows("SELECT * FROM companies ORDER BY name COLLATE NOCASE")]
def update_company(self, key: str, **fields: Any) -> Company:
"""Change any of name, domain, affinity, status, careers_url, notes."""
unknown = set(fields) - set(_COMPANY_FIELDS)
if unknown:
raise ValueError(f"not a company field: {', '.join(sorted(unknown))}")
if "affinity" in fields:
_check_affinity(fields["affinity"])
if fields:
sets = ", ".join(f"{f}=?" for f in fields)
changed = self._write(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?",
(*fields.values(), self.now(), key))
if not changed:
raise KeyError(key)
return self._company(key)
def remove_company(self, key: str) -> bool:
return self._write("DELETE FROM companies WHERE key=?", (key,)) > 0
def _company(self, key: str) -> Company:
found = self.company(key)
if found is None:
raise KeyError(key)
return found
# -------------------------------------------------------- subscriptions
def subscriptions(self, source: str | None = None,
fetchable: bool = False) -> list[Subscription]:
"""`fetchable` is what a pull reads: subscribed and with fetch on."""
where, params = [], []
if source:
where.append("source = ?")
params.append(source)
if fetchable:
where.append("fetch = 1 AND subscribed = 1")
sql = "SELECT * FROM subscriptions"
if where:
sql += " WHERE " + " AND ".join(where)
sql += " ORDER BY company, board, label"
return [Subscription.from_row(r) for r in self._rows(sql, params)]
def subscription(self, sub_id: int) -> Subscription | None:
row = self._row("SELECT * FROM subscriptions WHERE id=?", (sub_id,))
return Subscription.from_row(row) if row else None
def subscription_for(self, source: str, label: str) -> Subscription | None:
"""The row an item names: `Item.subscription` is the label."""
row = self._row("SELECT * FROM subscriptions WHERE source=? AND label=?",
(source, label))
return Subscription.from_row(row) if row else None
def add_subscription(self, source: str, target: str, label: str = "",
company: str = "", board: str = "",
params: dict[str, Any] | None = None,
target_id: str = "") -> Subscription:
"""Raises ValueError on a duplicate target or label within one plugin."""
target = target.strip()
if not target:
raise ValueError("a subscription needs a target")
label = label.strip() or target
try:
with self.transaction() as conn:
new_id = conn.execute(
"""INSERT INTO subscriptions (source, target, label, company, board,
params, added_at, target_id)
VALUES (?,?,?,?,?,?,?,?)""",
(source, target, label, company, board, json.dumps(params or {}),
self.now(), target_id)).lastrowid
except sqlite3.IntegrityError as exc:
raise ValueError(f"{source} already has {target!r} or the label {label!r}") from exc
assert new_id is not None
return self._subscription(new_id)
def update_subscription(self, sub_id: int, **fields: Any) -> Subscription:
"""Change labels, params, target_id or the three switches."""
unknown = set(fields) - set(_SUBSCRIPTION_FIELDS)
if unknown:
raise ValueError(f"not a subscription field: {', '.join(sorted(unknown))}")
if "params" in fields:
fields["params"] = json.dumps(fields["params"] or {})
for switch in ("subscribed", "fetch", "show"):
if switch in fields:
fields[switch] = int(bool(fields[switch]))
if fields:
sets = ", ".join(f"{f}=?" for f in fields)
try:
changed = self._write(f"UPDATE subscriptions SET {sets} WHERE id=?",
(*fields.values(), sub_id))
except sqlite3.IntegrityError as exc:
raise ValueError(f"label {fields.get('label')!r} is already in use") from exc
if not changed:
raise KeyError(sub_id)
return self._subscription(sub_id)
def unsubscribe(self, sub_id: int) -> Subscription:
"""Stop following it and hide it. The row stays, and so do its roles."""
return self.update_subscription(sub_id, subscribed=0, fetch=0, show=0)
def _subscription(self, sub_id: int) -> Subscription:
found = self.subscription(sub_id)
if found is None:
raise KeyError(sub_id)
return found
# --------------------------------------------------------- applications
def save_application(self, application: Application) -> Application:
"""Insert or update one application. An empty `id` is minted. Entering a
status, on insert or by change, appends an `application_events` row in
the same transaction."""
_check_status(application.status)
app_id = application.id or uuid4().hex
stamp = self.now()
fields = application.model_dump(exclude={"id", "created_at", "updated_at"})
with self.transaction() as conn:
before = conn.execute("SELECT status FROM applications WHERE id=?",
(app_id,)).fetchone()
if before is None:
columns = ["id", *fields, "created_at", "updated_at"]
conn.execute(
f"INSERT INTO applications ({', '.join(columns)}) "
f"VALUES ({', '.join('?' for _ in columns)})",
(app_id, *fields.values(), stamp, stamp))
else:
sets = ", ".join(f"{f}=?" for f in fields)
conn.execute(f"UPDATE applications SET {sets}, updated_at=? WHERE id=?",
(*fields.values(), stamp, app_id))
if before is None or before["status"] != application.status:
conn.execute("INSERT INTO application_events (application_id, status, at) "
"VALUES (?,?,?)", (app_id, application.status, stamp))
return self._application(app_id)
def set_application_status(self, app_id: str, status: str) -> Application:
"""Move one application to `status`. Raises ValueError for a status
outside `STATUSES` and KeyError for an unknown id."""
_check_status(status)
current = self._application(app_id)
return self.save_application(current.model_copy(update={"status": status}))
def application(self, app_id: str) -> Application | None:
row = self._row("SELECT * FROM applications WHERE id=?", (app_id,))
return Application.from_row(row) if row else None
def applications(self, status: str | None = None) -> list[Application]:
sql, params = "SELECT * FROM applications", []
if status:
sql += " WHERE status = ?"
params.append(status)
sql += " ORDER BY updated_at DESC, id"
return [Application.from_row(r) for r in self._rows(sql, params)]
def application_events(self, app_id: str) -> list[ApplicationEvent]:
return [ApplicationEvent.from_row(r) for r in self._rows(
"SELECT * FROM application_events WHERE application_id=? ORDER BY id",
(app_id,))]
def _application(self, app_id: str) -> Application:
found = self.application(app_id)
if found is None:
raise KeyError(app_id)
return found
# ---------------------------------------------------------------- state
@property
def state(self) -> MutableMapping[str, str]:
"""The `state` table as a mapping. What the runner hands a plugin as
`self.state`: it can keep a counter without being able to read roles."""
return _StateMap(self)
def _check_status(status: str) -> None:
if status not in STATUSES:
raise ValueError(f"{status!r} is not an application status "
f"({', '.join(STATUSES)})")
def _check_affinity(affinity: Any) -> None:
if not isinstance(affinity, int) or affinity not in AFFINITY:
raise ValueError(f"affinity is -1, 0, 1 or 2, not {affinity!r}")