Storage: roles, companies, subscriptions, applications, agents
roledesk/storage/schema.py creates the tables at v1 with WAL and the versioned-migration machinery from newsdesk; store.py adds upsert_role (unpacks raw into gate columns, first_seen on insert, last_seen every time), roles(), mark_closed, companies and subscriptions CRUD, and save_application writing an append-only application_events row on every status entered. Key helpers company_key, title_key, place_key live in models.py. docs/decisions.md records the key decisions. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
36dbcbb29e
commit
ac5eca874a
5 files changed
+949
-7
No files matched your search
+343
-1
@@ -8,14 +8,57 @@ that costs a file handle.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from collections.abc import Iterator, MutableMapping
|
||||
from collections.abc import Callable, Iterator, MutableMapping, Sequence
|
||||
from contextlib import closing, contextmanager
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from desk.models import Item
|
||||
from roledesk.models import (
|
||||
AFFINITY, REMOTE, STATUSES, Application, ApplicationEvent, Company, Role,
|
||||
Subscription, company_key, place_key, title_key,
|
||||
)
|
||||
from roledesk.storage import schema
|
||||
|
||||
#: Text columns a plugin may fill from `Item.raw` under the same name. An
|
||||
#: empty value never overwrites a filled column, so an upsert from a board that
|
||||
#: says less than `extract` found keeps what `extract` found.
|
||||
_RAW_TEXT = ("place", "country", "currency", "seniority", "function")
|
||||
_RAW_INT = ("salary_min", "salary_max")
|
||||
|
||||
_COMPANY_FIELDS = ("name", "domain", "affinity", "status", "careers_url", "notes")
|
||||
_SUBSCRIPTION_FIELDS = ("label", "company", "board", "params", "subscribed", "fetch",
|
||||
"show", "target_id")
|
||||
|
||||
|
||||
def _iso(moment: datetime) -> str:
|
||||
"""UTC, to the second. Every timestamp column is compared as text, so every
|
||||
writer has to spell it the same way."""
|
||||
return moment.astimezone(UTC).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def _remote(value: Any) -> str:
|
||||
"""A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything
|
||||
unrecognised is '' (unknown), which `extract` may fill later."""
|
||||
if value is True:
|
||||
return "remote"
|
||||
text = str(value or "").strip().lower().replace("-", "").replace(" ", "")
|
||||
text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text)
|
||||
return text if text in REMOTE else ""
|
||||
|
||||
|
||||
def _int_or_none(value: Any) -> int | None:
|
||||
if value is None or value == "" or isinstance(value, bool):
|
||||
return None
|
||||
try:
|
||||
return int(float(value))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
class _StateMap(MutableMapping[str, str]):
|
||||
"""`Store.state`: the key/value table, read and written one key at a time."""
|
||||
@@ -51,6 +94,8 @@ class Store:
|
||||
def __init__(self, path: str | Path) -> None:
|
||||
self.path = Path(path).expanduser()
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
#: What "now" is. Replaced by tests that need two writes a second apart.
|
||||
self.clock: Callable[[], datetime] = lambda: datetime.now(UTC)
|
||||
with self.transaction() as conn:
|
||||
# Persistent: recorded in the file, inherited by every connection.
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
@@ -94,6 +139,292 @@ class Store:
|
||||
with self.transaction() as conn:
|
||||
return conn.execute(sql, params).rowcount
|
||||
|
||||
def now(self) -> str:
|
||||
return _iso(self.clock())
|
||||
|
||||
# ---------------------------------------------------------------- roles
|
||||
|
||||
def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool:
|
||||
"""Store one posting. Returns True when it was new.
|
||||
|
||||
`first_seen` and `created_at` are written on insert only; `last_seen` on
|
||||
every call, and a role seen again is open again. The gate columns come
|
||||
from `item.raw` under their own names, with `company` falling back to
|
||||
the subscription's; an empty value never overwrites a filled column.
|
||||
"""
|
||||
if not item.source:
|
||||
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
|
||||
raw = item.raw or {}
|
||||
company = str(raw.get("company") or (subscription.company if subscription else ""))
|
||||
place = str(raw.get("place") or "")
|
||||
stamp = self.now()
|
||||
row: dict[str, Any] = {
|
||||
"uid": item.uid,
|
||||
"source": item.source,
|
||||
"source_id": item.source_id,
|
||||
"subscription_id": subscription.id if subscription else None,
|
||||
"company_key": company_key(company),
|
||||
"company": company,
|
||||
"title": item.title,
|
||||
"title_key": title_key(item.title),
|
||||
"place_key": place_key(place),
|
||||
"remote": _remote(raw.get("remote")),
|
||||
"description": item.body,
|
||||
"feed_text": item.excerpt,
|
||||
"original_url": item.location,
|
||||
"published_at": _iso(item.published_at),
|
||||
"first_seen": stamp,
|
||||
"last_seen": stamp,
|
||||
"raw": json.dumps(raw, default=str),
|
||||
"created_at": stamp,
|
||||
}
|
||||
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
|
||||
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
||||
|
||||
keep_filled = ("company", "company_key", "place_key", "remote", "description",
|
||||
"feed_text", "original_url", *_RAW_TEXT)
|
||||
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
|
||||
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
|
||||
for c in (*_RAW_INT, "subscription_id")]
|
||||
updates += [f"{c}=excluded.{c}"
|
||||
for c in ("title", "title_key", "published_at", "raw", "last_seen")]
|
||||
updates.append("closed_at=NULL")
|
||||
columns = ", ".join(row)
|
||||
marks = ", ".join(f":{c}" for c in row)
|
||||
with self.transaction() as conn:
|
||||
existed = conn.execute("SELECT 1 FROM roles WHERE uid=?",
|
||||
(row["uid"],)).fetchone() is not None
|
||||
conn.execute(
|
||||
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
|
||||
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
|
||||
return not existed
|
||||
|
||||
def role(self, uid: str) -> Role | None:
|
||||
row = self._row("SELECT * FROM roles WHERE uid=?", (uid,))
|
||||
return Role.from_row(row) if row else None
|
||||
|
||||
def roles(self, limit: int = 100, company: str | None = None,
|
||||
place: str | None = None, open_only: bool = True) -> list[Role]:
|
||||
"""Newest first by `first_seen`. `company` and `place` are matched by
|
||||
key, so any spelling that keys the same finds the same rows."""
|
||||
where, params = [], []
|
||||
if company:
|
||||
where.append("company_key = ?")
|
||||
params.append(company_key(company))
|
||||
if place:
|
||||
where.append("place_key = ?")
|
||||
params.append(place_key(place))
|
||||
if open_only:
|
||||
where.append("closed_at IS NULL")
|
||||
sql = "SELECT * FROM roles"
|
||||
if where:
|
||||
sql += " WHERE " + " AND ".join(where)
|
||||
sql += " ORDER BY first_seen DESC, uid LIMIT ?"
|
||||
return [Role.from_row(r) for r in self._rows(sql, [*params, limit])]
|
||||
|
||||
def mark_closed(self, source: str, seen_ids: Sequence[str],
|
||||
subscription_id: int) -> int:
|
||||
"""Close every open role of one board that a complete fetch of it did not
|
||||
list. Returns how many closed. Only call it after a fetch that finished:
|
||||
a failed one lists nothing, and would close the whole board."""
|
||||
return self._write(
|
||||
"""UPDATE roles SET closed_at = ?
|
||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||
AND source_id NOT IN (SELECT value FROM json_each(?))""",
|
||||
(self.now(), source, subscription_id, json.dumps(list(seen_ids))))
|
||||
|
||||
# ------------------------------------------------------------ companies
|
||||
|
||||
def add_company(self, name: str, *, key: str = "", domain: str = "",
|
||||
affinity: int = 0, status: str = "researching",
|
||||
careers_url: str = "", notes: str = "") -> Company:
|
||||
"""Raises ValueError on a key already taken or an affinity out of range."""
|
||||
name = name.strip()
|
||||
key = key or company_key(name)
|
||||
if not key:
|
||||
raise ValueError("a company needs a name")
|
||||
_check_affinity(affinity)
|
||||
stamp = self.now()
|
||||
try:
|
||||
self._write(
|
||||
"""INSERT INTO companies (key, name, domain, affinity, status,
|
||||
careers_url, notes, created_at, updated_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?)""",
|
||||
(key, name, domain, affinity, status, careers_url, notes, stamp, stamp))
|
||||
except sqlite3.IntegrityError as exc:
|
||||
raise ValueError(f"company {key!r} already exists") from exc
|
||||
return self._company(key)
|
||||
|
||||
def company(self, key: str) -> Company | None:
|
||||
row = self._row("SELECT * FROM companies WHERE key=?", (key,))
|
||||
return Company.from_row(row) if row else None
|
||||
|
||||
def companies(self) -> list[Company]:
|
||||
return [Company.from_row(r)
|
||||
for r in self._rows("SELECT * FROM companies ORDER BY name COLLATE NOCASE")]
|
||||
|
||||
def update_company(self, key: str, **fields: Any) -> Company:
|
||||
"""Change any of name, domain, affinity, status, careers_url, notes."""
|
||||
unknown = set(fields) - set(_COMPANY_FIELDS)
|
||||
if unknown:
|
||||
raise ValueError(f"not a company field: {', '.join(sorted(unknown))}")
|
||||
if "affinity" in fields:
|
||||
_check_affinity(fields["affinity"])
|
||||
if fields:
|
||||
sets = ", ".join(f"{f}=?" for f in fields)
|
||||
changed = self._write(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?",
|
||||
(*fields.values(), self.now(), key))
|
||||
if not changed:
|
||||
raise KeyError(key)
|
||||
return self._company(key)
|
||||
|
||||
def remove_company(self, key: str) -> bool:
|
||||
return self._write("DELETE FROM companies WHERE key=?", (key,)) > 0
|
||||
|
||||
def _company(self, key: str) -> Company:
|
||||
found = self.company(key)
|
||||
if found is None:
|
||||
raise KeyError(key)
|
||||
return found
|
||||
|
||||
# -------------------------------------------------------- subscriptions
|
||||
|
||||
def subscriptions(self, source: str | None = None,
|
||||
fetchable: bool = False) -> list[Subscription]:
|
||||
"""`fetchable` is what a pull reads: subscribed and with fetch on."""
|
||||
where, params = [], []
|
||||
if source:
|
||||
where.append("source = ?")
|
||||
params.append(source)
|
||||
if fetchable:
|
||||
where.append("fetch = 1 AND subscribed = 1")
|
||||
sql = "SELECT * FROM subscriptions"
|
||||
if where:
|
||||
sql += " WHERE " + " AND ".join(where)
|
||||
sql += " ORDER BY company, board, label"
|
||||
return [Subscription.from_row(r) for r in self._rows(sql, params)]
|
||||
|
||||
def subscription(self, sub_id: int) -> Subscription | None:
|
||||
row = self._row("SELECT * FROM subscriptions WHERE id=?", (sub_id,))
|
||||
return Subscription.from_row(row) if row else None
|
||||
|
||||
def subscription_for(self, source: str, label: str) -> Subscription | None:
|
||||
"""The row an item names: `Item.subscription` is the label."""
|
||||
row = self._row("SELECT * FROM subscriptions WHERE source=? AND label=?",
|
||||
(source, label))
|
||||
return Subscription.from_row(row) if row else None
|
||||
|
||||
def add_subscription(self, source: str, target: str, label: str = "",
|
||||
company: str = "", board: str = "",
|
||||
params: dict[str, Any] | None = None,
|
||||
target_id: str = "") -> Subscription:
|
||||
"""Raises ValueError on a duplicate target or label within one plugin."""
|
||||
target = target.strip()
|
||||
if not target:
|
||||
raise ValueError("a subscription needs a target")
|
||||
label = label.strip() or target
|
||||
try:
|
||||
with self.transaction() as conn:
|
||||
new_id = conn.execute(
|
||||
"""INSERT INTO subscriptions (source, target, label, company, board,
|
||||
params, added_at, target_id)
|
||||
VALUES (?,?,?,?,?,?,?,?)""",
|
||||
(source, target, label, company, board, json.dumps(params or {}),
|
||||
self.now(), target_id)).lastrowid
|
||||
except sqlite3.IntegrityError as exc:
|
||||
raise ValueError(f"{source} already has {target!r} or the label {label!r}") from exc
|
||||
assert new_id is not None
|
||||
return self._subscription(new_id)
|
||||
|
||||
def update_subscription(self, sub_id: int, **fields: Any) -> Subscription:
|
||||
"""Change labels, params, target_id or the three switches."""
|
||||
unknown = set(fields) - set(_SUBSCRIPTION_FIELDS)
|
||||
if unknown:
|
||||
raise ValueError(f"not a subscription field: {', '.join(sorted(unknown))}")
|
||||
if "params" in fields:
|
||||
fields["params"] = json.dumps(fields["params"] or {})
|
||||
for switch in ("subscribed", "fetch", "show"):
|
||||
if switch in fields:
|
||||
fields[switch] = int(bool(fields[switch]))
|
||||
if fields:
|
||||
sets = ", ".join(f"{f}=?" for f in fields)
|
||||
try:
|
||||
changed = self._write(f"UPDATE subscriptions SET {sets} WHERE id=?",
|
||||
(*fields.values(), sub_id))
|
||||
except sqlite3.IntegrityError as exc:
|
||||
raise ValueError(f"label {fields.get('label')!r} is already in use") from exc
|
||||
if not changed:
|
||||
raise KeyError(sub_id)
|
||||
return self._subscription(sub_id)
|
||||
|
||||
def unsubscribe(self, sub_id: int) -> Subscription:
|
||||
"""Stop following it and hide it. The row stays, and so do its roles."""
|
||||
return self.update_subscription(sub_id, subscribed=0, fetch=0, show=0)
|
||||
|
||||
def _subscription(self, sub_id: int) -> Subscription:
|
||||
found = self.subscription(sub_id)
|
||||
if found is None:
|
||||
raise KeyError(sub_id)
|
||||
return found
|
||||
|
||||
# --------------------------------------------------------- applications
|
||||
|
||||
def save_application(self, application: Application) -> Application:
|
||||
"""Insert or update one application. An empty `id` is minted. Entering a
|
||||
status, on insert or by change, appends an `application_events` row in
|
||||
the same transaction."""
|
||||
_check_status(application.status)
|
||||
app_id = application.id or uuid4().hex
|
||||
stamp = self.now()
|
||||
fields = application.model_dump(exclude={"id", "created_at", "updated_at"})
|
||||
with self.transaction() as conn:
|
||||
before = conn.execute("SELECT status FROM applications WHERE id=?",
|
||||
(app_id,)).fetchone()
|
||||
if before is None:
|
||||
columns = ["id", *fields, "created_at", "updated_at"]
|
||||
conn.execute(
|
||||
f"INSERT INTO applications ({', '.join(columns)}) "
|
||||
f"VALUES ({', '.join('?' for _ in columns)})",
|
||||
(app_id, *fields.values(), stamp, stamp))
|
||||
else:
|
||||
sets = ", ".join(f"{f}=?" for f in fields)
|
||||
conn.execute(f"UPDATE applications SET {sets}, updated_at=? WHERE id=?",
|
||||
(*fields.values(), stamp, app_id))
|
||||
if before is None or before["status"] != application.status:
|
||||
conn.execute("INSERT INTO application_events (application_id, status, at) "
|
||||
"VALUES (?,?,?)", (app_id, application.status, stamp))
|
||||
return self._application(app_id)
|
||||
|
||||
def set_application_status(self, app_id: str, status: str) -> Application:
|
||||
"""Move one application to `status`. Raises ValueError for a status
|
||||
outside `STATUSES` and KeyError for an unknown id."""
|
||||
_check_status(status)
|
||||
current = self._application(app_id)
|
||||
return self.save_application(current.model_copy(update={"status": status}))
|
||||
|
||||
def application(self, app_id: str) -> Application | None:
|
||||
row = self._row("SELECT * FROM applications WHERE id=?", (app_id,))
|
||||
return Application.from_row(row) if row else None
|
||||
|
||||
def applications(self, status: str | None = None) -> list[Application]:
|
||||
sql, params = "SELECT * FROM applications", []
|
||||
if status:
|
||||
sql += " WHERE status = ?"
|
||||
params.append(status)
|
||||
sql += " ORDER BY updated_at DESC, id"
|
||||
return [Application.from_row(r) for r in self._rows(sql, params)]
|
||||
|
||||
def application_events(self, app_id: str) -> list[ApplicationEvent]:
|
||||
return [ApplicationEvent.from_row(r) for r in self._rows(
|
||||
"SELECT * FROM application_events WHERE application_id=? ORDER BY id",
|
||||
(app_id,))]
|
||||
|
||||
def _application(self, app_id: str) -> Application:
|
||||
found = self.application(app_id)
|
||||
if found is None:
|
||||
raise KeyError(app_id)
|
||||
return found
|
||||
|
||||
# ---------------------------------------------------------------- state
|
||||
|
||||
@property
|
||||
@@ -101,3 +432,14 @@ class Store:
|
||||
"""The `state` table as a mapping. What the runner hands a plugin as
|
||||
`self.state`: it can keep a counter without being able to read roles."""
|
||||
return _StateMap(self)
|
||||
|
||||
|
||||
def _check_status(status: str) -> None:
|
||||
if status not in STATUSES:
|
||||
raise ValueError(f"{status!r} is not an application status "
|
||||
f"({', '.join(STATUSES)})")
|
||||
|
||||
|
||||
def _check_affinity(affinity: Any) -> None:
|
||||
if not isinstance(affinity, int) or affinity not in AFFINITY:
|
||||
raise ValueError(f"affinity is -1, 0, 1 or 2, not {affinity!r}")
|
||||
Reference in new issue
Block a user