Files
jobboard/app/storage/store.py
T
Wolf 43773efc53 Merge branch 'worktree-agent-a3bd292551395dda0'
# Conflicts:
#	app/storage/schema.py
2026-09-29 14:16:35 +02:00

1371 lines
68 KiB
Python

"""`Store`: every SQL statement in the application.
The connection plumbing is a copy of newsdesk's `Store`, on purpose: `desk`
has no storage package. A connection is opened per call and closed again, so
one caller's commit can never land inside another's transaction; in WAL mode
that costs a file handle.
"""
from __future__ import annotations
import json
import sqlite3
from collections.abc import Callable, Iterator, MutableMapping, Sequence
from contextlib import closing, contextmanager
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, Literal
from uuid import uuid4
import numpy as np
from desk.models import Item
from app.models import (
AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate,
Company, Gate, Limits, Role, ScoredRole, Subscription, Weights, company_key,
country_of, place_key, posted_at, posted_host, remote_of, title_key,
)
from app.storage import schema
#: Text columns a plugin may fill from `Item.raw` under the same name. An
#: empty value never overwrites a filled column, so an upsert from a board that
#: says less than `extract` found keeps what `extract` found.
_RAW_TEXT = ("place", "country", "currency", "seniority", "function")
_RAW_INT = ("salary_min", "salary_max")
#: What `extract` may fill. Only an empty column is written: a board's own
#: structured field wins over what a model read out of the prose, and the next
#: upsert from that board would put it back anyway.
_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "function",
"permit")
#: How many replies may leave a role unanswered before `extract` stops
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
#: never answers is otherwise paid for on every pull for as long as it exists.
GIVE_UP_AFTER = 3
#: A role `extract` has yet to read: never answered, still open, not given up.
_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL "
f"AND extract_tries < {GIVE_UP_AFTER}")
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
VOTES = (-1, 0, 1, None)
#: The `judged` filter of `Store.roles`, as a `WHERE` on the agent's vote,
#: joined as `v`. A neutral vote is judged, and is neither liked nor passed.
JUDGED: dict[str, str] = {
"liked": "v.vote = 1",
"unjudged": "v.vote IS NULL",
"passed": "v.vote = -1",
}
#: A ranking's score as `Store.roles` serves and orders it: halved by a pass,
#: lifted a fifth by a like, capped at 1. Applied at read time, so a vote
#: moves the role at once and the stored score stays what `score` computed.
_VOTED_SCORE = ("MIN(1.0, s.score * CASE v.vote WHEN -1 THEN 0.5 "
"WHEN 1 THEN 1.2 ELSE 1 END)")
#: The agent's votes beside every role. With no agent the join matches
#: nothing, so `vote` and `voted_at` are NULL: one query shape either way.
_WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
_COMPANY_FIELDS = ("name", "domain", "status", "careers_url", "notes", "ats", "country", "city",
"ats_slug")
#: A company as the store serves it: the row and the asked-for agent's affinity.
_COMPANY_SELECT = """SELECT c.*, COALESCE(ac.affinity, 0) AS affinity FROM companies c
LEFT JOIN agent_companies ac ON ac.company_key = c.key AND ac.agent_id = ?"""
_SUBSCRIPTION_FIELDS = ("label", "company", "board", "params", "subscribed", "fetch",
"show", "target_id")
#: A tag's longest stored form, and how many a role may carry. Long enough for
#: "senior backend engineer" and short enough that the filter row stays a row.
_MAX_TAG_LEN = 40
_MAX_TAGS = 12
def _normalise_tags(tags: Sequence[str]) -> list[str]:
"""Each tag stripped, lowercased, its internal whitespace collapsed to one
space and cut to `_MAX_TAG_LEN`; empties and repeats dropped, order kept,
capped at `_MAX_TAGS`. What `set_tags` and `set_summary` store, so the
filter row never has to trust what a model or a person typed."""
seen: set[str] = set()
out: list[str] = []
for tag in tags:
cleaned = " ".join(str(tag).split()).lower()[:_MAX_TAG_LEN]
if not cleaned or cleaned in seen:
continue
seen.add(cleaned)
out.append(cleaned)
if len(out) >= _MAX_TAGS:
break
return out
def _iso(moment: datetime) -> str:
"""UTC, to the second. Every timestamp column is compared as text, so every
writer has to spell it the same way."""
return moment.astimezone(UTC).isoformat(timespec="seconds")
def _remote(value: Any) -> str:
"""A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything
unrecognised is '' (unknown), which `extract` may fill later."""
if value is True:
return "remote"
text = str(value or "").strip().lower().replace("-", "").replace(" ", "")
text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text)
return text if text in REMOTE else ""
def _like(text: str) -> str:
"""A `LIKE` pattern for a case-insensitive substring match, with SQLite's
own wildcards escaped so a search for "50%" or "a_b" stays literal."""
escaped = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
return f"%{escaped}%"
#: Vectors are stored little-endian float32, fixed here so a file written on
#: one machine reads on another. newsdesk's encoding, written again.
_DTYPE = np.dtype("<f4")
def pack(vec: np.ndarray) -> bytes:
return np.asarray(vec, dtype=_DTYPE).tobytes()
def unpack(blob: bytes) -> np.ndarray:
return np.frombuffer(blob, dtype=_DTYPE)
def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
"""One gate as `(empty, match, params)`: SQL that is true when the role's
column is empty, and SQL that is true when it matches, over `roles r`.
`params` belong to `match`; `empty` takes none."""
column = f"r.{gate.field}"
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
marks = ", ".join("?" for _ in values) or "NULL"
if gate.field == "salary_min":
floor = min(_int_or_none(v) or 0 for v in gate.values)
return f"{column} IS NULL", f"{column} >= ?", [floor]
if gate.field == "place":
likes = " OR ".join("LOWER(r.place) LIKE ? ESCAPE '\\'" for _ in values) or "0"
return "r.place = ''", f"({likes})", [_like(v) for v in values]
if gate.field == "company":
keys = [company_key(v) for v in values]
return ("r.company_key = ''",
f"r.company_key IN ({', '.join('?' for _ in keys) or 'NULL'})", keys)
if gate.field == "languages":
return ("r.languages IN ('', '[]')",
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
f"WHERE LOWER(value) NOT IN ({marks}))", values)
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
def _int_or_none(value: Any) -> int | None:
if value is None or value == "" or isinstance(value, bool):
return None
try:
return int(float(value))
except (TypeError, ValueError):
return None
class _StateMap(MutableMapping[str, str]):
"""`Store.state`: the key/value table, read and written one key at a time."""
def __init__(self, store: Store) -> None:
self._store = store
def __getitem__(self, key: str) -> str:
row = self._store._row("SELECT value FROM state WHERE key=?", (key,))
if row is None:
raise KeyError(key)
return str(row["value"])
def __setitem__(self, key: str, value: str) -> None:
self._store._write(
"""INSERT INTO state (key, value) VALUES (?,?)
ON CONFLICT(key) DO UPDATE SET value=excluded.value""",
(key, value))
def __delitem__(self, key: str) -> None:
if not self._store._write("DELETE FROM state WHERE key=?", (key,)):
raise KeyError(key)
def __iter__(self) -> Iterator[str]:
return iter([r["key"] for r in self._store._rows("SELECT key FROM state ORDER BY key")])
def __len__(self) -> int:
row = self._store._row("SELECT COUNT(*) AS n FROM state")
return int(row["n"]) if row else 0
class Store:
def __init__(self, path: str | Path) -> None:
self.path = Path(path).expanduser()
self.path.parent.mkdir(parents=True, exist_ok=True)
#: What "now" is. Replaced by tests that need two writes a second apart.
self.clock: Callable[[], datetime] = lambda: datetime.now(UTC)
with self.transaction() as conn:
# Persistent: recorded in the file, inherited by every connection.
conn.execute("PRAGMA journal_mode=WAL")
#: Migration versions this open applied; empty when already current.
self.migrations_applied: list[int] = schema.initialise(conn)
# ------------------------------------------------------------ connections
def _connect(self) -> sqlite3.Connection:
"""A fresh connection. `foreign_keys` is off by default per connection;
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
`synchronous=NORMAL` is the standard WAL companion."""
conn = sqlite3.connect(self.path)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA foreign_keys=ON")
conn.execute("PRAGMA busy_timeout=30000")
conn.execute("PRAGMA synchronous=NORMAL")
return conn
@contextmanager
def transaction(self) -> Iterator[sqlite3.Connection]:
"""One connection, one transaction, closed on the way out."""
conn = self._connect()
try:
with conn:
yield conn
finally:
conn.close()
def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]:
with closing(self._connect()) as conn:
return conn.execute(sql, params).fetchall()
def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None:
with closing(self._connect()) as conn:
row: sqlite3.Row | None = conn.execute(sql, params).fetchone()
return row
def _write(self, sql: str, params: Any = ()) -> int:
"""One statement in its own transaction. Returns the rowcount."""
with self.transaction() as conn:
return conn.execute(sql, params).rowcount
def now(self) -> str:
return _iso(self.clock())
# ---------------------------------------------------------------- roles
def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool:
"""Store one posting. Returns True when it was new.
`first_seen` and `created_at` are written on insert only; `last_seen` on
every call, and a role seen again is open again. The gate columns come
from `item.raw` under their own names, with `company` falling back to
the subscription's and `country` to what `place` names; an empty value
never overwrites a filled column.
`posted_at` (see `app.models.posted_at`) is set from this pull's own
`published_at` and `stamp`; on an update the `ON CONFLICT` clause below
keeps the earlier of that and what is already stored, so an edit on
the board — which can only ever move `published_at` later, never
earlier — cannot move `posted_at` later either.
"""
if not item.source:
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
raw = item.raw or {}
company = str(raw.get("company") or (subscription.company if subscription else ""))
place = str(raw.get("place") or "")
stamp = self.now()
published = _iso(item.published_at)
row: dict[str, Any] = {
"uid": item.uid,
"source": item.source,
"source_id": item.source_id,
"subscription_id": subscription.id if subscription else None,
"company_key": company_key(company),
"company": company,
"title": item.title,
"title_key": title_key(item.title),
"place_key": place_key(place),
"remote": _remote(raw.get("remote")),
"description": item.body,
"feed_text": item.excerpt,
"original_url": item.location,
"published_at": published,
"posted_at": posted_at(published, stamp, posted_host(raw)),
"first_seen": stamp,
"last_seen": stamp,
"raw": json.dumps(raw, default=str),
"created_at": stamp,
}
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
row["country"] = row["country"] or country_of(place)
row["remote"] = row["remote"] or remote_of(place)
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
keep_filled = ("company", "company_key", "place_key", "remote", "description",
"feed_text", "original_url", *_RAW_TEXT)
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
for c in (*_RAW_INT, "subscription_id")]
updates += [f"{c}=excluded.{c}"
for c in ("title", "title_key", "published_at", "raw", "last_seen")]
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
updates.append("closed_at=NULL")
columns = ", ".join(row)
marks = ", ".join(f":{c}" for c in row)
with self.transaction() as conn:
existed = conn.execute("SELECT 1 FROM roles WHERE uid=?",
(row["uid"],)).fetchone() is not None
conn.execute(
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
return not existed
def role(self, uid: str, agent_id: int | None = None) -> Role | None:
"""One role with its tags, and with `vote` and `voted_at` as
`agent_id` judged it; None (unjudged) without an agent."""
row = self._row(f"SELECT roles.*, v.vote, v.voted_at FROM roles {_WITH_VOTES} "
"WHERE roles.uid=?", (agent_id, uid))
if row is None:
return None
role = Role.from_row(row)
role.tags = self._tags_of(uid)
return role
def set_read(self, uid: str, read: bool = True) -> Role:
"""Mark a role read, or unread again. Stamps `read_at` only the first
time — a role already read keeps the moment it was first opened —
and clears it outright when `read` is False. Raises KeyError for an
unknown uid."""
if read:
changed = self._write(
"UPDATE roles SET read_at = COALESCE(read_at, ?) WHERE uid = ?",
(self.now(), uid))
else:
changed = self._write("UPDATE roles SET read_at = NULL WHERE uid = ?", (uid,))
if not changed:
raise KeyError(uid)
role = self.role(uid)
assert role is not None
return role
def set_vote(self, agent_id: int, uid: str, value: int | None) -> Role:
"""One agent's judgement of one role: 1 like, 0 neither way, -1 pass,
None unjudged again, which deletes the row. A vote never hides a
role; it moves it in that agent's ranking (see `roles`) and shapes its
taste direction (see `jobs/score.py`). Voting is reading, so a vote
stamps the role's `read_at` too, the same once-only way `set_read`
does; clearing a vote is not un-reading, and leaves it alone. Returns
the role with this agent's vote. Raises ValueError for another value
and KeyError for an unknown uid or agent."""
if isinstance(value, bool) or value not in VOTES:
raise ValueError(f"a vote is -1, 0 or 1, not {value!r}")
with self.transaction() as conn:
self._known(conn, "roles", "uid", uid)
self._known(conn, "agents", "id", agent_id)
if value is None:
conn.execute("DELETE FROM role_votes WHERE agent_id=? AND uid=?",
(agent_id, uid))
else:
now = self.now()
conn.execute(
"""INSERT INTO role_votes (agent_id, uid, vote, voted_at) VALUES (?,?,?,?)
ON CONFLICT(agent_id, uid) DO UPDATE
SET vote=excluded.vote, voted_at=excluded.voted_at""",
(agent_id, uid, value, now))
conn.execute("UPDATE roles SET read_at = COALESCE(read_at, ?) WHERE uid = ?",
(now, uid))
role = self.role(uid, agent_id)
assert role is not None
return role
def votes(self, agent_id: int) -> dict[str, int]:
"""`uid -> vote` for one agent, open roles and closed alike: a closed
role somebody liked still says what they like."""
return {r["uid"]: int(r["vote"]) for r in self._rows(
"SELECT uid, vote FROM role_votes WHERE agent_id=?", (agent_id,))}
@staticmethod
def _known(conn: sqlite3.Connection, table: str, column: str, value: Any) -> None:
"""KeyError unless the row exists: the caller's 404, rather than the
foreign key's IntegrityError, which does not say which side is unknown."""
if conn.execute(f"SELECT 1 FROM {table} WHERE {column}=?", (value,)).fetchone() is None:
raise KeyError(value)
def set_tags(self, uid: str, tags: Sequence[str]) -> list[str]:
"""Replace one role's tags in one transaction. See `_normalise_tags`
for how each one is cleaned up. Returns what was stored, sorted the
way `role` and `roles` read it back. Raises KeyError for an unknown
uid."""
with self.transaction() as conn:
if conn.execute("SELECT 1 FROM roles WHERE uid=?", (uid,)).fetchone() is None:
raise KeyError(uid)
return self._replace_tags(conn, uid, tags)
def _replace_tags(self, conn: sqlite3.Connection, uid: str,
tags: Sequence[str]) -> list[str]:
"""`set_tags`'s write, against an already-open transaction so
`set_summary` can share it."""
cleaned = _normalise_tags(tags)
conn.execute("DELETE FROM role_tags WHERE uid=?", (uid,))
conn.executemany("INSERT INTO role_tags (uid, tag) VALUES (?,?)",
[(uid, tag) for tag in cleaned])
return sorted(cleaned)
def _tags_of(self, uid: str) -> list[str]:
return [str(r["tag"]) for r in self._rows(
"SELECT tag FROM role_tags WHERE uid=? ORDER BY tag", (uid,))]
def _tags_by_uid(self, uids: Sequence[str]) -> dict[str, list[str]]:
"""Every tag of these roles, by uid, sorted — one query for a whole
page rather than one per role."""
if not uids:
return {}
out: dict[str, list[str]] = {}
rows = self._rows(
"SELECT uid, tag FROM role_tags WHERE uid IN (SELECT value FROM json_each(?)) "
"ORDER BY uid, tag", (json.dumps(list(uids)),))
for r in rows:
out.setdefault(r["uid"], []).append(str(r["tag"]))
return out
def roles(self, limit: int = 100, company: str | None = None,
place: str | None = None, open_only: bool = True,
q: str | None = None, function: str | None = None,
remote: str | None = None, days: int | None = None,
hide_muted: bool = False, include_read: bool = True,
tags: Sequence[str] = (), tag_mode: str = "any",
agent: int | None = None, by_score: bool = True, gated: bool = False,
judged: str | None = None, offset: int = 0) -> list[ScoredRole]:
"""Newest first by `posted_at`. `company` and `place` are
case-insensitive substrings of the columns a board wrote — not the
key, so a search does not have to know the exact slug a company folds
to. `q` is matched word by word against `title`; a role must match
every word. `function` and `remote` match a gate column exactly.
`days` keeps only roles posted within that many days of now.
`agent` says whose votes, hidden companies and scores are meant. Each
row carries that agent's `vote` and `voted_at`; `judged` (a key of
`JUDGED`: liked, unjudged or passed) filters on them. `hide_muted`
excludes the companies that agent hid, before `limit` is applied, in
SQL rather than after the fact: a company that has published a lot,
all at once — a fresh board added, a re-pull that finally reached
it — can otherwise fill the whole page with roles that then all turn
out to be from the one company someone hid, leaving nothing to show
for a default query that would otherwise have plenty. With no agent
every vote is None, nothing is hidden and `judged` reads as such.
`include_read` set False excludes a role `set_read` or `set_vote` has
stamped, the same way and for the same reason as `hide_muted`:
before `limit`, not after. `tags` keeps only roles carrying at least
one of them (`tag_mode="any"`, the default) or every one
(`tag_mode="all"`), also before `limit`.
With `agent` and `by_score`, ordered by that agent's score instead,
highest first, carrying `score` and `gated_reason` on each row. Only
roles it has scored are listed, and a gated one only when `gated` is
set. The score is the stored one moved by the agent's vote
(`_VOTED_SCORE`), so a vote shows in the order at once and needs no
rescore. With `by_score` off the order is newest, with the votes and
no score.
`offset` skips that many rows of the same order, for the page that
loads the next 60 as it is scrolled; `count_roles` is the total.
Every returned role carries its tags, read with one extra query for
the whole page rather than one per role."""
sql, params = self._roles_from(
company=company, place=place, open_only=open_only, q=q, function=function,
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
judged=judged)
select = "SELECT roles.*, v.vote, v.voted_at"
order = "roles.posted_at DESC, roles.uid"
scored = agent is not None and by_score
if scored:
select += f", {_VOTED_SCORE} AS score, s.gated_reason"
order = "score DESC, " + order
elif open_only:
# Newest first, open only: idx_roles_posted covers this exactly
# (posted_at DESC, uid, WHERE closed_at IS NULL), so it earns the
# LIMIT rather than sorting every open role first. Left to the
# planner, sqlite3's bundled build (no STAT4) reads the near-100%
# selectivity of `closed_at IS NULL` from idx_roles_closed's plain
# average and picks that instead, sorting the lot — checked with
# EXPLAIN QUERY PLAN on a copy of the live database.
sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1)
found = [ScoredRole.from_row(r) for r in self._rows(
f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?", [*params, limit, offset])]
by_uid = self._tags_by_uid([r.uid for r in found])
for r in found:
r.tags = by_uid.get(r.uid, [])
return found
def count_roles(self, company: str | None = None, place: str | None = None,
open_only: bool = True, q: str | None = None,
function: str | None = None, remote: str | None = None,
days: int | None = None, hide_muted: bool = False,
include_read: bool = True, tags: Sequence[str] = (),
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
gated: bool = False, judged: str | None = None) -> int:
"""How many roles `roles` would list with the same filters and no limit."""
sql, params = self._roles_from(
company=company, place=place, open_only=open_only, q=q, function=function,
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
judged=judged)
row = self._row(f"SELECT COUNT(*) AS n {sql}", params)
return int(row["n"]) if row else 0
def _roles_from(self, *, company: str | None, place: str | None, open_only: bool,
q: str | None, function: str | None, remote: str | None,
days: int | None, hide_muted: bool, include_read: bool,
tags: Sequence[str], tag_mode: str, agent: int | None,
by_score: bool, gated: bool, judged: str | None) -> tuple[str, list[Any]]:
"""The `FROM ... WHERE ...` that `roles` and `count_roles` share, with
its parameters in order. The agent's votes are always joined as `v`
(see `_WITH_VOTES`); its scores as `s` only when ordering by them."""
where: list[str] = []
params: list[Any] = []
if company:
where.append("company LIKE ? ESCAPE '\\'")
params.append(_like(company))
if place:
where.append("place LIKE ? ESCAPE '\\'")
params.append(_like(place))
if function:
where.append("function = ?")
params.append(function)
if remote:
where.append("remote = ?")
params.append(remote)
if days is not None:
where.append("posted_at >= ?")
params.append(_iso(self.clock() - timedelta(days=days)))
for word in (q or "").split():
where.append("title LIKE ? ESCAPE '\\'")
params.append(_like(word))
if judged:
if judged not in JUDGED:
raise ValueError(f"judged is {', '.join(JUDGED)}, not {judged!r}")
where.append(JUDGED[judged])
if open_only:
where.append("closed_at IS NULL")
if hide_muted and agent is not None:
where.append("company_key NOT IN (SELECT company_key FROM agent_companies "
"WHERE agent_id = ? AND affinity = -1)")
params.append(agent)
if not include_read:
where.append("roles.read_at IS NULL")
if tags:
marks = ", ".join("?" for _ in tags)
if tag_mode == "all":
where.append(
f"(SELECT COUNT(DISTINCT rt.tag) FROM role_tags rt "
f"WHERE rt.uid = roles.uid AND rt.tag IN ({marks})) = ?")
params += [*tags, len(tags)]
elif tag_mode == "any":
where.append(
f"EXISTS (SELECT 1 FROM role_tags rt "
f"WHERE rt.uid = roles.uid AND rt.tag IN ({marks}))")
params += list(tags)
else:
raise ValueError(f"tag_mode is any or all, not {tag_mode!r}")
sql = f"FROM roles {_WITH_VOTES}"
joins: list[Any] = [agent]
if agent is not None and by_score:
sql += " JOIN scores s ON s.uid = roles.uid AND s.agent_id = ?"
joins.append(agent)
if not gated:
where.append("s.gated_reason = ''")
if where:
sql += " WHERE " + " AND ".join(where)
return sql, [*joins, *params]
def tag_counts(self, agent_id: int | None = None,
show_muted: bool = False) -> list[tuple[str, int, int]]:
"""`(tag, open_count, unread_count)` over open roles, most-used tag
first and then alphabetically. What `/api/filters` serves; the
companies `agent_id` hid are left out unless `show_muted`, the same
default as `roles`, and with no agent nothing is hidden."""
where = "r.closed_at IS NULL"
params: list[Any] = []
if not show_muted and agent_id is not None:
where += (" AND r.company_key NOT IN (SELECT company_key FROM agent_companies "
"WHERE agent_id = ? AND affinity = -1)")
params.append(agent_id)
rows = self._rows(
f"""SELECT rt.tag AS tag, COUNT(*) AS n, SUM(r.read_at IS NULL) AS unread
FROM role_tags rt JOIN roles r ON r.uid = rt.uid
WHERE {where}
GROUP BY rt.tag
ORDER BY n DESC, rt.tag ASC""", params)
return [(str(r["tag"]), int(r["n"]), int(r["unread"])) for r in rows]
def mark_closed(self, source: str, seen_ids: Sequence[str],
subscription_id: int) -> int:
"""Close every open role of one board that a complete fetch of it did not
list. Returns how many closed. Only call it after a fetch that finished:
a failed one lists nothing, and would close the whole board."""
return self._write(
"""UPDATE roles SET closed_at = ?
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
AND source_id NOT IN (SELECT value FROM json_each(?))""",
(self.now(), source, subscription_id, json.dumps(list(seen_ids))))
def close_unseen(self, source: str, before: str) -> int:
"""Close every open role of `source` last seen before `before`. For a
source whose fetch is never a complete list, a search, where a role
dropping out of one run's results is not the role closing."""
return self._write(
"""UPDATE roles SET closed_at = ?
WHERE source = ? AND closed_at IS NULL AND last_seen < ?""",
(self.now(), source, before))
def close_subscription(self, subscription_id: int) -> int:
"""Close every open role of one subscription, for a board that is no
longer fetched: nothing would ever close them otherwise."""
return self._write(
"UPDATE roles SET closed_at = ? WHERE subscription_id = ? AND closed_at IS NULL",
(self.now(), subscription_id))
def backfill_from_place(self) -> int:
"""Fill `country` and `remote` from `place` on every role that has
none, the way `upsert_role` does for a new one, so a word added to the
lookups reaches the roles already stored. Returns how many columns it
filled."""
rows = self._rows("SELECT uid, place FROM roles "
"WHERE place != '' AND (country = '' OR remote = '')")
countries = [(code, r["uid"]) for r in rows if (code := country_of(r["place"]))]
remotes = [(mode, r["uid"]) for r in rows if (mode := remote_of(r["place"]))]
with self.transaction() as conn:
conn.executemany("UPDATE roles SET country = ? WHERE uid = ? AND country = ''",
countries)
conn.executemany("UPDATE roles SET remote = ? WHERE uid = ? AND remote = ''",
remotes)
return len(countries) + len(remotes)
# -------------------------------------------------------------- extract
def _top_scope(self, stage: Literal["extract", "summarise"]) -> tuple[str, list[Any]]:
"""The `WHERE` fragment for a role among the best ungated scores of
an enabled agent, as deep and as low as that agent's own `limits`
say for `stage`: what the two model stages, extract and summarise,
are bounded to. The rank is numbered per agent and the agent's
limits ride out of the window subquery beside it, so each row is
held to the limits of the agent that ranked it; a key the JSON lacks
is the `Limits` default. The role is `r`. A company an agent hid
needs no clause: `score` writes no rows for it under that agent."""
defaults = Limits()
return (f"""r.uid IN (SELECT uid FROM (
SELECT s.uid, s.score, a.limits, ROW_NUMBER() OVER (
PARTITION BY s.agent_id
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
FROM scores s
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
WHERE s.gated_reason = '')
WHERE n <= COALESCE(json_extract(limits, '$.{stage}_top'), ?)
AND score >= COALESCE(json_extract(limits, '$.{stage}_min'), ?))""",
[getattr(defaults, f"{stage}_top"), getattr(defaults, f"{stage}_min")])
def _extract_scope(self, uids: Sequence[str] | None) -> tuple[str, list[Any]]:
"""The `WHERE` for a role `extract` may read: open, never answered,
not given up. Given `uids`, those of them and nothing else, since a
batch asks again when it runs and must finish what it was dealt;
otherwise only the roles in `_top_scope`."""
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")]
if uids is not None:
where.append("r.uid IN (SELECT value FROM json_each(?))")
return " AND ".join(where), [json.dumps(list(uids))]
top, params = self._top_scope("extract")
return " AND ".join([*where, top]), params
def extract_pending(self) -> int:
where, params = self._extract_scope(None)
row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params)
return int(row["n"]) if row else 0
def extract_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]:
"""Roles `extract` has yet to read, newest first: the best of every
enabled agent's scores, as deep and as low as that agent's own
`limits` say, so the backlog follows the rankings rather than the
boards. Given `uids`, only those of them still waiting, whatever
their rank."""
where, params = self._extract_scope(uids)
return [Role.from_row(r) for r in self._rows(
f"SELECT r.* FROM roles r WHERE {where} "
"ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
def set_extracted(self, uid: str, fields: dict[str, Any]) -> int | None:
"""Write what `extract` read for one role and stamp `extracted_at`.
Only into empty columns, and only a non-empty value: a board's own
field is never replaced and nothing is ever blanked. `place` brings
`place_key` with it. Returns how many columns were filled, or None
for an unknown uid. Values are trusted to be in vocabulary; the
job's `apply` is where a reply is checked."""
with self.transaction() as conn:
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
if row is None:
return None
sets: dict[str, Any] = {}
for name in _EXTRACT_TEXT:
value = str(fields.get(name) or "")
if value and not row[name]:
sets[name] = value
for name in _RAW_INT:
number = _int_or_none(fields.get(name))
if number is not None and row[name] is None:
sets[name] = number
languages = [str(code) for code in fields.get("languages") or []]
if languages and json.loads(row["languages"] or "[]") == []:
sets["languages"] = json.dumps(languages)
filled = len(sets)
if "place" in sets:
sets["place_key"] = place_key(sets["place"])
sets["extracted_at"] = self.now()
conn.execute(
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
(*sets.values(), uid))
return filled
def extract_missed(self, uids: Sequence[str], why: str = "") -> int:
"""One reply that left these roles unanswered. Counted only against
those still waiting, so a role the reply did answer is not charged.
Returns how many this pushed to `GIVE_UP_AFTER`. `why` is accepted
for `ModelTask.missed` and not kept: there is no Given up list yet."""
if not uids:
return 0
where = f"uid IN (SELECT value FROM json_each(?)) AND {_EXTRACT_WAITING}"
wanted = json.dumps(list(uids))
with self.transaction() as conn:
crossed = conn.execute(
f"SELECT COUNT(*) FROM roles WHERE {where} AND extract_tries + 1 >= ?",
(wanted, GIVE_UP_AFTER)).fetchone()[0]
conn.execute(f"UPDATE roles SET extract_tries = extract_tries + 1 "
f"WHERE {where}", (wanted,))
return int(crossed)
# ------------------------------------------------------------ summarise
#: The `state` key counting replies that left a role without a summary.
#: `roles` has no column for it, and the count only matters for the few
#: roles in some agent's top list.
_SUMMARY_MISSED = "summarise.missed:"
def _summary_scope(self, uids: Sequence[str] | None) -> tuple[str, list[Any]]:
"""The `WHERE` for a role `summarise` may be sent: no summary yet and
some text to summarise. Given `uids`, those of them and nothing else,
open or not and given up or not, because somebody asked for them.
Otherwise open, not given up, and in `_top_scope`."""
where = ["r.summary = ''",
"(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"]
if uids is not None:
where.append("r.uid IN (SELECT value FROM json_each(?))")
return " AND ".join(where), [json.dumps(list(uids))]
top, params = self._top_scope("summarise")
where += [
"r.closed_at IS NULL",
f"""NOT EXISTS (SELECT 1 FROM state
WHERE key = '{self._SUMMARY_MISSED}' || r.uid
AND CAST(value AS INTEGER) >= {GIVE_UP_AFTER})""",
top,
]
return " AND ".join(where), params
def summarise_pending(self) -> int:
where, params = self._summary_scope(None)
row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params)
return int(row["n"]) if row else 0
def summarise_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]:
"""Roles `summarise` has yet to write, newest first: the top of every
enabled agent's scores, as deep and as low as that agent's own
`limits` say, so the backlog is bounded by the agents rather than by
the number of open roles; given `uids`, those still without a
summary, whatever their rank. A batch asks again when it runs."""
where, params = self._summary_scope(uids)
return [Role.from_row(r) for r in self._rows(
f"SELECT r.* FROM roles r WHERE {where} "
"ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
def set_summary(self, uid: str, summary: str, tags: Sequence[str] = ()) -> bool:
"""Write a role's summary and stamp `summarised_at`. Only over an empty
one, so a second batch that carried the same role does not replace the
first answer. Given `tags`, they replace the role's tags in the same
transaction — see `_replace_tags` — but only when the summary was
actually written, so a rejected late answer cannot plant tags on a
role it did not get to summarise. False for an unknown uid or one
already summarised."""
with self.transaction() as conn:
written = conn.execute(
"UPDATE roles SET summary = ?, summarised_at = ? "
"WHERE uid = ? AND summary = ''", (summary, self.now(), uid)).rowcount
conn.execute("DELETE FROM state WHERE key = ?", (self._SUMMARY_MISSED + uid,))
if written and tags:
self._replace_tags(conn, uid, tags)
return bool(written)
def summarise_missed(self, uids: Sequence[str], why: str = "") -> int:
"""One reply that left these roles without a summary. Counted only
against those still without one. Returns how many this pushed to
`GIVE_UP_AFTER`. `why` is accepted for `ModelTask.missed` and not kept."""
if not uids:
return 0
rows = self._rows("SELECT uid FROM roles WHERE summary = '' "
"AND uid IN (SELECT value FROM json_each(?))",
(json.dumps(list(uids)),))
keys = [self._SUMMARY_MISSED + r["uid"] for r in rows]
with self.transaction() as conn:
conn.executemany(
"""INSERT INTO state (key, value) VALUES (?, '1')
ON CONFLICT(key) DO UPDATE
SET value = CAST(CAST(value AS INTEGER) + 1 AS TEXT)""",
[(key,) for key in keys])
crossed = conn.execute(
"SELECT COUNT(*) FROM state WHERE key IN (SELECT value FROM json_each(?)) "
"AND CAST(value AS INTEGER) = ?", (json.dumps(keys), GIVE_UP_AFTER)).fetchone()[0]
return int(crossed)
def set_article_text(self, uid: str, text: str) -> Role | None:
"""Store the posting page as plain text and stamp `article_at`. The
role as it now stands, or None for an unknown uid."""
if not self._write("UPDATE roles SET article_text = ?, article_at = ? WHERE uid = ?",
(text, self.now(), uid)):
return None
return self.role(uid)
# -------------------------------------------------------------- vectors
def roles_to_embed(self, model: str, limit: int) -> list[Role]:
"""Open roles with no vector from `model`, newest first, so a backlog
embedded from the top improves what is on screen first."""
return [Role.from_row(r) for r in self._rows(
"""SELECT r.* FROM roles r
LEFT JOIN vectors v ON v.uid = r.uid AND v.model = ?
WHERE v.uid IS NULL AND r.closed_at IS NULL
ORDER BY r.first_seen DESC, r.uid LIMIT ?""", (model, limit))]
def put_vectors(self, model: str, pairs: Sequence[tuple[str, np.ndarray]]) -> int:
stamp = self.now()
with self.transaction() as conn:
return conn.executemany(
"""INSERT INTO vectors (uid, model, dim, vec, embedded_at)
VALUES (?,?,?,?,?)
ON CONFLICT(uid) DO UPDATE SET model=excluded.model,
dim=excluded.dim, vec=excluded.vec,
embedded_at=excluded.embedded_at""",
[(uid, model, int(vec.shape[-1]), pack(vec), stamp)
for uid, vec in pairs]).rowcount
def vectors(self, model: str) -> dict[str, np.ndarray]:
"""Every vector from `model`, by uid. The whole table: at ~30k roles of
384 floats that is ~45 MB, and one read beats one per agent."""
return {r["uid"]: unpack(r["vec"]) for r in self._rows(
"SELECT uid, vec FROM vectors WHERE model = ?", (model,))}
def vector_counts(self) -> dict[str, int]:
return {r["model"]: int(r["n"]) for r in self._rows(
"SELECT model, COUNT(*) AS n FROM vectors GROUP BY model")}
def drop_vectors(self, keep: str) -> int:
"""Delete vectors from every model but `keep`: another model's are a
different geometry, never compared, and only make counts lie."""
return self._write("DELETE FROM vectors WHERE model <> ?", (keep,))
# --------------------------------------------------------------- agents
def agents(self, enabled_only: bool = False) -> list[Agent]:
sql = "SELECT * FROM agents"
if enabled_only:
sql += " WHERE enabled = 1"
return [Agent.from_row(r)
for r in self._rows(sql + " ORDER BY name COLLATE NOCASE")]
def agent(self, agent_id: int) -> Agent | None:
row = self._row("SELECT * FROM agents WHERE id=?", (agent_id,))
return Agent.from_row(row) if row else None
def default_agent(self) -> Agent | None:
"""The agent a vote or an affinity goes to when none is named: the
enabled one with the lowest id, else the lowest id, else None."""
row = self._row("SELECT * FROM agents ORDER BY enabled DESC, id LIMIT 1")
return Agent.from_row(row) if row else None
def save_agent(self, name: str, intent: str = "", gates: Sequence[Gate] = (),
weights: Weights | None = None, enabled: bool = True,
agent_id: int | None = None, limits: Limits | None = None) -> Agent:
"""Create an agent, or with `agent_id` replace that one's fields.
Raises ValueError on an empty or taken name, KeyError on an unknown id."""
name = name.strip()
if not name:
raise ValueError("an agent needs a name")
fields = (name, intent, json.dumps([g.model_dump() for g in gates]),
(weights or Weights()).model_dump_json(),
(limits or Limits()).model_dump_json(), int(enabled))
try:
with self.transaction() as conn:
if agent_id is None:
agent_id = conn.execute(
"""INSERT INTO agents (name, intent, gates, weights, limits,
enabled, created_at) VALUES (?,?,?,?,?,?,?)""",
(*fields, self.now())).lastrowid
elif not conn.execute(
"""UPDATE agents SET name=?, intent=?, gates=?, weights=?,
limits=?, enabled=? WHERE id=?""",
(*fields, agent_id)).rowcount:
raise KeyError(agent_id)
except sqlite3.IntegrityError as exc:
raise ValueError(f"an agent named {name!r} already exists") from exc
assert agent_id is not None
found = self.agent(agent_id)
assert found is not None
return found
def delete_agent(self, agent_id: int) -> bool:
"""Its scores, votes and affinities go with it, by the foreign keys."""
return self._write("DELETE FROM agents WHERE id=?", (agent_id,)) > 0
def intent_vector(self, agent_id: int, key: str) -> np.ndarray | None:
"""The agent's cached intent vector, if it was made under `key`."""
row = self._row("SELECT intent_key, intent_vec FROM agents WHERE id=?",
(agent_id,))
if row is None or row["intent_key"] != key or row["intent_vec"] is None:
return None
return unpack(row["intent_vec"])
def set_intent_vector(self, agent_id: int, key: str, vec: np.ndarray) -> None:
self._write("UPDATE agents SET intent_key=?, intent_vec=? WHERE id=?",
(key, pack(vec), agent_id))
# --------------------------------------------------------------- scores
def score_candidates(self, agent_id: int, gates: Sequence[Gate]) -> list[Candidate]:
"""Every open role of a company this agent has not hidden, with the
agent's affinity for the company and how the role meets each gate: 1
match, 0 no match, -1 the column is empty. The gates are evaluated
here, in SQL, so a `require` gate is the `WHERE` it reads as, written
as a column so a gated role keeps its row and reason."""
columns: list[str] = []
params: list[Any] = []
for n, gate in enumerate(gates):
empty, match, values = _gate_sql(gate)
columns.append(
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
params += values
select = ", ".join(["r.uid", "r.posted_at",
"COALESCE(ac.affinity, 0) AS affinity", *columns])
rows = self._rows(
f"""SELECT {select} FROM roles r
LEFT JOIN agent_companies ac
ON ac.company_key = r.company_key AND ac.agent_id = ?
WHERE r.closed_at IS NULL AND COALESCE(ac.affinity, 0) <> -1""",
[*params, agent_id])
return [Candidate(uid=r["uid"], posted_at=r["posted_at"],
affinity=int(r["affinity"]),
gates=[int(r[f"g{n}"]) for n in range(len(gates))])
for r in rows]
def replace_scores(self, agent_id: int,
rows: Sequence[tuple[str, float, str]]) -> int:
"""`(uid, score, gated_reason)` for every role this agent scored,
replacing whatever it had: a role closed or muted since the last run
drops out rather than keeping a stale score. One transaction, so a
reader sees the old list or the new one. An agent deleted while its
run was scoring gets nothing written and 0 back: its rows are gone
with it, and an insert would only fail the foreign key."""
stamp = self.now()
with self.transaction() as conn:
if conn.execute("SELECT 1 FROM agents WHERE id=?", (agent_id,)).fetchone() is None:
return 0
conn.execute("DELETE FROM scores WHERE agent_id=?", (agent_id,))
conn.executemany(
"""INSERT INTO scores (uid, agent_id, score, gated_reason, scored_at)
VALUES (?,?,?,?,?)""",
[(uid, agent_id, score, reason, stamp) for uid, score, reason in rows])
return len(rows)
def scores(self, agent_id: int) -> dict[str, tuple[float, str]]:
"""`uid -> (score, gated_reason)` for one agent."""
return {r["uid"]: (float(r["score"]), str(r["gated_reason"]))
for r in self._rows(
"SELECT uid, score, gated_reason FROM scores WHERE agent_id=?",
(agent_id,))}
# ------------------------------------------------------------ companies
def add_company(self, name: str, *, key: str = "", domain: str = "",
status: str = "researching", careers_url: str = "",
notes: str = "", ats: str = "", country: str = "",
city: str = "", ats_slug: str = "") -> Company:
"""Raises ValueError on a key already taken."""
name = name.strip()
key = key or company_key(name)
if not key:
raise ValueError("a company needs a name")
try:
with self.transaction() as conn:
self._insert_company(conn, key, name, {
"domain": domain, "status": status, "careers_url": careers_url,
"notes": notes, "ats": ats, "country": country, "city": city,
"ats_slug": ats_slug})
except sqlite3.IntegrityError as exc:
raise ValueError(f"company {key!r} already exists") from exc
return self._company(key)
def _insert_company(self, conn: sqlite3.Connection, key: str, name: str,
fields: dict[str, Any]) -> None:
stamp = self.now()
columns = ["key", "name", *fields, "created_at", "updated_at"]
conn.execute(
f"INSERT INTO companies ({', '.join(columns)}) "
f"VALUES ({', '.join('?' * len(columns))})",
(key, name, *fields.values(), stamp, stamp))
def company(self, key: str, agent_id: int | None = None) -> Company | None:
"""One company, with `affinity` as `agent_id` set it; 0 without an agent."""
row = self._row(f"{_COMPANY_SELECT} WHERE c.key=?", (agent_id, key))
return Company.from_row(row) if row else None
def companies(self, agent_id: int | None = None) -> list[Company]:
"""Every company by name, with `affinity` as `agent_id` set it."""
return [Company.from_row(r) for r in self._rows(
f"{_COMPANY_SELECT} ORDER BY c.name COLLATE NOCASE", (agent_id,))]
def set_affinity(self, agent_id: int, key: str, affinity: int) -> Company:
"""One agent's stance on a company: -1 hidden, 0 neutral, 1 liked, 2
favourite. 0 deletes the row, since no row means neutral. Returns the
company with this agent's affinity. Raises ValueError for another
value and KeyError for an unknown company or agent."""
_check_affinity(affinity)
with self.transaction() as conn:
self._known(conn, "companies", "key", key)
self._known(conn, "agents", "id", agent_id)
if affinity == 0:
conn.execute("DELETE FROM agent_companies WHERE agent_id=? AND company_key=?",
(agent_id, key))
else:
conn.execute(
"""INSERT INTO agent_companies (agent_id, company_key, affinity, updated_at)
VALUES (?,?,?,?)
ON CONFLICT(agent_id, company_key) DO UPDATE
SET affinity=excluded.affinity, updated_at=excluded.updated_at""",
(agent_id, key, affinity, self.now()))
return self._company(key, agent_id)
def update_company(self, key: str, **fields: Any) -> Company:
"""Change any of `_COMPANY_FIELDS`. Affinity is per agent:
`set_affinity`."""
unknown = set(fields) - set(_COMPANY_FIELDS)
if unknown:
raise ValueError(f"not a company field: {', '.join(sorted(unknown))}")
if fields:
sets = ", ".join(f"{f}=?" for f in fields)
changed = self._write(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?",
(*fields.values(), self.now(), key))
if not changed:
raise KeyError(key)
return self._company(key)
def remove_company(self, key: str) -> bool:
return self._write("DELETE FROM companies WHERE key=?", (key,)) > 0
def _company(self, key: str, agent_id: int | None = None) -> Company:
found = self.company(key, agent_id)
if found is None:
raise KeyError(key)
return found
def open_role_counts(self) -> dict[str, int]:
"""`company_key -> open role count`, for the company list's row
count. A key with no open roles is simply absent."""
return {str(r["company_key"]): int(r["n"]) for r in self._rows(
"SELECT company_key, COUNT(*) AS n FROM roles "
"WHERE closed_at IS NULL GROUP BY company_key")}
def open_countries_by_subscription(self) -> dict[int, dict[str, int]]:
"""Per subscription id, its open roles counted by `country`."""
rows = self._rows("""SELECT subscription_id, country, COUNT(*) AS n FROM roles
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
GROUP BY subscription_id, country""")
out: dict[int, dict[str, int]] = {}
for r in rows:
out.setdefault(r["subscription_id"], {})[r["country"]] = r["n"]
return out
def company_boards(self) -> dict[str, str]:
"""`company_key -> ats host`, read off each company's own
subscription. `Subscription.company` is the display name it and the
company row were both added with, so `company_key` of one finds the
other; a company with no subscription, or a detached one (`show` 0,
see `set_company_board`), is simply absent."""
boards: dict[str, str] = {}
for sub in self.subscriptions(source="ats"):
if sub.show:
boards.setdefault(company_key(sub.company), sub.board)
return boards
def company_subscription(self, key: str) -> Subscription | None:
"""Company `key`'s own `ats` subscription, detached or not; None when
it never had one. A company has at most one: `set_company_board`
rewrites it in place rather than adding a second."""
with closing(self._connect()) as conn:
return self._company_subscription(conn, key)
@staticmethod
def _company_subscription(conn: sqlite3.Connection, key: str) -> Subscription | None:
rows = conn.execute("SELECT * FROM subscriptions WHERE source = 'ats' "
"ORDER BY show DESC, id").fetchall()
return next((Subscription.from_row(r) for r in rows
if company_key(r["company"]) == key), None)
def set_company_board(self, key: str, target: str, *, host: str = "", name: str = "",
fields: dict[str, Any] | None = None,
) -> tuple[Company, Subscription | None, int]:
"""Point company `key` at the `ats` board `target`, or at none when
`target` is "", in one transaction, so a clash on either table leaves
nothing half written.
A missing company is created from `name` and `fields`; an existing one
has `fields` written over it. Its subscription is rewritten in place:
a new target closes the old target's open roles, as switching a board
off does, and "" detaches it (off, hidden, its roles closed) while the
row stays for the next board. Returns the company, its subscription
(None when there is none) and how many roles closed. Raises KeyError
for a missing company with no board, ValueError on a UNIQUE clash."""
fields = dict(fields or {})
unknown = set(fields) - set(_COMPANY_FIELDS)
if unknown:
raise ValueError(f"not a company field: {', '.join(sorted(unknown))}")
stamp = self.now()
closed = 0
try:
with self.transaction() as conn:
row = conn.execute("SELECT name FROM companies WHERE key=?", (key,)).fetchone()
if row is None:
name = fields.pop("name", "") or name.strip()
if not target:
raise KeyError(key)
if not name:
raise ValueError("a company needs a name")
self._insert_company(conn, key, name, fields)
else:
name = fields.get("name") or row["name"]
if fields:
sets = ", ".join(f"{f}=?" for f in fields)
conn.execute(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?",
(*fields.values(), stamp, key))
sub = self._company_subscription(conn, key)
sub_id = sub.id if sub else None
if sub is not None and sub.target != target:
closed = conn.execute(
"UPDATE roles SET closed_at = ? "
"WHERE subscription_id = ? AND closed_at IS NULL",
(stamp, sub.id)).rowcount
if not target:
if sub is not None:
conn.execute("UPDATE subscriptions SET subscribed=0, fetch=0, show=0 "
"WHERE id=?", (sub.id,))
elif sub is not None:
conn.execute(
"""UPDATE subscriptions
SET target=?, board=?, label=?, company=?, subscribed=1,
fetch=1, show=1,
target_id=CASE WHEN target=? THEN target_id ELSE '' END
WHERE id=?""",
(target, host, name, name, target, sub.id))
else:
sub_id = conn.execute(
"""INSERT INTO subscriptions (source, target, label, company, board,
params, added_at)
VALUES ('ats',?,?,?,?,'{}',?)""",
(target, name, name, host, stamp)).lastrowid
except sqlite3.IntegrityError as exc:
raise ValueError(f"another board already has {target!r} or the label "
f"{name!r}") from exc
return (self._company(key), self.subscription(sub_id) if sub_id else None, closed)
# -------------------------------------------------------- subscriptions
def subscriptions(self, source: str | None = None,
fetchable: bool = False) -> list[Subscription]:
"""`fetchable` is what a pull reads: subscribed and with fetch on."""
where, params = [], []
if source:
where.append("source = ?")
params.append(source)
if fetchable:
where.append("fetch = 1 AND subscribed = 1")
sql = "SELECT * FROM subscriptions"
if where:
sql += " WHERE " + " AND ".join(where)
sql += " ORDER BY company, board, label"
return [Subscription.from_row(r) for r in self._rows(sql, params)]
def subscription(self, sub_id: int) -> Subscription | None:
row = self._row("SELECT * FROM subscriptions WHERE id=?", (sub_id,))
return Subscription.from_row(row) if row else None
def subscription_for(self, source: str, label: str) -> Subscription | None:
"""The row an item names: `Item.subscription` is the label."""
row = self._row("SELECT * FROM subscriptions WHERE source=? AND label=?",
(source, label))
return Subscription.from_row(row) if row else None
def add_subscription(self, source: str, target: str, label: str = "",
company: str = "", board: str = "",
params: dict[str, Any] | None = None,
target_id: str = "") -> Subscription:
"""Raises ValueError on a duplicate target or label within one plugin."""
target = target.strip()
if not target:
raise ValueError("a subscription needs a target")
label = label.strip() or target
try:
with self.transaction() as conn:
new_id = conn.execute(
"""INSERT INTO subscriptions (source, target, label, company, board,
params, added_at, target_id)
VALUES (?,?,?,?,?,?,?,?)""",
(source, target, label, company, board, json.dumps(params or {}),
self.now(), target_id)).lastrowid
except sqlite3.IntegrityError as exc:
raise ValueError(f"{source} already has {target!r} or the label {label!r}") from exc
assert new_id is not None
return self._subscription(new_id)
def update_subscription(self, sub_id: int, **fields: Any) -> Subscription:
"""Change labels, params, target_id or the three switches."""
unknown = set(fields) - set(_SUBSCRIPTION_FIELDS)
if unknown:
raise ValueError(f"not a subscription field: {', '.join(sorted(unknown))}")
if "params" in fields:
fields["params"] = json.dumps(fields["params"] or {})
for switch in ("subscribed", "fetch", "show"):
if switch in fields:
fields[switch] = int(bool(fields[switch]))
if fields:
sets = ", ".join(f"{f}=?" for f in fields)
try:
changed = self._write(f"UPDATE subscriptions SET {sets} WHERE id=?",
(*fields.values(), sub_id))
except sqlite3.IntegrityError as exc:
raise ValueError(f"label {fields.get('label')!r} is already in use") from exc
if not changed:
raise KeyError(sub_id)
return self._subscription(sub_id)
def unsubscribe(self, sub_id: int) -> Subscription:
"""Stop following it and hide it. The row stays, and so do its roles."""
return self.update_subscription(sub_id, subscribed=0, fetch=0, show=0)
def _subscription(self, sub_id: int) -> Subscription:
found = self.subscription(sub_id)
if found is None:
raise KeyError(sub_id)
return found
# --------------------------------------------------------- applications
def save_application(self, application: Application) -> Application:
"""Insert or update one application. An empty `id` is minted. Entering a
status, on insert or by change, appends an `application_events` row in
the same transaction."""
_check_status(application.status)
app_id = application.id or uuid4().hex
stamp = self.now()
fields = application.model_dump(exclude={"id", "created_at", "updated_at"})
with self.transaction() as conn:
before = conn.execute("SELECT status FROM applications WHERE id=?",
(app_id,)).fetchone()
if before is None:
columns = ["id", *fields, "created_at", "updated_at"]
conn.execute(
f"INSERT INTO applications ({', '.join(columns)}) "
f"VALUES ({', '.join('?' for _ in columns)})",
(app_id, *fields.values(), stamp, stamp))
else:
sets = ", ".join(f"{f}=?" for f in fields)
conn.execute(f"UPDATE applications SET {sets}, updated_at=? WHERE id=?",
(*fields.values(), stamp, app_id))
if before is None or before["status"] != application.status:
conn.execute("INSERT INTO application_events (application_id, status, at) "
"VALUES (?,?,?)", (app_id, application.status, stamp))
return self._application(app_id)
def set_application_status(self, app_id: str, status: str) -> Application:
"""Move one application to `status`. Raises ValueError for a status
outside `STATUSES` and KeyError for an unknown id."""
_check_status(status)
current = self._application(app_id)
return self.save_application(current.model_copy(update={"status": status}))
def application(self, app_id: str) -> Application | None:
row = self._row("SELECT * FROM applications WHERE id=?", (app_id,))
return Application.from_row(row) if row else None
def delete_application(self, app_id: str) -> bool:
"""Drop one application and its events. Whether that is allowed (only
while it is still `saved`) is the caller's rule, not the store's."""
return self._write("DELETE FROM applications WHERE id=?", (app_id,)) > 0
def applications(self, status: str | None = None) -> list[Application]:
sql, params = "SELECT * FROM applications", []
if status:
sql += " WHERE status = ?"
params.append(status)
sql += " ORDER BY updated_at DESC, id"
return [Application.from_row(r) for r in self._rows(sql, params)]
def application_events(self, app_id: str) -> list[ApplicationEvent]:
return [ApplicationEvent.from_row(r) for r in self._rows(
"SELECT * FROM application_events WHERE application_id=? ORDER BY id",
(app_id,))]
def _application(self, app_id: str) -> Application:
found = self.application(app_id)
if found is None:
raise KeyError(app_id)
return found
# ---------------------------------------------------------------- state
@property
def state(self) -> MutableMapping[str, str]:
"""The `state` table as a mapping. What the runner hands a plugin as
`self.state`: it can keep a counter without being able to read roles."""
return _StateMap(self)
def state_items(self, prefix: str) -> dict[str, str]:
"""Every `state` key that starts with `prefix`, with its value, in one
query, for a reader that wants one kind of key across every target."""
return {str(r["key"]): str(r["value"]) for r in self._rows(
"SELECT key, value FROM state WHERE substr(key, 1, ?) = ? ORDER BY key",
(len(prefix), prefix))}
def claim_state(self, key: str, value: str, *, released: str, stale: str) -> bool:
"""Set `key` to `value` only while it is free: unset, equal to the
value under `released` (the claim last let go of), or no later than
`stale`. Returns whether it was set. One statement, so of two
processes claiming at once exactly one wins."""
return self._write(
"""INSERT INTO state (key, value) VALUES (?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value
WHERE state.value <= ?
OR state.value = COALESCE(
(SELECT r.value FROM state AS r WHERE r.key = ?), '')""",
(key, value, stale, released)) > 0
def _check_status(status: str) -> None:
if status not in STATUSES:
raise ValueError(f"{status!r} is not an application status "
f"({', '.join(STATUSES)})")
def _check_affinity(affinity: Any) -> None:
if isinstance(affinity, bool) or not isinstance(affinity, int) or affinity not in AFFINITY:
raise ValueError(f"affinity is -1, 0, 1 or 2, not {affinity!r}")