1503 lines
76 KiB
Python
1503 lines
76 KiB
Python
"""`Store`: every SQL statement in the application.
|
|
|
|
The connection plumbing is a copy of newsdesk's `Store`, on purpose: `desk`
|
|
has no storage package. A connection is opened per call and closed again, so
|
|
one caller's commit can never land inside another's transaction; in WAL mode
|
|
that costs a file handle.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sqlite3
|
|
from collections.abc import Callable, Iterator, MutableMapping, Sequence
|
|
from contextlib import closing, contextmanager
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any, Literal
|
|
from uuid import uuid4
|
|
|
|
import numpy as np
|
|
|
|
from desk.models import Item
|
|
from app.models import (
|
|
AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate,
|
|
Company, Gate, Limits, Role, RoleRow, ScoredRole, Subscription, Weights, company_key,
|
|
country_of, excerpt, place_key, posted_at, posted_host, remote_of, title_key,
|
|
)
|
|
from app.storage import schema
|
|
|
|
#: Text columns a plugin may fill from `Item.raw` under the same name. An
|
|
#: empty value never overwrites a filled column, so an upsert from a board that
|
|
#: says less than `extract` found keeps what `extract` found. `function` is
|
|
#: not here: it is not a board field (see the note by `keep_filled` below).
|
|
_RAW_TEXT = ("place", "country", "currency", "seniority")
|
|
_RAW_INT = ("salary_min", "salary_max")
|
|
|
|
#: What `extract` may fill. Only an empty column is written: a board's own
|
|
#: structured field wins over what a model read out of the prose, and the next
|
|
#: upsert from that board would put it back anyway. `function` is not here
|
|
#: either, for the opposite reason — see `set_extracted`.
|
|
_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "permit")
|
|
|
|
#: How many replies may leave a role unanswered before `extract` stops
|
|
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
|
|
#: never answers is otherwise paid for on every pull for as long as it exists.
|
|
GIVE_UP_AFTER = 3
|
|
|
|
#: A role `extract` has yet to read: never answered, still open, not given up.
|
|
_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL "
|
|
f"AND extract_tries < {GIVE_UP_AFTER}")
|
|
|
|
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
|
|
VOTES = (-1, 0, 1, None)
|
|
|
|
#: The `judged` filter of `Store.roles`, as a `WHERE` on the agent's vote,
|
|
#: joined as `v`. A neutral vote is judged, and is neither liked nor passed.
|
|
JUDGED: dict[str, str] = {
|
|
"liked": "v.vote = 1",
|
|
"unjudged": "v.vote IS NULL",
|
|
"passed": "v.vote = -1",
|
|
}
|
|
|
|
#: The feed's Pay filter, `_roles_from`'s `pay`: the SQL that is true for
|
|
#: each choice, over `roles`. `salary_state` (`app.models.Role`) is derived,
|
|
#: not stored, so the filter spells the same rule out in SQL rather than
|
|
#: reading a column: 'normalised' is `salary_min` or `salary_max` set,
|
|
#: 'stated' is `salary_text` set and neither of those is.
|
|
PAY: dict[str, str] = {
|
|
"stated": "(salary_min IS NOT NULL OR salary_max IS NOT NULL OR salary_text != '')",
|
|
"normalised": "(salary_min IS NOT NULL OR salary_max IS NOT NULL)",
|
|
"stated_only": "(salary_text != '' AND salary_min IS NULL AND salary_max IS NULL)",
|
|
}
|
|
|
|
#: A ranking's score as `Store.roles` serves and orders it: halved by a pass,
|
|
#: lifted a fifth by a like, capped at 1. Applied at read time, so a vote
|
|
#: moves the role at once and the stored score stays what `score` computed.
|
|
_VOTED_SCORE = ("MIN(1.0, s.score * CASE v.vote WHEN -1 THEN 0.5 "
|
|
"WHEN 1 THEN 1.2 ELSE 1 END)")
|
|
|
|
#: The agent's votes beside every role. With no agent the join matches
|
|
#: nothing, so `vote` and `voted_at` are NULL: one query shape either way.
|
|
_WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
|
|
|
|
#: What `roles_page` reads for `RoleRow.excerpt`: the summary when there is
|
|
#: one, else the description's opening. A `SUBSTR` of the source column, not
|
|
#: the column itself, so a big `description` never leaves SQLite whole for a
|
|
#: row that only ever shows `EXCERPT_LIMIT` characters of it.
|
|
_EXCERPT_SOURCE = "COALESCE(NULLIF(roles.summary, ''), roles.description)"
|
|
|
|
#: How many raw characters `roles_page` pulls before `app.models.excerpt`
|
|
#: cuts it to size: slack over `EXCERPT_LIMIT` so the word-boundary trim
|
|
#: still has somewhere to land.
|
|
_EXCERPT_RAW = 240
|
|
|
|
_COMPANY_FIELDS = ("name", "domain", "status", "careers_url", "notes", "ats", "country", "city",
|
|
"ats_slug")
|
|
|
|
#: A company as the store serves it: the row and the asked-for agent's affinity.
|
|
_COMPANY_SELECT = """SELECT c.*, COALESCE(ac.affinity, 0) AS affinity FROM companies c
|
|
LEFT JOIN agent_companies ac ON ac.company_key = c.key AND ac.agent_id = ?"""
|
|
_SUBSCRIPTION_FIELDS = ("label", "company", "board", "params", "subscribed", "fetch",
|
|
"show", "target_id")
|
|
|
|
#: A tag's longest stored form, and how many a role may carry. Long enough for
|
|
#: "senior backend engineer" and short enough that the filter row stays a row.
|
|
_MAX_TAG_LEN = 40
|
|
_MAX_TAGS = 12
|
|
|
|
|
|
def _normalise_tags(tags: Sequence[str]) -> list[str]:
|
|
"""Each tag stripped, lowercased, its internal whitespace collapsed to one
|
|
space and cut to `_MAX_TAG_LEN`; empties and repeats dropped, order kept,
|
|
capped at `_MAX_TAGS`. What `set_tags` and `set_summary` store, so the
|
|
filter row never has to trust what a model or a person typed."""
|
|
seen: set[str] = set()
|
|
out: list[str] = []
|
|
for tag in tags:
|
|
cleaned = " ".join(str(tag).split()).lower()[:_MAX_TAG_LEN]
|
|
if not cleaned or cleaned in seen:
|
|
continue
|
|
seen.add(cleaned)
|
|
out.append(cleaned)
|
|
if len(out) >= _MAX_TAGS:
|
|
break
|
|
return out
|
|
|
|
|
|
def _iso(moment: datetime) -> str:
|
|
"""UTC, to the second. Every timestamp column is compared as text, so every
|
|
writer has to spell it the same way."""
|
|
return moment.astimezone(UTC).isoformat(timespec="seconds")
|
|
|
|
|
|
def _remote(value: Any) -> str:
|
|
"""A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything
|
|
unrecognised is '' (unknown), which `extract` may fill later."""
|
|
if value is True:
|
|
return "remote"
|
|
text = str(value or "").strip().lower().replace("-", "").replace(" ", "")
|
|
text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text)
|
|
return text if text in REMOTE else ""
|
|
|
|
|
|
def _like(text: str) -> str:
|
|
"""A `LIKE` pattern for a case-insensitive substring match, with SQLite's
|
|
own wildcards escaped so a search for "50%" or "a_b" stays literal."""
|
|
escaped = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
|
return f"%{escaped}%"
|
|
|
|
|
|
#: Vectors are stored little-endian float32, fixed here so a file written on
|
|
#: one machine reads on another. newsdesk's encoding, written again.
|
|
_DTYPE = np.dtype("<f4")
|
|
|
|
|
|
def pack(vec: np.ndarray) -> bytes:
|
|
return np.asarray(vec, dtype=_DTYPE).tobytes()
|
|
|
|
|
|
def unpack(blob: bytes) -> np.ndarray:
|
|
return np.frombuffer(blob, dtype=_DTYPE)
|
|
|
|
|
|
def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
|
|
"""One gate as `(empty, match, params)`: SQL that is true when the role's
|
|
column is empty, and SQL that is true when it matches, over `roles r`.
|
|
`params` belong to `match`; `empty` takes none."""
|
|
column = f"r.{gate.field}"
|
|
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
|
|
marks = ", ".join("?" for _ in values) or "NULL"
|
|
if gate.field == "salary_min":
|
|
floor = min(_int_or_none(v) or 0 for v in gate.values)
|
|
return f"{column} IS NULL", f"{column} >= ?", [floor]
|
|
if gate.field == "place":
|
|
likes = " OR ".join("LOWER(r.place) LIKE ? ESCAPE '\\'" for _ in values) or "0"
|
|
return "r.place = ''", f"({likes})", [_like(v) for v in values]
|
|
if gate.field == "company":
|
|
keys = [company_key(v) for v in values]
|
|
return ("r.company_key = ''",
|
|
f"r.company_key IN ({', '.join('?' for _ in keys) or 'NULL'})", keys)
|
|
if gate.field == "languages":
|
|
return ("r.languages IN ('', '[]')",
|
|
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
|
|
f"WHERE LOWER(value) NOT IN ({marks}))", values)
|
|
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
|
|
|
|
|
|
def _int_or_none(value: Any) -> int | None:
|
|
if value is None or value == "" or isinstance(value, bool):
|
|
return None
|
|
try:
|
|
return int(float(value))
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
class _StateMap(MutableMapping[str, str]):
|
|
"""`Store.state`: the key/value table, read and written one key at a time."""
|
|
|
|
def __init__(self, store: Store) -> None:
|
|
self._store = store
|
|
|
|
def __getitem__(self, key: str) -> str:
|
|
row = self._store._row("SELECT value FROM state WHERE key=?", (key,))
|
|
if row is None:
|
|
raise KeyError(key)
|
|
return str(row["value"])
|
|
|
|
def __setitem__(self, key: str, value: str) -> None:
|
|
self._store._write(
|
|
"""INSERT INTO state (key, value) VALUES (?,?)
|
|
ON CONFLICT(key) DO UPDATE SET value=excluded.value""",
|
|
(key, value))
|
|
|
|
def __delitem__(self, key: str) -> None:
|
|
if not self._store._write("DELETE FROM state WHERE key=?", (key,)):
|
|
raise KeyError(key)
|
|
|
|
def __iter__(self) -> Iterator[str]:
|
|
return iter([r["key"] for r in self._store._rows("SELECT key FROM state ORDER BY key")])
|
|
|
|
def __len__(self) -> int:
|
|
row = self._store._row("SELECT COUNT(*) AS n FROM state")
|
|
return int(row["n"]) if row else 0
|
|
|
|
|
|
class Store:
|
|
def __init__(self, path: str | Path) -> None:
|
|
self.path = Path(path).expanduser()
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
#: What "now" is. Replaced by tests that need two writes a second apart.
|
|
self.clock: Callable[[], datetime] = lambda: datetime.now(UTC)
|
|
with self.transaction() as conn:
|
|
# Persistent: recorded in the file, inherited by every connection.
|
|
conn.execute("PRAGMA journal_mode=WAL")
|
|
#: Migration versions this open applied; empty when already current.
|
|
self.migrations_applied: list[int] = schema.initialise(conn)
|
|
|
|
# ------------------------------------------------------------ connections
|
|
|
|
def _connect(self) -> sqlite3.Connection:
|
|
"""A fresh connection. `foreign_keys` is off by default per connection;
|
|
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
|
|
`synchronous=NORMAL` is the standard WAL companion."""
|
|
conn = sqlite3.connect(self.path)
|
|
conn.row_factory = sqlite3.Row
|
|
conn.execute("PRAGMA foreign_keys=ON")
|
|
conn.execute("PRAGMA busy_timeout=30000")
|
|
conn.execute("PRAGMA synchronous=NORMAL")
|
|
return conn
|
|
|
|
@contextmanager
|
|
def transaction(self) -> Iterator[sqlite3.Connection]:
|
|
"""One connection, one transaction, closed on the way out."""
|
|
conn = self._connect()
|
|
try:
|
|
with conn:
|
|
yield conn
|
|
finally:
|
|
conn.close()
|
|
|
|
def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]:
|
|
with closing(self._connect()) as conn:
|
|
return conn.execute(sql, params).fetchall()
|
|
|
|
def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None:
|
|
with closing(self._connect()) as conn:
|
|
row: sqlite3.Row | None = conn.execute(sql, params).fetchone()
|
|
return row
|
|
|
|
def _write(self, sql: str, params: Any = ()) -> int:
|
|
"""One statement in its own transaction. Returns the rowcount."""
|
|
with self.transaction() as conn:
|
|
return conn.execute(sql, params).rowcount
|
|
|
|
def now(self) -> str:
|
|
return _iso(self.clock())
|
|
|
|
# ---------------------------------------------------------------- roles
|
|
|
|
def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool:
|
|
"""Store one posting. Returns True when it was new.
|
|
|
|
`first_seen` and `created_at` are written on insert only; `last_seen` on
|
|
every call, and a role seen again is open again. The gate columns come
|
|
from `item.raw` under their own names, with `company` falling back to
|
|
the subscription's and `country` to what `place` names; an empty value
|
|
never overwrites a filled column.
|
|
|
|
`posted_at` (see `app.models.posted_at`) is set from this pull's own
|
|
`published_at` and `stamp`; on an update the `ON CONFLICT` clause below
|
|
keeps the earlier of that and what is already stored, so an edit on
|
|
the board — which can only ever move `published_at` later, never
|
|
earlier — cannot move `posted_at` later either.
|
|
|
|
`function` is the one gate column no board actually sends: the `ats`
|
|
plugin's own keyword guess rides in `raw` under that name because
|
|
`raw` is the only channel a plugin has, but a guess is not first-party
|
|
the way a board's own structured field is (docs/decisions.md, "a
|
|
guess is not a board field"). So once a role has been through
|
|
`extract` (`extracted_at` is set), this upsert leaves `function`
|
|
alone for good, whatever the plugin's raw guess now says — see
|
|
`set_extracted` for the other half of the rule.
|
|
"""
|
|
if not item.source:
|
|
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
|
|
raw = item.raw or {}
|
|
company = str(raw.get("company") or (subscription.company if subscription else ""))
|
|
place = str(raw.get("place") or "")
|
|
stamp = self.now()
|
|
published = _iso(item.published_at)
|
|
row: dict[str, Any] = {
|
|
"uid": item.uid,
|
|
"source": item.source,
|
|
"source_id": item.source_id,
|
|
"subscription_id": subscription.id if subscription else None,
|
|
"company_key": company_key(company),
|
|
"company": company,
|
|
"title": item.title,
|
|
"title_key": title_key(item.title),
|
|
"place_key": place_key(place),
|
|
"remote": _remote(raw.get("remote")),
|
|
"description": item.body,
|
|
"feed_text": item.excerpt,
|
|
"original_url": item.location,
|
|
"published_at": published,
|
|
"posted_at": posted_at(published, stamp, posted_host(raw)),
|
|
"first_seen": stamp,
|
|
"last_seen": stamp,
|
|
"raw": json.dumps(raw, default=str),
|
|
"created_at": stamp,
|
|
}
|
|
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
|
|
row["country"] = row["country"] or country_of(place)
|
|
row["remote"] = row["remote"] or remote_of(place)
|
|
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
|
row["function"] = str(raw.get("function") or "")
|
|
# A board's own pay line, one per pull like title or published_at —
|
|
# not kept-filled like _RAW_TEXT: a board that stops stating pay
|
|
# must be able to clear it, not leave a stale line behind.
|
|
row["salary_text"] = str(raw.get("salary_text") or "")
|
|
|
|
keep_filled = ("company", "company_key", "place_key", "remote", "description",
|
|
"feed_text", "original_url", *_RAW_TEXT)
|
|
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
|
|
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
|
|
for c in (*_RAW_INT, "subscription_id")]
|
|
updates += [f"{c}=excluded.{c}"
|
|
for c in ("title", "title_key", "published_at", "raw", "last_seen",
|
|
"salary_text")]
|
|
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
|
|
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
|
|
# pull's classifier guess overwrite what extract decided, which is
|
|
# the flip the "board wins" rule exists to prevent for an actual
|
|
# board field — see the docstring above. Once extracted_at is set,
|
|
# function is extract's to own; before that, it is the guess, same
|
|
# as any other raw field.
|
|
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
|
|
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
|
|
"ELSE roles.function END")
|
|
updates.append("closed_at=NULL")
|
|
columns = ", ".join(row)
|
|
marks = ", ".join(f":{c}" for c in row)
|
|
with self.transaction() as conn:
|
|
existed = conn.execute("SELECT 1 FROM roles WHERE uid=?",
|
|
(row["uid"],)).fetchone() is not None
|
|
conn.execute(
|
|
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
|
|
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
|
|
return not existed
|
|
|
|
def role(self, uid: str, agent_id: int | None = None) -> Role | None:
|
|
"""One role with its tags, and with `vote` and `voted_at` as
|
|
`agent_id` judged it; None (unjudged) without an agent."""
|
|
row = self._row(f"SELECT roles.*, v.vote, v.voted_at FROM roles {_WITH_VOTES} "
|
|
"WHERE roles.uid=?", (agent_id, uid))
|
|
if row is None:
|
|
return None
|
|
role = Role.from_row(row)
|
|
role.tags = self._tags_of(uid)
|
|
return role
|
|
|
|
def set_read(self, uid: str, read: bool = True) -> Role:
|
|
"""Mark a role read, or unread again. Stamps `read_at` only the first
|
|
time — a role already read keeps the moment it was first opened —
|
|
and clears it outright when `read` is False. Raises KeyError for an
|
|
unknown uid."""
|
|
if read:
|
|
changed = self._write(
|
|
"UPDATE roles SET read_at = COALESCE(read_at, ?) WHERE uid = ?",
|
|
(self.now(), uid))
|
|
else:
|
|
changed = self._write("UPDATE roles SET read_at = NULL WHERE uid = ?", (uid,))
|
|
if not changed:
|
|
raise KeyError(uid)
|
|
role = self.role(uid)
|
|
assert role is not None
|
|
return role
|
|
|
|
def set_vote(self, agent_id: int, uid: str, value: int | None) -> Role:
|
|
"""One agent's judgement of one role: 1 like, 0 neither way, -1 pass,
|
|
None unjudged again, which deletes the row. A vote never hides a
|
|
role; it moves it in that agent's ranking (see `roles`) and shapes its
|
|
taste direction (see `jobs/score.py`). Voting is reading, so a vote
|
|
stamps the role's `read_at` too, the same once-only way `set_read`
|
|
does; clearing a vote is not un-reading, and leaves it alone. Returns
|
|
the role with this agent's vote. Raises ValueError for another value
|
|
and KeyError for an unknown uid or agent."""
|
|
if isinstance(value, bool) or value not in VOTES:
|
|
raise ValueError(f"a vote is -1, 0 or 1, not {value!r}")
|
|
with self.transaction() as conn:
|
|
self._known(conn, "roles", "uid", uid)
|
|
self._known(conn, "agents", "id", agent_id)
|
|
if value is None:
|
|
conn.execute("DELETE FROM role_votes WHERE agent_id=? AND uid=?",
|
|
(agent_id, uid))
|
|
else:
|
|
now = self.now()
|
|
conn.execute(
|
|
"""INSERT INTO role_votes (agent_id, uid, vote, voted_at) VALUES (?,?,?,?)
|
|
ON CONFLICT(agent_id, uid) DO UPDATE
|
|
SET vote=excluded.vote, voted_at=excluded.voted_at""",
|
|
(agent_id, uid, value, now))
|
|
conn.execute("UPDATE roles SET read_at = COALESCE(read_at, ?) WHERE uid = ?",
|
|
(now, uid))
|
|
role = self.role(uid, agent_id)
|
|
assert role is not None
|
|
return role
|
|
|
|
def votes(self, agent_id: int) -> dict[str, int]:
|
|
"""`uid -> vote` for one agent, open roles and closed alike: a closed
|
|
role somebody liked still says what they like."""
|
|
return {r["uid"]: int(r["vote"]) for r in self._rows(
|
|
"SELECT uid, vote FROM role_votes WHERE agent_id=?", (agent_id,))}
|
|
|
|
@staticmethod
|
|
def _known(conn: sqlite3.Connection, table: str, column: str, value: Any) -> None:
|
|
"""KeyError unless the row exists: the caller's 404, rather than the
|
|
foreign key's IntegrityError, which does not say which side is unknown."""
|
|
if conn.execute(f"SELECT 1 FROM {table} WHERE {column}=?", (value,)).fetchone() is None:
|
|
raise KeyError(value)
|
|
|
|
def set_tags(self, uid: str, tags: Sequence[str]) -> list[str]:
|
|
"""Replace one role's tags in one transaction. See `_normalise_tags`
|
|
for how each one is cleaned up. Returns what was stored, sorted the
|
|
way `role` and `roles` read it back. Raises KeyError for an unknown
|
|
uid."""
|
|
with self.transaction() as conn:
|
|
if conn.execute("SELECT 1 FROM roles WHERE uid=?", (uid,)).fetchone() is None:
|
|
raise KeyError(uid)
|
|
return self._replace_tags(conn, uid, tags)
|
|
|
|
def _replace_tags(self, conn: sqlite3.Connection, uid: str,
|
|
tags: Sequence[str]) -> list[str]:
|
|
"""`set_tags`'s write, against an already-open transaction so
|
|
`set_summary` can share it."""
|
|
cleaned = _normalise_tags(tags)
|
|
conn.execute("DELETE FROM role_tags WHERE uid=?", (uid,))
|
|
conn.executemany("INSERT INTO role_tags (uid, tag) VALUES (?,?)",
|
|
[(uid, tag) for tag in cleaned])
|
|
return sorted(cleaned)
|
|
|
|
def _tags_of(self, uid: str) -> list[str]:
|
|
return [str(r["tag"]) for r in self._rows(
|
|
"SELECT tag FROM role_tags WHERE uid=? ORDER BY tag", (uid,))]
|
|
|
|
def _tags_by_uid(self, uids: Sequence[str]) -> dict[str, list[str]]:
|
|
"""Every tag of these roles, by uid, sorted — one query for a whole
|
|
page rather than one per role."""
|
|
if not uids:
|
|
return {}
|
|
out: dict[str, list[str]] = {}
|
|
rows = self._rows(
|
|
"SELECT uid, tag FROM role_tags WHERE uid IN (SELECT value FROM json_each(?)) "
|
|
"ORDER BY uid, tag", (json.dumps(list(uids)),))
|
|
for r in rows:
|
|
out.setdefault(r["uid"], []).append(str(r["tag"]))
|
|
return out
|
|
|
|
def roles(self, limit: int = 100, company: str | None = None,
|
|
place: str | None = None, open_only: bool = True,
|
|
q: str | None = None, function: str | None = None,
|
|
remote: str | None = None, days: int | None = None,
|
|
hide_muted: bool = False, include_read: bool = True,
|
|
tags: Sequence[str] = (), tag_mode: str = "any",
|
|
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
|
judged: str | None = None, pay: str | None = None,
|
|
offset: int = 0) -> list[ScoredRole]:
|
|
"""Newest first by `posted_at`. `company` and `place` are
|
|
case-insensitive substrings of the columns a board wrote — not the
|
|
key, so a search does not have to know the exact slug a company folds
|
|
to. `q` is matched word by word against `title`; a role must match
|
|
every word. `function` and `remote` match a gate column exactly.
|
|
`days` keeps only roles posted within that many days of now. `pay`
|
|
(a key of `PAY`) filters on `salary_state`.
|
|
|
|
`agent` says whose votes, hidden companies and scores are meant. Each
|
|
row carries that agent's `vote` and `voted_at`; `judged` (a key of
|
|
`JUDGED`: liked, unjudged or passed) filters on them. `hide_muted`
|
|
excludes the companies that agent hid, before `limit` is applied, in
|
|
SQL rather than after the fact: a company that has published a lot,
|
|
all at once — a fresh board added, a re-pull that finally reached
|
|
it — can otherwise fill the whole page with roles that then all turn
|
|
out to be from the one company someone hid, leaving nothing to show
|
|
for a default query that would otherwise have plenty. With no agent
|
|
every vote is None, nothing is hidden and `judged` reads as such.
|
|
|
|
`include_read` set False excludes a role `set_read` or `set_vote` has
|
|
stamped, the same way and for the same reason as `hide_muted`:
|
|
before `limit`, not after. `tags` keeps only roles carrying at least
|
|
one of them (`tag_mode="any"`, the default) or every one
|
|
(`tag_mode="all"`), also before `limit`.
|
|
|
|
With `agent` and `by_score`, ordered by that agent's score instead,
|
|
highest first, carrying `score` and `gated_reason` on each row. Only
|
|
roles it has scored are listed, and a gated one only when `gated` is
|
|
set. The score is the stored one moved by the agent's vote
|
|
(`_VOTED_SCORE`), so a vote shows in the order at once and needs no
|
|
rescore. With `by_score` off the order is newest, with the votes and
|
|
no score.
|
|
|
|
`offset` skips that many rows of the same order, for the page that
|
|
loads the next 60 as it is scrolled; `count_roles` is the total.
|
|
Every returned role carries its tags, read with one extra query for
|
|
the whole page rather than one per role."""
|
|
sql, params = self._roles_from(
|
|
company=company, place=place, open_only=open_only, q=q, function=function,
|
|
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
|
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
|
judged=judged, pay=pay)
|
|
select = "SELECT roles.*, v.vote, v.voted_at"
|
|
order = "roles.posted_at DESC, roles.uid"
|
|
scored = agent is not None and by_score
|
|
if scored:
|
|
select += f", {_VOTED_SCORE} AS score, s.gated_reason"
|
|
order = "score DESC, " + order
|
|
elif open_only:
|
|
# Newest first, open only: idx_roles_posted covers this exactly
|
|
# (posted_at DESC, uid, WHERE closed_at IS NULL), so it earns the
|
|
# LIMIT rather than sorting every open role first. Left to the
|
|
# planner, sqlite3's bundled build (no STAT4) reads the near-100%
|
|
# selectivity of `closed_at IS NULL` from idx_roles_closed's plain
|
|
# average and picks that instead, sorting the lot — checked with
|
|
# EXPLAIN QUERY PLAN on a copy of the live database.
|
|
sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1)
|
|
found = [ScoredRole.from_row(r) for r in self._rows(
|
|
f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?", [*params, limit, offset])]
|
|
by_uid = self._tags_by_uid([r.uid for r in found])
|
|
for r in found:
|
|
r.tags = by_uid.get(r.uid, [])
|
|
return found
|
|
|
|
def roles_page(self, limit: int = 100, company: str | None = None,
|
|
place: str | None = None, open_only: bool = True,
|
|
q: str | None = None, function: str | None = None,
|
|
remote: str | None = None, days: int | None = None,
|
|
hide_muted: bool = False, include_read: bool = True,
|
|
tags: Sequence[str] = (), tag_mode: str = "any",
|
|
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
|
judged: str | None = None, pay: str | None = None,
|
|
offset: int = 0) -> list[RoleRow]:
|
|
"""What `GET /api/roles` serves: the same rows `roles` would, in the
|
|
same order, with the same filters and paging, but only the fields a
|
|
row draws (see `RoleRow`) — never `description`, `feed_text`, `raw`
|
|
or `article_text`, which a full `roles.*` select would carry whether
|
|
or not a page ever renders them. The reader still gets the whole role
|
|
by uid, see `role`.
|
|
|
|
`excerpt` is `app.models.excerpt` over a `SUBSTR` of `_EXCERPT_SOURCE`
|
|
(`_EXCERPT_RAW` characters, trimmed further in Python), so the source
|
|
column itself never crosses out of SQLite for a list."""
|
|
sql, params = self._roles_from(
|
|
company=company, place=place, open_only=open_only, q=q, function=function,
|
|
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
|
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
|
judged=judged, pay=pay)
|
|
select = (
|
|
"SELECT roles.uid, roles.company_key, roles.company, roles.title, roles.place, "
|
|
"roles.country, roles.remote, roles.function, roles.salary_min, "
|
|
"roles.salary_max, roles.currency, roles.salary_text, roles.original_url, "
|
|
"roles.posted_at, roles.read_at, "
|
|
"v.vote, v.voted_at, "
|
|
f"SUBSTR({_EXCERPT_SOURCE}, 1, {_EXCERPT_RAW}) AS excerpt")
|
|
order = "roles.posted_at DESC, roles.uid"
|
|
scored = agent is not None and by_score
|
|
if scored:
|
|
select += f", {_VOTED_SCORE} AS score, s.gated_reason"
|
|
order = "score DESC, " + order
|
|
elif open_only:
|
|
# Same reasoning as `roles`: the planner otherwise sorts every
|
|
# open role before applying LIMIT.
|
|
sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1)
|
|
rows = self._rows(f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?",
|
|
[*params, limit, offset])
|
|
found = []
|
|
for r in rows:
|
|
row = dict(r)
|
|
row["excerpt"] = excerpt(row["excerpt"])
|
|
found.append(RoleRow.from_row(row))
|
|
by_uid = self._tags_by_uid([r.uid for r in found])
|
|
for r in found:
|
|
r.tags = by_uid.get(r.uid, [])
|
|
return found
|
|
|
|
def count_roles(self, company: str | None = None, place: str | None = None,
|
|
open_only: bool = True, q: str | None = None,
|
|
function: str | None = None, remote: str | None = None,
|
|
days: int | None = None, hide_muted: bool = False,
|
|
include_read: bool = True, tags: Sequence[str] = (),
|
|
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
|
|
gated: bool = False, judged: str | None = None,
|
|
pay: str | None = None) -> int:
|
|
"""How many roles `roles` would list with the same filters and no limit."""
|
|
sql, params = self._roles_from(
|
|
company=company, place=place, open_only=open_only, q=q, function=function,
|
|
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
|
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
|
judged=judged, pay=pay)
|
|
row = self._row(f"SELECT COUNT(*) AS n {sql}", params)
|
|
return int(row["n"]) if row else 0
|
|
|
|
def _roles_from(self, *, company: str | None, place: str | None, open_only: bool,
|
|
q: str | None, function: str | None, remote: str | None,
|
|
days: int | None, hide_muted: bool, include_read: bool,
|
|
tags: Sequence[str], tag_mode: str, agent: int | None,
|
|
by_score: bool, gated: bool, judged: str | None,
|
|
pay: str | None = None) -> tuple[str, list[Any]]:
|
|
"""The `FROM ... WHERE ...` that `roles` and `count_roles` share, with
|
|
its parameters in order. The agent's votes are always joined as `v`
|
|
(see `_WITH_VOTES`); its scores as `s` only when ordering by them."""
|
|
where: list[str] = []
|
|
params: list[Any] = []
|
|
if company:
|
|
where.append("company LIKE ? ESCAPE '\\'")
|
|
params.append(_like(company))
|
|
if place:
|
|
where.append("place LIKE ? ESCAPE '\\'")
|
|
params.append(_like(place))
|
|
if function:
|
|
where.append("function = ?")
|
|
params.append(function)
|
|
if remote:
|
|
where.append("remote = ?")
|
|
params.append(remote)
|
|
if days is not None:
|
|
where.append("posted_at >= ?")
|
|
params.append(_iso(self.clock() - timedelta(days=days)))
|
|
for word in (q or "").split():
|
|
where.append("title LIKE ? ESCAPE '\\'")
|
|
params.append(_like(word))
|
|
if judged:
|
|
if judged not in JUDGED:
|
|
raise ValueError(f"judged is {', '.join(JUDGED)}, not {judged!r}")
|
|
where.append(JUDGED[judged])
|
|
if pay:
|
|
if pay not in PAY:
|
|
raise ValueError(f"pay is {', '.join(PAY)}, not {pay!r}")
|
|
where.append(PAY[pay])
|
|
if open_only:
|
|
where.append("closed_at IS NULL")
|
|
if hide_muted and agent is not None:
|
|
where.append("company_key NOT IN (SELECT company_key FROM agent_companies "
|
|
"WHERE agent_id = ? AND affinity = -1)")
|
|
params.append(agent)
|
|
if not include_read:
|
|
where.append("roles.read_at IS NULL")
|
|
if tags:
|
|
marks = ", ".join("?" for _ in tags)
|
|
if tag_mode == "all":
|
|
where.append(
|
|
f"(SELECT COUNT(DISTINCT rt.tag) FROM role_tags rt "
|
|
f"WHERE rt.uid = roles.uid AND rt.tag IN ({marks})) = ?")
|
|
params += [*tags, len(tags)]
|
|
elif tag_mode == "any":
|
|
where.append(
|
|
f"EXISTS (SELECT 1 FROM role_tags rt "
|
|
f"WHERE rt.uid = roles.uid AND rt.tag IN ({marks}))")
|
|
params += list(tags)
|
|
else:
|
|
raise ValueError(f"tag_mode is any or all, not {tag_mode!r}")
|
|
sql = f"FROM roles {_WITH_VOTES}"
|
|
joins: list[Any] = [agent]
|
|
if agent is not None and by_score:
|
|
sql += " JOIN scores s ON s.uid = roles.uid AND s.agent_id = ?"
|
|
joins.append(agent)
|
|
if not gated:
|
|
where.append("s.gated_reason = ''")
|
|
if where:
|
|
sql += " WHERE " + " AND ".join(where)
|
|
return sql, [*joins, *params]
|
|
|
|
def tag_counts(self, agent_id: int | None = None,
|
|
show_muted: bool = False) -> list[tuple[str, int, int]]:
|
|
"""`(tag, open_count, unread_count)` over open roles, most-used tag
|
|
first and then alphabetically. What `/api/filters` serves; the
|
|
companies `agent_id` hid are left out unless `show_muted`, the same
|
|
default as `roles`, and with no agent nothing is hidden."""
|
|
where = "r.closed_at IS NULL"
|
|
params: list[Any] = []
|
|
if not show_muted and agent_id is not None:
|
|
where += (" AND r.company_key NOT IN (SELECT company_key FROM agent_companies "
|
|
"WHERE agent_id = ? AND affinity = -1)")
|
|
params.append(agent_id)
|
|
rows = self._rows(
|
|
f"""SELECT rt.tag AS tag, COUNT(*) AS n, SUM(r.read_at IS NULL) AS unread
|
|
FROM role_tags rt JOIN roles r ON r.uid = rt.uid
|
|
WHERE {where}
|
|
GROUP BY rt.tag
|
|
ORDER BY n DESC, rt.tag ASC""", params)
|
|
return [(str(r["tag"]), int(r["n"]), int(r["unread"])) for r in rows]
|
|
|
|
def mark_closed(self, source: str, seen_ids: Sequence[str],
|
|
subscription_id: int) -> int:
|
|
"""Close every open role of one board that a complete fetch of it did not
|
|
list. Returns how many closed. Only call it after a fetch that finished:
|
|
a failed one lists nothing, and would close the whole board."""
|
|
return self._write(
|
|
"""UPDATE roles SET closed_at = ?
|
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
|
AND source_id NOT IN (SELECT value FROM json_each(?))""",
|
|
(self.now(), source, subscription_id, json.dumps(list(seen_ids))))
|
|
|
|
def close_unseen(self, source: str, before: str) -> int:
|
|
"""Close every open role of `source` last seen before `before`. For a
|
|
source whose fetch is never a complete list, a search, where a role
|
|
dropping out of one run's results is not the role closing."""
|
|
return self._write(
|
|
"""UPDATE roles SET closed_at = ?
|
|
WHERE source = ? AND closed_at IS NULL AND last_seen < ?""",
|
|
(self.now(), source, before))
|
|
|
|
def close_subscription(self, subscription_id: int) -> int:
|
|
"""Close every open role of one subscription, for a board that is no
|
|
longer fetched: nothing would ever close them otherwise."""
|
|
return self._write(
|
|
"UPDATE roles SET closed_at = ? WHERE subscription_id = ? AND closed_at IS NULL",
|
|
(self.now(), subscription_id))
|
|
|
|
def backfill_from_place(self) -> int:
|
|
"""Fill `country` and `remote` from `place` on every role that has
|
|
none, the way `upsert_role` does for a new one, so a word added to the
|
|
lookups reaches the roles already stored. Returns how many columns it
|
|
filled."""
|
|
rows = self._rows("SELECT uid, place FROM roles "
|
|
"WHERE place != '' AND (country = '' OR remote = '')")
|
|
countries = [(code, r["uid"]) for r in rows if (code := country_of(r["place"]))]
|
|
remotes = [(mode, r["uid"]) for r in rows if (mode := remote_of(r["place"]))]
|
|
with self.transaction() as conn:
|
|
conn.executemany("UPDATE roles SET country = ? WHERE uid = ? AND country = ''",
|
|
countries)
|
|
conn.executemany("UPDATE roles SET remote = ? WHERE uid = ? AND remote = ''",
|
|
remotes)
|
|
return len(countries) + len(remotes)
|
|
|
|
# -------------------------------------------------------------- extract
|
|
|
|
def _top_scope(self, stage: Literal["extract", "summarise"]) -> tuple[str, list[Any]]:
|
|
"""The `WHERE` fragment for a role among the best ungated scores of
|
|
an enabled agent, as deep and as low as that agent's own `limits`
|
|
say for `stage`: what the two model stages, extract and summarise,
|
|
are bounded to. The rank is numbered per agent and the agent's
|
|
limits ride out of the window subquery beside it, so each row is
|
|
held to the limits of the agent that ranked it; a key the JSON lacks
|
|
is the `Limits` default. The role is `r`. A company an agent hid
|
|
needs no clause: `score` writes no rows for it under that agent. An
|
|
agent without an intent counts for nothing here: its scores have no
|
|
fit, so they say how fresh a role is, not whether it is wanted."""
|
|
defaults = Limits()
|
|
return (f"""r.uid IN (SELECT uid FROM (
|
|
SELECT s.uid, s.score, a.limits, ROW_NUMBER() OVER (
|
|
PARTITION BY s.agent_id
|
|
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
|
|
FROM scores s
|
|
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
|
|
AND TRIM(a.intent, ' ' || char(9, 10, 13)) <> ''
|
|
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
|
WHERE s.gated_reason = '')
|
|
WHERE n <= COALESCE(json_extract(limits, '$.{stage}_top'), ?)
|
|
AND score >= COALESCE(json_extract(limits, '$.{stage}_min'), ?))""",
|
|
[getattr(defaults, f"{stage}_top"), getattr(defaults, f"{stage}_min")])
|
|
|
|
def _extract_scope(self, uids: Sequence[str] | None) -> tuple[str, list[Any]]:
|
|
"""The `WHERE` for a role `extract` may read: open, never answered,
|
|
not given up. Given `uids`, those of them and nothing else, since a
|
|
batch asks again when it runs and must finish what it was dealt;
|
|
otherwise only the roles in `_top_scope`."""
|
|
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
|
|
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")]
|
|
if uids is not None:
|
|
where.append("r.uid IN (SELECT value FROM json_each(?))")
|
|
return " AND ".join(where), [json.dumps(list(uids))]
|
|
top, params = self._top_scope("extract")
|
|
return " AND ".join([*where, top]), params
|
|
|
|
def extract_pending(self) -> int:
|
|
where, params = self._extract_scope(None)
|
|
row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params)
|
|
return int(row["n"]) if row else 0
|
|
|
|
def extract_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]:
|
|
"""Roles `extract` has yet to read, newest first: the best of every
|
|
enabled agent's scores, as deep and as low as that agent's own
|
|
`limits` say, so the backlog follows the rankings rather than the
|
|
boards. Given `uids`, only those of them still waiting, whatever
|
|
their rank."""
|
|
where, params = self._extract_scope(uids)
|
|
return [Role.from_row(r) for r in self._rows(
|
|
f"SELECT r.* FROM roles r WHERE {where} "
|
|
"ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
|
|
|
|
def set_extracted(self, uid: str, fields: dict[str, Any]) -> int | None:
|
|
"""Write what `extract` read for one role and stamp `extracted_at`.
|
|
|
|
Only into empty columns, and only a non-empty value: a board's own
|
|
field is never replaced and nothing is ever blanked. `place` brings
|
|
`place_key` with it. Returns how many columns were filled, or None
|
|
for an unknown uid. Values are trusted to be in vocabulary; the
|
|
job's `apply` is where a reply is checked.
|
|
|
|
`function` is the one exception to "only into empty columns": it is
|
|
the `ats` classifier's keyword guess, not a board field, so a
|
|
non-empty answer overwrites it outright rather than only filling an
|
|
empty column — see `upsert_role`'s docstring for the other half of
|
|
the rule that then keeps a later pull from putting the guess back."""
|
|
with self.transaction() as conn:
|
|
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
|
|
if row is None:
|
|
return None
|
|
sets: dict[str, Any] = {}
|
|
for name in _EXTRACT_TEXT:
|
|
value = str(fields.get(name) or "")
|
|
if value and not row[name]:
|
|
sets[name] = value
|
|
function = str(fields.get("function") or "")
|
|
if function:
|
|
sets["function"] = function
|
|
for name in _RAW_INT:
|
|
number = _int_or_none(fields.get(name))
|
|
if number is not None and row[name] is None:
|
|
sets[name] = number
|
|
languages = [str(code) for code in fields.get("languages") or []]
|
|
if languages and json.loads(row["languages"] or "[]") == []:
|
|
sets["languages"] = json.dumps(languages)
|
|
filled = len(sets)
|
|
if "place" in sets:
|
|
sets["place_key"] = place_key(sets["place"])
|
|
sets["extracted_at"] = self.now()
|
|
conn.execute(
|
|
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
|
|
(*sets.values(), uid))
|
|
return filled
|
|
|
|
def extract_missed(self, uids: Sequence[str], why: str = "") -> int:
|
|
"""One reply that left these roles unanswered. Counted only against
|
|
those still waiting, so a role the reply did answer is not charged.
|
|
Returns how many this pushed to `GIVE_UP_AFTER`. `why` is accepted
|
|
for `ModelTask.missed` and not kept: there is no Given up list yet."""
|
|
if not uids:
|
|
return 0
|
|
where = f"uid IN (SELECT value FROM json_each(?)) AND {_EXTRACT_WAITING}"
|
|
wanted = json.dumps(list(uids))
|
|
with self.transaction() as conn:
|
|
crossed = conn.execute(
|
|
f"SELECT COUNT(*) FROM roles WHERE {where} AND extract_tries + 1 >= ?",
|
|
(wanted, GIVE_UP_AFTER)).fetchone()[0]
|
|
conn.execute(f"UPDATE roles SET extract_tries = extract_tries + 1 "
|
|
f"WHERE {where}", (wanted,))
|
|
return int(crossed)
|
|
|
|
# ------------------------------------------------------------ summarise
|
|
|
|
#: The `state` key counting replies that left a role without a summary.
|
|
#: `roles` has no column for it, and the count only matters for the few
|
|
#: roles in some agent's top list.
|
|
_SUMMARY_MISSED = "summarise.missed:"
|
|
|
|
def _summary_scope(self, uids: Sequence[str] | None) -> tuple[str, list[Any]]:
|
|
"""The `WHERE` for a role `summarise` may be sent: no summary yet and
|
|
some text to summarise. Given `uids`, those of them and nothing else,
|
|
open or not and given up or not, because somebody asked for them.
|
|
Otherwise open, not given up, and in `_top_scope`."""
|
|
where = ["r.summary = ''",
|
|
"(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"]
|
|
if uids is not None:
|
|
where.append("r.uid IN (SELECT value FROM json_each(?))")
|
|
return " AND ".join(where), [json.dumps(list(uids))]
|
|
top, params = self._top_scope("summarise")
|
|
where += [
|
|
"r.closed_at IS NULL",
|
|
f"""NOT EXISTS (SELECT 1 FROM state
|
|
WHERE key = '{self._SUMMARY_MISSED}' || r.uid
|
|
AND CAST(value AS INTEGER) >= {GIVE_UP_AFTER})""",
|
|
top,
|
|
]
|
|
return " AND ".join(where), params
|
|
|
|
def summarise_pending(self) -> int:
|
|
where, params = self._summary_scope(None)
|
|
row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params)
|
|
return int(row["n"]) if row else 0
|
|
|
|
def summarise_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]:
|
|
"""Roles `summarise` has yet to write, newest first: the top of every
|
|
enabled agent's scores, as deep and as low as that agent's own
|
|
`limits` say, so the backlog is bounded by the agents rather than by
|
|
the number of open roles; given `uids`, those still without a
|
|
summary, whatever their rank. A batch asks again when it runs."""
|
|
where, params = self._summary_scope(uids)
|
|
return [Role.from_row(r) for r in self._rows(
|
|
f"SELECT r.* FROM roles r WHERE {where} "
|
|
"ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
|
|
|
|
def set_summary(self, uid: str, summary: str, tags: Sequence[str] = ()) -> bool:
|
|
"""Write a role's summary and stamp `summarised_at`. Only over an empty
|
|
one, so a second batch that carried the same role does not replace the
|
|
first answer. Given `tags`, they replace the role's tags in the same
|
|
transaction — see `_replace_tags` — but only when the summary was
|
|
actually written, so a rejected late answer cannot plant tags on a
|
|
role it did not get to summarise. False for an unknown uid or one
|
|
already summarised."""
|
|
with self.transaction() as conn:
|
|
written = conn.execute(
|
|
"UPDATE roles SET summary = ?, summarised_at = ? "
|
|
"WHERE uid = ? AND summary = ''", (summary, self.now(), uid)).rowcount
|
|
conn.execute("DELETE FROM state WHERE key = ?", (self._SUMMARY_MISSED + uid,))
|
|
if written and tags:
|
|
self._replace_tags(conn, uid, tags)
|
|
return bool(written)
|
|
|
|
def summarise_missed(self, uids: Sequence[str], why: str = "") -> int:
|
|
"""One reply that left these roles without a summary. Counted only
|
|
against those still without one. Returns how many this pushed to
|
|
`GIVE_UP_AFTER`. `why` is accepted for `ModelTask.missed` and not kept."""
|
|
if not uids:
|
|
return 0
|
|
rows = self._rows("SELECT uid FROM roles WHERE summary = '' "
|
|
"AND uid IN (SELECT value FROM json_each(?))",
|
|
(json.dumps(list(uids)),))
|
|
keys = [self._SUMMARY_MISSED + r["uid"] for r in rows]
|
|
with self.transaction() as conn:
|
|
conn.executemany(
|
|
"""INSERT INTO state (key, value) VALUES (?, '1')
|
|
ON CONFLICT(key) DO UPDATE
|
|
SET value = CAST(CAST(value AS INTEGER) + 1 AS TEXT)""",
|
|
[(key,) for key in keys])
|
|
crossed = conn.execute(
|
|
"SELECT COUNT(*) FROM state WHERE key IN (SELECT value FROM json_each(?)) "
|
|
"AND CAST(value AS INTEGER) = ?", (json.dumps(keys), GIVE_UP_AFTER)).fetchone()[0]
|
|
return int(crossed)
|
|
|
|
def set_article_text(self, uid: str, text: str) -> Role | None:
|
|
"""Store the posting page as plain text and stamp `article_at`. The
|
|
role as it now stands, or None for an unknown uid."""
|
|
if not self._write("UPDATE roles SET article_text = ?, article_at = ? WHERE uid = ?",
|
|
(text, self.now(), uid)):
|
|
return None
|
|
return self.role(uid)
|
|
|
|
# -------------------------------------------------------------- vectors
|
|
|
|
def roles_to_embed(self, model: str, limit: int) -> list[Role]:
|
|
"""Open roles with no vector from `model`, newest first, so a backlog
|
|
embedded from the top improves what is on screen first."""
|
|
return [Role.from_row(r) for r in self._rows(
|
|
"""SELECT r.* FROM roles r
|
|
LEFT JOIN vectors v ON v.uid = r.uid AND v.model = ?
|
|
WHERE v.uid IS NULL AND r.closed_at IS NULL
|
|
ORDER BY r.first_seen DESC, r.uid LIMIT ?""", (model, limit))]
|
|
|
|
def put_vectors(self, model: str, pairs: Sequence[tuple[str, np.ndarray]]) -> int:
|
|
stamp = self.now()
|
|
with self.transaction() as conn:
|
|
return conn.executemany(
|
|
"""INSERT INTO vectors (uid, model, dim, vec, embedded_at)
|
|
VALUES (?,?,?,?,?)
|
|
ON CONFLICT(uid) DO UPDATE SET model=excluded.model,
|
|
dim=excluded.dim, vec=excluded.vec,
|
|
embedded_at=excluded.embedded_at""",
|
|
[(uid, model, int(vec.shape[-1]), pack(vec), stamp)
|
|
for uid, vec in pairs]).rowcount
|
|
|
|
def vectors(self, model: str) -> dict[str, np.ndarray]:
|
|
"""Every vector from `model`, by uid. The whole table: at ~30k roles of
|
|
384 floats that is ~45 MB, and one read beats one per agent."""
|
|
return {r["uid"]: unpack(r["vec"]) for r in self._rows(
|
|
"SELECT uid, vec FROM vectors WHERE model = ?", (model,))}
|
|
|
|
def vector_counts(self) -> dict[str, int]:
|
|
return {r["model"]: int(r["n"]) for r in self._rows(
|
|
"SELECT model, COUNT(*) AS n FROM vectors GROUP BY model")}
|
|
|
|
def drop_vectors(self, keep: str) -> int:
|
|
"""Delete vectors from every model but `keep`: another model's are a
|
|
different geometry, never compared, and only make counts lie."""
|
|
return self._write("DELETE FROM vectors WHERE model <> ?", (keep,))
|
|
|
|
# --------------------------------------------------------------- agents
|
|
|
|
def agents(self, enabled_only: bool = False) -> list[Agent]:
|
|
sql = "SELECT * FROM agents"
|
|
if enabled_only:
|
|
sql += " WHERE enabled = 1"
|
|
return [Agent.from_row(r)
|
|
for r in self._rows(sql + " ORDER BY name COLLATE NOCASE")]
|
|
|
|
def agent(self, agent_id: int) -> Agent | None:
|
|
row = self._row("SELECT * FROM agents WHERE id=?", (agent_id,))
|
|
return Agent.from_row(row) if row else None
|
|
|
|
def default_agent(self) -> Agent | None:
|
|
"""The agent a vote or an affinity goes to when none is named: the
|
|
enabled one with the lowest id, else the lowest id, else None."""
|
|
row = self._row("SELECT * FROM agents ORDER BY enabled DESC, id LIMIT 1")
|
|
return Agent.from_row(row) if row else None
|
|
|
|
def save_agent(self, name: str, intent: str = "", gates: Sequence[Gate] = (),
|
|
weights: Weights | None = None, enabled: bool = True,
|
|
agent_id: int | None = None, limits: Limits | None = None) -> Agent:
|
|
"""Create an agent, or with `agent_id` replace that one's fields.
|
|
Raises ValueError on an empty or taken name, KeyError on an unknown id."""
|
|
name = name.strip()
|
|
if not name:
|
|
raise ValueError("an agent needs a name")
|
|
fields = (name, intent, json.dumps([g.model_dump() for g in gates]),
|
|
(weights or Weights()).model_dump_json(),
|
|
(limits or Limits()).model_dump_json(), int(enabled))
|
|
try:
|
|
with self.transaction() as conn:
|
|
if agent_id is None:
|
|
agent_id = conn.execute(
|
|
"""INSERT INTO agents (name, intent, gates, weights, limits,
|
|
enabled, created_at) VALUES (?,?,?,?,?,?,?)""",
|
|
(*fields, self.now())).lastrowid
|
|
elif not conn.execute(
|
|
"""UPDATE agents SET name=?, intent=?, gates=?, weights=?,
|
|
limits=?, enabled=? WHERE id=?""",
|
|
(*fields, agent_id)).rowcount:
|
|
raise KeyError(agent_id)
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"an agent named {name!r} already exists") from exc
|
|
assert agent_id is not None
|
|
found = self.agent(agent_id)
|
|
assert found is not None
|
|
return found
|
|
|
|
def delete_agent(self, agent_id: int) -> bool:
|
|
"""Its scores, votes and affinities go with it, by the foreign keys."""
|
|
return self._write("DELETE FROM agents WHERE id=?", (agent_id,)) > 0
|
|
|
|
def intent_vector(self, agent_id: int, key: str) -> np.ndarray | None:
|
|
"""The agent's cached intent vector, if it was made under `key`."""
|
|
row = self._row("SELECT intent_key, intent_vec FROM agents WHERE id=?",
|
|
(agent_id,))
|
|
if row is None or row["intent_key"] != key or row["intent_vec"] is None:
|
|
return None
|
|
return unpack(row["intent_vec"])
|
|
|
|
def set_intent_vector(self, agent_id: int, key: str, vec: np.ndarray) -> None:
|
|
self._write("UPDATE agents SET intent_key=?, intent_vec=? WHERE id=?",
|
|
(key, pack(vec), agent_id))
|
|
|
|
# --------------------------------------------------------------- scores
|
|
|
|
def score_candidates(self, agent_id: int, gates: Sequence[Gate]) -> list[Candidate]:
|
|
"""Every open role of a company this agent has not hidden, with the
|
|
agent's affinity for the company and how the role meets each gate: 1
|
|
match, 0 no match, -1 the column is empty. The gates are evaluated
|
|
here, in SQL, so a `require` gate is the `WHERE` it reads as, written
|
|
as a column so a gated role keeps its row and reason."""
|
|
columns: list[str] = []
|
|
params: list[Any] = []
|
|
for n, gate in enumerate(gates):
|
|
empty, match, values = _gate_sql(gate)
|
|
columns.append(
|
|
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
|
|
params += values
|
|
select = ", ".join(["r.uid", "r.posted_at",
|
|
"COALESCE(ac.affinity, 0) AS affinity", *columns])
|
|
rows = self._rows(
|
|
f"""SELECT {select} FROM roles r
|
|
LEFT JOIN agent_companies ac
|
|
ON ac.company_key = r.company_key AND ac.agent_id = ?
|
|
WHERE r.closed_at IS NULL AND COALESCE(ac.affinity, 0) <> -1""",
|
|
[*params, agent_id])
|
|
return [Candidate(uid=r["uid"], posted_at=r["posted_at"],
|
|
affinity=int(r["affinity"]),
|
|
gates=[int(r[f"g{n}"]) for n in range(len(gates))])
|
|
for r in rows]
|
|
|
|
def replace_scores(self, agent_id: int,
|
|
rows: Sequence[tuple[str, float, str]]) -> int:
|
|
"""`(uid, score, gated_reason)` for every role this agent scored,
|
|
replacing whatever it had: a role closed or muted since the last run
|
|
drops out rather than keeping a stale score. One transaction, so a
|
|
reader sees the old list or the new one. An agent deleted while its
|
|
run was scoring gets nothing written and 0 back: its rows are gone
|
|
with it, and an insert would only fail the foreign key."""
|
|
stamp = self.now()
|
|
with self.transaction() as conn:
|
|
if conn.execute("SELECT 1 FROM agents WHERE id=?", (agent_id,)).fetchone() is None:
|
|
return 0
|
|
conn.execute("DELETE FROM scores WHERE agent_id=?", (agent_id,))
|
|
conn.executemany(
|
|
"""INSERT INTO scores (uid, agent_id, score, gated_reason, scored_at)
|
|
VALUES (?,?,?,?,?)""",
|
|
[(uid, agent_id, score, reason, stamp) for uid, score, reason in rows])
|
|
return len(rows)
|
|
|
|
def scores(self, agent_id: int) -> dict[str, tuple[float, str]]:
|
|
"""`uid -> (score, gated_reason)` for one agent."""
|
|
return {r["uid"]: (float(r["score"]), str(r["gated_reason"]))
|
|
for r in self._rows(
|
|
"SELECT uid, score, gated_reason FROM scores WHERE agent_id=?",
|
|
(agent_id,))}
|
|
|
|
# ------------------------------------------------------------ companies
|
|
|
|
def add_company(self, name: str, *, key: str = "", domain: str = "",
|
|
status: str = "researching", careers_url: str = "",
|
|
notes: str = "", ats: str = "", country: str = "",
|
|
city: str = "", ats_slug: str = "") -> Company:
|
|
"""Raises ValueError on a key already taken."""
|
|
name = name.strip()
|
|
key = key or company_key(name)
|
|
if not key:
|
|
raise ValueError("a company needs a name")
|
|
try:
|
|
with self.transaction() as conn:
|
|
self._insert_company(conn, key, name, {
|
|
"domain": domain, "status": status, "careers_url": careers_url,
|
|
"notes": notes, "ats": ats, "country": country, "city": city,
|
|
"ats_slug": ats_slug})
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"company {key!r} already exists") from exc
|
|
return self._company(key)
|
|
|
|
def _insert_company(self, conn: sqlite3.Connection, key: str, name: str,
|
|
fields: dict[str, Any]) -> None:
|
|
stamp = self.now()
|
|
columns = ["key", "name", *fields, "created_at", "updated_at"]
|
|
conn.execute(
|
|
f"INSERT INTO companies ({', '.join(columns)}) "
|
|
f"VALUES ({', '.join('?' * len(columns))})",
|
|
(key, name, *fields.values(), stamp, stamp))
|
|
|
|
def company(self, key: str, agent_id: int | None = None) -> Company | None:
|
|
"""One company, with `affinity` as `agent_id` set it; 0 without an agent."""
|
|
row = self._row(f"{_COMPANY_SELECT} WHERE c.key=?", (agent_id, key))
|
|
return Company.from_row(row) if row else None
|
|
|
|
def companies(self, agent_id: int | None = None) -> list[Company]:
|
|
"""Every company by name, with `affinity` as `agent_id` set it."""
|
|
return [Company.from_row(r) for r in self._rows(
|
|
f"{_COMPANY_SELECT} ORDER BY c.name COLLATE NOCASE", (agent_id,))]
|
|
|
|
def set_affinity(self, agent_id: int, key: str, affinity: int) -> Company:
|
|
"""One agent's stance on a company: -1 hidden, 0 neutral, 1 liked, 2
|
|
favourite. 0 deletes the row, since no row means neutral. Returns the
|
|
company with this agent's affinity. Raises ValueError for another
|
|
value and KeyError for an unknown company or agent.
|
|
|
|
A key with roles and no company row, an employer known only from a
|
|
search, gets a row here holding only its key and the name its roles
|
|
give it, the most frequent one: nothing else can be judged without
|
|
one. It has no board, `ats`, `careers_url` or `domain`, which is how
|
|
the Companies list tells it from a followed company."""
|
|
_check_affinity(affinity)
|
|
with self.transaction() as conn:
|
|
self._known(conn, "agents", "id", agent_id)
|
|
if conn.execute("SELECT 1 FROM companies WHERE key=?", (key,)).fetchone() is None:
|
|
named = conn.execute(
|
|
"""SELECT company FROM roles WHERE company_key=? AND company<>''
|
|
GROUP BY company ORDER BY count(*) DESC, max(last_seen) DESC
|
|
LIMIT 1""", (key,)).fetchone()
|
|
if named is None:
|
|
raise KeyError(key)
|
|
self._insert_company(conn, key, named[0], {})
|
|
if affinity == 0:
|
|
conn.execute("DELETE FROM agent_companies WHERE agent_id=? AND company_key=?",
|
|
(agent_id, key))
|
|
else:
|
|
conn.execute(
|
|
"""INSERT INTO agent_companies (agent_id, company_key, affinity, updated_at)
|
|
VALUES (?,?,?,?)
|
|
ON CONFLICT(agent_id, company_key) DO UPDATE
|
|
SET affinity=excluded.affinity, updated_at=excluded.updated_at""",
|
|
(agent_id, key, affinity, self.now()))
|
|
return self._company(key, agent_id)
|
|
|
|
def update_company(self, key: str, **fields: Any) -> Company:
|
|
"""Change any of `_COMPANY_FIELDS`. Affinity is per agent:
|
|
`set_affinity`."""
|
|
unknown = set(fields) - set(_COMPANY_FIELDS)
|
|
if unknown:
|
|
raise ValueError(f"not a company field: {', '.join(sorted(unknown))}")
|
|
if fields:
|
|
sets = ", ".join(f"{f}=?" for f in fields)
|
|
changed = self._write(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?",
|
|
(*fields.values(), self.now(), key))
|
|
if not changed:
|
|
raise KeyError(key)
|
|
return self._company(key)
|
|
|
|
def remove_company(self, key: str) -> bool:
|
|
return self._write("DELETE FROM companies WHERE key=?", (key,)) > 0
|
|
|
|
def _company(self, key: str, agent_id: int | None = None) -> Company:
|
|
found = self.company(key, agent_id)
|
|
if found is None:
|
|
raise KeyError(key)
|
|
return found
|
|
|
|
def open_role_counts(self) -> dict[str, int]:
|
|
"""`company_key -> open role count`, for the company list's row
|
|
count. A key with no open roles is simply absent."""
|
|
return {str(r["company_key"]): int(r["n"]) for r in self._rows(
|
|
"SELECT company_key, COUNT(*) AS n FROM roles "
|
|
"WHERE closed_at IS NULL GROUP BY company_key")}
|
|
|
|
def open_countries_by_subscription(self) -> dict[int, dict[str, int]]:
|
|
"""Per subscription id, its open roles counted by `country`."""
|
|
rows = self._rows("""SELECT subscription_id, country, COUNT(*) AS n FROM roles
|
|
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
|
|
GROUP BY subscription_id, country""")
|
|
out: dict[int, dict[str, int]] = {}
|
|
for r in rows:
|
|
out.setdefault(r["subscription_id"], {})[r["country"]] = r["n"]
|
|
return out
|
|
|
|
def company_boards(self) -> dict[str, str]:
|
|
"""`company_key -> ats host`, read off each company's own
|
|
subscription. `Subscription.company` is the display name it and the
|
|
company row were both added with, so `company_key` of one finds the
|
|
other; a company with no subscription, or a detached one (`show` 0,
|
|
see `set_company_board`), is simply absent."""
|
|
boards: dict[str, str] = {}
|
|
for sub in self.subscriptions(source="ats"):
|
|
if sub.show:
|
|
boards.setdefault(company_key(sub.company), sub.board)
|
|
return boards
|
|
|
|
def company_subscription(self, key: str) -> Subscription | None:
|
|
"""Company `key`'s own `ats` subscription, detached or not; None when
|
|
it never had one. A company has at most one: `set_company_board`
|
|
rewrites it in place rather than adding a second."""
|
|
with closing(self._connect()) as conn:
|
|
return self._company_subscription(conn, key)
|
|
|
|
@staticmethod
|
|
def _company_subscription(conn: sqlite3.Connection, key: str) -> Subscription | None:
|
|
rows = conn.execute("SELECT * FROM subscriptions WHERE source = 'ats' "
|
|
"ORDER BY show DESC, id").fetchall()
|
|
return next((Subscription.from_row(r) for r in rows
|
|
if company_key(r["company"]) == key), None)
|
|
|
|
def set_company_board(self, key: str, target: str, *, host: str = "", name: str = "",
|
|
fields: dict[str, Any] | None = None,
|
|
) -> tuple[Company, Subscription | None, int]:
|
|
"""Point company `key` at the `ats` board `target`, or at none when
|
|
`target` is "", in one transaction, so a clash on either table leaves
|
|
nothing half written.
|
|
|
|
A missing company is created from `name` and `fields`; an existing one
|
|
has `fields` written over it. Its subscription is rewritten in place:
|
|
a new target closes the old target's open roles, as switching a board
|
|
off does, and "" detaches it (off, hidden, its roles closed) while the
|
|
row stays for the next board. Returns the company, its subscription
|
|
(None when there is none) and how many roles closed. Raises KeyError
|
|
for a missing company with no board, ValueError on a UNIQUE clash."""
|
|
fields = dict(fields or {})
|
|
unknown = set(fields) - set(_COMPANY_FIELDS)
|
|
if unknown:
|
|
raise ValueError(f"not a company field: {', '.join(sorted(unknown))}")
|
|
stamp = self.now()
|
|
closed = 0
|
|
try:
|
|
with self.transaction() as conn:
|
|
row = conn.execute("SELECT name FROM companies WHERE key=?", (key,)).fetchone()
|
|
if row is None:
|
|
name = fields.pop("name", "") or name.strip()
|
|
if not target:
|
|
raise KeyError(key)
|
|
if not name:
|
|
raise ValueError("a company needs a name")
|
|
self._insert_company(conn, key, name, fields)
|
|
else:
|
|
name = fields.get("name") or row["name"]
|
|
if fields:
|
|
sets = ", ".join(f"{f}=?" for f in fields)
|
|
conn.execute(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?",
|
|
(*fields.values(), stamp, key))
|
|
sub = self._company_subscription(conn, key)
|
|
sub_id = sub.id if sub else None
|
|
if sub is not None and sub.target != target:
|
|
closed = conn.execute(
|
|
"UPDATE roles SET closed_at = ? "
|
|
"WHERE subscription_id = ? AND closed_at IS NULL",
|
|
(stamp, sub.id)).rowcount
|
|
if not target:
|
|
if sub is not None:
|
|
conn.execute("UPDATE subscriptions SET subscribed=0, fetch=0, show=0 "
|
|
"WHERE id=?", (sub.id,))
|
|
elif sub is not None:
|
|
conn.execute(
|
|
"""UPDATE subscriptions
|
|
SET target=?, board=?, label=?, company=?, subscribed=1,
|
|
fetch=1, show=1,
|
|
target_id=CASE WHEN target=? THEN target_id ELSE '' END
|
|
WHERE id=?""",
|
|
(target, host, name, name, target, sub.id))
|
|
else:
|
|
sub_id = conn.execute(
|
|
"""INSERT INTO subscriptions (source, target, label, company, board,
|
|
params, added_at)
|
|
VALUES ('ats',?,?,?,?,'{}',?)""",
|
|
(target, name, name, host, stamp)).lastrowid
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"another board already has {target!r} or the label "
|
|
f"{name!r}") from exc
|
|
return (self._company(key), self.subscription(sub_id) if sub_id else None, closed)
|
|
|
|
# -------------------------------------------------------- subscriptions
|
|
|
|
def subscriptions(self, source: str | None = None,
|
|
fetchable: bool = False) -> list[Subscription]:
|
|
"""`fetchable` is what a pull reads: subscribed and with fetch on."""
|
|
where, params = [], []
|
|
if source:
|
|
where.append("source = ?")
|
|
params.append(source)
|
|
if fetchable:
|
|
where.append("fetch = 1 AND subscribed = 1")
|
|
sql = "SELECT * FROM subscriptions"
|
|
if where:
|
|
sql += " WHERE " + " AND ".join(where)
|
|
sql += " ORDER BY company, board, label"
|
|
return [Subscription.from_row(r) for r in self._rows(sql, params)]
|
|
|
|
def subscription(self, sub_id: int) -> Subscription | None:
|
|
row = self._row("SELECT * FROM subscriptions WHERE id=?", (sub_id,))
|
|
return Subscription.from_row(row) if row else None
|
|
|
|
def subscription_for(self, source: str, label: str) -> Subscription | None:
|
|
"""The row an item names: `Item.subscription` is the label."""
|
|
row = self._row("SELECT * FROM subscriptions WHERE source=? AND label=?",
|
|
(source, label))
|
|
return Subscription.from_row(row) if row else None
|
|
|
|
def add_subscription(self, source: str, target: str, label: str = "",
|
|
company: str = "", board: str = "",
|
|
params: dict[str, Any] | None = None,
|
|
target_id: str = "") -> Subscription:
|
|
"""Raises ValueError on a duplicate target or label within one plugin."""
|
|
target = target.strip()
|
|
if not target:
|
|
raise ValueError("a subscription needs a target")
|
|
label = label.strip() or target
|
|
try:
|
|
with self.transaction() as conn:
|
|
new_id = conn.execute(
|
|
"""INSERT INTO subscriptions (source, target, label, company, board,
|
|
params, added_at, target_id)
|
|
VALUES (?,?,?,?,?,?,?,?)""",
|
|
(source, target, label, company, board, json.dumps(params or {}),
|
|
self.now(), target_id)).lastrowid
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"{source} already has {target!r} or the label {label!r}") from exc
|
|
assert new_id is not None
|
|
return self._subscription(new_id)
|
|
|
|
def update_subscription(self, sub_id: int, **fields: Any) -> Subscription:
|
|
"""Change labels, params, target_id or the three switches."""
|
|
unknown = set(fields) - set(_SUBSCRIPTION_FIELDS)
|
|
if unknown:
|
|
raise ValueError(f"not a subscription field: {', '.join(sorted(unknown))}")
|
|
if "params" in fields:
|
|
fields["params"] = json.dumps(fields["params"] or {})
|
|
for switch in ("subscribed", "fetch", "show"):
|
|
if switch in fields:
|
|
fields[switch] = int(bool(fields[switch]))
|
|
if fields:
|
|
sets = ", ".join(f"{f}=?" for f in fields)
|
|
try:
|
|
changed = self._write(f"UPDATE subscriptions SET {sets} WHERE id=?",
|
|
(*fields.values(), sub_id))
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"label {fields.get('label')!r} is already in use") from exc
|
|
if not changed:
|
|
raise KeyError(sub_id)
|
|
return self._subscription(sub_id)
|
|
|
|
def unsubscribe(self, sub_id: int) -> Subscription:
|
|
"""Stop following it and hide it. The row stays, and so do its roles."""
|
|
return self.update_subscription(sub_id, subscribed=0, fetch=0, show=0)
|
|
|
|
def _subscription(self, sub_id: int) -> Subscription:
|
|
found = self.subscription(sub_id)
|
|
if found is None:
|
|
raise KeyError(sub_id)
|
|
return found
|
|
|
|
# --------------------------------------------------------- applications
|
|
|
|
def save_application(self, application: Application) -> Application:
|
|
"""Insert or update one application. An empty `id` is minted. Entering a
|
|
status, on insert or by change, appends an `application_events` row in
|
|
the same transaction."""
|
|
_check_status(application.status)
|
|
app_id = application.id or uuid4().hex
|
|
stamp = self.now()
|
|
fields = application.model_dump(exclude={"id", "created_at", "updated_at"})
|
|
with self.transaction() as conn:
|
|
before = conn.execute("SELECT status FROM applications WHERE id=?",
|
|
(app_id,)).fetchone()
|
|
if before is None:
|
|
columns = ["id", *fields, "created_at", "updated_at"]
|
|
conn.execute(
|
|
f"INSERT INTO applications ({', '.join(columns)}) "
|
|
f"VALUES ({', '.join('?' for _ in columns)})",
|
|
(app_id, *fields.values(), stamp, stamp))
|
|
else:
|
|
sets = ", ".join(f"{f}=?" for f in fields)
|
|
conn.execute(f"UPDATE applications SET {sets}, updated_at=? WHERE id=?",
|
|
(*fields.values(), stamp, app_id))
|
|
if before is None or before["status"] != application.status:
|
|
conn.execute("INSERT INTO application_events (application_id, status, at) "
|
|
"VALUES (?,?,?)", (app_id, application.status, stamp))
|
|
return self._application(app_id)
|
|
|
|
def set_application_status(self, app_id: str, status: str) -> Application:
|
|
"""Move one application to `status`. Raises ValueError for a status
|
|
outside `STATUSES` and KeyError for an unknown id."""
|
|
_check_status(status)
|
|
current = self._application(app_id)
|
|
return self.save_application(current.model_copy(update={"status": status}))
|
|
|
|
def application(self, app_id: str) -> Application | None:
|
|
row = self._row("SELECT * FROM applications WHERE id=?", (app_id,))
|
|
return Application.from_row(row) if row else None
|
|
|
|
def delete_application(self, app_id: str) -> bool:
|
|
"""Drop one application and its events. Whether that is allowed (only
|
|
while it is still `saved`) is the caller's rule, not the store's."""
|
|
return self._write("DELETE FROM applications WHERE id=?", (app_id,)) > 0
|
|
|
|
def applications(self, status: str | None = None) -> list[Application]:
|
|
sql, params = "SELECT * FROM applications", []
|
|
if status:
|
|
sql += " WHERE status = ?"
|
|
params.append(status)
|
|
sql += " ORDER BY updated_at DESC, id"
|
|
return [Application.from_row(r) for r in self._rows(sql, params)]
|
|
|
|
def application_events(self, app_id: str) -> list[ApplicationEvent]:
|
|
return [ApplicationEvent.from_row(r) for r in self._rows(
|
|
"SELECT * FROM application_events WHERE application_id=? ORDER BY id",
|
|
(app_id,))]
|
|
|
|
def _application(self, app_id: str) -> Application:
|
|
found = self.application(app_id)
|
|
if found is None:
|
|
raise KeyError(app_id)
|
|
return found
|
|
|
|
# ---------------------------------------------------------------- state
|
|
|
|
@property
|
|
def state(self) -> MutableMapping[str, str]:
|
|
"""The `state` table as a mapping. What the runner hands a plugin as
|
|
`self.state`: it can keep a counter without being able to read roles."""
|
|
return _StateMap(self)
|
|
|
|
def state_items(self, prefix: str) -> dict[str, str]:
|
|
"""Every `state` key that starts with `prefix`, with its value, in one
|
|
query, for a reader that wants one kind of key across every target."""
|
|
return {str(r["key"]): str(r["value"]) for r in self._rows(
|
|
"SELECT key, value FROM state WHERE substr(key, 1, ?) = ? ORDER BY key",
|
|
(len(prefix), prefix))}
|
|
|
|
def claim_state(self, key: str, value: str, *, released: str, stale: str) -> bool:
|
|
"""Set `key` to `value` only while it is free: unset, equal to the
|
|
value under `released` (the claim last let go of), or no later than
|
|
`stale`. Returns whether it was set. One statement, so of two
|
|
processes claiming at once exactly one wins."""
|
|
return self._write(
|
|
"""INSERT INTO state (key, value) VALUES (?, ?)
|
|
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
|
WHERE state.value <= ?
|
|
OR state.value = COALESCE(
|
|
(SELECT r.value FROM state AS r WHERE r.key = ?), '')""",
|
|
(key, value, stale, released)) > 0
|
|
|
|
|
|
def _check_status(status: str) -> None:
|
|
if status not in STATUSES:
|
|
raise ValueError(f"{status!r} is not an application status "
|
|
f"({', '.join(STATUSES)})")
|
|
|
|
|
|
def _check_affinity(affinity: Any) -> None:
|
|
if isinstance(affinity, bool) or not isinstance(affinity, int) or affinity not in AFFINITY:
|
|
raise ValueError(f"affinity is -1, 0, 1 or 2, not {affinity!r}")
|