The Roles tab takes newsdesk's shell, ported from its index.html and
app.css rather than rewritten: a filter rail (Show read, Best first or
Newest first, first seen, tag chips with counts, the old filters behind a
disclosure), the feed in newsdesk's row layout (score gutter, title,
company, place, badges, age, tags, the summary as excerpt, judge buttons)
and the reader as a third column that replaces the list on a phone. The
router pushes /roles/{uid}, which the server also renders with the reader
open, so the separate read page and role.js go. j/k, u/n/d, S, Enter,
Escape, o, /, f and c work as in newsdesk. The phone rules come with their
comments.
Two things the row lacked: roles.read_at, stamped when the reader opens a
role or a vote is cast, behind include_read on /api/roles; and role_tags,
written by summarise alongside the summary and served with counts by
/api/filters, filtered in SQL before LIMIT. Migration 5.
PyCharm run configurations in .run/ for serve, pull, seed and pytest.
core.autocrlf=input is set locally so a Windows-side checkout commits LF.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1129 lines
53 KiB
Python
1129 lines
53 KiB
Python
"""`Store`: every SQL statement in the application.
|
|
|
|
The connection plumbing is a copy of newsdesk's `Store`, on purpose: `desk`
|
|
has no storage package. A connection is opened per call and closed again, so
|
|
one caller's commit can never land inside another's transaction; in WAL mode
|
|
that costs a file handle.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sqlite3
|
|
from collections.abc import Callable, Iterator, MutableMapping, Sequence
|
|
from contextlib import closing, contextmanager
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
import numpy as np
|
|
|
|
from desk.models import Item
|
|
from app.models import (
|
|
AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate,
|
|
Company, Gate, Role, ScoredRole, Subscription, Weights, company_key, country_of,
|
|
place_key, title_key,
|
|
)
|
|
from app.storage import schema
|
|
|
|
#: Text columns a plugin may fill from `Item.raw` under the same name. An
|
|
#: empty value never overwrites a filled column, so an upsert from a board that
|
|
#: says less than `extract` found keeps what `extract` found.
|
|
_RAW_TEXT = ("place", "country", "currency", "seniority", "function")
|
|
_RAW_INT = ("salary_min", "salary_max")
|
|
|
|
#: What `extract` may fill. Only an empty column is written: a board's own
|
|
#: structured field wins over what a model read out of the prose, and the next
|
|
#: upsert from that board would put it back anyway.
|
|
_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "function",
|
|
"permit")
|
|
|
|
#: How many replies may leave a role unanswered before `extract` stops
|
|
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
|
|
#: never answers is otherwise paid for on every pull for as long as it exists.
|
|
GIVE_UP_AFTER = 3
|
|
|
|
#: A role `extract` has yet to read: never answered, still open, not given up.
|
|
_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL "
|
|
f"AND extract_tries < {GIVE_UP_AFTER}")
|
|
|
|
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
|
|
VOTES = (-1, 0, 1, None)
|
|
|
|
#: The `judged` filter of `Store.roles`, as a `WHERE` on `roles.vote`. A
|
|
#: neutral vote is judged, and is neither liked nor passed.
|
|
JUDGED: dict[str, str] = {
|
|
"liked": "roles.vote = 1",
|
|
"unjudged": "roles.vote IS NULL",
|
|
"passed": "roles.vote = -1",
|
|
}
|
|
|
|
#: A ranking's score as `Store.roles` serves and orders it: halved by a pass,
|
|
#: lifted a fifth by a like, capped at 1. Applied at read time, so a vote
|
|
#: moves the role at once and the stored score stays what `score` computed.
|
|
_VOTED_SCORE = ("MIN(1.0, s.score * CASE roles.vote WHEN -1 THEN 0.5 "
|
|
"WHEN 1 THEN 1.2 ELSE 1 END)")
|
|
|
|
_COMPANY_FIELDS = ("name", "domain", "affinity", "status", "careers_url", "notes")
|
|
_SUBSCRIPTION_FIELDS = ("label", "company", "board", "params", "subscribed", "fetch",
|
|
"show", "target_id")
|
|
|
|
#: A tag's longest stored form, and how many a role may carry. Long enough for
|
|
#: "senior backend engineer" and short enough that the filter row stays a row.
|
|
_MAX_TAG_LEN = 40
|
|
_MAX_TAGS = 12
|
|
|
|
|
|
def _normalise_tags(tags: Sequence[str]) -> list[str]:
|
|
"""Each tag stripped, lowercased, its internal whitespace collapsed to one
|
|
space and cut to `_MAX_TAG_LEN`; empties and repeats dropped, order kept,
|
|
capped at `_MAX_TAGS`. What `set_tags` and `set_summary` store, so the
|
|
filter row never has to trust what a model or a person typed."""
|
|
seen: set[str] = set()
|
|
out: list[str] = []
|
|
for tag in tags:
|
|
cleaned = " ".join(str(tag).split()).lower()[:_MAX_TAG_LEN]
|
|
if not cleaned or cleaned in seen:
|
|
continue
|
|
seen.add(cleaned)
|
|
out.append(cleaned)
|
|
if len(out) >= _MAX_TAGS:
|
|
break
|
|
return out
|
|
|
|
|
|
def _iso(moment: datetime) -> str:
|
|
"""UTC, to the second. Every timestamp column is compared as text, so every
|
|
writer has to spell it the same way."""
|
|
return moment.astimezone(UTC).isoformat(timespec="seconds")
|
|
|
|
|
|
def _remote(value: Any) -> str:
|
|
"""A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything
|
|
unrecognised is '' (unknown), which `extract` may fill later."""
|
|
if value is True:
|
|
return "remote"
|
|
text = str(value or "").strip().lower().replace("-", "").replace(" ", "")
|
|
text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text)
|
|
return text if text in REMOTE else ""
|
|
|
|
|
|
def _like(text: str) -> str:
|
|
"""A `LIKE` pattern for a case-insensitive substring match, with SQLite's
|
|
own wildcards escaped so a search for "50%" or "a_b" stays literal."""
|
|
escaped = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
|
return f"%{escaped}%"
|
|
|
|
|
|
#: Vectors are stored little-endian float32, fixed here so a file written on
|
|
#: one machine reads on another. newsdesk's encoding, written again.
|
|
_DTYPE = np.dtype("<f4")
|
|
|
|
|
|
def pack(vec: np.ndarray) -> bytes:
|
|
return np.asarray(vec, dtype=_DTYPE).tobytes()
|
|
|
|
|
|
def unpack(blob: bytes) -> np.ndarray:
|
|
return np.frombuffer(blob, dtype=_DTYPE)
|
|
|
|
|
|
def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
|
|
"""One gate as `(empty, match, params)`: SQL that is true when the role's
|
|
column is empty, and SQL that is true when it matches, over `roles r`.
|
|
`params` belong to `match`; `empty` takes none."""
|
|
column = f"r.{gate.field}"
|
|
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
|
|
marks = ", ".join("?" for _ in values) or "NULL"
|
|
if gate.field == "salary_min":
|
|
floor = min(_int_or_none(v) or 0 for v in gate.values)
|
|
return f"{column} IS NULL", f"{column} >= ?", [floor]
|
|
if gate.field == "place":
|
|
likes = " OR ".join("LOWER(r.place) LIKE ? ESCAPE '\\'" for _ in values) or "0"
|
|
return "r.place = ''", f"({likes})", [_like(v) for v in values]
|
|
if gate.field == "company":
|
|
keys = [company_key(v) for v in values]
|
|
return ("r.company_key = ''",
|
|
f"r.company_key IN ({', '.join('?' for _ in keys) or 'NULL'})", keys)
|
|
if gate.field == "languages":
|
|
return ("r.languages IN ('', '[]')",
|
|
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
|
|
f"WHERE LOWER(value) NOT IN ({marks}))", values)
|
|
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
|
|
|
|
|
|
def _int_or_none(value: Any) -> int | None:
|
|
if value is None or value == "" or isinstance(value, bool):
|
|
return None
|
|
try:
|
|
return int(float(value))
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
class _StateMap(MutableMapping[str, str]):
|
|
"""`Store.state`: the key/value table, read and written one key at a time."""
|
|
|
|
def __init__(self, store: Store) -> None:
|
|
self._store = store
|
|
|
|
def __getitem__(self, key: str) -> str:
|
|
row = self._store._row("SELECT value FROM state WHERE key=?", (key,))
|
|
if row is None:
|
|
raise KeyError(key)
|
|
return str(row["value"])
|
|
|
|
def __setitem__(self, key: str, value: str) -> None:
|
|
self._store._write(
|
|
"""INSERT INTO state (key, value) VALUES (?,?)
|
|
ON CONFLICT(key) DO UPDATE SET value=excluded.value""",
|
|
(key, value))
|
|
|
|
def __delitem__(self, key: str) -> None:
|
|
if not self._store._write("DELETE FROM state WHERE key=?", (key,)):
|
|
raise KeyError(key)
|
|
|
|
def __iter__(self) -> Iterator[str]:
|
|
return iter([r["key"] for r in self._store._rows("SELECT key FROM state ORDER BY key")])
|
|
|
|
def __len__(self) -> int:
|
|
row = self._store._row("SELECT COUNT(*) AS n FROM state")
|
|
return int(row["n"]) if row else 0
|
|
|
|
|
|
class Store:
|
|
def __init__(self, path: str | Path) -> None:
|
|
self.path = Path(path).expanduser()
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
#: What "now" is. Replaced by tests that need two writes a second apart.
|
|
self.clock: Callable[[], datetime] = lambda: datetime.now(UTC)
|
|
with self.transaction() as conn:
|
|
# Persistent: recorded in the file, inherited by every connection.
|
|
conn.execute("PRAGMA journal_mode=WAL")
|
|
#: Migration versions this open applied; empty when already current.
|
|
self.migrations_applied: list[int] = schema.initialise(conn)
|
|
|
|
# ------------------------------------------------------------ connections
|
|
|
|
def _connect(self) -> sqlite3.Connection:
|
|
"""A fresh connection. `foreign_keys` is off by default per connection;
|
|
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
|
|
`synchronous=NORMAL` is the standard WAL companion."""
|
|
conn = sqlite3.connect(self.path)
|
|
conn.row_factory = sqlite3.Row
|
|
conn.execute("PRAGMA foreign_keys=ON")
|
|
conn.execute("PRAGMA busy_timeout=30000")
|
|
conn.execute("PRAGMA synchronous=NORMAL")
|
|
return conn
|
|
|
|
@contextmanager
|
|
def transaction(self) -> Iterator[sqlite3.Connection]:
|
|
"""One connection, one transaction, closed on the way out."""
|
|
conn = self._connect()
|
|
try:
|
|
with conn:
|
|
yield conn
|
|
finally:
|
|
conn.close()
|
|
|
|
def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]:
|
|
with closing(self._connect()) as conn:
|
|
return conn.execute(sql, params).fetchall()
|
|
|
|
def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None:
|
|
with closing(self._connect()) as conn:
|
|
row: sqlite3.Row | None = conn.execute(sql, params).fetchone()
|
|
return row
|
|
|
|
def _write(self, sql: str, params: Any = ()) -> int:
|
|
"""One statement in its own transaction. Returns the rowcount."""
|
|
with self.transaction() as conn:
|
|
return conn.execute(sql, params).rowcount
|
|
|
|
def now(self) -> str:
|
|
return _iso(self.clock())
|
|
|
|
# ---------------------------------------------------------------- roles
|
|
|
|
def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool:
|
|
"""Store one posting. Returns True when it was new.
|
|
|
|
`first_seen` and `created_at` are written on insert only; `last_seen` on
|
|
every call, and a role seen again is open again. The gate columns come
|
|
from `item.raw` under their own names, with `company` falling back to
|
|
the subscription's and `country` to what `place` names; an empty value
|
|
never overwrites a filled column.
|
|
"""
|
|
if not item.source:
|
|
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
|
|
raw = item.raw or {}
|
|
company = str(raw.get("company") or (subscription.company if subscription else ""))
|
|
place = str(raw.get("place") or "")
|
|
stamp = self.now()
|
|
row: dict[str, Any] = {
|
|
"uid": item.uid,
|
|
"source": item.source,
|
|
"source_id": item.source_id,
|
|
"subscription_id": subscription.id if subscription else None,
|
|
"company_key": company_key(company),
|
|
"company": company,
|
|
"title": item.title,
|
|
"title_key": title_key(item.title),
|
|
"place_key": place_key(place),
|
|
"remote": _remote(raw.get("remote")),
|
|
"description": item.body,
|
|
"feed_text": item.excerpt,
|
|
"original_url": item.location,
|
|
"published_at": _iso(item.published_at),
|
|
"first_seen": stamp,
|
|
"last_seen": stamp,
|
|
"raw": json.dumps(raw, default=str),
|
|
"created_at": stamp,
|
|
}
|
|
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
|
|
row["country"] = row["country"] or country_of(place)
|
|
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
|
|
|
keep_filled = ("company", "company_key", "place_key", "remote", "description",
|
|
"feed_text", "original_url", *_RAW_TEXT)
|
|
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
|
|
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
|
|
for c in (*_RAW_INT, "subscription_id")]
|
|
updates += [f"{c}=excluded.{c}"
|
|
for c in ("title", "title_key", "published_at", "raw", "last_seen")]
|
|
updates.append("closed_at=NULL")
|
|
columns = ", ".join(row)
|
|
marks = ", ".join(f":{c}" for c in row)
|
|
with self.transaction() as conn:
|
|
existed = conn.execute("SELECT 1 FROM roles WHERE uid=?",
|
|
(row["uid"],)).fetchone() is not None
|
|
conn.execute(
|
|
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
|
|
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
|
|
return not existed
|
|
|
|
def role(self, uid: str) -> Role | None:
|
|
row = self._row("SELECT * FROM roles WHERE uid=?", (uid,))
|
|
if row is None:
|
|
return None
|
|
role = Role.from_row(row)
|
|
role.tags = self._tags_of(uid)
|
|
return role
|
|
|
|
def set_read(self, uid: str, read: bool = True) -> Role:
|
|
"""Mark a role read, or unread again. Stamps `read_at` only the first
|
|
time — a role already read keeps the moment it was first opened —
|
|
and clears it outright when `read` is False. Raises KeyError for an
|
|
unknown uid."""
|
|
if read:
|
|
changed = self._write(
|
|
"UPDATE roles SET read_at = COALESCE(read_at, ?) WHERE uid = ?",
|
|
(self.now(), uid))
|
|
else:
|
|
changed = self._write("UPDATE roles SET read_at = NULL WHERE uid = ?", (uid,))
|
|
if not changed:
|
|
raise KeyError(uid)
|
|
role = self.role(uid)
|
|
assert role is not None
|
|
return role
|
|
|
|
def set_vote(self, uid: str, value: int | None) -> Role:
|
|
"""Judge one role: 1 like, 0 neither way, -1 pass, None unjudged again.
|
|
Stamps `voted_at`, or clears it with the vote. A vote never hides a
|
|
role; it moves it in a ranking (see `roles`). Voting is reading, so
|
|
it stamps `read_at` too, the same once-only way `set_read` does;
|
|
clearing a vote is not un-reading, and leaves it alone. Raises
|
|
ValueError for another value and KeyError for an unknown uid."""
|
|
if isinstance(value, bool) or value not in VOTES:
|
|
raise ValueError(f"a vote is -1, 0 or 1, not {value!r}")
|
|
stamp = None if value is None else self.now()
|
|
if not self._write(
|
|
"UPDATE roles SET vote=?, voted_at=?, read_at=COALESCE(read_at, ?) "
|
|
"WHERE uid=?", (value, stamp, stamp, uid)):
|
|
raise KeyError(uid)
|
|
role = self.role(uid)
|
|
assert role is not None
|
|
return role
|
|
|
|
def set_tags(self, uid: str, tags: Sequence[str]) -> list[str]:
|
|
"""Replace one role's tags in one transaction. See `_normalise_tags`
|
|
for how each one is cleaned up. Returns what was stored, sorted the
|
|
way `role` and `roles` read it back. Raises KeyError for an unknown
|
|
uid."""
|
|
with self.transaction() as conn:
|
|
if conn.execute("SELECT 1 FROM roles WHERE uid=?", (uid,)).fetchone() is None:
|
|
raise KeyError(uid)
|
|
return self._replace_tags(conn, uid, tags)
|
|
|
|
def _replace_tags(self, conn: sqlite3.Connection, uid: str,
|
|
tags: Sequence[str]) -> list[str]:
|
|
"""`set_tags`'s write, against an already-open transaction so
|
|
`set_summary` can share it."""
|
|
cleaned = _normalise_tags(tags)
|
|
conn.execute("DELETE FROM role_tags WHERE uid=?", (uid,))
|
|
conn.executemany("INSERT INTO role_tags (uid, tag) VALUES (?,?)",
|
|
[(uid, tag) for tag in cleaned])
|
|
return sorted(cleaned)
|
|
|
|
def _tags_of(self, uid: str) -> list[str]:
|
|
return [str(r["tag"]) for r in self._rows(
|
|
"SELECT tag FROM role_tags WHERE uid=? ORDER BY tag", (uid,))]
|
|
|
|
def _tags_by_uid(self, uids: Sequence[str]) -> dict[str, list[str]]:
|
|
"""Every tag of these roles, by uid, sorted — one query for a whole
|
|
page rather than one per role."""
|
|
if not uids:
|
|
return {}
|
|
out: dict[str, list[str]] = {}
|
|
rows = self._rows(
|
|
"SELECT uid, tag FROM role_tags WHERE uid IN (SELECT value FROM json_each(?)) "
|
|
"ORDER BY uid, tag", (json.dumps(list(uids)),))
|
|
for r in rows:
|
|
out.setdefault(r["uid"], []).append(str(r["tag"]))
|
|
return out
|
|
|
|
def roles(self, limit: int = 100, company: str | None = None,
|
|
place: str | None = None, open_only: bool = True,
|
|
q: str | None = None, function: str | None = None,
|
|
remote: str | None = None, days: int | None = None,
|
|
hide_muted: bool = False, include_read: bool = True,
|
|
tags: Sequence[str] = (), tag_mode: str = "any",
|
|
agent: int | None = None, gated: bool = False,
|
|
judged: str | None = None, offset: int = 0) -> list[ScoredRole]:
|
|
"""Newest first by `first_seen`. `company` and `place` are
|
|
case-insensitive substrings of the columns a board wrote — not the
|
|
key, so a search does not have to know the exact slug a company folds
|
|
to. `q` is matched word by word against `title`; a role must match
|
|
every word. `function` and `remote` match a gate column exactly.
|
|
`days` keeps only roles first seen within that many days of now.
|
|
`judged` is a key of `JUDGED`: liked, unjudged or passed.
|
|
|
|
`hide_muted` excludes a muted company's roles before `limit` is
|
|
applied, in SQL rather than after the fact: a company that has
|
|
published a lot, all at once — a fresh board added, a re-pull that
|
|
finally reached it — can otherwise fill the whole page with roles
|
|
that then all turn out to be from the one company someone muted,
|
|
leaving nothing to show for a default query that would otherwise
|
|
have plenty.
|
|
|
|
`include_read` set False excludes a role `set_read` or `set_vote` has
|
|
stamped, the same way and for the same reason as `hide_muted`:
|
|
before `limit`, not after. `tags` keeps only roles carrying at least
|
|
one of them (`tag_mode="any"`, the default) or every one
|
|
(`tag_mode="all"`), also before `limit`.
|
|
|
|
`agent` orders by that agent's score instead, highest first, and
|
|
carries `score` and `gated_reason` on each row. Only roles it has
|
|
scored are listed, and a gated one only when `gated` is set. The score
|
|
is the stored one moved by the role's vote (`_VOTED_SCORE`), so a vote
|
|
shows in the order at once and needs no rescore.
|
|
|
|
`offset` skips that many rows of the same order, for the page that
|
|
loads the next 60 as it is scrolled; `count_roles` is the total.
|
|
Every returned role carries its tags, read with one extra query for
|
|
the whole page rather than one per role."""
|
|
sql, params = self._roles_from(
|
|
company=company, place=place, open_only=open_only, q=q, function=function,
|
|
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
|
tags=tags, tag_mode=tag_mode, agent=agent, gated=gated, judged=judged)
|
|
if agent is None:
|
|
select = "SELECT roles.* "
|
|
order = "roles.first_seen DESC, roles.uid"
|
|
else:
|
|
select = f"SELECT roles.*, {_VOTED_SCORE} AS score, s.gated_reason "
|
|
order = "score DESC, roles.first_seen DESC, roles.uid"
|
|
found = [ScoredRole.from_row(r) for r in self._rows(
|
|
f"{select}{sql} ORDER BY {order} LIMIT ? OFFSET ?", [*params, limit, offset])]
|
|
by_uid = self._tags_by_uid([r.uid for r in found])
|
|
for r in found:
|
|
r.tags = by_uid.get(r.uid, [])
|
|
return found
|
|
|
|
def count_roles(self, company: str | None = None, place: str | None = None,
|
|
open_only: bool = True, q: str | None = None,
|
|
function: str | None = None, remote: str | None = None,
|
|
days: int | None = None, hide_muted: bool = False,
|
|
include_read: bool = True, tags: Sequence[str] = (),
|
|
tag_mode: str = "any", agent: int | None = None,
|
|
gated: bool = False, judged: str | None = None) -> int:
|
|
"""How many roles `roles` would list with the same filters and no limit."""
|
|
sql, params = self._roles_from(
|
|
company=company, place=place, open_only=open_only, q=q, function=function,
|
|
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
|
tags=tags, tag_mode=tag_mode, agent=agent, gated=gated, judged=judged)
|
|
row = self._row(f"SELECT COUNT(*) AS n {sql}", params)
|
|
return int(row["n"]) if row else 0
|
|
|
|
def _roles_from(self, *, company: str | None, place: str | None, open_only: bool,
|
|
q: str | None, function: str | None, remote: str | None,
|
|
days: int | None, hide_muted: bool, include_read: bool,
|
|
tags: Sequence[str], tag_mode: str, agent: int | None,
|
|
gated: bool, judged: str | None) -> tuple[str, list[Any]]:
|
|
"""The `FROM ... WHERE ...` that `roles` and `count_roles` share, with
|
|
its parameters in order."""
|
|
where: list[str] = []
|
|
params: list[Any] = []
|
|
if company:
|
|
where.append("company LIKE ? ESCAPE '\\'")
|
|
params.append(_like(company))
|
|
if place:
|
|
where.append("place LIKE ? ESCAPE '\\'")
|
|
params.append(_like(place))
|
|
if function:
|
|
where.append("function = ?")
|
|
params.append(function)
|
|
if remote:
|
|
where.append("remote = ?")
|
|
params.append(remote)
|
|
if days is not None:
|
|
where.append("first_seen >= ?")
|
|
params.append(_iso(self.clock() - timedelta(days=days)))
|
|
for word in (q or "").split():
|
|
where.append("title LIKE ? ESCAPE '\\'")
|
|
params.append(_like(word))
|
|
if judged:
|
|
if judged not in JUDGED:
|
|
raise ValueError(f"judged is {', '.join(JUDGED)}, not {judged!r}")
|
|
where.append(JUDGED[judged])
|
|
if open_only:
|
|
where.append("closed_at IS NULL")
|
|
if hide_muted:
|
|
where.append("company_key NOT IN "
|
|
"(SELECT key FROM companies WHERE affinity = -1)")
|
|
if not include_read:
|
|
where.append("roles.read_at IS NULL")
|
|
if tags:
|
|
marks = ", ".join("?" for _ in tags)
|
|
if tag_mode == "all":
|
|
where.append(
|
|
f"(SELECT COUNT(DISTINCT rt.tag) FROM role_tags rt "
|
|
f"WHERE rt.uid = roles.uid AND rt.tag IN ({marks})) = ?")
|
|
params += [*tags, len(tags)]
|
|
elif tag_mode == "any":
|
|
where.append(
|
|
f"EXISTS (SELECT 1 FROM role_tags rt "
|
|
f"WHERE rt.uid = roles.uid AND rt.tag IN ({marks}))")
|
|
params += list(tags)
|
|
else:
|
|
raise ValueError(f"tag_mode is any or all, not {tag_mode!r}")
|
|
joins: list[Any] = []
|
|
if agent is None:
|
|
sql = "FROM roles"
|
|
else:
|
|
sql = "FROM roles JOIN scores s ON s.uid = roles.uid AND s.agent_id = ?"
|
|
joins.append(agent)
|
|
if not gated:
|
|
where.append("s.gated_reason = ''")
|
|
if where:
|
|
sql += " WHERE " + " AND ".join(where)
|
|
return sql, [*joins, *params]
|
|
|
|
def tag_counts(self, show_muted: bool = False) -> list[tuple[str, int, int]]:
|
|
"""`(tag, open_count, unread_count)` over open roles, most-used tag
|
|
first and then alphabetically. What `/api/filters` serves; muted
|
|
companies are left out unless `show_muted`, the same default as
|
|
`roles`."""
|
|
where = "r.closed_at IS NULL"
|
|
if not show_muted:
|
|
where += (" AND r.company_key NOT IN "
|
|
"(SELECT key FROM companies WHERE affinity = -1)")
|
|
rows = self._rows(
|
|
f"""SELECT rt.tag AS tag, COUNT(*) AS n, SUM(r.read_at IS NULL) AS unread
|
|
FROM role_tags rt JOIN roles r ON r.uid = rt.uid
|
|
WHERE {where}
|
|
GROUP BY rt.tag
|
|
ORDER BY n DESC, rt.tag ASC""")
|
|
return [(str(r["tag"]), int(r["n"]), int(r["unread"])) for r in rows]
|
|
|
|
def mark_closed(self, source: str, seen_ids: Sequence[str],
|
|
subscription_id: int) -> int:
|
|
"""Close every open role of one board that a complete fetch of it did not
|
|
list. Returns how many closed. Only call it after a fetch that finished:
|
|
a failed one lists nothing, and would close the whole board."""
|
|
return self._write(
|
|
"""UPDATE roles SET closed_at = ?
|
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
|
AND source_id NOT IN (SELECT value FROM json_each(?))""",
|
|
(self.now(), source, subscription_id, json.dumps(list(seen_ids))))
|
|
|
|
def close_unseen(self, source: str, before: str) -> int:
|
|
"""Close every open role of `source` last seen before `before`. For a
|
|
source whose fetch is never a complete list, a search, where a role
|
|
dropping out of one run's results is not the role closing."""
|
|
return self._write(
|
|
"""UPDATE roles SET closed_at = ?
|
|
WHERE source = ? AND closed_at IS NULL AND last_seen < ?""",
|
|
(self.now(), source, before))
|
|
|
|
def close_subscription(self, subscription_id: int) -> int:
|
|
"""Close every open role of one subscription, for a board that is no
|
|
longer fetched: nothing would ever close them otherwise."""
|
|
return self._write(
|
|
"UPDATE roles SET closed_at = ? WHERE subscription_id = ? AND closed_at IS NULL",
|
|
(self.now(), subscription_id))
|
|
|
|
def backfill_country(self) -> int:
|
|
"""Fill `country` from `place` on every role that has none, the way
|
|
`upsert_role` does for a new one. Returns how many it filled."""
|
|
rows = self._rows("SELECT uid, place FROM roles WHERE country = '' AND place != ''")
|
|
filled = [(code, r["uid"]) for r in rows if (code := country_of(r["place"]))]
|
|
with self.transaction() as conn:
|
|
conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", filled)
|
|
return len(filled)
|
|
|
|
# -------------------------------------------------------------- extract
|
|
|
|
def extract_pending(self) -> int:
|
|
row = self._row(f"SELECT COUNT(*) AS n FROM roles WHERE {_EXTRACT_WAITING}")
|
|
return int(row["n"]) if row else 0
|
|
|
|
def extract_waiting(self, limit: int,
|
|
uids: Sequence[str] | None = None) -> list[Role]:
|
|
"""Roles `extract` has yet to read, newest first. Given `uids`, only
|
|
those of them still waiting: a batch asks again when it runs, since a
|
|
role may have been answered since the task shared the work out."""
|
|
sql, params = f"SELECT * FROM roles WHERE {_EXTRACT_WAITING}", []
|
|
if uids is not None:
|
|
sql += " AND uid IN (SELECT value FROM json_each(?))"
|
|
params.append(json.dumps(list(uids)))
|
|
sql += " ORDER BY first_seen DESC, uid LIMIT ?"
|
|
return [Role.from_row(r) for r in self._rows(sql, [*params, limit])]
|
|
|
|
def set_extracted(self, uid: str, fields: dict[str, Any]) -> int | None:
|
|
"""Write what `extract` read for one role and stamp `extracted_at`.
|
|
|
|
Only into empty columns, and only a non-empty value: a board's own
|
|
field is never replaced and nothing is ever blanked. `place` brings
|
|
`place_key` with it. Returns how many columns were filled, or None
|
|
for an unknown uid. Values are trusted to be in vocabulary; the
|
|
job's `apply` is where a reply is checked."""
|
|
with self.transaction() as conn:
|
|
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
|
|
if row is None:
|
|
return None
|
|
sets: dict[str, Any] = {}
|
|
for name in _EXTRACT_TEXT:
|
|
value = str(fields.get(name) or "")
|
|
if value and not row[name]:
|
|
sets[name] = value
|
|
for name in _RAW_INT:
|
|
number = _int_or_none(fields.get(name))
|
|
if number is not None and row[name] is None:
|
|
sets[name] = number
|
|
languages = [str(code) for code in fields.get("languages") or []]
|
|
if languages and json.loads(row["languages"] or "[]") == []:
|
|
sets["languages"] = json.dumps(languages)
|
|
filled = len(sets)
|
|
if "place" in sets:
|
|
sets["place_key"] = place_key(sets["place"])
|
|
sets["extracted_at"] = self.now()
|
|
conn.execute(
|
|
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
|
|
(*sets.values(), uid))
|
|
return filled
|
|
|
|
def extract_missed(self, uids: Sequence[str], why: str = "") -> int:
|
|
"""One reply that left these roles unanswered. Counted only against
|
|
those still waiting, so a role the reply did answer is not charged.
|
|
Returns how many this pushed to `GIVE_UP_AFTER`. `why` is accepted
|
|
for `ModelTask.missed` and not kept: there is no Given up list yet."""
|
|
if not uids:
|
|
return 0
|
|
where = f"uid IN (SELECT value FROM json_each(?)) AND {_EXTRACT_WAITING}"
|
|
wanted = json.dumps(list(uids))
|
|
with self.transaction() as conn:
|
|
crossed = conn.execute(
|
|
f"SELECT COUNT(*) FROM roles WHERE {where} AND extract_tries + 1 >= ?",
|
|
(wanted, GIVE_UP_AFTER)).fetchone()[0]
|
|
conn.execute(f"UPDATE roles SET extract_tries = extract_tries + 1 "
|
|
f"WHERE {where}", (wanted,))
|
|
return int(crossed)
|
|
|
|
# ------------------------------------------------------------ summarise
|
|
|
|
#: The `state` key counting replies that left a role without a summary.
|
|
#: `roles` has no column for it, and the count only matters for the few
|
|
#: roles in some agent's top list.
|
|
_SUMMARY_MISSED = "summarise.missed:"
|
|
|
|
def _summary_scope(self, per_agent: int,
|
|
uids: Sequence[str] | None) -> tuple[str, list[Any]]:
|
|
"""The `WHERE` for a role `summarise` may be sent: no summary yet and
|
|
some text to summarise. Given `uids`, those of them and nothing else,
|
|
open or not and given up or not, because somebody asked for them.
|
|
Otherwise open, not of a hidden company, not given up, and among the
|
|
`per_agent` best ungated scores of an enabled agent."""
|
|
where = ["r.summary = ''",
|
|
"(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"]
|
|
if uids is not None:
|
|
where.append("r.uid IN (SELECT value FROM json_each(?))")
|
|
return " AND ".join(where), [json.dumps(list(uids))]
|
|
where += [
|
|
"r.closed_at IS NULL",
|
|
"r.company_key NOT IN (SELECT key FROM companies WHERE affinity = -1)",
|
|
f"""NOT EXISTS (SELECT 1 FROM state
|
|
WHERE key = '{self._SUMMARY_MISSED}' || r.uid
|
|
AND CAST(value AS INTEGER) >= {GIVE_UP_AFTER})""",
|
|
"""r.uid IN (SELECT uid FROM (
|
|
SELECT s.uid, ROW_NUMBER() OVER (
|
|
PARTITION BY s.agent_id
|
|
ORDER BY s.score DESC, top.first_seen DESC, s.uid) AS n
|
|
FROM scores s
|
|
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
|
|
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
|
WHERE s.gated_reason = '')
|
|
WHERE n <= ?)""",
|
|
]
|
|
return " AND ".join(where), [per_agent]
|
|
|
|
def summarise_pending(self, per_agent: int) -> int:
|
|
where, params = self._summary_scope(per_agent, None)
|
|
row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params)
|
|
return int(row["n"]) if row else 0
|
|
|
|
def summarise_waiting(self, limit: int, per_agent: int,
|
|
uids: Sequence[str] | None = None) -> list[Role]:
|
|
"""Roles `summarise` has yet to write, newest first. The top
|
|
`per_agent` of every enabled agent's scores, so the backlog is bounded
|
|
by the number of agents rather than by the number of open roles; given
|
|
`uids`, those still without a summary. A batch asks again when it runs."""
|
|
where, params = self._summary_scope(per_agent, uids)
|
|
return [Role.from_row(r) for r in self._rows(
|
|
f"SELECT r.* FROM roles r WHERE {where} "
|
|
"ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
|
|
|
|
def set_summary(self, uid: str, summary: str, tags: Sequence[str] = ()) -> bool:
|
|
"""Write a role's summary and stamp `summarised_at`. Only over an empty
|
|
one, so a second batch that carried the same role does not replace the
|
|
first answer. Given `tags`, they replace the role's tags in the same
|
|
transaction — see `_replace_tags` — but only when the summary was
|
|
actually written, so a rejected late answer cannot plant tags on a
|
|
role it did not get to summarise. False for an unknown uid or one
|
|
already summarised."""
|
|
with self.transaction() as conn:
|
|
written = conn.execute(
|
|
"UPDATE roles SET summary = ?, summarised_at = ? "
|
|
"WHERE uid = ? AND summary = ''", (summary, self.now(), uid)).rowcount
|
|
conn.execute("DELETE FROM state WHERE key = ?", (self._SUMMARY_MISSED + uid,))
|
|
if written and tags:
|
|
self._replace_tags(conn, uid, tags)
|
|
return bool(written)
|
|
|
|
def summarise_missed(self, uids: Sequence[str], why: str = "") -> int:
|
|
"""One reply that left these roles without a summary. Counted only
|
|
against those still without one. Returns how many this pushed to
|
|
`GIVE_UP_AFTER`. `why` is accepted for `ModelTask.missed` and not kept."""
|
|
if not uids:
|
|
return 0
|
|
rows = self._rows("SELECT uid FROM roles WHERE summary = '' "
|
|
"AND uid IN (SELECT value FROM json_each(?))",
|
|
(json.dumps(list(uids)),))
|
|
keys = [self._SUMMARY_MISSED + r["uid"] for r in rows]
|
|
with self.transaction() as conn:
|
|
conn.executemany(
|
|
"""INSERT INTO state (key, value) VALUES (?, '1')
|
|
ON CONFLICT(key) DO UPDATE
|
|
SET value = CAST(CAST(value AS INTEGER) + 1 AS TEXT)""",
|
|
[(key,) for key in keys])
|
|
crossed = conn.execute(
|
|
"SELECT COUNT(*) FROM state WHERE key IN (SELECT value FROM json_each(?)) "
|
|
"AND CAST(value AS INTEGER) = ?", (json.dumps(keys), GIVE_UP_AFTER)).fetchone()[0]
|
|
return int(crossed)
|
|
|
|
def set_article_text(self, uid: str, text: str) -> Role | None:
|
|
"""Store the posting page as plain text and stamp `article_at`. The
|
|
role as it now stands, or None for an unknown uid."""
|
|
if not self._write("UPDATE roles SET article_text = ?, article_at = ? WHERE uid = ?",
|
|
(text, self.now(), uid)):
|
|
return None
|
|
return self.role(uid)
|
|
|
|
# -------------------------------------------------------------- vectors
|
|
|
|
def roles_to_embed(self, model: str, limit: int) -> list[Role]:
|
|
"""Open roles with no vector from `model`, newest first, so a backlog
|
|
embedded from the top improves what is on screen first."""
|
|
return [Role.from_row(r) for r in self._rows(
|
|
"""SELECT r.* FROM roles r
|
|
LEFT JOIN vectors v ON v.uid = r.uid AND v.model = ?
|
|
WHERE v.uid IS NULL AND r.closed_at IS NULL
|
|
ORDER BY r.first_seen DESC, r.uid LIMIT ?""", (model, limit))]
|
|
|
|
def put_vectors(self, model: str, pairs: Sequence[tuple[str, np.ndarray]]) -> int:
|
|
stamp = self.now()
|
|
with self.transaction() as conn:
|
|
return conn.executemany(
|
|
"""INSERT INTO vectors (uid, model, dim, vec, embedded_at)
|
|
VALUES (?,?,?,?,?)
|
|
ON CONFLICT(uid) DO UPDATE SET model=excluded.model,
|
|
dim=excluded.dim, vec=excluded.vec,
|
|
embedded_at=excluded.embedded_at""",
|
|
[(uid, model, int(vec.shape[-1]), pack(vec), stamp)
|
|
for uid, vec in pairs]).rowcount
|
|
|
|
def vectors(self, model: str) -> dict[str, np.ndarray]:
|
|
"""Every vector from `model`, by uid. The whole table: at ~30k roles of
|
|
384 floats that is ~45 MB, and one read beats one per agent."""
|
|
return {r["uid"]: unpack(r["vec"]) for r in self._rows(
|
|
"SELECT uid, vec FROM vectors WHERE model = ?", (model,))}
|
|
|
|
def vector_counts(self) -> dict[str, int]:
|
|
return {r["model"]: int(r["n"]) for r in self._rows(
|
|
"SELECT model, COUNT(*) AS n FROM vectors GROUP BY model")}
|
|
|
|
def drop_vectors(self, keep: str) -> int:
|
|
"""Delete vectors from every model but `keep`: another model's are a
|
|
different geometry, never compared, and only make counts lie."""
|
|
return self._write("DELETE FROM vectors WHERE model <> ?", (keep,))
|
|
|
|
# --------------------------------------------------------------- agents
|
|
|
|
def agents(self, enabled_only: bool = False) -> list[Agent]:
|
|
sql = "SELECT * FROM agents"
|
|
if enabled_only:
|
|
sql += " WHERE enabled = 1"
|
|
return [Agent.from_row(r)
|
|
for r in self._rows(sql + " ORDER BY name COLLATE NOCASE")]
|
|
|
|
def agent(self, agent_id: int) -> Agent | None:
|
|
row = self._row("SELECT * FROM agents WHERE id=?", (agent_id,))
|
|
return Agent.from_row(row) if row else None
|
|
|
|
def save_agent(self, name: str, intent: str = "", gates: Sequence[Gate] = (),
|
|
weights: Weights | None = None, enabled: bool = True,
|
|
agent_id: int | None = None) -> Agent:
|
|
"""Create an agent, or with `agent_id` replace that one's fields.
|
|
Raises ValueError on an empty or taken name, KeyError on an unknown id."""
|
|
name = name.strip()
|
|
if not name:
|
|
raise ValueError("an agent needs a name")
|
|
fields = (name, intent, json.dumps([g.model_dump() for g in gates]),
|
|
(weights or Weights()).model_dump_json(), int(enabled))
|
|
try:
|
|
with self.transaction() as conn:
|
|
if agent_id is None:
|
|
agent_id = conn.execute(
|
|
"""INSERT INTO agents (name, intent, gates, weights, enabled,
|
|
created_at) VALUES (?,?,?,?,?,?)""",
|
|
(*fields, self.now())).lastrowid
|
|
elif not conn.execute(
|
|
"""UPDATE agents SET name=?, intent=?, gates=?, weights=?,
|
|
enabled=? WHERE id=?""",
|
|
(*fields, agent_id)).rowcount:
|
|
raise KeyError(agent_id)
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"an agent named {name!r} already exists") from exc
|
|
assert agent_id is not None
|
|
found = self.agent(agent_id)
|
|
assert found is not None
|
|
return found
|
|
|
|
def delete_agent(self, agent_id: int) -> bool:
|
|
"""Its scores go with it, by the foreign key."""
|
|
return self._write("DELETE FROM agents WHERE id=?", (agent_id,)) > 0
|
|
|
|
def intent_vector(self, agent_id: int, key: str) -> np.ndarray | None:
|
|
"""The agent's cached intent vector, if it was made under `key`."""
|
|
row = self._row("SELECT intent_key, intent_vec FROM agents WHERE id=?",
|
|
(agent_id,))
|
|
if row is None or row["intent_key"] != key or row["intent_vec"] is None:
|
|
return None
|
|
return unpack(row["intent_vec"])
|
|
|
|
def set_intent_vector(self, agent_id: int, key: str, vec: np.ndarray) -> None:
|
|
self._write("UPDATE agents SET intent_key=?, intent_vec=? WHERE id=?",
|
|
(key, pack(vec), agent_id))
|
|
|
|
# --------------------------------------------------------------- scores
|
|
|
|
def score_candidates(self, gates: Sequence[Gate]) -> list[Candidate]:
|
|
"""Every open role of a company that is not muted, with how it meets
|
|
each gate: 1 match, 0 no match, -1 the column is empty. The gates are
|
|
evaluated here, in SQL, so a `require` gate is the `WHERE` it reads
|
|
as, written as a column so a gated role keeps its row and reason."""
|
|
columns: list[str] = []
|
|
params: list[Any] = []
|
|
for n, gate in enumerate(gates):
|
|
empty, match, values = _gate_sql(gate)
|
|
columns.append(
|
|
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
|
|
params += values
|
|
select = ", ".join(["r.uid", "r.first_seen",
|
|
"COALESCE(c.affinity, 0) AS affinity", *columns])
|
|
rows = self._rows(
|
|
f"""SELECT {select} FROM roles r
|
|
LEFT JOIN companies c ON c.key = r.company_key
|
|
WHERE r.closed_at IS NULL AND COALESCE(c.affinity, 0) <> -1""",
|
|
params)
|
|
return [Candidate(uid=r["uid"], first_seen=r["first_seen"],
|
|
affinity=int(r["affinity"]),
|
|
gates=[int(r[f"g{n}"]) for n in range(len(gates))])
|
|
for r in rows]
|
|
|
|
def replace_scores(self, agent_id: int,
|
|
rows: Sequence[tuple[str, float, str]]) -> int:
|
|
"""`(uid, score, gated_reason)` for every role this agent scored,
|
|
replacing whatever it had: a role closed or muted since the last run
|
|
drops out rather than keeping a stale score. One transaction, so a
|
|
reader sees the old list or the new one."""
|
|
stamp = self.now()
|
|
with self.transaction() as conn:
|
|
conn.execute("DELETE FROM scores WHERE agent_id=?", (agent_id,))
|
|
conn.executemany(
|
|
"""INSERT INTO scores (uid, agent_id, score, gated_reason, scored_at)
|
|
VALUES (?,?,?,?,?)""",
|
|
[(uid, agent_id, score, reason, stamp) for uid, score, reason in rows])
|
|
return len(rows)
|
|
|
|
def scores(self, agent_id: int) -> dict[str, tuple[float, str]]:
|
|
"""`uid -> (score, gated_reason)` for one agent."""
|
|
return {r["uid"]: (float(r["score"]), str(r["gated_reason"]))
|
|
for r in self._rows(
|
|
"SELECT uid, score, gated_reason FROM scores WHERE agent_id=?",
|
|
(agent_id,))}
|
|
|
|
# ------------------------------------------------------------ companies
|
|
|
|
def add_company(self, name: str, *, key: str = "", domain: str = "",
|
|
affinity: int = 0, status: str = "researching",
|
|
careers_url: str = "", notes: str = "") -> Company:
|
|
"""Raises ValueError on a key already taken or an affinity out of range."""
|
|
name = name.strip()
|
|
key = key or company_key(name)
|
|
if not key:
|
|
raise ValueError("a company needs a name")
|
|
_check_affinity(affinity)
|
|
stamp = self.now()
|
|
try:
|
|
self._write(
|
|
"""INSERT INTO companies (key, name, domain, affinity, status,
|
|
careers_url, notes, created_at, updated_at)
|
|
VALUES (?,?,?,?,?,?,?,?,?)""",
|
|
(key, name, domain, affinity, status, careers_url, notes, stamp, stamp))
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"company {key!r} already exists") from exc
|
|
return self._company(key)
|
|
|
|
def company(self, key: str) -> Company | None:
|
|
row = self._row("SELECT * FROM companies WHERE key=?", (key,))
|
|
return Company.from_row(row) if row else None
|
|
|
|
def companies(self) -> list[Company]:
|
|
return [Company.from_row(r)
|
|
for r in self._rows("SELECT * FROM companies ORDER BY name COLLATE NOCASE")]
|
|
|
|
def update_company(self, key: str, **fields: Any) -> Company:
|
|
"""Change any of name, domain, affinity, status, careers_url, notes."""
|
|
unknown = set(fields) - set(_COMPANY_FIELDS)
|
|
if unknown:
|
|
raise ValueError(f"not a company field: {', '.join(sorted(unknown))}")
|
|
if "affinity" in fields:
|
|
_check_affinity(fields["affinity"])
|
|
if fields:
|
|
sets = ", ".join(f"{f}=?" for f in fields)
|
|
changed = self._write(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?",
|
|
(*fields.values(), self.now(), key))
|
|
if not changed:
|
|
raise KeyError(key)
|
|
return self._company(key)
|
|
|
|
def remove_company(self, key: str) -> bool:
|
|
return self._write("DELETE FROM companies WHERE key=?", (key,)) > 0
|
|
|
|
def _company(self, key: str) -> Company:
|
|
found = self.company(key)
|
|
if found is None:
|
|
raise KeyError(key)
|
|
return found
|
|
|
|
def open_role_counts(self) -> dict[str, int]:
|
|
"""`company_key -> open role count`, for the company list's row
|
|
count. A key with no open roles is simply absent."""
|
|
return {str(r["company_key"]): int(r["n"]) for r in self._rows(
|
|
"SELECT company_key, COUNT(*) AS n FROM roles "
|
|
"WHERE closed_at IS NULL GROUP BY company_key")}
|
|
|
|
def open_countries_by_subscription(self) -> dict[int, dict[str, int]]:
|
|
"""Per subscription id, its open roles counted by `country`."""
|
|
rows = self._rows("""SELECT subscription_id, country, COUNT(*) AS n FROM roles
|
|
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
|
|
GROUP BY subscription_id, country""")
|
|
out: dict[int, dict[str, int]] = {}
|
|
for r in rows:
|
|
out.setdefault(r["subscription_id"], {})[r["country"]] = r["n"]
|
|
return out
|
|
|
|
def company_boards(self) -> dict[str, str]:
|
|
"""`company_key -> ats host`, read off each company's own
|
|
subscription. `Subscription.company` is the display name it and the
|
|
company row were both added with, so `company_key` of one finds the
|
|
other; a company added by hand with no subscription is simply
|
|
absent."""
|
|
boards: dict[str, str] = {}
|
|
for sub in self.subscriptions(source="ats"):
|
|
boards.setdefault(company_key(sub.company), sub.board)
|
|
return boards
|
|
|
|
# -------------------------------------------------------- subscriptions
|
|
|
|
def subscriptions(self, source: str | None = None,
|
|
fetchable: bool = False) -> list[Subscription]:
|
|
"""`fetchable` is what a pull reads: subscribed and with fetch on."""
|
|
where, params = [], []
|
|
if source:
|
|
where.append("source = ?")
|
|
params.append(source)
|
|
if fetchable:
|
|
where.append("fetch = 1 AND subscribed = 1")
|
|
sql = "SELECT * FROM subscriptions"
|
|
if where:
|
|
sql += " WHERE " + " AND ".join(where)
|
|
sql += " ORDER BY company, board, label"
|
|
return [Subscription.from_row(r) for r in self._rows(sql, params)]
|
|
|
|
def subscription(self, sub_id: int) -> Subscription | None:
|
|
row = self._row("SELECT * FROM subscriptions WHERE id=?", (sub_id,))
|
|
return Subscription.from_row(row) if row else None
|
|
|
|
def subscription_for(self, source: str, label: str) -> Subscription | None:
|
|
"""The row an item names: `Item.subscription` is the label."""
|
|
row = self._row("SELECT * FROM subscriptions WHERE source=? AND label=?",
|
|
(source, label))
|
|
return Subscription.from_row(row) if row else None
|
|
|
|
def add_subscription(self, source: str, target: str, label: str = "",
|
|
company: str = "", board: str = "",
|
|
params: dict[str, Any] | None = None,
|
|
target_id: str = "") -> Subscription:
|
|
"""Raises ValueError on a duplicate target or label within one plugin."""
|
|
target = target.strip()
|
|
if not target:
|
|
raise ValueError("a subscription needs a target")
|
|
label = label.strip() or target
|
|
try:
|
|
with self.transaction() as conn:
|
|
new_id = conn.execute(
|
|
"""INSERT INTO subscriptions (source, target, label, company, board,
|
|
params, added_at, target_id)
|
|
VALUES (?,?,?,?,?,?,?,?)""",
|
|
(source, target, label, company, board, json.dumps(params or {}),
|
|
self.now(), target_id)).lastrowid
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"{source} already has {target!r} or the label {label!r}") from exc
|
|
assert new_id is not None
|
|
return self._subscription(new_id)
|
|
|
|
def update_subscription(self, sub_id: int, **fields: Any) -> Subscription:
|
|
"""Change labels, params, target_id or the three switches."""
|
|
unknown = set(fields) - set(_SUBSCRIPTION_FIELDS)
|
|
if unknown:
|
|
raise ValueError(f"not a subscription field: {', '.join(sorted(unknown))}")
|
|
if "params" in fields:
|
|
fields["params"] = json.dumps(fields["params"] or {})
|
|
for switch in ("subscribed", "fetch", "show"):
|
|
if switch in fields:
|
|
fields[switch] = int(bool(fields[switch]))
|
|
if fields:
|
|
sets = ", ".join(f"{f}=?" for f in fields)
|
|
try:
|
|
changed = self._write(f"UPDATE subscriptions SET {sets} WHERE id=?",
|
|
(*fields.values(), sub_id))
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"label {fields.get('label')!r} is already in use") from exc
|
|
if not changed:
|
|
raise KeyError(sub_id)
|
|
return self._subscription(sub_id)
|
|
|
|
def unsubscribe(self, sub_id: int) -> Subscription:
|
|
"""Stop following it and hide it. The row stays, and so do its roles."""
|
|
return self.update_subscription(sub_id, subscribed=0, fetch=0, show=0)
|
|
|
|
def _subscription(self, sub_id: int) -> Subscription:
|
|
found = self.subscription(sub_id)
|
|
if found is None:
|
|
raise KeyError(sub_id)
|
|
return found
|
|
|
|
# --------------------------------------------------------- applications
|
|
|
|
def save_application(self, application: Application) -> Application:
|
|
"""Insert or update one application. An empty `id` is minted. Entering a
|
|
status, on insert or by change, appends an `application_events` row in
|
|
the same transaction."""
|
|
_check_status(application.status)
|
|
app_id = application.id or uuid4().hex
|
|
stamp = self.now()
|
|
fields = application.model_dump(exclude={"id", "created_at", "updated_at"})
|
|
with self.transaction() as conn:
|
|
before = conn.execute("SELECT status FROM applications WHERE id=?",
|
|
(app_id,)).fetchone()
|
|
if before is None:
|
|
columns = ["id", *fields, "created_at", "updated_at"]
|
|
conn.execute(
|
|
f"INSERT INTO applications ({', '.join(columns)}) "
|
|
f"VALUES ({', '.join('?' for _ in columns)})",
|
|
(app_id, *fields.values(), stamp, stamp))
|
|
else:
|
|
sets = ", ".join(f"{f}=?" for f in fields)
|
|
conn.execute(f"UPDATE applications SET {sets}, updated_at=? WHERE id=?",
|
|
(*fields.values(), stamp, app_id))
|
|
if before is None or before["status"] != application.status:
|
|
conn.execute("INSERT INTO application_events (application_id, status, at) "
|
|
"VALUES (?,?,?)", (app_id, application.status, stamp))
|
|
return self._application(app_id)
|
|
|
|
def set_application_status(self, app_id: str, status: str) -> Application:
|
|
"""Move one application to `status`. Raises ValueError for a status
|
|
outside `STATUSES` and KeyError for an unknown id."""
|
|
_check_status(status)
|
|
current = self._application(app_id)
|
|
return self.save_application(current.model_copy(update={"status": status}))
|
|
|
|
def application(self, app_id: str) -> Application | None:
|
|
row = self._row("SELECT * FROM applications WHERE id=?", (app_id,))
|
|
return Application.from_row(row) if row else None
|
|
|
|
def delete_application(self, app_id: str) -> bool:
|
|
"""Drop one application and its events. Whether that is allowed (only
|
|
while it is still `saved`) is the caller's rule, not the store's."""
|
|
return self._write("DELETE FROM applications WHERE id=?", (app_id,)) > 0
|
|
|
|
def applications(self, status: str | None = None) -> list[Application]:
|
|
sql, params = "SELECT * FROM applications", []
|
|
if status:
|
|
sql += " WHERE status = ?"
|
|
params.append(status)
|
|
sql += " ORDER BY updated_at DESC, id"
|
|
return [Application.from_row(r) for r in self._rows(sql, params)]
|
|
|
|
def application_events(self, app_id: str) -> list[ApplicationEvent]:
|
|
return [ApplicationEvent.from_row(r) for r in self._rows(
|
|
"SELECT * FROM application_events WHERE application_id=? ORDER BY id",
|
|
(app_id,))]
|
|
|
|
def _application(self, app_id: str) -> Application:
|
|
found = self.application(app_id)
|
|
if found is None:
|
|
raise KeyError(app_id)
|
|
return found
|
|
|
|
# ---------------------------------------------------------------- state
|
|
|
|
@property
|
|
def state(self) -> MutableMapping[str, str]:
|
|
"""The `state` table as a mapping. What the runner hands a plugin as
|
|
`self.state`: it can keep a counter without being able to read roles."""
|
|
return _StateMap(self)
|
|
|
|
|
|
def _check_status(status: str) -> None:
|
|
if status not in STATUSES:
|
|
raise ValueError(f"{status!r} is not an application status "
|
|
f"({', '.join(STATUSES)})")
|
|
|
|
|
|
def _check_affinity(affinity: Any) -> None:
|
|
if not isinstance(affinity, int) or affinity not in AFFINITY:
|
|
raise ValueError(f"affinity is -1, 0, 1 or 2, not {affinity!r}")
|