"""`Store`: every SQL statement in the application. The connection plumbing is a copy of newsdesk's `Store`, on purpose: `desk` has no storage package. A connection is opened per call and closed again, so one caller's commit can never land inside another's transaction; in WAL mode that costs a file handle. """ from __future__ import annotations import json import sqlite3 from collections.abc import Callable, Iterator, MutableMapping, Sequence from contextlib import closing, contextmanager from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any from uuid import uuid4 import numpy as np from desk.models import Item from app.models import ( AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate, Company, Gate, Role, ScoredRole, Subscription, Weights, company_key, country_of, place_key, title_key, ) from app.storage import schema #: Text columns a plugin may fill from `Item.raw` under the same name. An #: empty value never overwrites a filled column, so an upsert from a board that #: says less than `extract` found keeps what `extract` found. _RAW_TEXT = ("place", "country", "currency", "seniority", "function") _RAW_INT = ("salary_min", "salary_max") #: What `extract` may fill. Only an empty column is written: a board's own #: structured field wins over what a model read out of the prose, and the next #: upsert from that board would put it back anyway. _EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "function", "permit") #: How many replies may leave a role unanswered before `extract` stops #: offering it. newsdesk's number, for newsdesk's reason: a row the model #: never answers is otherwise paid for on every pull for as long as it exists. GIVE_UP_AFTER = 3 #: A role `extract` has yet to read: never answered, still open, not given up. _EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL " f"AND extract_tries < {GIVE_UP_AFTER}") _COMPANY_FIELDS = ("name", "domain", "affinity", "status", "careers_url", "notes") _SUBSCRIPTION_FIELDS = ("label", "company", "board", "params", "subscribed", "fetch", "show", "target_id") def _iso(moment: datetime) -> str: """UTC, to the second. Every timestamp column is compared as text, so every writer has to spell it the same way.""" return moment.astimezone(UTC).isoformat(timespec="seconds") def _remote(value: Any) -> str: """A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything unrecognised is '' (unknown), which `extract` may fill later.""" if value is True: return "remote" text = str(value or "").strip().lower().replace("-", "").replace(" ", "") text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text) return text if text in REMOTE else "" def _like(text: str) -> str: """A `LIKE` pattern for a case-insensitive substring match, with SQLite's own wildcards escaped so a search for "50%" or "a_b" stays literal.""" escaped = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") return f"%{escaped}%" #: Vectors are stored little-endian float32, fixed here so a file written on #: one machine reads on another. newsdesk's encoding, written again. _DTYPE = np.dtype(" bytes: return np.asarray(vec, dtype=_DTYPE).tobytes() def unpack(blob: bytes) -> np.ndarray: return np.frombuffer(blob, dtype=_DTYPE) def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]: """One gate as `(empty, match, params)`: SQL that is true when the role's column is empty, and SQL that is true when it matches, over `roles r`. `params` belong to `match`; `empty` takes none.""" column = f"r.{gate.field}" values = [str(v).strip().lower() for v in gate.values if str(v).strip()] marks = ", ".join("?" for _ in values) or "NULL" if gate.field == "salary_min": floor = min(_int_or_none(v) or 0 for v in gate.values) return f"{column} IS NULL", f"{column} >= ?", [floor] if gate.field == "place": likes = " OR ".join("LOWER(r.place) LIKE ? ESCAPE '\\'" for _ in values) or "0" return "r.place = ''", f"({likes})", [_like(v) for v in values] if gate.field == "company": keys = [company_key(v) for v in values] return ("r.company_key = ''", f"r.company_key IN ({', '.join('?' for _ in keys) or 'NULL'})", keys) if gate.field == "languages": return ("r.languages IN ('', '[]')", f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) " f"WHERE LOWER(value) NOT IN ({marks}))", values) return f"{column} = ''", f"LOWER({column}) IN ({marks})", values def _int_or_none(value: Any) -> int | None: if value is None or value == "" or isinstance(value, bool): return None try: return int(float(value)) except (TypeError, ValueError): return None class _StateMap(MutableMapping[str, str]): """`Store.state`: the key/value table, read and written one key at a time.""" def __init__(self, store: Store) -> None: self._store = store def __getitem__(self, key: str) -> str: row = self._store._row("SELECT value FROM state WHERE key=?", (key,)) if row is None: raise KeyError(key) return str(row["value"]) def __setitem__(self, key: str, value: str) -> None: self._store._write( """INSERT INTO state (key, value) VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value""", (key, value)) def __delitem__(self, key: str) -> None: if not self._store._write("DELETE FROM state WHERE key=?", (key,)): raise KeyError(key) def __iter__(self) -> Iterator[str]: return iter([r["key"] for r in self._store._rows("SELECT key FROM state ORDER BY key")]) def __len__(self) -> int: row = self._store._row("SELECT COUNT(*) AS n FROM state") return int(row["n"]) if row else 0 class Store: def __init__(self, path: str | Path) -> None: self.path = Path(path).expanduser() self.path.parent.mkdir(parents=True, exist_ok=True) #: What "now" is. Replaced by tests that need two writes a second apart. self.clock: Callable[[], datetime] = lambda: datetime.now(UTC) with self.transaction() as conn: # Persistent: recorded in the file, inherited by every connection. conn.execute("PRAGMA journal_mode=WAL") #: Migration versions this open applied; empty when already current. self.migrations_applied: list[int] = schema.initialise(conn) # ------------------------------------------------------------ connections def _connect(self) -> sqlite3.Connection: """A fresh connection. `foreign_keys` is off by default per connection; `busy_timeout` matches `pipeline/table.py`, which writes the same file; `synchronous=NORMAL` is the standard WAL companion.""" conn = sqlite3.connect(self.path) conn.row_factory = sqlite3.Row conn.execute("PRAGMA foreign_keys=ON") conn.execute("PRAGMA busy_timeout=30000") conn.execute("PRAGMA synchronous=NORMAL") return conn @contextmanager def transaction(self) -> Iterator[sqlite3.Connection]: """One connection, one transaction, closed on the way out.""" conn = self._connect() try: with conn: yield conn finally: conn.close() def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]: with closing(self._connect()) as conn: return conn.execute(sql, params).fetchall() def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None: with closing(self._connect()) as conn: row: sqlite3.Row | None = conn.execute(sql, params).fetchone() return row def _write(self, sql: str, params: Any = ()) -> int: """One statement in its own transaction. Returns the rowcount.""" with self.transaction() as conn: return conn.execute(sql, params).rowcount def now(self) -> str: return _iso(self.clock()) # ---------------------------------------------------------------- roles def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool: """Store one posting. Returns True when it was new. `first_seen` and `created_at` are written on insert only; `last_seen` on every call, and a role seen again is open again. The gate columns come from `item.raw` under their own names, with `company` falling back to the subscription's and `country` to what `place` names; an empty value never overwrites a filled column. """ if not item.source: raise ValueError(f"item {item.source_id!r} has no source; the runner sets it") raw = item.raw or {} company = str(raw.get("company") or (subscription.company if subscription else "")) place = str(raw.get("place") or "") stamp = self.now() row: dict[str, Any] = { "uid": item.uid, "source": item.source, "source_id": item.source_id, "subscription_id": subscription.id if subscription else None, "company_key": company_key(company), "company": company, "title": item.title, "title_key": title_key(item.title), "place_key": place_key(place), "remote": _remote(raw.get("remote")), "description": item.body, "feed_text": item.excerpt, "original_url": item.location, "published_at": _iso(item.published_at), "first_seen": stamp, "last_seen": stamp, "raw": json.dumps(raw, default=str), "created_at": stamp, } row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT} row["country"] = row["country"] or country_of(place) row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT} keep_filled = ("company", "company_key", "place_key", "remote", "description", "feed_text", "original_url", *_RAW_TEXT) updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled] updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})" for c in (*_RAW_INT, "subscription_id")] updates += [f"{c}=excluded.{c}" for c in ("title", "title_key", "published_at", "raw", "last_seen")] updates.append("closed_at=NULL") columns = ", ".join(row) marks = ", ".join(f":{c}" for c in row) with self.transaction() as conn: existed = conn.execute("SELECT 1 FROM roles WHERE uid=?", (row["uid"],)).fetchone() is not None conn.execute( f"INSERT INTO roles ({columns}) VALUES ({marks}) " f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row) return not existed def role(self, uid: str) -> Role | None: row = self._row("SELECT * FROM roles WHERE uid=?", (uid,)) return Role.from_row(row) if row else None def roles(self, limit: int = 100, company: str | None = None, place: str | None = None, open_only: bool = True, q: str | None = None, function: str | None = None, remote: str | None = None, days: int | None = None, hide_muted: bool = False, agent: int | None = None, gated: bool = False) -> list[ScoredRole]: """Newest first by `first_seen`. `company` and `place` are case-insensitive substrings of the columns a board wrote — not the key, so a search does not have to know the exact slug a company folds to. `q` is matched word by word against `title`; a role must match every word. `function` and `remote` match a gate column exactly. `days` keeps only roles first seen within that many days of now. `hide_muted` excludes a muted company's roles before `limit` is applied, in SQL rather than after the fact: a company that has published a lot, all at once — a fresh board added, a re-pull that finally reached it — can otherwise fill the whole page with roles that then all turn out to be from the one company someone muted, leaving nothing to show for a default query that would otherwise have plenty. `agent` orders by that agent's score instead, highest first, and carries `score` and `gated_reason` on each row. Only roles it has scored are listed, and a gated one only when `gated` is set.""" where, params = [], [] if company: where.append("company LIKE ? ESCAPE '\\'") params.append(_like(company)) if place: where.append("place LIKE ? ESCAPE '\\'") params.append(_like(place)) if function: where.append("function = ?") params.append(function) if remote: where.append("remote = ?") params.append(remote) if days is not None: where.append("first_seen >= ?") params.append(_iso(self.clock() - timedelta(days=days))) for word in (q or "").split(): where.append("title LIKE ? ESCAPE '\\'") params.append(_like(word)) if open_only: where.append("closed_at IS NULL") if hide_muted: where.append("company_key NOT IN " "(SELECT key FROM companies WHERE affinity = -1)") joins: list[Any] = [] if agent is None: sql = "SELECT roles.* FROM roles" order = "roles.first_seen DESC, roles.uid" else: sql = ("SELECT roles.*, s.score, s.gated_reason FROM roles " "JOIN scores s ON s.uid = roles.uid AND s.agent_id = ?") joins.append(agent) if not gated: where.append("s.gated_reason = ''") order = "s.score DESC, roles.first_seen DESC, roles.uid" if where: sql += " WHERE " + " AND ".join(where) sql += f" ORDER BY {order} LIMIT ?" return [ScoredRole.from_row(r) for r in self._rows(sql, [*joins, *params, limit])] def mark_closed(self, source: str, seen_ids: Sequence[str], subscription_id: int) -> int: """Close every open role of one board that a complete fetch of it did not list. Returns how many closed. Only call it after a fetch that finished: a failed one lists nothing, and would close the whole board.""" return self._write( """UPDATE roles SET closed_at = ? WHERE source = ? AND subscription_id = ? AND closed_at IS NULL AND source_id NOT IN (SELECT value FROM json_each(?))""", (self.now(), source, subscription_id, json.dumps(list(seen_ids)))) def close_unseen(self, source: str, before: str) -> int: """Close every open role of `source` last seen before `before`. For a source whose fetch is never a complete list, a search, where a role dropping out of one run's results is not the role closing.""" return self._write( """UPDATE roles SET closed_at = ? WHERE source = ? AND closed_at IS NULL AND last_seen < ?""", (self.now(), source, before)) def close_subscription(self, subscription_id: int) -> int: """Close every open role of one subscription, for a board that is no longer fetched: nothing would ever close them otherwise.""" return self._write( "UPDATE roles SET closed_at = ? WHERE subscription_id = ? AND closed_at IS NULL", (self.now(), subscription_id)) def backfill_country(self) -> int: """Fill `country` from `place` on every role that has none, the way `upsert_role` does for a new one. Returns how many it filled.""" rows = self._rows("SELECT uid, place FROM roles WHERE country = '' AND place != ''") filled = [(code, r["uid"]) for r in rows if (code := country_of(r["place"]))] with self.transaction() as conn: conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", filled) return len(filled) # -------------------------------------------------------------- extract def extract_pending(self) -> int: row = self._row(f"SELECT COUNT(*) AS n FROM roles WHERE {_EXTRACT_WAITING}") return int(row["n"]) if row else 0 def extract_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]: """Roles `extract` has yet to read, newest first. Given `uids`, only those of them still waiting: a batch asks again when it runs, since a role may have been answered since the task shared the work out.""" sql, params = f"SELECT * FROM roles WHERE {_EXTRACT_WAITING}", [] if uids is not None: sql += " AND uid IN (SELECT value FROM json_each(?))" params.append(json.dumps(list(uids))) sql += " ORDER BY first_seen DESC, uid LIMIT ?" return [Role.from_row(r) for r in self._rows(sql, [*params, limit])] def set_extracted(self, uid: str, fields: dict[str, Any]) -> int | None: """Write what `extract` read for one role and stamp `extracted_at`. Only into empty columns, and only a non-empty value: a board's own field is never replaced and nothing is ever blanked. `place` brings `place_key` with it. Returns how many columns were filled, or None for an unknown uid. Values are trusted to be in vocabulary; the job's `apply` is where a reply is checked.""" with self.transaction() as conn: row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone() if row is None: return None sets: dict[str, Any] = {} for name in _EXTRACT_TEXT: value = str(fields.get(name) or "") if value and not row[name]: sets[name] = value for name in _RAW_INT: number = _int_or_none(fields.get(name)) if number is not None and row[name] is None: sets[name] = number languages = [str(code) for code in fields.get("languages") or []] if languages and json.loads(row["languages"] or "[]") == []: sets["languages"] = json.dumps(languages) filled = len(sets) if "place" in sets: sets["place_key"] = place_key(sets["place"]) sets["extracted_at"] = self.now() conn.execute( f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?", (*sets.values(), uid)) return filled def extract_missed(self, uids: Sequence[str], why: str = "") -> int: """One reply that left these roles unanswered. Counted only against those still waiting, so a role the reply did answer is not charged. Returns how many this pushed to `GIVE_UP_AFTER`. `why` is accepted for `ModelTask.missed` and not kept: there is no Given up list yet.""" if not uids: return 0 where = f"uid IN (SELECT value FROM json_each(?)) AND {_EXTRACT_WAITING}" wanted = json.dumps(list(uids)) with self.transaction() as conn: crossed = conn.execute( f"SELECT COUNT(*) FROM roles WHERE {where} AND extract_tries + 1 >= ?", (wanted, GIVE_UP_AFTER)).fetchone()[0] conn.execute(f"UPDATE roles SET extract_tries = extract_tries + 1 " f"WHERE {where}", (wanted,)) return int(crossed) # ------------------------------------------------------------ summarise #: The `state` key counting replies that left a role without a summary. #: `roles` has no column for it, and the count only matters for the few #: roles in some agent's top list. _SUMMARY_MISSED = "summarise.missed:" def _summary_scope(self, per_agent: int, uids: Sequence[str] | None) -> tuple[str, list[Any]]: """The `WHERE` for a role `summarise` may be sent: no summary yet and some text to summarise. Given `uids`, those of them and nothing else, open or not and given up or not, because somebody asked for them. Otherwise open, not of a hidden company, not given up, and among the `per_agent` best ungated scores of an enabled agent.""" where = ["r.summary = ''", "(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"] if uids is not None: where.append("r.uid IN (SELECT value FROM json_each(?))") return " AND ".join(where), [json.dumps(list(uids))] where += [ "r.closed_at IS NULL", "r.company_key NOT IN (SELECT key FROM companies WHERE affinity = -1)", f"""NOT EXISTS (SELECT 1 FROM state WHERE key = '{self._SUMMARY_MISSED}' || r.uid AND CAST(value AS INTEGER) >= {GIVE_UP_AFTER})""", """r.uid IN (SELECT uid FROM ( SELECT s.uid, ROW_NUMBER() OVER ( PARTITION BY s.agent_id ORDER BY s.score DESC, top.first_seen DESC, s.uid) AS n FROM scores s JOIN agents a ON a.id = s.agent_id AND a.enabled = 1 JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL WHERE s.gated_reason = '') WHERE n <= ?)""", ] return " AND ".join(where), [per_agent] def summarise_pending(self, per_agent: int) -> int: where, params = self._summary_scope(per_agent, None) row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params) return int(row["n"]) if row else 0 def summarise_waiting(self, limit: int, per_agent: int, uids: Sequence[str] | None = None) -> list[Role]: """Roles `summarise` has yet to write, newest first. The top `per_agent` of every enabled agent's scores, so the backlog is bounded by the number of agents rather than by the number of open roles; given `uids`, those still without a summary. A batch asks again when it runs.""" where, params = self._summary_scope(per_agent, uids) return [Role.from_row(r) for r in self._rows( f"SELECT r.* FROM roles r WHERE {where} " "ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])] def set_summary(self, uid: str, summary: str) -> bool: """Write a role's summary and stamp `summarised_at`. Only over an empty one, so a second batch that carried the same role does not replace the first answer. False for an unknown uid or one already summarised.""" with self.transaction() as conn: written = conn.execute( "UPDATE roles SET summary = ?, summarised_at = ? " "WHERE uid = ? AND summary = ''", (summary, self.now(), uid)).rowcount conn.execute("DELETE FROM state WHERE key = ?", (self._SUMMARY_MISSED + uid,)) return bool(written) def summarise_missed(self, uids: Sequence[str], why: str = "") -> int: """One reply that left these roles without a summary. Counted only against those still without one. Returns how many this pushed to `GIVE_UP_AFTER`. `why` is accepted for `ModelTask.missed` and not kept.""" if not uids: return 0 rows = self._rows("SELECT uid FROM roles WHERE summary = '' " "AND uid IN (SELECT value FROM json_each(?))", (json.dumps(list(uids)),)) keys = [self._SUMMARY_MISSED + r["uid"] for r in rows] with self.transaction() as conn: conn.executemany( """INSERT INTO state (key, value) VALUES (?, '1') ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + 1 AS TEXT)""", [(key,) for key in keys]) crossed = conn.execute( "SELECT COUNT(*) FROM state WHERE key IN (SELECT value FROM json_each(?)) " "AND CAST(value AS INTEGER) = ?", (json.dumps(keys), GIVE_UP_AFTER)).fetchone()[0] return int(crossed) def set_article_text(self, uid: str, text: str) -> Role | None: """Store the posting page as plain text and stamp `article_at`. The role as it now stands, or None for an unknown uid.""" if not self._write("UPDATE roles SET article_text = ?, article_at = ? WHERE uid = ?", (text, self.now(), uid)): return None return self.role(uid) # -------------------------------------------------------------- vectors def roles_to_embed(self, model: str, limit: int) -> list[Role]: """Open roles with no vector from `model`, newest first, so a backlog embedded from the top improves what is on screen first.""" return [Role.from_row(r) for r in self._rows( """SELECT r.* FROM roles r LEFT JOIN vectors v ON v.uid = r.uid AND v.model = ? WHERE v.uid IS NULL AND r.closed_at IS NULL ORDER BY r.first_seen DESC, r.uid LIMIT ?""", (model, limit))] def put_vectors(self, model: str, pairs: Sequence[tuple[str, np.ndarray]]) -> int: stamp = self.now() with self.transaction() as conn: return conn.executemany( """INSERT INTO vectors (uid, model, dim, vec, embedded_at) VALUES (?,?,?,?,?) ON CONFLICT(uid) DO UPDATE SET model=excluded.model, dim=excluded.dim, vec=excluded.vec, embedded_at=excluded.embedded_at""", [(uid, model, int(vec.shape[-1]), pack(vec), stamp) for uid, vec in pairs]).rowcount def vectors(self, model: str) -> dict[str, np.ndarray]: """Every vector from `model`, by uid. The whole table: at ~30k roles of 384 floats that is ~45 MB, and one read beats one per agent.""" return {r["uid"]: unpack(r["vec"]) for r in self._rows( "SELECT uid, vec FROM vectors WHERE model = ?", (model,))} def vector_counts(self) -> dict[str, int]: return {r["model"]: int(r["n"]) for r in self._rows( "SELECT model, COUNT(*) AS n FROM vectors GROUP BY model")} def drop_vectors(self, keep: str) -> int: """Delete vectors from every model but `keep`: another model's are a different geometry, never compared, and only make counts lie.""" return self._write("DELETE FROM vectors WHERE model <> ?", (keep,)) # --------------------------------------------------------------- agents def agents(self, enabled_only: bool = False) -> list[Agent]: sql = "SELECT * FROM agents" if enabled_only: sql += " WHERE enabled = 1" return [Agent.from_row(r) for r in self._rows(sql + " ORDER BY name COLLATE NOCASE")] def agent(self, agent_id: int) -> Agent | None: row = self._row("SELECT * FROM agents WHERE id=?", (agent_id,)) return Agent.from_row(row) if row else None def save_agent(self, name: str, intent: str = "", gates: Sequence[Gate] = (), weights: Weights | None = None, enabled: bool = True, agent_id: int | None = None) -> Agent: """Create an agent, or with `agent_id` replace that one's fields. Raises ValueError on an empty or taken name, KeyError on an unknown id.""" name = name.strip() if not name: raise ValueError("an agent needs a name") fields = (name, intent, json.dumps([g.model_dump() for g in gates]), (weights or Weights()).model_dump_json(), int(enabled)) try: with self.transaction() as conn: if agent_id is None: agent_id = conn.execute( """INSERT INTO agents (name, intent, gates, weights, enabled, created_at) VALUES (?,?,?,?,?,?)""", (*fields, self.now())).lastrowid elif not conn.execute( """UPDATE agents SET name=?, intent=?, gates=?, weights=?, enabled=? WHERE id=?""", (*fields, agent_id)).rowcount: raise KeyError(agent_id) except sqlite3.IntegrityError as exc: raise ValueError(f"an agent named {name!r} already exists") from exc assert agent_id is not None found = self.agent(agent_id) assert found is not None return found def delete_agent(self, agent_id: int) -> bool: """Its scores go with it, by the foreign key.""" return self._write("DELETE FROM agents WHERE id=?", (agent_id,)) > 0 def intent_vector(self, agent_id: int, key: str) -> np.ndarray | None: """The agent's cached intent vector, if it was made under `key`.""" row = self._row("SELECT intent_key, intent_vec FROM agents WHERE id=?", (agent_id,)) if row is None or row["intent_key"] != key or row["intent_vec"] is None: return None return unpack(row["intent_vec"]) def set_intent_vector(self, agent_id: int, key: str, vec: np.ndarray) -> None: self._write("UPDATE agents SET intent_key=?, intent_vec=? WHERE id=?", (key, pack(vec), agent_id)) # --------------------------------------------------------------- scores def score_candidates(self, gates: Sequence[Gate]) -> list[Candidate]: """Every open role of a company that is not muted, with how it meets each gate: 1 match, 0 no match, -1 the column is empty. The gates are evaluated here, in SQL, so a `require` gate is the `WHERE` it reads as, written as a column so a gated role keeps its row and reason.""" columns: list[str] = [] params: list[Any] = [] for n, gate in enumerate(gates): empty, match, values = _gate_sql(gate) columns.append( f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}") params += values select = ", ".join(["r.uid", "r.first_seen", "COALESCE(c.affinity, 0) AS affinity", *columns]) rows = self._rows( f"""SELECT {select} FROM roles r LEFT JOIN companies c ON c.key = r.company_key WHERE r.closed_at IS NULL AND COALESCE(c.affinity, 0) <> -1""", params) return [Candidate(uid=r["uid"], first_seen=r["first_seen"], affinity=int(r["affinity"]), gates=[int(r[f"g{n}"]) for n in range(len(gates))]) for r in rows] def replace_scores(self, agent_id: int, rows: Sequence[tuple[str, float, str]]) -> int: """`(uid, score, gated_reason)` for every role this agent scored, replacing whatever it had: a role closed or muted since the last run drops out rather than keeping a stale score. One transaction, so a reader sees the old list or the new one.""" stamp = self.now() with self.transaction() as conn: conn.execute("DELETE FROM scores WHERE agent_id=?", (agent_id,)) conn.executemany( """INSERT INTO scores (uid, agent_id, score, gated_reason, scored_at) VALUES (?,?,?,?,?)""", [(uid, agent_id, score, reason, stamp) for uid, score, reason in rows]) return len(rows) def scores(self, agent_id: int) -> dict[str, tuple[float, str]]: """`uid -> (score, gated_reason)` for one agent.""" return {r["uid"]: (float(r["score"]), str(r["gated_reason"])) for r in self._rows( "SELECT uid, score, gated_reason FROM scores WHERE agent_id=?", (agent_id,))} # ------------------------------------------------------------ companies def add_company(self, name: str, *, key: str = "", domain: str = "", affinity: int = 0, status: str = "researching", careers_url: str = "", notes: str = "") -> Company: """Raises ValueError on a key already taken or an affinity out of range.""" name = name.strip() key = key or company_key(name) if not key: raise ValueError("a company needs a name") _check_affinity(affinity) stamp = self.now() try: self._write( """INSERT INTO companies (key, name, domain, affinity, status, careers_url, notes, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?)""", (key, name, domain, affinity, status, careers_url, notes, stamp, stamp)) except sqlite3.IntegrityError as exc: raise ValueError(f"company {key!r} already exists") from exc return self._company(key) def company(self, key: str) -> Company | None: row = self._row("SELECT * FROM companies WHERE key=?", (key,)) return Company.from_row(row) if row else None def companies(self) -> list[Company]: return [Company.from_row(r) for r in self._rows("SELECT * FROM companies ORDER BY name COLLATE NOCASE")] def update_company(self, key: str, **fields: Any) -> Company: """Change any of name, domain, affinity, status, careers_url, notes.""" unknown = set(fields) - set(_COMPANY_FIELDS) if unknown: raise ValueError(f"not a company field: {', '.join(sorted(unknown))}") if "affinity" in fields: _check_affinity(fields["affinity"]) if fields: sets = ", ".join(f"{f}=?" for f in fields) changed = self._write(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?", (*fields.values(), self.now(), key)) if not changed: raise KeyError(key) return self._company(key) def remove_company(self, key: str) -> bool: return self._write("DELETE FROM companies WHERE key=?", (key,)) > 0 def _company(self, key: str) -> Company: found = self.company(key) if found is None: raise KeyError(key) return found def open_role_counts(self) -> dict[str, int]: """`company_key -> open role count`, for the company list's row count. A key with no open roles is simply absent.""" return {str(r["company_key"]): int(r["n"]) for r in self._rows( "SELECT company_key, COUNT(*) AS n FROM roles " "WHERE closed_at IS NULL GROUP BY company_key")} def open_countries_by_subscription(self) -> dict[int, dict[str, int]]: """Per subscription id, its open roles counted by `country`.""" rows = self._rows("""SELECT subscription_id, country, COUNT(*) AS n FROM roles WHERE closed_at IS NULL AND subscription_id IS NOT NULL GROUP BY subscription_id, country""") out: dict[int, dict[str, int]] = {} for r in rows: out.setdefault(r["subscription_id"], {})[r["country"]] = r["n"] return out def company_boards(self) -> dict[str, str]: """`company_key -> ats host`, read off each company's own subscription. `Subscription.company` is the display name it and the company row were both added with, so `company_key` of one finds the other; a company added by hand with no subscription is simply absent.""" boards: dict[str, str] = {} for sub in self.subscriptions(source="ats"): boards.setdefault(company_key(sub.company), sub.board) return boards # -------------------------------------------------------- subscriptions def subscriptions(self, source: str | None = None, fetchable: bool = False) -> list[Subscription]: """`fetchable` is what a pull reads: subscribed and with fetch on.""" where, params = [], [] if source: where.append("source = ?") params.append(source) if fetchable: where.append("fetch = 1 AND subscribed = 1") sql = "SELECT * FROM subscriptions" if where: sql += " WHERE " + " AND ".join(where) sql += " ORDER BY company, board, label" return [Subscription.from_row(r) for r in self._rows(sql, params)] def subscription(self, sub_id: int) -> Subscription | None: row = self._row("SELECT * FROM subscriptions WHERE id=?", (sub_id,)) return Subscription.from_row(row) if row else None def subscription_for(self, source: str, label: str) -> Subscription | None: """The row an item names: `Item.subscription` is the label.""" row = self._row("SELECT * FROM subscriptions WHERE source=? AND label=?", (source, label)) return Subscription.from_row(row) if row else None def add_subscription(self, source: str, target: str, label: str = "", company: str = "", board: str = "", params: dict[str, Any] | None = None, target_id: str = "") -> Subscription: """Raises ValueError on a duplicate target or label within one plugin.""" target = target.strip() if not target: raise ValueError("a subscription needs a target") label = label.strip() or target try: with self.transaction() as conn: new_id = conn.execute( """INSERT INTO subscriptions (source, target, label, company, board, params, added_at, target_id) VALUES (?,?,?,?,?,?,?,?)""", (source, target, label, company, board, json.dumps(params or {}), self.now(), target_id)).lastrowid except sqlite3.IntegrityError as exc: raise ValueError(f"{source} already has {target!r} or the label {label!r}") from exc assert new_id is not None return self._subscription(new_id) def update_subscription(self, sub_id: int, **fields: Any) -> Subscription: """Change labels, params, target_id or the three switches.""" unknown = set(fields) - set(_SUBSCRIPTION_FIELDS) if unknown: raise ValueError(f"not a subscription field: {', '.join(sorted(unknown))}") if "params" in fields: fields["params"] = json.dumps(fields["params"] or {}) for switch in ("subscribed", "fetch", "show"): if switch in fields: fields[switch] = int(bool(fields[switch])) if fields: sets = ", ".join(f"{f}=?" for f in fields) try: changed = self._write(f"UPDATE subscriptions SET {sets} WHERE id=?", (*fields.values(), sub_id)) except sqlite3.IntegrityError as exc: raise ValueError(f"label {fields.get('label')!r} is already in use") from exc if not changed: raise KeyError(sub_id) return self._subscription(sub_id) def unsubscribe(self, sub_id: int) -> Subscription: """Stop following it and hide it. The row stays, and so do its roles.""" return self.update_subscription(sub_id, subscribed=0, fetch=0, show=0) def _subscription(self, sub_id: int) -> Subscription: found = self.subscription(sub_id) if found is None: raise KeyError(sub_id) return found # --------------------------------------------------------- applications def save_application(self, application: Application) -> Application: """Insert or update one application. An empty `id` is minted. Entering a status, on insert or by change, appends an `application_events` row in the same transaction.""" _check_status(application.status) app_id = application.id or uuid4().hex stamp = self.now() fields = application.model_dump(exclude={"id", "created_at", "updated_at"}) with self.transaction() as conn: before = conn.execute("SELECT status FROM applications WHERE id=?", (app_id,)).fetchone() if before is None: columns = ["id", *fields, "created_at", "updated_at"] conn.execute( f"INSERT INTO applications ({', '.join(columns)}) " f"VALUES ({', '.join('?' for _ in columns)})", (app_id, *fields.values(), stamp, stamp)) else: sets = ", ".join(f"{f}=?" for f in fields) conn.execute(f"UPDATE applications SET {sets}, updated_at=? WHERE id=?", (*fields.values(), stamp, app_id)) if before is None or before["status"] != application.status: conn.execute("INSERT INTO application_events (application_id, status, at) " "VALUES (?,?,?)", (app_id, application.status, stamp)) return self._application(app_id) def set_application_status(self, app_id: str, status: str) -> Application: """Move one application to `status`. Raises ValueError for a status outside `STATUSES` and KeyError for an unknown id.""" _check_status(status) current = self._application(app_id) return self.save_application(current.model_copy(update={"status": status})) def application(self, app_id: str) -> Application | None: row = self._row("SELECT * FROM applications WHERE id=?", (app_id,)) return Application.from_row(row) if row else None def applications(self, status: str | None = None) -> list[Application]: sql, params = "SELECT * FROM applications", [] if status: sql += " WHERE status = ?" params.append(status) sql += " ORDER BY updated_at DESC, id" return [Application.from_row(r) for r in self._rows(sql, params)] def application_events(self, app_id: str) -> list[ApplicationEvent]: return [ApplicationEvent.from_row(r) for r in self._rows( "SELECT * FROM application_events WHERE application_id=? ORDER BY id", (app_id,))] def _application(self, app_id: str) -> Application: found = self.application(app_id) if found is None: raise KeyError(app_id) return found # ---------------------------------------------------------------- state @property def state(self) -> MutableMapping[str, str]: """The `state` table as a mapping. What the runner hands a plugin as `self.state`: it can keep a counter without being able to read roles.""" return _StateMap(self) def _check_status(status: str) -> None: if status not in STATUSES: raise ValueError(f"{status!r} is not an application status " f"({', '.join(STATUSES)})") def _check_affinity(affinity: Any) -> None: if not isinstance(affinity, int) or affinity not in AFFINITY: raise ValueError(f"affinity is -1, 0, 1 or 2, not {affinity!r}")