The ats boards only read companies someone listed, and the seed list is US-centred: 31 of 29,000 open roles were in Denmark. A `search` plugin runs saved queries on boards that span companies, pulled in the same task as the boards: Jobindex (Denmark, RSS, no key) and Adzuna (UK and most of the EU, free key, built to its documented API). Jobnet and Jooble were checked and left out. `country` is now filled from the board's location line by a lookup, with EU for Europe-wide roles, so country gates work without extract. `app companies prune --in` unsubscribes boards with no open role in a region. The starter agents and searches are Copenhagen and Europe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
821 lines
38 KiB
Python
821 lines
38 KiB
Python
"""`Store`: every SQL statement in the application.
|
|
|
|
The connection plumbing is a copy of newsdesk's `Store`, on purpose: `desk`
|
|
has no storage package. A connection is opened per call and closed again, so
|
|
one caller's commit can never land inside another's transaction; in WAL mode
|
|
that costs a file handle.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sqlite3
|
|
from collections.abc import Callable, Iterator, MutableMapping, Sequence
|
|
from contextlib import closing, contextmanager
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
import numpy as np
|
|
|
|
from desk.models import Item
|
|
from app.models import (
|
|
AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate,
|
|
Company, Gate, Role, ScoredRole, Subscription, Weights, company_key, country_of,
|
|
place_key, title_key,
|
|
)
|
|
from app.storage import schema
|
|
|
|
#: Text columns a plugin may fill from `Item.raw` under the same name. An
|
|
#: empty value never overwrites a filled column, so an upsert from a board that
|
|
#: says less than `extract` found keeps what `extract` found.
|
|
_RAW_TEXT = ("place", "country", "currency", "seniority", "function")
|
|
_RAW_INT = ("salary_min", "salary_max")
|
|
|
|
#: What `extract` may fill. Only an empty column is written: a board's own
|
|
#: structured field wins over what a model read out of the prose, and the next
|
|
#: upsert from that board would put it back anyway.
|
|
_EXTRACT_TEXT = ("place", "country", "remote", "seniority", "currency", "function",
|
|
"permit")
|
|
|
|
#: How many replies may leave a role unanswered before `extract` stops
|
|
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
|
|
#: never answers is otherwise paid for on every pull for as long as it exists.
|
|
GIVE_UP_AFTER = 3
|
|
|
|
#: A role `extract` has yet to read: never answered, still open, not given up.
|
|
_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL "
|
|
f"AND extract_tries < {GIVE_UP_AFTER}")
|
|
|
|
_COMPANY_FIELDS = ("name", "domain", "affinity", "status", "careers_url", "notes")
|
|
_SUBSCRIPTION_FIELDS = ("label", "company", "board", "params", "subscribed", "fetch",
|
|
"show", "target_id")
|
|
|
|
|
|
def _iso(moment: datetime) -> str:
|
|
"""UTC, to the second. Every timestamp column is compared as text, so every
|
|
writer has to spell it the same way."""
|
|
return moment.astimezone(UTC).isoformat(timespec="seconds")
|
|
|
|
|
|
def _remote(value: Any) -> str:
|
|
"""A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything
|
|
unrecognised is '' (unknown), which `extract` may fill later."""
|
|
if value is True:
|
|
return "remote"
|
|
text = str(value or "").strip().lower().replace("-", "").replace(" ", "")
|
|
text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text)
|
|
return text if text in REMOTE else ""
|
|
|
|
|
|
def _like(text: str) -> str:
|
|
"""A `LIKE` pattern for a case-insensitive substring match, with SQLite's
|
|
own wildcards escaped so a search for "50%" or "a_b" stays literal."""
|
|
escaped = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
|
return f"%{escaped}%"
|
|
|
|
|
|
#: Vectors are stored little-endian float32, fixed here so a file written on
|
|
#: one machine reads on another. newsdesk's encoding, written again.
|
|
_DTYPE = np.dtype("<f4")
|
|
|
|
|
|
def pack(vec: np.ndarray) -> bytes:
|
|
return np.asarray(vec, dtype=_DTYPE).tobytes()
|
|
|
|
|
|
def unpack(blob: bytes) -> np.ndarray:
|
|
return np.frombuffer(blob, dtype=_DTYPE)
|
|
|
|
|
|
def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
|
|
"""One gate as `(empty, match, params)`: SQL that is true when the role's
|
|
column is empty, and SQL that is true when it matches, over `roles r`.
|
|
`params` belong to `match`; `empty` takes none."""
|
|
column = f"r.{gate.field}"
|
|
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
|
|
marks = ", ".join("?" for _ in values) or "NULL"
|
|
if gate.field == "salary_min":
|
|
floor = min(_int_or_none(v) or 0 for v in gate.values)
|
|
return f"{column} IS NULL", f"{column} >= ?", [floor]
|
|
if gate.field == "place":
|
|
likes = " OR ".join("LOWER(r.place) LIKE ? ESCAPE '\\'" for _ in values) or "0"
|
|
return "r.place = ''", f"({likes})", [_like(v) for v in values]
|
|
if gate.field == "company":
|
|
keys = [company_key(v) for v in values]
|
|
return ("r.company_key = ''",
|
|
f"r.company_key IN ({', '.join('?' for _ in keys) or 'NULL'})", keys)
|
|
if gate.field == "languages":
|
|
return ("r.languages IN ('', '[]')",
|
|
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
|
|
f"WHERE LOWER(value) NOT IN ({marks}))", values)
|
|
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
|
|
|
|
|
|
def _int_or_none(value: Any) -> int | None:
|
|
if value is None or value == "" or isinstance(value, bool):
|
|
return None
|
|
try:
|
|
return int(float(value))
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
class _StateMap(MutableMapping[str, str]):
|
|
"""`Store.state`: the key/value table, read and written one key at a time."""
|
|
|
|
def __init__(self, store: Store) -> None:
|
|
self._store = store
|
|
|
|
def __getitem__(self, key: str) -> str:
|
|
row = self._store._row("SELECT value FROM state WHERE key=?", (key,))
|
|
if row is None:
|
|
raise KeyError(key)
|
|
return str(row["value"])
|
|
|
|
def __setitem__(self, key: str, value: str) -> None:
|
|
self._store._write(
|
|
"""INSERT INTO state (key, value) VALUES (?,?)
|
|
ON CONFLICT(key) DO UPDATE SET value=excluded.value""",
|
|
(key, value))
|
|
|
|
def __delitem__(self, key: str) -> None:
|
|
if not self._store._write("DELETE FROM state WHERE key=?", (key,)):
|
|
raise KeyError(key)
|
|
|
|
def __iter__(self) -> Iterator[str]:
|
|
return iter([r["key"] for r in self._store._rows("SELECT key FROM state ORDER BY key")])
|
|
|
|
def __len__(self) -> int:
|
|
row = self._store._row("SELECT COUNT(*) AS n FROM state")
|
|
return int(row["n"]) if row else 0
|
|
|
|
|
|
class Store:
|
|
def __init__(self, path: str | Path) -> None:
|
|
self.path = Path(path).expanduser()
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
#: What "now" is. Replaced by tests that need two writes a second apart.
|
|
self.clock: Callable[[], datetime] = lambda: datetime.now(UTC)
|
|
with self.transaction() as conn:
|
|
# Persistent: recorded in the file, inherited by every connection.
|
|
conn.execute("PRAGMA journal_mode=WAL")
|
|
#: Migration versions this open applied; empty when already current.
|
|
self.migrations_applied: list[int] = schema.initialise(conn)
|
|
|
|
# ------------------------------------------------------------ connections
|
|
|
|
def _connect(self) -> sqlite3.Connection:
|
|
"""A fresh connection. `foreign_keys` is off by default per connection;
|
|
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
|
|
`synchronous=NORMAL` is the standard WAL companion."""
|
|
conn = sqlite3.connect(self.path)
|
|
conn.row_factory = sqlite3.Row
|
|
conn.execute("PRAGMA foreign_keys=ON")
|
|
conn.execute("PRAGMA busy_timeout=30000")
|
|
conn.execute("PRAGMA synchronous=NORMAL")
|
|
return conn
|
|
|
|
@contextmanager
|
|
def transaction(self) -> Iterator[sqlite3.Connection]:
|
|
"""One connection, one transaction, closed on the way out."""
|
|
conn = self._connect()
|
|
try:
|
|
with conn:
|
|
yield conn
|
|
finally:
|
|
conn.close()
|
|
|
|
def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]:
|
|
with closing(self._connect()) as conn:
|
|
return conn.execute(sql, params).fetchall()
|
|
|
|
def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None:
|
|
with closing(self._connect()) as conn:
|
|
row: sqlite3.Row | None = conn.execute(sql, params).fetchone()
|
|
return row
|
|
|
|
def _write(self, sql: str, params: Any = ()) -> int:
|
|
"""One statement in its own transaction. Returns the rowcount."""
|
|
with self.transaction() as conn:
|
|
return conn.execute(sql, params).rowcount
|
|
|
|
def now(self) -> str:
|
|
return _iso(self.clock())
|
|
|
|
# ---------------------------------------------------------------- roles
|
|
|
|
def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool:
|
|
"""Store one posting. Returns True when it was new.
|
|
|
|
`first_seen` and `created_at` are written on insert only; `last_seen` on
|
|
every call, and a role seen again is open again. The gate columns come
|
|
from `item.raw` under their own names, with `company` falling back to
|
|
the subscription's and `country` to what `place` names; an empty value
|
|
never overwrites a filled column.
|
|
"""
|
|
if not item.source:
|
|
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
|
|
raw = item.raw or {}
|
|
company = str(raw.get("company") or (subscription.company if subscription else ""))
|
|
place = str(raw.get("place") or "")
|
|
stamp = self.now()
|
|
row: dict[str, Any] = {
|
|
"uid": item.uid,
|
|
"source": item.source,
|
|
"source_id": item.source_id,
|
|
"subscription_id": subscription.id if subscription else None,
|
|
"company_key": company_key(company),
|
|
"company": company,
|
|
"title": item.title,
|
|
"title_key": title_key(item.title),
|
|
"place_key": place_key(place),
|
|
"remote": _remote(raw.get("remote")),
|
|
"description": item.body,
|
|
"feed_text": item.excerpt,
|
|
"original_url": item.location,
|
|
"published_at": _iso(item.published_at),
|
|
"first_seen": stamp,
|
|
"last_seen": stamp,
|
|
"raw": json.dumps(raw, default=str),
|
|
"created_at": stamp,
|
|
}
|
|
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
|
|
row["country"] = row["country"] or country_of(place)
|
|
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
|
|
|
keep_filled = ("company", "company_key", "place_key", "remote", "description",
|
|
"feed_text", "original_url", *_RAW_TEXT)
|
|
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
|
|
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
|
|
for c in (*_RAW_INT, "subscription_id")]
|
|
updates += [f"{c}=excluded.{c}"
|
|
for c in ("title", "title_key", "published_at", "raw", "last_seen")]
|
|
updates.append("closed_at=NULL")
|
|
columns = ", ".join(row)
|
|
marks = ", ".join(f":{c}" for c in row)
|
|
with self.transaction() as conn:
|
|
existed = conn.execute("SELECT 1 FROM roles WHERE uid=?",
|
|
(row["uid"],)).fetchone() is not None
|
|
conn.execute(
|
|
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
|
|
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
|
|
return not existed
|
|
|
|
def role(self, uid: str) -> Role | None:
|
|
row = self._row("SELECT * FROM roles WHERE uid=?", (uid,))
|
|
return Role.from_row(row) if row else None
|
|
|
|
def roles(self, limit: int = 100, company: str | None = None,
|
|
place: str | None = None, open_only: bool = True,
|
|
q: str | None = None, function: str | None = None,
|
|
remote: str | None = None, days: int | None = None,
|
|
hide_muted: bool = False, agent: int | None = None,
|
|
gated: bool = False) -> list[ScoredRole]:
|
|
"""Newest first by `first_seen`. `company` and `place` are
|
|
case-insensitive substrings of the columns a board wrote — not the
|
|
key, so a search does not have to know the exact slug a company folds
|
|
to. `q` is matched word by word against `title`; a role must match
|
|
every word. `function` and `remote` match a gate column exactly.
|
|
`days` keeps only roles first seen within that many days of now.
|
|
|
|
`hide_muted` excludes a muted company's roles before `limit` is
|
|
applied, in SQL rather than after the fact: a company that has
|
|
published a lot, all at once — a fresh board added, a re-pull that
|
|
finally reached it — can otherwise fill the whole page with roles
|
|
that then all turn out to be from the one company someone muted,
|
|
leaving nothing to show for a default query that would otherwise
|
|
have plenty.
|
|
|
|
`agent` orders by that agent's score instead, highest first, and
|
|
carries `score` and `gated_reason` on each row. Only roles it has
|
|
scored are listed, and a gated one only when `gated` is set."""
|
|
where, params = [], []
|
|
if company:
|
|
where.append("company LIKE ? ESCAPE '\\'")
|
|
params.append(_like(company))
|
|
if place:
|
|
where.append("place LIKE ? ESCAPE '\\'")
|
|
params.append(_like(place))
|
|
if function:
|
|
where.append("function = ?")
|
|
params.append(function)
|
|
if remote:
|
|
where.append("remote = ?")
|
|
params.append(remote)
|
|
if days is not None:
|
|
where.append("first_seen >= ?")
|
|
params.append(_iso(self.clock() - timedelta(days=days)))
|
|
for word in (q or "").split():
|
|
where.append("title LIKE ? ESCAPE '\\'")
|
|
params.append(_like(word))
|
|
if open_only:
|
|
where.append("closed_at IS NULL")
|
|
if hide_muted:
|
|
where.append("company_key NOT IN "
|
|
"(SELECT key FROM companies WHERE affinity = -1)")
|
|
joins: list[Any] = []
|
|
if agent is None:
|
|
sql = "SELECT roles.* FROM roles"
|
|
order = "roles.first_seen DESC, roles.uid"
|
|
else:
|
|
sql = ("SELECT roles.*, s.score, s.gated_reason FROM roles "
|
|
"JOIN scores s ON s.uid = roles.uid AND s.agent_id = ?")
|
|
joins.append(agent)
|
|
if not gated:
|
|
where.append("s.gated_reason = ''")
|
|
order = "s.score DESC, roles.first_seen DESC, roles.uid"
|
|
if where:
|
|
sql += " WHERE " + " AND ".join(where)
|
|
sql += f" ORDER BY {order} LIMIT ?"
|
|
return [ScoredRole.from_row(r)
|
|
for r in self._rows(sql, [*joins, *params, limit])]
|
|
|
|
def mark_closed(self, source: str, seen_ids: Sequence[str],
|
|
subscription_id: int) -> int:
|
|
"""Close every open role of one board that a complete fetch of it did not
|
|
list. Returns how many closed. Only call it after a fetch that finished:
|
|
a failed one lists nothing, and would close the whole board."""
|
|
return self._write(
|
|
"""UPDATE roles SET closed_at = ?
|
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
|
AND source_id NOT IN (SELECT value FROM json_each(?))""",
|
|
(self.now(), source, subscription_id, json.dumps(list(seen_ids))))
|
|
|
|
def close_unseen(self, source: str, before: str) -> int:
|
|
"""Close every open role of `source` last seen before `before`. For a
|
|
source whose fetch is never a complete list, a search, where a role
|
|
dropping out of one run's results is not the role closing."""
|
|
return self._write(
|
|
"""UPDATE roles SET closed_at = ?
|
|
WHERE source = ? AND closed_at IS NULL AND last_seen < ?""",
|
|
(self.now(), source, before))
|
|
|
|
def close_subscription(self, subscription_id: int) -> int:
|
|
"""Close every open role of one subscription, for a board that is no
|
|
longer fetched: nothing would ever close them otherwise."""
|
|
return self._write(
|
|
"UPDATE roles SET closed_at = ? WHERE subscription_id = ? AND closed_at IS NULL",
|
|
(self.now(), subscription_id))
|
|
|
|
def backfill_country(self) -> int:
|
|
"""Fill `country` from `place` on every role that has none, the way
|
|
`upsert_role` does for a new one. Returns how many it filled."""
|
|
rows = self._rows("SELECT uid, place FROM roles WHERE country = '' AND place != ''")
|
|
filled = [(code, r["uid"]) for r in rows if (code := country_of(r["place"]))]
|
|
with self.transaction() as conn:
|
|
conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", filled)
|
|
return len(filled)
|
|
|
|
# -------------------------------------------------------------- extract
|
|
|
|
def extract_pending(self) -> int:
|
|
row = self._row(f"SELECT COUNT(*) AS n FROM roles WHERE {_EXTRACT_WAITING}")
|
|
return int(row["n"]) if row else 0
|
|
|
|
def extract_waiting(self, limit: int,
|
|
uids: Sequence[str] | None = None) -> list[Role]:
|
|
"""Roles `extract` has yet to read, newest first. Given `uids`, only
|
|
those of them still waiting: a batch asks again when it runs, since a
|
|
role may have been answered since the task shared the work out."""
|
|
sql, params = f"SELECT * FROM roles WHERE {_EXTRACT_WAITING}", []
|
|
if uids is not None:
|
|
sql += " AND uid IN (SELECT value FROM json_each(?))"
|
|
params.append(json.dumps(list(uids)))
|
|
sql += " ORDER BY first_seen DESC, uid LIMIT ?"
|
|
return [Role.from_row(r) for r in self._rows(sql, [*params, limit])]
|
|
|
|
def set_extracted(self, uid: str, fields: dict[str, Any]) -> int | None:
|
|
"""Write what `extract` read for one role and stamp `extracted_at`.
|
|
|
|
Only into empty columns, and only a non-empty value: a board's own
|
|
field is never replaced and nothing is ever blanked. `place` brings
|
|
`place_key` with it. Returns how many columns were filled, or None
|
|
for an unknown uid. Values are trusted to be in vocabulary; the
|
|
job's `apply` is where a reply is checked."""
|
|
with self.transaction() as conn:
|
|
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
|
|
if row is None:
|
|
return None
|
|
sets: dict[str, Any] = {}
|
|
for name in _EXTRACT_TEXT:
|
|
value = str(fields.get(name) or "")
|
|
if value and not row[name]:
|
|
sets[name] = value
|
|
for name in _RAW_INT:
|
|
number = _int_or_none(fields.get(name))
|
|
if number is not None and row[name] is None:
|
|
sets[name] = number
|
|
languages = [str(code) for code in fields.get("languages") or []]
|
|
if languages and json.loads(row["languages"] or "[]") == []:
|
|
sets["languages"] = json.dumps(languages)
|
|
filled = len(sets)
|
|
if "place" in sets:
|
|
sets["place_key"] = place_key(sets["place"])
|
|
sets["extracted_at"] = self.now()
|
|
conn.execute(
|
|
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
|
|
(*sets.values(), uid))
|
|
return filled
|
|
|
|
def extract_missed(self, uids: Sequence[str], why: str = "") -> int:
|
|
"""One reply that left these roles unanswered. Counted only against
|
|
those still waiting, so a role the reply did answer is not charged.
|
|
Returns how many this pushed to `GIVE_UP_AFTER`. `why` is accepted
|
|
for `ModelTask.missed` and not kept: there is no Given up list yet."""
|
|
if not uids:
|
|
return 0
|
|
where = f"uid IN (SELECT value FROM json_each(?)) AND {_EXTRACT_WAITING}"
|
|
wanted = json.dumps(list(uids))
|
|
with self.transaction() as conn:
|
|
crossed = conn.execute(
|
|
f"SELECT COUNT(*) FROM roles WHERE {where} AND extract_tries + 1 >= ?",
|
|
(wanted, GIVE_UP_AFTER)).fetchone()[0]
|
|
conn.execute(f"UPDATE roles SET extract_tries = extract_tries + 1 "
|
|
f"WHERE {where}", (wanted,))
|
|
return int(crossed)
|
|
|
|
# -------------------------------------------------------------- vectors
|
|
|
|
def roles_to_embed(self, model: str, limit: int) -> list[Role]:
|
|
"""Open roles with no vector from `model`, newest first, so a backlog
|
|
embedded from the top improves what is on screen first."""
|
|
return [Role.from_row(r) for r in self._rows(
|
|
"""SELECT r.* FROM roles r
|
|
LEFT JOIN vectors v ON v.uid = r.uid AND v.model = ?
|
|
WHERE v.uid IS NULL AND r.closed_at IS NULL
|
|
ORDER BY r.first_seen DESC, r.uid LIMIT ?""", (model, limit))]
|
|
|
|
def put_vectors(self, model: str, pairs: Sequence[tuple[str, np.ndarray]]) -> int:
|
|
stamp = self.now()
|
|
with self.transaction() as conn:
|
|
return conn.executemany(
|
|
"""INSERT INTO vectors (uid, model, dim, vec, embedded_at)
|
|
VALUES (?,?,?,?,?)
|
|
ON CONFLICT(uid) DO UPDATE SET model=excluded.model,
|
|
dim=excluded.dim, vec=excluded.vec,
|
|
embedded_at=excluded.embedded_at""",
|
|
[(uid, model, int(vec.shape[-1]), pack(vec), stamp)
|
|
for uid, vec in pairs]).rowcount
|
|
|
|
def vectors(self, model: str) -> dict[str, np.ndarray]:
|
|
"""Every vector from `model`, by uid. The whole table: at ~30k roles of
|
|
384 floats that is ~45 MB, and one read beats one per agent."""
|
|
return {r["uid"]: unpack(r["vec"]) for r in self._rows(
|
|
"SELECT uid, vec FROM vectors WHERE model = ?", (model,))}
|
|
|
|
def vector_counts(self) -> dict[str, int]:
|
|
return {r["model"]: int(r["n"]) for r in self._rows(
|
|
"SELECT model, COUNT(*) AS n FROM vectors GROUP BY model")}
|
|
|
|
def drop_vectors(self, keep: str) -> int:
|
|
"""Delete vectors from every model but `keep`: another model's are a
|
|
different geometry, never compared, and only make counts lie."""
|
|
return self._write("DELETE FROM vectors WHERE model <> ?", (keep,))
|
|
|
|
# --------------------------------------------------------------- agents
|
|
|
|
def agents(self, enabled_only: bool = False) -> list[Agent]:
|
|
sql = "SELECT * FROM agents"
|
|
if enabled_only:
|
|
sql += " WHERE enabled = 1"
|
|
return [Agent.from_row(r)
|
|
for r in self._rows(sql + " ORDER BY name COLLATE NOCASE")]
|
|
|
|
def agent(self, agent_id: int) -> Agent | None:
|
|
row = self._row("SELECT * FROM agents WHERE id=?", (agent_id,))
|
|
return Agent.from_row(row) if row else None
|
|
|
|
def save_agent(self, name: str, intent: str = "", gates: Sequence[Gate] = (),
|
|
weights: Weights | None = None, enabled: bool = True,
|
|
agent_id: int | None = None) -> Agent:
|
|
"""Create an agent, or with `agent_id` replace that one's fields.
|
|
Raises ValueError on an empty or taken name, KeyError on an unknown id."""
|
|
name = name.strip()
|
|
if not name:
|
|
raise ValueError("an agent needs a name")
|
|
fields = (name, intent, json.dumps([g.model_dump() for g in gates]),
|
|
(weights or Weights()).model_dump_json(), int(enabled))
|
|
try:
|
|
with self.transaction() as conn:
|
|
if agent_id is None:
|
|
agent_id = conn.execute(
|
|
"""INSERT INTO agents (name, intent, gates, weights, enabled,
|
|
created_at) VALUES (?,?,?,?,?,?)""",
|
|
(*fields, self.now())).lastrowid
|
|
elif not conn.execute(
|
|
"""UPDATE agents SET name=?, intent=?, gates=?, weights=?,
|
|
enabled=? WHERE id=?""",
|
|
(*fields, agent_id)).rowcount:
|
|
raise KeyError(agent_id)
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"an agent named {name!r} already exists") from exc
|
|
assert agent_id is not None
|
|
found = self.agent(agent_id)
|
|
assert found is not None
|
|
return found
|
|
|
|
def delete_agent(self, agent_id: int) -> bool:
|
|
"""Its scores go with it, by the foreign key."""
|
|
return self._write("DELETE FROM agents WHERE id=?", (agent_id,)) > 0
|
|
|
|
def intent_vector(self, agent_id: int, key: str) -> np.ndarray | None:
|
|
"""The agent's cached intent vector, if it was made under `key`."""
|
|
row = self._row("SELECT intent_key, intent_vec FROM agents WHERE id=?",
|
|
(agent_id,))
|
|
if row is None or row["intent_key"] != key or row["intent_vec"] is None:
|
|
return None
|
|
return unpack(row["intent_vec"])
|
|
|
|
def set_intent_vector(self, agent_id: int, key: str, vec: np.ndarray) -> None:
|
|
self._write("UPDATE agents SET intent_key=?, intent_vec=? WHERE id=?",
|
|
(key, pack(vec), agent_id))
|
|
|
|
# --------------------------------------------------------------- scores
|
|
|
|
def score_candidates(self, gates: Sequence[Gate]) -> list[Candidate]:
|
|
"""Every open role of a company that is not muted, with how it meets
|
|
each gate: 1 match, 0 no match, -1 the column is empty. The gates are
|
|
evaluated here, in SQL, so a `require` gate is the `WHERE` it reads
|
|
as, written as a column so a gated role keeps its row and reason."""
|
|
columns: list[str] = []
|
|
params: list[Any] = []
|
|
for n, gate in enumerate(gates):
|
|
empty, match, values = _gate_sql(gate)
|
|
columns.append(
|
|
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
|
|
params += values
|
|
select = ", ".join(["r.uid", "r.first_seen",
|
|
"COALESCE(c.affinity, 0) AS affinity", *columns])
|
|
rows = self._rows(
|
|
f"""SELECT {select} FROM roles r
|
|
LEFT JOIN companies c ON c.key = r.company_key
|
|
WHERE r.closed_at IS NULL AND COALESCE(c.affinity, 0) <> -1""",
|
|
params)
|
|
return [Candidate(uid=r["uid"], first_seen=r["first_seen"],
|
|
affinity=int(r["affinity"]),
|
|
gates=[int(r[f"g{n}"]) for n in range(len(gates))])
|
|
for r in rows]
|
|
|
|
def replace_scores(self, agent_id: int,
|
|
rows: Sequence[tuple[str, float, str]]) -> int:
|
|
"""`(uid, score, gated_reason)` for every role this agent scored,
|
|
replacing whatever it had: a role closed or muted since the last run
|
|
drops out rather than keeping a stale score. One transaction, so a
|
|
reader sees the old list or the new one."""
|
|
stamp = self.now()
|
|
with self.transaction() as conn:
|
|
conn.execute("DELETE FROM scores WHERE agent_id=?", (agent_id,))
|
|
conn.executemany(
|
|
"""INSERT INTO scores (uid, agent_id, score, gated_reason, scored_at)
|
|
VALUES (?,?,?,?,?)""",
|
|
[(uid, agent_id, score, reason, stamp) for uid, score, reason in rows])
|
|
return len(rows)
|
|
|
|
def scores(self, agent_id: int) -> dict[str, tuple[float, str]]:
|
|
"""`uid -> (score, gated_reason)` for one agent."""
|
|
return {r["uid"]: (float(r["score"]), str(r["gated_reason"]))
|
|
for r in self._rows(
|
|
"SELECT uid, score, gated_reason FROM scores WHERE agent_id=?",
|
|
(agent_id,))}
|
|
|
|
# ------------------------------------------------------------ companies
|
|
|
|
def add_company(self, name: str, *, key: str = "", domain: str = "",
|
|
affinity: int = 0, status: str = "researching",
|
|
careers_url: str = "", notes: str = "") -> Company:
|
|
"""Raises ValueError on a key already taken or an affinity out of range."""
|
|
name = name.strip()
|
|
key = key or company_key(name)
|
|
if not key:
|
|
raise ValueError("a company needs a name")
|
|
_check_affinity(affinity)
|
|
stamp = self.now()
|
|
try:
|
|
self._write(
|
|
"""INSERT INTO companies (key, name, domain, affinity, status,
|
|
careers_url, notes, created_at, updated_at)
|
|
VALUES (?,?,?,?,?,?,?,?,?)""",
|
|
(key, name, domain, affinity, status, careers_url, notes, stamp, stamp))
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"company {key!r} already exists") from exc
|
|
return self._company(key)
|
|
|
|
def company(self, key: str) -> Company | None:
|
|
row = self._row("SELECT * FROM companies WHERE key=?", (key,))
|
|
return Company.from_row(row) if row else None
|
|
|
|
def companies(self) -> list[Company]:
|
|
return [Company.from_row(r)
|
|
for r in self._rows("SELECT * FROM companies ORDER BY name COLLATE NOCASE")]
|
|
|
|
def update_company(self, key: str, **fields: Any) -> Company:
|
|
"""Change any of name, domain, affinity, status, careers_url, notes."""
|
|
unknown = set(fields) - set(_COMPANY_FIELDS)
|
|
if unknown:
|
|
raise ValueError(f"not a company field: {', '.join(sorted(unknown))}")
|
|
if "affinity" in fields:
|
|
_check_affinity(fields["affinity"])
|
|
if fields:
|
|
sets = ", ".join(f"{f}=?" for f in fields)
|
|
changed = self._write(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?",
|
|
(*fields.values(), self.now(), key))
|
|
if not changed:
|
|
raise KeyError(key)
|
|
return self._company(key)
|
|
|
|
def remove_company(self, key: str) -> bool:
|
|
return self._write("DELETE FROM companies WHERE key=?", (key,)) > 0
|
|
|
|
def _company(self, key: str) -> Company:
|
|
found = self.company(key)
|
|
if found is None:
|
|
raise KeyError(key)
|
|
return found
|
|
|
|
def open_role_counts(self) -> dict[str, int]:
|
|
"""`company_key -> open role count`, for the company list's row
|
|
count. A key with no open roles is simply absent."""
|
|
return {str(r["company_key"]): int(r["n"]) for r in self._rows(
|
|
"SELECT company_key, COUNT(*) AS n FROM roles "
|
|
"WHERE closed_at IS NULL GROUP BY company_key")}
|
|
|
|
def open_countries_by_subscription(self) -> dict[int, dict[str, int]]:
|
|
"""Per subscription id, its open roles counted by `country`."""
|
|
rows = self._rows("""SELECT subscription_id, country, COUNT(*) AS n FROM roles
|
|
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
|
|
GROUP BY subscription_id, country""")
|
|
out: dict[int, dict[str, int]] = {}
|
|
for r in rows:
|
|
out.setdefault(r["subscription_id"], {})[r["country"]] = r["n"]
|
|
return out
|
|
|
|
def company_boards(self) -> dict[str, str]:
|
|
"""`company_key -> ats host`, read off each company's own
|
|
subscription. `Subscription.company` is the display name it and the
|
|
company row were both added with, so `company_key` of one finds the
|
|
other; a company added by hand with no subscription is simply
|
|
absent."""
|
|
boards: dict[str, str] = {}
|
|
for sub in self.subscriptions(source="ats"):
|
|
boards.setdefault(company_key(sub.company), sub.board)
|
|
return boards
|
|
|
|
# -------------------------------------------------------- subscriptions
|
|
|
|
def subscriptions(self, source: str | None = None,
|
|
fetchable: bool = False) -> list[Subscription]:
|
|
"""`fetchable` is what a pull reads: subscribed and with fetch on."""
|
|
where, params = [], []
|
|
if source:
|
|
where.append("source = ?")
|
|
params.append(source)
|
|
if fetchable:
|
|
where.append("fetch = 1 AND subscribed = 1")
|
|
sql = "SELECT * FROM subscriptions"
|
|
if where:
|
|
sql += " WHERE " + " AND ".join(where)
|
|
sql += " ORDER BY company, board, label"
|
|
return [Subscription.from_row(r) for r in self._rows(sql, params)]
|
|
|
|
def subscription(self, sub_id: int) -> Subscription | None:
|
|
row = self._row("SELECT * FROM subscriptions WHERE id=?", (sub_id,))
|
|
return Subscription.from_row(row) if row else None
|
|
|
|
def subscription_for(self, source: str, label: str) -> Subscription | None:
|
|
"""The row an item names: `Item.subscription` is the label."""
|
|
row = self._row("SELECT * FROM subscriptions WHERE source=? AND label=?",
|
|
(source, label))
|
|
return Subscription.from_row(row) if row else None
|
|
|
|
def add_subscription(self, source: str, target: str, label: str = "",
|
|
company: str = "", board: str = "",
|
|
params: dict[str, Any] | None = None,
|
|
target_id: str = "") -> Subscription:
|
|
"""Raises ValueError on a duplicate target or label within one plugin."""
|
|
target = target.strip()
|
|
if not target:
|
|
raise ValueError("a subscription needs a target")
|
|
label = label.strip() or target
|
|
try:
|
|
with self.transaction() as conn:
|
|
new_id = conn.execute(
|
|
"""INSERT INTO subscriptions (source, target, label, company, board,
|
|
params, added_at, target_id)
|
|
VALUES (?,?,?,?,?,?,?,?)""",
|
|
(source, target, label, company, board, json.dumps(params or {}),
|
|
self.now(), target_id)).lastrowid
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"{source} already has {target!r} or the label {label!r}") from exc
|
|
assert new_id is not None
|
|
return self._subscription(new_id)
|
|
|
|
def update_subscription(self, sub_id: int, **fields: Any) -> Subscription:
|
|
"""Change labels, params, target_id or the three switches."""
|
|
unknown = set(fields) - set(_SUBSCRIPTION_FIELDS)
|
|
if unknown:
|
|
raise ValueError(f"not a subscription field: {', '.join(sorted(unknown))}")
|
|
if "params" in fields:
|
|
fields["params"] = json.dumps(fields["params"] or {})
|
|
for switch in ("subscribed", "fetch", "show"):
|
|
if switch in fields:
|
|
fields[switch] = int(bool(fields[switch]))
|
|
if fields:
|
|
sets = ", ".join(f"{f}=?" for f in fields)
|
|
try:
|
|
changed = self._write(f"UPDATE subscriptions SET {sets} WHERE id=?",
|
|
(*fields.values(), sub_id))
|
|
except sqlite3.IntegrityError as exc:
|
|
raise ValueError(f"label {fields.get('label')!r} is already in use") from exc
|
|
if not changed:
|
|
raise KeyError(sub_id)
|
|
return self._subscription(sub_id)
|
|
|
|
def unsubscribe(self, sub_id: int) -> Subscription:
|
|
"""Stop following it and hide it. The row stays, and so do its roles."""
|
|
return self.update_subscription(sub_id, subscribed=0, fetch=0, show=0)
|
|
|
|
def _subscription(self, sub_id: int) -> Subscription:
|
|
found = self.subscription(sub_id)
|
|
if found is None:
|
|
raise KeyError(sub_id)
|
|
return found
|
|
|
|
# --------------------------------------------------------- applications
|
|
|
|
def save_application(self, application: Application) -> Application:
|
|
"""Insert or update one application. An empty `id` is minted. Entering a
|
|
status, on insert or by change, appends an `application_events` row in
|
|
the same transaction."""
|
|
_check_status(application.status)
|
|
app_id = application.id or uuid4().hex
|
|
stamp = self.now()
|
|
fields = application.model_dump(exclude={"id", "created_at", "updated_at"})
|
|
with self.transaction() as conn:
|
|
before = conn.execute("SELECT status FROM applications WHERE id=?",
|
|
(app_id,)).fetchone()
|
|
if before is None:
|
|
columns = ["id", *fields, "created_at", "updated_at"]
|
|
conn.execute(
|
|
f"INSERT INTO applications ({', '.join(columns)}) "
|
|
f"VALUES ({', '.join('?' for _ in columns)})",
|
|
(app_id, *fields.values(), stamp, stamp))
|
|
else:
|
|
sets = ", ".join(f"{f}=?" for f in fields)
|
|
conn.execute(f"UPDATE applications SET {sets}, updated_at=? WHERE id=?",
|
|
(*fields.values(), stamp, app_id))
|
|
if before is None or before["status"] != application.status:
|
|
conn.execute("INSERT INTO application_events (application_id, status, at) "
|
|
"VALUES (?,?,?)", (app_id, application.status, stamp))
|
|
return self._application(app_id)
|
|
|
|
def set_application_status(self, app_id: str, status: str) -> Application:
|
|
"""Move one application to `status`. Raises ValueError for a status
|
|
outside `STATUSES` and KeyError for an unknown id."""
|
|
_check_status(status)
|
|
current = self._application(app_id)
|
|
return self.save_application(current.model_copy(update={"status": status}))
|
|
|
|
def application(self, app_id: str) -> Application | None:
|
|
row = self._row("SELECT * FROM applications WHERE id=?", (app_id,))
|
|
return Application.from_row(row) if row else None
|
|
|
|
def applications(self, status: str | None = None) -> list[Application]:
|
|
sql, params = "SELECT * FROM applications", []
|
|
if status:
|
|
sql += " WHERE status = ?"
|
|
params.append(status)
|
|
sql += " ORDER BY updated_at DESC, id"
|
|
return [Application.from_row(r) for r in self._rows(sql, params)]
|
|
|
|
def application_events(self, app_id: str) -> list[ApplicationEvent]:
|
|
return [ApplicationEvent.from_row(r) for r in self._rows(
|
|
"SELECT * FROM application_events WHERE application_id=? ORDER BY id",
|
|
(app_id,))]
|
|
|
|
def _application(self, app_id: str) -> Application:
|
|
found = self.application(app_id)
|
|
if found is None:
|
|
raise KeyError(app_id)
|
|
return found
|
|
|
|
# ---------------------------------------------------------------- state
|
|
|
|
@property
|
|
def state(self) -> MutableMapping[str, str]:
|
|
"""The `state` table as a mapping. What the runner hands a plugin as
|
|
`self.state`: it can keep a counter without being able to read roles."""
|
|
return _StateMap(self)
|
|
|
|
|
|
def _check_status(status: str) -> None:
|
|
if status not in STATUSES:
|
|
raise ValueError(f"{status!r} is not an application status "
|
|
f"({', '.join(STATUSES)})")
|
|
|
|
|
|
def _check_affinity(affinity: Any) -> None:
|
|
if not isinstance(affinity, int) or affinity not in AFFINITY:
|
|
raise ValueError(f"affinity is -1, 0, 1 or 2, not {affinity!r}")
|