"""`Store`: every SQL statement in the application. The connection plumbing is a copy of newsdesk's `Store`, on purpose: `desk` has no storage package. A connection is opened per call and closed again, so one caller's commit can never land inside another's transaction; in WAL mode that costs a file handle. """ from __future__ import annotations import json import sqlite3 from collections.abc import Callable, Iterable, Iterator, MutableMapping, Sequence from contextlib import closing, contextmanager from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any, Literal, NamedTuple from uuid import uuid4 import numpy as np from desk.models import Item from app import dupes, places from app.models import ( AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application, ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace, RoleRow, ScoredRole, Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key, host_word, posted_at, posted_host, remote_of, title_key, ) from app.storage import gazetteer, schema #: Text columns a plugin may fill from `Item.raw` under the same name. An #: empty value never overwrites a filled column, so an upsert from a board that #: says less than `extract` found keeps what `extract` found. `function` is #: not here: it is not a board field (see the note by `keep_filled` below). #: Nor is `seniority`, which is the title's level unless a board states one. _RAW_TEXT = ("place", "currency") #: The host's own words a reader hands on in `Item.raw`, stored as #: `_host` (`host_word`) for `app.prep` to map. `seniority_host` is #: rewritten every pull, as the host's level always was; the others are kept #: filled, since a role's own page (`enrich`) may state what its listing #: does not. _HOST_WORDS = ("seniority", "employment", "remote", "country") _KEPT_HOST_WORDS = ("employment_host", "remote_host", "country_host") #: What prep maps from that a pull rewrites whatever it brings. _PREP_OVERWRITTEN = ("title", "seniority_host") _RAW_INT = ("salary_min", "salary_max") #: What `extract` may fill. Only an empty column is written: a board's own #: structured field wins over what a model read out of the prose, and the next #: upsert from that board would put it back anyway. `function` and #: `seniority` are not here either, for the opposite reason — see #: `set_extracted`. _EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit") #: How many replies may leave a role unanswered before `extract` stops #: offering it. newsdesk's number, for newsdesk's reason: a row the model #: never answers is otherwise paid for on every pull for as long as it exists. GIVE_UP_AFTER = 3 #: A role `extract` has yet to read: still open, no copy of another #: (`rebuild_dupes`), not given up, and either never answered or without a #: model level. The second half makes `extract` the afterburner for the level: #: a role read before the level was asked for is read again, and only fills #: what is still empty (`set_extracted`). A reply that names no level leaves #: the role waiting and is charged a try (`extract.apply_batch`), so it ends. _EXTRACT_WAITING = (f"(extracted_at IS NULL OR seniority_model = '') AND closed_at IS NULL " f"AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}") #: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged. VOTES = (-1, 0, 1, None) #: The `judged` filter of `Store.roles`, as a `WHERE` on the agent's vote, #: joined as `v`. A neutral vote is judged, and is neither liked nor passed. JUDGED: dict[str, str] = { "liked": "v.vote = 1", "unjudged": "v.vote IS NULL", "passed": "v.vote = -1", } #: The feed's Pay filter, `_roles_from`'s `pay`: the SQL that is true for #: each choice, over `roles`. `salary_state` (`app.models.Role`) is derived, #: not stored, so the filter spells the same rule out in SQL rather than #: reading a column: 'normalised' is `salary_min` or `salary_max` set, #: 'stated' is `salary_text` set and neither of those is. PAY: dict[str, str] = { "stated": "(salary_min IS NOT NULL OR salary_max IS NOT NULL OR salary_text != '')", "normalised": "(salary_min IS NOT NULL OR salary_max IS NOT NULL)", "stated_only": "(salary_text != '' AND salary_min IS NULL AND salary_max IS NULL)", } #: A role that states no pay at all, what `pay_unknown` lets through the Pay #: filter, and the `""` of its facet. _NO_PAY = "(salary_min IS NULL AND salary_max IS NULL AND salary_text = '')" #: `salary_state` in SQL, what the Pay facet counts by: 'normalised', #: 'stated' (a line and no figure) or ''. _PAY_STATE = ("CASE WHEN salary_min IS NOT NULL OR salary_max IS NOT NULL THEN 'normalised' " "WHEN salary_text != '' THEN 'stated' ELSE '' END") #: A ranking's score as `Store.roles` serves and orders it: halved by a pass, #: lifted a fifth by a like, capped at 1. Applied at read time, so a vote #: moves the role at once and the stored score stays what `score` computed. _VOTED_SCORE = ("MIN(1.0, s.score * CASE v.vote WHEN -1 THEN 0.5 " "WHEN 1 THEN 1.2 ELSE 1 END)") #: The agent's votes beside every role. With no agent the join matches #: nothing, so `vote` and `voted_at` are NULL: one query shape either way. _WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?" #: What `roles_page` reads for `RoleRow.excerpt`: the summary when there is #: one, else the description's opening. A `SUBSTR` of the source column, not #: the column itself, so a big `description` never leaves SQLite whole for a #: row that only ever shows `EXCERPT_LIMIT` characters of it. _EXCERPT_SOURCE = ("COALESCE(NULLIF(roles.summary, ''), NULLIF(roles.description, ''), " "roles.teaser)") #: How many raw characters `roles_page` pulls before `app.models.excerpt` #: cuts it to size: slack over `EXCERPT_LIMIT` so the word-boundary trim #: still has somewhere to land. _EXCERPT_RAW = 240 _COMPANY_FIELDS = ("name", "domain", "status", "careers_url", "notes", "ats", "country", "city", "ats_slug", "watch", "careers_opened_at", "hand_reason", "sector", "stage", "size_band", "founded", "last_funding", "last_funding_currency", "facts_source", "facts_at", "all_roles") #: A company as the store serves it: the row and the asked-for agent's affinity. _COMPANY_SELECT = """SELECT c.*, COALESCE(ac.affinity, 0) AS affinity FROM companies c LEFT JOIN agent_companies ac ON ac.company_key = c.key AND ac.agent_id = ?""" _SUBSCRIPTION_FIELDS = ("label", "company", "board", "params", "subscribed", "fetch", "show", "target_id") #: A tag's longest stored form, and how many a role may carry. Long enough for #: "senior backend engineer" and short enough that the filter row stays a row. _MAX_TAG_LEN = 40 _MAX_TAGS = 12 def _normalise_tags(tags: Sequence[str]) -> list[str]: """Each tag stripped, lowercased, its internal whitespace collapsed to one space and cut to `_MAX_TAG_LEN`; empties and repeats dropped, order kept, capped at `_MAX_TAGS`. What `set_tags` and `set_summary` store, so the filter row never has to trust what a model or a person typed.""" seen: set[str] = set() out: list[str] = [] for tag in tags: cleaned = " ".join(str(tag).split()).lower()[:_MAX_TAG_LEN] if not cleaned or cleaned in seen: continue seen.add(cleaned) out.append(cleaned) if len(out) >= _MAX_TAGS: break return out def _iso(moment: datetime) -> str: """UTC, to the second. Every timestamp column is compared as text, so every writer has to spell it the same way.""" return moment.astimezone(UTC).isoformat(timespec="seconds") def _own_texts(conn: sqlite3.Connection, uid: str) -> list[str]: """One role's own place texts as stored, in the host's order.""" return list(dict.fromkeys(r[0] for r in conn.execute( "SELECT text FROM role_places WHERE uid = ? AND url = '' ORDER BY position, rowid", (uid,)))) def _like(text: str) -> str: """A `LIKE` pattern for a case-insensitive substring match, with SQLite's own wildcards escaped so a search for "50%" or "a_b" stays literal.""" escaped = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") return f"%{escaped}%" #: Vectors are stored little-endian float32, fixed here so a file written on #: one machine reads on another. newsdesk's encoding, written again. _DTYPE = np.dtype(" bytes: return np.asarray(vec, dtype=_DTYPE).tobytes() def unpack(blob: bytes) -> np.ndarray: return np.frombuffer(blob, dtype=_DTYPE) #: A role with no place the gazetteer resolved, over `roles r`. _UNPLACED = "r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL)" def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]: """One gate as `(empty, match, params)`: SQL that is true when the role's column is empty, and SQL that is true when it matches, over `roles r`. `params` belong to `match`; `empty` takes none. `country` and `place` match over the gazetteer, as the feed's Place filter does (`_places_sql`): a country gate is met by a role whose `country` is one of its codes or that has a place in one of them, so `DK` is met by `Copenhagen; London`. Its column is empty only when the role has neither. A place gate's value is a place token, bare text a `place:`.""" column = f"r.{gate.field}" values = [str(v).strip().lower() for v in gate.values if str(v).strip()] marks = ", ".join("?" for _ in values) or "NULL" if gate.field == "salary_min": floor = min(_int_or_none(v) or 0 for v in gate.values) return f"{column} IS NULL", f"{column} >= ?", [floor] if gate.field == "country": match, params = _places_sql([f"country:{v}" for v in values], gaz, table="r", codes=[v.upper() for v in values]) return f"(r.country = '' AND {_UNPLACED})", match, params if gate.field == "place": match, params = _places_sql(values, gaz, table="r") return "r.place = ''", match, params if gate.field == "company": keys = [company_key(v) for v in values] return ("r.company_key = ''", f"r.company_key IN ({', '.join('?' for _ in keys) or 'NULL'})", keys) if gate.field == "languages": return ("r.languages IN ('', '[]')", f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) " f"WHERE LOWER(value) NOT IN ({marks}))", values) return f"{column} = ''", f"LOWER({column}) IN ({marks})", values def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles", codes: Sequence[str] = ()) -> tuple[str, list[Any]]: """Any of the place tokens over `table`, as a join over the gazetteer: a role with a place under one of the places the tokens name (`country:DK` every place in Denmark, `region:skane` every town in Skåne, `place:malmo` Malmö by any spelling), or whose `country` is one of their countries (a role whose board gave a country and no place). `codes` are more values of `country` to accept. A `place:` token the gazetteer does not know is matched on whole words of the role's folded line (`places.match_sql`), so a town nobody has entered yet is still found.""" ids: set[int] = set() found = {c for c in codes if c} clauses: list[str] = [] params: list[Any] = [] for token in tokens: named = gaz.lookup(token) if named is None: sql, values = places.match_sql(token, place=f"{table}.place_fold", country=f"{table}.country") clauses.append(sql) params += values continue under = gaz.under(named) ids |= under found |= gaz.codes(under) if ids: clauses.insert(0, f"{table}.uid IN (SELECT uid FROM role_places " "WHERE place_id IN (SELECT value FROM json_each(?)))") params.insert(0, json.dumps(sorted(ids))) if found: clauses.insert(1 if ids else 0, f"{table}.country IN (SELECT value FROM json_each(?))") params.insert(1 if ids else 0, json.dumps(sorted(found))) return f"({' OR '.join(clauses) or '0'})", params def _known_places(tokens: Sequence[str]) -> None: """ValueError for a token that names nothing, which would otherwise match no role without saying why.""" for token in tokens: if not places.known(token): raise ValueError(f"{token!r} is not a place this app knows") def _text_rank(description: str, teaser: str) -> int: """What a role is embedded from, as a rank: 2 its description, 1 its teaser, 0 its title alone.""" return 2 if description else 1 if teaser else 0 def _int_or_none(value: Any) -> int | None: if value is None or value == "" or isinstance(value, bool): return None try: return int(float(value)) except (TypeError, ValueError): return None #: The host `app.dupes.Copy.host` and `RoleCopy.host` read: the provider of #: a search's row, else the ATS a board runs on, as `posted_host` reads it. _HOST = "COALESCE(json_extract(raw, '$.provider'), json_extract(raw, '$.board'), '')" #: A copy that is not one place of its kept role: the copies a role lists #: under "Also posted" and counts as "x N". `{kept}` is the kept role's uid #: as SQL, and a place copy is the one whose link the kept role holds. _NOT_A_PLACE = ("original_url NOT IN (SELECT url FROM role_places " "WHERE uid = {kept} AND url != '')") class DupeCounts(NamedTuple): """What `rebuild_dupes` left and did: kept roles with at least one copy, open roles that are a copy, and rows whose `dup_of` it rewrote.""" groups: int copies: int changed: int def _assign_only(rows: Iterable[dupes.Copy]) -> dupes.Folded: return dupes.Folded(dupes.assign(rows), frozenset()) def _sync_copy_places(conn: sqlite3.Connection, placed: dict[str, str | None]) -> None: """Make the `role_places` rows with a `url` those of the copies `placed` (`copy uid -> kept uid`): each own place of a copy, on the kept role, with the copy's link. Written only when they differ from what is stored, so an unchanged database is not touched. A copy that closed, or a group that dissolved, is no longer in `placed`, and its rows go.""" want: set[tuple[str, int, str, str, int | None, str]] = set() if placed: marks = ",".join("?" * len(placed)) for uid, pos, text, fold, place_id, url in conn.execute( f"""SELECT rp.uid, rp.position, rp.text, rp.fold, rp.place_id, r.original_url FROM role_places rp JOIN roles r ON r.uid = rp.uid WHERE rp.url = '' AND rp.uid IN ({marks})""", list(placed)): want.add((str(placed[uid]), pos, text, fold, place_id, url)) have = set(conn.execute( "SELECT uid, position, text, fold, place_id, url FROM role_places WHERE url != ''")) if want != have: conn.execute("DELETE FROM role_places WHERE url != ''") conn.executemany( "INSERT INTO role_places (uid, position, text, fold, place_id, url) " "VALUES (?, ?, ?, ?, ?, ?)", sorted(want, key=lambda r: (r[0], r[5], r[1], r[2]))) def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts: """Recompute `roles.dup_of` from scratch with `app.dupes.fold`, inside the caller's transaction: `Store.rebuild_dupes` and migration 18. Only the open roles of a `(company_key, title_key)` with more than one open role can be a copy, a few thousand of the ~34k, so only those are read. A row whose `dup_of` differs from the rule's answer is rewritten, and a `dup_of` left on a closed role, or on one no longer in such a pair, is cleared: a copy never outlives its posting, and a closed kept role hands its group to the next copy here. Where a group changed (a member's `dup_of` was rewritten), its kept role takes over what was done to the posting under another uid. The sources are the group's members and the roles they were copies of before, so a closed kept role still hands over. The kept role takes the earliest `read_at` of them when it has none, and each agent's latest vote among them moves to it unless that agent already voted on it; a vote that cannot move stays where it is. An unchanged group is not touched, so marking a kept role unread again sticks. Applications keep their `role_uid`, and scores of a copy go at the agent's next `score`. A copy the host published for another place (`dupes.Folded.placed`) leaves its places on the kept role as `role_places` rows with the copy's link, see `_sync_copy_places`.""" # Migration 18 runs this on a file that has no `teaser` and no # `role_places` yet: there it groups as before and folds no places. current = any(r[1] == "teaser" for r in conn.execute("PRAGMA table_info(roles)")) text = "COALESCE(NULLIF(description, ''), teaser)" if current else "description" rows = conn.execute( f"""SELECT uid, company_key, title_key, country, place, {_HOST} AS host, description != '' AS described, posted_at, SUBSTR({text}, 1, {dupes.TEXT_CHARS}), original_url FROM roles WHERE closed_at IS NULL AND company_key != '' AND (company_key, title_key) IN ( SELECT company_key, title_key FROM roles WHERE closed_at IS NULL AND company_key != '' GROUP BY company_key, title_key HAVING COUNT(*) > 1)""").fetchall() made = dupes.fold if current else _assign_only folded = made( dupes.Copy(uid=r[0], company_key=r[1], title_key=r[2], country=r[3], place=r[4], host=r[5], has_description=bool(r[6]), posted_at=r[7] or None, text=r[8] or "", url=r[9] or "") for r in rows) new = folded.dup_of old: dict[str, str | None] = dict.fromkeys(new) old.update((r[0], r[1]) for r in conn.execute( "SELECT uid, dup_of FROM roles WHERE dup_of IS NOT NULL")) updates = [(new.get(uid), uid) for uid, was in old.items() if new.get(uid) != was] conn.executemany("UPDATE roles SET dup_of = ? WHERE uid = ?", updates) if current: _sync_copy_places(conn, {uid: new[uid] for uid in folded.placed}) changed = {uid for _, uid in updates} members: dict[str, set[str]] = {} for uid, dup in new.items(): members.setdefault(dup or uid, set()).add(uid) groups = {kept: group for kept, group in members.items() if len(group) > 1} for kept, group in groups.items(): if not group & changed: continue sources = (group | {old.get(uid) or uid for uid in group}) - {kept} _carry(conn, kept, sources) return DupeCounts(groups=len(groups), copies=sum(dup is not None for dup in new.values()), changed=len(updates)) def _carry(conn: sqlite3.Connection, kept: str, sources: set[str]) -> None: """`rebuild_dupes`' hand-over to a group's kept role: the earliest `read_at` of `sources` if it has none, and each agent's latest vote among them unless it has one of its own.""" wanted = json.dumps(sorted(sources)) conn.execute( """UPDATE roles SET read_at = (SELECT MIN(read_at) FROM roles WHERE uid IN (SELECT value FROM json_each(?))) WHERE uid = ? AND read_at IS NULL""", (wanted, kept)) voted = {r[0] for r in conn.execute("SELECT agent_id FROM role_votes WHERE uid = ?", (kept,))} moves: dict[int, str] = {} for agent_id, uid in conn.execute( """SELECT agent_id, uid FROM role_votes WHERE uid IN (SELECT value FROM json_each(?)) ORDER BY voted_at, uid""", (wanted,)): if agent_id not in voted: moves[agent_id] = uid # the latest vote wins, by the order above conn.executemany("UPDATE role_votes SET uid = ? WHERE agent_id = ? AND uid = ?", [(kept, agent_id, uid) for agent_id, uid in moves.items()]) class _StateMap(MutableMapping[str, str]): """`Store.state`: the key/value table, read and written one key at a time.""" def __init__(self, store: Store) -> None: self._store = store def __getitem__(self, key: str) -> str: row = self._store._row("SELECT value FROM state WHERE key=?", (key,)) if row is None: raise KeyError(key) return str(row["value"]) def __setitem__(self, key: str, value: str) -> None: self._store._write( """INSERT INTO state (key, value) VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value""", (key, value)) def __delitem__(self, key: str) -> None: if not self._store._write("DELETE FROM state WHERE key=?", (key,)): raise KeyError(key) def __iter__(self) -> Iterator[str]: return iter([r["key"] for r in self._store._rows("SELECT key FROM state ORDER BY key")]) def __len__(self) -> int: row = self._store._row("SELECT COUNT(*) AS n FROM state") return int(row["n"]) if row else 0 class Store: def __init__(self, path: str | Path) -> None: self.path = Path(path).expanduser() self.path.parent.mkdir(parents=True, exist_ok=True) #: What "now" is. Replaced by tests that need two writes a second apart. self.clock: Callable[[], datetime] = lambda: datetime.now(UTC) #: The gazetteer as last read, with the `gazetteer.VERSION` it was read at. self._gaz: tuple[tuple[Any, ...], places.Gazetteer] | None = None with self.transaction() as conn: # Persistent: recorded in the file, inherited by every connection. conn.execute("PRAGMA journal_mode=WAL") #: Migration versions this open applied; empty when already current. self.migrations_applied: list[int] = schema.initialise(conn) # ------------------------------------------------------------ connections def _connect(self) -> sqlite3.Connection: """A fresh connection. `foreign_keys` is off by default per connection; `busy_timeout` matches `pipeline/table.py`, which writes the same file; `synchronous=NORMAL` is the standard WAL companion. `place_match` is registered on every one, since the place filter and the place gate call it for a town the gazetteer does not know (`_places_sql`).""" conn = sqlite3.connect(self.path) conn.row_factory = sqlite3.Row places.register(conn) conn.execute("PRAGMA foreign_keys=ON") conn.execute("PRAGMA busy_timeout=30000") conn.execute("PRAGMA synchronous=NORMAL") return conn @contextmanager def transaction(self) -> Iterator[sqlite3.Connection]: """One connection, one transaction, closed on the way out.""" conn = self._connect() try: with conn: yield conn finally: conn.close() def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]: with closing(self._connect()) as conn: return conn.execute(sql, params).fetchall() def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None: with closing(self._connect()) as conn: row: sqlite3.Row | None = conn.execute(sql, params).fetchone() return row def _write(self, sql: str, params: Any = ()) -> int: """One statement in its own transaction. Returns the rowcount.""" with self.transaction() as conn: return conn.execute(sql, params).rowcount def now(self) -> str: return _iso(self.clock()) # ------------------------------------------------------------- gazetteer def _gazetteer(self, conn: sqlite3.Connection) -> places.Gazetteer: """The gazetteer, read again only when its tables have grown, which another process may have done: one cheap query a call.""" version = tuple(conn.execute(gazetteer.VERSION).fetchone()) if self._gaz is None or self._gaz[0] != version: self._gaz = (version, gazetteer.load(conn)) return self._gaz[1] def gazetteer(self) -> places.Gazetteer: with closing(self._connect()) as conn: return self._gazetteer(conn) def role_places(self, uid: str) -> list[RolePlace]: """One role's places in the host's order, a copy's after the role's own.""" return [RolePlace.from_row(r) for r in self._rows( """SELECT rp.position, rp.text, rp.place_id, rp.url, COALESCE(p.name, '') AS name, COALESCE(p.kind, '') AS kind FROM role_places rp LEFT JOIN places p ON p.id = rp.place_id WHERE rp.uid = ? ORDER BY rp.url != '', rp.url, rp.position, rp.rowid""", (uid,))] def unresolved_places(self, open_only: bool = True, limit: int | None = None) -> list[tuple[str, str, int]]: """`(fold, text, roles)` for every place text no place of the gazetteer resolves, most roles first: the list a person works through with `add_place_alias` and `add_place`. `text` is one spelling of the fold. A text that names no place ("Remote", `places.placeless`) is left out; with `open_only`, so are closed roles and copies.""" where = "rp.place_id IS NULL" if open_only: where += " AND r.closed_at IS NULL AND r.dup_of IS NULL" rows = self._rows( f"""SELECT rp.fold, MIN(rp.text) AS text, COUNT(DISTINCT rp.uid) AS n FROM role_places rp JOIN roles r ON r.uid = rp.uid WHERE {where} GROUP BY rp.fold ORDER BY n DESC, rp.fold""") found = [(str(r["fold"]), str(r["text"]), int(r["n"])) for r in rows if not places.placeless(r["fold"])] return found[:limit] if limit is not None else found def _place_id(self, gaz: places.Gazetteer, place: int | str) -> int: """A place by id, or by a token that names exactly one place.""" if isinstance(place, int): if place not in gaz.places: raise ValueError(f"there is no place {place}") return place named = gaz.lookup(place) or frozenset() if len(named) != 1: raise ValueError(f"{place!r} names {len(named)} places, not one") return next(iter(named)) def add_place_alias(self, text: str, place: int | str) -> int: """Write `text` as a spelling of `place` (an id, or a token such as `place:copenhagen` or `country:DK` naming one place), and resolve again every role with a place text that holds it. Returns how many roles now resolve differently.""" folded = places.fold(text) if not folded: raise ValueError("an alias needs a letter or a digit") with self.transaction() as conn: place_id = self._place_id(self._gazetteer(conn), place) conn.execute("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)", (folded, place_id)) return gazetteer.relink(conn, self._gazetteer(conn), folded) def add_place(self, name: str, kind: str, parent: int | str, aliases: Sequence[str] = ()) -> int: """Add a city or a region the gazetteer lacks, in `parent` (an id or a token), written as `name` and each of `aliases`, and resolve again the roles with a place text that holds one of them. Returns its id.""" if kind not in ("city", "region"): raise ValueError(f"a place added by hand is a city or a region, not {kind!r}") folds = list(dict.fromkeys(f for f in map(places.fold, (name, *aliases)) if f)) if not folds: raise ValueError("a place needs a name") with self.transaction() as conn: parent_id = self._place_id(self._gazetteer(conn), parent) if conn.execute("SELECT 1 FROM places WHERE kind = ? AND key = ?", (kind, folds[0])).fetchone(): raise ValueError(f"there is already a {kind} {name!r}") place_id = int(conn.execute( "INSERT INTO places (kind, key, name, parent_id) VALUES (?, ?, ?, ?)", (kind, folds[0], name.strip(), parent_id)).lastrowid or 0) conn.executemany("INSERT OR IGNORE INTO place_aliases (fold, place_id) VALUES (?, ?)", [(f, place_id) for f in folds]) gaz = self._gazetteer(conn) for folded in folds: gazetteer.relink(conn, gaz, folded) return place_id def relink_places(self) -> int: """Resolve every role's own places again, after the gazetteer was seeded anew. Returns how many roles now resolve differently.""" with self.transaction() as conn: return gazetteer.relink(conn, self._gazetteer(conn)) # ---------------------------------------------------------------- roles def upsert_role(self, item: Item, subscription: Subscription | None = None) -> bool: """Store one posting. Returns True when it was new. `first_seen` and `created_at` are written on insert only; `last_seen` on every call, and a role seen again is open again. The raw columns come from `item.raw` under their own names, with `company` falling back to the subscription's; an empty value never overwrites a filled column. Nothing is mapped here: that is `app.prep`, which runs after every pull. The host's words go to `seniority_host`, `employment_host`, `remote_host` and `country_host` (`host_word`), `company_key` is written unresolved on a new role only, and the mapped columns (`seniority_stated`, `seniority_title`, `seniority`, `employment`, `remote`, `country`) are left as prep last wrote them. `prep_due` is 1 on a new role and grows by one when the title, the company, a host word or the place texts changed or the role reopens, which is what the next prep reads. `posted_at` (see `app.models.posted_at`) is set from this pull's own `published_at` and `stamp`; on an update the `ON CONFLICT` clause below keeps the earlier of that and what is already stored, so an edit on the board — which can only ever move `published_at` later, never earlier — cannot move `posted_at` later either. `function` is the one gate column no board actually sends: the `ats` plugin's own keyword guess rides in `raw` under that name because `raw` is the only channel a plugin has, but a guess is not first-party the way a board's own structured field is (docs/decisions.md, "a guess is not a board field"). So once a role has been through `extract` (`extracted_at` is set), this upsert leaves `function` alone for good, whatever the plugin's raw guess now says — see `set_extracted` for the other half of the rule. The role's places (`role_places`) are the texts of `raw["places"]`, else `raw["place"]` cut at `;` (`gazetteer.texts`), written unresolved when they differ from the stored ones; prep resolves them. Texts that did not change keep their resolved rows, so the feed's Place filter stays right between a pull and its prep. `place` is the line shown, the texts joined when `raw` has none. Like `place`, a pull with no place leaves them as they are. `place_fold` is written with `place`, under the same keep-filled rule. `teaser` is kept-filled too. A role is embedded from its description, else its teaser, else its title (`app.jobs.embed.document`). When a pull brings a better one of these than the role had, its vector is dropped so the next `embed` reads it. """ if not item.source: raise ValueError(f"item {item.source_id!r} has no source; the runner sets it") raw = item.raw or {} company = str(raw.get("company") or (subscription.company if subscription else "")) place_texts = gazetteer.texts(raw) place = str(raw.get("place") or "") or "; ".join(place_texts) stamp = self.now() published = _iso(item.published_at) row: dict[str, Any] = { "uid": item.uid, "source": item.source, "source_id": item.source_id, "subscription_id": subscription.id if subscription else None, "company_key": company_key(company), "company": company, "title": item.title, "title_key": title_key(item.title), "place_key": place_key(place), "description": item.body, "feed_text": item.excerpt, "original_url": item.location, "published_at": published, "posted_at": posted_at(published, stamp, posted_host(raw)), "first_seen": stamp, "last_seen": stamp, "raw": json.dumps(raw, default=str), "created_at": stamp, } row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT} row |= {f"{name}_host": host_word(raw.get(name)) for name in _HOST_WORDS} row["place"] = place row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT} row["function"] = str(raw.get("function") or "") row["teaser"] = str(raw.get("teaser") or "") row["place_fold"] = places.fold(row["place"]) # A board's own pay line, one per pull like title or published_at — # not kept-filled like _RAW_TEXT: a board that stops stating pay # must be able to clear it, not leave a stale line behind. row["salary_text"] = str(raw.get("salary_text") or "") row["prep_due"] = 1 keep_filled = ("company", "place_key", "description", "feed_text", "original_url", "teaser", *_RAW_TEXT, *_KEPT_HOST_WORDS) updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled] updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})" for c in (*_RAW_INT, "subscription_id")] updates += [f"{c}=excluded.{c}" for c in ("title", "title_key", "published_at", "raw", "last_seen", "salary_text", "seniority_host")] updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)") # Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh # pull's classifier guess overwrite what extract decided, which is # the flip the "board wins" rule exists to prevent for an actual # board field — see the docstring above. Once extracted_at is set, # function is extract's to own; before that, it is the guess, same # as any other raw field. updates.append("function=CASE WHEN roles.extracted_at IS NULL " "THEN COALESCE(NULLIF(excluded.function, ''), roles.function) " "ELSE roles.function END") updates.append("place_fold=CASE WHEN excluded.place != '' " "THEN excluded.place_fold ELSE roles.place_fold END") # One more than before when what prep maps from changed, or the role # reopens, so a prep that read the old values leaves it due # (`Store.prepped`). `roles.` is the stored row here, `excluded.` # what this pull brought. changed = [f"roles.{c} IS NOT excluded.{c}" for c in _PREP_OVERWRITTEN] changed += [f"(excluded.{c} != '' AND roles.{c} IS NOT excluded.{c})" for c in ("company", *_KEPT_HOST_WORDS)] updates.append(f"prep_due=CASE WHEN roles.closed_at IS NOT NULL OR {' OR '.join(changed)} " "THEN roles.prep_due + 1 ELSE roles.prep_due END") updates.append("closed_at=NULL") columns = ", ".join(row) marks = ", ".join(f":{c}" for c in row) with self.transaction() as conn: before = conn.execute("SELECT description, teaser FROM roles WHERE uid=?", (row["uid"],)).fetchone() conn.execute( f"INSERT INTO roles ({columns}) VALUES ({marks}) " f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row) if place_texts and (before is None or _own_texts(conn, row["uid"]) != place_texts): gazetteer.write_texts(conn, row["uid"], place_texts) if before is not None: conn.execute("UPDATE roles SET prep_due = prep_due + 1 WHERE uid = ?", (row["uid"],)) # `embed` encodes a role once, from the best text it has. When a # later pull brings a better one, as a host that states the text # on the role's own page does, the vector goes, so the next # `embed` reads it. if before is not None and _text_rank(row["description"] or before["description"], row["teaser"] or before["teaser"]) \ > _text_rank(before["description"], before["teaser"]): conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],)) return before is None def role(self, uid: str, agent_id: int | None = None) -> ScoredRole | None: """One role with its tags, and with `vote`, `voted_at`, `score`, `gated_reason` and `prompt` as `agent_id` judged and scored it; None (unjudged, unscored) without an agent. The score is moved by the vote as `roles` serves it.""" row = self._row( f"SELECT roles.*, v.vote, v.voted_at, {_VOTED_SCORE} AS score, " "COALESCE(s.gated_reason, '') AS gated_reason, COALESCE(s.prompt, '') AS prompt " f"FROM roles {_WITH_VOTES} " "LEFT JOIN scores s ON s.uid = roles.uid AND s.agent_id = ? WHERE roles.uid=?", (agent_id, agent_id, uid)) if row is None: return None role = ScoredRole.from_row(row) role.tags = self._tags_of(uid) role.copies = self._copies_of(uid) role.places = self.role_places(uid) return role def _copies_of(self, uid: str) -> list[RoleCopy]: """The open copies of one kept role, oldest first; empty for a copy or a role without any. See `Role.copies`.""" return [RoleCopy.from_row(r) for r in self._rows( f"""SELECT uid, {_HOST} AS host, place, original_url, posted_at FROM roles WHERE dup_of = ? AND closed_at IS NULL AND {_NOT_A_PLACE.format(kept="?")} ORDER BY posted_at, uid""", (uid, uid))] def _copy_counts(self, uids: Sequence[str]) -> dict[str, int]: """`uid -> open copies` for a page of kept roles, one grouped query over `idx_roles_dup_of`. A role without copies is absent. The index is named because the planner otherwise takes `idx_roles_closed` for `closed_at IS NULL` and reads every open role, 19 ms against 0.1 on the live database.""" if not uids: return {} return {str(r["dup_of"]): int(r["n"]) for r in self._rows( "SELECT dup_of, COUNT(*) AS n FROM roles INDEXED BY idx_roles_dup_of " "WHERE dup_of IN (SELECT value FROM json_each(?)) AND closed_at IS NULL " f"AND {_NOT_A_PLACE.format(kept='dup_of')} " "GROUP BY dup_of", (json.dumps(list(uids)),))} def _place_lines(self, uids: Sequence[str]) -> dict[str, str]: """`uid -> the line the feed shows` for a page of roles that have more than one place (`dupes.place_line`); a role with one place or none is absent and keeps `roles.place`.""" if not uids: return {} found: dict[str, list[str]] = {} for r in self._rows( """SELECT uid, text FROM role_places WHERE uid IN (SELECT value FROM json_each(?)) ORDER BY uid, url != '', url, position, rowid""", (json.dumps(list(uids)),)): found.setdefault(str(r["uid"]), []).append(str(r["text"])) return {uid: dupes.place_line(texts) for uid, texts in found.items() if len(set(texts)) > 1} def rebuild_dupes(self) -> DupeCounts: """Recompute which open roles are copies of one posting, in one transaction; see the module's `rebuild_dupes`. `BEGIN IMMEDIATE` because it reads before it writes: a deferred transaction that a pull's upsert wrote under in between could not take the write lock and would fail rather than wait.""" with self.transaction() as conn: conn.execute("BEGIN IMMEDIATE") return rebuild_dupes(conn) def set_read(self, uid: str, read: bool = True) -> Role: """Mark a role read, or unread again. Stamps `read_at` only the first time — a role already read keeps the moment it was first opened — and clears it outright when `read` is False. Raises KeyError for an unknown uid.""" if read: changed = self._write( "UPDATE roles SET read_at = COALESCE(read_at, ?) WHERE uid = ?", (self.now(), uid)) else: changed = self._write("UPDATE roles SET read_at = NULL WHERE uid = ?", (uid,)) if not changed: raise KeyError(uid) role = self.role(uid) assert role is not None return role def set_vote(self, agent_id: int, uid: str, value: int | None) -> ScoredRole: """One agent's judgement of one role: 1 like, 0 neither way, -1 pass, None unjudged again, which deletes the row. A vote never hides a role; it moves it in that agent's ranking (see `roles`) and shapes its taste direction (see `jobs/score.py`). Voting is reading, so a vote stamps the role's `read_at` too, the same once-only way `set_read` does; clearing a vote is not un-reading, and leaves it alone. Returns the role with this agent's vote. Raises ValueError for another value and KeyError for an unknown uid or agent.""" if isinstance(value, bool) or value not in VOTES: raise ValueError(f"a vote is -1, 0 or 1, not {value!r}") with self.transaction() as conn: self._known(conn, "roles", "uid", uid) self._known(conn, "agents", "id", agent_id) if value is None: conn.execute("DELETE FROM role_votes WHERE agent_id=? AND uid=?", (agent_id, uid)) else: now = self.now() conn.execute( """INSERT INTO role_votes (agent_id, uid, vote, voted_at) VALUES (?,?,?,?) ON CONFLICT(agent_id, uid) DO UPDATE SET vote=excluded.vote, voted_at=excluded.voted_at""", (agent_id, uid, value, now)) conn.execute("UPDATE roles SET read_at = COALESCE(read_at, ?) WHERE uid = ?", (now, uid)) role = self.role(uid, agent_id) assert role is not None return role def votes(self, agent_id: int) -> dict[str, int]: """`uid -> vote` for one agent, open roles and closed alike: a closed role somebody liked still says what they like.""" return {r["uid"]: int(r["vote"]) for r in self._rows( "SELECT uid, vote FROM role_votes WHERE agent_id=?", (agent_id,))} @staticmethod def _known(conn: sqlite3.Connection, table: str, column: str, value: Any) -> None: """KeyError unless the row exists: the caller's 404, rather than the foreign key's IntegrityError, which does not say which side is unknown.""" if conn.execute(f"SELECT 1 FROM {table} WHERE {column}=?", (value,)).fetchone() is None: raise KeyError(value) def set_tags(self, uid: str, tags: Sequence[str]) -> list[str]: """Replace one role's tags in one transaction. See `_normalise_tags` for how each one is cleaned up. Returns what was stored, sorted the way `role` and `roles` read it back. Raises KeyError for an unknown uid.""" with self.transaction() as conn: if conn.execute("SELECT 1 FROM roles WHERE uid=?", (uid,)).fetchone() is None: raise KeyError(uid) return self._replace_tags(conn, uid, tags) def _replace_tags(self, conn: sqlite3.Connection, uid: str, tags: Sequence[str]) -> list[str]: """`set_tags`'s write, against an already-open transaction so `set_summary` can share it.""" cleaned = _normalise_tags(tags) conn.execute("DELETE FROM role_tags WHERE uid=?", (uid,)) conn.executemany("INSERT INTO role_tags (uid, tag) VALUES (?,?)", [(uid, tag) for tag in cleaned]) return sorted(cleaned) def _tags_of(self, uid: str) -> list[str]: return [str(r["tag"]) for r in self._rows( "SELECT tag FROM role_tags WHERE uid=? ORDER BY tag", (uid,))] def _tags_by_uid(self, uids: Sequence[str]) -> dict[str, list[str]]: """Every tag of these roles, by uid, sorted — one query for a whole page rather than one per role.""" if not uids: return {} out: dict[str, list[str]] = {} rows = self._rows( "SELECT uid, tag FROM role_tags WHERE uid IN (SELECT value FROM json_each(?)) " "ORDER BY uid, tag", (json.dumps(list(uids)),)) for r in rows: out.setdefault(r["uid"], []).append(str(r["tag"])) return out def roles(self, limit: int = 100, company: str | None = None, place: str | None = None, open_only: bool = True, q: str | None = None, function: str | None = None, remote: str | None = None, days: int | None = None, hide_muted: bool = False, include_read: bool = True, tags: Sequence[str] = (), tag_mode: str = "any", agent: int | None = None, by_score: bool = True, gated: bool = False, judged: str | None = None, pay: str | None = None, company_keys: Sequence[str] = (), places: Sequence[str] = (), seniority: Sequence[str] = (), seniority_unknown: bool = True, pay_unknown: bool = False, offset: int = 0) -> list[ScoredRole]: """Newest first by `posted_at`. `company` and `place` are case-insensitive substrings of the columns a board wrote — not the key, so a search does not have to know the exact slug a company folds to. `q` is matched word by word against `title` and `company`; a role must match every word in one or the other. `function` and `remote` match a gate column exactly. `days` keeps only roles posted within that many days of now. `pay` (a key of `PAY`) filters on `salary_state`; with `pay_unknown` a role that states no pay passes it too. `company_keys` keeps those exact companies. `places` keeps a role any of the place tokens finds (`app.places`: `country:DK`, `region:skane`, `place:malmo`), over the gazetteer (`_places_sql`); a token that names nothing is a ValueError. `seniority` keeps the roles at any of those levels (`LEVELS`) and, with `seniority_unknown` (the default), those with no level, which is most of them. An empty list is no filter. `agent` says whose votes, hidden companies and scores are meant. Each row carries that agent's `vote` and `voted_at`; `judged` (a key of `JUDGED`: liked, unjudged or passed) filters on them. `hide_muted` excludes the companies that agent hid, before `limit` is applied, in SQL rather than after the fact: a company that has published a lot, all at once — a fresh board added, a re-pull that finally reached it — can otherwise fill the whole page with roles that then all turn out to be from the one company someone hid, leaving nothing to show for a default query that would otherwise have plenty. With no agent every vote is None, nothing is hidden and `judged` reads as such. `include_read` set False excludes a role `set_read` or `set_vote` has stamped, the same way and for the same reason as `hide_muted`: before `limit`, not after. `tags` keeps only roles carrying at least one of them (`tag_mode="any"`, the default) or every one (`tag_mode="all"`), also before `limit`. With `agent` and `by_score`, ordered by that agent's score instead, highest first, carrying `score` and `gated_reason` on each row. Only roles it has scored are listed, and a gated one only when `gated` is set. The score is the stored one moved by the agent's vote (`_VOTED_SCORE`), so a vote shows in the order at once and needs no rescore. With `by_score` off the order is newest, with the votes and no score. `offset` skips that many rows of the same order, for the page that loads the next 60 as it is scrolled; `count_roles` is the total. Every returned role carries its tags, read with one extra query for the whole page rather than one per role.""" sql, params = self._roles_from( company=company, place=place, open_only=open_only, q=q, function=function, remote=remote, days=days, hide_muted=hide_muted, include_read=include_read, tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated, judged=judged, pay=pay, company_keys=company_keys, places=places, seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown) select = "SELECT roles.*, v.vote, v.voted_at" order = "roles.posted_at DESC, roles.uid" scored = agent is not None and by_score if scored: select += f", {_VOTED_SCORE} AS score, s.gated_reason, s.prompt" order = "score DESC, " + order elif open_only: # Newest first, open only: idx_roles_posted covers this exactly # (posted_at DESC, uid, WHERE closed_at IS NULL), so it earns the # LIMIT rather than sorting every open role first. Left to the # planner, sqlite3's bundled build (no STAT4) reads the near-100% # selectivity of `closed_at IS NULL` from idx_roles_closed's plain # average and picks that instead, sorting the lot — checked with # EXPLAIN QUERY PLAN on a copy of the live database. sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1) found = [ScoredRole.from_row(r) for r in self._rows( f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?", [*params, limit, offset])] by_uid = self._tags_by_uid([r.uid for r in found]) for r in found: r.tags = by_uid.get(r.uid, []) return found def roles_page(self, limit: int = 100, company: str | None = None, place: str | None = None, open_only: bool = True, q: str | None = None, function: str | None = None, remote: str | None = None, days: int | None = None, hide_muted: bool = False, include_read: bool = True, tags: Sequence[str] = (), tag_mode: str = "any", agent: int | None = None, by_score: bool = True, gated: bool = False, judged: str | None = None, pay: str | None = None, company_keys: Sequence[str] = (), places: Sequence[str] = (), seniority: Sequence[str] = (), seniority_unknown: bool = True, pay_unknown: bool = False, offset: int = 0) -> list[RoleRow]: """What `GET /api/roles` serves: the same rows `roles` would, in the same order, with the same filters and paging, but only the fields a row draws (see `RoleRow`) — never `description`, `feed_text`, `raw` or `article_text`, which a full `roles.*` select would carry whether or not a page ever renders them. The reader still gets the whole role by uid, see `role`. `excerpt` is `app.models.excerpt` over a `SUBSTR` of `_EXCERPT_SOURCE` (`_EXCERPT_RAW` characters, trimmed further in Python), so the source column itself never crosses out of SQLite for a list.""" sql, params = self._roles_from( company=company, place=place, open_only=open_only, q=q, function=function, remote=remote, days=days, hide_muted=hide_muted, include_read=include_read, tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated, judged=judged, pay=pay, company_keys=company_keys, places=places, seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown) select = ( "SELECT roles.uid, roles.company_key, roles.company, roles.title, roles.place, " "roles.country, roles.remote, roles.function, roles.seniority, roles.salary_min, " "roles.salary_max, roles.currency, roles.salary_text, roles.original_url, " "roles.posted_at, roles.read_at, " "v.vote, v.voted_at, " f"SUBSTR({_EXCERPT_SOURCE}, 1, {_EXCERPT_RAW}) AS excerpt") order = "roles.posted_at DESC, roles.uid" scored = agent is not None and by_score if scored: select += f", {_VOTED_SCORE} AS score, s.gated_reason, s.prompt" order = "score DESC, " + order elif open_only: # Same reasoning as `roles`: the planner otherwise sorts every # open role before applying LIMIT. sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1) rows = self._rows(f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?", [*params, limit, offset]) found = [] for r in rows: row = dict(r) row["excerpt"] = excerpt(row["excerpt"]) found.append(RoleRow.from_row(row)) by_uid = self._tags_by_uid([f.uid for f in found]) copies = self._copy_counts([f.uid for f in found]) lines = self._place_lines([f.uid for f in found]) for f in found: f.tags = by_uid.get(f.uid, []) f.copies = copies.get(f.uid, 0) f.place = lines.get(f.uid, f.place) return found def count_roles(self, company: str | None = None, place: str | None = None, open_only: bool = True, q: str | None = None, function: str | None = None, remote: str | None = None, days: int | None = None, hide_muted: bool = False, include_read: bool = True, tags: Sequence[str] = (), tag_mode: str = "any", agent: int | None = None, by_score: bool = True, gated: bool = False, judged: str | None = None, pay: str | None = None, company_keys: Sequence[str] = (), places: Sequence[str] = (), seniority: Sequence[str] = (), seniority_unknown: bool = True, pay_unknown: bool = False) -> int: """How many roles `roles` would list with the same filters and no limit.""" sql, params = self._roles_from( company=company, place=place, open_only=open_only, q=q, function=function, remote=remote, days=days, hide_muted=hide_muted, include_read=include_read, tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated, judged=judged, pay=pay, company_keys=company_keys, places=places, seniority=seniority, seniority_unknown=seniority_unknown, pay_unknown=pay_unknown) row = self._row(f"SELECT COUNT(*) AS n {sql}", params) return int(row["n"]) if row else 0 def role_facets(self, company: str | None = None, place: str | None = None, open_only: bool = True, q: str | None = None, function: str | None = None, remote: str | None = None, days: int | None = None, hide_muted: bool = False, include_read: bool = True, tags: Sequence[str] = (), tag_mode: str = "any", agent: int | None = None, by_score: bool = True, gated: bool = False, judged: str | None = None, pay: str | None = None, company_keys: Sequence[str] = (), places: Sequence[str] = (), seniority: Sequence[str] = (), seniority_unknown: bool = True, pay_unknown: bool = False) -> dict[str, list[tuple[str, int]]]: """How many roles `roles` would list at each seniority level and each pay state, each counted with every other filter and its own left out, so the rail can say what a click would add. `seniority` is every level of `LEVELS` in order and then `""`, no level; `pay` is `salary_state`'s 'normalised' and 'stated' (a line with no figure) and then `""`, no pay stated. A value no role has counts 0.""" shared: dict[str, Any] = { "company": company, "place": place, "open_only": open_only, "q": q, "function": function, "remote": remote, "days": days, "hide_muted": hide_muted, "include_read": include_read, "tags": tags, "tag_mode": tag_mode, "agent": agent, "by_score": by_score, "gated": gated, "judged": judged, "company_keys": company_keys, "places": places} by_level, level_params = self._roles_from( **shared, pay=pay, pay_unknown=pay_unknown) by_pay, pay_params = self._roles_from( **shared, seniority=seniority, seniority_unknown=seniority_unknown) levels = {r["value"]: int(r["n"]) for r in self._rows( f"SELECT roles.seniority AS value, COUNT(*) AS n {by_level} " "GROUP BY roles.seniority", level_params)} pays = {r["value"]: int(r["n"]) for r in self._rows( f"SELECT {_PAY_STATE} AS value, COUNT(*) AS n {by_pay} GROUP BY value", pay_params)} return {"seniority": [(v, levels.get(v, 0)) for v in (*LEVELS, "")], "pay": [(v, pays.get(v, 0)) for v in ("normalised", "stated", "")]} def _roles_from(self, *, company: str | None, place: str | None, open_only: bool, q: str | None, function: str | None, remote: str | None, days: int | None, hide_muted: bool, include_read: bool, tags: Sequence[str], tag_mode: str, agent: int | None, by_score: bool, gated: bool, judged: str | None, pay: str | None = None, company_keys: Sequence[str] = (), places: Sequence[str] = (), seniority: Sequence[str] = (), seniority_unknown: bool = True, pay_unknown: bool = False) -> tuple[str, list[Any]]: """The `FROM ... WHERE ...` that `roles` and `count_roles` share, with its parameters in order. The agent's votes are always joined as `v` (see `_WITH_VOTES`); its scores as `s` only when ordering by them. A copy of another role (`rebuild_dupes`) is never listed: its kept role stands for it and counts its copies.""" where: list[str] = ["roles.dup_of IS NULL"] params: list[Any] = [] if company: where.append("company LIKE ? ESCAPE '\\'") params.append(_like(company)) if place: where.append("place LIKE ? ESCAPE '\\'") params.append(_like(place)) if function: where.append("function = ?") params.append(function) if remote: where.append("remote = ?") params.append(remote) if days is not None: where.append("posted_at >= ?") params.append(_iso(self.clock() - timedelta(days=days))) for word in (q or "").split(): where.append("(title LIKE ? ESCAPE '\\' OR company LIKE ? ESCAPE '\\')") params += [_like(word)] * 2 if judged: if judged not in JUDGED: raise ValueError(f"judged is {', '.join(JUDGED)}, not {judged!r}") where.append(JUDGED[judged]) if pay: if pay not in PAY: raise ValueError(f"pay is {', '.join(PAY)}, not {pay!r}") where.append(f"({PAY[pay]} OR {_NO_PAY})" if pay_unknown else PAY[pay]) if company_keys: where.append(f"company_key IN ({', '.join('?' for _ in company_keys)})") params += list(company_keys) if seniority: if unknown := sorted(set(seniority) - set(LEVELS)): raise ValueError(f"seniority is {', '.join(LEVELS)}, not {unknown[0]!r}") levels = f"roles.seniority IN ({', '.join('?' for _ in seniority)})" where.append(f"({levels} OR roles.seniority = '')" if seniority_unknown else levels) params += list(seniority) if open_only: where.append("closed_at IS NULL") if hide_muted and agent is not None: where.append("company_key NOT IN (SELECT company_key FROM agent_companies " "WHERE agent_id = ? AND affinity = -1)") params.append(agent) if not include_read: where.append("roles.read_at IS NULL") if tags: marks = ", ".join("?" for _ in tags) if tag_mode == "all": where.append( f"(SELECT COUNT(DISTINCT rt.tag) FROM role_tags rt " f"WHERE rt.uid = roles.uid AND rt.tag IN ({marks})) = ?") params += [*tags, len(tags)] elif tag_mode == "any": where.append( f"EXISTS (SELECT 1 FROM role_tags rt " f"WHERE rt.uid = roles.uid AND rt.tag IN ({marks}))") params += list(tags) else: raise ValueError(f"tag_mode is any or all, not {tag_mode!r}") if places: _known_places(places) clause, values = _places_sql(places, self.gazetteer()) where.append(clause) params += values sql = f"FROM roles {_WITH_VOTES}" joins: list[Any] = [agent] if agent is not None and by_score: sql += " JOIN scores s ON s.uid = roles.uid AND s.agent_id = ?" joins.append(agent) if not gated: where.append("s.gated_reason = ''") if where: sql += " WHERE " + " AND ".join(where) return sql, [*joins, *params] def tag_counts(self, agent_id: int | None = None, show_muted: bool = False) -> list[tuple[str, int, int]]: """`(tag, open_count, unread_count)` over open roles, most-used tag first and then alphabetically. What `/api/filters` serves; the companies `agent_id` hid are left out unless `show_muted`, the same default as `roles`, and with no agent nothing is hidden.""" where = "r.closed_at IS NULL AND r.dup_of IS NULL" params: list[Any] = [] if not show_muted and agent_id is not None: where += (" AND r.company_key NOT IN (SELECT company_key FROM agent_companies " "WHERE agent_id = ? AND affinity = -1)") params.append(agent_id) rows = self._rows( f"""SELECT rt.tag AS tag, COUNT(*) AS n, SUM(r.read_at IS NULL) AS unread FROM role_tags rt JOIN roles r ON r.uid = rt.uid WHERE {where} GROUP BY rt.tag ORDER BY n DESC, rt.tag ASC""", params) return [(str(r["tag"]), int(r["n"]), int(r["unread"])) for r in rows] def mark_closed(self, source: str, seen_ids: Sequence[str], subscription_id: int) -> int: """Close every open role of one board that a complete fetch of it did not list. Returns how many closed. Only call it after a fetch that finished: a failed one lists nothing, and would close the whole board.""" return self._write( """UPDATE roles SET closed_at = ? WHERE source = ? AND subscription_id = ? AND closed_at IS NULL AND source_id NOT IN (SELECT value FROM json_each(?))""", (self.now(), source, subscription_id, json.dumps(list(seen_ids)))) def close_unseen(self, source: str, before: str) -> int: """Close every open role of `source` last seen before `before`. For a source whose fetch is never a complete list, a search, where a role dropping out of one run's results is not the role closing.""" return self._write( """UPDATE roles SET closed_at = ? WHERE source = ? AND closed_at IS NULL AND last_seen < ?""", (self.now(), source, before)) def close_subscription(self, subscription_id: int) -> int: """Close every open role of one subscription, for a board that is no longer fetched: nothing would ever close them otherwise.""" return self._write( "UPDATE roles SET closed_at = ? WHERE subscription_id = ? AND closed_at IS NULL", (self.now(), subscription_id)) def backfill_from_place(self) -> int: """Fill `country` and `remote` from `place` on every role that has none, the way `upsert_role` does for a new one, so a word added to the lookups reaches the roles already stored. Returns how many columns it filled.""" rows = self._rows("SELECT uid, place FROM roles " "WHERE place != '' AND (country = '' OR remote = '')") countries = [(code, r["uid"]) for r in rows if (code := country_of(r["place"]))] remotes = [(mode, r["uid"]) for r in rows if (mode := remote_of(r["place"]))] with self.transaction() as conn: conn.executemany("UPDATE roles SET country = ? WHERE uid = ? AND country = ''", countries) conn.executemany("UPDATE roles SET remote = ? WHERE uid = ? AND remote = ''", remotes) return len(countries) + len(remotes) # ----------------------------------------------------------------- prep def prep_rows(self, scope: Literal["due", "open", "all"] = "due") -> list[sqlite3.Row]: """What `app.prep` maps a role from, and what it holds now: the roles due (`prep_due` above 0), every open role, or every role.""" where = {"due": "prep_due > 0", "open": "closed_at IS NULL", "all": "1"}[scope] return self._rows( f"""SELECT uid, prep_due, {_HOST} AS host, title, company, company_key, place, seniority_host, employment_host, remote_host, country_host, seniority_model, seniority_stated, seniority_title, seniority, employment, remote, country, json_extract(raw, '$.places') AS raw_places FROM roles WHERE {where}""") def resolve_role_places(self, uids: Sequence[str]) -> dict[str, list[int]]: """Resolve these roles' own places against the gazetteer, writing the rows that resolve differently; `uid -> place ids` for each that has places (`gazetteer.relink_uids`).""" with self.transaction() as conn: return gazetteer.relink_uids(conn, self._gazetteer(conn), list(uids)) def company_folds(self) -> dict[str, list[str]]: """`fold -> company keys` of `company_aliases`, sorted: what `resolve_company` asks one name at a time, for prep's whole batch.""" out: dict[str, list[str]] = {} for r in self._rows("SELECT fold, company_key FROM company_aliases " "ORDER BY fold, company_key"): out.setdefault(str(r["fold"]), []).append(str(r["company_key"])) return out def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int: """Write what prep mapped, one dict per role with `uid`, `due` (the `prep_due` it read), `company_key`, `seniority_stated`, `seniority_title`, `employment`, `remote`, `remote_places` and `country`. `seniority` is the model's level, else the stated, else the title's. An empty `remote` or `country` leaves the column as it is, since `extract` fills an empty one. `remote_places`, what the place texts say, only fills an empty column: it is a guess from a line, weaker than what `extract` read in the posting. `prep_due` goes back to 0 only when no pull changed the role since it was read. Returns the rows written.""" with self.transaction() as conn: return conn.executemany( """UPDATE roles SET company_key = :company_key, seniority_stated = :seniority_stated, seniority_title = :seniority_title, seniority = COALESCE(NULLIF(seniority_model, ''), NULLIF(:seniority_stated, ''), :seniority_title), employment = :employment, remote = COALESCE(NULLIF(:remote, ''), NULLIF(remote, ''), :remote_places), country = COALESCE(NULLIF(:country, ''), country), prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END WHERE uid = :uid""", mapped).rowcount def host_words(self) -> list[tuple[str, str, str, int]]: """`(field, host, word, open roles)` for every host word an open role carries, the word lowercased: what prep checks against its tables.""" found: list[tuple[str, str, str, int]] = [] for field in ("seniority", "employment", "remote", "country"): found += [(field, str(r[0]), str(r[1]), int(r[2])) for r in self._rows( f"""SELECT {_HOST}, LOWER({field}_host), COUNT(*) FROM roles WHERE closed_at IS NULL AND {field}_host != '' GROUP BY 1, 2""")] return found def replace_unmapped(self, rows: Iterable[tuple[str, str, str, int]]) -> None: """Make `unmapped` these `(field, host, word, roles)`, keeping the `first_seen` of a word already listed.""" stamp = self.now() with self.transaction() as conn: seen = {(r[0], r[1], r[2]): r[3] for r in conn.execute( "SELECT field, host, word, first_seen FROM unmapped")} conn.execute("DELETE FROM unmapped") conn.executemany( "INSERT INTO unmapped (field, host, word, roles, first_seen) " "VALUES (?, ?, ?, ?, ?)", [(*r, seen.get(r[:3], stamp)) for r in rows]) def unmapped(self) -> list[dict[str, Any]]: """The host words no table maps, by field and host, most roles first.""" return [dict(r) for r in self._rows( "SELECT field, host, word, roles, first_seen FROM unmapped " "ORDER BY field, host, roles DESC, word")] # -------------------------------------------------------------- extract def _top_scope(self, stage: Literal["extract", "summarise"]) -> tuple[str, list[Any]]: """The `WHERE` fragment for a role among the best ungated scores of an enabled agent, as deep and as low as that agent's own `limits` say for `stage`: what the two model stages, extract and summarise, are bounded to. The rank is numbered per agent and the agent's limits ride out of the window subquery beside it, so each row is held to the limits of the agent that ranked it; a key the JSON lacks is the `Limits` default. The role is `r`. A company an agent hid needs no clause: `score` writes no rows for it under that agent. An agent without a prompt with text counts for nothing here: its scores have no fit, so they say how fresh a role is, not whether it is wanted.""" defaults = Limits() return (f"""r.uid IN (SELECT uid FROM ( SELECT s.uid, s.score, a.limits, ROW_NUMBER() OVER ( PARTITION BY s.agent_id ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n FROM scores s JOIN agents a ON a.id = s.agent_id AND a.enabled = 1 AND EXISTS (SELECT 1 FROM agent_prompts p WHERE p.agent_id = a.id AND TRIM(p.text, ' ' || char(9, 10, 13)) <> '') JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL AND top.dup_of IS NULL WHERE s.gated_reason = '') WHERE n <= COALESCE(json_extract(limits, '$.{stage}_top'), ?) AND score >= COALESCE(json_extract(limits, '$.{stage}_min'), ?))""", [getattr(defaults, f"{stage}_top"), getattr(defaults, f"{stage}_min")]) def _extract_scope(self, uids: Sequence[str] | None) -> tuple[str, list[Any]]: """The `WHERE` for a role `extract` may read: open, never answered, not given up. Given `uids`, those of them and nothing else, since a batch asks again when it runs and must finish what it was dealt; otherwise only the roles in `_top_scope`.""" where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at") .replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries") .replace("dup_of", "r.dup_of") .replace("seniority_model", "r.seniority_model")] if uids is not None: where.append("r.uid IN (SELECT value FROM json_each(?))") return " AND ".join(where), [json.dumps(list(uids))] top, params = self._top_scope("extract") return " AND ".join([*where, top]), params def extract_pending(self) -> int: where, params = self._extract_scope(None) row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params) return int(row["n"]) if row else 0 def extract_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]: """Roles `extract` has yet to read: the best of every enabled agent's scores, as deep and as low as that agent's own `limits` say, so the backlog follows the rankings rather than the boards. Ordered by the best rank any agent gives the role (each agent's first, then each agent's second ...), then by its score, so every agent's top comes before anyone's tail whatever limits cut the batch. Given `uids`, only those of them still waiting, whatever their rank; a role no agent ranks comes last.""" where, params = self._extract_scope(uids) return [Role.from_row(r) for r in self._rows( "SELECT r.* FROM roles r LEFT JOIN (" " SELECT uid, MIN(n) AS best_rank, MAX(score) AS best_score FROM (" " SELECT s.uid, s.score, ROW_NUMBER() OVER (" " PARTITION BY s.agent_id ORDER BY s.score DESC, s.uid) AS n" " FROM scores s" " JOIN agents a ON a.id = s.agent_id AND a.enabled = 1" " WHERE s.gated_reason = '') GROUP BY uid) k ON k.uid = r.uid " f"WHERE {where} " "ORDER BY k.best_rank IS NULL, k.best_rank, k.best_score DESC, " "r.first_seen DESC, r.uid LIMIT ?", [*params, limit])] def set_extracted(self, uid: str, fields: dict[str, Any]) -> int | None: """Write what `extract` read for one role and stamp `extracted_at`. Only into empty columns, and only a non-empty value: a board's own field is never replaced and nothing is ever blanked. `place` brings `place_key` and `place_fold` with it. Returns how many columns were filled, or None for an unknown uid. Values are trusted to be in vocabulary; the job's `apply` is where a reply is checked. `function` is the one exception to "only into empty columns": it is the `ats` classifier's keyword guess, not a board field, so a non-empty answer overwrites it outright rather than only filling an empty column — see `upsert_role`'s docstring for the other half of the rule that then keeps a later pull from putting the guess back. `seniority` is the other: a non-empty answer is `seniority_model`, which comes before the stated and the title's level, so it becomes `seniority` too.""" with self.transaction() as conn: row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone() if row is None: return None sets: dict[str, Any] = {} for name in _EXTRACT_TEXT: value = str(fields.get(name) or "") if value and not row[name]: sets[name] = value function = str(fields.get("function") or "") if function: sets["function"] = function level = str(fields.get("seniority") or "") if level: sets["seniority_model"] = sets["seniority"] = level for name in _RAW_INT: number = _int_or_none(fields.get(name)) if number is not None and row[name] is None: sets[name] = number languages = [str(code) for code in fields.get("languages") or []] if languages and json.loads(row["languages"] or "[]") == []: sets["languages"] = json.dumps(languages) filled = len(sets) - ("seniority" in sets) if "place" in sets: sets["place_key"] = place_key(sets["place"]) sets["place_fold"] = places.fold(sets["place"]) gazetteer.write(conn, self._gazetteer(conn), uid, gazetteer.texts({"place": sets["place"]})) sets["extracted_at"] = self.now() conn.execute( f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?", (*sets.values(), uid)) return filled def extract_missed(self, uids: Sequence[str], why: str = "") -> int: """One reply that left these roles unanswered. Counted only against those still waiting, so a role the reply did answer is not charged. Returns how many this pushed to `GIVE_UP_AFTER`. `why` is accepted for `ModelTask.missed` and not kept: there is no Given up list yet.""" if not uids: return 0 where = f"uid IN (SELECT value FROM json_each(?)) AND {_EXTRACT_WAITING}" wanted = json.dumps(list(uids)) with self.transaction() as conn: crossed = conn.execute( f"SELECT COUNT(*) FROM roles WHERE {where} AND extract_tries + 1 >= ?", (wanted, GIVE_UP_AFTER)).fetchone()[0] conn.execute(f"UPDATE roles SET extract_tries = extract_tries + 1 " f"WHERE {where}", (wanted,)) return int(crossed) def seniority_agreement(self) -> list[dict[str, Any]]: """How far each host's stated level matches the model's, over the open roles that have both: `host`, `compared`, `agreed` and `rate` (agreed / compared), the least trusted host first. `stated_only` counts the roles that state a level and have no model value yet, so a small `compared` can be told from a host that states nothing.""" rows = self._rows( f"SELECT {_HOST} AS host, " "SUM(seniority_stated <> '' AND seniority_model <> '') AS compared, " "SUM(seniority_stated <> '' AND seniority_stated = seniority_model) AS agreed, " "SUM(seniority_stated <> '' AND seniority_model = '') AS stated_only " "FROM roles WHERE closed_at IS NULL AND dup_of IS NULL " "GROUP BY host HAVING compared > 0 OR stated_only > 0") found = [{"host": r["host"], "compared": int(r["compared"]), "agreed": int(r["agreed"]), "stated_only": int(r["stated_only"]), "rate": round(r["agreed"] / r["compared"], 3) if r["compared"] else None} for r in rows] return sorted(found, key=lambda h: (h["rate"] is None, h["rate"] or 0, h["host"])) # ------------------------------------------------------------ summarise #: The `state` key counting replies that left a role without a summary. #: `roles` has no column for it, and the count only matters for the few #: roles in some agent's top list. _SUMMARY_MISSED = "summarise.missed:" def _summary_scope(self, uids: Sequence[str] | None) -> tuple[str, list[Any]]: """The `WHERE` for a role `summarise` may be sent: no summary yet and some text to summarise. Given `uids`, those of them and nothing else, open or not and given up or not, because somebody asked for them. Otherwise open, not given up, and in `_top_scope`.""" where = ["r.summary = ''", "(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"] if uids is not None: where.append("r.uid IN (SELECT value FROM json_each(?))") return " AND ".join(where), [json.dumps(list(uids))] top, params = self._top_scope("summarise") where += [ "r.closed_at IS NULL", "r.dup_of IS NULL", f"""NOT EXISTS (SELECT 1 FROM state WHERE key = '{self._SUMMARY_MISSED}' || r.uid AND CAST(value AS INTEGER) >= {GIVE_UP_AFTER})""", top, ] return " AND ".join(where), params def summarise_pending(self) -> int: where, params = self._summary_scope(None) row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params) return int(row["n"]) if row else 0 def summarise_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]: """Roles `summarise` has yet to write, newest first: the top of every enabled agent's scores, as deep and as low as that agent's own `limits` say, so the backlog is bounded by the agents rather than by the number of open roles; given `uids`, those still without a summary, whatever their rank. A batch asks again when it runs.""" where, params = self._summary_scope(uids) return [Role.from_row(r) for r in self._rows( f"SELECT r.* FROM roles r WHERE {where} " "ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])] def set_summary(self, uid: str, summary: str, tags: Sequence[str] = ()) -> bool: """Write a role's summary and stamp `summarised_at`. Only over an empty one, so a second batch that carried the same role does not replace the first answer. Given `tags`, they replace the role's tags in the same transaction — see `_replace_tags` — but only when the summary was actually written, so a rejected late answer cannot plant tags on a role it did not get to summarise. False for an unknown uid or one already summarised.""" with self.transaction() as conn: written = conn.execute( "UPDATE roles SET summary = ?, summarised_at = ? " "WHERE uid = ? AND summary = ''", (summary, self.now(), uid)).rowcount conn.execute("DELETE FROM state WHERE key = ?", (self._SUMMARY_MISSED + uid,)) if written and tags: self._replace_tags(conn, uid, tags) return bool(written) def summarise_missed(self, uids: Sequence[str], why: str = "") -> int: """One reply that left these roles without a summary. Counted only against those still without one. Returns how many this pushed to `GIVE_UP_AFTER`. `why` is accepted for `ModelTask.missed` and not kept.""" if not uids: return 0 rows = self._rows("SELECT uid FROM roles WHERE summary = '' " "AND uid IN (SELECT value FROM json_each(?))", (json.dumps(list(uids)),)) keys = [self._SUMMARY_MISSED + r["uid"] for r in rows] with self.transaction() as conn: conn.executemany( """INSERT INTO state (key, value) VALUES (?, '1') ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + 1 AS TEXT)""", [(key,) for key in keys]) crossed = conn.execute( "SELECT COUNT(*) FROM state WHERE key IN (SELECT value FROM json_each(?)) " "AND CAST(value AS INTEGER) = ?", (json.dumps(keys), GIVE_UP_AFTER)).fetchone()[0] return int(crossed) def set_article_text(self, uid: str, text: str) -> Role | None: """Store the posting page as plain text and stamp `article_at`. The role as it now stands, or None for an unknown uid.""" if not self._write("UPDATE roles SET article_text = ?, article_at = ? WHERE uid = ?", (text, self.now(), uid)): return None return self.role(uid) # --------------------------------------------------------------- enrich def enrich_waiting(self, top: int, floor: float, uids: Sequence[str] | None = None) -> list[dict[str, Any]]: """Open `ats` roles without text that an enabled agent with a prompt ranks ungated among its best `top` and at `floor` or above, best first by the highest such score, as `{uid, source_id, subscription_id, title, url, published_at, raw, score}`. The ranking is `_top_scope`'s, with the bar given rather than the agent's own. Given `uids`, those of them still without text, whatever their rank, in no order (`score` is None), since a batch finishes what it was dealt.""" fields = ("r.uid, r.source_id, r.subscription_id, r.title, r.original_url AS url, " "r.published_at, r.raw") waiting = ("r.closed_at IS NULL AND r.dup_of IS NULL AND r.source = 'ats' " "AND r.subscription_id IS NOT NULL AND r.description = '' " "AND r.article_text = ''") if uids is not None: rows = self._rows( f"SELECT {fields}, NULL AS score FROM roles r WHERE {waiting} " "AND r.uid IN (SELECT value FROM json_each(?))", (json.dumps(list(uids)),)) else: rows = self._rows( f"""SELECT {fields}, MAX(ranked.score) AS score FROM ( SELECT s.uid, s.score, ROW_NUMBER() OVER ( PARTITION BY s.agent_id ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n FROM scores s JOIN agents a ON a.id = s.agent_id AND a.enabled = 1 AND EXISTS (SELECT 1 FROM agent_prompts p WHERE p.agent_id = a.id AND TRIM(p.text, ' ' || char(9, 10, 13)) <> '') JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL AND top.dup_of IS NULL WHERE s.gated_reason = '') ranked JOIN roles r ON r.uid = ranked.uid WHERE ranked.n <= ? AND ranked.score >= ? AND {waiting} GROUP BY r.uid ORDER BY score DESC, r.uid""", (top, floor)) return [dict(row) for row in rows] # -------------------------------------------------------------- vectors def roles_to_embed(self, model: str, limit: int) -> list[Role]: """Open roles with no vector from `model`, newest first, so a backlog embedded from the top improves what is on screen first. A copy is not embedded: it is never scored.""" return [Role.from_row(r) for r in self._rows( """SELECT r.* FROM roles r LEFT JOIN vectors v ON v.uid = r.uid AND v.model = ? WHERE v.uid IS NULL AND r.closed_at IS NULL AND r.dup_of IS NULL ORDER BY r.first_seen DESC, r.uid LIMIT ?""", (model, limit))] def put_vectors(self, model: str, pairs: Sequence[tuple[str, np.ndarray]]) -> int: stamp = self.now() with self.transaction() as conn: return conn.executemany( """INSERT INTO vectors (uid, model, dim, vec, embedded_at) VALUES (?,?,?,?,?) ON CONFLICT(uid) DO UPDATE SET model=excluded.model, dim=excluded.dim, vec=excluded.vec, embedded_at=excluded.embedded_at""", [(uid, model, int(vec.shape[-1]), pack(vec), stamp) for uid, vec in pairs]).rowcount def vectors(self, model: str) -> dict[str, np.ndarray]: """Every vector from `model`, by uid. The whole table: at ~30k roles of 384 floats that is ~45 MB, and one read beats one per agent.""" return {r["uid"]: unpack(r["vec"]) for r in self._rows( "SELECT uid, vec FROM vectors WHERE model = ?", (model,))} def vector_counts(self) -> dict[str, int]: return {r["model"]: int(r["n"]) for r in self._rows( "SELECT model, COUNT(*) AS n FROM vectors GROUP BY model")} def drop_vectors(self, keep: str) -> int: """Delete vectors from every model but `keep`: another model's are a different geometry, never compared, and only make counts lie.""" return self._write("DELETE FROM vectors WHERE model <> ?", (keep,)) # --------------------------------------------------------------- agents def agents(self, enabled_only: bool = False) -> list[Agent]: sql = "SELECT * FROM agents" if enabled_only: sql += " WHERE enabled = 1" return self._agents(self._rows(sql + " ORDER BY name COLLATE NOCASE")) def agent(self, agent_id: int) -> Agent | None: found = self._agents(self._rows("SELECT * FROM agents WHERE id=?", (agent_id,))) return found[0] if found else None def default_agent(self) -> Agent | None: """The agent a vote or an affinity goes to when none is named: the enabled one with the lowest id, else the lowest id, else None.""" found = self._agents(self._rows( "SELECT * FROM agents ORDER BY enabled DESC, id LIMIT 1")) return found[0] if found else None def _agents(self, rows: Sequence[sqlite3.Row]) -> list[Agent]: """The rows as agents, each with its prompts in order, read with one query for them all.""" prompts: dict[int, list[dict[str, str]]] = {} for p in self._rows( "SELECT agent_id, name, text FROM agent_prompts WHERE agent_id IN " "(SELECT value FROM json_each(?)) ORDER BY agent_id, position", (json.dumps([r["id"] for r in rows]),)): prompts.setdefault(p["agent_id"], []).append({"name": p["name"], "text": p["text"]}) return [Agent.from_row({**dict(r), "prompts": prompts.get(r["id"], [])}) for r in rows] def save_agent(self, name: str, intent: str = "", gates: Sequence[Gate] = (), weights: Weights | None = None, enabled: bool = True, agent_id: int | None = None, limits: Limits | None = None, prompts: Sequence[Prompt] | None = None) -> Agent: """Create an agent, or with `agent_id` replace that one's fields and prompts. Without `prompts`, a non-empty `intent` is its one prompt, named after the agent. A prompt whose text is unchanged keeps its cached vector, whatever its name or place, so an edit re-embeds only the prompts it changed. Raises ValueError on an empty or taken name, or two prompts of one name, KeyError on an unknown id.""" name = name.strip() if not name: raise ValueError("an agent needs a name") if prompts is None: prompts = [Prompt(name=name, text=intent)] if intent.strip() else [] prompts = check_prompts(list(prompts)) fields = (name, json.dumps([g.model_dump() for g in gates]), (weights or Weights()).model_dump_json(), (limits or Limits()).model_dump_json(), int(enabled)) try: with self.transaction() as conn: if agent_id is None: agent_id = conn.execute( """INSERT INTO agents (name, gates, weights, limits, enabled, created_at) VALUES (?,?,?,?,?,?)""", (*fields, self.now())).lastrowid elif not conn.execute( """UPDATE agents SET name=?, gates=?, weights=?, limits=?, enabled=? WHERE id=?""", (*fields, agent_id)).rowcount: raise KeyError(agent_id) cached = {r["text"]: (r["vec_key"], r["vec"]) for r in conn.execute( "SELECT text, vec_key, vec FROM agent_prompts WHERE agent_id=?", (agent_id,))} conn.execute("DELETE FROM agent_prompts WHERE agent_id=?", (agent_id,)) conn.executemany( """INSERT INTO agent_prompts (agent_id, position, name, text, vec_key, vec) VALUES (?,?,?,?,?,?)""", [(agent_id, n, p.name, p.text, *cached.get(p.text, ("", None))) for n, p in enumerate(prompts)]) except sqlite3.IntegrityError as exc: raise ValueError(f"an agent named {name!r} already exists") from exc assert agent_id is not None found = self.agent(agent_id) assert found is not None return found def delete_agent(self, agent_id: int) -> bool: """Its prompts, scores, votes and affinities go with it, by the foreign keys.""" return self._write("DELETE FROM agents WHERE id=?", (agent_id,)) > 0 def prompt_vector(self, agent_id: int, name: str, key: str) -> np.ndarray | None: """The cached vector of the agent's prompt `name`, if it was made under `key`.""" row = self._row("SELECT vec_key, vec FROM agent_prompts WHERE agent_id=? AND name=?", (agent_id, name)) if row is None or row["vec_key"] != key or row["vec"] is None: return None return unpack(row["vec"]) def set_prompt_vector(self, agent_id: int, name: str, key: str, vec: np.ndarray) -> None: self._write("UPDATE agent_prompts SET vec_key=?, vec=? WHERE agent_id=? AND name=?", (key, pack(vec), agent_id, name)) # --------------------------------------------------------------- scores def score_candidates(self, agent_id: int, gates: Sequence[Gate]) -> list[Candidate]: """Every open role of a company this agent has not hidden, copies (`rebuild_dupes`) aside, with the agent's affinity for the company and how the role meets each gate: 1 match, 0 no match, -1 the column is empty. The gates are evaluated here, in SQL, so a `require` gate is the `WHERE` it reads as, written as a column so a gated role keeps its row and reason.""" columns: list[str] = [] params: list[Any] = [] gaz = self.gazetteer() for n, gate in enumerate(gates): empty, match, values = _gate_sql(gate, gaz) columns.append( f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}") params += values select = ", ".join(["r.uid", "r.posted_at", "COALESCE(ac.affinity, 0) AS affinity", *columns]) rows = self._rows( f"""SELECT {select} FROM roles r LEFT JOIN agent_companies ac ON ac.company_key = r.company_key AND ac.agent_id = ? WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND COALESCE(ac.affinity, 0) <> -1""", [*params, agent_id]) return [Candidate(uid=r["uid"], posted_at=r["posted_at"], affinity=int(r["affinity"]), gates=[int(r[f"g{n}"]) for n in range(len(gates))]) for r in rows] def replace_scores(self, agent_id: int, rows: Sequence[tuple[str, float, str] | tuple[str, float, str, str]] ) -> int: """`(uid, score, gated_reason)` for every role this agent scored, with the name of the prompt that gave the fit as a fourth item ('' when left out), replacing whatever it had: a role closed or muted since the last run drops out rather than keeping a stale score. One transaction, so a reader sees the old list or the new one. An agent deleted while its run was scoring gets nothing written and 0 back: its rows are gone with it, and an insert would only fail the foreign key.""" stamp = self.now() with self.transaction() as conn: if conn.execute("SELECT 1 FROM agents WHERE id=?", (agent_id,)).fetchone() is None: return 0 conn.execute("DELETE FROM scores WHERE agent_id=?", (agent_id,)) conn.executemany( """INSERT INTO scores (uid, agent_id, score, gated_reason, prompt, scored_at) VALUES (?,?,?,?,?,?)""", [(row[0], agent_id, row[1], row[2], row[3] if len(row) > 3 else "", stamp) for row in rows]) return len(rows) def scores(self, agent_id: int) -> dict[str, tuple[float, str]]: """`uid -> (score, gated_reason)` for one agent.""" return {r["uid"]: (float(r["score"]), str(r["gated_reason"])) for r in self._rows( "SELECT uid, score, gated_reason FROM scores WHERE agent_id=?", (agent_id,))} # ------------------------------------------------------------ companies def add_company(self, name: str, *, key: str = "", domain: str = "", status: str = "researching", careers_url: str = "", notes: str = "", ats: str = "", country: str = "", city: str = "", ats_slug: str = "", hand_reason: str = "") -> Company: """Raises ValueError on a key already taken or a `hand_reason` outside `HAND_REASONS`.""" name = name.strip() key = key or company_key(name) if not key: raise ValueError("a company needs a name") _check_hand_reason(hand_reason) try: with self.transaction() as conn: self._insert_company(conn, key, name, { "domain": domain, "status": status, "careers_url": careers_url, "notes": notes, "ats": ats, "country": country, "city": city, "ats_slug": ats_slug, "hand_reason": hand_reason}) except sqlite3.IntegrityError as exc: raise ValueError(f"company {key!r} already exists") from exc return self._company(key) def _insert_company(self, conn: sqlite3.Connection, key: str, name: str, fields: dict[str, Any]) -> None: stamp = self.now() columns = ["key", "name", *fields, "created_at", "updated_at"] conn.execute( f"INSERT INTO companies ({', '.join(columns)}) " f"VALUES ({', '.join('?' * len(columns))})", (key, name, *fields.values(), stamp, stamp)) self._alias(conn, key, name) def company(self, key: str, agent_id: int | None = None) -> Company | None: """One company, with `affinity` as `agent_id` set it; 0 without an agent.""" row = self._row(f"{_COMPANY_SELECT} WHERE c.key=?", (agent_id, key)) return Company.from_row(row) if row else None def companies(self, agent_id: int | None = None) -> list[Company]: """Every company by name, with `affinity` as `agent_id` set it.""" return [Company.from_row(r) for r in self._rows( f"{_COMPANY_SELECT} ORDER BY c.name COLLATE NOCASE", (agent_id,))] def set_affinity(self, agent_id: int, key: str, affinity: int) -> Company: """One agent's stance on a company: -1 hidden, 0 neutral, 1 liked, 2 favourite. 0 deletes the row, since no row means neutral. Returns the company with this agent's affinity. Raises ValueError for another value and KeyError for an unknown company or agent. A key with roles and no company row, an employer known only from a search, gets a row here holding only its key and the name its roles give it, the most frequent one: nothing else can be judged without one. It has no board, `ats`, `careers_url` or `domain`, which is how the Companies list tells it from a followed company.""" _check_affinity(affinity) with self.transaction() as conn: self._known(conn, "agents", "id", agent_id) if conn.execute("SELECT 1 FROM companies WHERE key=?", (key,)).fetchone() is None: named = conn.execute( """SELECT company FROM roles WHERE company_key=? AND company<>'' GROUP BY company ORDER BY count(*) DESC, max(last_seen) DESC LIMIT 1""", (key,)).fetchone() if named is None: raise KeyError(key) self._insert_company(conn, key, named[0], {}) if affinity == 0: conn.execute("DELETE FROM agent_companies WHERE agent_id=? AND company_key=?", (agent_id, key)) else: conn.execute( """INSERT INTO agent_companies (agent_id, company_key, affinity, updated_at) VALUES (?,?,?,?) ON CONFLICT(agent_id, company_key) DO UPDATE SET affinity=excluded.affinity, updated_at=excluded.updated_at""", (agent_id, key, affinity, self.now())) return self._company(key, agent_id) def update_company(self, key: str, **fields: Any) -> Company: """Change any of `_COMPANY_FIELDS`. Affinity is per agent: `set_affinity`.""" unknown = set(fields) - set(_COMPANY_FIELDS) if unknown: raise ValueError(f"not a company field: {', '.join(sorted(unknown))}") if "hand_reason" in fields: _check_hand_reason(fields["hand_reason"]) if fields: sets = ", ".join(f"{f}=?" for f in fields) with self.transaction() as conn: changed = conn.execute(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?", (*fields.values(), self.now(), key)).rowcount if not changed: raise KeyError(key) if "name" in fields: self._alias(conn, key, fields["name"]) return self._company(key) def set_all_roles(self, key: str, on: bool) -> Company: """Tag a company All roles, or untag it: its boards are read with the agents' wide reach, Europe today, from the next pull (`app/jobs/reach.py`). Global, not per agent. Raises KeyError for an unknown company.""" return self.update_company(key, all_roles=int(bool(on))) def remove_company(self, key: str) -> bool: """Delete the row and every agent's stance on it, in one transaction: `agent_companies` holds the key with no foreign key to hold it back, so a row left behind would give a company added again under the same name an affinity it never got. Roles and subscriptions are not touched.""" with self.transaction() as conn: conn.execute("DELETE FROM agent_companies WHERE company_key=?", (key,)) conn.execute("DELETE FROM company_aliases WHERE company_key=?", (key,)) return conn.execute("DELETE FROM companies WHERE key=?", (key,)).rowcount > 0 # ---------------------------------------------------- company aliases @staticmethod def _alias(conn: sqlite3.Connection, key: str, name: str) -> None: if fold := company_fold(name): conn.execute("INSERT OR IGNORE INTO company_aliases (fold, company_key) VALUES (?, ?)", (fold, key)) def resolve_company(self, name: str) -> str: """The key a role from `name` belongs to: the company an alias names, the one with the name's own key first when several share the fold, else `company_key(name)`. `Orsted` lands on the `ørsted` company.""" own = company_key(name) row = self._row("SELECT company_key FROM company_aliases WHERE fold=? " "ORDER BY company_key=? DESC, company_key LIMIT 1", (company_fold(name), own)) return str(row["company_key"]) if row else own def add_alias(self, key: str, name: str) -> None: """Another name for company `key`: a search and a role from `name` find it. Raises KeyError for an unknown company, ValueError for a name that folds to nothing.""" if not company_fold(name): raise ValueError("an alias needs a name") with self.transaction() as conn: if conn.execute("SELECT 1 FROM companies WHERE key=?", (key,)).fetchone() is None: raise KeyError(key) self._alias(conn, key, name) def company_aliases(self) -> dict[str, list[str]]: """`company_key -> folds`, for the company search.""" out: dict[str, list[str]] = {} for r in self._rows("SELECT company_key, fold FROM company_aliases ORDER BY fold"): out.setdefault(str(r["company_key"]), []).append(str(r["fold"])) return out def duplicate_companies(self) -> list[tuple[str, list[str]]]: """`(fold, keys)` for each fold more than one company has, likely the same company under two spellings. Nothing is merged.""" return [(str(r["fold"]), str(r["keys"]).split("\t")) for r in self._rows( "SELECT fold, group_concat(company_key, char(9)) AS keys FROM company_aliases " "GROUP BY fold HAVING COUNT(*) > 1 ORDER BY fold")] def merge_companies(self, keep: str, other: str) -> Company: """Fold company `other` into `keep`, in one transaction: its roles and subscriptions move to `keep`, each agent's stance moves unless the agent has one on `keep` already, an All roles tag carries over, its aliases and its name become aliases of `keep`, and its row goes. Copies are not recomputed (`rebuild_dupes`). Raises KeyError for an unknown company and ValueError for merging one into itself.""" if keep == other: raise ValueError("a company cannot be merged into itself") with self.transaction() as conn: rows = {r["key"]: r["name"] for r in conn.execute( "SELECT key, name FROM companies WHERE key IN (?, ?)", (keep, other))} for key in (keep, other): if key not in rows: raise KeyError(key) conn.execute("UPDATE roles SET company_key=? WHERE company_key=?", (keep, other)) for sub in conn.execute("SELECT id, company FROM subscriptions").fetchall(): if company_key(sub["company"]) == other: conn.execute("UPDATE subscriptions SET company=? WHERE id=?", (rows[keep], sub["id"])) conn.execute("UPDATE OR IGNORE agent_companies SET company_key=? WHERE company_key=?", (keep, other)) conn.execute("DELETE FROM agent_companies WHERE company_key=?", (other,)) conn.execute("UPDATE OR IGNORE company_aliases SET company_key=? WHERE company_key=?", (keep, other)) conn.execute("DELETE FROM company_aliases WHERE company_key=?", (other,)) self._alias(conn, keep, rows[other]) conn.execute("UPDATE companies SET all_roles = MAX(all_roles, " "(SELECT all_roles FROM companies WHERE key=?)) WHERE key=?", (other, keep)) conn.execute("DELETE FROM companies WHERE key=?", (other,)) return self._company(keep) def _company(self, key: str, agent_id: int | None = None) -> Company: found = self.company(key, agent_id) if found is None: raise KeyError(key) return found def open_role_counts(self) -> dict[str, int]: """`company_key -> open role count`, for the company list's row count. A key with no open roles is simply absent. Copies are not counted, so the number is the rows the feed lists for the company.""" return {str(r["company_key"]): int(r["n"]) for r in self._rows( "SELECT company_key, COUNT(*) AS n FROM roles " "WHERE closed_at IS NULL AND dup_of IS NULL GROUP BY company_key")} def open_companies(self, agent_id: int | None = None ) -> list[tuple[str, list[str], int, int]]: """`(company_key, names, open roles, affinity)` for every company with an open role, what the Company field suggests from. `names` are the company row's name and then every spelling its roles carry, commonest first, so a search finds a company by any of them. A company `agent_id` hid is left out; `affinity` is that agent's, 0 without one. Copies are not counted, as in the feed.""" rows = self._rows( """SELECT r.company_key AS key, r.company AS name, COUNT(*) AS n, c.name AS own, COALESCE(ac.affinity, 0) AS affinity FROM roles r LEFT JOIN companies c ON c.key = r.company_key LEFT JOIN agent_companies ac ON ac.company_key = r.company_key AND ac.agent_id = ? WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.company_key != '' AND COALESCE(ac.affinity, 0) != -1 GROUP BY r.company_key, r.company ORDER BY r.company_key, n DESC, r.company""", (agent_id,)) found: dict[str, tuple[list[str], list[int], int]] = {} for r in rows: names, counts, _ = found.setdefault( r["key"], ([r["own"]] if r["own"] else [], [], int(r["affinity"]))) if r["name"] and r["name"] not in names: names.append(r["name"]) counts.append(int(r["n"])) return [(key, names, sum(counts), affinity) for key, (names, counts, affinity) in found.items()] def open_places(self) -> list[tuple[str, str, int]]: """`(place, country, open roles)` for every place line an open role carries, what `places.suggest` reads. Copies are not counted, as in the feed.""" return [(str(r["place"]), str(r["country"]), int(r["n"])) for r in self._rows( "SELECT place, country, COUNT(*) AS n FROM roles " "WHERE closed_at IS NULL AND dup_of IS NULL AND place != '' " "GROUP BY place, country")] def open_countries_by_subscription(self) -> dict[int, dict[str, int]]: """Per subscription id, its open roles counted by `country`, copies included: what a board or a search holds, not what the feed lists.""" rows = self._rows("""SELECT subscription_id, country, COUNT(*) AS n FROM roles WHERE closed_at IS NULL AND subscription_id IS NOT NULL GROUP BY subscription_id, country""") out: dict[int, dict[str, int]] = {} for r in rows: out.setdefault(r["subscription_id"], {})[r["country"]] = r["n"] return out def company_boards(self) -> dict[str, str]: """`company_key -> ats host`, read off each company's own subscription. `Subscription.company` is the display name it and the company row were both added with, so `company_key` of one finds the other; a company with no subscription, or a detached one (`show` 0, see `set_company_board`), is simply absent.""" boards: dict[str, str] = {} for sub in self.subscriptions(source="ats"): if sub.show: boards.setdefault(company_key(sub.company), sub.board) return boards def company_subscription(self, key: str) -> Subscription | None: """Company `key`'s own `ats` subscription, detached or not; None when it never had one. A company has at most one: `set_company_board` rewrites it in place rather than adding a second.""" with closing(self._connect()) as conn: return self._company_subscription(conn, key) @staticmethod def _company_subscription(conn: sqlite3.Connection, key: str) -> Subscription | None: rows = conn.execute("SELECT * FROM subscriptions WHERE source = 'ats' " "ORDER BY show DESC, id").fetchall() return next((Subscription.from_row(r) for r in rows if company_key(r["company"]) == key), None) def set_company_board(self, key: str, target: str, *, host: str = "", name: str = "", fields: dict[str, Any] | None = None, ) -> tuple[Company, Subscription | None, int]: """Point company `key` at the `ats` board `target`, or at none when `target` is "", in one transaction, so a clash on either table leaves nothing half written. A missing company is created from `name` and `fields`; an existing one has `fields` written over it. Its subscription is rewritten in place: a new target closes the old target's open roles, as switching a board off does, and "" detaches it (off, hidden, its roles closed) while the row stays for the next board. Returns the company, its subscription (None when there is none) and how many roles closed. Raises KeyError for a missing company with no board, ValueError on a UNIQUE clash.""" fields = dict(fields or {}) unknown = set(fields) - set(_COMPANY_FIELDS) if unknown: raise ValueError(f"not a company field: {', '.join(sorted(unknown))}") stamp = self.now() closed = 0 try: with self.transaction() as conn: row = conn.execute("SELECT name FROM companies WHERE key=?", (key,)).fetchone() if row is None: name = fields.pop("name", "") or name.strip() if not target: raise KeyError(key) if not name: raise ValueError("a company needs a name") self._insert_company(conn, key, name, fields) else: name = fields.get("name") or row["name"] if fields: sets = ", ".join(f"{f}=?" for f in fields) conn.execute(f"UPDATE companies SET {sets}, updated_at=? WHERE key=?", (*fields.values(), stamp, key)) sub = self._company_subscription(conn, key) sub_id = sub.id if sub else None if sub is not None and sub.target != target: closed = conn.execute( "UPDATE roles SET closed_at = ? " "WHERE subscription_id = ? AND closed_at IS NULL", (stamp, sub.id)).rowcount if not target: if sub is not None: conn.execute("UPDATE subscriptions SET subscribed=0, fetch=0, show=0 " "WHERE id=?", (sub.id,)) elif sub is not None: conn.execute( """UPDATE subscriptions SET target=?, board=?, label=?, company=?, subscribed=1, fetch=1, show=1, target_id=CASE WHEN target=? THEN target_id ELSE '' END WHERE id=?""", (target, host, name, name, target, sub.id)) else: sub_id = conn.execute( """INSERT INTO subscriptions (source, target, label, company, board, params, added_at) VALUES ('ats',?,?,?,?,'{}',?)""", (target, name, name, host, stamp)).lastrowid except sqlite3.IntegrityError as exc: raise ValueError(f"another board already has {target!r} or the label " f"{name!r}") from exc return (self._company(key), self.subscription(sub_id) if sub_id else None, closed) # -------------------------------------------------------- subscriptions def subscriptions(self, source: str | None = None, fetchable: bool = False) -> list[Subscription]: """`fetchable` is what a pull reads: subscribed and with fetch on.""" where, params = [], [] if source: where.append("source = ?") params.append(source) if fetchable: where.append("fetch = 1 AND subscribed = 1") sql = "SELECT * FROM subscriptions" if where: sql += " WHERE " + " AND ".join(where) sql += " ORDER BY company, board, label" return [Subscription.from_row(r) for r in self._rows(sql, params)] def subscription(self, sub_id: int) -> Subscription | None: row = self._row("SELECT * FROM subscriptions WHERE id=?", (sub_id,)) return Subscription.from_row(row) if row else None def subscription_for(self, source: str, label: str) -> Subscription | None: """The row an item names: `Item.subscription` is the label.""" row = self._row("SELECT * FROM subscriptions WHERE source=? AND label=?", (source, label)) return Subscription.from_row(row) if row else None def add_subscription(self, source: str, target: str, label: str = "", company: str = "", board: str = "", params: dict[str, Any] | None = None, target_id: str = "") -> Subscription: """Raises ValueError on a duplicate target or label within one plugin.""" target = target.strip() if not target: raise ValueError("a subscription needs a target") label = label.strip() or target try: with self.transaction() as conn: new_id = conn.execute( """INSERT INTO subscriptions (source, target, label, company, board, params, added_at, target_id) VALUES (?,?,?,?,?,?,?,?)""", (source, target, label, company, board, json.dumps(params or {}), self.now(), target_id)).lastrowid except sqlite3.IntegrityError as exc: raise ValueError(f"{source} already has {target!r} or the label {label!r}") from exc assert new_id is not None return self._subscription(new_id) def update_subscription(self, sub_id: int, **fields: Any) -> Subscription: """Change labels, params, target_id or the three switches.""" unknown = set(fields) - set(_SUBSCRIPTION_FIELDS) if unknown: raise ValueError(f"not a subscription field: {', '.join(sorted(unknown))}") if "params" in fields: fields["params"] = json.dumps(fields["params"] or {}) for switch in ("subscribed", "fetch", "show"): if switch in fields: fields[switch] = int(bool(fields[switch])) if fields: sets = ", ".join(f"{f}=?" for f in fields) try: changed = self._write(f"UPDATE subscriptions SET {sets} WHERE id=?", (*fields.values(), sub_id)) except sqlite3.IntegrityError as exc: raise ValueError(f"label {fields.get('label')!r} is already in use") from exc if not changed: raise KeyError(sub_id) return self._subscription(sub_id) def unsubscribe(self, sub_id: int) -> Subscription: """Stop following it and hide it. The row stays, and so do its roles.""" return self.update_subscription(sub_id, subscribed=0, fetch=0, show=0) def _subscription(self, sub_id: int) -> Subscription: found = self.subscription(sub_id) if found is None: raise KeyError(sub_id) return found # --------------------------------------------------------- applications def save_application(self, application: Application) -> Application: """Insert or update one application. An empty `id` is minted. Entering a status, on insert or by change, appends an `application_events` row in the same transaction.""" _check_status(application.status) app_id = application.id or uuid4().hex stamp = self.now() fields = application.model_dump(exclude={"id", "created_at", "updated_at"}) with self.transaction() as conn: before = conn.execute("SELECT status FROM applications WHERE id=?", (app_id,)).fetchone() if before is None: columns = ["id", *fields, "created_at", "updated_at"] conn.execute( f"INSERT INTO applications ({', '.join(columns)}) " f"VALUES ({', '.join('?' for _ in columns)})", (app_id, *fields.values(), stamp, stamp)) else: sets = ", ".join(f"{f}=?" for f in fields) conn.execute(f"UPDATE applications SET {sets}, updated_at=? WHERE id=?", (*fields.values(), stamp, app_id)) if before is None or before["status"] != application.status: conn.execute("INSERT INTO application_events (application_id, status, at) " "VALUES (?,?,?)", (app_id, application.status, stamp)) return self._application(app_id) def set_application_status(self, app_id: str, status: str) -> Application: """Move one application to `status`. Raises ValueError for a status outside `STATUSES` and KeyError for an unknown id.""" _check_status(status) current = self._application(app_id) return self.save_application(current.model_copy(update={"status": status})) def application(self, app_id: str) -> Application | None: row = self._row("SELECT * FROM applications WHERE id=?", (app_id,)) return Application.from_row(row) if row else None def delete_application(self, app_id: str) -> bool: """Drop one application and its events. Whether that is allowed (only while it is still `saved`) is the caller's rule, not the store's.""" return self._write("DELETE FROM applications WHERE id=?", (app_id,)) > 0 def applications(self, status: str | None = None) -> list[Application]: sql, params = "SELECT * FROM applications", [] if status: sql += " WHERE status = ?" params.append(status) sql += " ORDER BY updated_at DESC, id" return [Application.from_row(r) for r in self._rows(sql, params)] def application_events(self, app_id: str) -> list[ApplicationEvent]: return [ApplicationEvent.from_row(r) for r in self._rows( "SELECT * FROM application_events WHERE application_id=? ORDER BY id", (app_id,))] def _application(self, app_id: str) -> Application: found = self.application(app_id) if found is None: raise KeyError(app_id) return found # ---------------------------------------------------------------- state @property def state(self) -> MutableMapping[str, str]: """The `state` table as a mapping. What the runner hands a plugin as `self.state`: it can keep a counter without being able to read roles.""" return _StateMap(self) def state_items(self, prefix: str) -> dict[str, str]: """Every `state` key that starts with `prefix`, with its value, in one query, for a reader that wants one kind of key across every target.""" return {str(r["key"]): str(r["value"]) for r in self._rows( "SELECT key, value FROM state WHERE substr(key, 1, ?) = ? ORDER BY key", (len(prefix), prefix))} def claim_state(self, key: str, value: str, *, released: str, stale: str) -> bool: """Set `key` to `value` only while it is free: unset, equal to the value under `released` (the claim last let go of), or no later than `stale`. Returns whether it was set. One statement, so of two processes claiming at once exactly one wins.""" return self._write( """INSERT INTO state (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value WHERE state.value <= ? OR state.value = COALESCE( (SELECT r.value FROM state AS r WHERE r.key = ?), '')""", (key, value, stale, released)) > 0 def _check_status(status: str) -> None: if status not in STATUSES: raise ValueError(f"{status!r} is not an application status " f"({', '.join(STATUSES)})") def _check_hand_reason(reason: Any) -> None: if reason not in HAND_REASONS: raise ValueError(f"hand_reason is one of {', '.join(repr(r) for r in HAND_REASONS)}, " f"not {reason!r}") def _check_affinity(affinity: Any) -> None: if isinstance(affinity, bool) or not isinstance(affinity, int) or affinity not in AFFINITY: raise ValueError(f"affinity is -1, 0, 1 or 2, not {affinity!r}")