Score roles per search agent, and run pull into extract, embed and score
embed stores one vector per open role; score rates every open role for every enabled agent from require and prefer gates, fit against the agent's intent, recency and company affinity, keeping gated rows with a reason. Schema v3 adds vectors and the cached intent vector. The page gets an agent order, the score on each row, and an Agents tab. Two starter agents in roles.yaml are imported once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4dfb8831aa
commit
bc60324fab
22 files changed
+1457
-44
No files matched your search
+207
-7
@@ -17,10 +17,13 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import numpy as np
|
||||
|
||||
from desk.models import Item
|
||||
from roledesk.models import (
|
||||
AFFINITY, REMOTE, STATUSES, Application, ApplicationEvent, Company, Role,
|
||||
Subscription, company_key, place_key, title_key,
|
||||
AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate,
|
||||
Company, Gate, Role, ScoredRole, Subscription, Weights, company_key, place_key,
|
||||
title_key,
|
||||
)
|
||||
from roledesk.storage import schema
|
||||
|
||||
@@ -73,6 +76,43 @@ def _like(text: str) -> str:
|
||||
return f"%{escaped}%"
|
||||
|
||||
|
||||
#: Vectors are stored little-endian float32, fixed here so a file written on
|
||||
#: one machine reads on another. newsdesk's encoding, written again.
|
||||
_DTYPE = np.dtype("<f4")
|
||||
|
||||
|
||||
def pack(vec: np.ndarray) -> bytes:
|
||||
return np.asarray(vec, dtype=_DTYPE).tobytes()
|
||||
|
||||
|
||||
def unpack(blob: bytes) -> np.ndarray:
|
||||
return np.frombuffer(blob, dtype=_DTYPE)
|
||||
|
||||
|
||||
def _gate_sql(gate: Gate) -> tuple[str, str, list[Any]]:
|
||||
"""One gate as `(empty, match, params)`: SQL that is true when the role's
|
||||
column is empty, and SQL that is true when it matches, over `roles r`.
|
||||
`params` belong to `match`; `empty` takes none."""
|
||||
column = f"r.{gate.field}"
|
||||
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
|
||||
marks = ", ".join("?" for _ in values) or "NULL"
|
||||
if gate.field == "salary_min":
|
||||
floor = min(_int_or_none(v) or 0 for v in gate.values)
|
||||
return f"{column} IS NULL", f"{column} >= ?", [floor]
|
||||
if gate.field == "place":
|
||||
likes = " OR ".join("LOWER(r.place) LIKE ? ESCAPE '\\'" for _ in values) or "0"
|
||||
return "r.place = ''", f"({likes})", [_like(v) for v in values]
|
||||
if gate.field == "company":
|
||||
keys = [company_key(v) for v in values]
|
||||
return ("r.company_key = ''",
|
||||
f"r.company_key IN ({', '.join('?' for _ in keys) or 'NULL'})", keys)
|
||||
if gate.field == "languages":
|
||||
return ("r.languages IN ('', '[]')",
|
||||
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
|
||||
f"WHERE LOWER(value) NOT IN ({marks}))", values)
|
||||
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
|
||||
|
||||
|
||||
def _int_or_none(value: Any) -> int | None:
|
||||
if value is None or value == "" or isinstance(value, bool):
|
||||
return None
|
||||
@@ -229,7 +269,8 @@ class Store:
|
||||
place: str | None = None, open_only: bool = True,
|
||||
q: str | None = None, function: str | None = None,
|
||||
remote: str | None = None, days: int | None = None,
|
||||
hide_muted: bool = False) -> list[Role]:
|
||||
hide_muted: bool = False, agent: int | None = None,
|
||||
gated: bool = False) -> list[ScoredRole]:
|
||||
"""Newest first by `first_seen`. `company` and `place` are
|
||||
case-insensitive substrings of the columns a board wrote — not the
|
||||
key, so a search does not have to know the exact slug a company folds
|
||||
@@ -243,7 +284,11 @@ class Store:
|
||||
finally reached it — can otherwise fill the whole page with roles
|
||||
that then all turn out to be from the one company someone muted,
|
||||
leaving nothing to show for a default query that would otherwise
|
||||
have plenty."""
|
||||
have plenty.
|
||||
|
||||
`agent` orders by that agent's score instead, highest first, and
|
||||
carries `score` and `gated_reason` on each row. Only roles it has
|
||||
scored are listed, and a gated one only when `gated` is set."""
|
||||
where, params = [], []
|
||||
if company:
|
||||
where.append("company LIKE ? ESCAPE '\\'")
|
||||
@@ -268,11 +313,22 @@ class Store:
|
||||
if hide_muted:
|
||||
where.append("company_key NOT IN "
|
||||
"(SELECT key FROM companies WHERE affinity = -1)")
|
||||
sql = "SELECT * FROM roles"
|
||||
joins: list[Any] = []
|
||||
if agent is None:
|
||||
sql = "SELECT roles.* FROM roles"
|
||||
order = "roles.first_seen DESC, roles.uid"
|
||||
else:
|
||||
sql = ("SELECT roles.*, s.score, s.gated_reason FROM roles "
|
||||
"JOIN scores s ON s.uid = roles.uid AND s.agent_id = ?")
|
||||
joins.append(agent)
|
||||
if not gated:
|
||||
where.append("s.gated_reason = ''")
|
||||
order = "s.score DESC, roles.first_seen DESC, roles.uid"
|
||||
if where:
|
||||
sql += " WHERE " + " AND ".join(where)
|
||||
sql += " ORDER BY first_seen DESC, uid LIMIT ?"
|
||||
return [Role.from_row(r) for r in self._rows(sql, [*params, limit])]
|
||||
sql += f" ORDER BY {order} LIMIT ?"
|
||||
return [ScoredRole.from_row(r)
|
||||
for r in self._rows(sql, [*joins, *params, limit])]
|
||||
|
||||
def mark_closed(self, source: str, seen_ids: Sequence[str],
|
||||
subscription_id: int) -> int:
|
||||
@@ -353,6 +409,150 @@ class Store:
|
||||
f"WHERE {where}", (wanted,))
|
||||
return int(crossed)
|
||||
|
||||
# -------------------------------------------------------------- vectors
|
||||
|
||||
def roles_to_embed(self, model: str, limit: int) -> list[Role]:
|
||||
"""Open roles with no vector from `model`, newest first, so a backlog
|
||||
embedded from the top improves what is on screen first."""
|
||||
return [Role.from_row(r) for r in self._rows(
|
||||
"""SELECT r.* FROM roles r
|
||||
LEFT JOIN vectors v ON v.uid = r.uid AND v.model = ?
|
||||
WHERE v.uid IS NULL AND r.closed_at IS NULL
|
||||
ORDER BY r.first_seen DESC, r.uid LIMIT ?""", (model, limit))]
|
||||
|
||||
def put_vectors(self, model: str, pairs: Sequence[tuple[str, np.ndarray]]) -> int:
|
||||
stamp = self.now()
|
||||
with self.transaction() as conn:
|
||||
return conn.executemany(
|
||||
"""INSERT INTO vectors (uid, model, dim, vec, embedded_at)
|
||||
VALUES (?,?,?,?,?)
|
||||
ON CONFLICT(uid) DO UPDATE SET model=excluded.model,
|
||||
dim=excluded.dim, vec=excluded.vec,
|
||||
embedded_at=excluded.embedded_at""",
|
||||
[(uid, model, int(vec.shape[-1]), pack(vec), stamp)
|
||||
for uid, vec in pairs]).rowcount
|
||||
|
||||
def vectors(self, model: str) -> dict[str, np.ndarray]:
|
||||
"""Every vector from `model`, by uid. The whole table: at ~30k roles of
|
||||
384 floats that is ~45 MB, and one read beats one per agent."""
|
||||
return {r["uid"]: unpack(r["vec"]) for r in self._rows(
|
||||
"SELECT uid, vec FROM vectors WHERE model = ?", (model,))}
|
||||
|
||||
def vector_counts(self) -> dict[str, int]:
|
||||
return {r["model"]: int(r["n"]) for r in self._rows(
|
||||
"SELECT model, COUNT(*) AS n FROM vectors GROUP BY model")}
|
||||
|
||||
def drop_vectors(self, keep: str) -> int:
|
||||
"""Delete vectors from every model but `keep`: another model's are a
|
||||
different geometry, never compared, and only make counts lie."""
|
||||
return self._write("DELETE FROM vectors WHERE model <> ?", (keep,))
|
||||
|
||||
# --------------------------------------------------------------- agents
|
||||
|
||||
def agents(self, enabled_only: bool = False) -> list[Agent]:
|
||||
sql = "SELECT * FROM agents"
|
||||
if enabled_only:
|
||||
sql += " WHERE enabled = 1"
|
||||
return [Agent.from_row(r)
|
||||
for r in self._rows(sql + " ORDER BY name COLLATE NOCASE")]
|
||||
|
||||
def agent(self, agent_id: int) -> Agent | None:
|
||||
row = self._row("SELECT * FROM agents WHERE id=?", (agent_id,))
|
||||
return Agent.from_row(row) if row else None
|
||||
|
||||
def save_agent(self, name: str, intent: str = "", gates: Sequence[Gate] = (),
|
||||
weights: Weights | None = None, enabled: bool = True,
|
||||
agent_id: int | None = None) -> Agent:
|
||||
"""Create an agent, or with `agent_id` replace that one's fields.
|
||||
Raises ValueError on an empty or taken name, KeyError on an unknown id."""
|
||||
name = name.strip()
|
||||
if not name:
|
||||
raise ValueError("an agent needs a name")
|
||||
fields = (name, intent, json.dumps([g.model_dump() for g in gates]),
|
||||
(weights or Weights()).model_dump_json(), int(enabled))
|
||||
try:
|
||||
with self.transaction() as conn:
|
||||
if agent_id is None:
|
||||
agent_id = conn.execute(
|
||||
"""INSERT INTO agents (name, intent, gates, weights, enabled,
|
||||
created_at) VALUES (?,?,?,?,?,?)""",
|
||||
(*fields, self.now())).lastrowid
|
||||
elif not conn.execute(
|
||||
"""UPDATE agents SET name=?, intent=?, gates=?, weights=?,
|
||||
enabled=? WHERE id=?""",
|
||||
(*fields, agent_id)).rowcount:
|
||||
raise KeyError(agent_id)
|
||||
except sqlite3.IntegrityError as exc:
|
||||
raise ValueError(f"an agent named {name!r} already exists") from exc
|
||||
assert agent_id is not None
|
||||
found = self.agent(agent_id)
|
||||
assert found is not None
|
||||
return found
|
||||
|
||||
def delete_agent(self, agent_id: int) -> bool:
|
||||
"""Its scores go with it, by the foreign key."""
|
||||
return self._write("DELETE FROM agents WHERE id=?", (agent_id,)) > 0
|
||||
|
||||
def intent_vector(self, agent_id: int, key: str) -> np.ndarray | None:
|
||||
"""The agent's cached intent vector, if it was made under `key`."""
|
||||
row = self._row("SELECT intent_key, intent_vec FROM agents WHERE id=?",
|
||||
(agent_id,))
|
||||
if row is None or row["intent_key"] != key or row["intent_vec"] is None:
|
||||
return None
|
||||
return unpack(row["intent_vec"])
|
||||
|
||||
def set_intent_vector(self, agent_id: int, key: str, vec: np.ndarray) -> None:
|
||||
self._write("UPDATE agents SET intent_key=?, intent_vec=? WHERE id=?",
|
||||
(key, pack(vec), agent_id))
|
||||
|
||||
# --------------------------------------------------------------- scores
|
||||
|
||||
def score_candidates(self, gates: Sequence[Gate]) -> list[Candidate]:
|
||||
"""Every open role of a company that is not muted, with how it meets
|
||||
each gate: 1 match, 0 no match, -1 the column is empty. The gates are
|
||||
evaluated here, in SQL, so a `require` gate is the `WHERE` it reads
|
||||
as, written as a column so a gated role keeps its row and reason."""
|
||||
columns: list[str] = []
|
||||
params: list[Any] = []
|
||||
for n, gate in enumerate(gates):
|
||||
empty, match, values = _gate_sql(gate)
|
||||
columns.append(
|
||||
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
|
||||
params += values
|
||||
select = ", ".join(["r.uid", "r.first_seen",
|
||||
"COALESCE(c.affinity, 0) AS affinity", *columns])
|
||||
rows = self._rows(
|
||||
f"""SELECT {select} FROM roles r
|
||||
LEFT JOIN companies c ON c.key = r.company_key
|
||||
WHERE r.closed_at IS NULL AND COALESCE(c.affinity, 0) <> -1""",
|
||||
params)
|
||||
return [Candidate(uid=r["uid"], first_seen=r["first_seen"],
|
||||
affinity=int(r["affinity"]),
|
||||
gates=[int(r[f"g{n}"]) for n in range(len(gates))])
|
||||
for r in rows]
|
||||
|
||||
def replace_scores(self, agent_id: int,
|
||||
rows: Sequence[tuple[str, float, str]]) -> int:
|
||||
"""`(uid, score, gated_reason)` for every role this agent scored,
|
||||
replacing whatever it had: a role closed or muted since the last run
|
||||
drops out rather than keeping a stale score. One transaction, so a
|
||||
reader sees the old list or the new one."""
|
||||
stamp = self.now()
|
||||
with self.transaction() as conn:
|
||||
conn.execute("DELETE FROM scores WHERE agent_id=?", (agent_id,))
|
||||
conn.executemany(
|
||||
"""INSERT INTO scores (uid, agent_id, score, gated_reason, scored_at)
|
||||
VALUES (?,?,?,?,?)""",
|
||||
[(uid, agent_id, score, reason, stamp) for uid, score, reason in rows])
|
||||
return len(rows)
|
||||
|
||||
def scores(self, agent_id: int) -> dict[str, tuple[float, str]]:
|
||||
"""`uid -> (score, gated_reason)` for one agent."""
|
||||
return {r["uid"]: (float(r["score"]), str(r["gated_reason"]))
|
||||
for r in self._rows(
|
||||
"SELECT uid, score, gated_reason FROM scores WHERE agent_id=?",
|
||||
(agent_id,))}
|
||||
|
||||
# ------------------------------------------------------------ companies
|
||||
|
||||
def add_company(self, name: str, *, key: str = "", domain: str = "",
|
||||
|
||||
Reference in new issue
Block a user