diff --git a/app/cli.py b/app/cli.py index beda7f5..2b33b21 100644 --- a/app/cli.py +++ b/app/cli.py @@ -5,6 +5,7 @@ app worker [--workers http=8,cpu=1] [--until-idle] app companies seed|add|set-board|detach|discover|check app searches add|list|remove + app agents list|prompt app stages [extract|summarise on|off] app db init @@ -110,6 +111,17 @@ def build_parser() -> argparse.ArgumentParser: "remove", help="stop running a search and close its open roles") search_remove.add_argument("name") + agents = sub.add_parser("agents", help="search agents and their prompts") + agents_sub = agents.add_subparsers(dest="action", required=True) + agents_sub.add_parser("list", help="every agent and its prompts") + prompt = agents_sub.add_parser( + "prompt", help="add a prompt to an agent, replace its text, or --remove it") + prompt.add_argument("agent", help="the agent's name") + prompt.add_argument("name", help='the prompt\'s name, e.g. "AI leadership"') + prompt.add_argument("text", nargs="?", default="", + help="what the agent looks for, in a sentence or a paragraph") + prompt.add_argument("--remove", action="store_true", help="remove the prompt instead") + stages = sub.add_parser( "stages", help="show whether extract and summarise run, or switch one") stages.add_argument("name", nargs="?", choices=["extract", "summarise"]) @@ -135,6 +147,8 @@ def main(argv: list[str] | None = None) -> int: return run_companies(args) if args.command == "searches": return run_searches(args) + if args.command == "agents": + return run_agents(args) if args.command == "stages": return run_stages(args) return init_db(args) @@ -420,6 +434,53 @@ def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace return 0 +def run_agents(args: argparse.Namespace) -> int: + """`app agents list` prints every agent and its prompts. `app agents + prompt "Wolf, next role" "AI leadership" "Head of AI ..."` adds that + prompt at the end, or replaces the text of the one of that name in its + place; `--remove` removes it. The agent's other fields stay as they are. + Scores follow at the next `score`.""" + from app.models import Prompt + from app.storage import Store + + config = config_mod.load(args.config) + store = Store(config.database) + if args.action == "list": + for each in store.agents(): + print(each.name + ("" if each.enabled else " (off)")) + for p in each.prompts: + text = " ".join(p.text.split()) + print(f" {p.name}: {text[:100] + '…' if len(text) > 100 else text}") + return 0 + agent = next((a for a in store.agents() if a.name == args.agent), None) + if agent is None: + print(f"agents prompt: no agent named {args.agent!r}", file=sys.stderr) + return 1 + named = args.name.strip().casefold() + prompts = [p for p in agent.prompts if not (args.remove and p.name.casefold() == named)] + if args.remove and len(prompts) == len(agent.prompts): + print(f"agents prompt: {agent.name!r} has no prompt named {args.name!r}", + file=sys.stderr) + return 1 + if not args.remove: + try: + new = Prompt(name=args.name, text=args.text) + except ValueError as exc: + print(f"agents prompt: {exc}", file=sys.stderr) + return 1 + at = next((n for n, p in enumerate(prompts) if p.name.casefold() == named), None) + if at is None: + prompts.append(new) + else: + prompts[at] = new + saved = store.save_agent(agent.name, gates=agent.gates, weights=agent.weights, + enabled=agent.enabled, agent_id=agent.id, limits=agent.limits, + prompts=prompts) + print(json.dumps({"agent": saved.name, "prompts": [p.name for p in saved.prompts]}, + ensure_ascii=False)) + return 0 + + def run_stages(args: argparse.Namespace) -> int: """`app stages` prints what `GET /api/stages` returns; `app stages extract on` stores the switch through the function `PUT /api/stages/extract` diff --git a/app/config.py b/app/config.py index 8083766..b11835d 100644 --- a/app/config.py +++ b/app/config.py @@ -22,7 +22,7 @@ from typing import Any, Literal import yaml from pydantic import Field, ValidationError, field_validator, model_validator -from app.models import Gate, Limits, Weights +from app.models import Gate, Limits, Prompts, Weights from auth import AuthConfig from desk.config import AgentConfig, ModelTaskConfig, Section, WorkersConfig @@ -129,9 +129,12 @@ class EmbedConfig(Section): class AgentSeed(Section): """A search agent as `config.yaml` lists it. Imported into an empty - `agents` table once; after that the table is the answer.""" + `agents` table once; after that the table is the answer. `prompts` is a + list of `{name, text}`; without it, `intent` is the one prompt, named + after the agent.""" name: str + prompts: Prompts | None = None intent: str = "" gates: list[Gate] = Field(default_factory=list) weights: Weights = Field(default_factory=Weights) diff --git a/app/jobs/score.py b/app/jobs/score.py index f031092..cc0171c 100644 --- a/app/jobs/score.py +++ b/app/jobs/score.py @@ -14,10 +14,11 @@ A gate rejects on evidence, never on absence: a role whose column is empty is not gated by it. `extract` is what fills an empty column, and it reads only the roles that rank, so gating on an empty column would keep those roles out of extract's reach for good. - fit cos(role vector, intent vector) as a level: the embedding + fit cos(role vector, prompt vector) as a level: the embedding model's calibrated `lo` is 0 and `hi` 1, clipped outside (see `scale`), so a role's fit does not depend on what - else is open + else is open; with several prompts, the best prompt's + level, and the score names that prompt recency 0.5 ** (days since posted_at / 14) affinity (the agent's affinity for the company + 1) / 3 taste cos(role vector, liked direction), scaled like fit with the @@ -32,10 +33,10 @@ company (`affinity = 2`) skips the gates and is floored at 0.9; a hidden one (`-1`) is not scored at all. A role with no vector is scored without `fit` and `taste`, unless `fit` is all there is, when it is gated as "no vector". An agent with no votes scores every role without `taste`; the direction is -newsdesk's `centroid(liked) - centroid(disliked)`. An agent with no intent -has no `fit` either: its scores are not a match, so it is left out of what -extract and summarise read (`Store._top_scope`), and a run names such agents -in one notice. +newsdesk's `centroid(liked) - centroid(disliked)`. An agent with no prompt +that has text has no `fit` either: its scores are not a match, so it is left +out of what extract and summarise read (`Store._top_scope`), and a run names +such agents in one notice. Recomputed in full on every run, and each agent's rows are replaced in one transaction: nothing is incremental, so nothing can drift. @@ -176,7 +177,7 @@ def score_one(candidate: Candidate, gates: Sequence[Gate], weights: Weights, missing: str = "no vector") -> tuple[float, str]: """One role for one agent: `(score, gated_reason)`. `fit` and `taste` are already scaled to 0..1 (see `scale`), or None when there is nothing to - measure: no role vector, no intent vector, no votes. A None term leaves + measure: no role vector, no prompt vector, no votes. A None term leaves the denominator; `missing` says why fit is out.""" favourite = candidate.affinity == 2 if failed := failed_gates(candidate, gates): @@ -207,8 +208,8 @@ def intent_key(model: str, intent: str) -> str: class _Encoder: - """The embedder, loaded on first use only: an agent whose intent vector - is cached costs no model load at all.""" + """The embedder, loaded on first use only: an agent whose prompt vectors + are cached costs no model load at all.""" def __init__(self, model: str, context: RunContext) -> None: self.model = model @@ -229,66 +230,85 @@ class _Encoder: return np.asarray(self._embedder.encode([text])[0]) -def _intent(store: Store, agent: Agent, model: str, - encoder: _Encoder) -> np.ndarray | None: - if not agent.intent.strip(): - return None - key = intent_key(model, agent.intent) - cached = store.intent_vector(agent.id, key) - if cached is not None: - return cached - vec = encoder.encode(embed.intent_text(model, agent.intent)) - if vec is not None: - store.set_intent_vector(agent.id, key, vec) - return vec +def _prompts(store: Store, agent: Agent, model: str, + encoder: _Encoder) -> list[tuple[str, np.ndarray]]: + """`(name, vector)` for each of the agent's prompts that has text and + could be embedded, in order. Each vector is cached under its own key, so + an edit re-embeds only the prompt it changed.""" + found = [] + for prompt in agent.prompts: + if not prompt.text.strip(): + continue + key = intent_key(model, prompt.text) + vec = store.prompt_vector(agent.id, prompt.name, key) + if vec is None: + vec = encoder.encode(embed.intent_text(model, prompt.text)) + if vec is None: + continue + store.set_prompt_vector(agent.id, prompt.name, key, vec) + found.append((prompt.name, vec)) + return found # ------------------------------------------------------------------- the job def no_intent(names: Sequence[str]) -> Notice: - """The notice for enabled agents without an intent: their scores have no - fit, so `Store._top_scope` keeps them from feeding extract and summarise.""" + """The notice for enabled agents without a prompt that has text: their + scores have no fit, so `Store._top_scope` keeps them from feeding extract + and summarise. The code keeps its name from when an agent had one intent.""" return Notice( code="score.no_intent", level="warning", kind="expected", key="score", - text=f"No intent, so ranked without fit and not read by extract or " - f"summarise: {', '.join(names)}.", - fix="Give the agent an intent under Settings, Search agents.", + text=f"No prompt with text, so ranked without fit and not read by extract " + f"or summarise: {', '.join(names)}.", + fix="Give the agent a prompt under Settings, Search agents.", where="app.jobs.score") def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray], - intent: np.ndarray | None, now: datetime, + prompts: Sequence[tuple[str, np.ndarray]], now: datetime, calibration: Calibration | None = None) -> dict[str, int]: - """Two passes: the cosines of the roles this agent scores, to its intent - and to its taste direction, then every score with both scaled by the - model's `calibration`, or across the run without one. A gated role is - left out of the cosines, so under the fallback a role that fails a - require gate cannot stretch the range the ranked roles share. The voted - roles are scored like any other; a liked one sits near the direction it - helped make, which is what a like should do.""" + """Two passes: the cosines of the roles this agent scores, to each of its + `prompts` and to its taste direction, then every score with both scaled + by the model's `calibration`, or across the run without one. A role's fit + is the best of its prompts' levels, each scaled on its own, and the score + keeps that prompt's name; the first prompt wins a tie. Under the min-max + fallback each prompt is ranged over the run on its own too, so every + prompt's closest role gets 1 and one prompt equals the old single intent. + A gated role is left out of the cosines, so under the fallback a role + that fails a require gate cannot stretch the range the ranked roles + share. The voted roles are scored like any other; a liked one sits near + the direction it helped make, which is what a like should do.""" candidates = store.score_candidates(agent.id, agent.gates) direction = taste_direction(vectors, store.votes(agent.id)) - cosines: dict[str, float] = {} + cosines: dict[str, dict[str, float]] = {name: {} for name, _ in prompts} tastes: dict[str, float] = {} for candidate in candidates: role_vec = vectors.get(candidate.uid) if role_vec is None or failed_gates(candidate, agent.gates): continue - if intent is not None: - cosines[candidate.uid] = cosine(role_vec, intent) + for name, prompt_vec in prompts: + cosines[name][candidate.uid] = cosine(role_vec, prompt_vec) if direction is not None: tastes[candidate.uid] = cosine(role_vec, direction) - fits = scale(cosines, calibration.fit if calibration else None) + fits: dict[str, float] = {} + best: dict[str, str] = {} + for name, levels in cosines.items(): + for uid, value in scale(levels, calibration.fit if calibration else None).items(): + if uid not in fits or value > fits[uid]: + fits[uid], best[uid] = value, name tastes = scale(tastes, calibration.taste if calibration else None) - rows: list[tuple[str, float, str]] = [] + rows: list[tuple[str, float, str, str]] = [] gated = 0 for candidate in candidates: + fit = fits.get(candidate.uid) value, reason = score_one( - candidate, agent.gates, agent.weights, fits.get(candidate.uid), now, + candidate, agent.gates, agent.weights, fit, now, taste=tastes.get(candidate.uid), - missing="no vector" if intent is not None else "no intent vector") + missing="no vector" if prompts else "no prompt vector") gated += bool(reason) - rows.append((candidate.uid, round(value, 6), reason)) + counted = fit is not None and agent.weights.fit and not reason + rows.append((candidate.uid, round(value, 6), reason, + best[candidate.uid] if counted else "")) store.replace_scores(agent.id, rows) return {"scored": len(rows), "gated": gated} @@ -307,14 +327,15 @@ def run_score(config: Config, store: Store, *, log.warning("no calibration for %s: fit and taste are min-max scaled across " "each agent's roles in this run, so they rank rather than state a " "level; add it under embed.calibration", model) - if blind := [agent.name for agent in agents if not agent.intent.strip()]: + if blind := [agent.name for agent in agents + if not any(p.text.strip() for p in agent.prompts)]: context.notice(no_intent(blind)) scored = gated = 0 for n, agent in enumerate(agents): context.check() context.progress(n, len(agents), f"scoring for {agent.name}, {n + 1} of {len(agents)}") - counts = score_agent(store, agent, vectors, _intent(store, agent, model, encoder), now, - calibration) + counts = score_agent(store, agent, vectors, _prompts(store, agent, model, encoder), + now, calibration) scored += counts["scored"] gated += counts["gated"] log.info("%-20s %6d scored, %6d gated", agent.name, counts["scored"], counts["gated"]) @@ -335,7 +356,7 @@ def seed_agents(config: Config, store: Store) -> list[str]: for seed in config.agents: try: store.save_agent(seed.name, seed.intent, seed.gates, seed.weights, - seed.enabled, limits=seed.limits) + seed.enabled, limits=seed.limits, prompts=seed.prompts) except ValueError as exc: log.warning("agents: %s", exc) continue diff --git a/app/models.py b/app/models.py index e06a986..04219c2 100644 --- a/app/models.py +++ b/app/models.py @@ -15,7 +15,7 @@ import re import unicodedata from typing import Annotated, Any, Literal, get_args -from pydantic import BeforeValidator, Field, computed_field +from pydantic import AfterValidator, BeforeValidator, Field, computed_field, field_validator from desk.models import JsonDict, Model, ModelTaskResult @@ -828,6 +828,9 @@ class ScoredRole(Role): score: float | None = None gated_reason: str = "" + #: The agent's prompt that gave the role its fit; "" when fit did not + #: count (gated, no vector, no prompt with text). + prompt: str = "" #: How many characters `RoleRow.excerpt` keeps at most: the feed row's own @@ -881,6 +884,7 @@ class RoleRow(Model): posted_at: str = "" score: float | None = None gated_reason: str = "" + prompt: str = "" vote: Literal[-1, 0, 1] | None = None voted_at: str | None = None read_at: str | None = None @@ -1016,13 +1020,47 @@ class Limits(Model): summarise_min: float = Field(0.65, ge=0, le=1) +class Prompt(Model): + """One of an agent's descriptions of what it looks for. Each is embedded + on its own and a role's fit is its best prompt's (see `app.jobs.score`), + so one agent can hunt several kinds of role without blending them. A + prompt with no text has no fit to give.""" + + name: str + text: str = "" + + @field_validator("name") + @classmethod + def _named(cls, name: str) -> str: + name = name.strip() + if not name: + raise ValueError("a prompt needs a name") + return name + + +def check_prompts(prompts: list[Prompt]) -> list[Prompt]: + """The prompts as they are, or ValueError when two share a name + regardless of case.""" + seen: set[str] = set() + for prompt in prompts: + if prompt.name.casefold() in seen: + raise ValueError(f"two prompts are named {prompt.name!r}") + seen.add(prompt.name.casefold()) + return prompts + + +#: An agent's prompts, in order, their names unique regardless of case. +Prompts = Annotated[list[Prompt], AfterValidator(check_prompts)] + + class Agent(Model): - """A row of `agents`: a named search with its own intent, gates, weights - and limits. The cached intent vector is a column this does not declare.""" + """A row of `agents`: a named search with its own prompts, gates, weights + and limits. The prompts are rows of `agent_prompts`, read alongside; their + cached vectors are columns this does not declare.""" id: int name: str - intent: str = "" + prompts: Prompts = [] gates: Annotated[list[Gate], BeforeValidator(lambda v: _json(v, "[]"))] = [] weights: Annotated[Weights, BeforeValidator(lambda v: _json(v, "{}"))] = Weights() limits: Annotated[Limits, BeforeValidator(lambda v: _json(v, "{}"))] = Limits() diff --git a/app/storage/schema.py b/app/storage/schema.py index 10d4185..cb887da 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -177,25 +177,34 @@ BEGIN SELECT RAISE(ABORT, 'application_events is append-only'); END; --- A search agent: a named saved search with its own intent text, gates and +-- A search agent: a named saved search with its own prompts, gates and -- weights. gates is a list of {field, values, mode: require|prefer, weight}. --- intent_vec caches the intent's vector; intent_key is sha1 of the embedding --- model and the intent text, so editing either makes it stale. limits is how --- deep into this agent's ranking extract and summarise read, as JSON: a key --- it lacks means the `Limits` default. +-- limits is how deep into this agent's ranking extract and summarise read, +-- as JSON: a key it lacks means the `Limits` default. CREATE TABLE IF NOT EXISTS agents ( id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, - intent TEXT NOT NULL DEFAULT '', gates TEXT NOT NULL DEFAULT '[]', weights TEXT NOT NULL DEFAULT '{}', enabled INTEGER NOT NULL DEFAULT 1, created_at TEXT NOT NULL, - intent_key TEXT NOT NULL DEFAULT '', - intent_vec BLOB, limits TEXT NOT NULL DEFAULT '{}' ); +-- What an agent looks for, as several named texts in order; a role's fit is +-- its best prompt's. vec caches the text's vector and vec_key is sha1 of the +-- embedding model and the text, so editing one prompt re-embeds that one. +CREATE TABLE IF NOT EXISTS agent_prompts ( + agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE, + position INTEGER NOT NULL, + name TEXT NOT NULL, + text TEXT NOT NULL DEFAULT '', + vec_key TEXT NOT NULL DEFAULT '', + vec BLOB, + PRIMARY KEY (agent_id, position), + UNIQUE (agent_id, name COLLATE NOCASE) +); + -- One agent's stance on a company: -1 hidden, 0 neutral, 1 liked, 2 -- favourite. No row is 0, so the table holds only what somebody set. Per -- agent, because a company that is noise for one search is the target of @@ -220,12 +229,14 @@ CREATE TABLE IF NOT EXISTS role_votes ( ); -- A role is scored per agent. A gated role keeps its row, with the reason. +-- prompt names the agent's prompt that gave the fit, '' when fit did not count. CREATE TABLE IF NOT EXISTS scores ( uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE, score REAL NOT NULL DEFAULT 0, gated_reason TEXT NOT NULL DEFAULT '', scored_at TEXT NOT NULL, + prompt TEXT NOT NULL DEFAULT '', PRIMARY KEY (uid, agent_id) ); @@ -388,6 +399,31 @@ ALTER TABLE roles ADD COLUMN salary_text TEXT NOT NULL DEFAULT ''; """), (13, "roles.country recomputed from place for every not-yet-extracted role whose " "board gave none, country lookup fix", lambda conn: _recompute_country(conn)), + # An agent's one intent becomes its first prompt, named after the agent, + # with its cached vector: the key hashes the model and the text, which + # are unchanged, so nothing is embedded again. An empty intent becomes no + # prompt. The three columns are then dropped, as v6 dropped the ones it + # moved. + (14, "agent_prompts and scores.prompt: an agent holds several prompts, and a " + "score names the one that gave the fit", """ +CREATE TABLE agent_prompts ( + agent_id INTEGER NOT NULL REFERENCES agents(id) ON DELETE CASCADE, + position INTEGER NOT NULL, + name TEXT NOT NULL, + text TEXT NOT NULL DEFAULT '', + vec_key TEXT NOT NULL DEFAULT '', + vec BLOB, + PRIMARY KEY (agent_id, position), + UNIQUE (agent_id, name COLLATE NOCASE) +); +INSERT INTO agent_prompts (agent_id, position, name, text, vec_key, vec) +SELECT id, 0, name, intent, intent_key, intent_vec FROM agents + WHERE TRIM(intent, ' ' || char(9, 10, 13)) <> ''; +ALTER TABLE agents DROP COLUMN intent; +ALTER TABLE agents DROP COLUMN intent_key; +ALTER TABLE agents DROP COLUMN intent_vec; +ALTER TABLE scores ADD COLUMN prompt TEXT NOT NULL DEFAULT ''; +"""), ) #: What a file created from `SCHEMA` today is already at. diff --git a/app/storage/store.py b/app/storage/store.py index bf597e4..1f7d22d 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -22,8 +22,9 @@ import numpy as np from desk.models import Item from app.models import ( AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate, - Company, Gate, Limits, Role, RoleRow, ScoredRole, Subscription, Weights, company_key, - country_of, excerpt, place_key, posted_at, posted_host, remote_of, title_key, + Company, Gate, Limits, Prompt, Role, RoleRow, ScoredRole, Subscription, Weights, + check_prompts, company_key, country_of, excerpt, place_key, posted_at, posted_host, + remote_of, title_key, ) from app.storage import schema @@ -369,14 +370,20 @@ class Store: f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row) return not existed - def role(self, uid: str, agent_id: int | None = None) -> Role | None: - """One role with its tags, and with `vote` and `voted_at` as - `agent_id` judged it; None (unjudged) without an agent.""" - row = self._row(f"SELECT roles.*, v.vote, v.voted_at FROM roles {_WITH_VOTES} " - "WHERE roles.uid=?", (agent_id, uid)) + def role(self, uid: str, agent_id: int | None = None) -> ScoredRole | None: + """One role with its tags, and with `vote`, `voted_at`, `score`, + `gated_reason` and `prompt` as `agent_id` judged and scored it; None + (unjudged, unscored) without an agent. The score is moved by the vote + as `roles` serves it.""" + row = self._row( + f"SELECT roles.*, v.vote, v.voted_at, {_VOTED_SCORE} AS score, " + "COALESCE(s.gated_reason, '') AS gated_reason, COALESCE(s.prompt, '') AS prompt " + f"FROM roles {_WITH_VOTES} " + "LEFT JOIN scores s ON s.uid = roles.uid AND s.agent_id = ? WHERE roles.uid=?", + (agent_id, agent_id, uid)) if row is None: return None - role = Role.from_row(row) + role = ScoredRole.from_row(row) role.tags = self._tags_of(uid) return role @@ -397,7 +404,7 @@ class Store: assert role is not None return role - def set_vote(self, agent_id: int, uid: str, value: int | None) -> Role: + def set_vote(self, agent_id: int, uid: str, value: int | None) -> ScoredRole: """One agent's judgement of one role: 1 like, 0 neither way, -1 pass, None unjudged again, which deletes the row. A vote never hides a role; it moves it in that agent's ranking (see `roles`) and shapes its @@ -532,7 +539,7 @@ class Store: order = "roles.posted_at DESC, roles.uid" scored = agent is not None and by_score if scored: - select += f", {_VOTED_SCORE} AS score, s.gated_reason" + select += f", {_VOTED_SCORE} AS score, s.gated_reason, s.prompt" order = "score DESC, " + order elif open_only: # Newest first, open only: idx_roles_posted covers this exactly @@ -584,7 +591,7 @@ class Store: order = "roles.posted_at DESC, roles.uid" scored = agent is not None and by_score if scored: - select += f", {_VOTED_SCORE} AS score, s.gated_reason" + select += f", {_VOTED_SCORE} AS score, s.gated_reason, s.prompt" order = "score DESC, " + order elif open_only: # Same reasoning as `roles`: the planner otherwise sorts every @@ -763,8 +770,9 @@ class Store: held to the limits of the agent that ranked it; a key the JSON lacks is the `Limits` default. The role is `r`. A company an agent hid needs no clause: `score` writes no rows for it under that agent. An - agent without an intent counts for nothing here: its scores have no - fit, so they say how fresh a role is, not whether it is wanted.""" + agent without a prompt with text counts for nothing here: its scores + have no fit, so they say how fresh a role is, not whether it is + wanted.""" defaults = Limits() return (f"""r.uid IN (SELECT uid FROM ( SELECT s.uid, s.score, a.limits, ROW_NUMBER() OVER ( @@ -772,7 +780,9 @@ class Store: ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n FROM scores s JOIN agents a ON a.id = s.agent_id AND a.enabled = 1 - AND TRIM(a.intent, ' ' || char(9, 10, 13)) <> '' + AND EXISTS (SELECT 1 FROM agent_prompts p + WHERE p.agent_id = a.id + AND TRIM(p.text, ' ' || char(9, 10, 13)) <> '') JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL WHERE s.gated_reason = '') WHERE n <= COALESCE(json_extract(limits, '$.{stage}_top'), ?) @@ -1000,42 +1010,71 @@ class Store: sql = "SELECT * FROM agents" if enabled_only: sql += " WHERE enabled = 1" - return [Agent.from_row(r) - for r in self._rows(sql + " ORDER BY name COLLATE NOCASE")] + return self._agents(self._rows(sql + " ORDER BY name COLLATE NOCASE")) def agent(self, agent_id: int) -> Agent | None: - row = self._row("SELECT * FROM agents WHERE id=?", (agent_id,)) - return Agent.from_row(row) if row else None + found = self._agents(self._rows("SELECT * FROM agents WHERE id=?", (agent_id,))) + return found[0] if found else None def default_agent(self) -> Agent | None: """The agent a vote or an affinity goes to when none is named: the enabled one with the lowest id, else the lowest id, else None.""" - row = self._row("SELECT * FROM agents ORDER BY enabled DESC, id LIMIT 1") - return Agent.from_row(row) if row else None + found = self._agents(self._rows( + "SELECT * FROM agents ORDER BY enabled DESC, id LIMIT 1")) + return found[0] if found else None + + def _agents(self, rows: Sequence[sqlite3.Row]) -> list[Agent]: + """The rows as agents, each with its prompts in order, read with one + query for them all.""" + prompts: dict[int, list[dict[str, str]]] = {} + for p in self._rows( + "SELECT agent_id, name, text FROM agent_prompts WHERE agent_id IN " + "(SELECT value FROM json_each(?)) ORDER BY agent_id, position", + (json.dumps([r["id"] for r in rows]),)): + prompts.setdefault(p["agent_id"], []).append({"name": p["name"], "text": p["text"]}) + return [Agent.from_row({**dict(r), "prompts": prompts.get(r["id"], [])}) + for r in rows] def save_agent(self, name: str, intent: str = "", gates: Sequence[Gate] = (), weights: Weights | None = None, enabled: bool = True, - agent_id: int | None = None, limits: Limits | None = None) -> Agent: - """Create an agent, or with `agent_id` replace that one's fields. - Raises ValueError on an empty or taken name, KeyError on an unknown id.""" + agent_id: int | None = None, limits: Limits | None = None, + prompts: Sequence[Prompt] | None = None) -> Agent: + """Create an agent, or with `agent_id` replace that one's fields and + prompts. Without `prompts`, a non-empty `intent` is its one prompt, + named after the agent. A prompt whose text is unchanged keeps its + cached vector, whatever its name or place, so an edit re-embeds only + the prompts it changed. Raises ValueError on an empty or taken name, + or two prompts of one name, KeyError on an unknown id.""" name = name.strip() if not name: raise ValueError("an agent needs a name") - fields = (name, intent, json.dumps([g.model_dump() for g in gates]), + if prompts is None: + prompts = [Prompt(name=name, text=intent)] if intent.strip() else [] + prompts = check_prompts(list(prompts)) + fields = (name, json.dumps([g.model_dump() for g in gates]), (weights or Weights()).model_dump_json(), (limits or Limits()).model_dump_json(), int(enabled)) try: with self.transaction() as conn: if agent_id is None: agent_id = conn.execute( - """INSERT INTO agents (name, intent, gates, weights, limits, - enabled, created_at) VALUES (?,?,?,?,?,?,?)""", + """INSERT INTO agents (name, gates, weights, limits, enabled, + created_at) VALUES (?,?,?,?,?,?)""", (*fields, self.now())).lastrowid elif not conn.execute( - """UPDATE agents SET name=?, intent=?, gates=?, weights=?, - limits=?, enabled=? WHERE id=?""", + """UPDATE agents SET name=?, gates=?, weights=?, limits=?, enabled=? + WHERE id=?""", (*fields, agent_id)).rowcount: raise KeyError(agent_id) + cached = {r["text"]: (r["vec_key"], r["vec"]) for r in conn.execute( + "SELECT text, vec_key, vec FROM agent_prompts WHERE agent_id=?", + (agent_id,))} + conn.execute("DELETE FROM agent_prompts WHERE agent_id=?", (agent_id,)) + conn.executemany( + """INSERT INTO agent_prompts (agent_id, position, name, text, vec_key, vec) + VALUES (?,?,?,?,?,?)""", + [(agent_id, n, p.name, p.text, *cached.get(p.text, ("", None))) + for n, p in enumerate(prompts)]) except sqlite3.IntegrityError as exc: raise ValueError(f"an agent named {name!r} already exists") from exc assert agent_id is not None @@ -1044,20 +1083,22 @@ class Store: return found def delete_agent(self, agent_id: int) -> bool: - """Its scores, votes and affinities go with it, by the foreign keys.""" + """Its prompts, scores, votes and affinities go with it, by the + foreign keys.""" return self._write("DELETE FROM agents WHERE id=?", (agent_id,)) > 0 - def intent_vector(self, agent_id: int, key: str) -> np.ndarray | None: - """The agent's cached intent vector, if it was made under `key`.""" - row = self._row("SELECT intent_key, intent_vec FROM agents WHERE id=?", - (agent_id,)) - if row is None or row["intent_key"] != key or row["intent_vec"] is None: + def prompt_vector(self, agent_id: int, name: str, key: str) -> np.ndarray | None: + """The cached vector of the agent's prompt `name`, if it was made + under `key`.""" + row = self._row("SELECT vec_key, vec FROM agent_prompts WHERE agent_id=? AND name=?", + (agent_id, name)) + if row is None or row["vec_key"] != key or row["vec"] is None: return None - return unpack(row["intent_vec"]) + return unpack(row["vec"]) - def set_intent_vector(self, agent_id: int, key: str, vec: np.ndarray) -> None: - self._write("UPDATE agents SET intent_key=?, intent_vec=? WHERE id=?", - (key, pack(vec), agent_id)) + def set_prompt_vector(self, agent_id: int, name: str, key: str, vec: np.ndarray) -> None: + self._write("UPDATE agent_prompts SET vec_key=?, vec=? WHERE agent_id=? AND name=?", + (key, pack(vec), agent_id, name)) # --------------------------------------------------------------- scores @@ -1088,9 +1129,11 @@ class Store: for r in rows] def replace_scores(self, agent_id: int, - rows: Sequence[tuple[str, float, str]]) -> int: + rows: Sequence[tuple[str, float, str] | tuple[str, float, str, str]] + ) -> int: """`(uid, score, gated_reason)` for every role this agent scored, - replacing whatever it had: a role closed or muted since the last run + with the name of the prompt that gave the fit as a fourth item ('' when + left out), replacing whatever it had: a role closed or muted since the last run drops out rather than keeping a stale score. One transaction, so a reader sees the old list or the new one. An agent deleted while its run was scoring gets nothing written and 0 back: its rows are gone @@ -1101,9 +1144,10 @@ class Store: return 0 conn.execute("DELETE FROM scores WHERE agent_id=?", (agent_id,)) conn.executemany( - """INSERT INTO scores (uid, agent_id, score, gated_reason, scored_at) - VALUES (?,?,?,?,?)""", - [(uid, agent_id, score, reason, stamp) for uid, score, reason in rows]) + """INSERT INTO scores (uid, agent_id, score, gated_reason, prompt, scored_at) + VALUES (?,?,?,?,?,?)""", + [(row[0], agent_id, row[1], row[2], row[3] if len(row) > 3 else "", stamp) + for row in rows]) return len(rows) def scores(self, agent_id: int) -> dict[str, tuple[float, str]]: diff --git a/app/web/api.py b/app/web/api.py index 0e34723..23ea8fe 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -16,8 +16,8 @@ from pydantic import BaseModel, Field from app import jobs from app.models import ( - AFFINITY, Agent, Application, Company, Gate, Limits, Role, RoleRow, Status, - Subscription, Weights, company_key, + AFFINITY, Agent, Application, Company, Gate, Limits, Prompts, Role, RoleRow, + ScoredRole, Status, Subscription, Weights, company_key, ) from app.storage import Store from app.web.deps import ConfigDep, StoreDep @@ -60,9 +60,10 @@ class VoteBody(BaseModel): @router.get("/roles/{uid}") -def get_role(uid: str, store: StoreDep, agent: int | None = None) -> Role: - """One role, open or closed, for the read page, with `agent`'s vote (the - default agent's when absent; None when there is no agent).""" +def get_role(uid: str, store: StoreDep, agent: int | None = None) -> ScoredRole: + """One role, open or closed, for the read page, with `agent`'s vote and + score, and the prompt that gave its fit (the default agent's when absent; + None when there is no agent).""" role = store.role(uid, _agent_id(store, agent)) if role is None: raise HTTPException(404, f"no such role: {uid}") @@ -70,7 +71,7 @@ def get_role(uid: str, store: StoreDep, agent: int | None = None) -> Role: @router.post("/roles/{uid}/vote") -def vote_role(uid: str, body: VoteBody, store: StoreDep) -> Role: +def vote_role(uid: str, body: VoteBody, store: StoreDep) -> ScoredRole: """1 like, 0 neither way, -1 pass, for `body.agent` or the default agent. The role stays listed either way; the vote moves it in that agent's ranking, see `Store.roles`, and shapes its taste at the next `score`.""" @@ -82,7 +83,7 @@ def vote_role(uid: str, body: VoteBody, store: StoreDep) -> Role: @router.delete("/roles/{uid}/vote") -def unvote_role(uid: str, store: StoreDep, agent: int | None = None) -> Role: +def unvote_role(uid: str, store: StoreDep, agent: int | None = None) -> ScoredRole: """Unjudged again for `agent` or the default agent: what pressing the stance a role already has does.""" owner = _agent_or_default(store, agent) @@ -677,10 +678,13 @@ def list_agents(store: StoreDep) -> list[Agent]: class AgentBody(BaseModel): - """Create when `id` is missing, replace that agent's fields when given.""" + """Create when `id` is missing, replace that agent's fields when given. + `intent` is the body from before an agent had prompts: without `prompts` + it is stored as one prompt named after the agent.""" id: int | None = None name: str = Field(min_length=1) + prompts: Prompts | None = None intent: str = "" gates: list[Gate] = [] weights: Weights = Weights() @@ -694,7 +698,8 @@ def save_agent(body: AgentBody, store: StoreDep) -> Agent: a save and waits on the job it answers with.""" try: return store.save_agent(body.name, body.intent, body.gates, body.weights, - body.enabled, agent_id=body.id, limits=body.limits) + body.enabled, agent_id=body.id, limits=body.limits, + prompts=body.prompts) except ValueError as exc: raise HTTPException(409, str(exc)) from exc except KeyError: @@ -703,7 +708,7 @@ def save_agent(body: AgentBody, store: StoreDep) -> Agent: @router.delete("/agents/{agent_id}", status_code=204) def delete_agent(agent_id: int, store: StoreDep) -> Response: - """Its scores, votes and affinities go with it.""" + """Its prompts, scores, votes and affinities go with it.""" if not store.delete_agent(agent_id): raise HTTPException(404, f"no such agent: {agent_id}") return Response(status_code=204) diff --git a/config.example.yaml b/config.example.yaml index 6bc322c..bf7c4f7 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -69,7 +69,7 @@ summarise: enabled: false model: claude-haiku-4-5-20251001 -# A role and each agent's intent as vectors, for the fit term of a score +# A role and each agent's prompts as vectors, for the fit term of a score # (app/jobs/embed.py). Needs the `embeddings` extra; without it `embed` posts # a notice and scores leave fit out. Changing the model re-embeds. # multilingual-e5-small, not English-only MiniLM, because the corpus is now @@ -117,15 +117,27 @@ sources: {} # 0.65}; measured for "Copenhagen engineering" on 16,841 open roles, 0.55 # admits about 300 roles to extract and 0.65 about 35 to summarise. A wide # monitoring search lowers the bars and raises the counts, a narrow hunt the -# reverse. An agent without an intent has no fit and feeds neither stage. +# reverse. An agent without a prompt that has text has no fit and feeds +# neither stage. +# prompts is a list of {name, text}, each what the agent looks for; a role's +# fit is its best prompt's, so one agent can hunt several kinds of role +# without blending them into one paragraph. Votes, gates and limits stay the +# agent's, shared by its prompts. intent, as in config.yaml, is one prompt +# named after the agent. # The Search agents sub-tab lists and edits them; the order menu on Roles # picks one by name. Two more, beside the starter in config.yaml: agents: [] # - name: Remote from Europe -# intent: >- -# Staff or lead engineer role, fully remote from Europe. Platform and -# infrastructure work, distributed systems, developer tooling, technical -# leadership across teams. +# prompts: +# - name: Platform lead +# text: >- +# Staff or lead engineer role, fully remote from Europe. Platform and +# infrastructure work, distributed systems, developer tooling, +# technical leadership across teams. +# - name: Forward deployed +# text: >- +# Forward deployed or solutions engineering lead, fully remote from +# Europe. Hands-on with customers' systems, owns technical delivery. # gates: # - {field: remote, values: [remote], mode: require} # - {field: country, values: [AT, BE, BG, HR, CY, CZ, DK, EE, FI, FR, DE, GR, HU, IE, IT, diff --git a/tests/test_score.py b/tests/test_score.py index 3629971..e673276 100644 --- a/tests/test_score.py +++ b/tests/test_score.py @@ -22,7 +22,7 @@ from pipeline import FINAL, JOBS, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, from plugins.ats import ATSSource # noqa: E402 from app.config import CALIBRATION, AgentSeed, Config, ConfigError # noqa: E402 from app.jobs import cascade, embed, score # noqa: E402 -from app.models import Candidate, Gate, Limits, Weights # noqa: E402 +from app.models import Candidate, Gate, Limits, Prompt, Weights # noqa: E402 from app.storage import Store # noqa: E402 NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC) @@ -252,11 +252,11 @@ def test_changing_the_embed_model_drops_old_vectors_and_reembeds( assert dim_a != dim_e5, "the fakes must differ in width for this test to mean anything" -def test_changing_the_embed_model_recomputes_the_agents_cached_intent_vector( +def test_changing_the_embed_model_recomputes_the_agents_cached_prompt_vector( store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """`intent_vector` only returns a cache hit when `intent_key` -- which + """`prompt_vector` only returns a cache hit when `intent_key` -- which hashes the model in -- matches what is stored, so a model change is a - cache miss and `_intent` re-encodes under the new model, never handing a + cache miss and `_prompts` re-encodes under the new model, never handing a stale-dimension vector into a run's cosines.""" fake_a, fake_e5 = FakeEmbedder(), FakeE5Embedder() embedders = {"fake": fake_a, "intfloat/multilingual-e5-small": fake_e5} @@ -267,7 +267,7 @@ def test_changing_the_embed_model_recomputes_the_agents_cached_intent_vector( config_a = Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"}) run(config_a, store, embed.run_embed) run(config_a, store, score.run_score) - vec_a = store.intent_vector(agent.id, score.intent_key("fake", "python backend")) + vec_a = store.prompt_vector(agent.id, "Backend", score.intent_key("fake", "python backend")) assert vec_a is not None config_e5 = Config(database=str(tmp_path / "roledesk.db"), @@ -277,9 +277,10 @@ def test_changing_the_embed_model_recomputes_the_agents_cached_intent_vector( # The old key no longer resolves (the row moved on); the new key does, # at the new model's own width, and scoring this role did not raise. - assert store.intent_vector(agent.id, score.intent_key("fake", "python backend")) is None - vec_e5 = store.intent_vector( - agent.id, score.intent_key("intfloat/multilingual-e5-small", "python backend")) + assert store.prompt_vector( + agent.id, "Backend", score.intent_key("fake", "python backend")) is None + vec_e5 = store.prompt_vector( + agent.id, "Backend", score.intent_key("intfloat/multilingual-e5-small", "python backend")) assert vec_e5 is not None assert vec_e5.shape != vec_a.shape assert uid in store.scores(agent.id) @@ -676,13 +677,16 @@ def test_an_agent_without_an_intent_feeds_neither_model_stage_and_is_named( store.save_agent("Backend", "python backend", limits=top_one) blind = store.save_agent("Fresh", "", limits=top_one) store.save_agent("Also fresh", " \n", limits=top_one) + store.save_agent("Blank prompts", limits=top_one, + prompts=[Prompt(name="One"), Prompt(name="Two", text=" ")]) store.save_agent("Off", "", enabled=False) run(config, store, embed.run_embed) result = run(config, store, score.run_score) assert result["notices"] == ["score.no_intent"] - assert result["notice_texts"][0].endswith(": Also fresh, Fresh.") + assert result["notice_texts"][0].startswith("No prompt with text") + assert result["notice_texts"][0].endswith(": Also fresh, Blank prompts, Fresh.") ranked = sorted(store.scores(blind.id).items(), key=lambda kv: -kv[1][0]) assert ranked[0][0] == fresh, "the intent-less agent would pick the newer role" assert [r.uid for r in store.extract_waiting(10)] == [wanted] @@ -744,6 +748,105 @@ def test_the_intent_vector_is_cached_until_the_intent_changes( assert fake.calls[-1] == ["react frontend"] +# ----------------------------------------------------------------- prompts + +def test_fit_is_the_best_prompts_level_and_the_score_names_that_prompt( + store: Store, config: Config, fake: FakeEmbedder) -> None: + backend = role(store, "1", "Backend Engineer", body="Python backend") + frontend = role(store, "2", "Frontend Engineer", body="React frontend") + gated = role(store, "3", "Backend Engineer", body="Python backend", place="Munich") + agent = store.save_agent( + "Two kinds", gates=[Gate(field="place", values=["Berlin"])], + weights=Weights(fit=1, recency=0, affinity=0), + prompts=[Prompt(name="Web", text="react frontend"), + Prompt(name="Empty"), + Prompt(name="Server", text="python backend")]) + + run(config, store, embed.run_embed) + run(config, store, score.run_score) + + texts = {backend: "Backend Engineer\nPython backend", + frontend: "Frontend Engineer\nReact frontend"} + served = {r.uid: r for r in store.roles(agent=agent.id, gated=True)} + assert served[backend].prompt == "Server" and served[frontend].prompt == "Web" + for uid, text in texts.items(): + best = max(level(text, "react frontend"), level(text, "python backend")) + assert served[uid].score == pytest.approx(best, abs=1e-5) + # A gated role has no fit that counted, so no prompt. + assert (served[gated].gated_reason, served[gated].prompt) == ("place not Berlin", "") + assert fake.calls[1:] == [["react frontend"], ["python backend"]] + + +def test_one_prompt_scores_exactly_as_the_intent_did( + store: Store, config: Config, uncalibrated: Config, fake: FakeEmbedder) -> None: + """The legacy intent, one named prompt, and one prompt beside an empty + one are the same agent, calibrated and under the min-max fallback.""" + for n, body in enumerate(["Python backend", "Python backend data", "React apps", "Data"]): + role(store, str(n), "Engineer", body=body) + legacy = store.save_agent("Legacy", "python backend") + named = store.save_agent("Named", prompts=[Prompt(name="Backend", text="python backend")]) + padded = store.save_agent("Padded", prompts=[Prompt(name="Later"), + Prompt(name="Backend", text="python backend")]) + run(config, store, embed.run_embed) + for cfg in (config, uncalibrated): + run(cfg, store, score.run_score) + assert store.scores(legacy.id) == store.scores(named.id) == store.scores(padded.id) + assert {r.prompt for r in store.roles(agent=legacy.id)} == {"Legacy"} + assert {r.prompt for r in store.roles(agent=padded.id)} == {"Backend"} + + +def test_under_the_fallback_each_prompt_is_ranged_on_its_own( + store: Store, uncalibrated: Config, fake: FakeEmbedder) -> None: + """Min-max per prompt: each prompt's closest role gets 1, however far + that prompt's roles sit from it in absolute terms.""" + backend = role(store, "1", "Backend Engineer", body="Python backend") + frontend = role(store, "2", "Frontend Engineer", body="React frontend") + agent = store.save_agent("Two", weights=Weights(fit=1, recency=0, affinity=0), + prompts=[Prompt(name="Server", text="python backend"), + Prompt(name="Web", text="react")]) + run(uncalibrated, store, embed.run_embed) + run(uncalibrated, store, score.run_score) + + served = {r.uid: (r.score, r.prompt) for r in store.roles(agent=agent.id)} + assert served == {backend: (1.0, "Server"), frontend: (1.0, "Web")} + + +def test_an_edit_re_embeds_only_the_prompt_it_changed( + store: Store, config: Config, fake: FakeEmbedder) -> None: + role(store, "1") + agent = store.save_agent("Two", prompts=[Prompt(name="Server", text="python backend"), + Prompt(name="Web", text="react frontend")]) + run(config, store, score.run_score) + run(config, store, score.run_score) + assert fake.calls == [["python backend"], ["react frontend"]] + + # Reordered, one renamed and one rewritten: only the new text is encoded. + store.save_agent("Two", agent_id=agent.id, + prompts=[Prompt(name="Frontend", text="react frontend"), + Prompt(name="Server", text="python data")]) + run(config, store, score.run_score) + assert fake.calls[2:] == [["python data"]] + + +def test_without_an_embedder_the_prompts_are_skipped_and_fit_is_out( + store: Store, config: Config, fake: FakeEmbedder, + monkeypatch: pytest.MonkeyPatch) -> None: + uid = role(store, "1") + run(config, store, embed.run_embed) + + def missing(model: str) -> Any: + raise ImportError("no sentence_transformers") + + monkeypatch.setattr(embed, "load_embedder", missing) + agent = store.save_agent("Fit only", weights=Weights(fit=1, recency=0, affinity=0), + prompts=[Prompt(name="Server", text="python backend")]) + + result = run(config, store, score.run_score) + + assert result["notices"] == ["embed.unavailable"] + assert store.scores(agent.id) == {uid: (0.0, "no prompt vector")} + + def test_a_disabled_agent_is_not_scored( store: Store, config: Config, fake: FakeEmbedder) -> None: role(store, "1") @@ -780,6 +883,47 @@ def test_the_shipped_starter_agents_validate() -> None: for g in a.gates if g.mode == "prefer"} == {0.05} +def test_a_starter_agent_may_list_prompts_or_give_an_intent( + store: Store, tmp_path: Path) -> None: + config = Config(database=str(tmp_path / "roledesk.db"), agents=[ + AgentSeed(name="Old", intent="python backend"), + AgentSeed.model_validate({"name": "New", "prompts": [ + {"name": "AI", "text": "Head of AI"}, {"name": "FDE", "text": "FDE lead"}]})]) + score.seed_agents(config, store) + assert {a.name: a.prompts for a in store.agents()} == { + "New": [Prompt(name="AI", text="Head of AI"), Prompt(name="FDE", text="FDE lead")], + "Old": [Prompt(name="Old", text="python backend")]} + with pytest.raises(ValueError, match="two prompts are named"): + AgentSeed.model_validate({"name": "Bad", "prompts": [{"name": "A"}, {"name": "a"}]}) + + +def test_the_cli_adds_replaces_removes_and_lists_prompts( + tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + import yaml + from app import cli + + db = tmp_path / "roledesk.db" + path = tmp_path / "config.yaml" + path.write_text(yaml.safe_dump({"database": str(db)})) + agent = Store(db).save_agent("Wolf", "Head of data") + + def app(*argv: str) -> int: + return cli.main(["-c", str(path), "agents", *argv]) + + assert app("prompt", "Wolf", "FDE", "Forward deployed lead") == 0 + assert app("prompt", "Wolf", "wolf", "Head of AI") == 0 # replaced in place + assert app("prompt", "Wolf", "Research") == 0 # no text yet + assert Store(db).agent(agent.id).prompts == [ # type: ignore[union-attr] + Prompt(name="wolf", text="Head of AI"), Prompt(name="FDE", text="Forward deployed lead"), + Prompt(name="Research")] + assert app("prompt", "Wolf", "fde", "--remove") == 0 + assert app("prompt", "Wolf", "Nothing", "--remove") == 1 + assert app("prompt", "Nobody", "X", "y") == 1 + capsys.readouterr() + assert app("list") == 0 + assert capsys.readouterr().out == "Wolf\n wolf: Head of AI\n Research: \n" + + def test_the_shipped_starter_searches_validate() -> None: from app import config as config_mod diff --git a/tests/test_storage.py b/tests/test_storage.py index 96308c4..210fb36 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -11,10 +11,13 @@ import pytest from pydantic import ValidationError from desk.models import Item +import numpy as np + from app.models import ( - Application, Limits, TRUSTED_POSTED_HOSTS, company_key, place_key, posted_at, + Application, Limits, Prompt, TRUSTED_POSTED_HOSTS, company_key, place_key, posted_at, posted_host, title_key, ) +from app.storage.store import pack from app.storage import LATEST, Store @@ -66,7 +69,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 13 + assert LATEST == 14 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -85,11 +88,23 @@ def _columns(path: Path, table: str) -> list[tuple[str, str, int, str]]: return [(r[1], r[2], r[3], r[4]) for r in conn.execute(f"PRAGMA table_info({table})")] +def _undo_v14(conn: sqlite3.Connection) -> None: + """A fresh file put back to the v13 shape: an agent's intent and its + cached vector on the agent row, no prompts, no prompt on a score. Every + downgrade below starts here, since v14 is the last migration.""" + conn.execute("DROP TABLE agent_prompts") + conn.execute("ALTER TABLE scores DROP COLUMN prompt") + conn.execute("ALTER TABLE agents ADD COLUMN intent TEXT NOT NULL DEFAULT ''") + conn.execute("ALTER TABLE agents ADD COLUMN intent_key TEXT NOT NULL DEFAULT ''") + conn.execute("ALTER TABLE agents ADD COLUMN intent_vec BLOB") + + def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None: fresh, old = tmp_path / "fresh.db", tmp_path / "old.db" Store(fresh) Store(old) with sqlite3.connect(old) as conn: + _undo_v14(conn) conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at for column in ("permit", "languages", "extract_tries", "summary", "summarised_at", "article_text", "article_at", "read_at", "posted_at", "salary_text"): @@ -107,8 +122,8 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None: assert Store(old).migrations_applied == list(range(2, LATEST + 1)) # ALTER appends, so the order differs; every read is by column name. - for table in ("roles", "agents", "vectors", "companies", "role_tags", - "agent_companies", "role_votes"): + for table in ("roles", "agents", "agent_prompts", "scores", "vectors", "companies", + "role_tags", "agent_companies", "role_votes"): assert sorted(_columns(old, table)) == sorted(_columns(fresh, table)) @@ -116,6 +131,7 @@ def test_a_v8_file_gains_ats_slug_and_keeps_its_companies(tmp_path: Path) -> Non path = tmp_path / "v8.db" Store(path).add_company("Acme", ats="teamtailor", country="DK") with sqlite3.connect(path) as conn: + _undo_v14(conn) conn.execute("ALTER TABLE companies DROP COLUMN ats_slug") conn.execute("DROP INDEX idx_roles_posted") # v10, which follows conn.execute("ALTER TABLE roles DROP COLUMN posted_at") @@ -134,6 +150,7 @@ def _downgrade_to_v4(path: Path) -> None: """A fresh file put back to the v4 shape: votes on the role, affinity on the company, no read state, no tags, no per-agent tables.""" with sqlite3.connect(path) as conn: + _undo_v14(conn) conn.execute("ALTER TABLE roles DROP COLUMN read_at") conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at conn.execute("ALTER TABLE roles DROP COLUMN posted_at") # v9 @@ -245,6 +262,7 @@ def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses( assert before_trusted is not None and before_untrusted is not None with sqlite3.connect(path) as conn: + _undo_v14(conn) conn.execute("DROP INDEX idx_roles_posted") conn.execute("ALTER TABLE roles DROP COLUMN posted_at") conn.execute("ALTER TABLE roles DROP COLUMN salary_text") # v12 @@ -272,6 +290,7 @@ def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it( for item in (wrong, several, given, extracted): store.upsert_role(item) with sqlite3.connect(path) as conn: + _undo_v14(conn) conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'") conn.execute("UPDATE roles SET country = '' WHERE uid = ?", (several.uid,)) conn.execute("UPDATE roles SET extracted_at = '2026-09-23' WHERE uid = ?", @@ -287,6 +306,95 @@ def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it( extracted.uid: "US"} +def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: Path) -> None: + path = tmp_path / "v13.db" + store = Store(path) + store.upsert_role(posting("a")) + uid = posting("a").uid + backend = store.save_agent("Backend") + blank = store.save_agent("Blank") + empty = store.save_agent("Empty") + store.replace_scores(backend.id, [(uid, 0.7, "")]) + vec = np.asarray([0.6, 0.8], dtype=np.float32) + with sqlite3.connect(path) as conn: + _undo_v14(conn) + conn.execute("UPDATE agents SET intent = 'python backend', intent_key = 'k', " + "intent_vec = ? WHERE id = ?", (pack(vec), backend.id)) + conn.execute("UPDATE agents SET intent = ' \n ' WHERE id = ?", (blank.id,)) + conn.execute("PRAGMA user_version = 13") + + migrated = Store(path) + + assert migrated.migrations_applied == [14] + assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr] + Prompt(name="Backend", text="python backend")] + assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr] + assert migrated.agent(empty.id).prompts == [] # type: ignore[union-attr] + # The key hashes the model and the text, both unchanged: nothing to re-embed. + cached = migrated.prompt_vector(backend.id, "Backend", "k") + assert cached is not None and cached.tolist() == pytest.approx(vec.tolist()) + assert migrated.scores(backend.id) == {uid: (0.7, "")} + assert migrated.role(uid, backend.id).prompt == "" # type: ignore[union-attr] + columns = {c[0] for c in _columns(path, "agents")} + assert not {"intent", "intent_key", "intent_vec"} & columns + + +def test_prompts_are_saved_replaced_reordered_and_deleted(store: Store) -> None: + ai = Prompt(name="AI leadership", text="Head of AI") + fde = Prompt(name="Forward deployed", text="FDE lead") + agent = store.save_agent("Wolf", prompts=[ai, fde]) + assert store.agent(agent.id).prompts == [ai, fde] # type: ignore[union-attr] + store.set_prompt_vector(agent.id, "Forward deployed", "k", np.ones(2)) + + # Reordered and renamed, the same text keeps its vector; a new text has none. + research = Prompt(name="Research", text="AI research") + renamed = Prompt(name="FDE", text="FDE lead") + store.save_agent("Wolf", agent_id=agent.id, prompts=[renamed, research]) + assert [p.name for p in store.agents()[0].prompts] == ["FDE", "Research"] + assert store.prompt_vector(agent.id, "FDE", "k") is not None + assert store.prompt_vector(agent.id, "Research", "k") is None + + # A prompt may have no text; its name is what must be there, and unique. + store.save_agent("Wolf", agent_id=agent.id, prompts=[Prompt(name="Later")]) + assert store.agents()[0].prompts == [Prompt(name="Later", text="")] + with pytest.raises(ValueError, match="two prompts are named 'fde'"): + store.save_agent("Wolf", agent_id=agent.id, + prompts=[renamed, Prompt(name="fde", text="other")]) + with pytest.raises(ValidationError, match="a prompt needs a name"): + Prompt(name=" ", text="x") + assert store.agents()[0].prompts == [Prompt(name="Later")] + + # Without prompts an intent is the one prompt, named after the agent. + store.save_agent("Wolf", "python backend", agent_id=agent.id) + assert store.agent(agent.id).prompts == [ # type: ignore[union-attr] + Prompt(name="Wolf", text="python backend")] + store.save_agent("Wolf", agent_id=agent.id) + assert store.agent(agent.id).prompts == [] # type: ignore[union-attr] + + store.save_agent("Wolf", agent_id=agent.id, prompts=[ai, fde]) + store.delete_agent(agent.id) + with store.transaction() as conn: + assert conn.execute("SELECT COUNT(*) FROM agent_prompts").fetchone()[0] == 0 + + +def test_an_agent_whose_prompts_are_all_empty_feeds_neither_model_stage( + store: Store) -> None: + store.upsert_role(posting("a")) + store.upsert_role(posting("b")) + a, b = posting("a").uid, posting("b").uid + top = Limits(extract_top=1, extract_min=0, summarise_top=1, summarise_min=0) + blank = store.save_agent("Blank", limits=top, + prompts=[Prompt(name="One"), Prompt(name="Two", text=" \n")]) + store.replace_scores(blank.id, [(a, 0.9, "")]) + assert store.extract_waiting(10) == [] and store.summarise_waiting(10) == [] + + # One prompt with text is enough. + store.save_agent("Blank", agent_id=blank.id, limits=top, + prompts=[Prompt(name="One"), Prompt(name="Two", text="python")]) + assert [r.uid for r in store.extract_waiting(10)] == [a] + assert b not in {r.uid for r in store.summarise_waiting(10)} + + def test_state_is_a_mapping(store: Store) -> None: store.state["ats:last"] = "2026-09-23" assert store.state["ats:last"] == "2026-09-23" diff --git a/tests/test_web.py b/tests/test_web.py index 883725e..a10dc2f 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -877,6 +877,59 @@ def test_agents_are_created_updated_listed_and_deleted(client: TestClient) -> No assert client.delete(f"/api/agents/{agent['id']}").status_code == 404 +def test_agents_take_and_serve_prompts_and_still_take_an_intent(client: TestClient) -> None: + prompts = [{"name": "AI and data leadership", "text": "VP, Head or Director of AI"}, + {"name": "Forward deployed", "text": ""}] + created = client.post("/api/agents", json={"name": "Wolf", "prompts": prompts}) + assert created.status_code == 200 + assert created.json()["prompts"] == prompts and "intent" not in created.json() + listed = client.get("/api/agents").json() + assert listed[0]["prompts"] == prompts and "intent" not in listed[0] + + # Reordered in place; then the body from before prompts, which becomes one. + agent_id = created.json()["id"] + swapped = client.post("/api/agents", json={"id": agent_id, "name": "Wolf", + "prompts": prompts[::-1]}) + assert swapped.json()["prompts"] == prompts[::-1] + legacy = client.post("/api/agents", json={"id": agent_id, "name": "Wolf", + "intent": "python backend"}) + assert legacy.json()["prompts"] == [{"name": "Wolf", "text": "python backend"}] + # `prompts` wins over `intent` when both are sent. + both = client.post("/api/agents", json={"id": agent_id, "name": "Wolf", "prompts": [], + "intent": "python backend"}) + assert both.json()["prompts"] == [] + + +@pytest.mark.parametrize("prompts, says", [ + ([{"name": "AI", "text": "a"}, {"name": "ai", "text": "b"}], "two prompts are named 'ai'"), + ([{"name": " ", "text": "a"}], "a prompt needs a name"), +]) +def test_a_duplicate_or_empty_prompt_name_is_a_422_sentence( + client: TestClient, prompts: list[dict[str, str]], says: str) -> None: + res = client.post("/api/agents", json={"name": "Bad", "prompts": prompts}) + assert res.status_code == 422 and says in res.json()["detail"] + assert client.get("/api/agents").json() == [] + + +def test_the_role_endpoint_serves_the_agents_score_and_its_prompt( + client: TestClient, store: Store) -> None: + store.upsert_role(posting("1")) + uid = posting("1").uid + agent = store.save_agent("A", prompts=[]) + store.replace_scores(agent.id, [(uid, 0.8, "", "Forward deployed")]) + + role = client.get(f"/api/roles/{uid}", params={"agent": agent.id}).json() + assert (role["score"], role["gated_reason"], role["prompt"]) == \ + (0.8, "", "Forward deployed") + row = client.get("/api/roles", params={"agent": agent.id}).json()["items"][0] + assert row["prompt"] == "Forward deployed" + voted = client.post(f"/api/roles/{uid}/vote", json={"value": 1, "agent": agent.id}) + assert voted.json()["score"] == pytest.approx(0.96) + other = store.save_agent("B") + unscored = client.get(f"/api/roles/{uid}", params={"agent": other.id}).json() + assert (unscored["score"], unscored["prompt"]) == (None, "") + + def test_deleting_an_agent_takes_its_votes_and_affinities_with_it( client: TestClient, store: Store) -> None: store.upsert_role(posting("1")) @@ -950,7 +1003,7 @@ def test_list_roles_serves_the_row_projection_and_not_the_big_columns( assert set(row) == { "uid", "company_key", "company", "title", "place", "country", "remote", "function", "salary_min", "salary_max", "currency", "salary_text", "salary_state", - "original_url", "posted_at", "score", "gated_reason", "vote", + "original_url", "posted_at", "score", "gated_reason", "prompt", "vote", "voted_at", "read_at", "tags", "excerpt", } assert row["uid"] == uid