Give a search agent several prompts and score a role by its best one

Migration 14 moves each non-empty agents.intent into agent_prompts as one
prompt named after the agent, with its cached vector, and drops the three
intent columns. A role's fit is the maximum over the agent's prompts of the
calibrated cosine, and scores.prompt names the prompt that won. The agents
API takes prompts and still takes intent; the role endpoint serves the
agent's score and prompt.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-29 16:19:57 +02:00
1 parent b6afe24eaa
commit 4e3c3f130e
9 files changed
+552 -127

No files matched your search

+126 -9
View File
@@ -22,7 +22,7 @@ from pipeline import FINAL, JOBS, SCHEDULES, TASKS, JobQueue, Jobs, RunContext,
from plugins.ats import ATSSource # noqa: E402
from app.config import CALIBRATION, AgentSeed, Config, ConfigError # noqa: E402
from app.jobs import cascade, embed, score # noqa: E402
from app.models import Candidate, Gate, Limits, Weights # noqa: E402
from app.models import Candidate, Gate, Limits, Prompt, Weights # noqa: E402
from app.storage import Store # noqa: E402
NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
@@ -252,11 +252,11 @@ def test_changing_the_embed_model_drops_old_vectors_and_reembeds(
assert dim_a != dim_e5, "the fakes must differ in width for this test to mean anything"
def test_changing_the_embed_model_recomputes_the_agents_cached_intent_vector(
def test_changing_the_embed_model_recomputes_the_agents_cached_prompt_vector(
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""`intent_vector` only returns a cache hit when `intent_key` -- which
"""`prompt_vector` only returns a cache hit when `intent_key` -- which
hashes the model in -- matches what is stored, so a model change is a
cache miss and `_intent` re-encodes under the new model, never handing a
cache miss and `_prompts` re-encodes under the new model, never handing a
stale-dimension vector into a run's cosines."""
fake_a, fake_e5 = FakeEmbedder(), FakeE5Embedder()
embedders = {"fake": fake_a, "intfloat/multilingual-e5-small": fake_e5}
@@ -267,7 +267,7 @@ def test_changing_the_embed_model_recomputes_the_agents_cached_intent_vector(
config_a = Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
run(config_a, store, embed.run_embed)
run(config_a, store, score.run_score)
vec_a = store.intent_vector(agent.id, score.intent_key("fake", "python backend"))
vec_a = store.prompt_vector(agent.id, "Backend", score.intent_key("fake", "python backend"))
assert vec_a is not None
config_e5 = Config(database=str(tmp_path / "roledesk.db"),
@@ -277,9 +277,10 @@ def test_changing_the_embed_model_recomputes_the_agents_cached_intent_vector(
# The old key no longer resolves (the row moved on); the new key does,
# at the new model's own width, and scoring this role did not raise.
assert store.intent_vector(agent.id, score.intent_key("fake", "python backend")) is None
vec_e5 = store.intent_vector(
agent.id, score.intent_key("intfloat/multilingual-e5-small", "python backend"))
assert store.prompt_vector(
agent.id, "Backend", score.intent_key("fake", "python backend")) is None
vec_e5 = store.prompt_vector(
agent.id, "Backend", score.intent_key("intfloat/multilingual-e5-small", "python backend"))
assert vec_e5 is not None
assert vec_e5.shape != vec_a.shape
assert uid in store.scores(agent.id)
@@ -676,13 +677,16 @@ def test_an_agent_without_an_intent_feeds_neither_model_stage_and_is_named(
store.save_agent("Backend", "python backend", limits=top_one)
blind = store.save_agent("Fresh", "", limits=top_one)
store.save_agent("Also fresh", " \n", limits=top_one)
store.save_agent("Blank prompts", limits=top_one,
prompts=[Prompt(name="One"), Prompt(name="Two", text=" ")])
store.save_agent("Off", "", enabled=False)
run(config, store, embed.run_embed)
result = run(config, store, score.run_score)
assert result["notices"] == ["score.no_intent"]
assert result["notice_texts"][0].endswith(": Also fresh, Fresh.")
assert result["notice_texts"][0].startswith("No prompt with text")
assert result["notice_texts"][0].endswith(": Also fresh, Blank prompts, Fresh.")
ranked = sorted(store.scores(blind.id).items(), key=lambda kv: -kv[1][0])
assert ranked[0][0] == fresh, "the intent-less agent would pick the newer role"
assert [r.uid for r in store.extract_waiting(10)] == [wanted]
@@ -744,6 +748,105 @@ def test_the_intent_vector_is_cached_until_the_intent_changes(
assert fake.calls[-1] == ["react frontend"]
# ----------------------------------------------------------------- prompts
def test_fit_is_the_best_prompts_level_and_the_score_names_that_prompt(
store: Store, config: Config, fake: FakeEmbedder) -> None:
backend = role(store, "1", "Backend Engineer", body="Python backend")
frontend = role(store, "2", "Frontend Engineer", body="React frontend")
gated = role(store, "3", "Backend Engineer", body="Python backend", place="Munich")
agent = store.save_agent(
"Two kinds", gates=[Gate(field="place", values=["Berlin"])],
weights=Weights(fit=1, recency=0, affinity=0),
prompts=[Prompt(name="Web", text="react frontend"),
Prompt(name="Empty"),
Prompt(name="Server", text="python backend")])
run(config, store, embed.run_embed)
run(config, store, score.run_score)
texts = {backend: "Backend Engineer\nPython backend",
frontend: "Frontend Engineer\nReact frontend"}
served = {r.uid: r for r in store.roles(agent=agent.id, gated=True)}
assert served[backend].prompt == "Server" and served[frontend].prompt == "Web"
for uid, text in texts.items():
best = max(level(text, "react frontend"), level(text, "python backend"))
assert served[uid].score == pytest.approx(best, abs=1e-5)
# A gated role has no fit that counted, so no prompt.
assert (served[gated].gated_reason, served[gated].prompt) == ("place not Berlin", "")
assert fake.calls[1:] == [["react frontend"], ["python backend"]]
def test_one_prompt_scores_exactly_as_the_intent_did(
store: Store, config: Config, uncalibrated: Config, fake: FakeEmbedder) -> None:
"""The legacy intent, one named prompt, and one prompt beside an empty
one are the same agent, calibrated and under the min-max fallback."""
for n, body in enumerate(["Python backend", "Python backend data", "React apps", "Data"]):
role(store, str(n), "Engineer", body=body)
legacy = store.save_agent("Legacy", "python backend")
named = store.save_agent("Named", prompts=[Prompt(name="Backend", text="python backend")])
padded = store.save_agent("Padded", prompts=[Prompt(name="Later"),
Prompt(name="Backend", text="python backend")])
run(config, store, embed.run_embed)
for cfg in (config, uncalibrated):
run(cfg, store, score.run_score)
assert store.scores(legacy.id) == store.scores(named.id) == store.scores(padded.id)
assert {r.prompt for r in store.roles(agent=legacy.id)} == {"Legacy"}
assert {r.prompt for r in store.roles(agent=padded.id)} == {"Backend"}
def test_under_the_fallback_each_prompt_is_ranged_on_its_own(
store: Store, uncalibrated: Config, fake: FakeEmbedder) -> None:
"""Min-max per prompt: each prompt's closest role gets 1, however far
that prompt's roles sit from it in absolute terms."""
backend = role(store, "1", "Backend Engineer", body="Python backend")
frontend = role(store, "2", "Frontend Engineer", body="React frontend")
agent = store.save_agent("Two", weights=Weights(fit=1, recency=0, affinity=0),
prompts=[Prompt(name="Server", text="python backend"),
Prompt(name="Web", text="react")])
run(uncalibrated, store, embed.run_embed)
run(uncalibrated, store, score.run_score)
served = {r.uid: (r.score, r.prompt) for r in store.roles(agent=agent.id)}
assert served == {backend: (1.0, "Server"), frontend: (1.0, "Web")}
def test_an_edit_re_embeds_only_the_prompt_it_changed(
store: Store, config: Config, fake: FakeEmbedder) -> None:
role(store, "1")
agent = store.save_agent("Two", prompts=[Prompt(name="Server", text="python backend"),
Prompt(name="Web", text="react frontend")])
run(config, store, score.run_score)
run(config, store, score.run_score)
assert fake.calls == [["python backend"], ["react frontend"]]
# Reordered, one renamed and one rewritten: only the new text is encoded.
store.save_agent("Two", agent_id=agent.id,
prompts=[Prompt(name="Frontend", text="react frontend"),
Prompt(name="Server", text="python data")])
run(config, store, score.run_score)
assert fake.calls[2:] == [["python data"]]
def test_without_an_embedder_the_prompts_are_skipped_and_fit_is_out(
store: Store, config: Config, fake: FakeEmbedder,
monkeypatch: pytest.MonkeyPatch) -> None:
uid = role(store, "1")
run(config, store, embed.run_embed)
def missing(model: str) -> Any:
raise ImportError("no sentence_transformers")
monkeypatch.setattr(embed, "load_embedder", missing)
agent = store.save_agent("Fit only", weights=Weights(fit=1, recency=0, affinity=0),
prompts=[Prompt(name="Server", text="python backend")])
result = run(config, store, score.run_score)
assert result["notices"] == ["embed.unavailable"]
assert store.scores(agent.id) == {uid: (0.0, "no prompt vector")}
def test_a_disabled_agent_is_not_scored(
store: Store, config: Config, fake: FakeEmbedder) -> None:
role(store, "1")
@@ -780,6 +883,20 @@ def test_the_shipped_starter_agents_validate() -> None:
for g in a.gates if g.mode == "prefer"} == {0.05}
def test_a_starter_agent_may_list_prompts_or_give_an_intent(
store: Store, tmp_path: Path) -> None:
config = Config(database=str(tmp_path / "roledesk.db"), agents=[
AgentSeed(name="Old", intent="python backend"),
AgentSeed.model_validate({"name": "New", "prompts": [
{"name": "AI", "text": "Head of AI"}, {"name": "FDE", "text": "FDE lead"}]})])
score.seed_agents(config, store)
assert {a.name: a.prompts for a in store.agents()} == {
"New": [Prompt(name="AI", text="Head of AI"), Prompt(name="FDE", text="FDE lead")],
"Old": [Prompt(name="Old", text="python backend")]}
with pytest.raises(ValueError, match="two prompts are named"):
AgentSeed.model_validate({"name": "Bad", "prompts": [{"name": "A"}, {"name": "a"}]})
def test_the_shipped_starter_searches_validate() -> None:
from app import config as config_mod