Give a search agent several prompts and score a role by its best one
Migration 14 moves each non-empty agents.intent into agent_prompts as one prompt named after the agent, with its cached vector, and drops the three intent columns. A role's fit is the maximum over the agent's prompts of the calibrated cosine, and scores.prompt names the prompt that won. The agents API takes prompts and still takes intent; the role endpoint serves the agent's score and prompt. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b6afe24eaa
commit
4e3c3f130e
9 files changed
+552
-127
No files matched your search
+126
-9
@@ -22,7 +22,7 @@ from pipeline import FINAL, JOBS, SCHEDULES, TASKS, JobQueue, Jobs, RunContext,
|
||||
from plugins.ats import ATSSource # noqa: E402
|
||||
from app.config import CALIBRATION, AgentSeed, Config, ConfigError # noqa: E402
|
||||
from app.jobs import cascade, embed, score # noqa: E402
|
||||
from app.models import Candidate, Gate, Limits, Weights # noqa: E402
|
||||
from app.models import Candidate, Gate, Limits, Prompt, Weights # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
|
||||
NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
|
||||
@@ -252,11 +252,11 @@ def test_changing_the_embed_model_drops_old_vectors_and_reembeds(
|
||||
assert dim_a != dim_e5, "the fakes must differ in width for this test to mean anything"
|
||||
|
||||
|
||||
def test_changing_the_embed_model_recomputes_the_agents_cached_intent_vector(
|
||||
def test_changing_the_embed_model_recomputes_the_agents_cached_prompt_vector(
|
||||
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""`intent_vector` only returns a cache hit when `intent_key` -- which
|
||||
"""`prompt_vector` only returns a cache hit when `intent_key` -- which
|
||||
hashes the model in -- matches what is stored, so a model change is a
|
||||
cache miss and `_intent` re-encodes under the new model, never handing a
|
||||
cache miss and `_prompts` re-encodes under the new model, never handing a
|
||||
stale-dimension vector into a run's cosines."""
|
||||
fake_a, fake_e5 = FakeEmbedder(), FakeE5Embedder()
|
||||
embedders = {"fake": fake_a, "intfloat/multilingual-e5-small": fake_e5}
|
||||
@@ -267,7 +267,7 @@ def test_changing_the_embed_model_recomputes_the_agents_cached_intent_vector(
|
||||
config_a = Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
|
||||
run(config_a, store, embed.run_embed)
|
||||
run(config_a, store, score.run_score)
|
||||
vec_a = store.intent_vector(agent.id, score.intent_key("fake", "python backend"))
|
||||
vec_a = store.prompt_vector(agent.id, "Backend", score.intent_key("fake", "python backend"))
|
||||
assert vec_a is not None
|
||||
|
||||
config_e5 = Config(database=str(tmp_path / "roledesk.db"),
|
||||
@@ -277,9 +277,10 @@ def test_changing_the_embed_model_recomputes_the_agents_cached_intent_vector(
|
||||
|
||||
# The old key no longer resolves (the row moved on); the new key does,
|
||||
# at the new model's own width, and scoring this role did not raise.
|
||||
assert store.intent_vector(agent.id, score.intent_key("fake", "python backend")) is None
|
||||
vec_e5 = store.intent_vector(
|
||||
agent.id, score.intent_key("intfloat/multilingual-e5-small", "python backend"))
|
||||
assert store.prompt_vector(
|
||||
agent.id, "Backend", score.intent_key("fake", "python backend")) is None
|
||||
vec_e5 = store.prompt_vector(
|
||||
agent.id, "Backend", score.intent_key("intfloat/multilingual-e5-small", "python backend"))
|
||||
assert vec_e5 is not None
|
||||
assert vec_e5.shape != vec_a.shape
|
||||
assert uid in store.scores(agent.id)
|
||||
@@ -676,13 +677,16 @@ def test_an_agent_without_an_intent_feeds_neither_model_stage_and_is_named(
|
||||
store.save_agent("Backend", "python backend", limits=top_one)
|
||||
blind = store.save_agent("Fresh", "", limits=top_one)
|
||||
store.save_agent("Also fresh", " \n", limits=top_one)
|
||||
store.save_agent("Blank prompts", limits=top_one,
|
||||
prompts=[Prompt(name="One"), Prompt(name="Two", text=" ")])
|
||||
store.save_agent("Off", "", enabled=False)
|
||||
|
||||
run(config, store, embed.run_embed)
|
||||
result = run(config, store, score.run_score)
|
||||
|
||||
assert result["notices"] == ["score.no_intent"]
|
||||
assert result["notice_texts"][0].endswith(": Also fresh, Fresh.")
|
||||
assert result["notice_texts"][0].startswith("No prompt with text")
|
||||
assert result["notice_texts"][0].endswith(": Also fresh, Blank prompts, Fresh.")
|
||||
ranked = sorted(store.scores(blind.id).items(), key=lambda kv: -kv[1][0])
|
||||
assert ranked[0][0] == fresh, "the intent-less agent would pick the newer role"
|
||||
assert [r.uid for r in store.extract_waiting(10)] == [wanted]
|
||||
@@ -744,6 +748,105 @@ def test_the_intent_vector_is_cached_until_the_intent_changes(
|
||||
assert fake.calls[-1] == ["react frontend"]
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- prompts
|
||||
|
||||
def test_fit_is_the_best_prompts_level_and_the_score_names_that_prompt(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
backend = role(store, "1", "Backend Engineer", body="Python backend")
|
||||
frontend = role(store, "2", "Frontend Engineer", body="React frontend")
|
||||
gated = role(store, "3", "Backend Engineer", body="Python backend", place="Munich")
|
||||
agent = store.save_agent(
|
||||
"Two kinds", gates=[Gate(field="place", values=["Berlin"])],
|
||||
weights=Weights(fit=1, recency=0, affinity=0),
|
||||
prompts=[Prompt(name="Web", text="react frontend"),
|
||||
Prompt(name="Empty"),
|
||||
Prompt(name="Server", text="python backend")])
|
||||
|
||||
run(config, store, embed.run_embed)
|
||||
run(config, store, score.run_score)
|
||||
|
||||
texts = {backend: "Backend Engineer\nPython backend",
|
||||
frontend: "Frontend Engineer\nReact frontend"}
|
||||
served = {r.uid: r for r in store.roles(agent=agent.id, gated=True)}
|
||||
assert served[backend].prompt == "Server" and served[frontend].prompt == "Web"
|
||||
for uid, text in texts.items():
|
||||
best = max(level(text, "react frontend"), level(text, "python backend"))
|
||||
assert served[uid].score == pytest.approx(best, abs=1e-5)
|
||||
# A gated role has no fit that counted, so no prompt.
|
||||
assert (served[gated].gated_reason, served[gated].prompt) == ("place not Berlin", "")
|
||||
assert fake.calls[1:] == [["react frontend"], ["python backend"]]
|
||||
|
||||
|
||||
def test_one_prompt_scores_exactly_as_the_intent_did(
|
||||
store: Store, config: Config, uncalibrated: Config, fake: FakeEmbedder) -> None:
|
||||
"""The legacy intent, one named prompt, and one prompt beside an empty
|
||||
one are the same agent, calibrated and under the min-max fallback."""
|
||||
for n, body in enumerate(["Python backend", "Python backend data", "React apps", "Data"]):
|
||||
role(store, str(n), "Engineer", body=body)
|
||||
legacy = store.save_agent("Legacy", "python backend")
|
||||
named = store.save_agent("Named", prompts=[Prompt(name="Backend", text="python backend")])
|
||||
padded = store.save_agent("Padded", prompts=[Prompt(name="Later"),
|
||||
Prompt(name="Backend", text="python backend")])
|
||||
run(config, store, embed.run_embed)
|
||||
for cfg in (config, uncalibrated):
|
||||
run(cfg, store, score.run_score)
|
||||
assert store.scores(legacy.id) == store.scores(named.id) == store.scores(padded.id)
|
||||
assert {r.prompt for r in store.roles(agent=legacy.id)} == {"Legacy"}
|
||||
assert {r.prompt for r in store.roles(agent=padded.id)} == {"Backend"}
|
||||
|
||||
|
||||
def test_under_the_fallback_each_prompt_is_ranged_on_its_own(
|
||||
store: Store, uncalibrated: Config, fake: FakeEmbedder) -> None:
|
||||
"""Min-max per prompt: each prompt's closest role gets 1, however far
|
||||
that prompt's roles sit from it in absolute terms."""
|
||||
backend = role(store, "1", "Backend Engineer", body="Python backend")
|
||||
frontend = role(store, "2", "Frontend Engineer", body="React frontend")
|
||||
agent = store.save_agent("Two", weights=Weights(fit=1, recency=0, affinity=0),
|
||||
prompts=[Prompt(name="Server", text="python backend"),
|
||||
Prompt(name="Web", text="react")])
|
||||
run(uncalibrated, store, embed.run_embed)
|
||||
run(uncalibrated, store, score.run_score)
|
||||
|
||||
served = {r.uid: (r.score, r.prompt) for r in store.roles(agent=agent.id)}
|
||||
assert served == {backend: (1.0, "Server"), frontend: (1.0, "Web")}
|
||||
|
||||
|
||||
def test_an_edit_re_embeds_only_the_prompt_it_changed(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
role(store, "1")
|
||||
agent = store.save_agent("Two", prompts=[Prompt(name="Server", text="python backend"),
|
||||
Prompt(name="Web", text="react frontend")])
|
||||
run(config, store, score.run_score)
|
||||
run(config, store, score.run_score)
|
||||
assert fake.calls == [["python backend"], ["react frontend"]]
|
||||
|
||||
# Reordered, one renamed and one rewritten: only the new text is encoded.
|
||||
store.save_agent("Two", agent_id=agent.id,
|
||||
prompts=[Prompt(name="Frontend", text="react frontend"),
|
||||
Prompt(name="Server", text="python data")])
|
||||
run(config, store, score.run_score)
|
||||
assert fake.calls[2:] == [["python data"]]
|
||||
|
||||
|
||||
def test_without_an_embedder_the_prompts_are_skipped_and_fit_is_out(
|
||||
store: Store, config: Config, fake: FakeEmbedder,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
uid = role(store, "1")
|
||||
run(config, store, embed.run_embed)
|
||||
|
||||
def missing(model: str) -> Any:
|
||||
raise ImportError("no sentence_transformers")
|
||||
|
||||
monkeypatch.setattr(embed, "load_embedder", missing)
|
||||
agent = store.save_agent("Fit only", weights=Weights(fit=1, recency=0, affinity=0),
|
||||
prompts=[Prompt(name="Server", text="python backend")])
|
||||
|
||||
result = run(config, store, score.run_score)
|
||||
|
||||
assert result["notices"] == ["embed.unavailable"]
|
||||
assert store.scores(agent.id) == {uid: (0.0, "no prompt vector")}
|
||||
|
||||
|
||||
def test_a_disabled_agent_is_not_scored(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
role(store, "1")
|
||||
@@ -780,6 +883,20 @@ def test_the_shipped_starter_agents_validate() -> None:
|
||||
for g in a.gates if g.mode == "prefer"} == {0.05}
|
||||
|
||||
|
||||
def test_a_starter_agent_may_list_prompts_or_give_an_intent(
|
||||
store: Store, tmp_path: Path) -> None:
|
||||
config = Config(database=str(tmp_path / "roledesk.db"), agents=[
|
||||
AgentSeed(name="Old", intent="python backend"),
|
||||
AgentSeed.model_validate({"name": "New", "prompts": [
|
||||
{"name": "AI", "text": "Head of AI"}, {"name": "FDE", "text": "FDE lead"}]})])
|
||||
score.seed_agents(config, store)
|
||||
assert {a.name: a.prompts for a in store.agents()} == {
|
||||
"New": [Prompt(name="AI", text="Head of AI"), Prompt(name="FDE", text="FDE lead")],
|
||||
"Old": [Prompt(name="Old", text="python backend")]}
|
||||
with pytest.raises(ValueError, match="two prompts are named"):
|
||||
AgentSeed.model_validate({"name": "Bad", "prompts": [{"name": "A"}, {"name": "a"}]})
|
||||
|
||||
|
||||
def test_the_shipped_starter_searches_validate() -> None:
|
||||
from app import config as config_mod
|
||||
|
||||
|
||||
Reference in new issue
Block a user