Calibrate fit and taste per embedding model instead of min-max per run

scale() maps a cosine to clip((cos - lo) / (hi - lo), 0, 1) with constants
from CALIBRATION in app/config.py, keyed by model and replaceable under
embed.calibration. multilingual-e5-small: fit 0.75..0.88, taste -0.14..0.19
(taste from synthetic votes). An uncalibrated model falls back to min-max
and logs a warning once per run.

The extract and summarise bars default to 0.55 and 0.65, about 300 and 35
roles for the starter agent on the measured data; stored limits are kept.
An agent without an intent is left out of the model-stage scope, and each
score run names such agents in one score.no_intent notice.

Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus committed 2026-09-29 14:59:34 +02:00
1 parent 706b127bc0
commit b021838959
12 files changed
+336 -76

No files matched your search

+3 -3
View File
@@ -659,7 +659,7 @@ def test_the_agents_page_lists_and_edits_search_agents(client: TestClient) -> No
for limit in ("a-extract-top", "a-extract-min", "a-summarise-top", "a-summarise-min"):
assert f'id="{limit}"' in template
assert 'id="a-extract-top" min="0" step="10" value="500"' in page
assert 'id="a-summarise-min" min="0" max="1" step="0.05" value="0.5"' in page
assert 'id="a-summarise-min" min="0" max="1" step="0.05" value="0.65"' in page
assert 'id="a-new"' in page and 'id="a-delete"' in page
assert "Save and score" in page
@@ -823,8 +823,8 @@ def test_an_agents_limits_are_saved_listed_and_checked(client: TestClient) -> No
"""How deep the model stages read is the agent's own; a score outside
0..1 is refused."""
default = client.post("/api/agents", json={"name": "Wide"}).json()
assert default["limits"] == {"extract_top": 500, "extract_min": 0.3,
"summarise_top": 30, "summarise_min": 0.5}
assert default["limits"] == {"extract_top": 500, "extract_min": 0.55,
"summarise_top": 30, "summarise_min": 0.65}
narrow = {"extract_top": 50, "extract_min": 0.6, "summarise_top": 5, "summarise_min": 0.7}
saved = client.post("/api/agents", json={"name": "Narrow", "limits": narrow})
assert saved.status_code == 200 and saved.json()["limits"] == narrow