Calibrate fit and taste per embedding model instead of min-max per run
scale() maps a cosine to clip((cos - lo) / (hi - lo), 0, 1) with constants from CALIBRATION in app/config.py, keyed by model and replaceable under embed.calibration. multilingual-e5-small: fit 0.75..0.88, taste -0.14..0.19 (taste from synthetic votes). An uncalibrated model falls back to min-max and logs a warning once per run. The extract and summarise bars default to 0.55 and 0.65, about 300 and 35 roles for the starter agent on the measured data; stored limits are kept. An agent without an intent is left out of the model-stage scope, and each score run names such agents in one score.no_intent notice. Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
1 parent
706b127bc0
commit
b021838959
12 files changed
+336
-76
No files matched your search
+3
-3
@@ -659,7 +659,7 @@ def test_the_agents_page_lists_and_edits_search_agents(client: TestClient) -> No
|
||||
for limit in ("a-extract-top", "a-extract-min", "a-summarise-top", "a-summarise-min"):
|
||||
assert f'id="{limit}"' in template
|
||||
assert 'id="a-extract-top" min="0" step="10" value="500"' in page
|
||||
assert 'id="a-summarise-min" min="0" max="1" step="0.05" value="0.5"' in page
|
||||
assert 'id="a-summarise-min" min="0" max="1" step="0.05" value="0.65"' in page
|
||||
assert 'id="a-new"' in page and 'id="a-delete"' in page
|
||||
assert "Save and score" in page
|
||||
|
||||
@@ -823,8 +823,8 @@ def test_an_agents_limits_are_saved_listed_and_checked(client: TestClient) -> No
|
||||
"""How deep the model stages read is the agent's own; a score outside
|
||||
0..1 is refused."""
|
||||
default = client.post("/api/agents", json={"name": "Wide"}).json()
|
||||
assert default["limits"] == {"extract_top": 500, "extract_min": 0.3,
|
||||
"summarise_top": 30, "summarise_min": 0.5}
|
||||
assert default["limits"] == {"extract_top": 500, "extract_min": 0.55,
|
||||
"summarise_top": 30, "summarise_min": 0.65}
|
||||
narrow = {"extract_top": 50, "extract_min": 0.6, "summarise_top": 5, "summarise_min": 0.7}
|
||||
saved = client.post("/api/agents", json={"name": "Narrow", "limits": narrow})
|
||||
assert saved.status_code == 200 and saved.json()["limits"] == narrow
|
||||
|
||||
Reference in new issue
Block a user