Calibrate fit and taste per embedding model instead of min-max per run
scale() maps a cosine to clip((cos - lo) / (hi - lo), 0, 1) with constants from CALIBRATION in app/config.py, keyed by model and replaceable under embed.calibration. multilingual-e5-small: fit 0.75..0.88, taste -0.14..0.19 (taste from synthetic votes). An uncalibrated model falls back to min-max and logs a warning once per run. The extract and summarise bars default to 0.55 and 0.65, about 300 and 35 roles for the starter agent on the measured data; stored limits are kept. An agent without an intent is left out of the model-stage scope, and each score run names such agents in one score.no_intent notice. Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
1 parent
706b127bc0
commit
b021838959
12 files changed
+336
-76
No files matched your search
+10
-6
@@ -62,7 +62,7 @@ def ranked(store: Store, *items: Item) -> None:
|
||||
"""Score the roles for one enabled agent, best first, so they are in the
|
||||
top extract reads. Every role given must already be stored."""
|
||||
agents = store.agents()
|
||||
agent_id = agents[0].id if agents else store.save_agent("Backend").id
|
||||
agent_id = agents[0].id if agents else store.save_agent("Backend", "python backend").id
|
||||
store.replace_scores(agent_id, [(item.uid, 1.0 - n / 10, "")
|
||||
for n, item in enumerate(items)])
|
||||
|
||||
@@ -171,8 +171,9 @@ def test_only_the_best_of_each_agent_above_the_floor_is_read(
|
||||
uid finishes it."""
|
||||
for sid in ("best", "second", "low", "unscored", "gated"):
|
||||
store.upsert_role(posting(sid))
|
||||
one = store.save_agent("One", limits=Limits(extract_top=1, extract_min=0.5))
|
||||
two = store.save_agent("Two", enabled=False)
|
||||
one = store.save_agent("One", "python backend",
|
||||
limits=Limits(extract_top=1, extract_min=0.5))
|
||||
two = store.save_agent("Two", "python backend", enabled=False)
|
||||
store.replace_scores(one.id, [(posting("best").uid, 0.9, ""),
|
||||
(posting("second").uid, 0.8, ""),
|
||||
(posting("low").uid, 0.2, ""),
|
||||
@@ -182,11 +183,14 @@ def test_only_the_best_of_each_agent_above_the_floor_is_read(
|
||||
model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles])
|
||||
|
||||
assert store.extract_pending() == 1
|
||||
store.save_agent("One", agent_id=one.id, limits=Limits(extract_top=5, extract_min=0.5))
|
||||
store.save_agent("One", "python backend", agent_id=one.id,
|
||||
limits=Limits(extract_top=5, extract_min=0.5))
|
||||
assert store.extract_pending() == 2, "the floor holds the low one back"
|
||||
store.save_agent("One", agent_id=one.id, limits=Limits(extract_top=5, extract_min=0))
|
||||
store.save_agent("One", "python backend", agent_id=one.id,
|
||||
limits=Limits(extract_top=5, extract_min=0))
|
||||
assert store.extract_pending() == 3, "the disabled agent's pick never counts"
|
||||
store.save_agent("One", agent_id=one.id, limits=Limits(extract_top=1, extract_min=0.5))
|
||||
store.save_agent("One", "python backend", agent_id=one.id,
|
||||
limits=Limits(extract_top=1, extract_min=0.5))
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
|
||||
assert [[r["uid"] for r in batch] for batch in model.asked] == [[posting("best").uid]]
|
||||
|
||||
Reference in new issue
Block a user