Calibrate fit and taste per embedding model instead of min-max per run
scale() maps a cosine to clip((cos - lo) / (hi - lo), 0, 1) with constants from CALIBRATION in app/config.py, keyed by model and replaceable under embed.calibration. multilingual-e5-small: fit 0.75..0.88, taste -0.14..0.19 (taste from synthetic votes). An uncalibrated model falls back to min-max and logs a warning once per run. The extract and summarise bars default to 0.55 and 0.65, about 300 and 35 roles for the starter agent on the measured data; stored limits are kept. An agent without an intent is left out of the model-stage scope, and each score run names such agents in one score.no_intent notice. Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
1 parent
706b127bc0
commit
b021838959
12 files changed
+336
-76
No files matched your search
+11
-6
@@ -1033,24 +1033,29 @@ def test_summarise_waits_on_the_top_of_each_enabled_agent_by_its_own_limits(
|
||||
a, b, c, d = (posting(s).uid for s in "abcd")
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (store.now(), d))
|
||||
one = store.save_agent("One", limits=Limits(summarise_top=1, summarise_min=0))
|
||||
two = store.save_agent("Two", limits=Limits(summarise_top=1, summarise_min=0))
|
||||
one = store.save_agent("One", "python backend",
|
||||
limits=Limits(summarise_top=1, summarise_min=0))
|
||||
two = store.save_agent("Two", "python backend",
|
||||
limits=Limits(summarise_top=1, summarise_min=0))
|
||||
store.replace_scores(one.id, [(d, 0.99, ""), (a, 0.9, ""), (b, 0.5, "")])
|
||||
store.replace_scores(two.id, [(a, 0.95, "remote"), (c, 0.7, ""), (b, 0.8, "")])
|
||||
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b}
|
||||
|
||||
# Two goes deeper: its second, c, joins; One's second, b, was in already.
|
||||
store.save_agent("Two", agent_id=two.id, limits=Limits(summarise_top=2, summarise_min=0))
|
||||
store.save_agent("Two", "python backend", agent_id=two.id,
|
||||
limits=Limits(summarise_top=2, summarise_min=0))
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b, c}
|
||||
assert store.summarise_pending() == 3
|
||||
assert [r.uid for r in store.summarise_waiting(10, uids=[d])] == [d]
|
||||
# The floor is Two's own: c at 0.7 drops, One's b at 0.5 stays.
|
||||
store.save_agent("Two", agent_id=two.id,
|
||||
store.save_agent("Two", "python backend", agent_id=two.id,
|
||||
limits=Limits(summarise_top=2, summarise_min=0.75))
|
||||
store.save_agent("One", agent_id=one.id, limits=Limits(summarise_top=2, summarise_min=0))
|
||||
store.save_agent("One", "python backend", agent_id=one.id,
|
||||
limits=Limits(summarise_top=2, summarise_min=0))
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b}
|
||||
# '{}' is the defaults, 30 at 0.5: One's b at 0.5 stays, Two's c at 0.7 is back.
|
||||
# '{}' is the defaults, 30 at 0.65: One's b at 0.5 drops, but Two's b at
|
||||
# 0.8 and c at 0.7 clear it.
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE agents SET limits = '{}'")
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b, c}
|
||||
|
||||
Reference in new issue
Block a user