Calibrate fit and taste per embedding model instead of min-max per run

scale() maps a cosine to clip((cos - lo) / (hi - lo), 0, 1) with constants
from CALIBRATION in app/config.py, keyed by model and replaceable under
embed.calibration. multilingual-e5-small: fit 0.75..0.88, taste -0.14..0.19
(taste from synthetic votes). An uncalibrated model falls back to min-max
and logs a warning once per run.

The extract and summarise bars default to 0.55 and 0.65, about 300 and 35
roles for the starter agent on the measured data; stored limits are kept.
An agent without an intent is left out of the model-stage scope, and each
score run names such agents in one score.no_intent notice.

Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus committed 2026-09-29 14:59:34 +02:00
1 parent 706b127bc0
commit b021838959
12 files changed
+336 -76

No files matched your search

+170 -17
View File
@@ -3,6 +3,7 @@ by a deterministic fake. Nothing here downloads a model or reaches a board."""
from __future__ import annotations
import re
import sys
from collections.abc import Iterator, Sequence
from datetime import UTC, datetime, timedelta
@@ -19,13 +20,17 @@ if str(ROOT) not in sys.path:
from desk.models import Item # noqa: E402
from pipeline import FINAL, JOBS, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, Tally # noqa: E402
from plugins.ats import ATSSource # noqa: E402
from app.config import AgentSeed, Config # noqa: E402
from app.config import CALIBRATION, AgentSeed, Config, ConfigError # noqa: E402
from app.jobs import cascade, embed, score # noqa: E402
from app.models import Candidate, Gate, Weights # noqa: E402
from app.models import Candidate, Gate, Limits, Weights # noqa: E402
from app.storage import Store # noqa: E402
NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
#: The shipped constants, which the fake is calibrated with so a test reads
#: the rule production runs; `uncalibrated` is the min-max fallback.
E5 = CALIBRATION["intfloat/multilingual-e5-small"]
#: The fake's whole vocabulary: one dimension per word, plus a constant one so
#: no text is the zero vector.
WORDS = ("backend", "python", "frontend", "react", "data")
@@ -76,6 +81,12 @@ def store(tmp_path: Path, clock: Clock) -> Iterator[Store]:
@pytest.fixture
def config(tmp_path: Path) -> Config:
return Config(database=str(tmp_path / "roledesk.db"),
embed={"model": "fake", "calibration": {"fake": E5.model_dump()}})
@pytest.fixture
def uncalibrated(tmp_path: Path) -> Config:
return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
@@ -94,9 +105,17 @@ def run(config: Config, store: Store, job: Any) -> dict[str, Any]:
notices: list[Any] = []
result = dict(job(config, store, context=RunContext(on_notice=notices.append)))
result["notices"] = [n.code for n in notices]
result["notice_texts"] = [n.text for n in notices]
return result
def level(text: str, intent: str, band: tuple[float, float] = E5.fit) -> float:
"""The calibrated fit the fake gives a role whose `document` is `text`."""
role_vec, intent_vec = FakeEmbedder().encode([text, intent])
lo, hi = band
return min(max((score.cosine(role_vec, intent_vec) - lo) / (hi - lo), 0.0), 1.0)
# ------------------------------------------------------------------ embed
def test_embed_writes_a_vector_per_open_role_once(
@@ -431,7 +450,8 @@ def test_two_agents_rank_the_same_roles_the_other_way_round(
result = run(config, store, score.run_score)
assert result["agents"] == 2
# Fit alone, scaled per agent: each agent's closer role is 1, the other 0.
# Fit alone, calibrated: each agent's own role is at a cosine past `hi`,
# so 1, the other's near 0, below `lo`, so 0.
assert store.scores(backend.id)[back][0] == 1.0
assert store.scores(backend.id)[front][0] == 0.0
assert store.scores(frontend.id)[front][0] == 1.0
@@ -481,6 +501,7 @@ def test_taste_ranks_a_role_near_the_liked_one_above_one_near_the_passed(
scores = store.scores(agent.id)
assert scores[near_liked][0] > scores[near_passed][0]
# Cosines of about 0.7 and -0.7 to the direction, past both ends of the band.
assert scores[liked][0] == 1.0 and scores[passed][0] == 0.0
assert all(reason == "" for _, reason in scores.values())
@@ -513,35 +534,167 @@ def test_a_closed_liked_role_still_shapes_the_taste(
scores = store.scores(agent.id)
assert liked not in scores
assert scores[near][0] == 1.0 and scores[far][0] == 0.0
# Near sits on the liked direction, past `hi`. Far is orthogonal to it,
# a cosine of about 0 and so a middling level, not the 0 min-max gave it.
assert scores[near][0] == 1.0
assert scores[far][0] == pytest.approx(
level("Web Engineer\nReact frontend", "Backend Engineer\nPython backend", E5.taste),
abs=1e-5)
assert 0 < scores[far][0] < 1
# ------------------------------------------------------------ fit scaling
def test_scale_maps_the_closest_to_one_and_the_farthest_to_zero() -> None:
# The spread the last real run showed under `(cos + 1) / 2`, on
# multilingual-e5-small.
assert score.scale({"a": 0.87, "b": 0.95, "c": 0.91}) == pytest.approx(
def test_scale_maps_a_cosine_to_its_calibrated_level_and_clips_both_ends() -> None:
# multilingual-e5-small's fit band: lo 0.75 is 0, hi 0.88 is 1.
assert E5.fit == (0.75, 0.88) and E5.taste == (-0.14, 0.19)
assert score.scale({"lo": 0.75, "mid": 0.815, "hi": 0.88, "p50": 0.7989},
E5.fit) == pytest.approx(
{"lo": 0.0, "mid": 0.5, "hi": 1.0, "p50": 0.0489 / 0.13})
assert score.scale({"under": 0.6, "negative": -0.2, "over": 0.9031, "one": 1.0},
E5.fit) == {"under": 0.0, "negative": 0.0, "over": 1.0, "one": 1.0}
assert score.scale({"none": 0.0, "liked": 0.25}, E5.taste) == pytest.approx(
{"none": 0.14 / 0.33, "liked": 1.0})
# A lone role, or roles that are all equally close, get their level.
assert score.scale({"only": 0.8}, E5.fit) == pytest.approx({"only": 0.05 / 0.13})
assert score.scale({}, E5.fit) == {}
def test_scale_without_a_band_falls_back_to_min_max() -> None:
assert score.scale({"a": 0.87, "b": 0.95, "c": 0.91}, None) == pytest.approx(
{"a": 0.0, "b": 1.0, "c": 0.5})
assert score.scale({"only": 0.3}) == {"only": 1.0}
assert score.scale({"a": 0.4, "b": 0.4}) == {"a": 1.0, "b": 1.0}
assert score.scale({}) == {}
assert score.scale({"only": 0.3}, None) == {"only": 1.0}
assert score.scale({"a": 0.4, "b": 0.4}, None) == {"a": 1.0, "b": 1.0}
assert score.scale({}, None) == {}
def test_a_single_candidate_gets_full_fit(
store: Store, config: Config, fake: FakeEmbedder) -> None:
def test_a_single_candidate_gets_its_level_not_full_fit(
store: Store, config: Config, uncalibrated: Config, fake: FakeEmbedder) -> None:
"""Min-max gave the best of a poor set 1; a level gives it what it is."""
uid = role(store, "1", "Frontend Engineer", body="React apps")
agent = store.save_agent("Backend", "python backend developer",
weights=Weights(fit=1, recency=0, affinity=0))
run(config, store, embed.run_embed)
run(config, store, score.run_score)
assert store.scores(agent.id)[uid] == (0.0, "")
run(uncalibrated, store, score.run_score)
assert store.scores(agent.id)[uid] == (1.0, "")
def test_a_gated_role_does_not_stretch_the_fit_range(
store: Store, config: Config, fake: FakeEmbedder) -> None:
def test_the_same_role_gets_the_same_fit_whatever_else_is_open(
store: Store, config: Config, uncalibrated: Config, fake: FakeEmbedder) -> None:
"""The point of a level: two runs over different candidate sets give the
role the same fit, where min-max gave it 1 beside a worse role and 0
beside a better one."""
mid = role(store, "1", "Backend Engineer", body="Python backend data")
worse = role(store, "2", "Frontend Engineer", body="React apps")
agent = store.save_agent("Backend", "python backend",
weights=Weights(fit=1, recency=0, affinity=0))
expected = level("Backend Engineer\nPython backend data", "python backend")
assert 0.5 < expected < 1
run(config, store, embed.run_embed)
run(config, store, score.run_score)
first = store.scores(agent.id)[mid][0]
run(uncalibrated, store, score.run_score)
assert store.scores(agent.id)[mid][0] == 1.0
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (NOW.isoformat(), worse))
role(store, "3", "Backend Engineer", body="Python")
run(config, store, embed.run_embed)
run(config, store, score.run_score)
second = store.scores(agent.id)[mid][0]
run(uncalibrated, store, score.run_score)
assert store.scores(agent.id)[mid][0] == 0.0
assert first == second == pytest.approx(expected, abs=1e-5)
def test_an_uncalibrated_model_falls_back_to_min_max_and_warns_once_a_run(
store: Store, uncalibrated: Config, fake: FakeEmbedder,
caplog: pytest.LogCaptureFixture) -> None:
close = role(store, "1", "Backend Engineer", body="Python backend data")
far = role(store, "2", "Data Engineer", body="Data")
only_fit = Weights(fit=1, recency=0, affinity=0)
one = store.save_agent("One", "python backend", weights=only_fit)
two = store.save_agent("Two", "python backend", weights=only_fit)
run(uncalibrated, store, embed.run_embed)
with caplog.at_level("WARNING", logger="app.jobs.score"):
run(uncalibrated, store, score.run_score)
warnings = [r.getMessage() for r in caplog.records if r.name == "app.jobs.score"]
assert len(warnings) == 1 and warnings[0].startswith("no calibration for fake")
for agent in (one, two):
assert store.scores(agent.id)[close][0] == 1.0
assert store.scores(agent.id)[far][0] == 0.0
def test_the_calibration_is_the_built_in_one_unless_the_config_replaces_it() -> None:
assert Config().embed.calibrated() == E5
assert Config(embed={"model": "fake"}).embed.calibrated() is None
mine = {"fit": [0.7, 0.9], "taste": [-0.2, 0.2]}
replaced = Config(embed={"calibration": {"intfloat/multilingual-e5-small": mine}})
assert replaced.embed.calibrated() is not None
assert replaced.embed.calibrated().fit == (0.7, 0.9) # type: ignore[union-attr]
# Keyed by model: another model's entry does not apply.
other = Config(embed={"model": "fake", "calibration": {"other": mine}})
assert other.embed.calibrated() is None
@pytest.mark.parametrize("entry, says", [
({"fit": [0.88, 0.75], "taste": [-0.14, 0.19]}, "embed.calibration.fake.fit"),
({"fit": [0.75, 0.75], "taste": [-0.14, 0.19]}, "lo < hi"),
({"fit": [0.75, 0.88], "taste": [-2, 0.19]}, "embed.calibration.fake.taste"),
({"fit": [0.75, 0.88]}, "embed.calibration.fake.taste"),
({"fit": [0.75, 0.88], "taste": [-0.14, 0.19], "spread": 1}, "unknown setting"),
])
def test_a_bad_calibration_is_a_config_error(
tmp_path: Path, entry: dict[str, Any], says: str) -> None:
import yaml
from app import config as config_mod
path = tmp_path / "config.yaml"
path.write_text(yaml.safe_dump({"embed": {"model": "fake",
"calibration": {"fake": entry}}}))
with pytest.raises(ConfigError, match=re.escape(says)):
config_mod.load(path)
def test_an_agent_without_an_intent_feeds_neither_model_stage_and_is_named(
store: Store, config: Config, clock: Clock, fake: FakeEmbedder) -> None:
"""Its scores have no fit, so they rank fresh roles, not wanted ones: the
newer role tops it, and extract and summarise must not read it on that."""
clock.at = NOW - timedelta(days=1)
wanted = role(store, "1", "Backend Engineer", body="Python backend")
clock.at = NOW
fresh = role(store, "2", "Frontend Engineer", body="React apps")
top_one = Limits(extract_top=1, extract_min=0, summarise_top=1, summarise_min=0)
store.save_agent("Backend", "python backend", limits=top_one)
blind = store.save_agent("Fresh", "", limits=top_one)
store.save_agent("Also fresh", " \n", limits=top_one)
store.save_agent("Off", "", enabled=False)
run(config, store, embed.run_embed)
result = run(config, store, score.run_score)
assert result["notices"] == ["score.no_intent"]
assert result["notice_texts"][0].endswith(": Also fresh, Fresh.")
ranked = sorted(store.scores(blind.id).items(), key=lambda kv: -kv[1][0])
assert ranked[0][0] == fresh, "the intent-less agent would pick the newer role"
assert [r.uid for r in store.extract_waiting(10)] == [wanted]
assert [r.uid for r in store.summarise_waiting(10)] == [wanted]
assert store.extract_pending() == store.summarise_pending() == 1
def test_a_gated_role_does_not_stretch_the_fit_range_under_the_fallback(
store: Store, uncalibrated: Config, fake: FakeEmbedder) -> None:
"""Calibrated, no other role moves a role's fit. Min-max, the fallback,
must still leave a gated role out of the range."""
config = uncalibrated
close = role(store, "1", "Backend Engineer", body="Python backend", place="Berlin")
far = role(store, "2", "Data Engineer", body="Data", place="Berlin")
# Farther from the intent than either, but gated out by place.
@@ -562,8 +715,8 @@ def test_a_small_prefer_gate_no_longer_outweighs_fit(
"""The flaw the last real run showed: real cosines sit close together, so
under `(cos + 1) / 2` a prefer gate decided the order over fit. Here the
cosines are about 0.95 and 0.87: the old fit terms differ by 0.04, which
at weight 0.6 is less than the gate's 0.05, so the remote role won. Scaled,
the closer role wins."""
at weight 0.6 is less than the gate's 0.05, so the remote role won.
Calibrated, they are 1 and about 0.89, and the closer role wins."""
close = role(store, "1", "Backend Engineer", body="Python backend", remote="onsite")
far = role(store, "2", "Backend Engineer", body="Python backend data", remote="remote")
agent = store.save_agent("Backend", "python backend",