Calibrate fit and taste per embedding model instead of min-max per run
scale() maps a cosine to clip((cos - lo) / (hi - lo), 0, 1) with constants from CALIBRATION in app/config.py, keyed by model and replaceable under embed.calibration. multilingual-e5-small: fit 0.75..0.88, taste -0.14..0.19 (taste from synthetic votes). An uncalibrated model falls back to min-max and logs a warning once per run. The extract and summarise bars default to 0.55 and 0.65, about 300 and 35 roles for the starter agent on the measured data; stored limits are kept. An agent without an intent is left out of the model-stage scope, and each score run names such agents in one score.no_intent notice. Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
1 parent
706b127bc0
commit
b021838959
12 files changed
+336
-76
No files matched your search
+170
-17
@@ -3,6 +3,7 @@ by a deterministic fake. Nothing here downloads a model or reaches a board."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Iterator, Sequence
|
||||
from datetime import UTC, datetime, timedelta
|
||||
@@ -19,13 +20,17 @@ if str(ROOT) not in sys.path:
|
||||
from desk.models import Item # noqa: E402
|
||||
from pipeline import FINAL, JOBS, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, Tally # noqa: E402
|
||||
from plugins.ats import ATSSource # noqa: E402
|
||||
from app.config import AgentSeed, Config # noqa: E402
|
||||
from app.config import CALIBRATION, AgentSeed, Config, ConfigError # noqa: E402
|
||||
from app.jobs import cascade, embed, score # noqa: E402
|
||||
from app.models import Candidate, Gate, Weights # noqa: E402
|
||||
from app.models import Candidate, Gate, Limits, Weights # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
|
||||
NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
|
||||
|
||||
#: The shipped constants, which the fake is calibrated with so a test reads
|
||||
#: the rule production runs; `uncalibrated` is the min-max fallback.
|
||||
E5 = CALIBRATION["intfloat/multilingual-e5-small"]
|
||||
|
||||
#: The fake's whole vocabulary: one dimension per word, plus a constant one so
|
||||
#: no text is the zero vector.
|
||||
WORDS = ("backend", "python", "frontend", "react", "data")
|
||||
@@ -76,6 +81,12 @@ def store(tmp_path: Path, clock: Clock) -> Iterator[Store]:
|
||||
|
||||
@pytest.fixture
|
||||
def config(tmp_path: Path) -> Config:
|
||||
return Config(database=str(tmp_path / "roledesk.db"),
|
||||
embed={"model": "fake", "calibration": {"fake": E5.model_dump()}})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def uncalibrated(tmp_path: Path) -> Config:
|
||||
return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
|
||||
|
||||
|
||||
@@ -94,9 +105,17 @@ def run(config: Config, store: Store, job: Any) -> dict[str, Any]:
|
||||
notices: list[Any] = []
|
||||
result = dict(job(config, store, context=RunContext(on_notice=notices.append)))
|
||||
result["notices"] = [n.code for n in notices]
|
||||
result["notice_texts"] = [n.text for n in notices]
|
||||
return result
|
||||
|
||||
|
||||
def level(text: str, intent: str, band: tuple[float, float] = E5.fit) -> float:
|
||||
"""The calibrated fit the fake gives a role whose `document` is `text`."""
|
||||
role_vec, intent_vec = FakeEmbedder().encode([text, intent])
|
||||
lo, hi = band
|
||||
return min(max((score.cosine(role_vec, intent_vec) - lo) / (hi - lo), 0.0), 1.0)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ embed
|
||||
|
||||
def test_embed_writes_a_vector_per_open_role_once(
|
||||
@@ -431,7 +450,8 @@ def test_two_agents_rank_the_same_roles_the_other_way_round(
|
||||
result = run(config, store, score.run_score)
|
||||
|
||||
assert result["agents"] == 2
|
||||
# Fit alone, scaled per agent: each agent's closer role is 1, the other 0.
|
||||
# Fit alone, calibrated: each agent's own role is at a cosine past `hi`,
|
||||
# so 1, the other's near 0, below `lo`, so 0.
|
||||
assert store.scores(backend.id)[back][0] == 1.0
|
||||
assert store.scores(backend.id)[front][0] == 0.0
|
||||
assert store.scores(frontend.id)[front][0] == 1.0
|
||||
@@ -481,6 +501,7 @@ def test_taste_ranks_a_role_near_the_liked_one_above_one_near_the_passed(
|
||||
|
||||
scores = store.scores(agent.id)
|
||||
assert scores[near_liked][0] > scores[near_passed][0]
|
||||
# Cosines of about 0.7 and -0.7 to the direction, past both ends of the band.
|
||||
assert scores[liked][0] == 1.0 and scores[passed][0] == 0.0
|
||||
assert all(reason == "" for _, reason in scores.values())
|
||||
|
||||
@@ -513,35 +534,167 @@ def test_a_closed_liked_role_still_shapes_the_taste(
|
||||
|
||||
scores = store.scores(agent.id)
|
||||
assert liked not in scores
|
||||
assert scores[near][0] == 1.0 and scores[far][0] == 0.0
|
||||
# Near sits on the liked direction, past `hi`. Far is orthogonal to it,
|
||||
# a cosine of about 0 and so a middling level, not the 0 min-max gave it.
|
||||
assert scores[near][0] == 1.0
|
||||
assert scores[far][0] == pytest.approx(
|
||||
level("Web Engineer\nReact frontend", "Backend Engineer\nPython backend", E5.taste),
|
||||
abs=1e-5)
|
||||
assert 0 < scores[far][0] < 1
|
||||
|
||||
|
||||
# ------------------------------------------------------------ fit scaling
|
||||
|
||||
def test_scale_maps_the_closest_to_one_and_the_farthest_to_zero() -> None:
|
||||
# The spread the last real run showed under `(cos + 1) / 2`, on
|
||||
# multilingual-e5-small.
|
||||
assert score.scale({"a": 0.87, "b": 0.95, "c": 0.91}) == pytest.approx(
|
||||
def test_scale_maps_a_cosine_to_its_calibrated_level_and_clips_both_ends() -> None:
|
||||
# multilingual-e5-small's fit band: lo 0.75 is 0, hi 0.88 is 1.
|
||||
assert E5.fit == (0.75, 0.88) and E5.taste == (-0.14, 0.19)
|
||||
assert score.scale({"lo": 0.75, "mid": 0.815, "hi": 0.88, "p50": 0.7989},
|
||||
E5.fit) == pytest.approx(
|
||||
{"lo": 0.0, "mid": 0.5, "hi": 1.0, "p50": 0.0489 / 0.13})
|
||||
assert score.scale({"under": 0.6, "negative": -0.2, "over": 0.9031, "one": 1.0},
|
||||
E5.fit) == {"under": 0.0, "negative": 0.0, "over": 1.0, "one": 1.0}
|
||||
assert score.scale({"none": 0.0, "liked": 0.25}, E5.taste) == pytest.approx(
|
||||
{"none": 0.14 / 0.33, "liked": 1.0})
|
||||
# A lone role, or roles that are all equally close, get their level.
|
||||
assert score.scale({"only": 0.8}, E5.fit) == pytest.approx({"only": 0.05 / 0.13})
|
||||
assert score.scale({}, E5.fit) == {}
|
||||
|
||||
|
||||
def test_scale_without_a_band_falls_back_to_min_max() -> None:
|
||||
assert score.scale({"a": 0.87, "b": 0.95, "c": 0.91}, None) == pytest.approx(
|
||||
{"a": 0.0, "b": 1.0, "c": 0.5})
|
||||
assert score.scale({"only": 0.3}) == {"only": 1.0}
|
||||
assert score.scale({"a": 0.4, "b": 0.4}) == {"a": 1.0, "b": 1.0}
|
||||
assert score.scale({}) == {}
|
||||
assert score.scale({"only": 0.3}, None) == {"only": 1.0}
|
||||
assert score.scale({"a": 0.4, "b": 0.4}, None) == {"a": 1.0, "b": 1.0}
|
||||
assert score.scale({}, None) == {}
|
||||
|
||||
|
||||
def test_a_single_candidate_gets_full_fit(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
def test_a_single_candidate_gets_its_level_not_full_fit(
|
||||
store: Store, config: Config, uncalibrated: Config, fake: FakeEmbedder) -> None:
|
||||
"""Min-max gave the best of a poor set 1; a level gives it what it is."""
|
||||
uid = role(store, "1", "Frontend Engineer", body="React apps")
|
||||
agent = store.save_agent("Backend", "python backend developer",
|
||||
weights=Weights(fit=1, recency=0, affinity=0))
|
||||
|
||||
run(config, store, embed.run_embed)
|
||||
run(config, store, score.run_score)
|
||||
assert store.scores(agent.id)[uid] == (0.0, "")
|
||||
|
||||
run(uncalibrated, store, score.run_score)
|
||||
assert store.scores(agent.id)[uid] == (1.0, "")
|
||||
|
||||
|
||||
def test_a_gated_role_does_not_stretch_the_fit_range(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
def test_the_same_role_gets_the_same_fit_whatever_else_is_open(
|
||||
store: Store, config: Config, uncalibrated: Config, fake: FakeEmbedder) -> None:
|
||||
"""The point of a level: two runs over different candidate sets give the
|
||||
role the same fit, where min-max gave it 1 beside a worse role and 0
|
||||
beside a better one."""
|
||||
mid = role(store, "1", "Backend Engineer", body="Python backend data")
|
||||
worse = role(store, "2", "Frontend Engineer", body="React apps")
|
||||
agent = store.save_agent("Backend", "python backend",
|
||||
weights=Weights(fit=1, recency=0, affinity=0))
|
||||
expected = level("Backend Engineer\nPython backend data", "python backend")
|
||||
assert 0.5 < expected < 1
|
||||
|
||||
run(config, store, embed.run_embed)
|
||||
run(config, store, score.run_score)
|
||||
first = store.scores(agent.id)[mid][0]
|
||||
run(uncalibrated, store, score.run_score)
|
||||
assert store.scores(agent.id)[mid][0] == 1.0
|
||||
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (NOW.isoformat(), worse))
|
||||
role(store, "3", "Backend Engineer", body="Python")
|
||||
run(config, store, embed.run_embed)
|
||||
run(config, store, score.run_score)
|
||||
second = store.scores(agent.id)[mid][0]
|
||||
run(uncalibrated, store, score.run_score)
|
||||
assert store.scores(agent.id)[mid][0] == 0.0
|
||||
|
||||
assert first == second == pytest.approx(expected, abs=1e-5)
|
||||
|
||||
|
||||
def test_an_uncalibrated_model_falls_back_to_min_max_and_warns_once_a_run(
|
||||
store: Store, uncalibrated: Config, fake: FakeEmbedder,
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
close = role(store, "1", "Backend Engineer", body="Python backend data")
|
||||
far = role(store, "2", "Data Engineer", body="Data")
|
||||
only_fit = Weights(fit=1, recency=0, affinity=0)
|
||||
one = store.save_agent("One", "python backend", weights=only_fit)
|
||||
two = store.save_agent("Two", "python backend", weights=only_fit)
|
||||
run(uncalibrated, store, embed.run_embed)
|
||||
|
||||
with caplog.at_level("WARNING", logger="app.jobs.score"):
|
||||
run(uncalibrated, store, score.run_score)
|
||||
|
||||
warnings = [r.getMessage() for r in caplog.records if r.name == "app.jobs.score"]
|
||||
assert len(warnings) == 1 and warnings[0].startswith("no calibration for fake")
|
||||
for agent in (one, two):
|
||||
assert store.scores(agent.id)[close][0] == 1.0
|
||||
assert store.scores(agent.id)[far][0] == 0.0
|
||||
|
||||
|
||||
def test_the_calibration_is_the_built_in_one_unless_the_config_replaces_it() -> None:
|
||||
assert Config().embed.calibrated() == E5
|
||||
assert Config(embed={"model": "fake"}).embed.calibrated() is None
|
||||
mine = {"fit": [0.7, 0.9], "taste": [-0.2, 0.2]}
|
||||
replaced = Config(embed={"calibration": {"intfloat/multilingual-e5-small": mine}})
|
||||
assert replaced.embed.calibrated() is not None
|
||||
assert replaced.embed.calibrated().fit == (0.7, 0.9) # type: ignore[union-attr]
|
||||
# Keyed by model: another model's entry does not apply.
|
||||
other = Config(embed={"model": "fake", "calibration": {"other": mine}})
|
||||
assert other.embed.calibrated() is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entry, says", [
|
||||
({"fit": [0.88, 0.75], "taste": [-0.14, 0.19]}, "embed.calibration.fake.fit"),
|
||||
({"fit": [0.75, 0.75], "taste": [-0.14, 0.19]}, "lo < hi"),
|
||||
({"fit": [0.75, 0.88], "taste": [-2, 0.19]}, "embed.calibration.fake.taste"),
|
||||
({"fit": [0.75, 0.88]}, "embed.calibration.fake.taste"),
|
||||
({"fit": [0.75, 0.88], "taste": [-0.14, 0.19], "spread": 1}, "unknown setting"),
|
||||
])
|
||||
def test_a_bad_calibration_is_a_config_error(
|
||||
tmp_path: Path, entry: dict[str, Any], says: str) -> None:
|
||||
import yaml
|
||||
from app import config as config_mod
|
||||
|
||||
path = tmp_path / "config.yaml"
|
||||
path.write_text(yaml.safe_dump({"embed": {"model": "fake",
|
||||
"calibration": {"fake": entry}}}))
|
||||
with pytest.raises(ConfigError, match=re.escape(says)):
|
||||
config_mod.load(path)
|
||||
|
||||
|
||||
def test_an_agent_without_an_intent_feeds_neither_model_stage_and_is_named(
|
||||
store: Store, config: Config, clock: Clock, fake: FakeEmbedder) -> None:
|
||||
"""Its scores have no fit, so they rank fresh roles, not wanted ones: the
|
||||
newer role tops it, and extract and summarise must not read it on that."""
|
||||
clock.at = NOW - timedelta(days=1)
|
||||
wanted = role(store, "1", "Backend Engineer", body="Python backend")
|
||||
clock.at = NOW
|
||||
fresh = role(store, "2", "Frontend Engineer", body="React apps")
|
||||
top_one = Limits(extract_top=1, extract_min=0, summarise_top=1, summarise_min=0)
|
||||
store.save_agent("Backend", "python backend", limits=top_one)
|
||||
blind = store.save_agent("Fresh", "", limits=top_one)
|
||||
store.save_agent("Also fresh", " \n", limits=top_one)
|
||||
store.save_agent("Off", "", enabled=False)
|
||||
|
||||
run(config, store, embed.run_embed)
|
||||
result = run(config, store, score.run_score)
|
||||
|
||||
assert result["notices"] == ["score.no_intent"]
|
||||
assert result["notice_texts"][0].endswith(": Also fresh, Fresh.")
|
||||
ranked = sorted(store.scores(blind.id).items(), key=lambda kv: -kv[1][0])
|
||||
assert ranked[0][0] == fresh, "the intent-less agent would pick the newer role"
|
||||
assert [r.uid for r in store.extract_waiting(10)] == [wanted]
|
||||
assert [r.uid for r in store.summarise_waiting(10)] == [wanted]
|
||||
assert store.extract_pending() == store.summarise_pending() == 1
|
||||
|
||||
|
||||
def test_a_gated_role_does_not_stretch_the_fit_range_under_the_fallback(
|
||||
store: Store, uncalibrated: Config, fake: FakeEmbedder) -> None:
|
||||
"""Calibrated, no other role moves a role's fit. Min-max, the fallback,
|
||||
must still leave a gated role out of the range."""
|
||||
config = uncalibrated
|
||||
close = role(store, "1", "Backend Engineer", body="Python backend", place="Berlin")
|
||||
far = role(store, "2", "Data Engineer", body="Data", place="Berlin")
|
||||
# Farther from the intent than either, but gated out by place.
|
||||
@@ -562,8 +715,8 @@ def test_a_small_prefer_gate_no_longer_outweighs_fit(
|
||||
"""The flaw the last real run showed: real cosines sit close together, so
|
||||
under `(cos + 1) / 2` a prefer gate decided the order over fit. Here the
|
||||
cosines are about 0.95 and 0.87: the old fit terms differ by 0.04, which
|
||||
at weight 0.6 is less than the gate's 0.05, so the remote role won. Scaled,
|
||||
the closer role wins."""
|
||||
at weight 0.6 is less than the gate's 0.05, so the remote role won.
|
||||
Calibrated, they are 1 and about 0.89, and the closer role wins."""
|
||||
close = role(store, "1", "Backend Engineer", body="Python backend", remote="onsite")
|
||||
far = role(store, "2", "Backend Engineer", body="Python backend data", remote="remote")
|
||||
agent = store.save_agent("Backend", "python backend",
|
||||
|
||||
Reference in new issue
Block a user