Scale fit across each agent's run, and lower the starter prefer weights

With fit as (cos + 1) / 2 the real run spanned about 0.55 to 0.79, so a
prefer gate at weight 0.2 moved a score more than the whole spread of fit.
fit is now the cosine min-max scaled across the roles an agent scores in
that run: the closest is 1, the farthest 0, a single role 1, and a role a
require gate rules out stays out of the range. The starter agents' prefer
weights drop to 0.05, enough to break a tie. A regression test pins the
case: two close cosines that the old mapping let a 0.05 gate reorder.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-23 12:51:08 +02:00
1 parent 59a301c351
commit 8b246c3035
5 files changed
+132 -31

No files matched your search

+75 -9
View File
@@ -216,19 +216,83 @@ def test_an_older_role_scores_lower(
assert scores[new][0] > scores[old][0]
def test_two_agents_score_the_same_role_differently(
def test_two_agents_rank_the_same_roles_the_other_way_round(
store: Store, config: Config, fake: FakeEmbedder) -> None:
uid = role(store, "1", "Backend Engineer", body="Python services")
backend = store.save_agent("Backend", "python backend developer",
weights=Weights(fit=1, recency=0, affinity=0))
frontend = store.save_agent("Frontend", "react frontend developer",
weights=Weights(fit=1, recency=0, affinity=0))
back = role(store, "1", "Backend Engineer", body="Python services")
front = role(store, "2", "Frontend Engineer", body="React apps")
only_fit = Weights(fit=1, recency=0, affinity=0)
backend = store.save_agent("Backend", "python backend developer", weights=only_fit)
frontend = store.save_agent("Frontend", "react frontend developer", weights=only_fit)
run(config, store, embed.run_embed)
result = run(config, store, score.run_score)
assert result["agents"] == 2
assert store.scores(backend.id)[uid][0] > store.scores(frontend.id)[uid][0]
# Fit alone, scaled per agent: each agent's closer role is 1, the other 0.
assert store.scores(backend.id)[back][0] == 1.0
assert store.scores(backend.id)[front][0] == 0.0
assert store.scores(frontend.id)[front][0] == 1.0
assert store.scores(frontend.id)[back][0] == 0.0
# ------------------------------------------------------------ fit scaling
def test_scale_maps_the_closest_to_one_and_the_farthest_to_zero() -> None:
# The spread the last real run showed under `(cos + 1) / 2`.
assert score.scale({"a": 0.55, "b": 0.79, "c": 0.67}) == pytest.approx(
{"a": 0.0, "b": 1.0, "c": 0.5})
assert score.scale({"only": 0.3}) == {"only": 1.0}
assert score.scale({"a": 0.4, "b": 0.4}) == {"a": 1.0, "b": 1.0}
assert score.scale({}) == {}
def test_a_single_candidate_gets_full_fit(
store: Store, config: Config, fake: FakeEmbedder) -> None:
uid = role(store, "1", "Frontend Engineer", body="React apps")
agent = store.save_agent("Backend", "python backend developer",
weights=Weights(fit=1, recency=0, affinity=0))
run(config, store, embed.run_embed)
run(config, store, score.run_score)
assert store.scores(agent.id)[uid] == (1.0, "")
def test_a_gated_role_does_not_stretch_the_fit_range(
store: Store, config: Config, fake: FakeEmbedder) -> None:
close = role(store, "1", "Backend Engineer", body="Python backend", place="Berlin")
far = role(store, "2", "Data Engineer", body="Data", place="Berlin")
# Farther from the intent than either, but gated out by place.
role(store, "3", "Frontend Engineer", body="React react", place="Munich")
agent = store.save_agent("Berlin", "python backend",
[Gate(field="place", values=["Berlin"])],
weights=Weights(fit=1, recency=0, affinity=0))
run(config, store, embed.run_embed)
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[close][0] == 1.0 and scores[far][0] == 0.0
def test_a_small_prefer_gate_no_longer_outweighs_fit(
store: Store, config: Config, fake: FakeEmbedder) -> None:
"""The flaw the last real run showed: real cosines sit close together, so
under `(cos + 1) / 2` a prefer gate decided the order over fit. Here the
cosines are about 0.95 and 0.87: the old fit terms differ by 0.04, which
at weight 0.6 is less than the gate's 0.05, so the remote role won. Scaled,
the closer role wins."""
close = role(store, "1", "Backend Engineer", body="Python backend", remote="onsite")
far = role(store, "2", "Backend Engineer", body="Python backend data", remote="remote")
agent = store.save_agent("Backend", "python backend",
[Gate(field="remote", values=["remote"], mode="prefer",
weight=0.05)])
run(config, store, embed.run_embed)
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[close][0] > scores[far][0]
def test_the_intent_vector_is_cached_until_the_intent_changes(
@@ -266,8 +330,10 @@ def test_starter_agents_are_imported_once(store: Store, tmp_path: Path) -> None:
def test_the_shipped_starter_agents_validate() -> None:
from app import config as config_mod
names = [a.name for a in config_mod.load(ROOT / "config.yaml").agents]
assert names == ["Berlin backend", "Remote EU stretch"]
agents = config_mod.load(ROOT / "config.yaml").agents
assert [a.name for a in agents] == ["Berlin backend", "Remote EU stretch"]
# Small enough that a prefer gate breaks ties rather than overrides fit.
assert {g.weight for a in agents for g in a.gates if g.mode == "prefer"} == {0.05}
# ---------------------------------------------------------------- cascade