Scale fit across each agent's run, and lower the starter prefer weights
With fit as (cos + 1) / 2 the real run spanned about 0.55 to 0.79, so a prefer gate at weight 0.2 moved a score more than the whole spread of fit. fit is now the cosine min-max scaled across the roles an agent scores in that run: the closest is 1, the farthest 0, a single role 1, and a role a require gate rules out stays out of the range. The starter agents' prefer weights drop to 0.05, enough to break a tie. A regression test pins the case: two close cosines that the old mapping let a 0.05 gate reorder. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
59a301c351
commit
8b246c3035
5 files changed
+132
-31
No files matched your search
+75
-9
@@ -216,19 +216,83 @@ def test_an_older_role_scores_lower(
|
||||
assert scores[new][0] > scores[old][0]
|
||||
|
||||
|
||||
def test_two_agents_score_the_same_role_differently(
|
||||
def test_two_agents_rank_the_same_roles_the_other_way_round(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
uid = role(store, "1", "Backend Engineer", body="Python services")
|
||||
backend = store.save_agent("Backend", "python backend developer",
|
||||
weights=Weights(fit=1, recency=0, affinity=0))
|
||||
frontend = store.save_agent("Frontend", "react frontend developer",
|
||||
weights=Weights(fit=1, recency=0, affinity=0))
|
||||
back = role(store, "1", "Backend Engineer", body="Python services")
|
||||
front = role(store, "2", "Frontend Engineer", body="React apps")
|
||||
only_fit = Weights(fit=1, recency=0, affinity=0)
|
||||
backend = store.save_agent("Backend", "python backend developer", weights=only_fit)
|
||||
frontend = store.save_agent("Frontend", "react frontend developer", weights=only_fit)
|
||||
|
||||
run(config, store, embed.run_embed)
|
||||
result = run(config, store, score.run_score)
|
||||
|
||||
assert result["agents"] == 2
|
||||
assert store.scores(backend.id)[uid][0] > store.scores(frontend.id)[uid][0]
|
||||
# Fit alone, scaled per agent: each agent's closer role is 1, the other 0.
|
||||
assert store.scores(backend.id)[back][0] == 1.0
|
||||
assert store.scores(backend.id)[front][0] == 0.0
|
||||
assert store.scores(frontend.id)[front][0] == 1.0
|
||||
assert store.scores(frontend.id)[back][0] == 0.0
|
||||
|
||||
|
||||
# ------------------------------------------------------------ fit scaling
|
||||
|
||||
def test_scale_maps_the_closest_to_one_and_the_farthest_to_zero() -> None:
|
||||
# The spread the last real run showed under `(cos + 1) / 2`.
|
||||
assert score.scale({"a": 0.55, "b": 0.79, "c": 0.67}) == pytest.approx(
|
||||
{"a": 0.0, "b": 1.0, "c": 0.5})
|
||||
assert score.scale({"only": 0.3}) == {"only": 1.0}
|
||||
assert score.scale({"a": 0.4, "b": 0.4}) == {"a": 1.0, "b": 1.0}
|
||||
assert score.scale({}) == {}
|
||||
|
||||
|
||||
def test_a_single_candidate_gets_full_fit(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
uid = role(store, "1", "Frontend Engineer", body="React apps")
|
||||
agent = store.save_agent("Backend", "python backend developer",
|
||||
weights=Weights(fit=1, recency=0, affinity=0))
|
||||
|
||||
run(config, store, embed.run_embed)
|
||||
run(config, store, score.run_score)
|
||||
|
||||
assert store.scores(agent.id)[uid] == (1.0, "")
|
||||
|
||||
|
||||
def test_a_gated_role_does_not_stretch_the_fit_range(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
close = role(store, "1", "Backend Engineer", body="Python backend", place="Berlin")
|
||||
far = role(store, "2", "Data Engineer", body="Data", place="Berlin")
|
||||
# Farther from the intent than either, but gated out by place.
|
||||
role(store, "3", "Frontend Engineer", body="React react", place="Munich")
|
||||
agent = store.save_agent("Berlin", "python backend",
|
||||
[Gate(field="place", values=["Berlin"])],
|
||||
weights=Weights(fit=1, recency=0, affinity=0))
|
||||
|
||||
run(config, store, embed.run_embed)
|
||||
run(config, store, score.run_score)
|
||||
|
||||
scores = store.scores(agent.id)
|
||||
assert scores[close][0] == 1.0 and scores[far][0] == 0.0
|
||||
|
||||
|
||||
def test_a_small_prefer_gate_no_longer_outweighs_fit(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
"""The flaw the last real run showed: real cosines sit close together, so
|
||||
under `(cos + 1) / 2` a prefer gate decided the order over fit. Here the
|
||||
cosines are about 0.95 and 0.87: the old fit terms differ by 0.04, which
|
||||
at weight 0.6 is less than the gate's 0.05, so the remote role won. Scaled,
|
||||
the closer role wins."""
|
||||
close = role(store, "1", "Backend Engineer", body="Python backend", remote="onsite")
|
||||
far = role(store, "2", "Backend Engineer", body="Python backend data", remote="remote")
|
||||
agent = store.save_agent("Backend", "python backend",
|
||||
[Gate(field="remote", values=["remote"], mode="prefer",
|
||||
weight=0.05)])
|
||||
|
||||
run(config, store, embed.run_embed)
|
||||
run(config, store, score.run_score)
|
||||
|
||||
scores = store.scores(agent.id)
|
||||
assert scores[close][0] > scores[far][0]
|
||||
|
||||
|
||||
def test_the_intent_vector_is_cached_until_the_intent_changes(
|
||||
@@ -266,8 +330,10 @@ def test_starter_agents_are_imported_once(store: Store, tmp_path: Path) -> None:
|
||||
def test_the_shipped_starter_agents_validate() -> None:
|
||||
from app import config as config_mod
|
||||
|
||||
names = [a.name for a in config_mod.load(ROOT / "config.yaml").agents]
|
||||
assert names == ["Berlin backend", "Remote EU stretch"]
|
||||
agents = config_mod.load(ROOT / "config.yaml").agents
|
||||
assert [a.name for a in agents] == ["Berlin backend", "Remote EU stretch"]
|
||||
# Small enough that a prefer gate breaks ties rather than overrides fit.
|
||||
assert {g.weight for a in agents for g in a.gates if g.mode == "prefer"} == {0.05}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- cascade
|
||||
|
||||
Reference in new issue
Block a user