Scale fit across each agent's run, and lower the starter prefer weights
With fit as (cos + 1) / 2 the real run spanned about 0.55 to 0.79, so a prefer gate at weight 0.2 moved a score more than the whole spread of fit. fit is now the cosine min-max scaled across the roles an agent scores in that run: the closest is 1, the farthest 0, a single role 1, and a role a require gate rules out stays out of the range. The starter agents' prefer weights drop to 0.05, enough to break a tie. A regression test pins the case: two close cosines that the old mapping let a 0.05 gate reorder. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
59a301c351
commit
8b246c3035
5 files changed
+132
-31
No files matched your search
+53
-18
@@ -6,7 +6,9 @@ good for a given search. For each agent:
|
|||||||
require gates a role that fails one keeps its row, with score 0 and the
|
require gates a role that fails one keeps its row, with score 0 and the
|
||||||
reason ("place unknown", "remote not remote")
|
reason ("place unknown", "remote not remote")
|
||||||
prefer gates `weight` when the role matches, 0 when it does not
|
prefer gates `weight` when the role matches, 0 when it does not
|
||||||
fit cos(role vector, intent vector), mapped to 0..1
|
fit cos(role vector, intent vector), min-max scaled across
|
||||||
|
the roles this agent scores in this run: the closest is 1,
|
||||||
|
the farthest 0, a lone role 1
|
||||||
recency 0.5 ** (days since first_seen / 14)
|
recency 0.5 ** (days since first_seen / 14)
|
||||||
affinity (companies.affinity + 1) / 3
|
affinity (companies.affinity + 1) / 3
|
||||||
|
|
||||||
@@ -65,13 +67,30 @@ def affinity_term(affinity: int) -> float:
|
|||||||
return (affinity + 1) / 3
|
return (affinity + 1) / 3
|
||||||
|
|
||||||
|
|
||||||
def fit_term(role: np.ndarray, intent: np.ndarray) -> float:
|
def cosine(role: np.ndarray, intent: np.ndarray) -> float:
|
||||||
"""Cosine to 0..1. Rescaled rather than clipped, as newsdesk does, so a
|
"""Raw cosine, -1..1; 0 when either vector is zero."""
|
||||||
role pointing away from the intent sits below an unrelated one."""
|
|
||||||
norms = float(np.linalg.norm(role) * np.linalg.norm(intent))
|
norms = float(np.linalg.norm(role) * np.linalg.norm(intent))
|
||||||
if norms == 0:
|
if norms == 0:
|
||||||
return 0.5
|
return 0.0
|
||||||
return (float(np.dot(role, intent)) / norms + 1.0) / 2.0
|
return float(np.dot(role, intent)) / norms
|
||||||
|
|
||||||
|
|
||||||
|
def scale(cosines: dict[str, float]) -> dict[str, float]:
|
||||||
|
"""Min-max to 0..1 across one agent's run, so `fit` ranks roles rather
|
||||||
|
than stating a level.
|
||||||
|
|
||||||
|
`(cos + 1) / 2` was the level, and on MiniLM it spans about 0.55 to 0.79
|
||||||
|
for real postings against a real intent, so a prefer gate weighted 0.2
|
||||||
|
moved a score more than the whole spread of fit did. Scaled, the closest
|
||||||
|
role is 1 and the farthest 0. A single role, or roles that are all equally
|
||||||
|
close, get 1: there is nothing to tell them apart by.
|
||||||
|
"""
|
||||||
|
if not cosines:
|
||||||
|
return {}
|
||||||
|
low, high = min(cosines.values()), max(cosines.values())
|
||||||
|
if high - low < 1e-9:
|
||||||
|
return dict.fromkeys(cosines, 1.0)
|
||||||
|
return {uid: (value - low) / (high - low) for uid, value in cosines.items()}
|
||||||
|
|
||||||
|
|
||||||
def _reason(gate: Gate, state: int) -> str:
|
def _reason(gate: Gate, state: int) -> str:
|
||||||
@@ -82,17 +101,24 @@ def _reason(gate: Gate, state: int) -> str:
|
|||||||
return f"{gate.field} not {' / '.join(str(v) for v in gate.values)}"
|
return f"{gate.field} not {' / '.join(str(v) for v in gate.values)}"
|
||||||
|
|
||||||
|
|
||||||
|
def failed_gates(candidate: Candidate, gates: Sequence[Gate]) -> list[str]:
|
||||||
|
"""Every require gate this role fails, as reasons. None for a favourite,
|
||||||
|
which skips the gates."""
|
||||||
|
if candidate.affinity == 2:
|
||||||
|
return []
|
||||||
|
return [_reason(g, state) for g, state in zip(gates, candidate.gates, strict=True)
|
||||||
|
if g.mode == "require" and state != 1]
|
||||||
|
|
||||||
|
|
||||||
def score_one(candidate: Candidate, gates: Sequence[Gate], weights: Weights,
|
def score_one(candidate: Candidate, gates: Sequence[Gate], weights: Weights,
|
||||||
fit: float | None, now: datetime, *,
|
fit: float | None, now: datetime, *,
|
||||||
missing: str = "no vector") -> tuple[float, str]:
|
missing: str = "no vector") -> tuple[float, str]:
|
||||||
"""One role for one agent: `(score, gated_reason)`. `fit` is None when
|
"""One role for one agent: `(score, gated_reason)`. `fit` is already
|
||||||
the role or the intent has no vector; `missing` says which."""
|
scaled to 0..1 (see `scale`), or None when the role or the intent has no
|
||||||
|
vector; `missing` says which."""
|
||||||
favourite = candidate.affinity == 2
|
favourite = candidate.affinity == 2
|
||||||
if not favourite:
|
if failed := failed_gates(candidate, gates):
|
||||||
failed = [_reason(g, state) for g, state in zip(gates, candidate.gates, strict=True)
|
return 0.0, ", ".join(failed)
|
||||||
if g.mode == "require" and state != 1]
|
|
||||||
if failed:
|
|
||||||
return 0.0, ", ".join(failed)
|
|
||||||
|
|
||||||
terms = [(weights.recency, recency(candidate.first_seen, now)),
|
terms = [(weights.recency, recency(candidate.first_seen, now)),
|
||||||
(weights.affinity, affinity_term(candidate.affinity))]
|
(weights.affinity, affinity_term(candidate.affinity))]
|
||||||
@@ -157,14 +183,23 @@ def _intent(store: Store, agent: Agent, model: str,
|
|||||||
|
|
||||||
def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
|
def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
|
||||||
intent: np.ndarray | None, now: datetime) -> dict[str, int]:
|
intent: np.ndarray | None, now: datetime) -> dict[str, int]:
|
||||||
|
"""Two passes: the cosines of the roles this agent scores, then every
|
||||||
|
score with fit scaled across them. A gated role is left out of the
|
||||||
|
scaling, so a role that fails a require gate cannot stretch the range the
|
||||||
|
ranked roles share."""
|
||||||
|
candidates = store.score_candidates(agent.gates)
|
||||||
|
cosines: dict[str, float] = {}
|
||||||
|
if intent is not None:
|
||||||
|
for candidate in candidates:
|
||||||
|
role_vec = vectors.get(candidate.uid)
|
||||||
|
if role_vec is not None and not failed_gates(candidate, agent.gates):
|
||||||
|
cosines[candidate.uid] = cosine(role_vec, intent)
|
||||||
|
fits = scale(cosines)
|
||||||
rows: list[tuple[str, float, str]] = []
|
rows: list[tuple[str, float, str]] = []
|
||||||
gated = 0
|
gated = 0
|
||||||
for candidate in store.score_candidates(agent.gates):
|
for candidate in candidates:
|
||||||
role_vec = vectors.get(candidate.uid)
|
|
||||||
fit = fit_term(role_vec, intent) if role_vec is not None and intent is not None \
|
|
||||||
else None
|
|
||||||
value, reason = score_one(
|
value, reason = score_one(
|
||||||
candidate, agent.gates, agent.weights, fit, now,
|
candidate, agent.gates, agent.weights, fits.get(candidate.uid), now,
|
||||||
missing="no vector" if intent is not None else "no intent vector")
|
missing="no vector" if intent is not None else "no intent vector")
|
||||||
gated += bool(reason)
|
gated += bool(reason)
|
||||||
rows.append((candidate.uid, round(value, 6), reason))
|
rows.append((candidate.uid, round(value, 6), reason))
|
||||||
|
|||||||
@@ -14,7 +14,7 @@
|
|||||||
<label>Name <input type="text" id="a-name" required></label>
|
<label>Name <input type="text" id="a-name" required></label>
|
||||||
<label>Intent <span class="hint">a CV paragraph, or what this search is for</span>
|
<label>Intent <span class="hint">a CV paragraph, or what this search is for</span>
|
||||||
<textarea id="a-intent" rows="5"></textarea></label>
|
<textarea id="a-intent" rows="5"></textarea></label>
|
||||||
<label>Gates <span class="hint">JSON: [{"field": "place", "values": ["Berlin"], "mode": "require"}, {"field": "remote", "values": ["remote"], "mode": "prefer", "weight": 0.2}]</span>
|
<label>Gates <span class="hint">JSON: [{"field": "place", "values": ["Berlin"], "mode": "require"}, {"field": "remote", "values": ["remote"], "mode": "prefer", "weight": 0.05}]</span>
|
||||||
<textarea id="a-gates" rows="5" spellcheck="false">[]</textarea></label>
|
<textarea id="a-gates" rows="5" spellcheck="false">[]</textarea></label>
|
||||||
<fieldset class="weights">
|
<fieldset class="weights">
|
||||||
<legend>Weights</legend>
|
<legend>Weights</legend>
|
||||||
|
|||||||
+2
-2
@@ -30,7 +30,7 @@ agents:
|
|||||||
ownership of production systems in a product company.
|
ownership of production systems in a product company.
|
||||||
gates:
|
gates:
|
||||||
- {field: place, values: [Berlin], mode: require}
|
- {field: place, values: [Berlin], mode: require}
|
||||||
- {field: remote, values: [hybrid, remote], mode: prefer, weight: 0.2}
|
- {field: remote, values: [hybrid, remote], mode: prefer, weight: 0.05}
|
||||||
- name: Remote EU stretch
|
- name: Remote EU stretch
|
||||||
intent: >-
|
intent: >-
|
||||||
Staff or lead engineer role, fully remote from Europe. Platform and
|
Staff or lead engineer role, fully remote from Europe. Platform and
|
||||||
@@ -38,4 +38,4 @@ agents:
|
|||||||
leadership across teams.
|
leadership across teams.
|
||||||
gates:
|
gates:
|
||||||
- {field: remote, values: [remote], mode: require}
|
- {field: remote, values: [remote], mode: require}
|
||||||
- {field: country, values: [DE, NL, AT, CH], mode: prefer, weight: 0.2}
|
- {field: country, values: [DE, NL, AT, CH], mode: prefer, weight: 0.05}
|
||||||
+1
-1
@@ -59,7 +59,7 @@ at least the listing.
|
|||||||
|
|
||||||
**Scoring is per search agent.** A role is scored as `(uid, agent) -> score`, because a role is only good for a given search. Each gate has a mode: `require` becomes a SQL `WHERE`, `prefer` becomes a weighted term in the score. Gated roles keep their row with a reason. `companies.affinity = 2` skips the gates and floors the score, `-1` mutes.
|
**Scoring is per search agent.** A role is scored as `(uid, agent) -> score`, because a role is only good for a given search. Each gate has a mode: `require` becomes a SQL `WHERE`, `prefer` becomes a weighted term in the score. Gated roles keep their row with a reason. `companies.affinity = 2` skips the gates and floors the score, `-1` mutes.
|
||||||
|
|
||||||
**`fit` is cosine between the role and the agent's intent text.** The intent is a CV or a paragraph per agent. This is why the embedding backend is effectively required: TF-IDF cannot relate "Kubernetes" to "container orchestration".
|
**`fit` is the cosine between a role and the agent's intent, min-max scaled across the roles that agent scores in the run.** `(cos + 1) / 2` spanned only about 0.55 to 0.79 in the real run, so a prefer gate at 0.2 outweighed fit; scaled, the closest role is 1 and the farthest 0. A single role gets 1, gated roles stay out of the range, and the starter agents' prefer weights are 0.05. The intent is a CV or a paragraph, so the embedding backend is effectively required: TF-IDF cannot relate "Kubernetes" to "container orchestration".
|
||||||
|
|
||||||
**Summary and tagging are off; `extract` replaces them.** Every profile declares `SummaryPolicy(enabled=False)` and `TaggingPolicy(enabled=False)`. `extract` is a roledesk `ModelTask` on Haiku that turns title and description into the gate columns, and fills only the columns a plugin left empty.
|
**Summary and tagging are off; `extract` replaces them.** Every profile declares `SummaryPolicy(enabled=False)` and `TaggingPolicy(enabled=False)`. `extract` is a roledesk `ModelTask` on Haiku that turns title and description into the gate columns, and fills only the columns a plugin left empty.
|
||||||
|
|
||||||
|
|||||||
+75
-9
@@ -216,19 +216,83 @@ def test_an_older_role_scores_lower(
|
|||||||
assert scores[new][0] > scores[old][0]
|
assert scores[new][0] > scores[old][0]
|
||||||
|
|
||||||
|
|
||||||
def test_two_agents_score_the_same_role_differently(
|
def test_two_agents_rank_the_same_roles_the_other_way_round(
|
||||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||||
uid = role(store, "1", "Backend Engineer", body="Python services")
|
back = role(store, "1", "Backend Engineer", body="Python services")
|
||||||
backend = store.save_agent("Backend", "python backend developer",
|
front = role(store, "2", "Frontend Engineer", body="React apps")
|
||||||
weights=Weights(fit=1, recency=0, affinity=0))
|
only_fit = Weights(fit=1, recency=0, affinity=0)
|
||||||
frontend = store.save_agent("Frontend", "react frontend developer",
|
backend = store.save_agent("Backend", "python backend developer", weights=only_fit)
|
||||||
weights=Weights(fit=1, recency=0, affinity=0))
|
frontend = store.save_agent("Frontend", "react frontend developer", weights=only_fit)
|
||||||
|
|
||||||
run(config, store, embed.run_embed)
|
run(config, store, embed.run_embed)
|
||||||
result = run(config, store, score.run_score)
|
result = run(config, store, score.run_score)
|
||||||
|
|
||||||
assert result["agents"] == 2
|
assert result["agents"] == 2
|
||||||
assert store.scores(backend.id)[uid][0] > store.scores(frontend.id)[uid][0]
|
# Fit alone, scaled per agent: each agent's closer role is 1, the other 0.
|
||||||
|
assert store.scores(backend.id)[back][0] == 1.0
|
||||||
|
assert store.scores(backend.id)[front][0] == 0.0
|
||||||
|
assert store.scores(frontend.id)[front][0] == 1.0
|
||||||
|
assert store.scores(frontend.id)[back][0] == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------ fit scaling
|
||||||
|
|
||||||
|
def test_scale_maps_the_closest_to_one_and_the_farthest_to_zero() -> None:
|
||||||
|
# The spread the last real run showed under `(cos + 1) / 2`.
|
||||||
|
assert score.scale({"a": 0.55, "b": 0.79, "c": 0.67}) == pytest.approx(
|
||||||
|
{"a": 0.0, "b": 1.0, "c": 0.5})
|
||||||
|
assert score.scale({"only": 0.3}) == {"only": 1.0}
|
||||||
|
assert score.scale({"a": 0.4, "b": 0.4}) == {"a": 1.0, "b": 1.0}
|
||||||
|
assert score.scale({}) == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_single_candidate_gets_full_fit(
|
||||||
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||||
|
uid = role(store, "1", "Frontend Engineer", body="React apps")
|
||||||
|
agent = store.save_agent("Backend", "python backend developer",
|
||||||
|
weights=Weights(fit=1, recency=0, affinity=0))
|
||||||
|
|
||||||
|
run(config, store, embed.run_embed)
|
||||||
|
run(config, store, score.run_score)
|
||||||
|
|
||||||
|
assert store.scores(agent.id)[uid] == (1.0, "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_gated_role_does_not_stretch_the_fit_range(
|
||||||
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||||
|
close = role(store, "1", "Backend Engineer", body="Python backend", place="Berlin")
|
||||||
|
far = role(store, "2", "Data Engineer", body="Data", place="Berlin")
|
||||||
|
# Farther from the intent than either, but gated out by place.
|
||||||
|
role(store, "3", "Frontend Engineer", body="React react", place="Munich")
|
||||||
|
agent = store.save_agent("Berlin", "python backend",
|
||||||
|
[Gate(field="place", values=["Berlin"])],
|
||||||
|
weights=Weights(fit=1, recency=0, affinity=0))
|
||||||
|
|
||||||
|
run(config, store, embed.run_embed)
|
||||||
|
run(config, store, score.run_score)
|
||||||
|
|
||||||
|
scores = store.scores(agent.id)
|
||||||
|
assert scores[close][0] == 1.0 and scores[far][0] == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_small_prefer_gate_no_longer_outweighs_fit(
|
||||||
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||||
|
"""The flaw the last real run showed: real cosines sit close together, so
|
||||||
|
under `(cos + 1) / 2` a prefer gate decided the order over fit. Here the
|
||||||
|
cosines are about 0.95 and 0.87: the old fit terms differ by 0.04, which
|
||||||
|
at weight 0.6 is less than the gate's 0.05, so the remote role won. Scaled,
|
||||||
|
the closer role wins."""
|
||||||
|
close = role(store, "1", "Backend Engineer", body="Python backend", remote="onsite")
|
||||||
|
far = role(store, "2", "Backend Engineer", body="Python backend data", remote="remote")
|
||||||
|
agent = store.save_agent("Backend", "python backend",
|
||||||
|
[Gate(field="remote", values=["remote"], mode="prefer",
|
||||||
|
weight=0.05)])
|
||||||
|
|
||||||
|
run(config, store, embed.run_embed)
|
||||||
|
run(config, store, score.run_score)
|
||||||
|
|
||||||
|
scores = store.scores(agent.id)
|
||||||
|
assert scores[close][0] > scores[far][0]
|
||||||
|
|
||||||
|
|
||||||
def test_the_intent_vector_is_cached_until_the_intent_changes(
|
def test_the_intent_vector_is_cached_until_the_intent_changes(
|
||||||
@@ -266,8 +330,10 @@ def test_starter_agents_are_imported_once(store: Store, tmp_path: Path) -> None:
|
|||||||
def test_the_shipped_starter_agents_validate() -> None:
|
def test_the_shipped_starter_agents_validate() -> None:
|
||||||
from app import config as config_mod
|
from app import config as config_mod
|
||||||
|
|
||||||
names = [a.name for a in config_mod.load(ROOT / "config.yaml").agents]
|
agents = config_mod.load(ROOT / "config.yaml").agents
|
||||||
assert names == ["Berlin backend", "Remote EU stretch"]
|
assert [a.name for a in agents] == ["Berlin backend", "Remote EU stretch"]
|
||||||
|
# Small enough that a prefer gate breaks ties rather than overrides fit.
|
||||||
|
assert {g.weight for a in agents for g in a.gates if g.mode == "prefer"} == {0.05}
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- cascade
|
# ---------------------------------------------------------------- cascade
|
||||||
|
|||||||
Reference in new issue
Block a user