Rank by the cut: fetch gate overrides in scoring, the cut per agent, stages read the cut, fetch text renamed from enrich, the feed kept to the fetch gate, and migration 32

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-10-04 15:11:43 +02:00
1 parent 060ccf66e6
commit 617356e60c
35 files changed
+1054 -751

No files matched your search

+80 -78
View File
@@ -12,6 +12,7 @@ from typing import Any
import numpy as np
import pytest
from pydantic import ValidationError
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
@@ -353,18 +354,25 @@ def test_any_gates_are_one_group_a_role_must_match_one_of(
def test_failed_gates_reads_evidence_only() -> None:
gates = [Gate(field="country", values=["DK"]),
Gate(field="remote", values=["remote"], mode="any"),
Gate(field="seniority", values=["senior"], mode="any"),
Gate(field="function", values=["engineering"], mode="prefer")]
Gate(field="seniority", values=["senior"], mode="any")]
def cand(*states: int) -> Candidate:
return Candidate(uid="u", posted_at=NOW.isoformat(), affinity=0, gates=list(states))
assert score.failed_gates(cand(1, 1, 0, 0), gates) == []
assert score.failed_gates(cand(-1, -1, -1, 0), gates) == []
assert score.failed_gates(cand(0, 0, -1, 1), gates) == [
assert score.failed_gates(cand(1, 1, 0), gates) == []
assert score.failed_gates(cand(-1, -1, -1), gates) == []
assert score.failed_gates(cand(0, 0, -1), gates) == [
"country not DK", "none of remote remote, seniority senior"]
assert score.failed_gates(cand(1, 0, 0, 1), gates) == [
assert score.failed_gates(cand(1, 0, 0), gates) == [
"none of remote remote, seniority senior"]
def test_a_gate_is_require_any_or_exclude_and_has_no_weight() -> None:
"""Prefer gates left the agent: hybrid is a feed filter now."""
with pytest.raises(ValidationError):
Gate(field="remote", values=["hybrid"], mode="prefer") # type: ignore[arg-type]
with pytest.raises(ValidationError):
Gate.model_validate({"field": "remote", "values": ["remote"], "weight": 1.0})
def test_an_exclude_gate_on_title_gates_whole_words_and_names_the_value(
store: Store, config: Config, fake: FakeEmbedder) -> None:
"""The sales and data engineering titles the owner does not want: a
@@ -389,7 +397,7 @@ def test_an_exclude_gate_on_title_gates_whole_words_and_names_the_value(
def test_a_favourite_is_gated_by_an_exclude_gate_it_matches(
store: Store, config: Config, fake: FakeEmbedder) -> None:
"""A star skips the `any` group, not the titles the agent leaves out."""
"""A star raises the level only: it skips no part of the fetch gate."""
store.add_company("Fave")
director = role(store, "1", "Account Director, Nordics", company="Fave")
engineer = role(store, "2", "Forward Deployed Engineer", company="Fave")
@@ -415,21 +423,6 @@ def test_failed_gates_names_the_value_an_exclude_gate_matched() -> None:
assert score.failed_gates(cand(0, 1), gates) == ["country not DK", "title account executive"]
def test_a_prefer_gate_adds_its_weight_to_the_score(
store: Store, config: Config, fake: FakeEmbedder) -> None:
remote = role(store, "1", remote="remote")
office = role(store, "2", remote="onsite")
agent = store.save_agent("Remote", "", [Gate(field="remote", values=["remote"],
mode="prefer", weight=0.5)])
run(config, store, score.run_score)
scores = store.scores(agent.id)
# No intent, so fit is out: recency 0.2 + affinity 0.2 + the gate's 0.5.
assert scores[remote][0] - scores[office][0] == pytest.approx(0.5 / 0.9, abs=1e-5)
assert scores[office][1] == ""
def test_a_favourite_is_gated_by_a_require_gate_it_fails(
store: Store, config: Config, fake: FakeEmbedder) -> None:
"""A star widens the search to the company, not past a require gate: the
@@ -447,40 +440,70 @@ def test_a_favourite_is_gated_by_a_require_gate_it_fails(
assert scores[berlin][1] == "" and scores[berlin][0] > 0
def test_a_favourite_passes_the_any_group_others_fail(
def test_all_countries_passes_the_any_group_a_favourite_does_not(
store: Store, config: Config, fake: FakeEmbedder) -> None:
""""Denmark or remote" gates the onsite German role of a neutral company
and lets the same role of a starred one through."""
store.add_company("Fave")
and of a favourite, and lets the same role of an All countries company
through."""
for name in ("Fave", "Wide", "Other"):
store.add_company(name)
fave = role(store, "1", company="Fave", place="Berlin, Germany", remote="onsite")
other = role(store, "2", company="Other", place="Berlin, Germany", remote="onsite")
wide = role(store, "2", company="Wide", place="Berlin, Germany", remote="onsite")
other = role(store, "3", company="Other", place="Berlin, Germany", remote="onsite")
agent = store.save_agent("Copenhagen or remote", "", [
Gate(field="country", values=["DK"], mode="any"),
Gate(field="remote", values=["remote"], mode="any")])
store.set_affinity(agent.id, "fave", 2)
store.set_all_countries("wide", True)
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[fave][1] == "" and scores[fave][0] > 0
assert scores[wide][1] == "" and scores[wide][0] > 0
assert scores[fave] == (0.0, "none of country DK, remote remote")
assert scores[other] == (0.0, "none of country DK, remote remote")
def test_a_favourite_takes_both_halves_of_the_rule_in_one_agent() -> None:
gates = [Gate(field="country", values=["DK"]),
def test_all_countries_skips_the_place_part_and_all_roles_the_titles() -> None:
"""The region applies to every company; All countries lifts the `any`
group, All roles the exclude gates on title and no other exclude gate."""
gates = [Gate(field="country", values=["DK", "SE", "EU"]),
Gate(field="country", values=["DK"], mode="any"),
Gate(field="remote", values=["remote"], mode="any"),
Gate(field="seniority", values=["senior"], mode="any")]
def cand(affinity: int, *states: int) -> Candidate:
return Candidate(uid="u", posted_at=NOW.isoformat(), affinity=affinity,
gates=list(states))
# The any group fails for both; only the neutral company is gated by it.
assert score.failed_gates(cand(2, 1, 0, 0), gates) == []
assert score.failed_gates(cand(0, 1, 0, 0), gates) == [
"none of remote remote, seniority senior"]
# The require gate fails for both.
assert score.failed_gates(cand(2, 0, 0, 0), gates) == ["country not DK"]
assert score.failed_gates(cand(0, 0, 0, 0), gates) == [
"country not DK", "none of remote remote, seniority senior"]
Gate(field="title", values=["account executive"], mode="exclude"),
Gate(field="company", values=["acme"], mode="exclude")]
def cand(*states: int, wide: bool = False, every: bool = False) -> Candidate:
return Candidate(uid="u", posted_at=NOW.isoformat(), all_countries=wide,
all_roles=every, gates=list(states))
assert score.failed_gates(cand(1, 0, 0, 0, 0), gates) == [
"none of country DK, remote remote"]
assert score.failed_gates(cand(1, 0, 0, 0, 0, wide=True), gates) == []
assert score.failed_gates(cand(0, 0, 0, 0, 0, wide=True, every=True), gates) == [
"country not DK / SE / EU"]
assert score.failed_gates(cand(1, 1, 0, 1, 0), gates) == ["title account executive"]
assert score.failed_gates(cand(1, 1, 0, 1, 0, wide=True), gates) == [
"title account executive"]
assert score.failed_gates(cand(1, 1, 0, 1, 0, every=True), gates) == []
assert score.failed_gates(cand(1, 1, 0, 1, 1, every=True), gates) == ["company acme"]
assert score.failed_gates(cand(1, 0, 0, 1, 0, every=True), gates) == [
"none of country DK, remote remote"]
def test_all_roles_keeps_a_title_the_agent_leaves_out(
store: Store, config: Config, fake: FakeEmbedder) -> None:
for name in ("Small", "Other"):
store.add_company(name)
small = role(store, "1", "Account Executive", company="Small")
other = role(store, "2", "Account Executive", company="Other")
agent = store.save_agent("No sales", "", [
Gate(field="title", values=["account executive"], mode="exclude")])
store.set_all_roles("small", True)
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[small][1] == ""
assert scores[other] == (0.0, "title account executive")
def test_a_favourite_with_an_empty_column_is_not_gated() -> None:
@@ -808,12 +831,13 @@ def test_a_bad_calibration_is_a_config_error(
def test_an_agent_without_an_intent_feeds_neither_model_stage_and_is_named(
store: Store, config: Config, clock: Clock, fake: FakeEmbedder) -> None:
"""Its scores have no fit, so they rank fresh roles, not wanted ones: the
newer role tops it, and extract and summarise must not read it on that."""
newer role tops it, and nothing of it enters the cut by fit, nor does its
top choose what summarise reads."""
clock.at = NOW - timedelta(days=1)
wanted = role(store, "1", "Backend Engineer", body="Python backend")
clock.at = NOW
fresh = role(store, "2", "Frontend Engineer", body="React apps")
top_one = Limits(extract_top=1, extract_min=0, summarise_top=1, summarise_min=0)
top_one = Limits(cut_fit_top=1, cut_fit_min=0, summarise_top=1)
store.save_agent("Backend", "python backend", limits=top_one)
blind = store.save_agent("Fresh", "", limits=top_one)
store.save_agent("Also fresh", " \n", limits=top_one)
@@ -854,26 +878,6 @@ def test_a_gated_role_does_not_stretch_the_fit_range_under_the_fallback(
assert scores[close][0] == 1.0 and scores[far][0] == 0.0
def test_a_small_prefer_gate_no_longer_outweighs_fit(
store: Store, config: Config, fake: FakeEmbedder) -> None:
"""The flaw the last real run showed: real cosines sit close together, so
under `(cos + 1) / 2` a prefer gate decided the order over fit. Here the
cosines are about 0.95 and 0.87: the old fit terms differ by 0.04, which
at weight 0.6 is less than the gate's 0.05, so the remote role won.
Calibrated, they are 1 and about 0.89, and the closer role wins."""
close = role(store, "1", "Backend Engineer", body="Python backend", remote="onsite")
far = role(store, "2", "Backend Engineer", body="Python backend data", remote="remote")
agent = store.save_agent("Backend", "python backend",
[Gate(field="remote", values=["remote"], mode="prefer",
weight=0.05)])
run(config, store, embed.run_embed)
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[close][0] > scores[far][0]
def test_the_intent_vector_is_cached_until_the_intent_changes(
store: Store, config: Config, fake: FakeEmbedder) -> None:
role(store, "1")
@@ -1018,9 +1022,7 @@ def test_the_shipped_starter_agents_validate() -> None:
block = [line[2:] for line in lines[start:] if line.startswith("# ")]
examples = [AgentSeed.model_validate(a) for a in yaml.safe_load("\n".join(block))]
assert [a.name for a in examples] == ["Remote from Europe", "Europe hybrid"]
# Small enough that a prefer gate breaks ties rather than overrides fit.
assert {g.weight for a in [*agents, *examples]
for g in a.gates if g.mode == "prefer"} == {0.05}
assert examples[0].limits.cut_fit_top == 1000
def test_a_starter_agent_may_list_prompts_or_give_an_intent(
@@ -1074,7 +1076,7 @@ def test_the_shipped_starter_searches_validate() -> None:
# ---------------------------------------------------------------- cascade
def test_the_pull_job_runs_pull_prep_embed_score_extract_score_then_summarise(
def test_the_pull_job_runs_pull_prep_embed_score_extract_score_judge_then_summarise(
store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
@@ -1104,10 +1106,10 @@ def test_the_pull_job_runs_pull_prep_embed_score_extract_score_then_summarise(
assert batch is not None
jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id)
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
# Enrich found no board with role pages, so the prep, embed and score
# Fetch text found no board with role pages, so the prep, embed and score
# behind it are passed over.
assert stages == ["pull", "prep", "embed", "score", "enrich", "extract", "score",
"summarise", "judge"]
assert stages == ["pull", "prep", "embed", "score", "fetch_text", "extract", "score",
"judge", "summarise"]
work = [j.kind for j in jobs if j.kind != FINAL]
# The `pull` job and the chain it asked for are one cascade, so the Jobs
# tab shows a scheduled pull as one run. Extract and summarise are off and
@@ -1159,14 +1161,14 @@ def test_the_model_batches_are_isolated_with_a_timeout() -> None:
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
"""By position, not by name: `score` is in the chain twice, once before
extract and once after, and the second must lead on to summarise."""
assert cascade.CASCADES["pull"] == ("pull", "prep", "embed", "score", "enrich", "prep",
"embed", "score", "extract", "score", "summarise",
"judge")
"""By position, not by name: `score` is in the chain three times, and the
one after extract must lead on to judge."""
assert cascade.CASCADES["pull"] == ("pull", "prep", "embed", "score", "fetch_text", "prep",
"embed", "score", "extract", "score", "judge",
"summarise")
assert [cascade.follows("pull", at) for at in range(12)] == \
["prep", "embed", "score", "enrich", "prep", "embed", "score", "extract", "score",
"summarise", "judge", ""]
["prep", "embed", "score", "fetch_text", "prep", "embed", "score", "extract", "score",
"judge", "summarise", ""]
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 11) == ""
assert cascade.follows("nope", 0) == ""
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
@@ -1186,4 +1188,4 @@ def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None:
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 8}
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 9},
context=Context())
assert out["next"] == "summarise" and made[1].carries["at"] == 10
assert out["next"] == "judge" and made[1].carries["at"] == 10