Rank by the cut: fetch gate overrides in scoring, the cut per agent, stages read the cut, fetch text renamed from enrich, the feed kept to the fetch gate, and migration 32
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
060ccf66e6
commit
617356e60c
35 files changed
+1054
-751
No files matched your search
+80
-78
@@ -12,6 +12,7 @@ from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
@@ -353,18 +354,25 @@ def test_any_gates_are_one_group_a_role_must_match_one_of(
|
||||
def test_failed_gates_reads_evidence_only() -> None:
|
||||
gates = [Gate(field="country", values=["DK"]),
|
||||
Gate(field="remote", values=["remote"], mode="any"),
|
||||
Gate(field="seniority", values=["senior"], mode="any"),
|
||||
Gate(field="function", values=["engineering"], mode="prefer")]
|
||||
Gate(field="seniority", values=["senior"], mode="any")]
|
||||
def cand(*states: int) -> Candidate:
|
||||
return Candidate(uid="u", posted_at=NOW.isoformat(), affinity=0, gates=list(states))
|
||||
assert score.failed_gates(cand(1, 1, 0, 0), gates) == []
|
||||
assert score.failed_gates(cand(-1, -1, -1, 0), gates) == []
|
||||
assert score.failed_gates(cand(0, 0, -1, 1), gates) == [
|
||||
assert score.failed_gates(cand(1, 1, 0), gates) == []
|
||||
assert score.failed_gates(cand(-1, -1, -1), gates) == []
|
||||
assert score.failed_gates(cand(0, 0, -1), gates) == [
|
||||
"country not DK", "none of remote remote, seniority senior"]
|
||||
assert score.failed_gates(cand(1, 0, 0, 1), gates) == [
|
||||
assert score.failed_gates(cand(1, 0, 0), gates) == [
|
||||
"none of remote remote, seniority senior"]
|
||||
|
||||
|
||||
def test_a_gate_is_require_any_or_exclude_and_has_no_weight() -> None:
|
||||
"""Prefer gates left the agent: hybrid is a feed filter now."""
|
||||
with pytest.raises(ValidationError):
|
||||
Gate(field="remote", values=["hybrid"], mode="prefer") # type: ignore[arg-type]
|
||||
with pytest.raises(ValidationError):
|
||||
Gate.model_validate({"field": "remote", "values": ["remote"], "weight": 1.0})
|
||||
|
||||
|
||||
def test_an_exclude_gate_on_title_gates_whole_words_and_names_the_value(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
"""The sales and data engineering titles the owner does not want: a
|
||||
@@ -389,7 +397,7 @@ def test_an_exclude_gate_on_title_gates_whole_words_and_names_the_value(
|
||||
|
||||
def test_a_favourite_is_gated_by_an_exclude_gate_it_matches(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
"""A star skips the `any` group, not the titles the agent leaves out."""
|
||||
"""A star raises the level only: it skips no part of the fetch gate."""
|
||||
store.add_company("Fave")
|
||||
director = role(store, "1", "Account Director, Nordics", company="Fave")
|
||||
engineer = role(store, "2", "Forward Deployed Engineer", company="Fave")
|
||||
@@ -415,21 +423,6 @@ def test_failed_gates_names_the_value_an_exclude_gate_matched() -> None:
|
||||
assert score.failed_gates(cand(0, 1), gates) == ["country not DK", "title account executive"]
|
||||
|
||||
|
||||
def test_a_prefer_gate_adds_its_weight_to_the_score(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
remote = role(store, "1", remote="remote")
|
||||
office = role(store, "2", remote="onsite")
|
||||
agent = store.save_agent("Remote", "", [Gate(field="remote", values=["remote"],
|
||||
mode="prefer", weight=0.5)])
|
||||
|
||||
run(config, store, score.run_score)
|
||||
|
||||
scores = store.scores(agent.id)
|
||||
# No intent, so fit is out: recency 0.2 + affinity 0.2 + the gate's 0.5.
|
||||
assert scores[remote][0] - scores[office][0] == pytest.approx(0.5 / 0.9, abs=1e-5)
|
||||
assert scores[office][1] == ""
|
||||
|
||||
|
||||
def test_a_favourite_is_gated_by_a_require_gate_it_fails(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
"""A star widens the search to the company, not past a require gate: the
|
||||
@@ -447,40 +440,70 @@ def test_a_favourite_is_gated_by_a_require_gate_it_fails(
|
||||
assert scores[berlin][1] == "" and scores[berlin][0] > 0
|
||||
|
||||
|
||||
def test_a_favourite_passes_the_any_group_others_fail(
|
||||
def test_all_countries_passes_the_any_group_a_favourite_does_not(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
""""Denmark or remote" gates the onsite German role of a neutral company
|
||||
and lets the same role of a starred one through."""
|
||||
store.add_company("Fave")
|
||||
and of a favourite, and lets the same role of an All countries company
|
||||
through."""
|
||||
for name in ("Fave", "Wide", "Other"):
|
||||
store.add_company(name)
|
||||
fave = role(store, "1", company="Fave", place="Berlin, Germany", remote="onsite")
|
||||
other = role(store, "2", company="Other", place="Berlin, Germany", remote="onsite")
|
||||
wide = role(store, "2", company="Wide", place="Berlin, Germany", remote="onsite")
|
||||
other = role(store, "3", company="Other", place="Berlin, Germany", remote="onsite")
|
||||
agent = store.save_agent("Copenhagen or remote", "", [
|
||||
Gate(field="country", values=["DK"], mode="any"),
|
||||
Gate(field="remote", values=["remote"], mode="any")])
|
||||
store.set_affinity(agent.id, "fave", 2)
|
||||
store.set_all_countries("wide", True)
|
||||
|
||||
run(config, store, score.run_score)
|
||||
|
||||
scores = store.scores(agent.id)
|
||||
assert scores[fave][1] == "" and scores[fave][0] > 0
|
||||
assert scores[wide][1] == "" and scores[wide][0] > 0
|
||||
assert scores[fave] == (0.0, "none of country DK, remote remote")
|
||||
assert scores[other] == (0.0, "none of country DK, remote remote")
|
||||
|
||||
|
||||
def test_a_favourite_takes_both_halves_of_the_rule_in_one_agent() -> None:
|
||||
gates = [Gate(field="country", values=["DK"]),
|
||||
def test_all_countries_skips_the_place_part_and_all_roles_the_titles() -> None:
|
||||
"""The region applies to every company; All countries lifts the `any`
|
||||
group, All roles the exclude gates on title and no other exclude gate."""
|
||||
gates = [Gate(field="country", values=["DK", "SE", "EU"]),
|
||||
Gate(field="country", values=["DK"], mode="any"),
|
||||
Gate(field="remote", values=["remote"], mode="any"),
|
||||
Gate(field="seniority", values=["senior"], mode="any")]
|
||||
def cand(affinity: int, *states: int) -> Candidate:
|
||||
return Candidate(uid="u", posted_at=NOW.isoformat(), affinity=affinity,
|
||||
gates=list(states))
|
||||
# The any group fails for both; only the neutral company is gated by it.
|
||||
assert score.failed_gates(cand(2, 1, 0, 0), gates) == []
|
||||
assert score.failed_gates(cand(0, 1, 0, 0), gates) == [
|
||||
"none of remote remote, seniority senior"]
|
||||
# The require gate fails for both.
|
||||
assert score.failed_gates(cand(2, 0, 0, 0), gates) == ["country not DK"]
|
||||
assert score.failed_gates(cand(0, 0, 0, 0), gates) == [
|
||||
"country not DK", "none of remote remote, seniority senior"]
|
||||
Gate(field="title", values=["account executive"], mode="exclude"),
|
||||
Gate(field="company", values=["acme"], mode="exclude")]
|
||||
def cand(*states: int, wide: bool = False, every: bool = False) -> Candidate:
|
||||
return Candidate(uid="u", posted_at=NOW.isoformat(), all_countries=wide,
|
||||
all_roles=every, gates=list(states))
|
||||
assert score.failed_gates(cand(1, 0, 0, 0, 0), gates) == [
|
||||
"none of country DK, remote remote"]
|
||||
assert score.failed_gates(cand(1, 0, 0, 0, 0, wide=True), gates) == []
|
||||
assert score.failed_gates(cand(0, 0, 0, 0, 0, wide=True, every=True), gates) == [
|
||||
"country not DK / SE / EU"]
|
||||
assert score.failed_gates(cand(1, 1, 0, 1, 0), gates) == ["title account executive"]
|
||||
assert score.failed_gates(cand(1, 1, 0, 1, 0, wide=True), gates) == [
|
||||
"title account executive"]
|
||||
assert score.failed_gates(cand(1, 1, 0, 1, 0, every=True), gates) == []
|
||||
assert score.failed_gates(cand(1, 1, 0, 1, 1, every=True), gates) == ["company acme"]
|
||||
assert score.failed_gates(cand(1, 0, 0, 1, 0, every=True), gates) == [
|
||||
"none of country DK, remote remote"]
|
||||
|
||||
|
||||
def test_all_roles_keeps_a_title_the_agent_leaves_out(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
for name in ("Small", "Other"):
|
||||
store.add_company(name)
|
||||
small = role(store, "1", "Account Executive", company="Small")
|
||||
other = role(store, "2", "Account Executive", company="Other")
|
||||
agent = store.save_agent("No sales", "", [
|
||||
Gate(field="title", values=["account executive"], mode="exclude")])
|
||||
store.set_all_roles("small", True)
|
||||
|
||||
run(config, store, score.run_score)
|
||||
|
||||
scores = store.scores(agent.id)
|
||||
assert scores[small][1] == ""
|
||||
assert scores[other] == (0.0, "title account executive")
|
||||
|
||||
|
||||
def test_a_favourite_with_an_empty_column_is_not_gated() -> None:
|
||||
@@ -808,12 +831,13 @@ def test_a_bad_calibration_is_a_config_error(
|
||||
def test_an_agent_without_an_intent_feeds_neither_model_stage_and_is_named(
|
||||
store: Store, config: Config, clock: Clock, fake: FakeEmbedder) -> None:
|
||||
"""Its scores have no fit, so they rank fresh roles, not wanted ones: the
|
||||
newer role tops it, and extract and summarise must not read it on that."""
|
||||
newer role tops it, and nothing of it enters the cut by fit, nor does its
|
||||
top choose what summarise reads."""
|
||||
clock.at = NOW - timedelta(days=1)
|
||||
wanted = role(store, "1", "Backend Engineer", body="Python backend")
|
||||
clock.at = NOW
|
||||
fresh = role(store, "2", "Frontend Engineer", body="React apps")
|
||||
top_one = Limits(extract_top=1, extract_min=0, summarise_top=1, summarise_min=0)
|
||||
top_one = Limits(cut_fit_top=1, cut_fit_min=0, summarise_top=1)
|
||||
store.save_agent("Backend", "python backend", limits=top_one)
|
||||
blind = store.save_agent("Fresh", "", limits=top_one)
|
||||
store.save_agent("Also fresh", " \n", limits=top_one)
|
||||
@@ -854,26 +878,6 @@ def test_a_gated_role_does_not_stretch_the_fit_range_under_the_fallback(
|
||||
assert scores[close][0] == 1.0 and scores[far][0] == 0.0
|
||||
|
||||
|
||||
def test_a_small_prefer_gate_no_longer_outweighs_fit(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
"""The flaw the last real run showed: real cosines sit close together, so
|
||||
under `(cos + 1) / 2` a prefer gate decided the order over fit. Here the
|
||||
cosines are about 0.95 and 0.87: the old fit terms differ by 0.04, which
|
||||
at weight 0.6 is less than the gate's 0.05, so the remote role won.
|
||||
Calibrated, they are 1 and about 0.89, and the closer role wins."""
|
||||
close = role(store, "1", "Backend Engineer", body="Python backend", remote="onsite")
|
||||
far = role(store, "2", "Backend Engineer", body="Python backend data", remote="remote")
|
||||
agent = store.save_agent("Backend", "python backend",
|
||||
[Gate(field="remote", values=["remote"], mode="prefer",
|
||||
weight=0.05)])
|
||||
|
||||
run(config, store, embed.run_embed)
|
||||
run(config, store, score.run_score)
|
||||
|
||||
scores = store.scores(agent.id)
|
||||
assert scores[close][0] > scores[far][0]
|
||||
|
||||
|
||||
def test_the_intent_vector_is_cached_until_the_intent_changes(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
role(store, "1")
|
||||
@@ -1018,9 +1022,7 @@ def test_the_shipped_starter_agents_validate() -> None:
|
||||
block = [line[2:] for line in lines[start:] if line.startswith("# ")]
|
||||
examples = [AgentSeed.model_validate(a) for a in yaml.safe_load("\n".join(block))]
|
||||
assert [a.name for a in examples] == ["Remote from Europe", "Europe hybrid"]
|
||||
# Small enough that a prefer gate breaks ties rather than overrides fit.
|
||||
assert {g.weight for a in [*agents, *examples]
|
||||
for g in a.gates if g.mode == "prefer"} == {0.05}
|
||||
assert examples[0].limits.cut_fit_top == 1000
|
||||
|
||||
|
||||
def test_a_starter_agent_may_list_prompts_or_give_an_intent(
|
||||
@@ -1074,7 +1076,7 @@ def test_the_shipped_starter_searches_validate() -> None:
|
||||
|
||||
# ---------------------------------------------------------------- cascade
|
||||
|
||||
def test_the_pull_job_runs_pull_prep_embed_score_extract_score_then_summarise(
|
||||
def test_the_pull_job_runs_pull_prep_embed_score_extract_score_judge_then_summarise(
|
||||
store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
||||
@@ -1104,10 +1106,10 @@ def test_the_pull_job_runs_pull_prep_embed_score_extract_score_then_summarise(
|
||||
assert batch is not None
|
||||
jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id)
|
||||
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
|
||||
# Enrich found no board with role pages, so the prep, embed and score
|
||||
# Fetch text found no board with role pages, so the prep, embed and score
|
||||
# behind it are passed over.
|
||||
assert stages == ["pull", "prep", "embed", "score", "enrich", "extract", "score",
|
||||
"summarise", "judge"]
|
||||
assert stages == ["pull", "prep", "embed", "score", "fetch_text", "extract", "score",
|
||||
"judge", "summarise"]
|
||||
work = [j.kind for j in jobs if j.kind != FINAL]
|
||||
# The `pull` job and the chain it asked for are one cascade, so the Jobs
|
||||
# tab shows a scheduled pull as one run. Extract and summarise are off and
|
||||
@@ -1159,14 +1161,14 @@ def test_the_model_batches_are_isolated_with_a_timeout() -> None:
|
||||
|
||||
|
||||
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
|
||||
"""By position, not by name: `score` is in the chain twice, once before
|
||||
extract and once after, and the second must lead on to summarise."""
|
||||
assert cascade.CASCADES["pull"] == ("pull", "prep", "embed", "score", "enrich", "prep",
|
||||
"embed", "score", "extract", "score", "summarise",
|
||||
"judge")
|
||||
"""By position, not by name: `score` is in the chain three times, and the
|
||||
one after extract must lead on to judge."""
|
||||
assert cascade.CASCADES["pull"] == ("pull", "prep", "embed", "score", "fetch_text", "prep",
|
||||
"embed", "score", "extract", "score", "judge",
|
||||
"summarise")
|
||||
assert [cascade.follows("pull", at) for at in range(12)] == \
|
||||
["prep", "embed", "score", "enrich", "prep", "embed", "score", "extract", "score",
|
||||
"summarise", "judge", ""]
|
||||
["prep", "embed", "score", "fetch_text", "prep", "embed", "score", "extract", "score",
|
||||
"judge", "summarise", ""]
|
||||
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 11) == ""
|
||||
assert cascade.follows("nope", 0) == ""
|
||||
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
|
||||
@@ -1186,4 +1188,4 @@ def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 8}
|
||||
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 9},
|
||||
context=Context())
|
||||
assert out["next"] == "summarise" and made[1].carries["at"] == 10
|
||||
assert out["next"] == "judge" and made[1].carries["at"] == 10
|
||||
Reference in new issue
Block a user