Test migration 32, the cut, saved filters and the override rescore, and bring the READMEs and config comments to fetch text and the cut
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
617356e60c
commit
8b6c6d5ac3
9 files changed
+260
-37
No files matched your search
@@ -210,7 +210,7 @@ def test_the_role_carries_its_judgement_and_languages(
|
||||
|
||||
|
||||
def test_an_agent_without_languages_flags_nothing(client: TestClient, store: Store) -> None:
|
||||
owner, uid = setup(store)
|
||||
owner, _uid = setup(store)
|
||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_languages=[])
|
||||
rows = client.get("/api/roles", params={"agent": owner.id}).json()["items"]
|
||||
assert {r["lang_flag"] for r in rows} == {""}
|
||||
|
||||
+70
-1
@@ -23,7 +23,7 @@ from pipeline import FINAL, JOBS, SCHEDULES, TASKS, JobQueue, Jobs, RunContext,
|
||||
from plugins.ats import ATSSource # noqa: E402
|
||||
from app.config import CALIBRATION, AgentSeed, Config, ConfigError # noqa: E402
|
||||
from app.jobs import cascade, embed, score # noqa: E402
|
||||
from app.models import Candidate, Gate, Limits, Prompt, Weights # noqa: E402
|
||||
from app.models import Application, Candidate, Gate, Limits, Prompt, Weights # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
from prepped import upsert # noqa: E402
|
||||
|
||||
@@ -991,6 +991,75 @@ def test_without_an_embedder_the_prompts_are_skipped_and_fit_is_out(
|
||||
assert store.scores(agent.id) == {uid: (0.0, "no prompt vector")}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- cut
|
||||
|
||||
def test_the_affinity_bonus_is_the_share_it_has_in_the_final_score() -> None:
|
||||
w = Weights()
|
||||
assert score.bonus(0, w) == 0
|
||||
assert score.bonus(1, w) == pytest.approx(0.2 / 3 / 1.2) # about 0.056
|
||||
assert score.bonus(2, w) == pytest.approx(0.4 / 3 / 1.2) # about 0.111
|
||||
assert score.bonus(2, Weights(fit=0, recency=0, affinity=0, taste=0)) == 0
|
||||
|
||||
|
||||
def _cands(*rows: tuple[str, str, int]) -> list[Candidate]:
|
||||
"""`(uid, posted_at day, affinity)` as candidates."""
|
||||
return [Candidate(uid=uid, posted_at=f"2026-09-{day}T00:00:00+00:00", affinity=affinity)
|
||||
for uid, day, affinity in rows]
|
||||
|
||||
|
||||
def test_the_cut_is_the_best_by_fit_then_by_cv_then_the_kept() -> None:
|
||||
cands = _cands(("a", "10", 0), ("b", "11", 0), ("c", "12", 0), ("d", "13", 0),
|
||||
("e", "14", 0), ("f", "15", 0), ("g", "16", 0))
|
||||
fits = {"a": 0.9, "b": 0.8, "c": 0.7, "d": 0.65, "e": 0.3, "f": 0.2}
|
||||
near = {"a": 0.99, "b": 0.1, "c": 0.1, "d": 0.75, "e": 0.8, "f": 0.5}
|
||||
passed = {"a", "b", "c", "d", "e", "f"}
|
||||
limits = Limits(cut_fit_top=2, cut_fit_min=0.6, cut_cv_top=2, cut_cv_min=0.6)
|
||||
cut = score.cut_of(cands, passed, fits, near, {"f", "g", "b"}, Weights(), limits)
|
||||
# a and b by fit; e and d the closest to the CV that those leave out (a,
|
||||
# the closest of all, is in already); f kept; g kept but gated, so out.
|
||||
assert cut == {"a": "fit", "b": "fit", "e": "cv", "d": "cv", "f": "kept"}
|
||||
|
||||
|
||||
def test_the_minimum_stops_a_list_before_its_count() -> None:
|
||||
cands = _cands(("a", "10", 0), ("b", "11", 0))
|
||||
limits = Limits(cut_fit_top=500, cut_fit_min=0.6, cut_cv_top=150, cut_cv_min=0.6)
|
||||
cut = score.cut_of(cands, {"a", "b"}, {"a": 0.61, "b": 0.59}, {"b": 0.59}, set(),
|
||||
Weights(), limits)
|
||||
assert cut == {"a": "fit"}
|
||||
|
||||
|
||||
def test_affinity_lifts_a_level_over_the_minimum_but_is_no_pass() -> None:
|
||||
cands = _cands(("liked", "10", 1), ("fave", "10", 2), ("plain", "10", 0),
|
||||
("weak", "10", 2))
|
||||
fits = {"liked": 0.56, "fave": 0.5, "plain": 0.59, "weak": 0.1}
|
||||
cut = score.cut_of(cands, {c.uid for c in cands}, fits, {}, set(), Weights(),
|
||||
Limits(cut_fit_min=0.6))
|
||||
assert cut == {"liked": "fit", "fave": "fit"}
|
||||
|
||||
|
||||
def test_a_tie_goes_to_the_newer_role_then_the_lower_uid() -> None:
|
||||
cands = _cands(("b", "10", 0), ("a", "10", 0), ("old", "09", 0), ("new", "11", 0))
|
||||
fits = dict.fromkeys(("a", "b", "old", "new"), 0.8)
|
||||
cut = score.cut_of(cands, {c.uid for c in cands}, fits, {}, set(), Weights(),
|
||||
Limits(cut_fit_top=3, cut_fit_min=0))
|
||||
assert set(cut) == {"new", "a", "b"}
|
||||
|
||||
|
||||
def test_a_like_and_a_save_put_a_role_in_the_cut_on_the_next_score(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
liked = role(store, "1", "Sales Manager", body="sales")
|
||||
saved = role(store, "2", "Office Manager", body="office")
|
||||
other = role(store, "3", "Barista", body="coffee")
|
||||
agent = store.save_agent("Backend", "python backend",
|
||||
limits=Limits(cut_fit_min=1, cut_cv_min=1))
|
||||
store.set_vote(agent.id, liked, 1)
|
||||
store.save_application(Application(role_uid=saved, role_title="Office Manager"))
|
||||
run(config, store, embed.run_embed)
|
||||
run(config, store, score.run_score)
|
||||
assert store.cut(agent.id) == {liked: "kept", saved: "kept"}
|
||||
assert other not in store.cut(agent.id)
|
||||
|
||||
|
||||
def test_a_disabled_agent_is_not_scored(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
role(store, "1")
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
@@ -1778,3 +1779,116 @@ def test_summarise_waits_on_the_top_of_each_enabled_agent_by_its_own_limits(
|
||||
conn.execute("UPDATE agents SET limits = '{}'")
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b, c}
|
||||
assert store.agents()[0].limits == Limits()
|
||||
|
||||
|
||||
# ------------------------------------------------------------- migration 32
|
||||
|
||||
def _v31(path: Path) -> None:
|
||||
"""Take a file `Store` made back to v31: no cut columns, no saved
|
||||
filters."""
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("DROP INDEX idx_scores_cut")
|
||||
conn.execute("ALTER TABLE scores DROP COLUMN cut")
|
||||
conn.execute("ALTER TABLE scores DROP COLUMN fit")
|
||||
conn.execute("DROP TABLE saved_filters")
|
||||
conn.execute("PRAGMA user_version = 31")
|
||||
|
||||
|
||||
def test_v32_resets_the_limits_drops_prefer_gates_and_renames_enrich(tmp_path: Path) -> None:
|
||||
"""An agent stored before the cut, with the old limits keys, a prefer
|
||||
gate and a `weight` on every gate, loads after migrating; the switch and
|
||||
the queued jobs of `enrich` follow the rename."""
|
||||
from pipeline import Jobs
|
||||
|
||||
path = tmp_path / "v31.db"
|
||||
store = Store(path)
|
||||
upsert(store, posting("a"))
|
||||
Jobs(path)
|
||||
old_gates = [{"field": "country", "values": ["DK"], "mode": "require", "weight": 1.0},
|
||||
{"field": "remote", "values": ["hybrid", "remote"], "mode": "prefer",
|
||||
"weight": 0.05},
|
||||
{"field": "title", "values": ["account executive"], "mode": "exclude",
|
||||
"weight": 1.0}]
|
||||
old_limits = {"extract_top": 500, "extract_min": 0.55, "summarise_top": 30,
|
||||
"summarise_min": 0.65, "judge_top": 150, "judge_min": 0.55,
|
||||
"judge_cv_top": 50}
|
||||
_v31(path)
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("INSERT INTO agents (name, gates, limits, created_at) VALUES (?,?,?,?)",
|
||||
("Wolf", json.dumps(old_gates), json.dumps(old_limits), "2026-10-01"))
|
||||
conn.execute("INSERT INTO agents (name, created_at) VALUES ('Bare', '2026-10-01')")
|
||||
conn.execute("INSERT INTO state (key, value) VALUES ('stage:enrich', 'off')")
|
||||
for kind, state in (("enrich", "queued"), ("enrich_batch", "queued"),
|
||||
("enrich", "done"), ("score", "queued")):
|
||||
conn.execute("INSERT INTO jobs (kind, state, created_at) VALUES (?,?,?)",
|
||||
(kind, state, "2026-10-01"))
|
||||
conn.execute("INSERT INTO schedules (name, kind, cadence, created_at) "
|
||||
"VALUES ('texts', 'enrich', 'every 1d', '2026-10-01')")
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [32]
|
||||
by_name = {a.name: a for a in migrated.agents()}
|
||||
assert by_name["Wolf"].limits == Limits() == by_name["Bare"].limits
|
||||
assert [(g.field, g.mode) for g in by_name["Wolf"].gates] == [
|
||||
("country", "require"), ("title", "exclude")]
|
||||
assert migrated.state["stage:fetch_text"] == "off" and "stage:enrich" not in migrated.state
|
||||
with sqlite3.connect(path) as conn:
|
||||
assert conn.execute("SELECT kind, state FROM jobs ORDER BY id").fetchall() == [
|
||||
("enrich", "done"), ("score", "queued")]
|
||||
assert conn.execute("SELECT kind FROM schedules WHERE name = 'texts'").fetchone() == (
|
||||
"fetch_text",)
|
||||
stored = json.loads(conn.execute(
|
||||
"SELECT gates FROM agents WHERE name = 'Wolf'").fetchone()[0])
|
||||
assert all("weight" not in g for g in stored)
|
||||
agent = by_name["Wolf"]
|
||||
migrated.replace_scores(agent.id, [(posting("a").uid, 0.5, "", "", 0.7, 0.6, "fit")])
|
||||
assert migrated.cut(agent.id) == {posting("a").uid: "fit"}
|
||||
assert migrated.save_filter(agent.id, "Remote", "remote=remote").name == "Remote"
|
||||
|
||||
|
||||
def test_v32_runs_twice_and_keeps_limits_set_since(tmp_path: Path) -> None:
|
||||
path = tmp_path / "v31.db"
|
||||
store = Store(path)
|
||||
agent = store.save_agent("Wolf", limits=Limits(cut_fit_top=900))
|
||||
_v31(path)
|
||||
assert Store(path).migrations_applied == [32]
|
||||
assert Store(path).agent(agent.id).limits == Limits() # type: ignore[union-attr]
|
||||
narrow = Limits(cut_fit_top=900, summarise_top=10)
|
||||
store.save_agent("Wolf", agent_id=agent.id, limits=narrow)
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("PRAGMA user_version = 31") # the columns are there already
|
||||
again = Store(path)
|
||||
assert again.migrations_applied == [32]
|
||||
assert again.agent(agent.id).limits == narrow # type: ignore[union-attr]
|
||||
|
||||
|
||||
def test_a_fresh_file_has_the_cut_columns_and_saved_filters(store: Store) -> None:
|
||||
with store.transaction() as conn:
|
||||
columns = {r[1] for r in conn.execute("PRAGMA table_info(scores)")}
|
||||
tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master")}
|
||||
assert {"fit", "cut", "cv_fit"} <= columns and "saved_filters" in tables
|
||||
|
||||
|
||||
# ------------------------------------------------------------ saved filters
|
||||
|
||||
def test_saved_filters_are_per_agent_named_once_regardless_of_case(store: Store) -> None:
|
||||
one, two = store.save_agent("One"), store.save_agent("Two")
|
||||
remote = store.save_filter(one.id, " Remote only ", "remote=remote&order=newest")
|
||||
store.save_filter(one.id, "berlin", "places=place:berlin")
|
||||
store.save_filter(two.id, "Remote only", "remote=remote")
|
||||
|
||||
assert (remote.name, remote.query) == ("Remote only", "remote=remote&order=newest")
|
||||
assert [f.name for f in store.saved_filters(one.id)] == ["berlin", "Remote only"]
|
||||
with pytest.raises(ValueError, match="already saved"):
|
||||
store.save_filter(one.id, "REMOTE ONLY", "x=1")
|
||||
with pytest.raises(ValueError, match="needs a name"):
|
||||
store.save_filter(one.id, " ", "x=1")
|
||||
with pytest.raises(KeyError):
|
||||
store.save_filter(999, "Nobody", "x=1")
|
||||
|
||||
assert store.delete_filter(remote.id) and not store.delete_filter(remote.id)
|
||||
assert [f.name for f in store.saved_filters(one.id)] == ["berlin"]
|
||||
store.delete_agent(one.id)
|
||||
assert store.saved_filters(one.id) == []
|
||||
assert [f.name for f in store.saved_filters(two.id)] == ["Remote only"]
|
||||
@@ -1874,3 +1874,42 @@ def test_saving_queues_no_summary_when_the_stage_is_off_or_it_has_one(
|
||||
store.upsert_role(posting("3"))
|
||||
client.post("/api/applications", json={"role_uid": posting("3").uid})
|
||||
assert _summary_jobs(queue) == []
|
||||
|
||||
|
||||
# ------------------------------------------------------------ saved filters
|
||||
|
||||
def test_saved_filters_are_listed_saved_and_deleted_per_agent(
|
||||
client: TestClient, store: Store) -> None:
|
||||
one, two = store.save_agent("One"), store.save_agent("Two")
|
||||
url = f"/api/agents/{one.id}/filters"
|
||||
|
||||
made = client.post(url, json={"name": "Remote", "query": "remote=remote&order=newest"})
|
||||
assert made.status_code == 201
|
||||
row = made.json()
|
||||
assert set(row) == {"id", "name", "query", "created_at"}
|
||||
assert (row["name"], row["query"]) == ("Remote", "remote=remote&order=newest")
|
||||
client.post(url, json={"name": "berlin", "query": "places=place:berlin"})
|
||||
assert [f["name"] for f in client.get(url).json()] == ["berlin", "Remote"]
|
||||
assert client.get(f"/api/agents/{two.id}/filters").json() == []
|
||||
|
||||
assert client.post(url, json={"name": "remote", "query": "x=1"}).status_code == 409
|
||||
assert client.post(url, json={"name": " ", "query": "x=1"}).status_code == 422
|
||||
assert client.post("/api/agents/999/filters", json={"name": "A"}).status_code == 404
|
||||
assert client.get("/api/agents/999/filters").status_code == 404
|
||||
|
||||
assert client.delete(f"/api/filters/{row['id']}").status_code == 204
|
||||
assert client.delete(f"/api/filters/{row['id']}").status_code == 404
|
||||
assert [f["name"] for f in client.get(url).json()] == ["berlin"]
|
||||
|
||||
|
||||
def test_an_override_queues_a_rescore_and_fetch_text_either_way(
|
||||
client: TestClient, store: Store, queue: Any) -> None:
|
||||
"""All countries and All roles change the agents' fetch gates for the
|
||||
roles already stored, so either switch, on or off, rescores."""
|
||||
states = ("queued", "running", "done", "failed", "cancelled")
|
||||
store.add_company("Acme")
|
||||
assert client.post("/api/companies/acme/all-roles", json={"on": False}).status_code == 200
|
||||
found = queue.jobs.last("fetch_text", states=states)
|
||||
assert found is not None
|
||||
assert [queue.jobs.get(i).kind for i in found.depends_on] == ["score"]
|
||||
assert queue.jobs.last("pull", states=states) is None
|
||||
Reference in new issue
Block a user