Let the model report a verdict per must-have and cap career at 39 in code when one is broken

The judge numbers an agent's must-haves (one per line) and answers met, broken or unclear for each; career is stored as the model gave it and counts as at most BROKEN_CAP when a role is served, in cv_score, Store.judgement and the SQL of Best CV match. Migration 29 adds role_judgements.musts and broken; the judge's input_key carries a format version so earlier judgements are made again. The reader lists the must-haves with their verdicts and the cap.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet 5.5 committed 2026-10-03 22:40:54 +02:00
1 parent 64fecf57ad
commit 001fc9002b
19 files changed
+336 -61

No files matched your search

+2 -2
View File
@@ -148,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
"VALUES (?, ?, '', '')", (key, name))
conn.execute("PRAGMA user_version = 19")
reopened = Store(path)
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27, 28]
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
assert len(reopened.companies()) == 3
@@ -186,7 +186,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
conn.execute("ALTER TABLE companies DROP COLUMN all_countries")
conn.execute("PRAGMA user_version = 20")
reopened = Store(path)
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27, 28]
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27, 28, 29]
# The tag of 21 is All countries and All roles since 24.
tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
assert {c.key: c.all_roles for c in reopened.companies()} == tagged
+31
View File
@@ -91,6 +91,37 @@ def test_best_cv_match_puts_the_judged_first_by_cv_score(
"order": "best"}).status_code == 422
def test_a_broken_must_have_caps_career_in_the_order_and_the_reader(
client: TestClient, store: Store) -> None:
owner, uid = setup(store)
# c: 90 on all three, but one must-have broken: career counts as 39.
musts = [{"id": "M1", "text": "Permanent", "verdict": "met"},
{"id": "M2", "text": "Remote", "verdict": "broken"}]
store.set_judgement(owner.id, uid["c"], "k", 90, 90, 90, [], [], musts)
capped = round((30 * 90 + 25 * 90 + 30 * 39) / 85, 1)
rows = client.get("/api/roles", params={"agent": owner.id, "order": "cv"}).json()["items"]
assert [(r["uid"], r["cv_score"]) for r in rows[:2]] == [(uid["c"], capped), (uid["b"], 60.0)]
# With career weighted up the capped value drops under b's 60.
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_text=owner.cv_text,
weights=Weights(cv_skills=0, cv_experience=0, cv_career=1))
rows = client.get("/api/roles", params={"agent": owner.id, "order": "cv"}).json()["items"]
assert [(r["uid"], r["cv_score"]) for r in rows[:2]] == [(uid["b"], 60.0), (uid["c"], 39.0)]
one = client.get(f"/api/roles/{uid['c']}", params={"agent": owner.id}).json()
j = one["judgement"]
assert (j["career"], j["career_capped"], j["broken"]) == (90, 39, 1)
assert j["musts"] == musts and one["cv_score"] == j["cv_score"] == 39.0
def test_a_met_or_unclear_must_have_leaves_career_as_the_model_gave_it(
client: TestClient, store: Store) -> None:
owner, uid = setup(store)
musts = [{"id": "M1", "text": "Permanent", "verdict": "met"},
{"id": "M2", "text": "Remote", "verdict": "unclear"}]
store.set_judgement(owner.id, uid["c"], "k", 90, 90, 90, [], [], musts)
j = client.get(f"/api/roles/{uid['c']}", params={"agent": owner.id}).json()["judgement"]
assert (j["career"], j["career_capped"], j["broken"], j["cv_score"]) == (90, 90, 0, 90.0)
def test_the_agents_weights_make_the_cv_score_at_read_time(
client: TestClient, store: Store) -> None:
owner, uid = setup(store)
+25 -3
View File
@@ -16,7 +16,7 @@ from pydantic import ValidationError
from app import prep
from app.config import Config
from app.jobs import extract
from app.models import CvLanguage, RoleLanguage, cv_score, iso_day, language_flag
from app.models import CvLanguage, RoleLanguage, cv_score, iso_day, language_flag, must_lines
from app.storage import Store
from desk import modeltask
from desk.models import Item
@@ -93,7 +93,7 @@ def test_v28_adds_the_cv_columns_and_the_host_deadline_and_runs_twice(tmp_path:
migrated = Store(path)
assert migrated.migrations_applied == [28]
assert migrated.migrations_applied == [28, 29]
rows = {r["source_id"]: r for r in migrated._rows(
"SELECT source_id, languages_required, deadline_host, deadline FROM roles")}
assert (rows["a"]["deadline_host"], rows["a"]["deadline"]) == ("2026-10-18", None)
@@ -105,7 +105,7 @@ def test_v28_adds_the_cv_columns_and_the_host_deadline_and_runs_twice(tmp_path:
assert migrated.role(posting("a").uid).deadline == "2026-10-18"
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 27")
assert Store(path).migrations_applied == [28]
assert Store(path).migrations_applied == [28, 29]
# ---------------------------------------------------------------- deadline
@@ -236,3 +236,25 @@ def test_a_stored_language_list_survives_a_bad_entry(store: Store) -> None:
(json.dumps([{"language": "da", "level": "fluent", "kind": "required"},
{"language": "en", "level": "superb"}]), item.uid))
assert [lang.language for lang in store.role(item.uid).languages_required] == ["da"]
def test_v29_adds_the_musts_and_broken_columns_and_runs_twice(tmp_path: Path) -> None:
path = tmp_path / "v28.db"
Store(path)
with sqlite3.connect(path) as conn:
for column in ("musts", "broken"):
conn.execute(f"ALTER TABLE role_judgements DROP COLUMN {column}")
conn.execute("PRAGMA user_version = 28")
assert Store(path).migrations_applied == [29]
with sqlite3.connect(path) as conn:
columns = {r[1] for r in conn.execute("PRAGMA table_info(role_judgements)")}
assert {"musts", "broken"} <= columns
conn.execute("PRAGMA user_version = 28")
assert Store(path).migrations_applied == [29]
def test_the_musts_text_is_one_must_have_per_line() -> None:
assert must_lines("- A\n* B c\n\u2022 D\n1. E\n2) F\n \nG\n-H") == [
"A", "B c", "D", "E", "F", "G", "-H"]
assert must_lines("") == []
+1 -1
View File
@@ -261,7 +261,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
migrated = Store(path)
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
with sqlite3.connect(path) as conn:
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
+1 -1
View File
@@ -434,7 +434,7 @@ def test_migration_23_gives_every_agent_no_terms(tmp_path: Path) -> None:
conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms")
conn.execute("PRAGMA user_version = 22")
migrated = Store(path)
assert migrated.migrations_applied == [23, 24, 25, 26, 27, 28]
assert migrated.migrations_applied == [23, 24, 25, 26, 27, 28, 29]
assert migrated.agents()[0].fetch_terms == []
+60
View File
@@ -4,6 +4,8 @@ the stage. Nothing here reaches a model."""
from __future__ import annotations
import hashlib
import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
@@ -109,6 +111,64 @@ def test_a_run_writes_the_judgement_and_its_total(
assert "- AI lead: Head of AI" in system and "deal-breaker" in system
def test_the_musts_are_numbered_in_the_profile_and_stored_with_their_text(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_text=CV,
must_text="- Permanent contract\n\n * Remote two days\n3) Danish team\n")
[uid] = ranked(store, owner, "1")
def reply(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [{**one, "career": 80, "musts": [
{"id": "M1", "verdict": "met"}, {"id": "M2", "verdict": "broken"},
{"id": "M9", "verdict": "broken"}, {"id": "M3", "verdict": "maybe"}]}
for one in judged(items)]
model = stub(monkeypatch, reply)
modeltask.run(judge.TASK, config, store)
[system] = model.systems
assert "M1: Permanent contract\nM2: Remote two days\nM3: Danish team" in system
found = store.judgement(owner.id, uid)
assert found is not None
assert [(m.id, m.text, m.verdict) for m in found.musts] == [
("M1", "Permanent contract", "met"), ("M2", "Remote two days", "broken"),
("M3", "Danish team", "unclear")]
assert (found.career, found.career_capped, found.broken) == (80, 39, 1)
assert found.cv_score == round((30 * 80 + 25 * 70 + 30 * 39) / 85, 1)
def test_a_must_the_model_left_out_is_unclear_and_nothing_is_broken_by_silence(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
[uid] = ranked(store, owner, "1")
stub(monkeypatch) # answers without any `musts`
modeltask.run(judge.TASK, config, store)
found = store.judgement(owner.id, uid)
assert found is not None
assert [(m.id, m.verdict) for m in found.musts] == [("M1", "unclear")]
assert (found.career, found.career_capped, found.broken) == (90, 90, 0)
def test_the_schema_requires_the_verdicts_and_the_prompt_leaves_the_cap_to_code() -> None:
item = judge.SCHEMA["properties"]["judgements"]["items"]
assert "musts" in item["required"]
verdict = item["properties"]["musts"]["items"]["properties"]["verdict"]
assert verdict["enum"] == ["met", "broken", "unclear"]
assert "career 0-39" not in judge.SYSTEM and "clearly contradicts" in judge.SYSTEM
def test_judgements_made_before_the_format_are_made_again(store: Store) -> None:
owner = hunter(store)
assert judge.input_key(owner, "text") != hashlib_old_key(owner, "text")
def hashlib_old_key(agent: Any, text: str) -> str:
blob = json.dumps([agent.cv_text, [[p.name, p.text] for p in agent.prompts],
agent.avoid_text, agent.must_text, text], ensure_ascii=False)
return hashlib.sha1(blob.encode()).hexdigest()[:16]
def test_only_enabled_agents_with_a_cv_and_a_prompt_are_judged(store: Store) -> None:
for name, kwargs in (("No CV", {"cv": " "}), ("Off", {"enabled": False})):
ranked(store, hunter(store, name, **kwargs), name)
+1 -1
View File
@@ -120,7 +120,7 @@ def test_migration_24_splits_the_tag_and_moves_terms_off(tmp_path: Path) -> None
conn.execute("UPDATE companies SET all_roles=1 WHERE key='mistral-ai'")
conn.execute("PRAGMA user_version = 23")
migrated = Store(path)
assert migrated.migrations_applied == [24, 25, 26, 27, 28]
assert migrated.migrations_applied == [24, 25, 26, 27, 28, 29]
flags = {c.key: (c.all_countries, c.all_roles) for c in migrated.companies()}
assert flags == {"mistral-ai": (1, 1), "acme": (0, 1), "plain": (0, 0)}
params = {s.target: s.params for s in migrated.subscriptions(source="ats")}
+1 -1
View File
@@ -316,7 +316,7 @@ def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
migrated = Store(path)
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28]
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28, 29]
one = row(migrated, "1")
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
one["country_host"]) == ("senior", "full_time", "true", "DK")
+2 -2
View File
@@ -530,7 +530,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path:
migrated = Store(path)
assert migrated.migrations_applied == [27, 28]
assert migrated.migrations_applied == [27, 28, 29]
assert migrated.gazetteer().anywhere is not None
row = migrated._row("SELECT remote_host, remote_model, remote_where_model FROM roles "
"WHERE uid = ?", (board,))
@@ -540,7 +540,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path:
assert area(migrated, search.uid) == ("", "")
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 26")
assert Store(path).migrations_applied == [27, 28]
assert Store(path).migrations_applied == [27, 28, 29]
def test_a_role_waits_for_extract_until_it_was_asked_where(store: Store) -> None:
+11 -9
View File
@@ -70,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 28
assert LATEST == 29
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -168,6 +168,8 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None:
conn.execute(f"ALTER TABLE companies DROP COLUMN {column}")
for table in ("vectors", "role_tags", "agent_companies", "role_votes"):
conn.execute(f"DROP TABLE {table}")
for column in ("musts", "broken"): # v29
conn.execute(f"ALTER TABLE role_judgements DROP COLUMN {column}")
conn.execute("PRAGMA user_version = 1")
assert Store(old).migrations_applied == list(range(2, LATEST + 1))
@@ -187,7 +189,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
migrated = Store(path)
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
acme = migrated.company("acme")
assert acme is not None
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
@@ -392,7 +394,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
migrated = Store(path)
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
Prompt(name="Backend", text="python backend")]
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
@@ -416,7 +418,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
migrated = Store(path)
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
acme = migrated.company("acme")
assert acme is not None
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
@@ -445,7 +447,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
migrated = Store(path)
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
with sqlite3.connect(path) as conn:
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
"SELECT uid, place_fold, seniority FROM roles")}
@@ -688,7 +690,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
migrated = Store(path)
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27, 28]
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
assert migrated.role_places(stated.uid) == []
@@ -730,7 +732,7 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level(
migrated = Store(path)
assert migrated.migrations_applied == [25, 26, 27, 28]
assert migrated.migrations_applied == [25, 26, 27, 28, 29]
with sqlite3.connect(path) as conn:
found = {sid: rest for sid, *rest in conn.execute(
"SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority "
@@ -748,12 +750,12 @@ def test_v26_adds_posting_url_empty_and_runs_twice_without_harm(tmp_path: Path)
migrated = Store(path)
assert migrated.migrations_applied == [26, 27, 28]
assert migrated.migrations_applied == [26, 27, 28, 29]
role = migrated.role(posting("a").uid)
assert role is not None and role.posting_url == ""
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 25") # the column is there already
assert Store(path).migrations_applied == [26, 27, 28]
assert Store(path).migrations_applied == [26, 27, 28, 29]
def test_a_pull_keeps_the_posting_url(store: Store) -> None: