Files
jobboard/tests/test_judge.py
T

364 lines
15 KiB
Python

"""The `judge` task, with the model call stubbed: which pairs are offered,
what one call carries, what an answer writes, when a role is read again, and
the stage. Nothing here reaches a model."""
from __future__ import annotations
import hashlib
import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import pytest
from app.config import Config
from app.jobs import judge, stages
from app.models import CvLanguage, Prompt
from app.storage import Store
from desk import modeltask
from desk.models import Item
from desk.tools import agent
from pipeline import TASKS
CV = "VP Data Science. Led 40 people in AI and analytics. Python, ML platforms."
def posting(source_id: str, body: str = "Head of AI. Lead the data science team.",
**raw: object) -> Item:
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
title=f"Head of AI {source_id}", body=body,
published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"company": "Acme", "place": "Copenhagen", **raw})
item.source = "ats"
return item
class Model:
"""Stands in for `agent.ask_json`: records each call's system prompt
and items, and hands back whatever `reply` builds from the items."""
def __init__(self, reply: Any) -> None:
self.reply = reply
self.asked: list[list[dict[str, Any]]] = []
self.systems: list[str] = []
def __call__(self, settings: Any, **kwargs: Any) -> agent.Reply:
items = kwargs["question"]["judgements"]
self.asked.append(items)
self.systems.append(kwargs["system"])
return agent.Reply(data={"judgements": self.reply(items)}, backend="api")
def judged(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [{"key": i["key"], "skills": 8, "experience": 7, "career": 9,
"strengths": ["Leads a data science team.", "", "Python.", "ML.", "Fourth."],
"gaps": ["No people budget stated."]} for i in items]
def stub(monkeypatch: pytest.MonkeyPatch, reply: Any = judged) -> Model:
model = Model(reply)
monkeypatch.setattr(agent, "backend", lambda settings: "api")
monkeypatch.setattr(agent, "ask_json", model)
return model
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
@pytest.fixture
def config(tmp_path: Path) -> Config:
return Config(database=str(tmp_path / "roledesk.db"), judge={"enabled": True})
def hunter(store: Store, name: str = "Data", cv: str = CV, **kwargs: Any) -> Any:
return store.save_agent(name, prompts=[Prompt(name="AI lead", text="Head of AI")],
cv_text=cv, like_text="Building teams.",
avoid_text="Pure reporting.",
must_text="A permanent contract.", **kwargs)
def ranked(store: Store, owner: Any, *ids: str) -> list[str]:
uids = []
for sid in ids:
store.upsert_role(posting(sid))
uids.append(posting(sid).uid)
store.replace_scores(owner.id, [(uid, 0.9 - n / 100, "", "", None, 0.9 - n / 100, "fit")
for n, uid in enumerate(uids)])
return uids
def test_a_run_writes_the_judgement_and_its_total(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
[uid] = ranked(store, owner, "1")
model = stub(monkeypatch)
result = modeltask.run(judge.TASK, config, store)
found = store.judgement(owner.id, uid)
assert found is not None
assert (found.skills, found.experience, found.career) == (8, 7, 9)
assert found.cv_score == round(10 * (30 * 8 + 25 * 7 + 30 * 9) / 85, 1)
assert found.strengths == ["Leads a data science team.", "Python.", "ML."]
assert found.gaps == ["No people budget stated."]
assert result["judged"] == 1 and result["pending"] == 0
[[item]] = model.asked
assert item == {"key": f"{owner.id}:{uid}", "title": "Head of AI 1", "company": "Acme",
"place": "Copenhagen", "text": "Head of AI. Lead the data science team."}
[system] = model.systems
assert CV in system and "Pure reporting." in system and "A permanent contract." in system
assert "- AI lead: Head of AI" in system and "deal-breaker" in system
assert '<likes title="what they like: work, workplace, situations">\nBuilding teams.' \
in system
assert '<dislikes title="what they do not like">\nPure reporting.' in system
assert "<avoid" not in system
def test_the_musts_are_numbered_in_the_profile_and_stored_with_their_text(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_text=CV,
must_text="- Permanent contract\n\n * Remote two days\n3) Danish team\n")
[uid] = ranked(store, owner, "1")
def reply(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [{**one, "career": 8, "musts": [
{"id": "M1", "verdict": "met"}, {"id": "M2", "verdict": "broken"},
{"id": "M9", "verdict": "broken"}, {"id": "M3", "verdict": "maybe"}]}
for one in judged(items)]
model = stub(monkeypatch, reply)
modeltask.run(judge.TASK, config, store)
[system] = model.systems
assert "M1: Permanent contract\nM2: Remote two days\nM3: Danish team" in system
found = store.judgement(owner.id, uid)
assert found is not None
assert [(m.id, m.text, m.verdict) for m in found.musts] == [
("M1", "Permanent contract", "met"), ("M2", "Remote two days", "broken"),
("M3", "Danish team", "unclear")]
assert (found.career, found.career_capped, found.broken) == (8, 4, 1)
assert found.cv_score == round(10 * (30 * 8 + 25 * 7 + 30 * 4) / 85, 1)
def test_a_must_the_model_left_out_is_unclear_and_nothing_is_broken_by_silence(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
[uid] = ranked(store, owner, "1")
stub(monkeypatch) # answers without any `musts`
modeltask.run(judge.TASK, config, store)
found = store.judgement(owner.id, uid)
assert found is not None
assert [(m.id, m.verdict) for m in found.musts] == [("M1", "unclear")]
assert (found.career, found.career_capped, found.broken) == (9, 9, 0)
def test_the_schema_requires_the_verdicts_and_the_prompt_leaves_the_cap_to_code() -> None:
item = judge.SCHEMA["properties"]["judgements"]["items"]
assert "musts" in item["required"]
verdict = item["properties"]["musts"]["items"]["properties"]["verdict"]
assert verdict["enum"] == ["met", "broken", "unclear"]
assert "career 0-3" not in judge.SYSTEM and "clearly contradicts" in judge.SYSTEM
assert "0 to 10" in judge.SYSTEM and "80-100" not in judge.SYSTEM
def test_judgements_made_before_the_format_are_made_again(store: Store) -> None:
owner = hunter(store)
assert judge.input_key(owner, "text") != hashlib_old_key(owner, "text")
def test_a_judgement_on_the_0_to_100_scale_is_made_again(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
now = judge.input_key(owner, "text")
monkeypatch.setattr(judge, "FORMAT", 3)
assert judge.input_key(owner, "text") != now
def test_what_the_person_likes_is_part_of_what_a_judgement_read(store: Store) -> None:
owner = hunter(store)
liked = store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
like_text="Small teams.")
assert judge.input_key(liked, "text") != judge.input_key(owner, "text")
assert "Small teams." in judge.system(liked) and "Building teams." not in judge.system(liked)
assert "<likes" not in judge.system(
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, like_text=""))
def hashlib_old_key(agent: Any, text: str) -> str:
blob = json.dumps([agent.cv_text, [[p.name, p.text] for p in agent.prompts],
agent.avoid_text, agent.must_text, text], ensure_ascii=False)
return hashlib.sha1(blob.encode()).hexdigest()[:16]
def test_only_enabled_agents_with_a_cv_and_a_prompt_are_judged(store: Store) -> None:
for name, kwargs in (("No CV", {"cv": " "}), ("Off", {"enabled": False})):
ranked(store, hunter(store, name, **kwargs), name)
bare = store.save_agent("No prompt", cv_text=CV)
ranked(store, bare, "bare")
assert judge.waiting(store) == []
def test_the_agents_cut_is_what_is_read_by_fit_then_cv_then_kept(store: Store) -> None:
owner = hunter(store)
for sid in "abcdef":
store.upsert_role(posting(sid))
uid = {sid: posting(sid).uid for sid in "abcdef"}
store.replace_scores(owner.id, [
(uid["a"], 0.5, "", "", 0.9, 0.5, "kept"),
(uid["b"], 0.6, "", "", 0.7, 0.4, "cv"),
(uid["c"], 0.7, "", "", 0.1, 0.7, "fit"),
(uid["d"], 0.8, "", "", 0.1, 0.8, "fit"),
(uid["e"], 0.9, "", "", 0.9, 0.9, ""),
(uid["f"], 0.9, "", "", 0.8, 0.3, "cv")])
assert [row["uid"] for _, row, _, _ in judge.waiting(store)] == [
uid["d"], uid["c"], uid["f"], uid["b"], uid["a"]]
def test_a_role_without_text_is_not_judged(store: Store) -> None:
owner = hunter(store)
item = posting("empty", body="")
store.upsert_role(item)
store.replace_scores(owner.id, [(item.uid, 0.9, "", "", None, 0.9, "fit")])
assert judge.waiting(store) == []
def test_a_judgement_is_read_again_only_when_what_it_read_changed(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
[uid] = ranked(store, owner, "1")
stub(monkeypatch)
modeltask.run(judge.TASK, config, store)
assert judge.pending(store) == 0
# A rescore and the role closing leave it be.
store.replace_scores(owner.id, [(uid, 0.8, "", "", None, 0.8, "fit")])
assert judge.pending(store) == 0
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = '2026-10-01' WHERE uid = ?", (uid,))
assert store.judgement(owner.id, uid) is not None
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = NULL WHERE uid = ?", (uid,))
# The languages are not part of what it read; the must text is.
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
cv_languages=[CvLanguage(language="da", level="native")])
assert judge.pending(store) == 0
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
must_text="Remote two days a week.")
assert judge.pending(store) == 1
assert store.judgement(owner.id, uid) is not None, "kept until the new one lands"
def test_a_key_not_in_the_batch_is_rejected_and_a_left_out_pair_given_up(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
[uid] = ranked(store, owner, "1")
stub(monkeypatch, lambda items: [{"key": f"{owner.id}:other", "skills": 1, "experience": 1,
"career": 1, "strengths": [], "gaps": []}])
for _ in range(3):
result = modeltask.run(judge.TASK, config, store)
assert store.judgement(owner.id, uid) is None
assert result["given_up"] == 1 and judge.pending(store) == 0
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id)
assert judge.pending(store) == 1, "a save gives the tries back"
def test_an_answer_without_its_three_numbers_is_rejected(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
[uid] = ranked(store, owner, "1")
stub(monkeypatch, lambda items: [{"key": items[0]["key"], "skills": None,
"experience": 5, "career": 25,
"strengths": [], "gaps": []}])
modeltask.run(judge.TASK, config, store)
assert store.judgement(owner.id, uid) is None
stub(monkeypatch, lambda items: [{"key": items[0]["key"], "skills": 4.4,
"experience": 5, "career": 25,
"strengths": [], "gaps": []}])
modeltask.run(judge.TASK, config, store)
found = store.judgement(owner.id, uid)
assert found is not None and (found.skills, found.career) == (4, 10)
def test_a_batch_holds_one_agents_pairs_with_that_agents_profile(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
first = hunter(store, "First", cv="First CV.")
second = hunter(store, "Second", cv="Second CV.")
shared = ranked(store, first, "a", "b")
store.replace_scores(second.id, [(uid, 0.9, "", "", None, 0.9, "fit") for uid in shared])
model = stub(monkeypatch)
modeltask.run(judge.TASK, config, store)
assert len(model.asked) == 2
for items, system in zip(model.asked, model.systems, strict=True):
owners = {i["key"].split(":")[0] for i in items}
assert len(owners) == 1
owner = store.agent(int(owners.pop()))
assert owner is not None and owner.cv_text in system
assert all(store.judgement(a.id, uid) for a in (first, second) for uid in shared)
batches = judge.judge_task().finder(config, store)
assert batches == []
def test_the_finder_cuts_per_agent_and_finds_nothing_while_the_switch_is_off(
store: Store, config: Config) -> None:
first = hunter(store, "First")
second = hunter(store, "Second")
uids = ranked(store, first, *"abcdefg")
store.replace_scores(second.id, [(uids[0], 0.9, "", "", None, 0.9, "fit")])
found = judge.judge_task().finder(config, store)
assert [len(b.payload["uids"]) for b in found] == [5, 2, 1]
assert found[0].label == "Judging 5 roles for First, 1 of 3"
stages.set_stage(store, "judge", False)
assert judge.judge_task().finder(config, store) == []
assert "judge" in TASKS
def test_a_job_given_two_agents_keys_asks_once_for_each(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
first = hunter(store, "First", cv="First CV.")
second = hunter(store, "Second", cv="Second CV.")
[uid] = ranked(store, first, "a")
store.replace_scores(second.id, [(uid, 0.9, "", "", None, 0.9, "fit")])
model = stub(monkeypatch)
result = judge.run_batch(config, store, uids=[f"{first.id}:{uid}", f"{second.id}:{uid}"])
assert result["judged"] == 2 and len(model.asked) == 2
# ----------------------------------------------------------- CV languages
def test_the_languages_of_a_cv_are_suggested_and_checked(
config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
asked: dict[str, Any] = {}
def reply(settings: Any, **kwargs: Any) -> agent.Reply:
asked.update(kwargs)
return agent.Reply(data={"languages": [
{"language": "DA", "level": "native"}, {"language": "en", "level": "fluent"},
{"language": "German", "level": "professional"},
{"language": "da", "level": "basic"}]}, backend="api")
monkeypatch.setattr(agent, "backend", lambda settings: "api")
monkeypatch.setattr(agent, "ask_json", reply)
found = judge.suggest_languages(config, " Danish native, English fluent. ")
assert [(lang.language, lang.level) for lang in found] == [("da", "native"),
("en", "fluent")]
assert asked["question"] == {"cv": "Danish native, English fluent."}
def test_no_model_is_an_agent_error(config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(agent, "backend", lambda settings: "")
with pytest.raises(agent.AgentError):
judge.suggest_languages(config, "CV")