Files
jobboard/tests/test_judge.py
T

344 lines
14 KiB
Python

"""The `judge` task, with the model call stubbed: which pairs are offered,
what one call carries, what an answer writes, when a role is read again, and
the stage. Nothing here reaches a model."""
from __future__ import annotations
import hashlib
import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import pytest
from app.config import Config
from app.jobs import judge, stages
from app.models import CvLanguage, Prompt, Weights
from app.storage import Store
from desk import modeltask
from desk.models import Item
from desk.tools import agent
from pipeline import TASKS
CV = "VP Data Science. Led 40 people in AI and analytics. Python, ML platforms."
def posting(source_id: str, body: str = "Head of AI. Lead the data science team.",
**raw: object) -> Item:
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
title=f"Head of AI {source_id}", body=body,
published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"company": "Acme", "place": "Copenhagen", **raw})
item.source = "ats"
return item
class Model:
"""Stands in for `agent.ask_json`: records each call's system prompt
and items, and hands back whatever `reply` builds from the items."""
def __init__(self, reply: Any) -> None:
self.reply = reply
self.asked: list[list[dict[str, Any]]] = []
self.systems: list[str] = []
def __call__(self, settings: Any, **kwargs: Any) -> agent.Reply:
items = kwargs["question"]["judgements"]
self.asked.append(items)
self.systems.append(kwargs["system"])
return agent.Reply(data={"judgements": self.reply(items)}, backend="api")
def judged(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [{"key": i["key"], "skills": 8, "experience": 7, "career": 9,
"strengths": ["Leads a data science team.", "", "Python.", "ML.", "Fourth."],
"gaps": ["No people budget stated."]} for i in items]
def stub(monkeypatch: pytest.MonkeyPatch, reply: Any = judged) -> Model:
model = Model(reply)
monkeypatch.setattr(agent, "backend", lambda settings: "api")
monkeypatch.setattr(agent, "ask_json", model)
return model
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
@pytest.fixture
def config(tmp_path: Path) -> Config:
return Config(database=str(tmp_path / "roledesk.db"), judge={"enabled": True})
def hunter(store: Store, name: str = "Data", cv: str = CV, **kwargs: Any) -> Any:
return store.save_agent(name, prompts=[Prompt(name="AI lead", text="Head of AI")],
cv_text=cv, like_text="Building teams.",
avoid_text="Pure reporting.", **kwargs)
def ranked(store: Store, owner: Any, *ids: str) -> list[str]:
uids = []
for sid in ids:
store.upsert_role(posting(sid))
uids.append(posting(sid).uid)
store.replace_scores(owner.id, [(uid, 0.9 - n / 100, "", "", None, 0.9 - n / 100, "fit")
for n, uid in enumerate(uids)])
return uids
def test_a_run_writes_the_judgement_and_its_total(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
[uid] = ranked(store, owner, "1")
model = stub(monkeypatch)
result = modeltask.run(judge.TASK, config, store)
found = store.judgement(owner.id, uid)
assert found is not None
assert (found.skills, found.experience, found.career) == (8, 7, 9)
assert found.cv_score == round(10 * (30 * 8 + 25 * 7 + 30 * 9) / 85, 1)
assert found.strengths == ["Leads a data science team.", "Python.", "ML."]
assert found.gaps == ["No people budget stated."]
assert result["judged"] == 1 and result["pending"] == 0
[[item]] = model.asked
assert item == {"key": f"{owner.id}:{uid}", "title": "Head of AI 1", "company": "Acme",
"place": "Copenhagen", "text": "Head of AI. Lead the data science team."}
[system] = model.systems
assert CV in system and "Pure reporting." in system
assert "- AI lead: Head of AI" in system
assert '<likes title="what they like: work, workplace, situations">\nBuilding teams.' \
in system
assert '<dislikes title="what they do not like">\nPure reporting.' in system
assert "<avoid" not in system
def test_the_schema_asks_for_three_numbers_strengths_and_gaps() -> None:
item = judge.SCHEMA["properties"]["judgements"]["items"]
assert item["required"] == ["key", "skills", "experience", "career", "strengths", "gaps"]
assert "must-have" not in judge.SYSTEM
assert "0 to 10" in judge.SYSTEM and "80-100" not in judge.SYSTEM
def test_judgements_made_before_the_format_are_made_again(store: Store) -> None:
owner = hunter(store)
assert judge.input_key(owner, "text") != hashlib_old_key(owner, "text")
def test_a_judgement_on_the_0_to_100_scale_is_made_again(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
now = judge.input_key(owner, "text")
monkeypatch.setattr(judge, "FORMAT", 3)
assert judge.input_key(owner, "text") != now
def test_what_the_person_likes_is_part_of_what_a_judgement_read(store: Store) -> None:
owner = hunter(store)
liked = store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
like_text="Small teams.")
assert judge.input_key(liked, "text") != judge.input_key(owner, "text")
assert "Small teams." in judge.system(liked) and "Building teams." not in judge.system(liked)
assert "<likes" not in judge.system(
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, like_text=""))
def hashlib_old_key(agent: Any, text: str) -> str:
blob = json.dumps([agent.cv_text, [[p.name, p.text] for p in agent.prompts],
agent.avoid_text, text], ensure_ascii=False)
return hashlib.sha1(blob.encode()).hexdigest()[:16]
def test_only_enabled_agents_with_a_cv_are_judged(store: Store) -> None:
for name, kwargs in (("No CV", {"cv": " "}), ("Off", {"enabled": False})):
ranked(store, hunter(store, name, **kwargs), name)
bare = store.save_agent("No prompt", cv_text=CV)
[uid] = ranked(store, bare, "bare")
assert [(a.id, row["uid"]) for a, row, _, _ in judge.waiting(store)] == [(bare.id, uid)]
def test_an_agent_without_a_prompt_is_judged_without_wants(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
"""A CV alone is enough: the cut is the CV list, and the profile has no
`<wants>` block."""
bare = store.save_agent("No prompt", cv_text=CV)
[uid] = ranked(store, bare, "1")
model = stub(monkeypatch)
modeltask.run(judge.TASK, config, store)
found = store.judgement(bare.id, uid)
assert found is not None and (found.skills, found.experience, found.career) == (8, 7, 9)
[system] = model.systems
assert CV in system and "<wants" not in system
def test_the_agents_cut_is_what_is_read_by_fit_then_cv_then_kept(store: Store) -> None:
owner = hunter(store)
for sid in "abcdef":
store.upsert_role(posting(sid))
uid = {sid: posting(sid).uid for sid in "abcdef"}
store.replace_scores(owner.id, [
(uid["a"], 0.5, "", "", 0.9, 0.5, "kept"),
(uid["b"], 0.6, "", "", 0.7, 0.4, "cv"),
(uid["c"], 0.7, "", "", 0.1, 0.7, "fit"),
(uid["d"], 0.8, "", "", 0.1, 0.8, "fit"),
(uid["e"], 0.9, "", "", 0.9, 0.9, ""),
(uid["f"], 0.9, "", "", 0.8, 0.3, "cv")])
assert [row["uid"] for _, row, _, _ in judge.waiting(store)] == [
uid["d"], uid["c"], uid["f"], uid["b"], uid["a"]]
def test_a_role_without_text_is_not_judged(store: Store) -> None:
owner = hunter(store)
item = posting("empty", body="")
store.upsert_role(item)
store.replace_scores(owner.id, [(item.uid, 0.9, "", "", None, 0.9, "fit")])
assert judge.waiting(store) == []
def test_a_judgement_is_read_again_only_when_what_it_read_changed(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
[uid] = ranked(store, owner, "1")
stub(monkeypatch)
modeltask.run(judge.TASK, config, store)
assert judge.pending(store) == 0
# A rescore and the role closing leave it be.
store.replace_scores(owner.id, [(uid, 0.8, "", "", None, 0.8, "fit")])
assert judge.pending(store) == 0
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = '2026-10-01' WHERE uid = ?", (uid,))
assert store.judgement(owner.id, uid) is not None
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = NULL WHERE uid = ?", (uid,))
# The languages and the dislikes are part of what it read; the weights
# are not.
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
weights=Weights(cv_skills=50))
assert judge.pending(store) == 0
spoken = store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
cv_languages=[CvLanguage(language="da", level="native")])
assert judge.pending(store) == 1
assert '<languages title="the languages they speak, and how well">\nDanish native' \
in judge.system(spoken)
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
avoid_text="Long commutes.")
assert judge.pending(store) == 1
assert store.judgement(owner.id, uid) is not None, "kept until the new one lands"
def test_a_key_not_in_the_batch_is_rejected_and_a_left_out_pair_given_up(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
[uid] = ranked(store, owner, "1")
stub(monkeypatch, lambda items: [{"key": f"{owner.id}:other", "skills": 1, "experience": 1,
"career": 1, "strengths": [], "gaps": []}])
for _ in range(3):
result = modeltask.run(judge.TASK, config, store)
assert store.judgement(owner.id, uid) is None
assert result["given_up"] == 1 and judge.pending(store) == 0
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id)
assert judge.pending(store) == 1, "a save gives the tries back"
def test_an_answer_without_its_three_numbers_is_rejected(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
[uid] = ranked(store, owner, "1")
stub(monkeypatch, lambda items: [{"key": items[0]["key"], "skills": None,
"experience": 5, "career": 25,
"strengths": [], "gaps": []}])
modeltask.run(judge.TASK, config, store)
assert store.judgement(owner.id, uid) is None
stub(monkeypatch, lambda items: [{"key": items[0]["key"], "skills": 4.4,
"experience": 5, "career": 25,
"strengths": [], "gaps": []}])
modeltask.run(judge.TASK, config, store)
found = store.judgement(owner.id, uid)
assert found is not None and (found.skills, found.career) == (4, 10)
def test_a_batch_holds_one_agents_pairs_with_that_agents_profile(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
first = hunter(store, "First", cv="First CV.")
second = hunter(store, "Second", cv="Second CV.")
shared = ranked(store, first, "a", "b")
store.replace_scores(second.id, [(uid, 0.9, "", "", None, 0.9, "fit") for uid in shared])
model = stub(monkeypatch)
modeltask.run(judge.TASK, config, store)
assert len(model.asked) == 2
for items, system in zip(model.asked, model.systems, strict=True):
owners = {i["key"].split(":")[0] for i in items}
assert len(owners) == 1
owner = store.agent(int(owners.pop()))
assert owner is not None and owner.cv_text in system
assert all(store.judgement(a.id, uid) for a in (first, second) for uid in shared)
batches = judge.judge_task().finder(config, store)
assert batches == []
def test_the_finder_cuts_per_agent_and_finds_nothing_while_the_switch_is_off(
store: Store, config: Config) -> None:
first = hunter(store, "First")
second = hunter(store, "Second")
uids = ranked(store, first, *"abcdefg")
store.replace_scores(second.id, [(uids[0], 0.9, "", "", None, 0.9, "fit")])
found = judge.judge_task().finder(config, store)
assert [len(b.payload["uids"]) for b in found] == [5, 2, 1]
assert found[0].label == "Judging 5 roles for First, 1 of 3"
stages.set_stage(store, "judge", False)
assert judge.judge_task().finder(config, store) == []
assert "judge" in TASKS
def test_a_job_given_two_agents_keys_asks_once_for_each(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
first = hunter(store, "First", cv="First CV.")
second = hunter(store, "Second", cv="Second CV.")
[uid] = ranked(store, first, "a")
store.replace_scores(second.id, [(uid, 0.9, "", "", None, 0.9, "fit")])
model = stub(monkeypatch)
result = judge.run_batch(config, store, uids=[f"{first.id}:{uid}", f"{second.id}:{uid}"])
assert result["judged"] == 2 and len(model.asked) == 2
# ----------------------------------------------------------- CV languages
def test_the_languages_of_a_cv_are_suggested_and_checked(
config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
asked: dict[str, Any] = {}
def reply(settings: Any, **kwargs: Any) -> agent.Reply:
asked.update(kwargs)
return agent.Reply(data={"languages": [
{"language": "DA", "level": "native"}, {"language": "en", "level": "fluent"},
{"language": "German", "level": "professional"},
{"language": "da", "level": "basic"}]}, backend="api")
monkeypatch.setattr(agent, "backend", lambda settings: "api")
monkeypatch.setattr(agent, "ask_json", reply)
found = judge.suggest_languages(config, " Danish native, English fluent. ")
assert [(lang.language, lang.level) for lang in found] == [("da", "native"),
("en", "fluent")]
assert asked["question"] == {"cv": "Danish native, English fluent."}
def test_no_model_is_an_agent_error(config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(agent, "backend", lambda settings: "")
with pytest.raises(agent.AgentError):
judge.suggest_languages(config, "CV")