"""The `judge` task, with the model call stubbed: which pairs are offered, what one call carries, what an answer writes, when a role is read again, and the stage. Nothing here reaches a model.""" from __future__ import annotations import hashlib import json from datetime import UTC, datetime from pathlib import Path from typing import Any import pytest from app.config import Config from app.jobs import judge, stages from app.models import CvLanguage, Limits, Prompt from app.storage import Store from desk import modeltask from desk.models import Item from desk.tools import agent from pipeline import TASKS CV = "VP Data Science. Led 40 people in AI and analytics. Python, ML platforms." def posting(source_id: str, body: str = "Head of AI. Lead the data science team.", **raw: object) -> Item: item = Item(source_id=source_id, location=f"https://boards.example/{source_id}", title=f"Head of AI {source_id}", body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC), raw={"company": "Acme", "place": "Copenhagen", **raw}) item.source = "ats" return item class Model: """Stands in for `agent.ask_json`: records each call's system prompt and items, and hands back whatever `reply` builds from the items.""" def __init__(self, reply: Any) -> None: self.reply = reply self.asked: list[list[dict[str, Any]]] = [] self.systems: list[str] = [] def __call__(self, settings: Any, **kwargs: Any) -> agent.Reply: items = kwargs["question"]["judgements"] self.asked.append(items) self.systems.append(kwargs["system"]) return agent.Reply(data={"judgements": self.reply(items)}, backend="api") def judged(items: list[dict[str, Any]]) -> list[dict[str, Any]]: return [{"key": i["key"], "skills": 80, "experience": 70, "career": 90, "strengths": ["Leads a data science team.", "", "Python.", "ML.", "Fourth."], "gaps": ["No people budget stated."]} for i in items] def stub(monkeypatch: pytest.MonkeyPatch, reply: Any = judged) -> Model: model = Model(reply) monkeypatch.setattr(agent, "backend", lambda settings: "api") monkeypatch.setattr(agent, "ask_json", model) return model @pytest.fixture def store(tmp_path: Path) -> Store: return Store(tmp_path / "roledesk.db") @pytest.fixture def config(tmp_path: Path) -> Config: return Config(database=str(tmp_path / "roledesk.db"), judge={"enabled": True}) def hunter(store: Store, name: str = "Data", cv: str = CV, **kwargs: Any) -> Any: return store.save_agent(name, prompts=[Prompt(name="AI lead", text="Head of AI")], cv_text=cv, like_text="Building teams.", avoid_text="Pure reporting.", must_text="A permanent contract.", **kwargs) def ranked(store: Store, owner: Any, *ids: str) -> list[str]: uids = [] for sid in ids: store.upsert_role(posting(sid)) uids.append(posting(sid).uid) store.replace_scores(owner.id, [(uid, 0.9 - n / 100, "") for n, uid in enumerate(uids)]) return uids def test_a_run_writes_the_judgement_and_its_total( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: owner = hunter(store) [uid] = ranked(store, owner, "1") model = stub(monkeypatch) result = modeltask.run(judge.TASK, config, store) found = store.judgement(owner.id, uid) assert found is not None assert (found.skills, found.experience, found.career) == (80, 70, 90) assert found.cv_score == round((30 * 80 + 25 * 70 + 30 * 90) / 85, 1) assert found.strengths == ["Leads a data science team.", "Python.", "ML."] assert found.gaps == ["No people budget stated."] assert result["judged"] == 1 and result["pending"] == 0 [[item]] = model.asked assert item == {"key": f"{owner.id}:{uid}", "title": "Head of AI 1", "company": "Acme", "place": "Copenhagen", "text": "Head of AI. Lead the data science team."} [system] = model.systems assert CV in system and "Pure reporting." in system and "A permanent contract." in system assert "- AI lead: Head of AI" in system and "deal-breaker" in system assert '\nBuilding teams.' \ in system assert '\nPure reporting.' in system assert " None: owner = hunter(store) store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_text=CV, must_text="- Permanent contract\n\n * Remote two days\n3) Danish team\n") [uid] = ranked(store, owner, "1") def reply(items: list[dict[str, Any]]) -> list[dict[str, Any]]: return [{**one, "career": 80, "musts": [ {"id": "M1", "verdict": "met"}, {"id": "M2", "verdict": "broken"}, {"id": "M9", "verdict": "broken"}, {"id": "M3", "verdict": "maybe"}]} for one in judged(items)] model = stub(monkeypatch, reply) modeltask.run(judge.TASK, config, store) [system] = model.systems assert "M1: Permanent contract\nM2: Remote two days\nM3: Danish team" in system found = store.judgement(owner.id, uid) assert found is not None assert [(m.id, m.text, m.verdict) for m in found.musts] == [ ("M1", "Permanent contract", "met"), ("M2", "Remote two days", "broken"), ("M3", "Danish team", "unclear")] assert (found.career, found.career_capped, found.broken) == (80, 39, 1) assert found.cv_score == round((30 * 80 + 25 * 70 + 30 * 39) / 85, 1) def test_a_must_the_model_left_out_is_unclear_and_nothing_is_broken_by_silence( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: owner = hunter(store) [uid] = ranked(store, owner, "1") stub(monkeypatch) # answers without any `musts` modeltask.run(judge.TASK, config, store) found = store.judgement(owner.id, uid) assert found is not None assert [(m.id, m.verdict) for m in found.musts] == [("M1", "unclear")] assert (found.career, found.career_capped, found.broken) == (90, 90, 0) def test_the_schema_requires_the_verdicts_and_the_prompt_leaves_the_cap_to_code() -> None: item = judge.SCHEMA["properties"]["judgements"]["items"] assert "musts" in item["required"] verdict = item["properties"]["musts"]["items"]["properties"]["verdict"] assert verdict["enum"] == ["met", "broken", "unclear"] assert "career 0-39" not in judge.SYSTEM and "clearly contradicts" in judge.SYSTEM def test_judgements_made_before_the_format_are_made_again(store: Store) -> None: owner = hunter(store) assert judge.input_key(owner, "text") != hashlib_old_key(owner, "text") def test_what_the_person_likes_is_part_of_what_a_judgement_read(store: Store) -> None: owner = hunter(store) liked = store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, like_text="Small teams.") assert judge.input_key(liked, "text") != judge.input_key(owner, "text") assert "Small teams." in judge.system(liked) and "Building teams." not in judge.system(liked) assert " str: blob = json.dumps([agent.cv_text, [[p.name, p.text] for p in agent.prompts], agent.avoid_text, agent.must_text, text], ensure_ascii=False) return hashlib.sha1(blob.encode()).hexdigest()[:16] def test_only_enabled_agents_with_a_cv_and_a_prompt_are_judged(store: Store) -> None: for name, kwargs in (("No CV", {"cv": " "}), ("Off", {"enabled": False})): ranked(store, hunter(store, name, **kwargs), name) bare = store.save_agent("No prompt", cv_text=CV) ranked(store, bare, "bare") assert judge.waiting(store) == [] def test_the_agents_judge_limits_bound_what_is_read(store: Store) -> None: owner = hunter(store, limits=Limits(judge_top=2, judge_min=0.885)) uids = ranked(store, owner, "a", "b", "c") assert [row["uid"] for _, row, _, _ in judge.waiting(store)] == uids[:2] store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, limits=Limits(judge_top=2, judge_min=0.895)) assert [row["uid"] for _, row, _, _ in judge.waiting(store)] == uids[:1] def test_a_role_without_text_is_not_judged(store: Store) -> None: owner = hunter(store) item = posting("empty", body="") store.upsert_role(item) store.replace_scores(owner.id, [(item.uid, 0.9, "")]) assert judge.waiting(store) == [] def test_a_judgement_is_read_again_only_when_what_it_read_changed( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: owner = hunter(store) [uid] = ranked(store, owner, "1") stub(monkeypatch) modeltask.run(judge.TASK, config, store) assert judge.pending(store) == 0 # A rescore and the role closing leave it be. store.replace_scores(owner.id, [(uid, 0.8, "")]) assert judge.pending(store) == 0 with store.transaction() as conn: conn.execute("UPDATE roles SET closed_at = '2026-10-01' WHERE uid = ?", (uid,)) assert store.judgement(owner.id, uid) is not None with store.transaction() as conn: conn.execute("UPDATE roles SET closed_at = NULL WHERE uid = ?", (uid,)) # The languages are not part of what it read; the must text is. store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_languages=[CvLanguage(language="da", level="native")]) assert judge.pending(store) == 0 store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, must_text="Remote two days a week.") assert judge.pending(store) == 1 assert store.judgement(owner.id, uid) is not None, "kept until the new one lands" def test_a_key_not_in_the_batch_is_rejected_and_a_left_out_pair_given_up( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: owner = hunter(store) [uid] = ranked(store, owner, "1") stub(monkeypatch, lambda items: [{"key": f"{owner.id}:other", "skills": 1, "experience": 1, "career": 1, "strengths": [], "gaps": []}]) for _ in range(3): result = modeltask.run(judge.TASK, config, store) assert store.judgement(owner.id, uid) is None assert result["given_up"] == 1 and judge.pending(store) == 0 store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id) assert judge.pending(store) == 1, "a save gives the tries back" def test_an_answer_without_its_three_numbers_is_rejected( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: owner = hunter(store) [uid] = ranked(store, owner, "1") stub(monkeypatch, lambda items: [{"key": items[0]["key"], "skills": None, "experience": 50, "career": 250, "strengths": [], "gaps": []}]) modeltask.run(judge.TASK, config, store) assert store.judgement(owner.id, uid) is None stub(monkeypatch, lambda items: [{"key": items[0]["key"], "skills": 40.4, "experience": 50, "career": 250, "strengths": [], "gaps": []}]) modeltask.run(judge.TASK, config, store) found = store.judgement(owner.id, uid) assert found is not None and (found.skills, found.career) == (40, 100) def test_a_batch_holds_one_agents_pairs_with_that_agents_profile( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: first = hunter(store, "First", cv="First CV.") second = hunter(store, "Second", cv="Second CV.") shared = ranked(store, first, "a", "b") store.replace_scores(second.id, [(uid, 0.9, "") for uid in shared]) model = stub(monkeypatch) modeltask.run(judge.TASK, config, store) assert len(model.asked) == 2 for items, system in zip(model.asked, model.systems, strict=True): owners = {i["key"].split(":")[0] for i in items} assert len(owners) == 1 owner = store.agent(int(owners.pop())) assert owner is not None and owner.cv_text in system assert all(store.judgement(a.id, uid) for a in (first, second) for uid in shared) batches = judge.judge_task().finder(config, store) assert batches == [] def test_the_finder_cuts_per_agent_and_finds_nothing_while_the_switch_is_off( store: Store, config: Config) -> None: first = hunter(store, "First") second = hunter(store, "Second") uids = ranked(store, first, *"abcdefg") store.replace_scores(second.id, [(uids[0], 0.9, "")]) found = judge.judge_task().finder(config, store) assert [len(b.payload["uids"]) for b in found] == [5, 2, 1] assert found[0].label == "Judging 5 roles for First, 1 of 3" stages.set_stage(store, "judge", False) assert judge.judge_task().finder(config, store) == [] assert "judge" in TASKS def test_a_job_given_two_agents_keys_asks_once_for_each( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: first = hunter(store, "First", cv="First CV.") second = hunter(store, "Second", cv="Second CV.") [uid] = ranked(store, first, "a") store.replace_scores(second.id, [(uid, 0.9, "")]) model = stub(monkeypatch) result = judge.run_batch(config, store, uids=[f"{first.id}:{uid}", f"{second.id}:{uid}"]) assert result["judged"] == 2 and len(model.asked) == 2 # ----------------------------------------------------------- CV languages def test_the_languages_of_a_cv_are_suggested_and_checked( config: Config, monkeypatch: pytest.MonkeyPatch) -> None: asked: dict[str, Any] = {} def reply(settings: Any, **kwargs: Any) -> agent.Reply: asked.update(kwargs) return agent.Reply(data={"languages": [ {"language": "DA", "level": "native"}, {"language": "en", "level": "fluent"}, {"language": "German", "level": "professional"}, {"language": "da", "level": "basic"}]}, backend="api") monkeypatch.setattr(agent, "backend", lambda settings: "api") monkeypatch.setattr(agent, "ask_json", reply) found = judge.suggest_languages(config, " Danish native, English fluent. ") assert [(lang.language, lang.level) for lang in found] == [("da", "native"), ("en", "fluent")] assert asked["question"] == {"cv": "Danish native, English fluent."} def test_no_model_is_an_agent_error(config: Config, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(agent, "backend", lambda settings: "") with pytest.raises(agent.AgentError): judge.suggest_languages(config, "CV")