Add agents.like_text (migration 30) beside avoid_text, now shown as "What I don't like". The judge reads both under <likes> and <dislikes>, its career rubric judges against the prompts, the likes and the dislikes, and FORMAT 3 makes earlier judgements be made again. The form drops the must-haves box and no longer sends must_text, so a stored one is kept and still caps career in code. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
347 lines
15 KiB
Python
347 lines
15 KiB
Python
"""The `judge` task, with the model call stubbed: which pairs are offered,
|
|
what one call carries, what an answer writes, when a role is read again, and
|
|
the stage. Nothing here reaches a model."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from app.config import Config
|
|
from app.jobs import judge, stages
|
|
from app.models import CvLanguage, Limits, Prompt
|
|
from app.storage import Store
|
|
from desk import modeltask
|
|
from desk.models import Item
|
|
from desk.tools import agent
|
|
from pipeline import TASKS
|
|
|
|
CV = "VP Data Science. Led 40 people in AI and analytics. Python, ML platforms."
|
|
|
|
|
|
def posting(source_id: str, body: str = "Head of AI. Lead the data science team.",
|
|
**raw: object) -> Item:
|
|
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
|
|
title=f"Head of AI {source_id}", body=body,
|
|
published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
|
raw={"company": "Acme", "place": "Copenhagen", **raw})
|
|
item.source = "ats"
|
|
return item
|
|
|
|
|
|
class Model:
|
|
"""Stands in for `agent.ask_json`: records each call's system prompt
|
|
and items, and hands back whatever `reply` builds from the items."""
|
|
|
|
def __init__(self, reply: Any) -> None:
|
|
self.reply = reply
|
|
self.asked: list[list[dict[str, Any]]] = []
|
|
self.systems: list[str] = []
|
|
|
|
def __call__(self, settings: Any, **kwargs: Any) -> agent.Reply:
|
|
items = kwargs["question"]["judgements"]
|
|
self.asked.append(items)
|
|
self.systems.append(kwargs["system"])
|
|
return agent.Reply(data={"judgements": self.reply(items)}, backend="api")
|
|
|
|
|
|
def judged(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
return [{"key": i["key"], "skills": 80, "experience": 70, "career": 90,
|
|
"strengths": ["Leads a data science team.", "", "Python.", "ML.", "Fourth."],
|
|
"gaps": ["No people budget stated."]} for i in items]
|
|
|
|
|
|
def stub(monkeypatch: pytest.MonkeyPatch, reply: Any = judged) -> Model:
|
|
model = Model(reply)
|
|
monkeypatch.setattr(agent, "backend", lambda settings: "api")
|
|
monkeypatch.setattr(agent, "ask_json", model)
|
|
return model
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path) -> Store:
|
|
return Store(tmp_path / "roledesk.db")
|
|
|
|
|
|
@pytest.fixture
|
|
def config(tmp_path: Path) -> Config:
|
|
return Config(database=str(tmp_path / "roledesk.db"), judge={"enabled": True})
|
|
|
|
|
|
def hunter(store: Store, name: str = "Data", cv: str = CV, **kwargs: Any) -> Any:
|
|
return store.save_agent(name, prompts=[Prompt(name="AI lead", text="Head of AI")],
|
|
cv_text=cv, like_text="Building teams.",
|
|
avoid_text="Pure reporting.",
|
|
must_text="A permanent contract.", **kwargs)
|
|
|
|
|
|
def ranked(store: Store, owner: Any, *ids: str) -> list[str]:
|
|
uids = []
|
|
for sid in ids:
|
|
store.upsert_role(posting(sid))
|
|
uids.append(posting(sid).uid)
|
|
store.replace_scores(owner.id, [(uid, 0.9 - n / 100, "") for n, uid in enumerate(uids)])
|
|
return uids
|
|
|
|
|
|
def test_a_run_writes_the_judgement_and_its_total(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
owner = hunter(store)
|
|
[uid] = ranked(store, owner, "1")
|
|
model = stub(monkeypatch)
|
|
|
|
result = modeltask.run(judge.TASK, config, store)
|
|
|
|
found = store.judgement(owner.id, uid)
|
|
assert found is not None
|
|
assert (found.skills, found.experience, found.career) == (80, 70, 90)
|
|
assert found.cv_score == round((30 * 80 + 25 * 70 + 30 * 90) / 85, 1)
|
|
assert found.strengths == ["Leads a data science team.", "Python.", "ML."]
|
|
assert found.gaps == ["No people budget stated."]
|
|
assert result["judged"] == 1 and result["pending"] == 0
|
|
[[item]] = model.asked
|
|
assert item == {"key": f"{owner.id}:{uid}", "title": "Head of AI 1", "company": "Acme",
|
|
"place": "Copenhagen", "text": "Head of AI. Lead the data science team."}
|
|
[system] = model.systems
|
|
assert CV in system and "Pure reporting." in system and "A permanent contract." in system
|
|
assert "- AI lead: Head of AI" in system and "deal-breaker" in system
|
|
assert '<likes title="what they like: work, workplace, situations">\nBuilding teams.' \
|
|
in system
|
|
assert '<dislikes title="what they do not like">\nPure reporting.' in system
|
|
assert "<avoid" not in system
|
|
|
|
|
|
def test_the_musts_are_numbered_in_the_profile_and_stored_with_their_text(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
owner = hunter(store)
|
|
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_text=CV,
|
|
must_text="- Permanent contract\n\n * Remote two days\n3) Danish team\n")
|
|
[uid] = ranked(store, owner, "1")
|
|
|
|
def reply(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
return [{**one, "career": 80, "musts": [
|
|
{"id": "M1", "verdict": "met"}, {"id": "M2", "verdict": "broken"},
|
|
{"id": "M9", "verdict": "broken"}, {"id": "M3", "verdict": "maybe"}]}
|
|
for one in judged(items)]
|
|
|
|
model = stub(monkeypatch, reply)
|
|
modeltask.run(judge.TASK, config, store)
|
|
|
|
[system] = model.systems
|
|
assert "M1: Permanent contract\nM2: Remote two days\nM3: Danish team" in system
|
|
found = store.judgement(owner.id, uid)
|
|
assert found is not None
|
|
assert [(m.id, m.text, m.verdict) for m in found.musts] == [
|
|
("M1", "Permanent contract", "met"), ("M2", "Remote two days", "broken"),
|
|
("M3", "Danish team", "unclear")]
|
|
assert (found.career, found.career_capped, found.broken) == (80, 39, 1)
|
|
assert found.cv_score == round((30 * 80 + 25 * 70 + 30 * 39) / 85, 1)
|
|
|
|
|
|
def test_a_must_the_model_left_out_is_unclear_and_nothing_is_broken_by_silence(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
owner = hunter(store)
|
|
[uid] = ranked(store, owner, "1")
|
|
stub(monkeypatch) # answers without any `musts`
|
|
modeltask.run(judge.TASK, config, store)
|
|
found = store.judgement(owner.id, uid)
|
|
assert found is not None
|
|
assert [(m.id, m.verdict) for m in found.musts] == [("M1", "unclear")]
|
|
assert (found.career, found.career_capped, found.broken) == (90, 90, 0)
|
|
|
|
|
|
def test_the_schema_requires_the_verdicts_and_the_prompt_leaves_the_cap_to_code() -> None:
|
|
item = judge.SCHEMA["properties"]["judgements"]["items"]
|
|
assert "musts" in item["required"]
|
|
verdict = item["properties"]["musts"]["items"]["properties"]["verdict"]
|
|
assert verdict["enum"] == ["met", "broken", "unclear"]
|
|
assert "career 0-39" not in judge.SYSTEM and "clearly contradicts" in judge.SYSTEM
|
|
|
|
|
|
def test_judgements_made_before_the_format_are_made_again(store: Store) -> None:
|
|
owner = hunter(store)
|
|
assert judge.input_key(owner, "text") != hashlib_old_key(owner, "text")
|
|
|
|
|
|
def test_what_the_person_likes_is_part_of_what_a_judgement_read(store: Store) -> None:
|
|
owner = hunter(store)
|
|
liked = store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
|
like_text="Small teams.")
|
|
assert judge.input_key(liked, "text") != judge.input_key(owner, "text")
|
|
assert "Small teams." in judge.system(liked) and "Building teams." not in judge.system(liked)
|
|
assert "<likes" not in judge.system(
|
|
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, like_text=""))
|
|
|
|
|
|
def hashlib_old_key(agent: Any, text: str) -> str:
|
|
blob = json.dumps([agent.cv_text, [[p.name, p.text] for p in agent.prompts],
|
|
agent.avoid_text, agent.must_text, text], ensure_ascii=False)
|
|
return hashlib.sha1(blob.encode()).hexdigest()[:16]
|
|
|
|
|
|
def test_only_enabled_agents_with_a_cv_and_a_prompt_are_judged(store: Store) -> None:
|
|
for name, kwargs in (("No CV", {"cv": " "}), ("Off", {"enabled": False})):
|
|
ranked(store, hunter(store, name, **kwargs), name)
|
|
bare = store.save_agent("No prompt", cv_text=CV)
|
|
ranked(store, bare, "bare")
|
|
assert judge.waiting(store) == []
|
|
|
|
|
|
def test_the_agents_judge_limits_bound_what_is_read(store: Store) -> None:
|
|
owner = hunter(store, limits=Limits(judge_top=2, judge_min=0.885))
|
|
uids = ranked(store, owner, "a", "b", "c")
|
|
assert [row["uid"] for _, row, _, _ in judge.waiting(store)] == uids[:2]
|
|
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
|
limits=Limits(judge_top=2, judge_min=0.895))
|
|
assert [row["uid"] for _, row, _, _ in judge.waiting(store)] == uids[:1]
|
|
|
|
|
|
def test_a_role_without_text_is_not_judged(store: Store) -> None:
|
|
owner = hunter(store)
|
|
item = posting("empty", body="")
|
|
store.upsert_role(item)
|
|
store.replace_scores(owner.id, [(item.uid, 0.9, "")])
|
|
assert judge.waiting(store) == []
|
|
|
|
|
|
def test_a_judgement_is_read_again_only_when_what_it_read_changed(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
owner = hunter(store)
|
|
[uid] = ranked(store, owner, "1")
|
|
stub(monkeypatch)
|
|
modeltask.run(judge.TASK, config, store)
|
|
assert judge.pending(store) == 0
|
|
|
|
# A rescore and the role closing leave it be.
|
|
store.replace_scores(owner.id, [(uid, 0.8, "")])
|
|
assert judge.pending(store) == 0
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET closed_at = '2026-10-01' WHERE uid = ?", (uid,))
|
|
assert store.judgement(owner.id, uid) is not None
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET closed_at = NULL WHERE uid = ?", (uid,))
|
|
|
|
# The languages are not part of what it read; the must text is.
|
|
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
|
cv_languages=[CvLanguage(language="da", level="native")])
|
|
assert judge.pending(store) == 0
|
|
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
|
must_text="Remote two days a week.")
|
|
assert judge.pending(store) == 1
|
|
assert store.judgement(owner.id, uid) is not None, "kept until the new one lands"
|
|
|
|
|
|
def test_a_key_not_in_the_batch_is_rejected_and_a_left_out_pair_given_up(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
owner = hunter(store)
|
|
[uid] = ranked(store, owner, "1")
|
|
stub(monkeypatch, lambda items: [{"key": f"{owner.id}:other", "skills": 1, "experience": 1,
|
|
"career": 1, "strengths": [], "gaps": []}])
|
|
|
|
for _ in range(3):
|
|
result = modeltask.run(judge.TASK, config, store)
|
|
assert store.judgement(owner.id, uid) is None
|
|
assert result["given_up"] == 1 and judge.pending(store) == 0
|
|
|
|
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id)
|
|
assert judge.pending(store) == 1, "a save gives the tries back"
|
|
|
|
|
|
def test_an_answer_without_its_three_numbers_is_rejected(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
owner = hunter(store)
|
|
[uid] = ranked(store, owner, "1")
|
|
stub(monkeypatch, lambda items: [{"key": items[0]["key"], "skills": None,
|
|
"experience": 50, "career": 250,
|
|
"strengths": [], "gaps": []}])
|
|
modeltask.run(judge.TASK, config, store)
|
|
assert store.judgement(owner.id, uid) is None
|
|
stub(monkeypatch, lambda items: [{"key": items[0]["key"], "skills": 40.4,
|
|
"experience": 50, "career": 250,
|
|
"strengths": [], "gaps": []}])
|
|
modeltask.run(judge.TASK, config, store)
|
|
found = store.judgement(owner.id, uid)
|
|
assert found is not None and (found.skills, found.career) == (40, 100)
|
|
|
|
|
|
def test_a_batch_holds_one_agents_pairs_with_that_agents_profile(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
first = hunter(store, "First", cv="First CV.")
|
|
second = hunter(store, "Second", cv="Second CV.")
|
|
shared = ranked(store, first, "a", "b")
|
|
store.replace_scores(second.id, [(uid, 0.9, "") for uid in shared])
|
|
model = stub(monkeypatch)
|
|
|
|
modeltask.run(judge.TASK, config, store)
|
|
|
|
assert len(model.asked) == 2
|
|
for items, system in zip(model.asked, model.systems, strict=True):
|
|
owners = {i["key"].split(":")[0] for i in items}
|
|
assert len(owners) == 1
|
|
owner = store.agent(int(owners.pop()))
|
|
assert owner is not None and owner.cv_text in system
|
|
assert all(store.judgement(a.id, uid) for a in (first, second) for uid in shared)
|
|
batches = judge.judge_task().finder(config, store)
|
|
assert batches == []
|
|
|
|
|
|
def test_the_finder_cuts_per_agent_and_finds_nothing_while_the_switch_is_off(
|
|
store: Store, config: Config) -> None:
|
|
first = hunter(store, "First")
|
|
second = hunter(store, "Second")
|
|
uids = ranked(store, first, *"abcdefg")
|
|
store.replace_scores(second.id, [(uids[0], 0.9, "")])
|
|
found = judge.judge_task().finder(config, store)
|
|
assert [len(b.payload["uids"]) for b in found] == [5, 2, 1]
|
|
assert found[0].label == "Judging 5 roles for First, 1 of 3"
|
|
stages.set_stage(store, "judge", False)
|
|
assert judge.judge_task().finder(config, store) == []
|
|
assert "judge" in TASKS
|
|
|
|
|
|
def test_a_job_given_two_agents_keys_asks_once_for_each(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
first = hunter(store, "First", cv="First CV.")
|
|
second = hunter(store, "Second", cv="Second CV.")
|
|
[uid] = ranked(store, first, "a")
|
|
store.replace_scores(second.id, [(uid, 0.9, "")])
|
|
model = stub(monkeypatch)
|
|
|
|
result = judge.run_batch(config, store, uids=[f"{first.id}:{uid}", f"{second.id}:{uid}"])
|
|
|
|
assert result["judged"] == 2 and len(model.asked) == 2
|
|
|
|
|
|
# ----------------------------------------------------------- CV languages
|
|
|
|
def test_the_languages_of_a_cv_are_suggested_and_checked(
|
|
config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
asked: dict[str, Any] = {}
|
|
|
|
def reply(settings: Any, **kwargs: Any) -> agent.Reply:
|
|
asked.update(kwargs)
|
|
return agent.Reply(data={"languages": [
|
|
{"language": "DA", "level": "native"}, {"language": "en", "level": "fluent"},
|
|
{"language": "German", "level": "professional"},
|
|
{"language": "da", "level": "basic"}]}, backend="api")
|
|
|
|
monkeypatch.setattr(agent, "backend", lambda settings: "api")
|
|
monkeypatch.setattr(agent, "ask_json", reply)
|
|
|
|
found = judge.suggest_languages(config, " Danish native, English fluent. ")
|
|
|
|
assert [(lang.language, lang.level) for lang in found] == [("da", "native"),
|
|
("en", "fluent")]
|
|
assert asked["question"] == {"cv": "Danish native, English fluent."}
|
|
|
|
|
|
def test_no_model_is_an_agent_error(config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(agent, "backend", lambda settings: "")
|
|
with pytest.raises(agent.AgentError):
|
|
judge.suggest_languages(config, "CV")
|