344 lines
14 KiB
Python
344 lines
14 KiB
Python
"""The `judge` task, with the model call stubbed: which pairs are offered,
|
|
what one call carries, what an answer writes, when a role is read again, and
|
|
the stage. Nothing here reaches a model."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from app.config import Config
|
|
from app.jobs import judge, stages
|
|
from app.models import CvLanguage, Prompt, Weights
|
|
from app.storage import Store
|
|
from desk import modeltask
|
|
from desk.models import Item
|
|
from desk.tools import agent
|
|
from pipeline import TASKS
|
|
|
|
CV = "VP Data Science. Led 40 people in AI and analytics. Python, ML platforms."
|
|
|
|
|
|
def posting(source_id: str, body: str = "Head of AI. Lead the data science team.",
|
|
**raw: object) -> Item:
|
|
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
|
|
title=f"Head of AI {source_id}", body=body,
|
|
published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
|
raw={"company": "Acme", "place": "Copenhagen", **raw})
|
|
item.source = "ats"
|
|
return item
|
|
|
|
|
|
class Model:
|
|
"""Stands in for `agent.ask_json`: records each call's system prompt
|
|
and items, and hands back whatever `reply` builds from the items."""
|
|
|
|
def __init__(self, reply: Any) -> None:
|
|
self.reply = reply
|
|
self.asked: list[list[dict[str, Any]]] = []
|
|
self.systems: list[str] = []
|
|
|
|
def __call__(self, settings: Any, **kwargs: Any) -> agent.Reply:
|
|
items = kwargs["question"]["judgements"]
|
|
self.asked.append(items)
|
|
self.systems.append(kwargs["system"])
|
|
return agent.Reply(data={"judgements": self.reply(items)}, backend="api")
|
|
|
|
|
|
def judged(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
return [{"key": i["key"], "skills": 8, "experience": 7, "career": 9,
|
|
"strengths": ["Leads a data science team.", "", "Python.", "ML.", "Fourth."],
|
|
"gaps": ["No people budget stated."]} for i in items]
|
|
|
|
|
|
def stub(monkeypatch: pytest.MonkeyPatch, reply: Any = judged) -> Model:
|
|
model = Model(reply)
|
|
monkeypatch.setattr(agent, "backend", lambda settings: "api")
|
|
monkeypatch.setattr(agent, "ask_json", model)
|
|
return model
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path) -> Store:
|
|
return Store(tmp_path / "roledesk.db")
|
|
|
|
|
|
@pytest.fixture
|
|
def config(tmp_path: Path) -> Config:
|
|
return Config(database=str(tmp_path / "roledesk.db"), judge={"enabled": True})
|
|
|
|
|
|
def hunter(store: Store, name: str = "Data", cv: str = CV, **kwargs: Any) -> Any:
|
|
return store.save_agent(name, prompts=[Prompt(name="AI lead", text="Head of AI")],
|
|
cv_text=cv, like_text="Building teams.",
|
|
avoid_text="Pure reporting.", **kwargs)
|
|
|
|
|
|
def ranked(store: Store, owner: Any, *ids: str) -> list[str]:
|
|
uids = []
|
|
for sid in ids:
|
|
store.upsert_role(posting(sid))
|
|
uids.append(posting(sid).uid)
|
|
store.replace_scores(owner.id, [(uid, 0.9 - n / 100, "", "", None, 0.9 - n / 100, "fit")
|
|
for n, uid in enumerate(uids)])
|
|
return uids
|
|
|
|
|
|
def test_a_run_writes_the_judgement_and_its_total(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
owner = hunter(store)
|
|
[uid] = ranked(store, owner, "1")
|
|
model = stub(monkeypatch)
|
|
|
|
result = modeltask.run(judge.TASK, config, store)
|
|
|
|
found = store.judgement(owner.id, uid)
|
|
assert found is not None
|
|
assert (found.skills, found.experience, found.career) == (8, 7, 9)
|
|
assert found.cv_score == round(10 * (30 * 8 + 25 * 7 + 30 * 9) / 85, 1)
|
|
assert found.strengths == ["Leads a data science team.", "Python.", "ML."]
|
|
assert found.gaps == ["No people budget stated."]
|
|
assert result["judged"] == 1 and result["pending"] == 0
|
|
[[item]] = model.asked
|
|
assert item == {"key": f"{owner.id}:{uid}", "title": "Head of AI 1", "company": "Acme",
|
|
"place": "Copenhagen", "text": "Head of AI. Lead the data science team."}
|
|
[system] = model.systems
|
|
assert CV in system and "Pure reporting." in system
|
|
assert "- AI lead: Head of AI" in system
|
|
assert '<likes title="what they like: work, workplace, situations">\nBuilding teams.' \
|
|
in system
|
|
assert '<dislikes title="what they do not like">\nPure reporting.' in system
|
|
assert "<avoid" not in system
|
|
|
|
|
|
def test_the_schema_asks_for_three_numbers_strengths_and_gaps() -> None:
|
|
item = judge.SCHEMA["properties"]["judgements"]["items"]
|
|
assert item["required"] == ["key", "skills", "experience", "career", "strengths", "gaps"]
|
|
assert "must-have" not in judge.SYSTEM
|
|
assert "0 to 10" in judge.SYSTEM and "80-100" not in judge.SYSTEM
|
|
|
|
|
|
def test_judgements_made_before_the_format_are_made_again(store: Store) -> None:
|
|
owner = hunter(store)
|
|
assert judge.input_key(owner, "text") != hashlib_old_key(owner, "text")
|
|
|
|
|
|
def test_a_judgement_on_the_0_to_100_scale_is_made_again(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
owner = hunter(store)
|
|
now = judge.input_key(owner, "text")
|
|
monkeypatch.setattr(judge, "FORMAT", 3)
|
|
assert judge.input_key(owner, "text") != now
|
|
|
|
|
|
def test_what_the_person_likes_is_part_of_what_a_judgement_read(store: Store) -> None:
|
|
owner = hunter(store)
|
|
liked = store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
|
like_text="Small teams.")
|
|
assert judge.input_key(liked, "text") != judge.input_key(owner, "text")
|
|
assert "Small teams." in judge.system(liked) and "Building teams." not in judge.system(liked)
|
|
assert "<likes" not in judge.system(
|
|
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, like_text=""))
|
|
|
|
|
|
def hashlib_old_key(agent: Any, text: str) -> str:
|
|
blob = json.dumps([agent.cv_text, [[p.name, p.text] for p in agent.prompts],
|
|
agent.avoid_text, text], ensure_ascii=False)
|
|
return hashlib.sha1(blob.encode()).hexdigest()[:16]
|
|
|
|
|
|
def test_only_enabled_agents_with_a_cv_are_judged(store: Store) -> None:
|
|
for name, kwargs in (("No CV", {"cv": " "}), ("Off", {"enabled": False})):
|
|
ranked(store, hunter(store, name, **kwargs), name)
|
|
bare = store.save_agent("No prompt", cv_text=CV)
|
|
[uid] = ranked(store, bare, "bare")
|
|
assert [(a.id, row["uid"]) for a, row, _, _ in judge.waiting(store)] == [(bare.id, uid)]
|
|
|
|
|
|
def test_an_agent_without_a_prompt_is_judged_without_wants(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A CV alone is enough: the cut is the CV list, and the profile has no
|
|
`<wants>` block."""
|
|
bare = store.save_agent("No prompt", cv_text=CV)
|
|
[uid] = ranked(store, bare, "1")
|
|
model = stub(monkeypatch)
|
|
|
|
modeltask.run(judge.TASK, config, store)
|
|
|
|
found = store.judgement(bare.id, uid)
|
|
assert found is not None and (found.skills, found.experience, found.career) == (8, 7, 9)
|
|
[system] = model.systems
|
|
assert CV in system and "<wants" not in system
|
|
|
|
|
|
def test_the_agents_cut_is_what_is_read_by_fit_then_cv_then_kept(store: Store) -> None:
|
|
owner = hunter(store)
|
|
for sid in "abcdef":
|
|
store.upsert_role(posting(sid))
|
|
uid = {sid: posting(sid).uid for sid in "abcdef"}
|
|
store.replace_scores(owner.id, [
|
|
(uid["a"], 0.5, "", "", 0.9, 0.5, "kept"),
|
|
(uid["b"], 0.6, "", "", 0.7, 0.4, "cv"),
|
|
(uid["c"], 0.7, "", "", 0.1, 0.7, "fit"),
|
|
(uid["d"], 0.8, "", "", 0.1, 0.8, "fit"),
|
|
(uid["e"], 0.9, "", "", 0.9, 0.9, ""),
|
|
(uid["f"], 0.9, "", "", 0.8, 0.3, "cv")])
|
|
assert [row["uid"] for _, row, _, _ in judge.waiting(store)] == [
|
|
uid["d"], uid["c"], uid["f"], uid["b"], uid["a"]]
|
|
|
|
|
|
def test_a_role_without_text_is_not_judged(store: Store) -> None:
|
|
owner = hunter(store)
|
|
item = posting("empty", body="")
|
|
store.upsert_role(item)
|
|
store.replace_scores(owner.id, [(item.uid, 0.9, "", "", None, 0.9, "fit")])
|
|
assert judge.waiting(store) == []
|
|
|
|
|
|
def test_a_judgement_is_read_again_only_when_what_it_read_changed(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
owner = hunter(store)
|
|
[uid] = ranked(store, owner, "1")
|
|
stub(monkeypatch)
|
|
modeltask.run(judge.TASK, config, store)
|
|
assert judge.pending(store) == 0
|
|
|
|
# A rescore and the role closing leave it be.
|
|
store.replace_scores(owner.id, [(uid, 0.8, "", "", None, 0.8, "fit")])
|
|
assert judge.pending(store) == 0
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET closed_at = '2026-10-01' WHERE uid = ?", (uid,))
|
|
assert store.judgement(owner.id, uid) is not None
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET closed_at = NULL WHERE uid = ?", (uid,))
|
|
|
|
# The languages and the dislikes are part of what it read; the weights
|
|
# are not.
|
|
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
|
weights=Weights(cv_skills=50))
|
|
assert judge.pending(store) == 0
|
|
spoken = store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
|
cv_languages=[CvLanguage(language="da", level="native")])
|
|
assert judge.pending(store) == 1
|
|
assert '<languages title="the languages they speak, and how well">\nDanish native' \
|
|
in judge.system(spoken)
|
|
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
|
avoid_text="Long commutes.")
|
|
assert judge.pending(store) == 1
|
|
assert store.judgement(owner.id, uid) is not None, "kept until the new one lands"
|
|
|
|
|
|
def test_a_key_not_in_the_batch_is_rejected_and_a_left_out_pair_given_up(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
owner = hunter(store)
|
|
[uid] = ranked(store, owner, "1")
|
|
stub(monkeypatch, lambda items: [{"key": f"{owner.id}:other", "skills": 1, "experience": 1,
|
|
"career": 1, "strengths": [], "gaps": []}])
|
|
|
|
for _ in range(3):
|
|
result = modeltask.run(judge.TASK, config, store)
|
|
assert store.judgement(owner.id, uid) is None
|
|
assert result["given_up"] == 1 and judge.pending(store) == 0
|
|
|
|
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id)
|
|
assert judge.pending(store) == 1, "a save gives the tries back"
|
|
|
|
|
|
def test_an_answer_without_its_three_numbers_is_rejected(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
owner = hunter(store)
|
|
[uid] = ranked(store, owner, "1")
|
|
stub(monkeypatch, lambda items: [{"key": items[0]["key"], "skills": None,
|
|
"experience": 5, "career": 25,
|
|
"strengths": [], "gaps": []}])
|
|
modeltask.run(judge.TASK, config, store)
|
|
assert store.judgement(owner.id, uid) is None
|
|
stub(monkeypatch, lambda items: [{"key": items[0]["key"], "skills": 4.4,
|
|
"experience": 5, "career": 25,
|
|
"strengths": [], "gaps": []}])
|
|
modeltask.run(judge.TASK, config, store)
|
|
found = store.judgement(owner.id, uid)
|
|
assert found is not None and (found.skills, found.career) == (4, 10)
|
|
|
|
|
|
def test_a_batch_holds_one_agents_pairs_with_that_agents_profile(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
first = hunter(store, "First", cv="First CV.")
|
|
second = hunter(store, "Second", cv="Second CV.")
|
|
shared = ranked(store, first, "a", "b")
|
|
store.replace_scores(second.id, [(uid, 0.9, "", "", None, 0.9, "fit") for uid in shared])
|
|
model = stub(monkeypatch)
|
|
|
|
modeltask.run(judge.TASK, config, store)
|
|
|
|
assert len(model.asked) == 2
|
|
for items, system in zip(model.asked, model.systems, strict=True):
|
|
owners = {i["key"].split(":")[0] for i in items}
|
|
assert len(owners) == 1
|
|
owner = store.agent(int(owners.pop()))
|
|
assert owner is not None and owner.cv_text in system
|
|
assert all(store.judgement(a.id, uid) for a in (first, second) for uid in shared)
|
|
batches = judge.judge_task().finder(config, store)
|
|
assert batches == []
|
|
|
|
|
|
def test_the_finder_cuts_per_agent_and_finds_nothing_while_the_switch_is_off(
|
|
store: Store, config: Config) -> None:
|
|
first = hunter(store, "First")
|
|
second = hunter(store, "Second")
|
|
uids = ranked(store, first, *"abcdefg")
|
|
store.replace_scores(second.id, [(uids[0], 0.9, "", "", None, 0.9, "fit")])
|
|
found = judge.judge_task().finder(config, store)
|
|
assert [len(b.payload["uids"]) for b in found] == [5, 2, 1]
|
|
assert found[0].label == "Judging 5 roles for First, 1 of 3"
|
|
stages.set_stage(store, "judge", False)
|
|
assert judge.judge_task().finder(config, store) == []
|
|
assert "judge" in TASKS
|
|
|
|
|
|
def test_a_job_given_two_agents_keys_asks_once_for_each(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
first = hunter(store, "First", cv="First CV.")
|
|
second = hunter(store, "Second", cv="Second CV.")
|
|
[uid] = ranked(store, first, "a")
|
|
store.replace_scores(second.id, [(uid, 0.9, "", "", None, 0.9, "fit")])
|
|
model = stub(monkeypatch)
|
|
|
|
result = judge.run_batch(config, store, uids=[f"{first.id}:{uid}", f"{second.id}:{uid}"])
|
|
|
|
assert result["judged"] == 2 and len(model.asked) == 2
|
|
|
|
|
|
# ----------------------------------------------------------- CV languages
|
|
|
|
def test_the_languages_of_a_cv_are_suggested_and_checked(
|
|
config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
asked: dict[str, Any] = {}
|
|
|
|
def reply(settings: Any, **kwargs: Any) -> agent.Reply:
|
|
asked.update(kwargs)
|
|
return agent.Reply(data={"languages": [
|
|
{"language": "DA", "level": "native"}, {"language": "en", "level": "fluent"},
|
|
{"language": "German", "level": "professional"},
|
|
{"language": "da", "level": "basic"}]}, backend="api")
|
|
|
|
monkeypatch.setattr(agent, "backend", lambda settings: "api")
|
|
monkeypatch.setattr(agent, "ask_json", reply)
|
|
|
|
found = judge.suggest_languages(config, " Danish native, English fluent. ")
|
|
|
|
assert [(lang.language, lang.level) for lang in found] == [("da", "native"),
|
|
("en", "fluent")]
|
|
assert asked["question"] == {"cv": "Danish native, English fluent."}
|
|
|
|
|
|
def test_no_model_is_an_agent_error(config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(agent, "backend", lambda settings: "")
|
|
with pytest.raises(agent.AgentError):
|
|
judge.suggest_languages(config, "CV")
|