Let the model report a verdict per must-have and cap career at 39 in code when one is broken
The judge numbers an agent's must-haves (one per line) and answers met, broken or unclear for each; career is stored as the model gave it and counts as at most BROKEN_CAP when a role is served, in cv_score, Store.judgement and the SQL of Best CV match. Migration 29 adds role_judgements.musts and broken; the judge's input_key carries a format version so earlier judgements are made again. The reader lists the must-haves with their verdicts and the cap. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
64fecf57ad
commit
001fc9002b
19 files changed
+336
-61
No files matched your search
@@ -4,6 +4,8 @@ the stage. Nothing here reaches a model."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -109,6 +111,64 @@ def test_a_run_writes_the_judgement_and_its_total(
|
||||
assert "- AI lead: Head of AI" in system and "deal-breaker" in system
|
||||
|
||||
|
||||
def test_the_musts_are_numbered_in_the_profile_and_stored_with_their_text(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
owner = hunter(store)
|
||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_text=CV,
|
||||
must_text="- Permanent contract\n\n * Remote two days\n3) Danish team\n")
|
||||
[uid] = ranked(store, owner, "1")
|
||||
|
||||
def reply(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return [{**one, "career": 80, "musts": [
|
||||
{"id": "M1", "verdict": "met"}, {"id": "M2", "verdict": "broken"},
|
||||
{"id": "M9", "verdict": "broken"}, {"id": "M3", "verdict": "maybe"}]}
|
||||
for one in judged(items)]
|
||||
|
||||
model = stub(monkeypatch, reply)
|
||||
modeltask.run(judge.TASK, config, store)
|
||||
|
||||
[system] = model.systems
|
||||
assert "M1: Permanent contract\nM2: Remote two days\nM3: Danish team" in system
|
||||
found = store.judgement(owner.id, uid)
|
||||
assert found is not None
|
||||
assert [(m.id, m.text, m.verdict) for m in found.musts] == [
|
||||
("M1", "Permanent contract", "met"), ("M2", "Remote two days", "broken"),
|
||||
("M3", "Danish team", "unclear")]
|
||||
assert (found.career, found.career_capped, found.broken) == (80, 39, 1)
|
||||
assert found.cv_score == round((30 * 80 + 25 * 70 + 30 * 39) / 85, 1)
|
||||
|
||||
|
||||
def test_a_must_the_model_left_out_is_unclear_and_nothing_is_broken_by_silence(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
owner = hunter(store)
|
||||
[uid] = ranked(store, owner, "1")
|
||||
stub(monkeypatch) # answers without any `musts`
|
||||
modeltask.run(judge.TASK, config, store)
|
||||
found = store.judgement(owner.id, uid)
|
||||
assert found is not None
|
||||
assert [(m.id, m.verdict) for m in found.musts] == [("M1", "unclear")]
|
||||
assert (found.career, found.career_capped, found.broken) == (90, 90, 0)
|
||||
|
||||
|
||||
def test_the_schema_requires_the_verdicts_and_the_prompt_leaves_the_cap_to_code() -> None:
|
||||
item = judge.SCHEMA["properties"]["judgements"]["items"]
|
||||
assert "musts" in item["required"]
|
||||
verdict = item["properties"]["musts"]["items"]["properties"]["verdict"]
|
||||
assert verdict["enum"] == ["met", "broken", "unclear"]
|
||||
assert "career 0-39" not in judge.SYSTEM and "clearly contradicts" in judge.SYSTEM
|
||||
|
||||
|
||||
def test_judgements_made_before_the_format_are_made_again(store: Store) -> None:
|
||||
owner = hunter(store)
|
||||
assert judge.input_key(owner, "text") != hashlib_old_key(owner, "text")
|
||||
|
||||
|
||||
def hashlib_old_key(agent: Any, text: str) -> str:
|
||||
blob = json.dumps([agent.cv_text, [[p.name, p.text] for p in agent.prompts],
|
||||
agent.avoid_text, agent.must_text, text], ensure_ascii=False)
|
||||
return hashlib.sha1(blob.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def test_only_enabled_agents_with_a_cv_and_a_prompt_are_judged(store: Store) -> None:
|
||||
for name, kwargs in (("No CV", {"cv": " "}), ("Off", {"enabled": False})):
|
||||
ranked(store, hunter(store, name, **kwargs), name)
|
||||
|
||||
Reference in new issue
Block a user