Remove the must-haves the form dropped on 2026-10-03: the agent's must text, the judge's verdicts and the career cap they set, with migration 34 dropping their columns and the judge's format at 6 so every role is judged again
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
38c31c830b
commit
2159c7925f
19 files changed
+157
-365
No files matched your search
+9
-51
@@ -76,8 +76,7 @@ def config(tmp_path: Path) -> Config:
|
||||
def hunter(store: Store, name: str = "Data", cv: str = CV, **kwargs: Any) -> Any:
|
||||
return store.save_agent(name, prompts=[Prompt(name="AI lead", text="Head of AI")],
|
||||
cv_text=cv, like_text="Building teams.",
|
||||
avoid_text="Pure reporting.",
|
||||
must_text="A permanent contract.", **kwargs)
|
||||
avoid_text="Pure reporting.", **kwargs)
|
||||
|
||||
|
||||
def ranked(store: Store, owner: Any, *ids: str) -> list[str]:
|
||||
@@ -109,59 +108,18 @@ def test_a_run_writes_the_judgement_and_its_total(
|
||||
assert item == {"key": f"{owner.id}:{uid}", "title": "Head of AI 1", "company": "Acme",
|
||||
"place": "Copenhagen", "text": "Head of AI. Lead the data science team."}
|
||||
[system] = model.systems
|
||||
assert CV in system and "Pure reporting." in system and "A permanent contract." in system
|
||||
assert "- AI lead: Head of AI" in system and "deal-breaker" in system
|
||||
assert CV in system and "Pure reporting." in system
|
||||
assert "- AI lead: Head of AI" in system
|
||||
assert '<likes title="what they like: work, workplace, situations">\nBuilding teams.' \
|
||||
in system
|
||||
assert '<dislikes title="what they do not like">\nPure reporting.' in system
|
||||
assert "<avoid" not in system
|
||||
|
||||
|
||||
def test_the_musts_are_numbered_in_the_profile_and_stored_with_their_text(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
owner = hunter(store)
|
||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_text=CV,
|
||||
must_text="- Permanent contract\n\n * Remote two days\n3) Danish team\n")
|
||||
[uid] = ranked(store, owner, "1")
|
||||
|
||||
def reply(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return [{**one, "career": 8, "musts": [
|
||||
{"id": "M1", "verdict": "met"}, {"id": "M2", "verdict": "broken"},
|
||||
{"id": "M9", "verdict": "broken"}, {"id": "M3", "verdict": "maybe"}]}
|
||||
for one in judged(items)]
|
||||
|
||||
model = stub(monkeypatch, reply)
|
||||
modeltask.run(judge.TASK, config, store)
|
||||
|
||||
[system] = model.systems
|
||||
assert "M1: Permanent contract\nM2: Remote two days\nM3: Danish team" in system
|
||||
found = store.judgement(owner.id, uid)
|
||||
assert found is not None
|
||||
assert [(m.id, m.text, m.verdict) for m in found.musts] == [
|
||||
("M1", "Permanent contract", "met"), ("M2", "Remote two days", "broken"),
|
||||
("M3", "Danish team", "unclear")]
|
||||
assert (found.career, found.career_capped, found.broken) == (8, 4, 1)
|
||||
assert found.cv_score == round(10 * (30 * 8 + 25 * 7 + 30 * 4) / 85, 1)
|
||||
|
||||
|
||||
def test_a_must_the_model_left_out_is_unclear_and_nothing_is_broken_by_silence(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
owner = hunter(store)
|
||||
[uid] = ranked(store, owner, "1")
|
||||
stub(monkeypatch) # answers without any `musts`
|
||||
modeltask.run(judge.TASK, config, store)
|
||||
found = store.judgement(owner.id, uid)
|
||||
assert found is not None
|
||||
assert [(m.id, m.verdict) for m in found.musts] == [("M1", "unclear")]
|
||||
assert (found.career, found.career_capped, found.broken) == (9, 9, 0)
|
||||
|
||||
|
||||
def test_the_schema_requires_the_verdicts_and_the_prompt_leaves_the_cap_to_code() -> None:
|
||||
def test_the_schema_asks_for_three_numbers_strengths_and_gaps() -> None:
|
||||
item = judge.SCHEMA["properties"]["judgements"]["items"]
|
||||
assert "musts" in item["required"]
|
||||
verdict = item["properties"]["musts"]["items"]["properties"]["verdict"]
|
||||
assert verdict["enum"] == ["met", "broken", "unclear"]
|
||||
assert "career 0-3" not in judge.SYSTEM and "clearly contradicts" in judge.SYSTEM
|
||||
assert item["required"] == ["key", "skills", "experience", "career", "strengths", "gaps"]
|
||||
assert "must-have" not in judge.SYSTEM
|
||||
assert "0 to 10" in judge.SYSTEM and "80-100" not in judge.SYSTEM
|
||||
|
||||
|
||||
@@ -190,7 +148,7 @@ def test_what_the_person_likes_is_part_of_what_a_judgement_read(store: Store) ->
|
||||
|
||||
def hashlib_old_key(agent: Any, text: str) -> str:
|
||||
blob = json.dumps([agent.cv_text, [[p.name, p.text] for p in agent.prompts],
|
||||
agent.avoid_text, agent.must_text, text], ensure_ascii=False)
|
||||
agent.avoid_text, text], ensure_ascii=False)
|
||||
return hashlib.sha1(blob.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
@@ -259,7 +217,7 @@ def test_a_judgement_is_read_again_only_when_what_it_read_changed(
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET closed_at = NULL WHERE uid = ?", (uid,))
|
||||
|
||||
# The languages and the must text are part of what it read; the weights
|
||||
# The languages and the dislikes are part of what it read; the weights
|
||||
# are not.
|
||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
||||
weights=Weights(cv_skills=50))
|
||||
@@ -270,7 +228,7 @@ def test_a_judgement_is_read_again_only_when_what_it_read_changed(
|
||||
assert '<languages title="the languages they speak, and how well">\nDanish native' \
|
||||
in judge.system(spoken)
|
||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
||||
must_text="Remote two days a week.")
|
||||
avoid_text="Long commutes.")
|
||||
assert judge.pending(store) == 1
|
||||
assert store.judgement(owner.id, uid) is not None, "kept until the new one lands"
|
||||
|
||||
|
||||
Reference in new issue
Block a user