Extract as the afterburner for the level: model verdict with stated and title hints, best rank first, agreement per host
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3adafd8950
commit
7ca1f02e26
4 files changed
+194
-23
No files matched your search
+95
-2
@@ -136,8 +136,8 @@ def test_closed_and_answered_roles_are_not_offered(
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' "
|
||||
"WHERE source_id = 'closed'")
|
||||
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00' "
|
||||
"WHERE source_id = 'done'")
|
||||
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
|
||||
"seniority_model = 'senior' WHERE source_id = 'done'")
|
||||
model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles])
|
||||
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
@@ -215,3 +215,96 @@ def test_disabled_finds_nothing_and_asks_nobody(
|
||||
result = modeltask.run(extract.TASK, Config(database=str(tmp_path / "x.db")), store)
|
||||
|
||||
assert model.asked == [] and result["extracted"] == 0
|
||||
|
||||
|
||||
def test_a_role_read_before_without_a_level_is_read_again_for_it(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
item = posting("1", seniority="senior")
|
||||
store.upsert_role(item)
|
||||
ranked(store, item)
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
|
||||
"country = 'DE' WHERE source_id = '1'")
|
||||
stub(monkeypatch, lambda roles: [answer(r["uid"], country="FR", seniority="staff")
|
||||
for r in roles])
|
||||
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
|
||||
role = store.role(item.uid)
|
||||
assert role is not None
|
||||
assert (role.seniority_model, role.seniority) == ("staff", "staff")
|
||||
assert role.seniority_stated == "senior", "the host's word stays beside the model's"
|
||||
assert role.country == "DE", "the other fields still only fill what is empty"
|
||||
assert store.extract_pending() == 0
|
||||
|
||||
|
||||
def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
withtext = posting("text", title="Senior Engineer", seniority="mid")
|
||||
noText = posting("teaser", title="Analyst")
|
||||
noText.body = ""
|
||||
store.upsert_role(withtext)
|
||||
store.upsert_role(noText)
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET teaser = 'Join our analytics team' "
|
||||
"WHERE source_id = 'teaser'")
|
||||
ranked(store, withtext, noText)
|
||||
model = stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="mid") for r in roles])
|
||||
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
|
||||
asked = {r["uid"]: r for batch in model.asked for r in batch}
|
||||
one, two = asked[withtext.uid], asked[noText.uid]
|
||||
assert (one["stated"], one["title_level"]) == ("mid", "senior") and "teaser" not in one
|
||||
assert two["teaser"] == "Join our analytics team" and "text" not in two
|
||||
assert "stated" not in two and "overrule" in extract.SYSTEM
|
||||
schema = extract.SCHEMA["properties"]["roles"]["items"]["properties"]["seniority"]
|
||||
assert "" not in schema["enum"], "the afterburner must pick a level"
|
||||
|
||||
|
||||
def test_an_answer_with_no_level_is_charged_until_given_up(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
item = posting("1")
|
||||
store.upsert_role(item)
|
||||
ranked(store, item)
|
||||
stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="") for r in roles])
|
||||
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
assert store.extract_pending() == 1, "answered without a level: still waiting"
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
assert store.extract_pending() == 0
|
||||
|
||||
|
||||
def test_each_agents_best_come_first_then_the_rest(store: Store) -> None:
|
||||
items = [posting(s) for s in ("a1", "a2", "a3", "b1")]
|
||||
for item in items:
|
||||
store.upsert_role(item)
|
||||
one = store.save_agent("One", "python backend")
|
||||
two = store.save_agent("Two", "python backend")
|
||||
store.replace_scores(one.id, [(items[0].uid, 0.9, ""), (items[1].uid, 0.8, ""),
|
||||
(items[2].uid, 0.7, "")])
|
||||
store.replace_scores(two.id, [(items[3].uid, 0.6, "")])
|
||||
|
||||
got = [r.uid for r in store.extract_waiting(10)]
|
||||
|
||||
assert got == [items[0].uid, items[3].uid, items[1].uid, items[2].uid]
|
||||
|
||||
|
||||
def test_agreement_per_host(store: Store) -> None:
|
||||
for sid, board, stated, model in [
|
||||
("1", "personio", "senior", "senior"), ("2", "personio", "junior", "mid"),
|
||||
("3", "lever", "mid", "mid"), ("4", "lever", "", "senior"),
|
||||
("5", "lever", "senior", "")]:
|
||||
store.upsert_role(posting(sid, board=board, seniority=stated))
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET seniority_model = ? WHERE source_id = ?",
|
||||
(model, sid))
|
||||
|
||||
got = {h["host"]: h for h in store.seniority_agreement()}
|
||||
|
||||
assert (got["personio"]["compared"], got["personio"]["agreed"], got["personio"]["rate"]) \
|
||||
== (2, 1, 0.5)
|
||||
assert (got["lever"]["compared"], got["lever"]["rate"], got["lever"]["stated_only"]) \
|
||||
== (1, 1.0, 1)
|
||||
assert [h["host"] for h in store.seniority_agreement()] == ["personio", "lever"]
|
||||
Reference in new issue
Block a user