Files
jobboard/tests/test_extract.py
T
WolfandClaude Opus 5.5 322db49773 Title levels on the ladder: principal, senior manager, distinguished, lead split (WP A)
from_title matches a title twice, for the highest people level and the
highest individual grade. A people level wins on its own rung; a grade
above that rung lifts it one rung (Principal Engineering Manager is a
senior manager). Senior Staff is principal, Senior Principal, Distinguished
and Technical Fellow are distinguished; Associate Director, AVP and Senior
Manager are senior managers. lead is now a team lead only (Team Lead, Shift
Lead, Supervisor, Teamleiter); Tech Lead, Lead X and X Lead(er) are staff.
A slash names two levels, so Senior/Staff is staff.

- Jobylon "lead" maps to staff; Recruitee maps student_college, manager,
  senior_manager and executive.
- extract's prompt describes the ladder and its two tracks; enum from LEVELS.
- prep VERSION 2, so the next prep maps every open role again.
- Migration 25 clears seniority_model where it is staff, lead, manager or
  director, so extract reads those roles again, and sets roles.seniority to
  the first filled of model, stated and title.
- The coverage test of tests/test_seniority.py passes; its xfail is gone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 13:11:26 +02:00

320 lines
14 KiB
Python

"""The `extract` task, with the model call stubbed: what is offered, what an
answer writes, and what it may not touch. Nothing here reaches a model."""
from __future__ import annotations
import re
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import pytest
from desk import modeltask
from desk.models import Item
from desk.tools import agent
from app.config import Config
from app.jobs import extract
from app.models import LEVELS, Limits
from app.storage import Store
from prepped import upsert
def posting(source_id: str, title: str = "Backend Engineer", **raw: object) -> Item:
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
title=title, body="We pay 80.000-95.000 EUR. Hybrid in Berlin.",
published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=dict(raw))
item.source = "ats"
return item
def answer(uid: str, **fields: Any) -> dict[str, Any]:
empty: dict[str, Any] = {
"place": "", "country": "", "remote": "", "seniority": "", "salary_min": None,
"salary_max": None, "currency": "", "function": "", "permit": "",
"languages_required": []}
return {"uid": uid, **empty, **fields}
class Model:
"""Stands in for `agent.ask_json`: records what it was asked and hands
back whatever `reply` builds from the batch."""
def __init__(self, reply: Any) -> None:
self.reply = reply
self.asked: list[list[dict[str, Any]]] = []
def __call__(self, settings: Any, **kwargs: Any) -> agent.Reply:
roles = kwargs["question"]["roles"]
self.asked.append(roles)
return agent.Reply(data={"roles": self.reply(roles)}, backend="api")
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
@pytest.fixture
def config(tmp_path: Path) -> Config:
return Config(database=str(tmp_path / "roledesk.db"), extract={"enabled": True})
def ranked(store: Store, *items: Item) -> None:
"""Score the roles for one enabled agent, best first, so they are in the
top extract reads. Every role given must already be stored."""
agents = store.agents()
agent_id = agents[0].id if agents else store.save_agent("Backend", "python backend").id
store.replace_scores(agent_id, [(item.uid, 1.0 - n / 10, "")
for n, item in enumerate(items)])
def stub(monkeypatch: pytest.MonkeyPatch, reply: Any) -> Model:
model = Model(reply)
monkeypatch.setattr(agent, "backend", lambda settings: "api")
monkeypatch.setattr(agent, "ask_json", model)
return model
def test_a_batch_writes_the_columns_and_stamps_extracted_at(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
uid = posting("1").uid
upsert(store, posting("1"))
ranked(store, posting("1"))
stub(monkeypatch, lambda roles: [answer(
r["uid"], place="Berlin", country="de", remote="hybrid", seniority="senior",
salary_min=80000, salary_max=95000, currency="eur", function="engineering",
permit="sponsors", languages_required=["EN", "de", "xx-long"]) for r in roles])
result = modeltask.run(extract.TASK, config, store)
role = store.role(uid)
assert role is not None
assert (role.place, role.place_key, role.country, role.remote) == \
("Berlin", "berlin", "DE", "hybrid")
assert (role.seniority, role.function, role.permit) == ("senior", "engineering", "sponsors")
assert (role.salary_min, role.salary_max, role.currency) == (80000, 95000, "EUR")
assert role.languages == ["en", "de"]
assert role.extracted_at is not None
assert result["extracted"] == 1 and result["pending"] == 0
def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
item = posting("1", place="Berlin", remote="hybrid", salary_min=70000)
upsert(store, item)
ranked(store, item)
stub(monkeypatch, lambda roles: [answer(
r["uid"], place="Munich", remote="", salary_min=None, salary_max=90000,
country="DE") for r in roles])
modeltask.run(extract.TASK, config, store)
role = store.role(item.uid)
assert role is not None
assert (role.place, role.remote, role.salary_min) == ("Berlin", "hybrid", 70000)
assert (role.country, role.salary_max) == ("DE", 90000)
def test_the_next_upsert_keeps_what_extract_filled(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
upsert(store, posting("1"))
ranked(store, posting("1"))
stub(monkeypatch, lambda roles: [answer(r["uid"], country="DE", seniority="mid")
for r in roles])
modeltask.run(extract.TASK, config, store)
upsert(store, posting("1"))
role = store.role(posting("1").uid)
assert role is not None and (role.country, role.seniority) == ("DE", "mid")
def test_closed_and_answered_roles_are_not_offered(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
for sid in ("open", "closed", "done"):
upsert(store, posting(sid))
ranked(store, posting("open"), posting("closed"), posting("done"))
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' "
"WHERE source_id = 'closed'")
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
"seniority_model = 'senior' WHERE source_id = 'done'")
model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles])
modeltask.run(extract.TASK, config, store)
assert [[r["uid"] for r in batch] for batch in model.asked] == [[posting("open").uid]]
def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
upsert(store, posting("1"))
ranked(store, posting("1"))
stub(monkeypatch, lambda roles: [answer("not-in-the-batch", country="DE")])
result = modeltask.run(extract.TASK, config, store)
role = store.role(posting("1").uid)
assert role is not None and role.country == "" and role.extracted_at is None
assert result["rejected"] == ["not-in-the-batch"]
assert store.extract_pending() == 1
modeltask.run(extract.TASK, config, store)
modeltask.run(extract.TASK, config, store)
assert store.extract_pending() == 0, "given up after three unanswered replies"
def test_only_the_best_of_each_agent_above_the_floor_is_read(
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""The bound that keeps extract cheap: the top of an enabled agent's
scores, as deep and as low as that agent's own `limits` say, whatever
the boards produced. A role nobody ranks is never sent; a batch dealt a
uid finishes it."""
for sid in ("best", "second", "low", "unscored", "gated"):
upsert(store, posting(sid))
one = store.save_agent("One", "python backend",
limits=Limits(extract_top=1, extract_min=0.5))
two = store.save_agent("Two", "python backend", enabled=False)
store.replace_scores(one.id, [(posting("best").uid, 0.9, ""),
(posting("second").uid, 0.8, ""),
(posting("low").uid, 0.2, ""),
(posting("gated").uid, 0.95, "country unknown")])
store.replace_scores(two.id, [(posting("unscored").uid, 0.99, "")])
config = Config(database=str(tmp_path / "x.db"), extract={"enabled": True})
model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles])
assert store.extract_pending() == 1
store.save_agent("One", "python backend", agent_id=one.id,
limits=Limits(extract_top=5, extract_min=0.5))
assert store.extract_pending() == 2, "the floor holds the low one back"
store.save_agent("One", "python backend", agent_id=one.id,
limits=Limits(extract_top=5, extract_min=0))
assert store.extract_pending() == 3, "the disabled agent's pick never counts"
store.save_agent("One", "python backend", agent_id=one.id,
limits=Limits(extract_top=1, extract_min=0.5))
modeltask.run(extract.TASK, config, store)
assert [[r["uid"] for r in batch] for batch in model.asked] == [[posting("best").uid]]
assert [r.uid for r in store.extract_waiting(10, uids=[posting("low").uid])] == \
[posting("low").uid], "a dealt uid is read whatever its rank"
def test_a_value_outside_the_vocabulary_is_dropped() -> None:
got = extract.fields(answer("u", remote="sometimes", seniority="Senior",
country="Germany", currency="€", function="ops",
permit="maybe", salary_min=-1, salary_max=True))
assert got["remote"] == "" and got["seniority"] == "senior"
assert got["country"] == "" and got["currency"] == "" and got["function"] == ""
assert got["permit"] == "" and got["salary_min"] is None and got["salary_max"] is None
def test_disabled_finds_nothing_and_asks_nobody(
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
upsert(store, posting("1"))
model = stub(monkeypatch, lambda roles: [])
result = modeltask.run(extract.TASK, Config(database=str(tmp_path / "x.db")), store)
assert model.asked == [] and result["extracted"] == 0
def test_a_role_read_before_without_a_level_is_read_again_for_it(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
item = posting("1", seniority="senior")
upsert(store, item)
ranked(store, item)
with store.transaction() as conn:
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
"country = 'DE' WHERE source_id = '1'")
stub(monkeypatch, lambda roles: [answer(r["uid"], country="FR", seniority="staff")
for r in roles])
modeltask.run(extract.TASK, config, store)
role = store.role(item.uid)
assert role is not None
assert (role.seniority_model, role.seniority) == ("staff", "staff")
assert role.seniority_stated == "senior", "the host's word stays beside the model's"
assert role.country == "DE", "the other fields still only fill what is empty"
assert store.extract_pending() == 0
def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
withtext = posting("text", title="Senior Engineer", seniority="mid")
noText = posting("teaser", title="Analyst")
noText.body = ""
upsert(store, withtext)
upsert(store, noText)
with store.transaction() as conn:
conn.execute("UPDATE roles SET teaser = 'Join our analytics team' "
"WHERE source_id = 'teaser'")
ranked(store, withtext, noText)
model = stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="mid") for r in roles])
modeltask.run(extract.TASK, config, store)
asked = {r["uid"]: r for batch in model.asked for r in batch}
one, two = asked[withtext.uid], asked[noText.uid]
assert (one["stated"], one["title_level"]) == ("mid", "senior") and "teaser" not in one
assert two["teaser"] == "Join our analytics team" and "text" not in two
assert "stated" not in two and "overrule" in extract.SYSTEM
schema = extract.SCHEMA["properties"]["roles"]["items"]["properties"]["seniority"]
assert "" not in schema["enum"], "the afterburner must pick a level"
def test_the_prompt_names_every_level_of_the_ladder() -> None:
schema = extract.SCHEMA["properties"]["roles"]["items"]["properties"]["seniority"]
assert schema["enum"] == list(LEVELS)
section = extract.SYSTEM.split("- seniority:")[1].split("- salary_min")[0]
assert all(re.search(rf"\b{level}\b", section) for level in LEVELS)
def test_an_answer_with_no_level_is_charged_until_given_up(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
item = posting("1")
upsert(store, item)
ranked(store, item)
stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="") for r in roles])
modeltask.run(extract.TASK, config, store)
assert store.extract_pending() == 1, "answered without a level: still waiting"
modeltask.run(extract.TASK, config, store)
modeltask.run(extract.TASK, config, store)
assert store.extract_pending() == 0
def test_each_agents_best_come_first_then_the_rest(store: Store) -> None:
items = [posting(s) for s in ("a1", "a2", "a3", "b1")]
for item in items:
upsert(store, item)
one = store.save_agent("One", "python backend")
two = store.save_agent("Two", "python backend")
store.replace_scores(one.id, [(items[0].uid, 0.9, ""), (items[1].uid, 0.8, ""),
(items[2].uid, 0.7, "")])
store.replace_scores(two.id, [(items[3].uid, 0.6, "")])
got = [r.uid for r in store.extract_waiting(10)]
assert got == [items[0].uid, items[3].uid, items[1].uid, items[2].uid]
def test_agreement_per_host(store: Store) -> None:
for sid, board, stated, model in [
("1", "personio", "senior", "senior"), ("2", "personio", "junior", "mid"),
("3", "lever", "mid", "mid"), ("4", "lever", "", "senior"),
("5", "lever", "senior", "")]:
upsert(store, posting(sid, board=board, seniority=stated))
with store.transaction() as conn:
conn.execute("UPDATE roles SET seniority_model = ? WHERE source_id = ?",
(model, sid))
got = {h["host"]: h for h in store.seniority_agreement()}
assert (got["personio"]["compared"], got["personio"]["agreed"], got["personio"]["rate"]) \
== (2, 1, 0.5)
assert (got["lever"]["compared"], got["lever"]["rate"], got["lever"]["stated_only"]) \
== (1, 1.0, 1)
assert [h["host"] for h in store.seniority_agreement()] == ["personio", "lever"]