311 lines
13 KiB
Python
311 lines
13 KiB
Python
"""The `extract` task, with the model call stubbed: what is offered, what an
|
|
answer writes, and what it may not touch. Nothing here reaches a model."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from desk import modeltask
|
|
from desk.models import Item
|
|
from desk.tools import agent
|
|
from app.config import Config
|
|
from app.jobs import extract
|
|
from app.models import Limits
|
|
from app.storage import Store
|
|
|
|
|
|
def posting(source_id: str, title: str = "Backend Engineer", **raw: object) -> Item:
|
|
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
|
|
title=title, body="We pay 80.000-95.000 EUR. Hybrid in Berlin.",
|
|
published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=dict(raw))
|
|
item.source = "ats"
|
|
return item
|
|
|
|
|
|
def answer(uid: str, **fields: Any) -> dict[str, Any]:
|
|
empty: dict[str, Any] = {
|
|
"place": "", "country": "", "remote": "", "seniority": "", "salary_min": None,
|
|
"salary_max": None, "currency": "", "function": "", "permit": "",
|
|
"languages_required": []}
|
|
return {"uid": uid, **empty, **fields}
|
|
|
|
|
|
class Model:
|
|
"""Stands in for `agent.ask_json`: records what it was asked and hands
|
|
back whatever `reply` builds from the batch."""
|
|
|
|
def __init__(self, reply: Any) -> None:
|
|
self.reply = reply
|
|
self.asked: list[list[dict[str, Any]]] = []
|
|
|
|
def __call__(self, settings: Any, **kwargs: Any) -> agent.Reply:
|
|
roles = kwargs["question"]["roles"]
|
|
self.asked.append(roles)
|
|
return agent.Reply(data={"roles": self.reply(roles)}, backend="api")
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path) -> Store:
|
|
return Store(tmp_path / "roledesk.db")
|
|
|
|
|
|
@pytest.fixture
|
|
def config(tmp_path: Path) -> Config:
|
|
return Config(database=str(tmp_path / "roledesk.db"), extract={"enabled": True})
|
|
|
|
|
|
def ranked(store: Store, *items: Item) -> None:
|
|
"""Score the roles for one enabled agent, best first, so they are in the
|
|
top extract reads. Every role given must already be stored."""
|
|
agents = store.agents()
|
|
agent_id = agents[0].id if agents else store.save_agent("Backend", "python backend").id
|
|
store.replace_scores(agent_id, [(item.uid, 1.0 - n / 10, "")
|
|
for n, item in enumerate(items)])
|
|
|
|
|
|
def stub(monkeypatch: pytest.MonkeyPatch, reply: Any) -> Model:
|
|
model = Model(reply)
|
|
monkeypatch.setattr(agent, "backend", lambda settings: "api")
|
|
monkeypatch.setattr(agent, "ask_json", model)
|
|
return model
|
|
|
|
|
|
def test_a_batch_writes_the_columns_and_stamps_extracted_at(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
uid = posting("1").uid
|
|
store.upsert_role(posting("1"))
|
|
ranked(store, posting("1"))
|
|
stub(monkeypatch, lambda roles: [answer(
|
|
r["uid"], place="Berlin", country="de", remote="hybrid", seniority="senior",
|
|
salary_min=80000, salary_max=95000, currency="eur", function="engineering",
|
|
permit="sponsors", languages_required=["EN", "de", "xx-long"]) for r in roles])
|
|
|
|
result = modeltask.run(extract.TASK, config, store)
|
|
|
|
role = store.role(uid)
|
|
assert role is not None
|
|
assert (role.place, role.place_key, role.country, role.remote) == \
|
|
("Berlin", "berlin", "DE", "hybrid")
|
|
assert (role.seniority, role.function, role.permit) == ("senior", "engineering", "sponsors")
|
|
assert (role.salary_min, role.salary_max, role.currency) == (80000, 95000, "EUR")
|
|
assert role.languages == ["en", "de"]
|
|
assert role.extracted_at is not None
|
|
assert result["extracted"] == 1 and result["pending"] == 0
|
|
|
|
|
|
def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
item = posting("1", place="Berlin", remote="hybrid", salary_min=70000)
|
|
store.upsert_role(item)
|
|
ranked(store, item)
|
|
stub(monkeypatch, lambda roles: [answer(
|
|
r["uid"], place="Munich", remote="", salary_min=None, salary_max=90000,
|
|
country="DE") for r in roles])
|
|
|
|
modeltask.run(extract.TASK, config, store)
|
|
|
|
role = store.role(item.uid)
|
|
assert role is not None
|
|
assert (role.place, role.remote, role.salary_min) == ("Berlin", "hybrid", 70000)
|
|
assert (role.country, role.salary_max) == ("DE", 90000)
|
|
|
|
|
|
def test_the_next_upsert_keeps_what_extract_filled(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.upsert_role(posting("1"))
|
|
ranked(store, posting("1"))
|
|
stub(monkeypatch, lambda roles: [answer(r["uid"], country="DE", seniority="mid")
|
|
for r in roles])
|
|
modeltask.run(extract.TASK, config, store)
|
|
|
|
store.upsert_role(posting("1"))
|
|
|
|
role = store.role(posting("1").uid)
|
|
assert role is not None and (role.country, role.seniority) == ("DE", "mid")
|
|
|
|
|
|
def test_closed_and_answered_roles_are_not_offered(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
for sid in ("open", "closed", "done"):
|
|
store.upsert_role(posting(sid))
|
|
ranked(store, posting("open"), posting("closed"), posting("done"))
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' "
|
|
"WHERE source_id = 'closed'")
|
|
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
|
|
"seniority_model = 'senior' WHERE source_id = 'done'")
|
|
model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles])
|
|
|
|
modeltask.run(extract.TASK, config, store)
|
|
|
|
assert [[r["uid"] for r in batch] for batch in model.asked] == [[posting("open").uid]]
|
|
|
|
|
|
def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.upsert_role(posting("1"))
|
|
ranked(store, posting("1"))
|
|
stub(monkeypatch, lambda roles: [answer("not-in-the-batch", country="DE")])
|
|
|
|
result = modeltask.run(extract.TASK, config, store)
|
|
|
|
role = store.role(posting("1").uid)
|
|
assert role is not None and role.country == "" and role.extracted_at is None
|
|
assert result["rejected"] == ["not-in-the-batch"]
|
|
assert store.extract_pending() == 1
|
|
|
|
modeltask.run(extract.TASK, config, store)
|
|
modeltask.run(extract.TASK, config, store)
|
|
assert store.extract_pending() == 0, "given up after three unanswered replies"
|
|
|
|
|
|
def test_only_the_best_of_each_agent_above_the_floor_is_read(
|
|
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The bound that keeps extract cheap: the top of an enabled agent's
|
|
scores, as deep and as low as that agent's own `limits` say, whatever
|
|
the boards produced. A role nobody ranks is never sent; a batch dealt a
|
|
uid finishes it."""
|
|
for sid in ("best", "second", "low", "unscored", "gated"):
|
|
store.upsert_role(posting(sid))
|
|
one = store.save_agent("One", "python backend",
|
|
limits=Limits(extract_top=1, extract_min=0.5))
|
|
two = store.save_agent("Two", "python backend", enabled=False)
|
|
store.replace_scores(one.id, [(posting("best").uid, 0.9, ""),
|
|
(posting("second").uid, 0.8, ""),
|
|
(posting("low").uid, 0.2, ""),
|
|
(posting("gated").uid, 0.95, "country unknown")])
|
|
store.replace_scores(two.id, [(posting("unscored").uid, 0.99, "")])
|
|
config = Config(database=str(tmp_path / "x.db"), extract={"enabled": True})
|
|
model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles])
|
|
|
|
assert store.extract_pending() == 1
|
|
store.save_agent("One", "python backend", agent_id=one.id,
|
|
limits=Limits(extract_top=5, extract_min=0.5))
|
|
assert store.extract_pending() == 2, "the floor holds the low one back"
|
|
store.save_agent("One", "python backend", agent_id=one.id,
|
|
limits=Limits(extract_top=5, extract_min=0))
|
|
assert store.extract_pending() == 3, "the disabled agent's pick never counts"
|
|
store.save_agent("One", "python backend", agent_id=one.id,
|
|
limits=Limits(extract_top=1, extract_min=0.5))
|
|
modeltask.run(extract.TASK, config, store)
|
|
|
|
assert [[r["uid"] for r in batch] for batch in model.asked] == [[posting("best").uid]]
|
|
assert [r.uid for r in store.extract_waiting(10, uids=[posting("low").uid])] == \
|
|
[posting("low").uid], "a dealt uid is read whatever its rank"
|
|
|
|
|
|
def test_a_value_outside_the_vocabulary_is_dropped() -> None:
|
|
got = extract.fields(answer("u", remote="sometimes", seniority="Senior",
|
|
country="Germany", currency="€", function="ops",
|
|
permit="maybe", salary_min=-1, salary_max=True))
|
|
assert got["remote"] == "" and got["seniority"] == "senior"
|
|
assert got["country"] == "" and got["currency"] == "" and got["function"] == ""
|
|
assert got["permit"] == "" and got["salary_min"] is None and got["salary_max"] is None
|
|
|
|
|
|
def test_disabled_finds_nothing_and_asks_nobody(
|
|
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.upsert_role(posting("1"))
|
|
model = stub(monkeypatch, lambda roles: [])
|
|
|
|
result = modeltask.run(extract.TASK, Config(database=str(tmp_path / "x.db")), store)
|
|
|
|
assert model.asked == [] and result["extracted"] == 0
|
|
|
|
|
|
def test_a_role_read_before_without_a_level_is_read_again_for_it(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
item = posting("1", seniority="senior")
|
|
store.upsert_role(item)
|
|
ranked(store, item)
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
|
|
"country = 'DE' WHERE source_id = '1'")
|
|
stub(monkeypatch, lambda roles: [answer(r["uid"], country="FR", seniority="staff")
|
|
for r in roles])
|
|
|
|
modeltask.run(extract.TASK, config, store)
|
|
|
|
role = store.role(item.uid)
|
|
assert role is not None
|
|
assert (role.seniority_model, role.seniority) == ("staff", "staff")
|
|
assert role.seniority_stated == "senior", "the host's word stays beside the model's"
|
|
assert role.country == "DE", "the other fields still only fill what is empty"
|
|
assert store.extract_pending() == 0
|
|
|
|
|
|
def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
withtext = posting("text", title="Senior Engineer", seniority="mid")
|
|
noText = posting("teaser", title="Analyst")
|
|
noText.body = ""
|
|
store.upsert_role(withtext)
|
|
store.upsert_role(noText)
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET teaser = 'Join our analytics team' "
|
|
"WHERE source_id = 'teaser'")
|
|
ranked(store, withtext, noText)
|
|
model = stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="mid") for r in roles])
|
|
|
|
modeltask.run(extract.TASK, config, store)
|
|
|
|
asked = {r["uid"]: r for batch in model.asked for r in batch}
|
|
one, two = asked[withtext.uid], asked[noText.uid]
|
|
assert (one["stated"], one["title_level"]) == ("mid", "senior") and "teaser" not in one
|
|
assert two["teaser"] == "Join our analytics team" and "text" not in two
|
|
assert "stated" not in two and "overrule" in extract.SYSTEM
|
|
schema = extract.SCHEMA["properties"]["roles"]["items"]["properties"]["seniority"]
|
|
assert "" not in schema["enum"], "the afterburner must pick a level"
|
|
|
|
|
|
def test_an_answer_with_no_level_is_charged_until_given_up(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
item = posting("1")
|
|
store.upsert_role(item)
|
|
ranked(store, item)
|
|
stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="") for r in roles])
|
|
|
|
modeltask.run(extract.TASK, config, store)
|
|
assert store.extract_pending() == 1, "answered without a level: still waiting"
|
|
modeltask.run(extract.TASK, config, store)
|
|
modeltask.run(extract.TASK, config, store)
|
|
assert store.extract_pending() == 0
|
|
|
|
|
|
def test_each_agents_best_come_first_then_the_rest(store: Store) -> None:
|
|
items = [posting(s) for s in ("a1", "a2", "a3", "b1")]
|
|
for item in items:
|
|
store.upsert_role(item)
|
|
one = store.save_agent("One", "python backend")
|
|
two = store.save_agent("Two", "python backend")
|
|
store.replace_scores(one.id, [(items[0].uid, 0.9, ""), (items[1].uid, 0.8, ""),
|
|
(items[2].uid, 0.7, "")])
|
|
store.replace_scores(two.id, [(items[3].uid, 0.6, "")])
|
|
|
|
got = [r.uid for r in store.extract_waiting(10)]
|
|
|
|
assert got == [items[0].uid, items[3].uid, items[1].uid, items[2].uid]
|
|
|
|
|
|
def test_agreement_per_host(store: Store) -> None:
|
|
for sid, board, stated, model in [
|
|
("1", "personio", "senior", "senior"), ("2", "personio", "junior", "mid"),
|
|
("3", "lever", "mid", "mid"), ("4", "lever", "", "senior"),
|
|
("5", "lever", "senior", "")]:
|
|
store.upsert_role(posting(sid, board=board, seniority=stated))
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET seniority_model = ? WHERE source_id = ?",
|
|
(model, sid))
|
|
|
|
got = {h["host"]: h for h in store.seniority_agreement()}
|
|
|
|
assert (got["personio"]["compared"], got["personio"]["agreed"], got["personio"]["rate"]) \
|
|
== (2, 1, 0.5)
|
|
assert (got["lever"]["compared"], got["lever"]["rate"], got["lever"]["stated_only"]) \
|
|
== (1, 1.0, 1)
|
|
assert [h["host"] for h in store.seniority_agreement()] == ["personio", "lever"]
|