Files
jobboard/tests/test_summarise.py
T
WolfandClaude Fable 5.1 7db77e83a3 Extract is bounded to the ranking: pull → embed → score → extract → score → summarise
The free stages run first and their score bounds the two model stages.
Extract reads only the best extract.per_agent roles of each agent at
extract.min_score or more, then score runs again so the gates read what it
filled; summarise gets a min_score of its own. The cascade steps by
position, since score is in it twice.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-28 15:26:58 +02:00

215 lines
8.0 KiB
Python

"""The `summarise` task, with the model call stubbed: which roles are offered,
what one item carries, what an answer writes, and the cascade stage. Nothing
here reaches a model."""
from __future__ import annotations
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import pytest
from app.config import Config
from app.jobs import cascade, summarise
from app.storage import Store
from desk import modeltask
from desk.models import Item
from desk.tools import agent
from pipeline import TASKS
def posting(source_id: str, body: str = "Python and Go on Kubernetes. Pays 700k DKK.",
**raw: object) -> Item:
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
title=f"Backend Engineer {source_id}", body=body,
published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"company": "Acme", "place": "Copenhagen", **raw})
item.source = "ats"
return item
class Model:
"""Stands in for `agent.ask_json`: records what it was asked and hands
back whatever `reply` builds from the batch."""
def __init__(self, reply: Any) -> None:
self.reply = reply
self.asked: list[list[dict[str, Any]]] = []
def __call__(self, settings: Any, **kwargs: Any) -> agent.Reply:
items = kwargs["question"]["summaries"]
self.asked.append(items)
return agent.Reply(data={"summaries": self.reply(items)}, backend="api")
def written(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [{"uid": i["uid"], "summary": f"A backend role at Acme, {i['uid']}.",
"tags": ["Backend Engineer", "python", "fintech"]}
for i in items]
def stub(monkeypatch: pytest.MonkeyPatch, reply: Any = written) -> Model:
model = Model(reply)
monkeypatch.setattr(agent, "backend", lambda settings: "api")
monkeypatch.setattr(agent, "ask_json", model)
return model
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
@pytest.fixture
def config(tmp_path: Path) -> Config:
return Config(database=str(tmp_path / "roledesk.db"),
summarise={"enabled": True, "per_agent": 2})
def ranked(store: Store, *ids: str, agent_name: str = "Backend",
enabled: bool = True) -> list[str]:
"""Store a role per id and score them for one agent, best first."""
uids = []
for sid in ids:
store.upsert_role(posting(sid))
uids.append(posting(sid).uid)
one = store.save_agent(agent_name, "python backend", enabled=enabled)
store.replace_scores(one.id, [(uid, 1.0 - n / 10, "") for n, uid in enumerate(uids)])
return uids
def test_a_batch_writes_the_summary_and_stamps_summarised_at(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
[uid] = ranked(store, "1")
model = stub(monkeypatch)
result = modeltask.run(summarise.TASK, config, store)
role = store.role(uid)
assert role is not None
assert role.summary == f"A backend role at Acme, {uid}."
assert role.summarised_at is not None
assert role.tags == ["backend engineer", "fintech", "python"]
assert result["summarised"] == 1 and result["pending"] == 0
[[item]] = model.asked
assert item == {"uid": uid, "title": "Backend Engineer 1", "company": "Acme",
"place": "Copenhagen", "text": "Python and Go on Kubernetes. Pays 700k DKK."}
def test_the_replys_tags_are_normalised_and_stored(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
[uid] = ranked(store, "1")
stub(monkeypatch, lambda items: [
{"uid": items[0]["uid"], "summary": "A backend role, long enough to store.",
"tags": [" Backend Engineer ", "PYTHON", "python", "", " "]}])
modeltask.run(summarise.TASK, config, store)
role = store.role(uid)
assert role is not None
# Collapsed whitespace, lowercased, deduplicated, empties dropped.
assert role.tags == ["backend engineer", "python"]
def test_the_fetched_page_is_read_before_the_description_and_capped(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
[uid] = ranked(store, "1")
store.set_article_text(uid, "word " * 3000)
model = stub(monkeypatch)
modeltask.run(summarise.TASK, config, store)
text = model.asked[0][0]["text"]
assert text.startswith("word word") and len(text) == summarise.EXCERPT
def test_only_the_top_of_an_enabled_agent_without_a_summary_is_offered(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
first, second, third = ranked(store, "1", "2", "3")
ranked(store, "4", agent_name="Off", enabled=False)
store.set_summary(first, "Already written, long enough.")
gated = posting("5")
store.upsert_role(gated)
backend = store.agents()[0]
store.replace_scores(backend.id, [(first, 0.9, ""), (second, 0.8, ""),
(third, 0.7, ""), (gated.uid, 0.95, "remote")])
model = stub(monkeypatch)
modeltask.run(summarise.TASK, config, store)
# The top two ungated are `first` and `second`; `first` has one already.
assert [[i["uid"] for i in batch] for batch in model.asked] == [[second]]
def test_a_role_with_no_text_is_skipped_not_sent(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
store.upsert_role(posting("1", body=" "))
empty = posting("1").uid
one = store.save_agent("Backend", "python")
store.replace_scores(one.id, [(empty, 0.9, "")])
model = stub(monkeypatch)
modeltask.run(summarise.TASK, config, store)
summarise.run_batch(config, store, uids=[empty])
assert model.asked == []
assert store.summarise_pending(config.summarise.per_agent) == 0
def test_a_named_uid_is_summarised_whatever_its_rank_or_the_switch(
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""The read page's button: a batch of one uid, with `enabled` off and the
role in nobody's top list."""
store.upsert_role(posting("1"))
uid = posting("1").uid
stub(monkeypatch)
off = Config(database=str(tmp_path / "roledesk.db"))
result = summarise.run_batch(off, store, uids=[uid])
assert result["summarised"] == 1
assert store.role(uid).summary # type: ignore[union-attr]
def test_an_unknown_or_empty_answer_is_rejected_and_the_role_given_up_after_three(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
[uid] = ranked(store, "1")
stub(monkeypatch, lambda items: [{"uid": "not-in-the-batch", "summary": "A role."},
{"uid": items[0]["uid"], "summary": "N/A"}])
result = modeltask.run(summarise.TASK, config, store)
assert result["rejected"] == ["not-in-the-batch", uid]
assert store.role(uid).summary == "" # type: ignore[union-attr]
assert store.summarise_pending(2) == 1
modeltask.run(summarise.TASK, config, store)
modeltask.run(summarise.TASK, config, store)
assert store.summarise_pending(2) == 0, "given up after three unanswered replies"
def test_disabled_finds_nothing_and_asks_nobody(
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
ranked(store, "1")
model = stub(monkeypatch)
result = modeltask.run(summarise.TASK, Config(database=str(tmp_path / "x.db")), store)
assert model.asked == [] and result["summarised"] == 0
def test_summarise_follows_score_and_finds_nothing_without_a_model(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
ranked(store, "1", "2")
assert cascade.follows("pull", cascade.CASCADES["pull"].index("summarise") - 1) == "summarise"
assert "summarise" in TASKS
stage = cascade.task("summarise", cascade="pull")
monkeypatch.setattr(agent, "backend", lambda settings: "")
assert stage.finder(config, store) == []
monkeypatch.setattr(agent, "backend", lambda settings: "api")
[batch] = stage.finder(config, store)
assert len(batch.payload["uids"]) == 2
assert stage.kind == "summarise_batch" and stage.callback == cascade.STEP