Summarise and fetch text: a summary stage after score, two role routes

app/jobs/summarise.py is a ModelTask beside extract: about 60 words per role
from title, company, place and the fetched page else the description (capped
at 8000 characters). The pull cascade gains a summarise stage after score that
takes only roles without a summary in the top summarise.per_agent of an
enabled agent's ungated scores, and finds nothing when summarise.enabled is
false or no model is reachable. A reply that leaves a role unanswered is
counted in the state table and the role is given up after three.

POST /api/roles/{uid}/text scrapes original_url once and stores
article_text; POST /api/roles/{uid}/summarise queues a summarise_batch for
that uid alone, deduped per role.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-25 21:39:43 +02:00
1 parent 7d31cca394
commit 9035529e0d
12 files changed
+733 -10

No files matched your search

+82
View File
@@ -496,3 +496,85 @@ def test_prune_previews_then_switches_off(client: TestClient, store: Store) -> N
done = client.post("/api/prune", json={"region": "DK,EU"}).json()
assert (done["pruned"], done["roles_closed"], done["boards_left"]) == (1, 1, 1)
assert client.get("/api/prune", params={"region": " "}).status_code == 422
# ------------------------------------------------- roles: text and summary
def test_the_posting_text_is_fetched_once(
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
from desk.tools import scrape
from desk.tools.article import ArticleContent
store.upsert_role(posting("1"))
uid = posting("1").uid
asked: list[str] = []
def gone(url: str) -> scrape.Scraped:
asked.append(url)
return scrape.Scraped(None, "not_found", "http", ())
monkeypatch.setattr(scrape, "scrape", gone)
assert client.post("/api/roles/nope/text").status_code == 404
res = client.post(f"/api/roles/{uid}/text")
assert res.status_code == 422 and "gone" in res.json()["detail"]
assert asked == ["https://boards.example/acme/1"]
def found(url: str) -> scrape.Scraped:
content = ArticleContent(format="plain", content=" The whole posting.\n",
source_url=url, method="test")
return scrape.Scraped(content, "", "http", ())
monkeypatch.setattr(scrape, "scrape", found)
res = client.post(f"/api/roles/{uid}/text")
assert res.status_code == 200
body = res.json()
assert body["uid"] == uid and body["article_text"] == "The whole posting."
assert body["article_at"]
assert client.post(f"/api/roles/{uid}/text").status_code == 409
def test_summarise_queues_one_role_and_the_job_writes_it(
app: FastAPI, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""No lifespan, so no worker: the job waits in the table until drained."""
from typing import Any
from desk.tools import agent
client = TestClient(app)
store.upsert_role(posting("1"))
store.upsert_role(posting("2"))
uid, bare = posting("1").uid, posting("2").uid
with store.transaction() as conn:
conn.execute("UPDATE roles SET description = '' WHERE uid = ?", (bare,))
monkeypatch.setattr(agent, "backend", lambda settings: "")
assert client.post("/api/roles/nope/summarise").status_code == 404
res = client.post(f"/api/roles/{uid}/summarise")
assert res.status_code == 422 and "model" in res.json()["detail"]
asked: list[Any] = []
def ask_json(settings: Any, **kwargs: Any) -> agent.Reply:
items = kwargs["question"]["summaries"]
asked.append([i["uid"] for i in items])
return agent.Reply(data={"summaries": [
{"uid": i["uid"], "summary": "Backend role in Berlin, Python."}
for i in items]}, backend="api")
monkeypatch.setattr(agent, "backend", lambda settings: "api")
monkeypatch.setattr(agent, "ask_json", ask_json)
res = client.post(f"/api/roles/{bare}/summarise")
assert res.status_code == 422 and "no text" in res.json()["detail"]
res = client.post(f"/api/roles/{uid}/summarise")
assert res.status_code == 202
job = res.json()
assert job["kind"] == "summarise_batch" and job["payload"] == {"uids": [uid]}
assert job["state"] == "queued"
assert client.post(f"/api/roles/{uid}/summarise").json()["id"] == job["id"]
app.state.queue.drain(timeout=30)
assert asked == [[uid]]
assert store.role(uid).summary == "Backend role in Berlin, Python." # type: ignore[union-attr]
assert client.post(f"/api/roles/{uid}/summarise").status_code == 409