Summarise and fetch text: a summary stage after score, two role routes
app/jobs/summarise.py is a ModelTask beside extract: about 60 words per role
from title, company, place and the fetched page else the description (capped
at 8000 characters). The pull cascade gains a summarise stage after score that
takes only roles without a summary in the top summarise.per_agent of an
enabled agent's ungated scores, and finds nothing when summarise.enabled is
false or no model is reachable. A reply that leaves a role unanswered is
counted in the state table and the role is given up after three.
POST /api/roles/{uid}/text scrapes original_url once and stores
article_text; POST /api/roles/{uid}/summarise queues a summarise_batch for
that uid alone, deduped per role.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
7d31cca394
commit
9035529e0d
12 files changed
+733
-10
No files matched your search
@@ -496,3 +496,85 @@ def test_prune_previews_then_switches_off(client: TestClient, store: Store) -> N
|
||||
done = client.post("/api/prune", json={"region": "DK,EU"}).json()
|
||||
assert (done["pruned"], done["roles_closed"], done["boards_left"]) == (1, 1, 1)
|
||||
assert client.get("/api/prune", params={"region": " "}).status_code == 422
|
||||
|
||||
|
||||
# ------------------------------------------------- roles: text and summary
|
||||
|
||||
def test_the_posting_text_is_fetched_once(
|
||||
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from desk.tools import scrape
|
||||
from desk.tools.article import ArticleContent
|
||||
|
||||
store.upsert_role(posting("1"))
|
||||
uid = posting("1").uid
|
||||
asked: list[str] = []
|
||||
|
||||
def gone(url: str) -> scrape.Scraped:
|
||||
asked.append(url)
|
||||
return scrape.Scraped(None, "not_found", "http", ())
|
||||
|
||||
monkeypatch.setattr(scrape, "scrape", gone)
|
||||
assert client.post("/api/roles/nope/text").status_code == 404
|
||||
res = client.post(f"/api/roles/{uid}/text")
|
||||
assert res.status_code == 422 and "gone" in res.json()["detail"]
|
||||
assert asked == ["https://boards.example/acme/1"]
|
||||
|
||||
def found(url: str) -> scrape.Scraped:
|
||||
content = ArticleContent(format="plain", content=" The whole posting.\n",
|
||||
source_url=url, method="test")
|
||||
return scrape.Scraped(content, "", "http", ())
|
||||
|
||||
monkeypatch.setattr(scrape, "scrape", found)
|
||||
res = client.post(f"/api/roles/{uid}/text")
|
||||
assert res.status_code == 200
|
||||
body = res.json()
|
||||
assert body["uid"] == uid and body["article_text"] == "The whole posting."
|
||||
assert body["article_at"]
|
||||
assert client.post(f"/api/roles/{uid}/text").status_code == 409
|
||||
|
||||
|
||||
def test_summarise_queues_one_role_and_the_job_writes_it(
|
||||
app: FastAPI, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""No lifespan, so no worker: the job waits in the table until drained."""
|
||||
from typing import Any
|
||||
|
||||
from desk.tools import agent
|
||||
|
||||
client = TestClient(app)
|
||||
store.upsert_role(posting("1"))
|
||||
store.upsert_role(posting("2"))
|
||||
uid, bare = posting("1").uid, posting("2").uid
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET description = '' WHERE uid = ?", (bare,))
|
||||
|
||||
monkeypatch.setattr(agent, "backend", lambda settings: "")
|
||||
assert client.post("/api/roles/nope/summarise").status_code == 404
|
||||
res = client.post(f"/api/roles/{uid}/summarise")
|
||||
assert res.status_code == 422 and "model" in res.json()["detail"]
|
||||
|
||||
asked: list[Any] = []
|
||||
|
||||
def ask_json(settings: Any, **kwargs: Any) -> agent.Reply:
|
||||
items = kwargs["question"]["summaries"]
|
||||
asked.append([i["uid"] for i in items])
|
||||
return agent.Reply(data={"summaries": [
|
||||
{"uid": i["uid"], "summary": "Backend role in Berlin, Python."}
|
||||
for i in items]}, backend="api")
|
||||
|
||||
monkeypatch.setattr(agent, "backend", lambda settings: "api")
|
||||
monkeypatch.setattr(agent, "ask_json", ask_json)
|
||||
res = client.post(f"/api/roles/{bare}/summarise")
|
||||
assert res.status_code == 422 and "no text" in res.json()["detail"]
|
||||
|
||||
res = client.post(f"/api/roles/{uid}/summarise")
|
||||
assert res.status_code == 202
|
||||
job = res.json()
|
||||
assert job["kind"] == "summarise_batch" and job["payload"] == {"uids": [uid]}
|
||||
assert job["state"] == "queued"
|
||||
assert client.post(f"/api/roles/{uid}/summarise").json()["id"] == job["id"]
|
||||
|
||||
app.state.queue.drain(timeout=30)
|
||||
|
||||
assert asked == [[uid]]
|
||||
assert store.role(uid).summary == "Backend role in Berlin, Python." # type: ignore[union-attr]
|
||||
assert client.post(f"/api/roles/{uid}/summarise").status_code == 409
|
||||
Reference in new issue
Block a user