Summarise and fetch text: a summary stage after score, two role routes

app/jobs/summarise.py is a ModelTask beside extract: about 60 words per role
from title, company, place and the fetched page else the description (capped
at 8000 characters). The pull cascade gains a summarise stage after score that
takes only roles without a summary in the top summarise.per_agent of an
enabled agent's ungated scores, and finds nothing when summarise.enabled is
false or no model is reachable. A reply that leaves a role unanswered is
counted in the state table and the role is given up after three.

POST /api/roles/{uid}/text scrapes original_url once and stores
article_text; POST /api/roles/{uid}/summarise queues a summarise_batch for
that uid alone, deduped per role.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-25 21:39:43 +02:00
1 parent 7d31cca394
commit 9035529e0d
12 files changed
+733 -10

No files matched your search

+93
View File
@@ -436,6 +436,99 @@ class Store:
f"WHERE {where}", (wanted,))
return int(crossed)
# ------------------------------------------------------------ summarise
#: The `state` key counting replies that left a role without a summary.
#: `roles` has no column for it, and the count only matters for the few
#: roles in some agent's top list.
_SUMMARY_MISSED = "summarise.missed:"
def _summary_scope(self, per_agent: int,
uids: Sequence[str] | None) -> tuple[str, list[Any]]:
"""The `WHERE` for a role `summarise` may be sent: no summary yet and
some text to summarise. Given `uids`, those of them and nothing else,
open or not and given up or not, because somebody asked for them.
Otherwise open, not of a hidden company, not given up, and among the
`per_agent` best ungated scores of an enabled agent."""
where = ["r.summary = ''",
"(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"]
if uids is not None:
where.append("r.uid IN (SELECT value FROM json_each(?))")
return " AND ".join(where), [json.dumps(list(uids))]
where += [
"r.closed_at IS NULL",
"r.company_key NOT IN (SELECT key FROM companies WHERE affinity = -1)",
f"""NOT EXISTS (SELECT 1 FROM state
WHERE key = '{self._SUMMARY_MISSED}' || r.uid
AND CAST(value AS INTEGER) >= {GIVE_UP_AFTER})""",
"""r.uid IN (SELECT uid FROM (
SELECT s.uid, ROW_NUMBER() OVER (
PARTITION BY s.agent_id
ORDER BY s.score DESC, top.first_seen DESC, s.uid) AS n
FROM scores s
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
WHERE s.gated_reason = '')
WHERE n <= ?)""",
]
return " AND ".join(where), [per_agent]
def summarise_pending(self, per_agent: int) -> int:
where, params = self._summary_scope(per_agent, None)
row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params)
return int(row["n"]) if row else 0
def summarise_waiting(self, limit: int, per_agent: int,
uids: Sequence[str] | None = None) -> list[Role]:
"""Roles `summarise` has yet to write, newest first. The top
`per_agent` of every enabled agent's scores, so the backlog is bounded
by the number of agents rather than by the number of open roles; given
`uids`, those still without a summary. A batch asks again when it runs."""
where, params = self._summary_scope(per_agent, uids)
return [Role.from_row(r) for r in self._rows(
f"SELECT r.* FROM roles r WHERE {where} "
"ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
def set_summary(self, uid: str, summary: str) -> bool:
"""Write a role's summary and stamp `summarised_at`. Only over an empty
one, so a second batch that carried the same role does not replace the
first answer. False for an unknown uid or one already summarised."""
with self.transaction() as conn:
written = conn.execute(
"UPDATE roles SET summary = ?, summarised_at = ? "
"WHERE uid = ? AND summary = ''", (summary, self.now(), uid)).rowcount
conn.execute("DELETE FROM state WHERE key = ?", (self._SUMMARY_MISSED + uid,))
return bool(written)
def summarise_missed(self, uids: Sequence[str], why: str = "") -> int:
"""One reply that left these roles without a summary. Counted only
against those still without one. Returns how many this pushed to
`GIVE_UP_AFTER`. `why` is accepted for `ModelTask.missed` and not kept."""
if not uids:
return 0
rows = self._rows("SELECT uid FROM roles WHERE summary = '' "
"AND uid IN (SELECT value FROM json_each(?))",
(json.dumps(list(uids)),))
keys = [self._SUMMARY_MISSED + r["uid"] for r in rows]
with self.transaction() as conn:
conn.executemany(
"""INSERT INTO state (key, value) VALUES (?, '1')
ON CONFLICT(key) DO UPDATE
SET value = CAST(CAST(value AS INTEGER) + 1 AS TEXT)""",
[(key,) for key in keys])
crossed = conn.execute(
"SELECT COUNT(*) FROM state WHERE key IN (SELECT value FROM json_each(?)) "
"AND CAST(value AS INTEGER) = ?", (json.dumps(keys), GIVE_UP_AFTER)).fetchone()[0]
return int(crossed)
def set_article_text(self, uid: str, text: str) -> Role | None:
"""Store the posting page as plain text and stamp `article_at`. The
role as it now stands, or None for an unknown uid."""
if not self._write("UPDATE roles SET article_text = ?, article_at = ? WHERE uid = ?",
(text, self.now(), uid)):
return None
return self.role(uid)
# -------------------------------------------------------------- vectors
def roles_to_embed(self, model: str, limit: int) -> list[Role]: