Summarise and fetch text: a summary stage after score, two role routes
app/jobs/summarise.py is a ModelTask beside extract: about 60 words per role
from title, company, place and the fetched page else the description (capped
at 8000 characters). The pull cascade gains a summarise stage after score that
takes only roles without a summary in the top summarise.per_agent of an
enabled agent's ungated scores, and finds nothing when summarise.enabled is
false or no model is reachable. A reply that leaves a role unanswered is
counted in the state table and the role is given up after three.
POST /api/roles/{uid}/text scrapes original_url once and stores
article_text; POST /api/roles/{uid}/summarise queues a summarise_batch for
that uid alone, deduped per role.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
7d31cca394
commit
9035529e0d
12 files changed
+733
-10
No files matched your search
@@ -436,6 +436,99 @@ class Store:
|
||||
f"WHERE {where}", (wanted,))
|
||||
return int(crossed)
|
||||
|
||||
# ------------------------------------------------------------ summarise
|
||||
|
||||
#: The `state` key counting replies that left a role without a summary.
|
||||
#: `roles` has no column for it, and the count only matters for the few
|
||||
#: roles in some agent's top list.
|
||||
_SUMMARY_MISSED = "summarise.missed:"
|
||||
|
||||
def _summary_scope(self, per_agent: int,
|
||||
uids: Sequence[str] | None) -> tuple[str, list[Any]]:
|
||||
"""The `WHERE` for a role `summarise` may be sent: no summary yet and
|
||||
some text to summarise. Given `uids`, those of them and nothing else,
|
||||
open or not and given up or not, because somebody asked for them.
|
||||
Otherwise open, not of a hidden company, not given up, and among the
|
||||
`per_agent` best ungated scores of an enabled agent."""
|
||||
where = ["r.summary = ''",
|
||||
"(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"]
|
||||
if uids is not None:
|
||||
where.append("r.uid IN (SELECT value FROM json_each(?))")
|
||||
return " AND ".join(where), [json.dumps(list(uids))]
|
||||
where += [
|
||||
"r.closed_at IS NULL",
|
||||
"r.company_key NOT IN (SELECT key FROM companies WHERE affinity = -1)",
|
||||
f"""NOT EXISTS (SELECT 1 FROM state
|
||||
WHERE key = '{self._SUMMARY_MISSED}' || r.uid
|
||||
AND CAST(value AS INTEGER) >= {GIVE_UP_AFTER})""",
|
||||
"""r.uid IN (SELECT uid FROM (
|
||||
SELECT s.uid, ROW_NUMBER() OVER (
|
||||
PARTITION BY s.agent_id
|
||||
ORDER BY s.score DESC, top.first_seen DESC, s.uid) AS n
|
||||
FROM scores s
|
||||
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
|
||||
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
||||
WHERE s.gated_reason = '')
|
||||
WHERE n <= ?)""",
|
||||
]
|
||||
return " AND ".join(where), [per_agent]
|
||||
|
||||
def summarise_pending(self, per_agent: int) -> int:
|
||||
where, params = self._summary_scope(per_agent, None)
|
||||
row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params)
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
def summarise_waiting(self, limit: int, per_agent: int,
|
||||
uids: Sequence[str] | None = None) -> list[Role]:
|
||||
"""Roles `summarise` has yet to write, newest first. The top
|
||||
`per_agent` of every enabled agent's scores, so the backlog is bounded
|
||||
by the number of agents rather than by the number of open roles; given
|
||||
`uids`, those still without a summary. A batch asks again when it runs."""
|
||||
where, params = self._summary_scope(per_agent, uids)
|
||||
return [Role.from_row(r) for r in self._rows(
|
||||
f"SELECT r.* FROM roles r WHERE {where} "
|
||||
"ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
|
||||
|
||||
def set_summary(self, uid: str, summary: str) -> bool:
|
||||
"""Write a role's summary and stamp `summarised_at`. Only over an empty
|
||||
one, so a second batch that carried the same role does not replace the
|
||||
first answer. False for an unknown uid or one already summarised."""
|
||||
with self.transaction() as conn:
|
||||
written = conn.execute(
|
||||
"UPDATE roles SET summary = ?, summarised_at = ? "
|
||||
"WHERE uid = ? AND summary = ''", (summary, self.now(), uid)).rowcount
|
||||
conn.execute("DELETE FROM state WHERE key = ?", (self._SUMMARY_MISSED + uid,))
|
||||
return bool(written)
|
||||
|
||||
def summarise_missed(self, uids: Sequence[str], why: str = "") -> int:
|
||||
"""One reply that left these roles without a summary. Counted only
|
||||
against those still without one. Returns how many this pushed to
|
||||
`GIVE_UP_AFTER`. `why` is accepted for `ModelTask.missed` and not kept."""
|
||||
if not uids:
|
||||
return 0
|
||||
rows = self._rows("SELECT uid FROM roles WHERE summary = '' "
|
||||
"AND uid IN (SELECT value FROM json_each(?))",
|
||||
(json.dumps(list(uids)),))
|
||||
keys = [self._SUMMARY_MISSED + r["uid"] for r in rows]
|
||||
with self.transaction() as conn:
|
||||
conn.executemany(
|
||||
"""INSERT INTO state (key, value) VALUES (?, '1')
|
||||
ON CONFLICT(key) DO UPDATE
|
||||
SET value = CAST(CAST(value AS INTEGER) + 1 AS TEXT)""",
|
||||
[(key,) for key in keys])
|
||||
crossed = conn.execute(
|
||||
"SELECT COUNT(*) FROM state WHERE key IN (SELECT value FROM json_each(?)) "
|
||||
"AND CAST(value AS INTEGER) = ?", (json.dumps(keys), GIVE_UP_AFTER)).fetchone()[0]
|
||||
return int(crossed)
|
||||
|
||||
def set_article_text(self, uid: str, text: str) -> Role | None:
|
||||
"""Store the posting page as plain text and stamp `article_at`. The
|
||||
role as it now stands, or None for an unknown uid."""
|
||||
if not self._write("UPDATE roles SET article_text = ?, article_at = ? WHERE uid = ?",
|
||||
(text, self.now(), uid)):
|
||||
return None
|
||||
return self.role(uid)
|
||||
|
||||
# -------------------------------------------------------------- vectors
|
||||
|
||||
def roles_to_embed(self, model: str, limit: int) -> list[Role]:
|
||||
|
||||
Reference in new issue
Block a user