Extract is bounded to the ranking: pull → embed → score → extract → score → summarise

The free stages run first and their score bounds the two model stages.
Extract reads only the best extract.per_agent roles of each agent at
extract.min_score or more, then score runs again so the gates read what it
filled; summarise gets a min_score of its own. The cascade steps by
position, since score is in it twice.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-28 15:26:58 +02:00
1 parent c4f2bcc0c9
commit 7db77e83a3
15 files changed
+215 -89

No files matched your search

+56 -33
View File
@@ -619,21 +619,51 @@ class Store:
# -------------------------------------------------------------- extract
def extract_pending(self) -> int:
row = self._row(f"SELECT COUNT(*) AS n FROM roles WHERE {_EXTRACT_WAITING}")
def _top_scope(self, per_agent: int, min_score: float) -> tuple[str, list[Any]]:
"""The `WHERE` fragment for a role among the `per_agent` best ungated
scores of an enabled agent, not below `min_score`: what the two model
stages, extract and summarise, are bounded to. The role is `r`. A
company an agent hid needs no clause: `score` writes no rows for it
under that agent."""
return ("""r.uid IN (SELECT uid FROM (
SELECT s.uid, s.score, ROW_NUMBER() OVER (
PARTITION BY s.agent_id
ORDER BY s.score DESC, top.first_seen DESC, s.uid) AS n
FROM scores s
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
WHERE s.gated_reason = '')
WHERE n <= ? AND score >= ?)""", [per_agent, min_score])
def _extract_scope(self, per_agent: int, min_score: float,
uids: Sequence[str] | None) -> tuple[str, list[Any]]:
"""The `WHERE` for a role `extract` may read: open, never answered,
not given up. Given `uids`, those of them and nothing else, since a
batch asks again when it runs and must finish what it was dealt;
otherwise only the roles in `_top_scope`."""
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")]
if uids is not None:
where.append("r.uid IN (SELECT value FROM json_each(?))")
return " AND ".join(where), [json.dumps(list(uids))]
top, params = self._top_scope(per_agent, min_score)
return " AND ".join([*where, top]), params
def extract_pending(self, per_agent: int, min_score: float = 0.0) -> int:
where, params = self._extract_scope(per_agent, min_score, None)
row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params)
return int(row["n"]) if row else 0
def extract_waiting(self, limit: int,
def extract_waiting(self, limit: int, per_agent: int, min_score: float = 0.0,
uids: Sequence[str] | None = None) -> list[Role]:
"""Roles `extract` has yet to read, newest first. Given `uids`, only
those of them still waiting: a batch asks again when it runs, since a
role may have been answered since the task shared the work out."""
sql, params = f"SELECT * FROM roles WHERE {_EXTRACT_WAITING}", []
if uids is not None:
sql += " AND uid IN (SELECT value FROM json_each(?))"
params.append(json.dumps(list(uids)))
sql += " ORDER BY first_seen DESC, uid LIMIT ?"
return [Role.from_row(r) for r in self._rows(sql, [*params, limit])]
"""Roles `extract` has yet to read, newest first: the best
`per_agent` of every enabled agent's scores not below `min_score`, so
the backlog follows the rankings rather than the boards. Given
`uids`, only those of them still waiting, whatever their rank."""
where, params = self._extract_scope(per_agent, min_score, uids)
return [Role.from_row(r) for r in self._rows(
f"SELECT r.* FROM roles r WHERE {where} "
"ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
def set_extracted(self, uid: str, fields: dict[str, Any]) -> int | None:
"""Write what `extract` read for one role and stamp `extracted_at`.
@@ -692,48 +722,41 @@ class Store:
#: roles in some agent's top list.
_SUMMARY_MISSED = "summarise.missed:"
def _summary_scope(self, per_agent: int,
def _summary_scope(self, per_agent: int, min_score: float,
uids: Sequence[str] | None) -> tuple[str, list[Any]]:
"""The `WHERE` for a role `summarise` may be sent: no summary yet and
some text to summarise. Given `uids`, those of them and nothing else,
open or not and given up or not, because somebody asked for them.
Otherwise open, not given up, and among the `per_agent` best ungated
scores of an enabled agent. A company an agent hid needs no clause:
`score` writes no rows for it under that agent."""
Otherwise open, not given up, and in `_top_scope`."""
where = ["r.summary = ''",
"(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"]
if uids is not None:
where.append("r.uid IN (SELECT value FROM json_each(?))")
return " AND ".join(where), [json.dumps(list(uids))]
top, params = self._top_scope(per_agent, min_score)
where += [
"r.closed_at IS NULL",
f"""NOT EXISTS (SELECT 1 FROM state
WHERE key = '{self._SUMMARY_MISSED}' || r.uid
AND CAST(value AS INTEGER) >= {GIVE_UP_AFTER})""",
"""r.uid IN (SELECT uid FROM (
SELECT s.uid, ROW_NUMBER() OVER (
PARTITION BY s.agent_id
ORDER BY s.score DESC, top.first_seen DESC, s.uid) AS n
FROM scores s
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
WHERE s.gated_reason = '')
WHERE n <= ?)""",
top,
]
return " AND ".join(where), [per_agent]
return " AND ".join(where), params
def summarise_pending(self, per_agent: int) -> int:
where, params = self._summary_scope(per_agent, None)
def summarise_pending(self, per_agent: int, min_score: float = 0.0) -> int:
where, params = self._summary_scope(per_agent, min_score, None)
row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params)
return int(row["n"]) if row else 0
def summarise_waiting(self, limit: int, per_agent: int,
uids: Sequence[str] | None = None) -> list[Role]:
uids: Sequence[str] | None = None,
min_score: float = 0.0) -> list[Role]:
"""Roles `summarise` has yet to write, newest first. The top
`per_agent` of every enabled agent's scores, so the backlog is bounded
by the number of agents rather than by the number of open roles; given
`uids`, those still without a summary. A batch asks again when it runs."""
where, params = self._summary_scope(per_agent, uids)
`per_agent` of every enabled agent's scores not below `min_score`, so
the backlog is bounded by the number of agents rather than by the
number of open roles; given `uids`, those still without a summary,
whatever their rank. A batch asks again when it runs."""
where, params = self._summary_scope(per_agent, min_score, uids)
return [Role.from_row(r) for r in self._rows(
f"SELECT r.* FROM roles r WHERE {where} "
"ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]