Extract is bounded to the ranking: pull → embed → score → extract → score → summarise
The free stages run first and their score bounds the two model stages. Extract reads only the best extract.per_agent roles of each agent at extract.min_score or more, then score runs again so the gates read what it filled; summarise gets a min_score of its own. The cascade steps by position, since score is in it twice. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
c4f2bcc0c9
commit
7db77e83a3
15 files changed
+215
-89
No files matched your search
+56
-33
@@ -619,21 +619,51 @@ class Store:
|
||||
|
||||
# -------------------------------------------------------------- extract
|
||||
|
||||
def extract_pending(self) -> int:
|
||||
row = self._row(f"SELECT COUNT(*) AS n FROM roles WHERE {_EXTRACT_WAITING}")
|
||||
def _top_scope(self, per_agent: int, min_score: float) -> tuple[str, list[Any]]:
|
||||
"""The `WHERE` fragment for a role among the `per_agent` best ungated
|
||||
scores of an enabled agent, not below `min_score`: what the two model
|
||||
stages, extract and summarise, are bounded to. The role is `r`. A
|
||||
company an agent hid needs no clause: `score` writes no rows for it
|
||||
under that agent."""
|
||||
return ("""r.uid IN (SELECT uid FROM (
|
||||
SELECT s.uid, s.score, ROW_NUMBER() OVER (
|
||||
PARTITION BY s.agent_id
|
||||
ORDER BY s.score DESC, top.first_seen DESC, s.uid) AS n
|
||||
FROM scores s
|
||||
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
|
||||
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
||||
WHERE s.gated_reason = '')
|
||||
WHERE n <= ? AND score >= ?)""", [per_agent, min_score])
|
||||
|
||||
def _extract_scope(self, per_agent: int, min_score: float,
|
||||
uids: Sequence[str] | None) -> tuple[str, list[Any]]:
|
||||
"""The `WHERE` for a role `extract` may read: open, never answered,
|
||||
not given up. Given `uids`, those of them and nothing else, since a
|
||||
batch asks again when it runs and must finish what it was dealt;
|
||||
otherwise only the roles in `_top_scope`."""
|
||||
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
|
||||
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")]
|
||||
if uids is not None:
|
||||
where.append("r.uid IN (SELECT value FROM json_each(?))")
|
||||
return " AND ".join(where), [json.dumps(list(uids))]
|
||||
top, params = self._top_scope(per_agent, min_score)
|
||||
return " AND ".join([*where, top]), params
|
||||
|
||||
def extract_pending(self, per_agent: int, min_score: float = 0.0) -> int:
|
||||
where, params = self._extract_scope(per_agent, min_score, None)
|
||||
row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params)
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
def extract_waiting(self, limit: int,
|
||||
def extract_waiting(self, limit: int, per_agent: int, min_score: float = 0.0,
|
||||
uids: Sequence[str] | None = None) -> list[Role]:
|
||||
"""Roles `extract` has yet to read, newest first. Given `uids`, only
|
||||
those of them still waiting: a batch asks again when it runs, since a
|
||||
role may have been answered since the task shared the work out."""
|
||||
sql, params = f"SELECT * FROM roles WHERE {_EXTRACT_WAITING}", []
|
||||
if uids is not None:
|
||||
sql += " AND uid IN (SELECT value FROM json_each(?))"
|
||||
params.append(json.dumps(list(uids)))
|
||||
sql += " ORDER BY first_seen DESC, uid LIMIT ?"
|
||||
return [Role.from_row(r) for r in self._rows(sql, [*params, limit])]
|
||||
"""Roles `extract` has yet to read, newest first: the best
|
||||
`per_agent` of every enabled agent's scores not below `min_score`, so
|
||||
the backlog follows the rankings rather than the boards. Given
|
||||
`uids`, only those of them still waiting, whatever their rank."""
|
||||
where, params = self._extract_scope(per_agent, min_score, uids)
|
||||
return [Role.from_row(r) for r in self._rows(
|
||||
f"SELECT r.* FROM roles r WHERE {where} "
|
||||
"ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
|
||||
|
||||
def set_extracted(self, uid: str, fields: dict[str, Any]) -> int | None:
|
||||
"""Write what `extract` read for one role and stamp `extracted_at`.
|
||||
@@ -692,48 +722,41 @@ class Store:
|
||||
#: roles in some agent's top list.
|
||||
_SUMMARY_MISSED = "summarise.missed:"
|
||||
|
||||
def _summary_scope(self, per_agent: int,
|
||||
def _summary_scope(self, per_agent: int, min_score: float,
|
||||
uids: Sequence[str] | None) -> tuple[str, list[Any]]:
|
||||
"""The `WHERE` for a role `summarise` may be sent: no summary yet and
|
||||
some text to summarise. Given `uids`, those of them and nothing else,
|
||||
open or not and given up or not, because somebody asked for them.
|
||||
Otherwise open, not given up, and among the `per_agent` best ungated
|
||||
scores of an enabled agent. A company an agent hid needs no clause:
|
||||
`score` writes no rows for it under that agent."""
|
||||
Otherwise open, not given up, and in `_top_scope`."""
|
||||
where = ["r.summary = ''",
|
||||
"(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"]
|
||||
if uids is not None:
|
||||
where.append("r.uid IN (SELECT value FROM json_each(?))")
|
||||
return " AND ".join(where), [json.dumps(list(uids))]
|
||||
top, params = self._top_scope(per_agent, min_score)
|
||||
where += [
|
||||
"r.closed_at IS NULL",
|
||||
f"""NOT EXISTS (SELECT 1 FROM state
|
||||
WHERE key = '{self._SUMMARY_MISSED}' || r.uid
|
||||
AND CAST(value AS INTEGER) >= {GIVE_UP_AFTER})""",
|
||||
"""r.uid IN (SELECT uid FROM (
|
||||
SELECT s.uid, ROW_NUMBER() OVER (
|
||||
PARTITION BY s.agent_id
|
||||
ORDER BY s.score DESC, top.first_seen DESC, s.uid) AS n
|
||||
FROM scores s
|
||||
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
|
||||
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
||||
WHERE s.gated_reason = '')
|
||||
WHERE n <= ?)""",
|
||||
top,
|
||||
]
|
||||
return " AND ".join(where), [per_agent]
|
||||
return " AND ".join(where), params
|
||||
|
||||
def summarise_pending(self, per_agent: int) -> int:
|
||||
where, params = self._summary_scope(per_agent, None)
|
||||
def summarise_pending(self, per_agent: int, min_score: float = 0.0) -> int:
|
||||
where, params = self._summary_scope(per_agent, min_score, None)
|
||||
row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params)
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
def summarise_waiting(self, limit: int, per_agent: int,
|
||||
uids: Sequence[str] | None = None) -> list[Role]:
|
||||
uids: Sequence[str] | None = None,
|
||||
min_score: float = 0.0) -> list[Role]:
|
||||
"""Roles `summarise` has yet to write, newest first. The top
|
||||
`per_agent` of every enabled agent's scores, so the backlog is bounded
|
||||
by the number of agents rather than by the number of open roles; given
|
||||
`uids`, those still without a summary. A batch asks again when it runs."""
|
||||
where, params = self._summary_scope(per_agent, uids)
|
||||
`per_agent` of every enabled agent's scores not below `min_score`, so
|
||||
the backlog is bounded by the number of agents rather than by the
|
||||
number of open roles; given `uids`, those still without a summary,
|
||||
whatever their rank. A batch asks again when it runs."""
|
||||
where, params = self._summary_scope(per_agent, min_score, uids)
|
||||
return [Role.from_row(r) for r in self._rows(
|
||||
f"SELECT r.* FROM roles r WHERE {where} "
|
||||
"ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
|
||||
|
||||
Reference in new issue
Block a user