Summarise and fetch text: a summary stage after score, two role routes
app/jobs/summarise.py is a ModelTask beside extract: about 60 words per role
from title, company, place and the fetched page else the description (capped
at 8000 characters). The pull cascade gains a summarise stage after score that
takes only roles without a summary in the top summarise.per_agent of an
enabled agent's ungated scores, and finds nothing when summarise.enabled is
false or no model is reachable. A reply that leaves a role unanswered is
counted in the state table and the role is given up after three.
POST /api/roles/{uid}/text scrapes original_url once and stores
article_text; POST /api/roles/{uid}/summarise queues a summarise_batch for
that uid alone, deduped per role.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
7d31cca394
commit
9035529e0d
12 files changed
+733
-10
No files matched your search
+26
-5
@@ -1,6 +1,6 @@
|
||||
"""The tasks roledesk offers, and the order a pull runs them in.
|
||||
|
||||
pull → extract → embed → score
|
||||
pull → extract → embed → score → summarise
|
||||
|
||||
newsdesk's cascade convention: a cascade is a tuple of task names, and the
|
||||
final job of one task starts the next (`step` is the callback it carries).
|
||||
@@ -11,6 +11,8 @@ nothing: `extract.enabled: false`, or an `on_ingest` this pull does not
|
||||
satisfy, is an extract stage with no batches whose final job moves straight
|
||||
on. Extract runs before score so the gates read what it filled; before embed
|
||||
too, since embedding does not read its columns and the order costs nothing.
|
||||
Summarise runs last because it reads the scores: it takes only the best
|
||||
`summarise.per_agent` roles of each enabled agent.
|
||||
|
||||
A batch that failed, or a stage somebody killed, does not stop the chain: the
|
||||
work comes round on the next pull, and a rescore of what did arrive is worth
|
||||
@@ -28,14 +30,14 @@ from dataclasses import replace
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
from app.jobs import embed, extract, pull, score # noqa: F401 -- registers the jobs
|
||||
from app.jobs import embed, extract, pull, score, summarise # noqa: F401 -- registers the jobs
|
||||
from desk.tools import agent
|
||||
from pipeline import NULL, TASKS, RunContext, Tally, Task, declare, job
|
||||
|
||||
STEP = "app.jobs.cascade:step"
|
||||
|
||||
CASCADES: dict[str, tuple[str, ...]] = {
|
||||
"pull": ("pull", "extract", "embed", "score"),
|
||||
"pull": ("pull", "extract", "embed", "score", "summarise"),
|
||||
}
|
||||
|
||||
|
||||
@@ -62,12 +64,31 @@ def _extract(cascade: str) -> Task:
|
||||
return replace(stage, finder=during_a_pull)
|
||||
|
||||
|
||||
def _summarise(cascade: str) -> Task:
|
||||
"""`summarise` as a stage: nothing with no model reachable, so a machine
|
||||
without one pulls in silence rather than failing a batch per pull. Off
|
||||
(`summarise.enabled: false`) is nothing in either case. Standing on its
|
||||
own, as the button, a missing model fails the batch with a notice."""
|
||||
stage = summarise.summarise_task()
|
||||
if not cascade:
|
||||
return stage
|
||||
find = stage.finder
|
||||
|
||||
def during_a_pull(config: Any, store: Any) -> list[Any]:
|
||||
if not agent.backend(config.agent):
|
||||
return []
|
||||
return find(config, store)
|
||||
|
||||
return replace(stage, finder=during_a_pull)
|
||||
|
||||
|
||||
STAGES: dict[str, Callable[[str], Task]] = {
|
||||
"pull": lambda cascade: pull.pull_task(),
|
||||
"extract": _extract,
|
||||
"embed": lambda cascade: Task(kind="embed", finder=_once, label="Embedding roles"),
|
||||
"score": lambda cascade: Task(kind="score", finder=_once,
|
||||
label="Scoring roles for every agent"),
|
||||
"summarise": _summarise,
|
||||
}
|
||||
|
||||
|
||||
@@ -123,7 +144,7 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
|
||||
#: What `app job <name>` and the Pulls page offer besides `pull`: one stage
|
||||
#: each, on its own. `pull` is not a task here, since it is a job: a name that
|
||||
#: is both would be two buttons on the Pulls page.
|
||||
for _name in ("extract", "embed", "score"):
|
||||
for _name in ("extract", "embed", "score", "summarise"):
|
||||
TASKS[_name] = partial(task, _name)
|
||||
|
||||
|
||||
@@ -133,4 +154,4 @@ for _name in ("extract", "embed", "score"):
|
||||
# setting: a cadence edited or switched off on the Pulls page survives
|
||||
# every restart (see pipeline/schedule.py).
|
||||
declare("pull", "pull", "every 6h",
|
||||
label="Pull every board, then extract, embed and score")
|
||||
label="Pull every board, then extract, embed, score and summarise")
|
||||
Reference in new issue
Block a user