Switch the extract and summarise stages on the page, over config.yaml

A stored `stage:extract` / `stage:summarise` in the state table overrides
extract.enabled / summarise.enabled; with none, the file applies. One
function, stages.stage_on, decides, and both tasks' finders ask it.
GET/PUT /api/stages and `app stages` show and set it with who would answer;
Runs has a "What the model reads" block, and a search agent's card says when
a stage is off and links to the switch.

Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet committed 2026-09-29 15:54:53 +02:00
1 parent a629eda8de
commit 471ace703c
15 files changed
+593 -23

No files matched your search

+3 -2
View File
@@ -8,6 +8,7 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
summarise.py about 60 words on each of the best roles, on Haiku
cascade.py pull → embed → score → extract → score → summarise, what `TASKS`
offers, the `pull` job a schedule fires, and that schedule
stages.py whether extract and summarise run: the switch, else config.yaml
sources.py what a pull reads, changed by hand: boards and searches
added, switched on or off
discover.py which ATS a company's careers site runs on, from its pages
@@ -17,7 +18,7 @@ from __future__ import annotations
from typing import Any
from app.jobs import cascade, discover, embed, extract, pull, score, sources, summarise
from app.jobs import cascade, discover, embed, extract, pull, score, sources, stages, summarise
from app.jobs.pull import search_target, seed_searches
from app.jobs.score import seed_agents
@@ -37,4 +38,4 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]:
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "pull",
"score", "search_target", "seed_agents", "seed_searches", "sources", "summarise"]
"score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"]
+2 -2
View File
@@ -8,7 +8,7 @@ Nothing is alive between two tasks. A stage is addressed by its position in
the tuple, not its name, since `score` runs twice.
Every stage is always in the chain, and a stage that is switched off finds
nothing: `extract.enabled: false`, or an `on_ingest` this pull does not
nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull does not
satisfy, is an extract stage with no batches whose final job moves straight
on. The free stages go first: embed and score cost no tokens, and the score
they produce is what bounds the two that do. Extract reads only the best
@@ -69,7 +69,7 @@ def _extract(cascade: str) -> Task:
def _summarise(cascade: str) -> Task:
"""`summarise` as a stage: nothing with no model reachable, so a machine
without one pulls in silence rather than failing a batch per pull. Off
(`summarise.enabled: false`) is nothing in either case. Standing on its
(`stages.stage_on`) is nothing in either case. Standing on its
own, as the button, a missing model fails the batch with a notice."""
stage = summarise.summarise_task()
if not cascade:
+19 -4
View File
@@ -19,7 +19,8 @@ fills columns that are still empty: a board's structured field wins over what
a model read out of the prose. See `Store.set_extracted`.
Off by default (`extract.enabled: false`), because on the API backend every
batch is a paid call.
batch is a paid call. Whether it runs is `stages.stage_on`: the switch under
Runs, else the config file.
"""
from __future__ import annotations
@@ -35,6 +36,7 @@ from desk import modeltask
from desk.modeltask import ModelTask
from pipeline import NULL, RunContext, Task, job
from app.config import Config
from app.jobs import stages
from app.models import PERMITS, REMOTE, SENIORITIES, ExtractResult, Role
from app.storage import Store
@@ -234,7 +236,20 @@ def run_batch(config: Config, store: Store, *, context: RunContext = NULL,
return modeltask.one(TASK, config, store, uids or [], context=context).model_dump()
def _always(config: Config) -> Any:
"""The settings with the file's flag out of the way: `batches` reads
`enabled` off them, and the switch is `stages.stage_on`, which needs the
store that `settings` is not given."""
return config.extract.model_copy(update={"enabled": True})
def extract_task() -> Task:
"""Finds nothing while `extract.enabled` is false, so the task, the
button and the cascade stage are all no-ops with the switch off."""
return replace(modeltask.of(TASK), label=TASK.label)
"""Finds nothing while the stage is off (`stages.stage_on`), so the task,
the button and the cascade stage are all no-ops with the switch off."""
stage = modeltask.of(replace(TASK, settings=_always))
find = stage.finder
def when_on(config: Config, store: Store) -> Any:
return find(config, store) if stages.stage_on(config, store, "extract") else []
return replace(stage, finder=when_on)
+91
View File
@@ -0,0 +1,91 @@
"""Whether the two model stages of a pull, `extract` and `summarise`, run.
`extract.enabled` and `summarise.enabled` in config.yaml are read at startup
and say nothing on the page, so the switch lives in the database as well: the
`state` keys `stage:extract` and `stage:summarise`, "on" or "off". A stored
value beats the file; with none, the file applies, so config.yaml is the
default for a fresh database and a deployment that never uses the switch
behaves as it did. `stage_on` is the one place that decides, and both tasks'
finders ask it (`extract.extract_task`, `summarise.summarise_task`).
`describe` is what `GET /api/stages` and `app stages` show: for each stage
whether it is on, where that comes from, the model it uses, and who would
answer.
"""
from __future__ import annotations
from typing import Literal
from pydantic import BaseModel
from app.config import Config
from app.storage import Store
from desk.tools import agent
NAMES = ("extract", "summarise")
NO_MODEL = ("no model is reachable: set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN, "
"see docs/llm-backends.md")
ANSWERS = {
"api": "answered by the Claude API",
"claude_code": "answered by the Claude Code sign-in on this machine",
}
class Stage(BaseModel):
"""One model stage as the page shows it. `source` is "stored" when the
switch set it and "config" when it is still config.yaml's default.
`backend` is `agent.backend`'s answer, "" for nobody, and `answer` says
it in words."""
name: Literal["extract", "summarise"]
on: bool
source: Literal["stored", "config"]
model: str
backend: str
answer: str
def _key(name: str) -> str:
return f"stage:{name}"
def stored(store: Store, name: str) -> bool | None:
"""The switch's value, or None while nobody has used it."""
value = store.state.get(_key(name))
return {"on": True, "off": False}.get(value or "")
def stage_on(config: Config, store: Store, name: str) -> bool:
"""Whether stage `name` runs: the stored setting, else config.yaml's."""
value = stored(store, name)
return getattr(config, name).enabled if value is None else value
def set_stage(store: Store, name: str, on: bool) -> None:
"""Store the switch. A `KeyError` for a name that is not a stage."""
if name not in NAMES:
raise KeyError(name)
store.state[_key(name)] = "on" if on else "off"
def answer(config: Config) -> tuple[str, str]:
"""Who would answer a model call, as `(backend, sentence)`."""
backend = agent.backend(config.agent)
if backend:
return backend, ANSWERS.get(backend, f"answered by {backend}")
if config.agent.backend == "off":
return "", "no model is reachable: `agent.backend` is `off` in config.yaml"
return "", NO_MODEL
def describe_one(config: Config, store: Store, name: str) -> Stage:
backend, said = answer(config)
return Stage(name=name, on=stage_on(config, store, name),
source="config" if stored(store, name) is None else "stored",
model=getattr(config, name).model, backend=backend, answer=said)
def describe(config: Config, store: Store) -> list[Stage]:
return [describe_one(config, store, name) for name in NAMES]
+17 -4
View File
@@ -15,7 +15,8 @@ button) takes those roles whatever their rank. See `Store.summarise_waiting`.
The text is `article_text` when the posting page was fetched, else
`description`. A role with neither is not waiting at all, so it is never sent.
Off by default (`summarise.enabled: false`), like extract.
Off by default (`summarise.enabled: false`), like extract. Whether it runs is
`stages.stage_on`: the switch under Runs, else the config file.
"""
from __future__ import annotations
@@ -29,6 +30,7 @@ from desk.models import ModelTaskResult
from desk.modeltask import ModelTask
from pipeline import NULL, RunContext, Task, job
from app.config import Config
from app.jobs import stages
from app.models import Role
from app.storage import Store
@@ -195,8 +197,19 @@ def run_batch(config: Config, store: Store, *, context: RunContext = NULL,
return modeltask.one(TASK, config, store, uids or [], context=context).model_dump()
def _always(config: Config) -> Any:
"""See `extract._always`."""
return config.summarise.model_copy(update={"enabled": True})
def summarise_task() -> Task:
"""Finds nothing while `summarise.enabled` is false, so the task, `app
job summarise` and the cascade stage are all no-ops with the switch
"""Finds nothing while the stage is off (`stages.stage_on`), so the task,
`app job summarise` and the cascade stage are all no-ops with the switch
off."""
return replace(modeltask.of(TASK), label=TASK.label)
stage = modeltask.of(replace(TASK, settings=_always))
find = stage.finder
def when_on(config: Config, store: Store) -> Any:
return find(config, store) if stages.stage_on(config, store, "summarise") else []
return replace(stage, finder=when_on)