Switch the extract and summarise stages on the page, over config.yaml
A stored `stage:extract` / `stage:summarise` in the state table overrides extract.enabled / summarise.enabled; with none, the file applies. One function, stages.stage_on, decides, and both tasks' finders ask it. GET/PUT /api/stages and `app stages` show and set it with who would answer; Runs has a "What the model reads" block, and a search agent's card says when a stage is off and links to the switch. Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
This commit is contained in:
1 parent
a629eda8de
commit
471ace703c
15 files changed
+593
-23
No files matched your search
@@ -8,6 +8,7 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
|
||||
summarise.py about 60 words on each of the best roles, on Haiku
|
||||
cascade.py pull → embed → score → extract → score → summarise, what `TASKS`
|
||||
offers, the `pull` job a schedule fires, and that schedule
|
||||
stages.py whether extract and summarise run: the switch, else config.yaml
|
||||
sources.py what a pull reads, changed by hand: boards and searches
|
||||
added, switched on or off
|
||||
discover.py which ATS a company's careers site runs on, from its pages
|
||||
@@ -17,7 +18,7 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.jobs import cascade, discover, embed, extract, pull, score, sources, summarise
|
||||
from app.jobs import cascade, discover, embed, extract, pull, score, sources, stages, summarise
|
||||
from app.jobs.pull import search_target, seed_searches
|
||||
from app.jobs.score import seed_agents
|
||||
|
||||
@@ -37,4 +38,4 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]:
|
||||
|
||||
|
||||
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "pull",
|
||||
"score", "search_target", "seed_agents", "seed_searches", "sources", "summarise"]
|
||||
"score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"]
|
||||
+2
-2
@@ -8,7 +8,7 @@ Nothing is alive between two tasks. A stage is addressed by its position in
|
||||
the tuple, not its name, since `score` runs twice.
|
||||
|
||||
Every stage is always in the chain, and a stage that is switched off finds
|
||||
nothing: `extract.enabled: false`, or an `on_ingest` this pull does not
|
||||
nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull does not
|
||||
satisfy, is an extract stage with no batches whose final job moves straight
|
||||
on. The free stages go first: embed and score cost no tokens, and the score
|
||||
they produce is what bounds the two that do. Extract reads only the best
|
||||
@@ -69,7 +69,7 @@ def _extract(cascade: str) -> Task:
|
||||
def _summarise(cascade: str) -> Task:
|
||||
"""`summarise` as a stage: nothing with no model reachable, so a machine
|
||||
without one pulls in silence rather than failing a batch per pull. Off
|
||||
(`summarise.enabled: false`) is nothing in either case. Standing on its
|
||||
(`stages.stage_on`) is nothing in either case. Standing on its
|
||||
own, as the button, a missing model fails the batch with a notice."""
|
||||
stage = summarise.summarise_task()
|
||||
if not cascade:
|
||||
|
||||
+19
-4
@@ -19,7 +19,8 @@ fills columns that are still empty: a board's structured field wins over what
|
||||
a model read out of the prose. See `Store.set_extracted`.
|
||||
|
||||
Off by default (`extract.enabled: false`), because on the API backend every
|
||||
batch is a paid call.
|
||||
batch is a paid call. Whether it runs is `stages.stage_on`: the switch under
|
||||
Runs, else the config file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -35,6 +36,7 @@ from desk import modeltask
|
||||
from desk.modeltask import ModelTask
|
||||
from pipeline import NULL, RunContext, Task, job
|
||||
from app.config import Config
|
||||
from app.jobs import stages
|
||||
from app.models import PERMITS, REMOTE, SENIORITIES, ExtractResult, Role
|
||||
from app.storage import Store
|
||||
|
||||
@@ -234,7 +236,20 @@ def run_batch(config: Config, store: Store, *, context: RunContext = NULL,
|
||||
return modeltask.one(TASK, config, store, uids or [], context=context).model_dump()
|
||||
|
||||
|
||||
def _always(config: Config) -> Any:
|
||||
"""The settings with the file's flag out of the way: `batches` reads
|
||||
`enabled` off them, and the switch is `stages.stage_on`, which needs the
|
||||
store that `settings` is not given."""
|
||||
return config.extract.model_copy(update={"enabled": True})
|
||||
|
||||
|
||||
def extract_task() -> Task:
|
||||
"""Finds nothing while `extract.enabled` is false, so the task, the
|
||||
button and the cascade stage are all no-ops with the switch off."""
|
||||
return replace(modeltask.of(TASK), label=TASK.label)
|
||||
"""Finds nothing while the stage is off (`stages.stage_on`), so the task,
|
||||
the button and the cascade stage are all no-ops with the switch off."""
|
||||
stage = modeltask.of(replace(TASK, settings=_always))
|
||||
find = stage.finder
|
||||
|
||||
def when_on(config: Config, store: Store) -> Any:
|
||||
return find(config, store) if stages.stage_on(config, store, "extract") else []
|
||||
|
||||
return replace(stage, finder=when_on)
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Whether the two model stages of a pull, `extract` and `summarise`, run.
|
||||
|
||||
`extract.enabled` and `summarise.enabled` in config.yaml are read at startup
|
||||
and say nothing on the page, so the switch lives in the database as well: the
|
||||
`state` keys `stage:extract` and `stage:summarise`, "on" or "off". A stored
|
||||
value beats the file; with none, the file applies, so config.yaml is the
|
||||
default for a fresh database and a deployment that never uses the switch
|
||||
behaves as it did. `stage_on` is the one place that decides, and both tasks'
|
||||
finders ask it (`extract.extract_task`, `summarise.summarise_task`).
|
||||
|
||||
`describe` is what `GET /api/stages` and `app stages` show: for each stage
|
||||
whether it is on, where that comes from, the model it uses, and who would
|
||||
answer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.config import Config
|
||||
from app.storage import Store
|
||||
from desk.tools import agent
|
||||
|
||||
NAMES = ("extract", "summarise")
|
||||
|
||||
NO_MODEL = ("no model is reachable: set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN, "
|
||||
"see docs/llm-backends.md")
|
||||
ANSWERS = {
|
||||
"api": "answered by the Claude API",
|
||||
"claude_code": "answered by the Claude Code sign-in on this machine",
|
||||
}
|
||||
|
||||
|
||||
class Stage(BaseModel):
|
||||
"""One model stage as the page shows it. `source` is "stored" when the
|
||||
switch set it and "config" when it is still config.yaml's default.
|
||||
`backend` is `agent.backend`'s answer, "" for nobody, and `answer` says
|
||||
it in words."""
|
||||
|
||||
name: Literal["extract", "summarise"]
|
||||
on: bool
|
||||
source: Literal["stored", "config"]
|
||||
model: str
|
||||
backend: str
|
||||
answer: str
|
||||
|
||||
|
||||
def _key(name: str) -> str:
|
||||
return f"stage:{name}"
|
||||
|
||||
|
||||
def stored(store: Store, name: str) -> bool | None:
|
||||
"""The switch's value, or None while nobody has used it."""
|
||||
value = store.state.get(_key(name))
|
||||
return {"on": True, "off": False}.get(value or "")
|
||||
|
||||
|
||||
def stage_on(config: Config, store: Store, name: str) -> bool:
|
||||
"""Whether stage `name` runs: the stored setting, else config.yaml's."""
|
||||
value = stored(store, name)
|
||||
return getattr(config, name).enabled if value is None else value
|
||||
|
||||
|
||||
def set_stage(store: Store, name: str, on: bool) -> None:
|
||||
"""Store the switch. A `KeyError` for a name that is not a stage."""
|
||||
if name not in NAMES:
|
||||
raise KeyError(name)
|
||||
store.state[_key(name)] = "on" if on else "off"
|
||||
|
||||
|
||||
def answer(config: Config) -> tuple[str, str]:
|
||||
"""Who would answer a model call, as `(backend, sentence)`."""
|
||||
backend = agent.backend(config.agent)
|
||||
if backend:
|
||||
return backend, ANSWERS.get(backend, f"answered by {backend}")
|
||||
if config.agent.backend == "off":
|
||||
return "", "no model is reachable: `agent.backend` is `off` in config.yaml"
|
||||
return "", NO_MODEL
|
||||
|
||||
|
||||
def describe_one(config: Config, store: Store, name: str) -> Stage:
|
||||
backend, said = answer(config)
|
||||
return Stage(name=name, on=stage_on(config, store, name),
|
||||
source="config" if stored(store, name) is None else "stored",
|
||||
model=getattr(config, name).model, backend=backend, answer=said)
|
||||
|
||||
|
||||
def describe(config: Config, store: Store) -> list[Stage]:
|
||||
return [describe_one(config, store, name) for name in NAMES]
|
||||
+17
-4
@@ -15,7 +15,8 @@ button) takes those roles whatever their rank. See `Store.summarise_waiting`.
|
||||
The text is `article_text` when the posting page was fetched, else
|
||||
`description`. A role with neither is not waiting at all, so it is never sent.
|
||||
|
||||
Off by default (`summarise.enabled: false`), like extract.
|
||||
Off by default (`summarise.enabled: false`), like extract. Whether it runs is
|
||||
`stages.stage_on`: the switch under Runs, else the config file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -29,6 +30,7 @@ from desk.models import ModelTaskResult
|
||||
from desk.modeltask import ModelTask
|
||||
from pipeline import NULL, RunContext, Task, job
|
||||
from app.config import Config
|
||||
from app.jobs import stages
|
||||
from app.models import Role
|
||||
from app.storage import Store
|
||||
|
||||
@@ -195,8 +197,19 @@ def run_batch(config: Config, store: Store, *, context: RunContext = NULL,
|
||||
return modeltask.one(TASK, config, store, uids or [], context=context).model_dump()
|
||||
|
||||
|
||||
def _always(config: Config) -> Any:
|
||||
"""See `extract._always`."""
|
||||
return config.summarise.model_copy(update={"enabled": True})
|
||||
|
||||
|
||||
def summarise_task() -> Task:
|
||||
"""Finds nothing while `summarise.enabled` is false, so the task, `app
|
||||
job summarise` and the cascade stage are all no-ops with the switch
|
||||
"""Finds nothing while the stage is off (`stages.stage_on`), so the task,
|
||||
`app job summarise` and the cascade stage are all no-ops with the switch
|
||||
off."""
|
||||
return replace(modeltask.of(TASK), label=TASK.label)
|
||||
stage = modeltask.of(replace(TASK, settings=_always))
|
||||
find = stage.finder
|
||||
|
||||
def when_on(config: Config, store: Store) -> Any:
|
||||
return find(config, store) if stages.stage_on(config, store, "summarise") else []
|
||||
|
||||
return replace(stage, finder=when_on)
|
||||
Reference in new issue
Block a user