"""The tasks roledesk offers, and the order a pull runs them in. pull → embed → score → extract → score → summarise newsdesk's cascade convention: a cascade is a tuple of task names, and the final job of one task starts the next (`step` is the callback it carries). Nothing is alive between two tasks. A stage is addressed by its position in the tuple, not its name, since `score` runs twice. Every stage is always in the chain, and a stage that is switched off finds nothing: `extract.enabled: false`, or an `on_ingest` this pull does not satisfy, is an extract stage with no batches whose final job moves straight on. The free stages go first: embed and score cost no tokens, and the score they produce is what bounds the two that do. Extract reads only the best roles of each enabled agent, as deep and as low as that agent's own `limits` say; the second score then applies the gates to the columns it filled. Summarise runs last, on the best roles by the same agent's summarise limits. A batch that failed, or a stage somebody killed, does not stop the chain: the work comes round on the next pull, and a rescore of what did arrive is worth having either way. `pull` is a job as well as a cascade, because a schedule enqueues a job kind and a cascade is started by one: the `pull` job's only work is to ask for the chain. Its default schedule, `every 6h`, is declared here beside it. """ from __future__ import annotations from collections.abc import Callable from dataclasses import replace from functools import partial from typing import Any from app.jobs import embed, extract, pull, score, summarise # noqa: F401 -- registers the jobs from desk.tools import agent from pipeline import NULL, TASKS, RunContext, Tally, Task, declare, job STEP = "app.jobs.cascade:step" CASCADES: dict[str, tuple[str, ...]] = { "pull": ("pull", "embed", "score", "extract", "score", "summarise"), } def _once(config: Any, store: Any) -> list[dict[str, Any]]: """One batch with nothing in its payload: a task that is a single job.""" return [{}] def _extract(cascade: str) -> Task: """`extract` as a stage, reading `on_ingest` when it is one of a pull. `never`, or `auto` with no model reachable, is a stage that finds nothing. Standing on its own, as the button, it reads neither.""" stage = extract.extract_task() if not cascade: return stage find = stage.finder def during_a_pull(config: Any, store: Any) -> list[Any]: when = config.extract.on_ingest if when == "never" or (when == "auto" and not agent.backend(config.agent)): return [] return find(config, store) return replace(stage, finder=during_a_pull) def _summarise(cascade: str) -> Task: """`summarise` as a stage: nothing with no model reachable, so a machine without one pulls in silence rather than failing a batch per pull. Off (`summarise.enabled: false`) is nothing in either case. Standing on its own, as the button, a missing model fails the batch with a notice.""" stage = summarise.summarise_task() if not cascade: return stage find = stage.finder def during_a_pull(config: Any, store: Any) -> list[Any]: if not agent.backend(config.agent): return [] return find(config, store) return replace(stage, finder=during_a_pull) STAGES: dict[str, Callable[[str], Task]] = { "pull": lambda cascade: pull.pull_task(), "extract": _extract, "embed": lambda cascade: Task(kind="embed", finder=_once, label="Embedding roles"), "score": lambda cascade: Task(kind="score", finder=_once, label="Scoring roles for every agent"), "summarise": _summarise, } def task(name: str, *, cascade: str = "", at: int = 0) -> Task: """One stage as a `Task`. With `cascade` its final job carries `step`, and the stage after position `at` follows; without, it stops when its batches do.""" return replace(STAGES[name](cascade), name=name, callback=STEP if cascade else "", carries={"cascade": cascade, "task": name, "at": at}) def follows(cascade: str, at: int) -> str: """The stage after position `at`, or "" at the end or off the cascade.""" tasks = CASCADES.get(cascade, ()) return tasks[at + 1] if 0 <= at < len(tasks) - 1 else "" def step(config: Any, store: Any, tally: Tally, payload: dict[str, Any], *, context: Any) -> dict[str, Any]: """The callback: make the next task. A killed stage is skipped and the chain goes on, as in newsdesk's cascade.""" name, done = payload.get("cascade", ""), payload.get("task", "") at = int(payload.get("at", -1)) following = follows(name, at) if following: context.task(task(following, cascade=name, at=at + 1)) out: dict[str, Any] = {"task": done, "next": following} if tally.killed: out["skipped"] = "cancelled" return out def chain(name: str) -> Task: """The first stage of cascade `name`, carrying `step`, so the rest follow.""" return task(CASCADES[name][0], cascade=name, at=0) @job("pull", "Pulling every board", resource="http") def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]: """Ask for the pull cascade once this job is done. Its own work is nil. Under `NULL` there is no queue to take the chain, so this raises rather than report a pull that never happened; `app job pull` starts the chain on a queue of its own instead of calling this. """ if not context.task(chain("pull")): raise RuntimeError("no queue to start the pull cascade on") return {"started": " > ".join(CASCADES["pull"])} #: What `app job ` and `/api/jobs` start besides `pull`: one stage #: each, on its own. `pull` is not a task here, since it is a job: a name that #: is both would be listed twice by `/api/jobs/kinds`. for _name in ("extract", "embed", "score", "summarise"): TASKS[_name] = partial(task, _name) # ---------------------------------------------------------------- schedules # The default, next to the job it fires. A starting point rather than a # setting: a cadence edited or switched off under Runs survives # every restart (see pipeline/schedule.py). declare("pull", "pull", "every 6h", label="Pull every board, then embed, score, extract, rescore and summarise")