The boilerplate's rule: the deploy kit's Dockerfile, the APP_CONFIG, APP_PASSWORD and APP_DATA variables and its commands all assume a package called app, so a project renames its display title and nothing else. The console script is `app`, and the launch entry runs `uv run app serve --port 8790`. The domain noun stays `role`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
106 lines
3.8 KiB
Python
106 lines
3.8 KiB
Python
"""The tasks roledesk offers, and the order a pull runs them in.
|
|
|
|
pull → extract → embed → score
|
|
|
|
newsdesk's cascade convention: a cascade is a tuple of task names, and the
|
|
final job of one task starts the next (`step` is the callback it carries).
|
|
Nothing is alive between two tasks.
|
|
|
|
Every stage is always in the chain, and a stage that is switched off finds
|
|
nothing: `extract.enabled: false`, or an `on_ingest` this pull does not
|
|
satisfy, is an extract stage with no batches whose final job moves straight
|
|
on. Extract runs before score so the gates read what it filled; before embed
|
|
too, since embedding does not read its columns and the order costs nothing.
|
|
|
|
A batch that failed, or a stage somebody killed, does not stop the chain: the
|
|
work comes round on the next pull, and a rescore of what did arrive is worth
|
|
having either way.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Callable
|
|
from dataclasses import replace
|
|
from functools import partial
|
|
from typing import Any
|
|
|
|
from desk.tools import agent
|
|
from pipeline import TASKS, Tally, Task
|
|
from app.jobs import embed, extract, pull, score # noqa: F401 -- registers the jobs
|
|
|
|
STEP = "app.jobs.cascade:step"
|
|
|
|
CASCADES: dict[str, tuple[str, ...]] = {
|
|
"pull": ("pull", "extract", "embed", "score"),
|
|
}
|
|
|
|
|
|
def _once(config: Any, store: Any) -> list[dict[str, Any]]:
|
|
"""One batch with nothing in its payload: a task that is a single job."""
|
|
return [{}]
|
|
|
|
|
|
def _extract(cascade: str) -> Task:
|
|
"""`extract` as a stage, reading `on_ingest` when it is one of a pull.
|
|
`never`, or `auto` with no model reachable, is a stage that finds
|
|
nothing. Standing on its own, as the button, it reads neither."""
|
|
stage = extract.extract_task()
|
|
if not cascade:
|
|
return stage
|
|
find = stage.finder
|
|
|
|
def during_a_pull(config: Any, store: Any) -> list[Any]:
|
|
when = config.extract.on_ingest
|
|
if when == "never" or (when == "auto" and not agent.backend(config.agent)):
|
|
return []
|
|
return find(config, store)
|
|
|
|
return replace(stage, finder=during_a_pull)
|
|
|
|
|
|
STAGES: dict[str, Callable[[str], Task]] = {
|
|
"pull": lambda cascade: pull.pull_task(),
|
|
"extract": _extract,
|
|
"embed": lambda cascade: Task(kind="embed", finder=_once, label="Embedding roles"),
|
|
"score": lambda cascade: Task(kind="score", finder=_once,
|
|
label="Scoring roles for every agent"),
|
|
}
|
|
|
|
|
|
def task(name: str, *, cascade: str = "") -> Task:
|
|
"""One stage as a `Task`. With `cascade` its final job carries `step`,
|
|
and the next stage follows; without, it stops when its batches do."""
|
|
return replace(STAGES[name](cascade), name=name,
|
|
callback=STEP if cascade else "",
|
|
carries={"cascade": cascade, "task": name})
|
|
|
|
|
|
def follows(cascade: str, name: str) -> str:
|
|
"""What runs after `name`, or "" at the end or for an unknown name."""
|
|
tasks = CASCADES.get(cascade, ())
|
|
if name not in tasks:
|
|
return ""
|
|
at = tasks.index(name)
|
|
return tasks[at + 1] if at + 1 < len(tasks) else ""
|
|
|
|
|
|
def step(config: Any, store: Any, tally: Tally, payload: dict[str, Any], *,
|
|
context: Any) -> dict[str, Any]:
|
|
"""The callback: make the next task. A killed stage is skipped and the
|
|
chain goes on, as in newsdesk's cascade."""
|
|
name, done = payload.get("cascade", ""), payload.get("task", "")
|
|
following = follows(name, done)
|
|
if following:
|
|
context.task(task(following, cascade=name))
|
|
out: dict[str, Any] = {"task": done, "next": following}
|
|
if tally.killed:
|
|
out["skipped"] = "cancelled"
|
|
return out
|
|
|
|
|
|
#: What `app job <name>` and the Jobs view offer. `pull` is the whole
|
|
#: chain; the others are one stage each, on their own.
|
|
TASKS["pull"] = partial(task, "pull", cascade="pull")
|
|
for _name in ("extract", "embed", "score"):
|
|
TASKS[_name] = partial(task, _name)
|