"""The tasks roledesk offers, and the order a pull runs them in. pull → prep → embed → score → enrich → prep → embed → score → extract → score → summarise newsdesk's cascade convention: a cascade is a tuple of task names, and the final job of one task starts the next (`step` is the callback it carries). Nothing is alive between two tasks. `prep` (`app/jobs/prep.py`) maps what the pull stored and folds the copies before anything reads the roles; after `enrich` it maps the roles their own pages filled. A stage is addressed by its position in the tuple, not its name, since `score` runs twice. Every stage is always in the chain, and a stage that is switched off finds nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull does not satisfy, is an extract stage with no batches whose final job moves straight on. The free stages go first: embed and score cost no tokens, and the score they produce is what bounds the rest. Enrich reads the best roles without text on their own pages (`app/jobs/enrich.py`), and prep, embed and score follow for the roles it filled; when it filled none, `step` passes over all three. Extract reads only the best roles of each enabled agent, as deep and as low as that agent's own `limits` say; the second score then applies the gates to the columns it filled. Summarise runs last, on the best roles by the same agent's summarise limits. A batch that failed, or a stage somebody killed, does not stop the chain: the work comes round on the next pull, and a rescore of what did arrive is worth having either way. `pull` is a job as well as a cascade, because a schedule enqueues a job kind and a cascade is started by one: the `pull` job's only work is to ask for the chain. Its default schedule, `every 6h`, is declared here beside it. `enrich` is a job and a cascade the same way, `enrich → prep → embed → score`, queued behind the `score` that follows a change to the ranking other than a pull's (`enrich_after`): an agent saved, a company's affinity changed. A vote moves one agent's taste a little and queues neither. """ from __future__ import annotations from collections.abc import Callable from dataclasses import replace from functools import partial from typing import Any from app.jobs import embed, enrich, extract, prep, pull, score, summarise # noqa: F401 -- registers the jobs from desk.tools import agent from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job STEP = "app.jobs.cascade:step" CASCADES: dict[str, tuple[str, ...]] = { "pull": ("pull", "prep", "embed", "score", "enrich", "prep", "embed", "score", "extract", "score", "summarise"), "enrich": ("enrich", "prep", "embed", "score"), } #: What follows `enrich` for the roles it filled, passed over when it filled none. REFILL = ("prep", "embed", "score") def _once(config: Any, store: Any) -> list[dict[str, Any]]: """One batch with nothing in its payload: a task that is a single job.""" return [{}] def _extract(cascade: str) -> Task: """`extract` as a stage, reading `on_ingest` when it is one of a pull. `never`, or `auto` with no model reachable, is a stage that finds nothing. Standing on its own, as the button, it reads neither.""" stage = extract.extract_task() if not cascade: return stage find = stage.finder def during_a_pull(config: Any, store: Any) -> list[Any]: when = config.extract.on_ingest if when == "never" or (when == "auto" and not agent.backend(config.agent)): return [] return find(config, store) return replace(stage, finder=during_a_pull) def _summarise(cascade: str) -> Task: """`summarise` as a stage: nothing with no model reachable, so a machine without one pulls in silence rather than failing a batch per pull. Off (`stages.stage_on`) is nothing in either case. Standing on its own, as the button, a missing model fails the batch with a notice.""" stage = summarise.summarise_task() if not cascade: return stage find = stage.finder def during_a_pull(config: Any, store: Any) -> list[Any]: if not agent.backend(config.agent): return [] return find(config, store) return replace(stage, finder=during_a_pull) STAGES: dict[str, Callable[[str], Task]] = { "pull": lambda cascade: pull.pull_task(), "extract": _extract, "embed": lambda cascade: Task(kind="embed", finder=_once, label="Embedding roles"), "score": lambda cascade: Task(kind="score", finder=_once, label="Scoring roles for every agent"), "prep": lambda cascade: Task(kind="prep", finder=_once, label="Mapping the roles' host words"), "summarise": _summarise, "enrich": lambda cascade: enrich.enrich_task(), } def task(name: str, *, cascade: str = "", at: int = 0) -> Task: """One stage as a `Task`. With `cascade` its final job carries `step`, and the stage after position `at` follows; without, it stops when its batches do.""" return replace(STAGES[name](cascade), name=name, callback=STEP if cascade else "", carries={"cascade": cascade, "task": name, "at": at}) def follows(cascade: str, at: int) -> str: """The stage after position `at`, or "" at the end or off the cascade.""" tasks = CASCADES.get(cascade, ()) return tasks[at + 1] if 0 <= at < len(tasks) - 1 else "" def step(config: Any, store: Any, tally: Tally, payload: dict[str, Any], *, context: Any) -> dict[str, Any]: """The callback: make the next task. A killed stage is skipped and the chain goes on, as in newsdesk's cascade.""" name, done = payload.get("cascade", ""), payload.get("task", "") at = int(payload.get("at", -1)) tasks = CASCADES.get(name, ()) if done == "enrich" and not enrich.filled(tally) \ and tasks[at + 1:at + 1 + len(REFILL)] == REFILL: at += len(REFILL) following = follows(name, at) if following: context.task(task(following, cascade=name, at=at + 1)) out: dict[str, Any] = {"task": done, "next": following} if tally.killed: out["skipped"] = "cancelled" return out def chain(name: str) -> Task: """The first stage of cascade `name`, carrying `step`, so the rest follow.""" return task(CASCADES[name][0], cascade=name, at=0) def pull_chain(run: str) -> Task: """The pull cascade for a run `pull.claim_run` has claimed: its finder reads under that marker rather than claiming one of its own.""" return replace(chain("pull"), finder=pull.pull_task(run).finder) def already_running(since: str) -> Notice: """The notice for a pull refused because another is in flight.""" return Notice( code="pull.running", level="info", kind="expected", key="pull", text=f"Not started: {pull.ALREADY_RUNNING}" + (f" (since {since[:19].replace('T', ' ')} UTC)." if since else "."), where="app.jobs.pull") @job("pull", "Pulling every board", resource="http") def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]: """Claim a pull run and ask for the pull cascade once this job is done. Its own work is nil. A pull whose claim is refused, because another run is still unsettled, queues nothing, embed to summarise included, and says so in its result and a notice: a second cascade would fetch every board again and could send the same roles to the model twice. Under `NULL` there is no queue to take the chain, so this raises rather than report a pull that never happened; `app job pull` claims the run and starts the chain on a queue of its own instead of calling this. """ run = pull.claim_run(store) if not run: since = pull.running(store) context.notice(already_running(since)) return {"skipped": pull.ALREADY_RUNNING, "running": since} if not context.task(pull_chain(run)): pull.release_run(store, run) raise RuntimeError("no queue to start the pull cascade on") return {"started": " > ".join(CASCADES["pull"]), "run": run} @job("enrich", "Reading the best roles on their own pages", solo=True, resource="http") def run_enrich(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]: """Ask for the enrich cascade, `enrich → prep → embed → score`. Its own work is nil, as the `pull` job's. Under `NULL` there is no queue to take it.""" if not context.task(chain("enrich")): raise RuntimeError("no queue to start the enrich cascade on") return {"started": " > ".join(CASCADES["enrich"])} def enrich_after(queue: Any, score_job: Any) -> Any: """Queue the `enrich` job behind `score_job`, the rescore that follows a change to the ranking, however that score ends. One per score job (`enrich:after:`), so ten changes answered by one score are one enrich. None without a score job, or when the queue refuses.""" if score_job is None: return None return queue.enqueue("enrich", depends_on=[score_job.id], after="done", dedup_key=f"enrich:after:{score_job.id}") #: What `app job ` and `/api/jobs` start besides `pull`: one stage #: each, on its own. `pull` is not a task here, since it is a job: a name that #: is both would be listed twice by `/api/jobs/kinds`. for _name in ("extract", "embed", "score", "summarise", "prep"): TASKS[_name] = partial(task, _name) # ---------------------------------------------------------------- schedules # The default, next to the job it fires. A starting point rather than a # setting: a cadence edited or switched off under Runs survives # every restart (see pipeline/schedule.py). declare("pull", "pull", "every 6h", label="Pull every board, then map, embed, score, read the best roles' pages, " "extract, rescore and summarise")