Files
jobboard/app/jobs/cascade.py
T
WolfandClaude Opus 5.5 77b45a4b0f Read role pages in an enrich stage after scoring, best roles first
The pull reads listings only. `enrich` (app/jobs/enrich.py) takes the open
roles without text that an enabled agent ranks ungated in its best
`enrich.top` at `enrich.min_score` or above, best score first, on boards
whose reader has `detail`, within `enrich.per_board` and `enrich.per_run`,
one batch per board. `plugins.ats.read_board_details` reads them and keeps
the per-board state; a read that gave no text is read again after
`enrich.retry_days` (finding 18). Filled roles are upserted, which drops
their vectors, and embed and score follow; `cascade.step` passes over both
when nothing was filled.

The pull cascade is pull, embed, score, enrich, embed, score, extract, score,
summarise. Saving an agent and changing a company's affinity queue the
`enrich` job (enrich, embed, score) behind their rescore; a vote does not.
The stage has a switch, `stage:enrich`, over `enrich.enabled`.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 09:53:29 +02:00

225 lines
9.4 KiB
Python

"""The tasks roledesk offers, and the order a pull runs them in.
pull → embed → score → enrich → embed → score → extract → score → summarise
newsdesk's cascade convention: a cascade is a tuple of task names, and the
final job of one task starts the next (`step` is the callback it carries).
Nothing is alive between two tasks. A stage is addressed by its position in
the tuple, not its name, since `score` runs twice.
Every stage is always in the chain, and a stage that is switched off finds
nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull does not
satisfy, is an extract stage with no batches whose final job moves straight
on. The free stages go first: embed and score cost no tokens, and the score
they produce is what bounds the rest. Enrich reads the best roles without
text on their own pages (`app/jobs/enrich.py`), and embed and score follow for
the roles it filled; when it filled none, `step` passes over both. Extract
reads only the best
roles of each enabled agent, as deep and as low as that agent's own `limits`
say; the second score then applies the gates to the columns it filled.
Summarise runs last, on the best roles by the same agent's summarise limits.
A batch that failed, or a stage somebody killed, does not stop the chain: the
work comes round on the next pull, and a rescore of what did arrive is worth
having either way.
`pull` is a job as well as a cascade, because a schedule enqueues a job kind
and a cascade is started by one: the `pull` job's only work is to ask for the
chain. Its default schedule, `every 6h`, is declared here beside it.
`enrich` is a job and a cascade the same way, `enrich → embed → score`, queued
behind the `score` that follows a change to the ranking other than a pull's
(`enrich_after`): an agent saved, a company's affinity changed. A vote moves
one agent's taste a little and queues neither.
"""
from __future__ import annotations
from collections.abc import Callable
from dataclasses import replace
from functools import partial
from typing import Any
from app.jobs import embed, enrich, extract, pull, score, summarise # noqa: F401 -- registers the jobs
from desk.tools import agent
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
STEP = "app.jobs.cascade:step"
CASCADES: dict[str, tuple[str, ...]] = {
"pull": ("pull", "embed", "score", "enrich", "embed", "score", "extract", "score",
"summarise"),
"enrich": ("enrich", "embed", "score"),
}
#: What follows `enrich` for the roles it filled, passed over when it filled none.
REFILL = ("embed", "score")
def _once(config: Any, store: Any) -> list[dict[str, Any]]:
"""One batch with nothing in its payload: a task that is a single job."""
return [{}]
def _extract(cascade: str) -> Task:
"""`extract` as a stage, reading `on_ingest` when it is one of a pull.
`never`, or `auto` with no model reachable, is a stage that finds
nothing. Standing on its own, as the button, it reads neither."""
stage = extract.extract_task()
if not cascade:
return stage
find = stage.finder
def during_a_pull(config: Any, store: Any) -> list[Any]:
when = config.extract.on_ingest
if when == "never" or (when == "auto" and not agent.backend(config.agent)):
return []
return find(config, store)
return replace(stage, finder=during_a_pull)
def _summarise(cascade: str) -> Task:
"""`summarise` as a stage: nothing with no model reachable, so a machine
without one pulls in silence rather than failing a batch per pull. Off
(`stages.stage_on`) is nothing in either case. Standing on its
own, as the button, a missing model fails the batch with a notice."""
stage = summarise.summarise_task()
if not cascade:
return stage
find = stage.finder
def during_a_pull(config: Any, store: Any) -> list[Any]:
if not agent.backend(config.agent):
return []
return find(config, store)
return replace(stage, finder=during_a_pull)
STAGES: dict[str, Callable[[str], Task]] = {
"pull": lambda cascade: pull.pull_task(),
"extract": _extract,
"embed": lambda cascade: Task(kind="embed", finder=_once, label="Embedding roles"),
"score": lambda cascade: Task(kind="score", finder=_once,
label="Scoring roles for every agent"),
"summarise": _summarise,
"enrich": lambda cascade: enrich.enrich_task(),
}
def task(name: str, *, cascade: str = "", at: int = 0) -> Task:
"""One stage as a `Task`. With `cascade` its final job carries `step`,
and the stage after position `at` follows; without, it stops when its
batches do."""
return replace(STAGES[name](cascade), name=name,
callback=STEP if cascade else "",
carries={"cascade": cascade, "task": name, "at": at})
def follows(cascade: str, at: int) -> str:
"""The stage after position `at`, or "" at the end or off the cascade."""
tasks = CASCADES.get(cascade, ())
return tasks[at + 1] if 0 <= at < len(tasks) - 1 else ""
def step(config: Any, store: Any, tally: Tally, payload: dict[str, Any], *,
context: Any) -> dict[str, Any]:
"""The callback: make the next task. A killed stage is skipped and the
chain goes on, as in newsdesk's cascade."""
name, done = payload.get("cascade", ""), payload.get("task", "")
at = int(payload.get("at", -1))
tasks = CASCADES.get(name, ())
if done == "enrich" and not enrich.filled(tally) \
and tasks[at + 1:at + 1 + len(REFILL)] == REFILL:
at += len(REFILL)
following = follows(name, at)
if following:
context.task(task(following, cascade=name, at=at + 1))
out: dict[str, Any] = {"task": done, "next": following}
if tally.killed:
out["skipped"] = "cancelled"
return out
def chain(name: str) -> Task:
"""The first stage of cascade `name`, carrying `step`, so the rest follow."""
return task(CASCADES[name][0], cascade=name, at=0)
def pull_chain(run: str) -> Task:
"""The pull cascade for a run `pull.claim_run` has claimed: its finder
reads under that marker rather than claiming one of its own."""
return replace(chain("pull"), finder=pull.pull_task(run).finder)
def already_running(since: str) -> Notice:
"""The notice for a pull refused because another is in flight."""
return Notice(
code="pull.running", level="info", kind="expected", key="pull",
text=f"Not started: {pull.ALREADY_RUNNING}"
+ (f" (since {since[:19].replace('T', ' ')} UTC)." if since else "."),
where="app.jobs.pull")
@job("pull", "Pulling every board", resource="http")
def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
"""Claim a pull run and ask for the pull cascade once this job is done.
Its own work is nil.
A pull whose claim is refused, because another run is still unsettled,
queues nothing, embed to summarise included, and says so in its result
and a notice: a second cascade would fetch every board again and could
send the same roles to the model twice.
Under `NULL` there is no queue to take the chain, so this raises rather
than report a pull that never happened; `app job pull` claims the run and
starts the chain on a queue of its own instead of calling this.
"""
run = pull.claim_run(store)
if not run:
since = pull.running(store)
context.notice(already_running(since))
return {"skipped": pull.ALREADY_RUNNING, "running": since}
if not context.task(pull_chain(run)):
pull.release_run(store, run)
raise RuntimeError("no queue to start the pull cascade on")
return {"started": " > ".join(CASCADES["pull"]), "run": run}
@job("enrich", "Reading the best roles on their own pages", solo=True, resource="http")
def run_enrich(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
"""Ask for the enrich cascade, `enrich → embed → score`. Its own work is
nil, as the `pull` job's. Under `NULL` there is no queue to take it."""
if not context.task(chain("enrich")):
raise RuntimeError("no queue to start the enrich cascade on")
return {"started": " > ".join(CASCADES["enrich"])}
def enrich_after(queue: Any, score_job: Any) -> Any:
"""Queue the `enrich` job behind `score_job`, the rescore that follows a
change to the ranking, however that score ends. One per score job
(`enrich:after:<id>`), so ten changes answered by one score are one
enrich. None without a score job, or when the queue refuses."""
if score_job is None:
return None
return queue.enqueue("enrich", depends_on=[score_job.id], after="done",
dedup_key=f"enrich:after:{score_job.id}")
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
#: each, on its own. `pull` is not a task here, since it is a job: a name that
#: is both would be listed twice by `/api/jobs/kinds`.
for _name in ("extract", "embed", "score", "summarise"):
TASKS[_name] = partial(task, _name)
# ---------------------------------------------------------------- schedules
# The default, next to the job it fires. A starting point rather than a
# setting: a cadence edited or switched off under Runs survives
# every restart (see pipeline/schedule.py).
declare("pull", "pull", "every 6h",
label="Pull every board, then embed, score, read the best roles' pages, "
"extract, rescore and summarise")