Files
jobboard/app/jobs/cascade.py
T

265 lines
11 KiB
Python

"""The tasks roledesk offers, and the order a pull runs them in.
pull → prep → embed → score → fetch_text → prep → embed → score → extract
→ score → judge → summarise
newsdesk's cascade convention: a cascade is a tuple of task names, and the
final job of one task starts the next (`step` is the callback it carries).
Nothing is alive between two tasks. `prep` (`app/jobs/prep.py`) maps what
the pull stored and folds the copies before anything reads the roles; after
`fetch_text` it maps the roles their own pages filled. A stage is addressed
by its position in the tuple, not its name, since `score` runs three times.
Every stage is always in the chain, and a stage that is switched off finds
nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this
pull does not satisfy, is a stage with no batches whose final job moves
straight on. The free stages go first: embed and score cost no tokens, and
score writes each agent's cut (`app.jobs.score`), which bounds the rest.
Fetch text reads the role's own page for the roles in the cut that came
without text (`app/jobs/fetch_text.py`), and prep, embed and score follow
for the roles it filled, so their levels and the cut are recomputed; when
it filled none, `step` passes over all three. Extract reads the cut; the
score after it applies the fetch gate and the cut again with the columns
extract filled, so a role can drop out or enter (one that enters is
extracted on the next pull). Judge reads each agent's cut as recomputed,
against its CV. Summarise runs last, on the top of each agent's feed in
the want order and in the match order, and the roles liked or saved: it is
no part of any score.
A batch that failed, or a stage somebody killed, does not stop the chain: the
work comes round on the next pull, and a rescore of what did arrive is worth
having either way.
`pull` is a job as well as a cascade, because a schedule enqueues a job kind
and a cascade is started by one: the `pull` job's only work is to ask for the
chain. Its default schedule, `every 6h`, is declared here beside it.
`fetch_text` is a job and a cascade the same way, the pull's chain from
fetch text on, queued behind the `score` that follows a change to the
ranking other than a pull's (`fetch_text_after`): an agent saved, a
company's affinity or overrides changed. The cut may have changed, so its
new roles get their text, extract and judge at once rather than at the next
pull; a judgement whose `input_key` did not change costs nothing. A vote
moves one agent's taste a little and queues neither.
"""
from __future__ import annotations
from collections.abc import Callable
from dataclasses import replace
from functools import partial
from typing import Any
from app.jobs import ( # noqa: F401 -- registers the jobs
embed, extract, fetch_text, judge, prep, pull, score, summarise,
)
from desk.tools import agent
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
STEP = "app.jobs.cascade:step"
CASCADES: dict[str, tuple[str, ...]] = {
"pull": ("pull", "prep", "embed", "score", "fetch_text", "prep", "embed", "score",
"extract", "score", "judge", "summarise"),
"fetch_text": ("fetch_text", "prep", "embed", "score", "extract", "score", "judge",
"summarise"),
}
#: What follows `fetch_text` for the roles it filled, passed over when it filled none.
REFILL = ("prep", "embed", "score")
def _once(config: Any, store: Any) -> list[dict[str, Any]]:
"""One batch with nothing in its payload: a task that is a single job."""
return [{}]
def _extract(cascade: str) -> Task:
"""`extract` as a stage, reading `on_ingest` when it is one of a pull.
`never`, or `auto` with no model reachable, is a stage that finds
nothing. Standing on its own, as the button, it reads neither."""
stage = extract.extract_task()
if not cascade:
return stage
find = stage.finder
def during_a_pull(config: Any, store: Any) -> list[Any]:
when = config.extract.on_ingest
if when == "never" or (when == "auto" and not agent.backend(config.agent)):
return []
return find(config, store)
return replace(stage, finder=during_a_pull)
def _summarise(cascade: str) -> Task:
"""`summarise` as a stage: nothing with no model reachable, so a machine
without one pulls in silence rather than failing a batch per pull. Off
(`stages.stage_on`) is nothing in either case. Standing on its
own, as the button, a missing model fails the batch with a notice."""
stage = summarise.summarise_task()
if not cascade:
return stage
find = stage.finder
def during_a_pull(config: Any, store: Any) -> list[Any]:
if not agent.backend(config.agent):
return []
return find(config, store)
return replace(stage, finder=during_a_pull)
def _judge(cascade: str) -> Task:
"""`judge` as a stage: nothing with no model reachable, as `_summarise`."""
stage = judge.judge_task()
if not cascade:
return stage
find = stage.finder
def during_a_pull(config: Any, store: Any) -> list[Any]:
if not agent.backend(config.agent):
return []
return find(config, store)
return replace(stage, finder=during_a_pull)
STAGES: dict[str, Callable[[str], Task]] = {
"pull": lambda cascade: pull.pull_task(),
"extract": _extract,
"embed": lambda cascade: Task(kind="embed", finder=_once, label="Embedding roles"),
"score": lambda cascade: Task(kind="score", finder=_once,
label="Scoring roles for every agent"),
"prep": lambda cascade: Task(kind="prep", finder=_once,
label="Mapping the roles' host words"),
"summarise": _summarise,
"judge": _judge,
"fetch_text": lambda cascade: fetch_text.fetch_text_task(),
}
def task(name: str, *, cascade: str = "", at: int = 0) -> Task:
"""One stage as a `Task`. With `cascade` its final job carries `step`,
and the stage after position `at` follows; without, it stops when its
batches do."""
return replace(STAGES[name](cascade), name=name,
callback=STEP if cascade else "",
carries={"cascade": cascade, "task": name, "at": at})
def follows(cascade: str, at: int) -> str:
"""The stage after position `at`, or "" at the end or off the cascade."""
tasks = CASCADES.get(cascade, ())
return tasks[at + 1] if 0 <= at < len(tasks) - 1 else ""
def step(config: Any, store: Any, tally: Tally, payload: dict[str, Any], *,
context: Any) -> dict[str, Any]:
"""The callback: make the next task. A killed stage is skipped and the
chain goes on, as in newsdesk's cascade."""
name, done = payload.get("cascade", ""), payload.get("task", "")
at = int(payload.get("at", -1))
tasks = CASCADES.get(name, ())
if done == "fetch_text" and not fetch_text.filled(tally) \
and tasks[at + 1:at + 1 + len(REFILL)] == REFILL:
at += len(REFILL)
following = follows(name, at)
if following:
context.task(task(following, cascade=name, at=at + 1))
out: dict[str, Any] = {"task": done, "next": following}
if tally.killed:
out["skipped"] = "cancelled"
return out
def chain(name: str) -> Task:
"""The first stage of cascade `name`, carrying `step`, so the rest follow."""
return task(CASCADES[name][0], cascade=name, at=0)
def pull_chain(run: str) -> Task:
"""The pull cascade for a run `pull.claim_run` has claimed: its finder
reads under that marker rather than claiming one of its own."""
return replace(chain("pull"), finder=pull.pull_task(run).finder)
def already_running(since: str) -> Notice:
"""The notice for a pull refused because another is in flight."""
return Notice(
code="pull.running", level="info", kind="expected", key="pull",
text=f"Not started: {pull.ALREADY_RUNNING}"
+ (f" (since {since[:19].replace('T', ' ')} UTC)." if since else "."),
where="app.jobs.pull")
@job("pull", "Pulling every board", resource="http")
def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
"""Claim a pull run and ask for the pull cascade once this job is done.
Its own work is nil.
A pull whose claim is refused, because another run is still unsettled,
queues nothing, embed to summarise included, and says so in its result
and a notice: a second cascade would fetch every board again and could
send the same roles to the model twice.
Under `NULL` there is no queue to take the chain, so this raises rather
than report a pull that never happened; `app job pull` claims the run and
starts the chain on a queue of its own instead of calling this. Anything
that raises once the run is claimed, building the chain or handing it
over, releases the claim first, or the next pull would wait
`pull.ABANDONED` for a run that never started.
"""
run = pull.claim_run(store)
if not run:
since = pull.running(store)
context.notice(already_running(since))
return {"skipped": pull.ALREADY_RUNNING, "running": since}
try:
started = context.task(pull_chain(run))
except Exception:
pull.release_run(store, run)
raise
if not started:
pull.release_run(store, run)
raise RuntimeError("no queue to start the pull cascade on")
return {"started": " > ".join(CASCADES["pull"]), "run": run}
@job("fetch_text", fetch_text.LABEL, solo=True, resource="http")
def run_fetch_text(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
"""Ask for the fetch text cascade, the pull's chain from fetch text on.
Its own work is nil, as the `pull` job's. Under `NULL` there is no queue
to take it."""
if not context.task(chain("fetch_text")):
raise RuntimeError("no queue to start the fetch text cascade on")
return {"started": " > ".join(CASCADES["fetch_text"])}
def fetch_text_after(queue: Any, score_job: Any) -> Any:
"""Queue the `fetch_text` job behind `score_job`, the rescore that
follows a change to the ranking, however that score ends. One per score
job (`fetch_text:after:<id>`), so ten changes answered by one score are
one cascade. None without a score job, or when the queue refuses."""
if score_job is None:
return None
return queue.enqueue("fetch_text", depends_on=[score_job.id], after="done",
dedup_key=f"fetch_text:after:{score_job.id}")
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
#: each, on its own. `pull` is not a task here, since it is a job: a name that
#: is both would be listed twice by `/api/jobs/kinds`.
for _name in ("extract", "embed", "score", "summarise", "judge", "prep"):
TASKS[_name] = partial(task, _name)
# ---------------------------------------------------------------- schedules
# The default, next to the job it fires. A starting point rather than a
# setting: a cadence edited or switched off under Runs survives
# every restart (see pipeline/schedule.py).
declare("pull", "pull", "every 6h",
label="Pull every board, then map, embed, score, fetch the text of the roles in the "
"cut, extract, rescore, judge against the CVs and summarise")