265 lines
11 KiB
Python
265 lines
11 KiB
Python
"""The tasks roledesk offers, and the order a pull runs them in.
|
|
|
|
pull → prep → embed → score → fetch_text → prep → embed → score → extract
|
|
→ score → judge → summarise
|
|
|
|
newsdesk's cascade convention: a cascade is a tuple of task names, and the
|
|
final job of one task starts the next (`step` is the callback it carries).
|
|
Nothing is alive between two tasks. `prep` (`app/jobs/prep.py`) maps what
|
|
the pull stored and folds the copies before anything reads the roles; after
|
|
`fetch_text` it maps the roles their own pages filled. A stage is addressed
|
|
by its position in the tuple, not its name, since `score` runs three times.
|
|
|
|
Every stage is always in the chain, and a stage that is switched off finds
|
|
nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this
|
|
pull does not satisfy, is a stage with no batches whose final job moves
|
|
straight on. The free stages go first: embed and score cost no tokens, and
|
|
score writes each agent's cut (`app.jobs.score`), which bounds the rest.
|
|
Fetch text reads the role's own page for the roles in the cut that came
|
|
without text (`app/jobs/fetch_text.py`), and prep, embed and score follow
|
|
for the roles it filled, so their levels and the cut are recomputed; when
|
|
it filled none, `step` passes over all three. Extract reads the cut; the
|
|
score after it applies the fetch gate and the cut again with the columns
|
|
extract filled, so a role can drop out or enter (one that enters is
|
|
extracted on the next pull). Judge reads each agent's cut as recomputed,
|
|
against its CV. Summarise runs last, on the top of each agent's feed in
|
|
the want order and in the match order, and the roles liked or saved: it is
|
|
no part of any score.
|
|
|
|
A batch that failed, or a stage somebody killed, does not stop the chain: the
|
|
work comes round on the next pull, and a rescore of what did arrive is worth
|
|
having either way.
|
|
|
|
`pull` is a job as well as a cascade, because a schedule enqueues a job kind
|
|
and a cascade is started by one: the `pull` job's only work is to ask for the
|
|
chain. Its default schedule, `every 6h`, is declared here beside it.
|
|
|
|
`fetch_text` is a job and a cascade the same way, the pull's chain from
|
|
fetch text on, queued behind the `score` that follows a change to the
|
|
ranking other than a pull's (`fetch_text_after`): an agent saved, a
|
|
company's affinity or overrides changed. The cut may have changed, so its
|
|
new roles get their text, extract and judge at once rather than at the next
|
|
pull; a judgement whose `input_key` did not change costs nothing. A vote
|
|
moves one agent's taste a little and queues neither.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Callable
|
|
from dataclasses import replace
|
|
from functools import partial
|
|
from typing import Any
|
|
|
|
from app.jobs import ( # noqa: F401 -- registers the jobs
|
|
embed, extract, fetch_text, judge, prep, pull, score, summarise,
|
|
)
|
|
from desk.tools import agent
|
|
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
|
|
|
|
STEP = "app.jobs.cascade:step"
|
|
|
|
CASCADES: dict[str, tuple[str, ...]] = {
|
|
"pull": ("pull", "prep", "embed", "score", "fetch_text", "prep", "embed", "score",
|
|
"extract", "score", "judge", "summarise"),
|
|
"fetch_text": ("fetch_text", "prep", "embed", "score", "extract", "score", "judge",
|
|
"summarise"),
|
|
}
|
|
|
|
#: What follows `fetch_text` for the roles it filled, passed over when it filled none.
|
|
REFILL = ("prep", "embed", "score")
|
|
|
|
|
|
def _once(config: Any, store: Any) -> list[dict[str, Any]]:
|
|
"""One batch with nothing in its payload: a task that is a single job."""
|
|
return [{}]
|
|
|
|
|
|
def _extract(cascade: str) -> Task:
|
|
"""`extract` as a stage, reading `on_ingest` when it is one of a pull.
|
|
`never`, or `auto` with no model reachable, is a stage that finds
|
|
nothing. Standing on its own, as the button, it reads neither."""
|
|
stage = extract.extract_task()
|
|
if not cascade:
|
|
return stage
|
|
find = stage.finder
|
|
|
|
def during_a_pull(config: Any, store: Any) -> list[Any]:
|
|
when = config.extract.on_ingest
|
|
if when == "never" or (when == "auto" and not agent.backend(config.agent)):
|
|
return []
|
|
return find(config, store)
|
|
|
|
return replace(stage, finder=during_a_pull)
|
|
|
|
|
|
def _summarise(cascade: str) -> Task:
|
|
"""`summarise` as a stage: nothing with no model reachable, so a machine
|
|
without one pulls in silence rather than failing a batch per pull. Off
|
|
(`stages.stage_on`) is nothing in either case. Standing on its
|
|
own, as the button, a missing model fails the batch with a notice."""
|
|
stage = summarise.summarise_task()
|
|
if not cascade:
|
|
return stage
|
|
find = stage.finder
|
|
|
|
def during_a_pull(config: Any, store: Any) -> list[Any]:
|
|
if not agent.backend(config.agent):
|
|
return []
|
|
return find(config, store)
|
|
|
|
return replace(stage, finder=during_a_pull)
|
|
|
|
|
|
def _judge(cascade: str) -> Task:
|
|
"""`judge` as a stage: nothing with no model reachable, as `_summarise`."""
|
|
stage = judge.judge_task()
|
|
if not cascade:
|
|
return stage
|
|
find = stage.finder
|
|
|
|
def during_a_pull(config: Any, store: Any) -> list[Any]:
|
|
if not agent.backend(config.agent):
|
|
return []
|
|
return find(config, store)
|
|
|
|
return replace(stage, finder=during_a_pull)
|
|
|
|
|
|
STAGES: dict[str, Callable[[str], Task]] = {
|
|
"pull": lambda cascade: pull.pull_task(),
|
|
"extract": _extract,
|
|
"embed": lambda cascade: Task(kind="embed", finder=_once, label="Embedding roles"),
|
|
"score": lambda cascade: Task(kind="score", finder=_once,
|
|
label="Scoring roles for every agent"),
|
|
"prep": lambda cascade: Task(kind="prep", finder=_once,
|
|
label="Mapping the roles' host words"),
|
|
"summarise": _summarise,
|
|
"judge": _judge,
|
|
"fetch_text": lambda cascade: fetch_text.fetch_text_task(),
|
|
}
|
|
|
|
|
|
def task(name: str, *, cascade: str = "", at: int = 0) -> Task:
|
|
"""One stage as a `Task`. With `cascade` its final job carries `step`,
|
|
and the stage after position `at` follows; without, it stops when its
|
|
batches do."""
|
|
return replace(STAGES[name](cascade), name=name,
|
|
callback=STEP if cascade else "",
|
|
carries={"cascade": cascade, "task": name, "at": at})
|
|
|
|
|
|
def follows(cascade: str, at: int) -> str:
|
|
"""The stage after position `at`, or "" at the end or off the cascade."""
|
|
tasks = CASCADES.get(cascade, ())
|
|
return tasks[at + 1] if 0 <= at < len(tasks) - 1 else ""
|
|
|
|
|
|
def step(config: Any, store: Any, tally: Tally, payload: dict[str, Any], *,
|
|
context: Any) -> dict[str, Any]:
|
|
"""The callback: make the next task. A killed stage is skipped and the
|
|
chain goes on, as in newsdesk's cascade."""
|
|
name, done = payload.get("cascade", ""), payload.get("task", "")
|
|
at = int(payload.get("at", -1))
|
|
tasks = CASCADES.get(name, ())
|
|
if done == "fetch_text" and not fetch_text.filled(tally) \
|
|
and tasks[at + 1:at + 1 + len(REFILL)] == REFILL:
|
|
at += len(REFILL)
|
|
following = follows(name, at)
|
|
if following:
|
|
context.task(task(following, cascade=name, at=at + 1))
|
|
out: dict[str, Any] = {"task": done, "next": following}
|
|
if tally.killed:
|
|
out["skipped"] = "cancelled"
|
|
return out
|
|
|
|
|
|
def chain(name: str) -> Task:
|
|
"""The first stage of cascade `name`, carrying `step`, so the rest follow."""
|
|
return task(CASCADES[name][0], cascade=name, at=0)
|
|
|
|
|
|
def pull_chain(run: str) -> Task:
|
|
"""The pull cascade for a run `pull.claim_run` has claimed: its finder
|
|
reads under that marker rather than claiming one of its own."""
|
|
return replace(chain("pull"), finder=pull.pull_task(run).finder)
|
|
|
|
|
|
def already_running(since: str) -> Notice:
|
|
"""The notice for a pull refused because another is in flight."""
|
|
return Notice(
|
|
code="pull.running", level="info", kind="expected", key="pull",
|
|
text=f"Not started: {pull.ALREADY_RUNNING}"
|
|
+ (f" (since {since[:19].replace('T', ' ')} UTC)." if since else "."),
|
|
where="app.jobs.pull")
|
|
|
|
|
|
@job("pull", "Pulling every board", resource="http")
|
|
def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
|
|
"""Claim a pull run and ask for the pull cascade once this job is done.
|
|
Its own work is nil.
|
|
|
|
A pull whose claim is refused, because another run is still unsettled,
|
|
queues nothing, embed to summarise included, and says so in its result
|
|
and a notice: a second cascade would fetch every board again and could
|
|
send the same roles to the model twice.
|
|
|
|
Under `NULL` there is no queue to take the chain, so this raises rather
|
|
than report a pull that never happened; `app job pull` claims the run and
|
|
starts the chain on a queue of its own instead of calling this. Anything
|
|
that raises once the run is claimed, building the chain or handing it
|
|
over, releases the claim first, or the next pull would wait
|
|
`pull.ABANDONED` for a run that never started.
|
|
"""
|
|
run = pull.claim_run(store)
|
|
if not run:
|
|
since = pull.running(store)
|
|
context.notice(already_running(since))
|
|
return {"skipped": pull.ALREADY_RUNNING, "running": since}
|
|
try:
|
|
started = context.task(pull_chain(run))
|
|
except Exception:
|
|
pull.release_run(store, run)
|
|
raise
|
|
if not started:
|
|
pull.release_run(store, run)
|
|
raise RuntimeError("no queue to start the pull cascade on")
|
|
return {"started": " > ".join(CASCADES["pull"]), "run": run}
|
|
|
|
|
|
@job("fetch_text", fetch_text.LABEL, solo=True, resource="http")
|
|
def run_fetch_text(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
|
|
"""Ask for the fetch text cascade, the pull's chain from fetch text on.
|
|
Its own work is nil, as the `pull` job's. Under `NULL` there is no queue
|
|
to take it."""
|
|
if not context.task(chain("fetch_text")):
|
|
raise RuntimeError("no queue to start the fetch text cascade on")
|
|
return {"started": " > ".join(CASCADES["fetch_text"])}
|
|
|
|
|
|
def fetch_text_after(queue: Any, score_job: Any) -> Any:
|
|
"""Queue the `fetch_text` job behind `score_job`, the rescore that
|
|
follows a change to the ranking, however that score ends. One per score
|
|
job (`fetch_text:after:<id>`), so ten changes answered by one score are
|
|
one cascade. None without a score job, or when the queue refuses."""
|
|
if score_job is None:
|
|
return None
|
|
return queue.enqueue("fetch_text", depends_on=[score_job.id], after="done",
|
|
dedup_key=f"fetch_text:after:{score_job.id}")
|
|
|
|
|
|
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
|
|
#: each, on its own. `pull` is not a task here, since it is a job: a name that
|
|
#: is both would be listed twice by `/api/jobs/kinds`.
|
|
for _name in ("extract", "embed", "score", "summarise", "judge", "prep"):
|
|
TASKS[_name] = partial(task, _name)
|
|
|
|
|
|
# ---------------------------------------------------------------- schedules
|
|
|
|
# The default, next to the job it fires. A starting point rather than a
|
|
# setting: a cadence edited or switched off under Runs survives
|
|
# every restart (see pipeline/schedule.py).
|
|
declare("pull", "pull", "every 6h",
|
|
label="Pull every board, then map, embed, score, fetch the text of the roles in the "
|
|
"cut, extract, rescore, judge against the CVs and summarise")
|