roledesk is built on the boilerplate from here on: pipeline 2.0 (queue and scheduler), auth (one password, sessions, passkeys), the deploy kit and its docs, plus the coupling and packaging tests. Downstream copies, unedited. Refreshed from boilerplate/d56949f7d0f6c0bfe5745478e09007bfad67ae87 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
433 lines
18 KiB
Python
433 lines
18 KiB
Python
"""The shapes the queue stores and serves: a job, a run, a snapshot, a notice.
|
|
|
|
One set of models, not two — there is no ORM here, so the row the table
|
|
returns and the object the API serves are the same declaration. `Model` is
|
|
the base an app's own shapes are welcome to extend as well, because the one
|
|
thing it does (tolerate a `sqlite3.Row`, ignore columns nobody declared) is
|
|
wanted by every table anyone will ever `SELECT *` from.
|
|
|
|
Pydantic is the package's only dependency outside the standard library, and
|
|
this is the file that wants it. Everything else here would run without it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import UTC, datetime
|
|
from typing import Annotated, Any, Literal, Self
|
|
|
|
from pydantic import (BaseModel, BeforeValidator, ConfigDict, computed_field)
|
|
|
|
#: A column that holds JSON. SQLite hands back the text; everything above the
|
|
#: storage layer wants the dict. Decoded here so neither side has to remember,
|
|
#: and tolerant of a malformed value because a job whose payload failed to
|
|
#: parse should still be listable and cancellable rather than un-readable.
|
|
JsonDict = Annotated[
|
|
dict[str, Any],
|
|
BeforeValidator(lambda v: v if isinstance(v, dict) else _loads(v)),
|
|
]
|
|
|
|
|
|
def _loads(value: Any) -> dict[str, Any]:
|
|
try:
|
|
parsed = json.loads(value or "{}")
|
|
except (TypeError, ValueError):
|
|
return {}
|
|
return parsed if isinstance(parsed, dict) else {}
|
|
|
|
|
|
#: The same for a column holding a JSON array of job ids. Tolerant for the same
|
|
#: reason: a job whose dependency list will not parse has already been let
|
|
#: through by the queue, which reads the column in SQL — so refusing to build
|
|
#: the model here would only hide it from the list it is holding up.
|
|
JsonIds = Annotated[
|
|
list[int],
|
|
BeforeValidator(lambda v: v if isinstance(v, list) else _ids(v)),
|
|
]
|
|
|
|
|
|
def _ids(value: Any) -> list[int]:
|
|
try:
|
|
parsed = json.loads(value or "[]")
|
|
except (TypeError, ValueError):
|
|
return []
|
|
return [int(i) for i in parsed] if isinstance(parsed, list) else []
|
|
|
|
|
|
class Model(BaseModel):
|
|
"""Base: tolerate sqlite3.Row and reject fields nobody declared."""
|
|
|
|
model_config = ConfigDict(from_attributes=True, extra="forbid")
|
|
|
|
@classmethod
|
|
def from_row(cls, row: Any) -> Self:
|
|
"""Build one from a database row, ignoring columns nobody declared.
|
|
|
|
`SELECT *` into a model that forbids extras means the next
|
|
`ALTER TABLE` is a 500 in whatever route reads that table — raised a
|
|
long way from the migration that caused it, over a column the API was
|
|
never going to serve anyway. A storage column the API has not heard of
|
|
is not something to hand out; it is something to ignore until someone
|
|
declares a field for it.
|
|
|
|
The half of `extra="forbid"` worth keeping still works. A column a
|
|
model *requires* going missing — a rename, a typo — is a validation
|
|
error exactly as before. Only additions go quiet.
|
|
"""
|
|
return cls.model_validate(
|
|
{k: v for k, v in dict(row).items() if k in cls.model_fields}
|
|
)
|
|
|
|
|
|
# ------------------------------------------------------------------- jobs
|
|
|
|
#: The five states a job can be in. `queued` and `running` are live; the other
|
|
#: three are terminal and a job never leaves them.
|
|
JobState = Literal["queued", "running", "done", "failed", "cancelled"]
|
|
|
|
#: Everything that is over, however it ended. Used to decide what the dedup
|
|
#: index constrains, what the UI greys out, and when to stop pushing updates.
|
|
TERMINAL: frozenset[str] = frozenset({"done", "failed", "cancelled"})
|
|
|
|
|
|
class Job(Model):
|
|
"""One unit of queued work, as stored.
|
|
|
|
`payload` is the handler's keyword arguments and `result` is what it
|
|
returned — both JSON in SQLite, both dicts everywhere else, translated
|
|
once here rather than at each call site.
|
|
|
|
`done`/`total` are deliberately untyped as units. A pull counts sources, a
|
|
transcription counts minutes of audio; the UI renders "3/12" and a bar and
|
|
does not need to know which.
|
|
"""
|
|
|
|
id: int
|
|
kind: str
|
|
label: str = ""
|
|
payload: JsonDict = {}
|
|
state: JobState = "queued"
|
|
done: int = 0
|
|
total: int = 0
|
|
message: str = ""
|
|
result: JsonDict = {}
|
|
error: str = ""
|
|
dedup_key: str = ""
|
|
#: Which jobs this one waits for, and whether it needs them to have
|
|
#: succeeded (`ok`) or merely to be over (`done`).
|
|
depends_on: JsonIds = []
|
|
after: str = "ok"
|
|
#: Which kind of worker runs it: `llm`, `http`, `cpu`.
|
|
resource: str = "cpu"
|
|
#: Which cascade this belongs to, and which of its tasks. A job queued on
|
|
#: its own is its own cascade and its own task. Both name a row in `runs`
|
|
#: — a series of their own, not this one's. See `Run`.
|
|
cascade_id: int = 0
|
|
task_id: int = 0
|
|
cancel_requested: bool = False
|
|
#: How many times this has been started, counted by `claim`. One after the
|
|
#: first run, whether that run ended well, badly, or with the worker dying
|
|
#: under it.
|
|
attempt: int = 0
|
|
#: The bound, counting the first attempt — so `1` is the default and means
|
|
#: no second try. Carried on the row rather than looked up from the job
|
|
#: kind, because the attempt that decides is the one after a restart, in a
|
|
#: worker that may be running a different version of the code.
|
|
max_attempts: int = 1
|
|
#: The wait before the next attempt, and what multiplies it each time
|
|
#: round. `1.0` is a flat wait.
|
|
retry_delay: float = 0.0
|
|
retry_backoff: float = 1.0
|
|
#: Not before this. Empty means now, which is why it is `''` rather than
|
|
#: NULL: the column is compared as text and `''` sorts before every stamp,
|
|
#: so "is it due?" is one comparison with no special case in it.
|
|
run_after: str = ""
|
|
created_at: str
|
|
started_at: str | None = None
|
|
finished_at: str | None = None
|
|
|
|
@property
|
|
def live(self) -> bool:
|
|
return self.state not in TERMINAL
|
|
|
|
@computed_field # type: ignore[prop-decorator]
|
|
@property
|
|
def waiting(self) -> bool:
|
|
"""Queued, but not until its next-attempt time comes round.
|
|
|
|
Serialised, unlike its two neighbours, because it is the one of the
|
|
three a client cannot work out for itself. `live` is a state, and
|
|
`exhausted` is `attempt`, `max_attempts` and `state`, all of which go
|
|
out on the row; this one is `run_after` against *now*, and the only
|
|
clock worth asking is the one the stamp was written by. A browser four
|
|
minutes fast would otherwise draw a job as due while the queue goes on
|
|
passing over it.
|
|
|
|
`run_after` travels too and still has to: this says *that* it is
|
|
waiting and the stamp says until when, which is the difference between
|
|
greying a row out and counting down on it.
|
|
|
|
The resting state of a failure that is going to be tried again, and the
|
|
thing that makes it different from a failure: it is still `queued`, so
|
|
nothing is running it, nothing holds a slot for it, and whatever waits
|
|
on it goes on waiting. Compared as text, which is safe because
|
|
`run_after` is written in the same UTC ISO form as every other stamp
|
|
here — see `table.py`.
|
|
"""
|
|
return self.state == "queued" and self.run_after > datetime.now(UTC).isoformat()
|
|
|
|
@property
|
|
def exhausted(self) -> bool:
|
|
"""Failed on the last attempt it was allowed, rather than the only one.
|
|
|
|
Worth telling apart, because the two want opposite things done about
|
|
them. A job that failed with attempts still to come is not this — it is
|
|
back in the queue and `waiting`. A job that failed at `max_attempts=1`
|
|
is not this either: nobody ever said it should be retried, so nothing
|
|
was used up. This is the third case, the one where the policy ran its
|
|
course and the work still did not land, and it is the only one where
|
|
trying again is a decision a person has to make.
|
|
"""
|
|
return self.state == "failed" and self.attempt >= self.max_attempts > 1
|
|
|
|
|
|
class Run(Model):
|
|
"""A cascade or a task: an identity, and whether it was killed.
|
|
|
|
No state and no progress: the jobs carrying this id are the record of what
|
|
happened, and a second copy of it here could only go out of step. What it
|
|
holds is what cannot be worked out from them — that this id is a cascade
|
|
rather than a task, what it was called, when it began, and the one fact its
|
|
jobs cannot tell you, which is that somebody stopped it.
|
|
"""
|
|
|
|
id: int
|
|
kind: str
|
|
name: str = ""
|
|
#: What to call it on screen. A task is not a job, so this is the only
|
|
#: place it can be written down.
|
|
label: str = ""
|
|
created_at: str
|
|
#: When it was killed, or "" for a run nobody stopped. A cascade exists
|
|
#: nowhere but this row, so this is where its Kill has to live.
|
|
cancelled_at: str = ""
|
|
|
|
|
|
class Schedule(Model):
|
|
"""A standing instruction: enqueue this kind, with this payload, this often.
|
|
|
|
A row rather than a timer, for the reason every other waiting thing here is
|
|
a row: nothing is alive between runs, so a process restarted at 05:59 still
|
|
fires the 06:00 one, and a second worker on the same file reads the same
|
|
schedule rather than keeping a second copy of it.
|
|
|
|
`name` is unique and is the handle declared defaults are matched on — see
|
|
`pipeline/schedule.py`. Everything else about a row is the user's to edit,
|
|
the cadence and whether it is on at all included, and redeclaring a default
|
|
never overwrites those.
|
|
|
|
`next_run` and `last_run` are UTC ISO stamps, like every other time in this
|
|
table, and are compared as text — so a cadence worded in a local timezone
|
|
still sorts correctly against `now` and reads the same from any process.
|
|
Empty means never, which for `next_run` also means not due, and is why it
|
|
is `''` rather than NULL.
|
|
"""
|
|
|
|
id: int
|
|
name: str
|
|
kind: str
|
|
#: What to call it on screen, from `declare(label=...)`. Empty means the
|
|
#: job kind's own label, which is what the queued job then carries.
|
|
label: str = ""
|
|
payload: JsonDict = {}
|
|
cadence: str
|
|
enabled: bool = True
|
|
next_run: str = ""
|
|
last_run: str = ""
|
|
#: The job the last firing queued, and so the answer to "is one still
|
|
#: going?" — which is what stops a slow job from piling up behind itself.
|
|
last_job_id: int = 0
|
|
#: How that job ended, or `queued`/`running` while it has not. Joined on in
|
|
#: `Jobs.schedules` rather than fetched per row: it is one column on a query
|
|
#: that is already running, and the time is no use on screen without it.
|
|
last_state: str = ""
|
|
created_at: str
|
|
|
|
|
|
class TaskProgress(Model):
|
|
"""One task of a live cascade, as a tally of the jobs carrying its id.
|
|
|
|
Counts and not the jobs themselves. A task is a body fetch shared over
|
|
twenty-three jobs and there is nothing to tell them apart — the ones still
|
|
going are in `JobsSnapshot.running` with their own `done`/`total`, and the
|
|
ones that landed are a number, because a landed job is a full box and full
|
|
boxes are all alike.
|
|
|
|
Derived on read, never stored, for the reason `runs` holds no state: the
|
|
job rows are the record of what happened and a second copy of the tally
|
|
could only go out of step with them.
|
|
"""
|
|
|
|
#: The `runs` row this task is, and the id `/api/tasks/{id}/cancel` takes.
|
|
id: int
|
|
#: What the task is — `body`, `tag`. Its own name, minted with the `runs`
|
|
#: row before any of its jobs existed, and how a cascade's list of tasks is
|
|
#: matched against it.
|
|
name: str = ""
|
|
#: How to say that on screen: "Fetching what the feeds left out". The
|
|
#: task's own, off the `runs` row, because a task is not a job and has no
|
|
#: handler to carry one.
|
|
label: str = ""
|
|
done: int = 0
|
|
failed: int = 0
|
|
cancelled: int = 0
|
|
running: int = 0
|
|
queued: int = 0
|
|
|
|
|
|
class CascadeProgress(Model):
|
|
"""A cascade that is still going, and how far each of its tasks has got.
|
|
|
|
Only the tasks that exist. A cascade splits at every boundary — how many
|
|
jobs summarising becomes is a query that is only meaningful once the
|
|
bodies have landed — so the ones ahead have no jobs, no ids and nothing to
|
|
count. Which of them are coming is the app's own business, in its cascade
|
|
module and served beside this; here there is nothing to say about them.
|
|
"""
|
|
|
|
#: The `runs` row, and the id `/api/cascades/{id}/cancel` takes.
|
|
id: int
|
|
#: Which cascade it is a run of — `pull`. For a job or a task queued on
|
|
#: its own, which is a cascade of one, that name and a word saying it
|
|
#: stands alone: `pull on its own`. The two are different runs and the
|
|
#: name is where they are told apart, so it is also what the strip shows.
|
|
name: str = ""
|
|
#: In the order they began, which for a cascade is the order they run.
|
|
tasks: list[TaskProgress] = []
|
|
|
|
|
|
class JobsSnapshot(Model):
|
|
"""What the queue looks like right now — the whole WebSocket payload.
|
|
|
|
Sent on connect and after every state change, rather than diffed. The
|
|
whole thing is a few hundred bytes and one shape that is always complete
|
|
is worth more here than the bytes a patch protocol would save.
|
|
"""
|
|
|
|
#: Everything under way. A list because the queue has several workers, so
|
|
#: "the running job" stopped being a single thing.
|
|
running: list[Job] = []
|
|
queued: list[Job] = []
|
|
recent: list[Job] = []
|
|
#: The cascades any of that belongs to, tallied by task. Here rather than
|
|
#: behind a URL of its own because a tally of job states changes exactly
|
|
#: when this message does: asked for separately it would arrive a beat
|
|
#: later and draw a bar that disagreed with the list above it. And it
|
|
#: cannot be counted from that list — `recent` is the last thirty jobs, so
|
|
#: by the time tagging starts the body batches that landed have dropped
|
|
#: off the end and a bar drawn from it would count backwards.
|
|
cascades: list[CascadeProgress] = []
|
|
|
|
|
|
# ---------------------------------------------------------------- notices
|
|
|
|
#: How loud a `Notice` is, and the four colours a reader renders them in.
|
|
#: `error` is red — "you have to do something about this". `warning` is amber,
|
|
#: the same amber a work panel uses for a row that wants you — "this went
|
|
#: wrong and carried on". `info` is grey, worth saying once. `good` is green
|
|
#: and is the odd one out: nothing in `notices.py` produces one, because that
|
|
#: module is about failure — it exists for a UI to confirm something worked
|
|
#: without inventing a second mechanism to say so.
|
|
NoticeLevel = Literal["info", "good", "warning", "error"]
|
|
|
|
#: Whether a failure is the world's or ours. See `Notice.kind`.
|
|
NoticeKind = Literal["expected", "bug"]
|
|
|
|
|
|
class Notice(Model):
|
|
"""Something went wrong that you, not the log, need to know about *now*.
|
|
|
|
Broadcast to whatever browsers are open, and written to the `notices`
|
|
table so that one opened afterwards can read it — see `NoticeEntry`, and
|
|
`pipeline/notices.py` for why it is kept after all.
|
|
|
|
So the division of labour is:
|
|
|
|
this "the tagger stopped, your claude login expired" — on
|
|
screen, the moment it happens, with enough to tell you
|
|
whether to shrug, fix something, or go and read the log.
|
|
the table the same sentence, findable once it has faded, with how
|
|
often it has been said. Cleared by you, or by a fortnight
|
|
of not being said again.
|
|
the log everything, with the traceback, still there tomorrow.
|
|
|
|
Which means a notice does not have to be complete. It has to be *enough to
|
|
orient you*, and `where` points at the rest.
|
|
"""
|
|
|
|
#: Stable dotted identifier — `agent.auth`, `source.failed`. What the UI
|
|
#: dedupes a burst on, and what a test asserts against, so the prose in
|
|
#: `text` can be reworded without breaking either.
|
|
code: str
|
|
level: NoticeLevel = "warning"
|
|
#: The distinction worth drawing when something breaks: is this the world
|
|
#: misbehaving in a way it is entitled to, or is this ours?
|
|
#:
|
|
#: expected an expired login, a feed returning 403, a rate limit. Not a
|
|
#: defect. Either you fix it or you wait, and either way there
|
|
#: is nothing to debug.
|
|
#: bug an exception we did not anticipate. Something to go and read
|
|
#: the log about, and probably to fix in the code.
|
|
#:
|
|
#: Getting this wrong in either direction is expensive — hunting a bug that
|
|
#: was a dead feed, or shrugging at a real one — so it is a field rather
|
|
#: than something to infer from the wording.
|
|
kind: NoticeKind = "expected"
|
|
text: str
|
|
#: What to do about it, when there is something: `claude auth login`. Empty
|
|
#: when there isn't, which is worth saying honestly rather than padding.
|
|
fix: str = ""
|
|
#: The raw error. Shown folded — it is the bit you would paste into a bug
|
|
#: report, and the bit that tells you which log line to look for.
|
|
detail: str = ""
|
|
#: Where to look for the whole story. Always the log; this names the part.
|
|
where: str = ""
|
|
#: What this is about within its code — a source name, a job kind. Lets the
|
|
#: UI collapse a burst of the same failure into one line with a count.
|
|
key: str = ""
|
|
|
|
|
|
class NoticeEntry(Notice):
|
|
"""A notice as the log holds it: one row per `(code, key)`.
|
|
|
|
What `Jobs.record_notice` hands back and what `/api/notices` lists. The
|
|
same failure said again is this row with a higher `count` and a later
|
|
`last_at`, never a second row — which is also how a client knows whether
|
|
to interrupt anyone: `count == 1` is news, anything above it is a repeat
|
|
of something already said and still sitting in the log.
|
|
"""
|
|
|
|
id: int
|
|
first_at: str
|
|
last_at: str
|
|
count: int = 1
|
|
|
|
|
|
__all__ = [
|
|
"TERMINAL",
|
|
"CascadeProgress",
|
|
"Job",
|
|
"JobState",
|
|
"JobsSnapshot",
|
|
"JsonDict",
|
|
"JsonIds",
|
|
"Model",
|
|
"Notice",
|
|
"NoticeEntry",
|
|
"NoticeKind",
|
|
"NoticeLevel",
|
|
"Run",
|
|
"Schedule",
|
|
"TaskProgress",
|
|
]
|