Bring in pipeline, auth, deploy and docs from the boilerplate
roledesk is built on the boilerplate from here on: pipeline 2.0 (queue and scheduler), auth (one password, sessions, passkeys), the deploy kit and its docs, plus the coupling and packaging tests. Downstream copies, unedited. Refreshed from boilerplate/d56949f7d0f6c0bfe5745478e09007bfad67ae87 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1f436104f3
commit
9d666d0937
76 files changed
+15235
No files matched your search
@@ -0,0 +1,432 @@
|
||||
"""The shapes the queue stores and serves: a job, a run, a snapshot, a notice.
|
||||
|
||||
One set of models, not two — there is no ORM here, so the row the table
|
||||
returns and the object the API serves are the same declaration. `Model` is
|
||||
the base an app's own shapes are welcome to extend as well, because the one
|
||||
thing it does (tolerate a `sqlite3.Row`, ignore columns nobody declared) is
|
||||
wanted by every table anyone will ever `SELECT *` from.
|
||||
|
||||
Pydantic is the package's only dependency outside the standard library, and
|
||||
this is the file that wants it. Everything else here would run without it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Annotated, Any, Literal, Self
|
||||
|
||||
from pydantic import (BaseModel, BeforeValidator, ConfigDict, computed_field)
|
||||
|
||||
#: A column that holds JSON. SQLite hands back the text; everything above the
|
||||
#: storage layer wants the dict. Decoded here so neither side has to remember,
|
||||
#: and tolerant of a malformed value because a job whose payload failed to
|
||||
#: parse should still be listable and cancellable rather than un-readable.
|
||||
JsonDict = Annotated[
|
||||
dict[str, Any],
|
||||
BeforeValidator(lambda v: v if isinstance(v, dict) else _loads(v)),
|
||||
]
|
||||
|
||||
|
||||
def _loads(value: Any) -> dict[str, Any]:
|
||||
try:
|
||||
parsed = json.loads(value or "{}")
|
||||
except (TypeError, ValueError):
|
||||
return {}
|
||||
return parsed if isinstance(parsed, dict) else {}
|
||||
|
||||
|
||||
#: The same for a column holding a JSON array of job ids. Tolerant for the same
|
||||
#: reason: a job whose dependency list will not parse has already been let
|
||||
#: through by the queue, which reads the column in SQL — so refusing to build
|
||||
#: the model here would only hide it from the list it is holding up.
|
||||
JsonIds = Annotated[
|
||||
list[int],
|
||||
BeforeValidator(lambda v: v if isinstance(v, list) else _ids(v)),
|
||||
]
|
||||
|
||||
|
||||
def _ids(value: Any) -> list[int]:
|
||||
try:
|
||||
parsed = json.loads(value or "[]")
|
||||
except (TypeError, ValueError):
|
||||
return []
|
||||
return [int(i) for i in parsed] if isinstance(parsed, list) else []
|
||||
|
||||
|
||||
class Model(BaseModel):
|
||||
"""Base: tolerate sqlite3.Row and reject fields nobody declared."""
|
||||
|
||||
model_config = ConfigDict(from_attributes=True, extra="forbid")
|
||||
|
||||
@classmethod
|
||||
def from_row(cls, row: Any) -> Self:
|
||||
"""Build one from a database row, ignoring columns nobody declared.
|
||||
|
||||
`SELECT *` into a model that forbids extras means the next
|
||||
`ALTER TABLE` is a 500 in whatever route reads that table — raised a
|
||||
long way from the migration that caused it, over a column the API was
|
||||
never going to serve anyway. A storage column the API has not heard of
|
||||
is not something to hand out; it is something to ignore until someone
|
||||
declares a field for it.
|
||||
|
||||
The half of `extra="forbid"` worth keeping still works. A column a
|
||||
model *requires* going missing — a rename, a typo — is a validation
|
||||
error exactly as before. Only additions go quiet.
|
||||
"""
|
||||
return cls.model_validate(
|
||||
{k: v for k, v in dict(row).items() if k in cls.model_fields}
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- jobs
|
||||
|
||||
#: The five states a job can be in. `queued` and `running` are live; the other
|
||||
#: three are terminal and a job never leaves them.
|
||||
JobState = Literal["queued", "running", "done", "failed", "cancelled"]
|
||||
|
||||
#: Everything that is over, however it ended. Used to decide what the dedup
|
||||
#: index constrains, what the UI greys out, and when to stop pushing updates.
|
||||
TERMINAL: frozenset[str] = frozenset({"done", "failed", "cancelled"})
|
||||
|
||||
|
||||
class Job(Model):
|
||||
"""One unit of queued work, as stored.
|
||||
|
||||
`payload` is the handler's keyword arguments and `result` is what it
|
||||
returned — both JSON in SQLite, both dicts everywhere else, translated
|
||||
once here rather than at each call site.
|
||||
|
||||
`done`/`total` are deliberately untyped as units. A pull counts sources, a
|
||||
transcription counts minutes of audio; the UI renders "3/12" and a bar and
|
||||
does not need to know which.
|
||||
"""
|
||||
|
||||
id: int
|
||||
kind: str
|
||||
label: str = ""
|
||||
payload: JsonDict = {}
|
||||
state: JobState = "queued"
|
||||
done: int = 0
|
||||
total: int = 0
|
||||
message: str = ""
|
||||
result: JsonDict = {}
|
||||
error: str = ""
|
||||
dedup_key: str = ""
|
||||
#: Which jobs this one waits for, and whether it needs them to have
|
||||
#: succeeded (`ok`) or merely to be over (`done`).
|
||||
depends_on: JsonIds = []
|
||||
after: str = "ok"
|
||||
#: Which kind of worker runs it: `llm`, `http`, `cpu`.
|
||||
resource: str = "cpu"
|
||||
#: Which cascade this belongs to, and which of its tasks. A job queued on
|
||||
#: its own is its own cascade and its own task. Both name a row in `runs`
|
||||
#: — a series of their own, not this one's. See `Run`.
|
||||
cascade_id: int = 0
|
||||
task_id: int = 0
|
||||
cancel_requested: bool = False
|
||||
#: How many times this has been started, counted by `claim`. One after the
|
||||
#: first run, whether that run ended well, badly, or with the worker dying
|
||||
#: under it.
|
||||
attempt: int = 0
|
||||
#: The bound, counting the first attempt — so `1` is the default and means
|
||||
#: no second try. Carried on the row rather than looked up from the job
|
||||
#: kind, because the attempt that decides is the one after a restart, in a
|
||||
#: worker that may be running a different version of the code.
|
||||
max_attempts: int = 1
|
||||
#: The wait before the next attempt, and what multiplies it each time
|
||||
#: round. `1.0` is a flat wait.
|
||||
retry_delay: float = 0.0
|
||||
retry_backoff: float = 1.0
|
||||
#: Not before this. Empty means now, which is why it is `''` rather than
|
||||
#: NULL: the column is compared as text and `''` sorts before every stamp,
|
||||
#: so "is it due?" is one comparison with no special case in it.
|
||||
run_after: str = ""
|
||||
created_at: str
|
||||
started_at: str | None = None
|
||||
finished_at: str | None = None
|
||||
|
||||
@property
|
||||
def live(self) -> bool:
|
||||
return self.state not in TERMINAL
|
||||
|
||||
@computed_field # type: ignore[prop-decorator]
|
||||
@property
|
||||
def waiting(self) -> bool:
|
||||
"""Queued, but not until its next-attempt time comes round.
|
||||
|
||||
Serialised, unlike its two neighbours, because it is the one of the
|
||||
three a client cannot work out for itself. `live` is a state, and
|
||||
`exhausted` is `attempt`, `max_attempts` and `state`, all of which go
|
||||
out on the row; this one is `run_after` against *now*, and the only
|
||||
clock worth asking is the one the stamp was written by. A browser four
|
||||
minutes fast would otherwise draw a job as due while the queue goes on
|
||||
passing over it.
|
||||
|
||||
`run_after` travels too and still has to: this says *that* it is
|
||||
waiting and the stamp says until when, which is the difference between
|
||||
greying a row out and counting down on it.
|
||||
|
||||
The resting state of a failure that is going to be tried again, and the
|
||||
thing that makes it different from a failure: it is still `queued`, so
|
||||
nothing is running it, nothing holds a slot for it, and whatever waits
|
||||
on it goes on waiting. Compared as text, which is safe because
|
||||
`run_after` is written in the same UTC ISO form as every other stamp
|
||||
here — see `table.py`.
|
||||
"""
|
||||
return self.state == "queued" and self.run_after > datetime.now(UTC).isoformat()
|
||||
|
||||
@property
|
||||
def exhausted(self) -> bool:
|
||||
"""Failed on the last attempt it was allowed, rather than the only one.
|
||||
|
||||
Worth telling apart, because the two want opposite things done about
|
||||
them. A job that failed with attempts still to come is not this — it is
|
||||
back in the queue and `waiting`. A job that failed at `max_attempts=1`
|
||||
is not this either: nobody ever said it should be retried, so nothing
|
||||
was used up. This is the third case, the one where the policy ran its
|
||||
course and the work still did not land, and it is the only one where
|
||||
trying again is a decision a person has to make.
|
||||
"""
|
||||
return self.state == "failed" and self.attempt >= self.max_attempts > 1
|
||||
|
||||
|
||||
class Run(Model):
|
||||
"""A cascade or a task: an identity, and whether it was killed.
|
||||
|
||||
No state and no progress: the jobs carrying this id are the record of what
|
||||
happened, and a second copy of it here could only go out of step. What it
|
||||
holds is what cannot be worked out from them — that this id is a cascade
|
||||
rather than a task, what it was called, when it began, and the one fact its
|
||||
jobs cannot tell you, which is that somebody stopped it.
|
||||
"""
|
||||
|
||||
id: int
|
||||
kind: str
|
||||
name: str = ""
|
||||
#: What to call it on screen. A task is not a job, so this is the only
|
||||
#: place it can be written down.
|
||||
label: str = ""
|
||||
created_at: str
|
||||
#: When it was killed, or "" for a run nobody stopped. A cascade exists
|
||||
#: nowhere but this row, so this is where its Kill has to live.
|
||||
cancelled_at: str = ""
|
||||
|
||||
|
||||
class Schedule(Model):
|
||||
"""A standing instruction: enqueue this kind, with this payload, this often.
|
||||
|
||||
A row rather than a timer, for the reason every other waiting thing here is
|
||||
a row: nothing is alive between runs, so a process restarted at 05:59 still
|
||||
fires the 06:00 one, and a second worker on the same file reads the same
|
||||
schedule rather than keeping a second copy of it.
|
||||
|
||||
`name` is unique and is the handle declared defaults are matched on — see
|
||||
`pipeline/schedule.py`. Everything else about a row is the user's to edit,
|
||||
the cadence and whether it is on at all included, and redeclaring a default
|
||||
never overwrites those.
|
||||
|
||||
`next_run` and `last_run` are UTC ISO stamps, like every other time in this
|
||||
table, and are compared as text — so a cadence worded in a local timezone
|
||||
still sorts correctly against `now` and reads the same from any process.
|
||||
Empty means never, which for `next_run` also means not due, and is why it
|
||||
is `''` rather than NULL.
|
||||
"""
|
||||
|
||||
id: int
|
||||
name: str
|
||||
kind: str
|
||||
#: What to call it on screen, from `declare(label=...)`. Empty means the
|
||||
#: job kind's own label, which is what the queued job then carries.
|
||||
label: str = ""
|
||||
payload: JsonDict = {}
|
||||
cadence: str
|
||||
enabled: bool = True
|
||||
next_run: str = ""
|
||||
last_run: str = ""
|
||||
#: The job the last firing queued, and so the answer to "is one still
|
||||
#: going?" — which is what stops a slow job from piling up behind itself.
|
||||
last_job_id: int = 0
|
||||
#: How that job ended, or `queued`/`running` while it has not. Joined on in
|
||||
#: `Jobs.schedules` rather than fetched per row: it is one column on a query
|
||||
#: that is already running, and the time is no use on screen without it.
|
||||
last_state: str = ""
|
||||
created_at: str
|
||||
|
||||
|
||||
class TaskProgress(Model):
|
||||
"""One task of a live cascade, as a tally of the jobs carrying its id.
|
||||
|
||||
Counts and not the jobs themselves. A task is a body fetch shared over
|
||||
twenty-three jobs and there is nothing to tell them apart — the ones still
|
||||
going are in `JobsSnapshot.running` with their own `done`/`total`, and the
|
||||
ones that landed are a number, because a landed job is a full box and full
|
||||
boxes are all alike.
|
||||
|
||||
Derived on read, never stored, for the reason `runs` holds no state: the
|
||||
job rows are the record of what happened and a second copy of the tally
|
||||
could only go out of step with them.
|
||||
"""
|
||||
|
||||
#: The `runs` row this task is, and the id `/api/tasks/{id}/cancel` takes.
|
||||
id: int
|
||||
#: What the task is — `body`, `tag`. Its own name, minted with the `runs`
|
||||
#: row before any of its jobs existed, and how a cascade's list of tasks is
|
||||
#: matched against it.
|
||||
name: str = ""
|
||||
#: How to say that on screen: "Fetching what the feeds left out". The
|
||||
#: task's own, off the `runs` row, because a task is not a job and has no
|
||||
#: handler to carry one.
|
||||
label: str = ""
|
||||
done: int = 0
|
||||
failed: int = 0
|
||||
cancelled: int = 0
|
||||
running: int = 0
|
||||
queued: int = 0
|
||||
|
||||
|
||||
class CascadeProgress(Model):
|
||||
"""A cascade that is still going, and how far each of its tasks has got.
|
||||
|
||||
Only the tasks that exist. A cascade splits at every boundary — how many
|
||||
jobs summarising becomes is a query that is only meaningful once the
|
||||
bodies have landed — so the ones ahead have no jobs, no ids and nothing to
|
||||
count. Which of them are coming is the app's own business, in its cascade
|
||||
module and served beside this; here there is nothing to say about them.
|
||||
"""
|
||||
|
||||
#: The `runs` row, and the id `/api/cascades/{id}/cancel` takes.
|
||||
id: int
|
||||
#: Which cascade it is a run of — `pull`. For a job or a task queued on
|
||||
#: its own, which is a cascade of one, that name and a word saying it
|
||||
#: stands alone: `pull on its own`. The two are different runs and the
|
||||
#: name is where they are told apart, so it is also what the strip shows.
|
||||
name: str = ""
|
||||
#: In the order they began, which for a cascade is the order they run.
|
||||
tasks: list[TaskProgress] = []
|
||||
|
||||
|
||||
class JobsSnapshot(Model):
|
||||
"""What the queue looks like right now — the whole WebSocket payload.
|
||||
|
||||
Sent on connect and after every state change, rather than diffed. The
|
||||
whole thing is a few hundred bytes and one shape that is always complete
|
||||
is worth more here than the bytes a patch protocol would save.
|
||||
"""
|
||||
|
||||
#: Everything under way. A list because the queue has several workers, so
|
||||
#: "the running job" stopped being a single thing.
|
||||
running: list[Job] = []
|
||||
queued: list[Job] = []
|
||||
recent: list[Job] = []
|
||||
#: The cascades any of that belongs to, tallied by task. Here rather than
|
||||
#: behind a URL of its own because a tally of job states changes exactly
|
||||
#: when this message does: asked for separately it would arrive a beat
|
||||
#: later and draw a bar that disagreed with the list above it. And it
|
||||
#: cannot be counted from that list — `recent` is the last thirty jobs, so
|
||||
#: by the time tagging starts the body batches that landed have dropped
|
||||
#: off the end and a bar drawn from it would count backwards.
|
||||
cascades: list[CascadeProgress] = []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- notices
|
||||
|
||||
#: How loud a `Notice` is, and the four colours a reader renders them in.
|
||||
#: `error` is red — "you have to do something about this". `warning` is amber,
|
||||
#: the same amber a work panel uses for a row that wants you — "this went
|
||||
#: wrong and carried on". `info` is grey, worth saying once. `good` is green
|
||||
#: and is the odd one out: nothing in `notices.py` produces one, because that
|
||||
#: module is about failure — it exists for a UI to confirm something worked
|
||||
#: without inventing a second mechanism to say so.
|
||||
NoticeLevel = Literal["info", "good", "warning", "error"]
|
||||
|
||||
#: Whether a failure is the world's or ours. See `Notice.kind`.
|
||||
NoticeKind = Literal["expected", "bug"]
|
||||
|
||||
|
||||
class Notice(Model):
|
||||
"""Something went wrong that you, not the log, need to know about *now*.
|
||||
|
||||
Broadcast to whatever browsers are open, and written to the `notices`
|
||||
table so that one opened afterwards can read it — see `NoticeEntry`, and
|
||||
`pipeline/notices.py` for why it is kept after all.
|
||||
|
||||
So the division of labour is:
|
||||
|
||||
this "the tagger stopped, your claude login expired" — on
|
||||
screen, the moment it happens, with enough to tell you
|
||||
whether to shrug, fix something, or go and read the log.
|
||||
the table the same sentence, findable once it has faded, with how
|
||||
often it has been said. Cleared by you, or by a fortnight
|
||||
of not being said again.
|
||||
the log everything, with the traceback, still there tomorrow.
|
||||
|
||||
Which means a notice does not have to be complete. It has to be *enough to
|
||||
orient you*, and `where` points at the rest.
|
||||
"""
|
||||
|
||||
#: Stable dotted identifier — `agent.auth`, `source.failed`. What the UI
|
||||
#: dedupes a burst on, and what a test asserts against, so the prose in
|
||||
#: `text` can be reworded without breaking either.
|
||||
code: str
|
||||
level: NoticeLevel = "warning"
|
||||
#: The distinction worth drawing when something breaks: is this the world
|
||||
#: misbehaving in a way it is entitled to, or is this ours?
|
||||
#:
|
||||
#: expected an expired login, a feed returning 403, a rate limit. Not a
|
||||
#: defect. Either you fix it or you wait, and either way there
|
||||
#: is nothing to debug.
|
||||
#: bug an exception we did not anticipate. Something to go and read
|
||||
#: the log about, and probably to fix in the code.
|
||||
#:
|
||||
#: Getting this wrong in either direction is expensive — hunting a bug that
|
||||
#: was a dead feed, or shrugging at a real one — so it is a field rather
|
||||
#: than something to infer from the wording.
|
||||
kind: NoticeKind = "expected"
|
||||
text: str
|
||||
#: What to do about it, when there is something: `claude auth login`. Empty
|
||||
#: when there isn't, which is worth saying honestly rather than padding.
|
||||
fix: str = ""
|
||||
#: The raw error. Shown folded — it is the bit you would paste into a bug
|
||||
#: report, and the bit that tells you which log line to look for.
|
||||
detail: str = ""
|
||||
#: Where to look for the whole story. Always the log; this names the part.
|
||||
where: str = ""
|
||||
#: What this is about within its code — a source name, a job kind. Lets the
|
||||
#: UI collapse a burst of the same failure into one line with a count.
|
||||
key: str = ""
|
||||
|
||||
|
||||
class NoticeEntry(Notice):
|
||||
"""A notice as the log holds it: one row per `(code, key)`.
|
||||
|
||||
What `Jobs.record_notice` hands back and what `/api/notices` lists. The
|
||||
same failure said again is this row with a higher `count` and a later
|
||||
`last_at`, never a second row — which is also how a client knows whether
|
||||
to interrupt anyone: `count == 1` is news, anything above it is a repeat
|
||||
of something already said and still sitting in the log.
|
||||
"""
|
||||
|
||||
id: int
|
||||
first_at: str
|
||||
last_at: str
|
||||
count: int = 1
|
||||
|
||||
|
||||
__all__ = [
|
||||
"TERMINAL",
|
||||
"CascadeProgress",
|
||||
"Job",
|
||||
"JobState",
|
||||
"JobsSnapshot",
|
||||
"JsonDict",
|
||||
"JsonIds",
|
||||
"Model",
|
||||
"Notice",
|
||||
"NoticeEntry",
|
||||
"NoticeKind",
|
||||
"NoticeLevel",
|
||||
"Run",
|
||||
"Schedule",
|
||||
"TaskProgress",
|
||||
]
|
||||
Reference in new issue
Block a user