"""The shapes the queue stores and serves: a job, a run, a snapshot, a notice. One set of models, not two — there is no ORM here, so the row the table returns and the object the API serves are the same declaration. `Model` is the base an app's own shapes are welcome to extend as well, because the one thing it does (tolerate a `sqlite3.Row`, ignore columns nobody declared) is wanted by every table anyone will ever `SELECT *` from. Pydantic is the package's only dependency outside the standard library, and this is the file that wants it. Everything else here would run without it. """ from __future__ import annotations import json from datetime import UTC, datetime from typing import Annotated, Any, Literal, Self from pydantic import (BaseModel, BeforeValidator, ConfigDict, computed_field) #: A column that holds JSON. SQLite hands back the text; everything above the #: storage layer wants the dict. Decoded here so neither side has to remember, #: and tolerant of a malformed value because a job whose payload failed to #: parse should still be listable and cancellable rather than un-readable. JsonDict = Annotated[ dict[str, Any], BeforeValidator(lambda v: v if isinstance(v, dict) else _loads(v)), ] def _loads(value: Any) -> dict[str, Any]: try: parsed = json.loads(value or "{}") except (TypeError, ValueError): return {} return parsed if isinstance(parsed, dict) else {} #: The same for a column holding a JSON array of job ids. Tolerant for the same #: reason: a job whose dependency list will not parse has already been let #: through by the queue, which reads the column in SQL — so refusing to build #: the model here would only hide it from the list it is holding up. JsonIds = Annotated[ list[int], BeforeValidator(lambda v: v if isinstance(v, list) else _ids(v)), ] def _ids(value: Any) -> list[int]: try: parsed = json.loads(value or "[]") except (TypeError, ValueError): return [] return [int(i) for i in parsed] if isinstance(parsed, list) else [] class Model(BaseModel): """Base: tolerate sqlite3.Row and reject fields nobody declared.""" model_config = ConfigDict(from_attributes=True, extra="forbid") @classmethod def from_row(cls, row: Any) -> Self: """Build one from a database row, ignoring columns nobody declared. `SELECT *` into a model that forbids extras means the next `ALTER TABLE` is a 500 in whatever route reads that table — raised a long way from the migration that caused it, over a column the API was never going to serve anyway. A storage column the API has not heard of is not something to hand out; it is something to ignore until someone declares a field for it. The half of `extra="forbid"` worth keeping still works. A column a model *requires* going missing — a rename, a typo — is a validation error exactly as before. Only additions go quiet. """ return cls.model_validate( {k: v for k, v in dict(row).items() if k in cls.model_fields} ) # ------------------------------------------------------------------- jobs #: The five states a job can be in. `queued` and `running` are live; the other #: three are terminal and a job never leaves them. JobState = Literal["queued", "running", "done", "failed", "cancelled"] #: Everything that is over, however it ended. Used to decide what the dedup #: index constrains, what the UI greys out, and when to stop pushing updates. TERMINAL: frozenset[str] = frozenset({"done", "failed", "cancelled"}) class Job(Model): """One unit of queued work, as stored. `payload` is the handler's keyword arguments and `result` is what it returned — both JSON in SQLite, both dicts everywhere else, translated once here rather than at each call site. `done`/`total` are deliberately untyped as units. A pull counts sources, a transcription counts minutes of audio; the UI renders "3/12" and a bar and does not need to know which. """ id: int kind: str label: str = "" payload: JsonDict = {} state: JobState = "queued" done: int = 0 total: int = 0 message: str = "" result: JsonDict = {} error: str = "" dedup_key: str = "" #: Which jobs this one waits for, and whether it needs them to have #: succeeded (`ok`) or merely to be over (`done`). depends_on: JsonIds = [] after: str = "ok" #: Which kind of worker runs it: `llm`, `http`, `cpu`. resource: str = "cpu" #: Which cascade this belongs to, and which of its tasks. A job queued on #: its own is its own cascade and its own task. Both name a row in `runs` #: — a series of their own, not this one's. See `Run`. cascade_id: int = 0 task_id: int = 0 cancel_requested: bool = False #: How many times this has been started, counted by `claim`. One after the #: first run, whether that run ended well, badly, or with the worker dying #: under it. attempt: int = 0 #: The bound, counting the first attempt — so `1` is the default and means #: no second try. Carried on the row rather than looked up from the job #: kind, because the attempt that decides is the one after a restart, in a #: worker that may be running a different version of the code. max_attempts: int = 1 #: The wait before the next attempt, and what multiplies it each time #: round. `1.0` is a flat wait. retry_delay: float = 0.0 retry_backoff: float = 1.0 #: Not before this. Empty means now, which is why it is `''` rather than #: NULL: the column is compared as text and `''` sorts before every stamp, #: so "is it due?" is one comparison with no special case in it. run_after: str = "" created_at: str started_at: str | None = None finished_at: str | None = None @property def live(self) -> bool: return self.state not in TERMINAL @computed_field # type: ignore[prop-decorator] @property def waiting(self) -> bool: """Queued, but not until its next-attempt time comes round. Serialised, unlike its two neighbours, because it is the one of the three a client cannot work out for itself. `live` is a state, and `exhausted` is `attempt`, `max_attempts` and `state`, all of which go out on the row; this one is `run_after` against *now*, and the only clock worth asking is the one the stamp was written by. A browser four minutes fast would otherwise draw a job as due while the queue goes on passing over it. `run_after` travels too and still has to: this says *that* it is waiting and the stamp says until when, which is the difference between greying a row out and counting down on it. The resting state of a failure that is going to be tried again, and the thing that makes it different from a failure: it is still `queued`, so nothing is running it, nothing holds a slot for it, and whatever waits on it goes on waiting. Compared as text, which is safe because `run_after` is written in the same UTC ISO form as every other stamp here — see `table.py`. """ return self.state == "queued" and self.run_after > datetime.now(UTC).isoformat() @property def exhausted(self) -> bool: """Failed on the last attempt it was allowed, rather than the only one. Worth telling apart, because the two want opposite things done about them. A job that failed with attempts still to come is not this — it is back in the queue and `waiting`. A job that failed at `max_attempts=1` is not this either: nobody ever said it should be retried, so nothing was used up. This is the third case, the one where the policy ran its course and the work still did not land, and it is the only one where trying again is a decision a person has to make. """ return self.state == "failed" and self.attempt >= self.max_attempts > 1 class Run(Model): """A cascade or a task: an identity, and whether it was killed. No state and no progress: the jobs carrying this id are the record of what happened, and a second copy of it here could only go out of step. What it holds is what cannot be worked out from them — that this id is a cascade rather than a task, what it was called, when it began, and the one fact its jobs cannot tell you, which is that somebody stopped it. """ id: int kind: str name: str = "" #: What to call it on screen. A task is not a job, so this is the only #: place it can be written down. label: str = "" created_at: str #: When it was killed, or "" for a run nobody stopped. A cascade exists #: nowhere but this row, so this is where its Kill has to live. cancelled_at: str = "" class Schedule(Model): """A standing instruction: enqueue this kind, with this payload, this often. A row rather than a timer, for the reason every other waiting thing here is a row: nothing is alive between runs, so a process restarted at 05:59 still fires the 06:00 one, and a second worker on the same file reads the same schedule rather than keeping a second copy of it. `name` is unique and is the handle declared defaults are matched on — see `pipeline/schedule.py`. Everything else about a row is the user's to edit, the cadence and whether it is on at all included, and redeclaring a default never overwrites those. `next_run` and `last_run` are UTC ISO stamps, like every other time in this table, and are compared as text — so a cadence worded in a local timezone still sorts correctly against `now` and reads the same from any process. Empty means never, which for `next_run` also means not due, and is why it is `''` rather than NULL. """ id: int name: str kind: str #: What to call it on screen, from `declare(label=...)`. Empty means the #: job kind's own label, which is what the queued job then carries. label: str = "" payload: JsonDict = {} cadence: str enabled: bool = True next_run: str = "" last_run: str = "" #: The job the last firing queued, and so the answer to "is one still #: going?" — which is what stops a slow job from piling up behind itself. last_job_id: int = 0 #: How that job ended, or `queued`/`running` while it has not. Joined on in #: `Jobs.schedules` rather than fetched per row: it is one column on a query #: that is already running, and the time is no use on screen without it. last_state: str = "" created_at: str class TaskProgress(Model): """One task of a live cascade, as a tally of the jobs carrying its id. Counts and not the jobs themselves. A task is a body fetch shared over twenty-three jobs and there is nothing to tell them apart — the ones still going are in `JobsSnapshot.running` with their own `done`/`total`, and the ones that landed are a number, because a landed job is a full box and full boxes are all alike. Derived on read, never stored, for the reason `runs` holds no state: the job rows are the record of what happened and a second copy of the tally could only go out of step with them. """ #: The `runs` row this task is, and the id `/api/tasks/{id}/cancel` takes. id: int #: What the task is — `body`, `tag`. Its own name, minted with the `runs` #: row before any of its jobs existed, and how a cascade's list of tasks is #: matched against it. name: str = "" #: How to say that on screen: "Fetching what the feeds left out". The #: task's own, off the `runs` row, because a task is not a job and has no #: handler to carry one. label: str = "" done: int = 0 failed: int = 0 cancelled: int = 0 running: int = 0 queued: int = 0 class CascadeProgress(Model): """A cascade that is still going, and how far each of its tasks has got. Only the tasks that exist. A cascade splits at every boundary — how many jobs summarising becomes is a query that is only meaningful once the bodies have landed — so the ones ahead have no jobs, no ids and nothing to count. Which of them are coming is the app's own business, in its cascade module and served beside this; here there is nothing to say about them. """ #: The `runs` row, and the id `/api/cascades/{id}/cancel` takes. id: int #: Which cascade it is a run of — `pull`. For a job or a task queued on #: its own, which is a cascade of one, that name and a word saying it #: stands alone: `pull on its own`. The two are different runs and the #: name is where they are told apart, so it is also what the strip shows. name: str = "" #: In the order they began, which for a cascade is the order they run. tasks: list[TaskProgress] = [] class JobsSnapshot(Model): """What the queue looks like right now — the whole WebSocket payload. Sent on connect and after every state change, rather than diffed. The whole thing is a few hundred bytes and one shape that is always complete is worth more here than the bytes a patch protocol would save. """ #: Everything under way. A list because the queue has several workers, so #: "the running job" stopped being a single thing. running: list[Job] = [] queued: list[Job] = [] recent: list[Job] = [] #: The cascades any of that belongs to, tallied by task. Here rather than #: behind a URL of its own because a tally of job states changes exactly #: when this message does: asked for separately it would arrive a beat #: later and draw a bar that disagreed with the list above it. And it #: cannot be counted from that list — `recent` is the last thirty jobs, so #: by the time tagging starts the body batches that landed have dropped #: off the end and a bar drawn from it would count backwards. cascades: list[CascadeProgress] = [] # ---------------------------------------------------------------- notices #: How loud a `Notice` is, and the four colours a reader renders them in. #: `error` is red — "you have to do something about this". `warning` is amber, #: the same amber a work panel uses for a row that wants you — "this went #: wrong and carried on". `info` is grey, worth saying once. `good` is green #: and is the odd one out: nothing in `notices.py` produces one, because that #: module is about failure — it exists for a UI to confirm something worked #: without inventing a second mechanism to say so. NoticeLevel = Literal["info", "good", "warning", "error"] #: Whether a failure is the world's or ours. See `Notice.kind`. NoticeKind = Literal["expected", "bug"] class Notice(Model): """Something went wrong that you, not the log, need to know about *now*. Broadcast to whatever browsers are open, and written to the `notices` table so that one opened afterwards can read it — see `NoticeEntry`, and `pipeline/notices.py` for why it is kept after all. So the division of labour is: this "the tagger stopped, your claude login expired" — on screen, the moment it happens, with enough to tell you whether to shrug, fix something, or go and read the log. the table the same sentence, findable once it has faded, with how often it has been said. Cleared by you, or by a fortnight of not being said again. the log everything, with the traceback, still there tomorrow. Which means a notice does not have to be complete. It has to be *enough to orient you*, and `where` points at the rest. """ #: Stable dotted identifier — `agent.auth`, `source.failed`. What the UI #: dedupes a burst on, and what a test asserts against, so the prose in #: `text` can be reworded without breaking either. code: str level: NoticeLevel = "warning" #: The distinction worth drawing when something breaks: is this the world #: misbehaving in a way it is entitled to, or is this ours? #: #: expected an expired login, a feed returning 403, a rate limit. Not a #: defect. Either you fix it or you wait, and either way there #: is nothing to debug. #: bug an exception we did not anticipate. Something to go and read #: the log about, and probably to fix in the code. #: #: Getting this wrong in either direction is expensive — hunting a bug that #: was a dead feed, or shrugging at a real one — so it is a field rather #: than something to infer from the wording. kind: NoticeKind = "expected" text: str #: What to do about it, when there is something: `claude auth login`. Empty #: when there isn't, which is worth saying honestly rather than padding. fix: str = "" #: The raw error. Shown folded — it is the bit you would paste into a bug #: report, and the bit that tells you which log line to look for. detail: str = "" #: Where to look for the whole story. Always the log; this names the part. where: str = "" #: What this is about within its code — a source name, a job kind. Lets the #: UI collapse a burst of the same failure into one line with a count. key: str = "" class NoticeEntry(Notice): """A notice as the log holds it: one row per `(code, key)`. What `Jobs.record_notice` hands back and what `/api/notices` lists. The same failure said again is this row with a higher `count` and a later `last_at`, never a second row — which is also how a client knows whether to interrupt anyone: `count == 1` is news, anything above it is a repeat of something already said and still sitting in the log. """ id: int first_at: str last_at: str count: int = 1 __all__ = [ "TERMINAL", "CascadeProgress", "Job", "JobState", "JobsSnapshot", "JsonDict", "JsonIds", "Model", "Notice", "NoticeEntry", "NoticeKind", "NoticeLevel", "Run", "Schedule", "TaskProgress", ]