"""The jobs table: the queue's own storage, on whatever SQLite file it is given. The pipeline used to keep its rows in the application's store, which meant the application's schema knew what a job was and the queue could not exist without it. Now the table is the pipeline's: it creates it on open, owns its shape, and asks nothing of the file beyond being SQLite. Point it at the app's database and the jobs sit next to the app's rows, one file to back up; point it at a file of its own and a worker runs with no app at all. Why a table rather than a list in memory is in `queue.py`. Why every method here is one statement is the same reason: the file is shared by whoever opens it, so a claim has to be atomic and a state change has to be a single `UPDATE` that a second process cannot half-see. The shape is `CREATE TABLE IF NOT EXISTS`, which leaves a file that already has the table exactly as it is — so a column added here would never reach one. `_reconcile` is how a column arrives instead: read the catalogue, add what is missing. Idempotent, so it runs on every open and there is no version to keep in step. There is deliberately no counter of our own — `PRAGMA user_version` on this file belongs to newsdesk (`newsdesk/storage/sql/schema.py`), and a second migration series numbering itself on the same pragma would fight it. """ from __future__ import annotations import json import logging import sqlite3 from collections.abc import Mapping, Sequence from contextlib import closing, contextmanager from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any from pipeline.models import ( CascadeProgress, Job, JobsSnapshot, Notice, NoticeEntry, Run, Schedule, TaskProgress, ) log = logging.getLogger(__name__) SCHEMA = """ CREATE TABLE IF NOT EXISTS jobs ( id INTEGER PRIMARY KEY, kind TEXT NOT NULL, label TEXT NOT NULL DEFAULT '', payload TEXT NOT NULL DEFAULT '{}', state TEXT NOT NULL DEFAULT 'queued', done INTEGER NOT NULL DEFAULT 0, total INTEGER NOT NULL DEFAULT 0, message TEXT NOT NULL DEFAULT '', result TEXT NOT NULL DEFAULT '{}', error TEXT NOT NULL DEFAULT '', dedup_key TEXT NOT NULL DEFAULT '', -- Which jobs this one waits for, and what it wants of them. `after='ok'` -- is one job needing another's work to have happened, and fails with it; -- `after='done'` waits only for them to be over, however they ended, and -- is what a job whose purpose is to report on them needs — success-only -- would mean the report never exists when there is something to report. -- An id that names nothing is satisfied: it finished before this was -- asked, or was never queued. depends_on TEXT NOT NULL DEFAULT '[]', after TEXT NOT NULL DEFAULT 'ok', -- What this job spends while it runs: `llm` for anything that asks a -- model, `http` for a fetch, `cpu` for work that wants the machine to -- itself. Named on the job rather than counted in the worker, because a -- second process opening this file has to be counted too — four model -- calls is a fact about the subscription, not about one program. resource TEXT NOT NULL DEFAULT 'cpu', -- Which cascade this job belongs to, and which of its tasks. Both name a -- row in `runs`, and both are set before the job is written — a task -- exists before any of its jobs do, so its id is minted rather than -- scavenged from whichever job it happened to queue first. -- -- Two of them rather than one, because they are two different groupings -- and the buttons want both: stopping a task is not stopping the cascade -- it is part of. cascade_id INTEGER NOT NULL DEFAULT 0, task_id INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL, started_at TEXT, -- The lease. A worker with a job in hand stamps this every few seconds -- from its event loop, so a `running` row says not only that somebody -- claimed it but that somebody is still there. Without it the only way to -- read a `running` row on a shared file is to assume its worker died, and -- a second process starting up would re-queue the first one's live work -- and run it twice. Written by `claim` as well, so a job is leased from -- the moment it is taken rather than from the first beat after it. seen_at TEXT, finished_at TEXT, -- Cancellation is a request in the table, not a flag in the process that -- happens to be running the job. A worker polls it, so Kill works whether -- the worker is the web app's own thread or a second process that -- opened the same file. cancel_requested INTEGER NOT NULL DEFAULT 0, -- The retry policy, and where it has got to. All five are on the row -- rather than read off the job kind when a failure happens, because the -- attempt that has to decide may be in another process, after a restart, -- running a version of the code where the kind's numbers are different or -- the kind is gone. A job carries its own bound for the same reason it -- carries its own payload. -- -- `attempt` is how many times this has been *started*, stamped by `claim` -- in the same statement that takes the row — so a worker dying under a job -- costs it an attempt, which is the reading that stops a job that kills -- workers from being fed them forever. attempt INTEGER NOT NULL DEFAULT 0, -- Counting the first attempt, so 1 is "run it once" and is the default: -- every job that existed before this column behaves exactly as it did. max_attempts INTEGER NOT NULL DEFAULT 1, retry_delay REAL NOT NULL DEFAULT 0, retry_backoff REAL NOT NULL DEFAULT 1, -- Not before this. The whole of how a job waits without being run: it -- stays `queued`, `claim` passes over it, and no worker, no thread and no -- lease is spent on the wait. Durable because it is a column — a next -- attempt an hour away outlives the process that scheduled it, which a -- timer in a worker could not. -- -- `''` rather than NULL, and compared as text: `''` sorts before every -- stamp, so "due" is one comparison with no coalesce and no special case. run_after TEXT NOT NULL DEFAULT '' ); -- Where a cascade's id and a task's come from. A row is identity, and one -- thing more: whether it was killed. No state and no progress — the jobs -- carrying the id are the record of what happened and a second copy could -- only go out of step. The kill is the one fact about a run its jobs cannot -- tell you, and a cascade exists nowhere else, so it is stored here beside -- the job's own `cancel_requested`. -- -- It exists so that a number can be handed out *before* there is a job to -- take it from, and so that "the pull you started at 09:04" has somewhere to -- be written down. -- -- `name` is what was asked for, and it is also how "is one already going?" -- is answered — so the two ways a cascade row comes about have to be told -- apart by it. One is a chain somebody named; the other is minted only -- because a lone job or a lone task needed an id to carry. The second is -- named by `on_its_own` and so never answers for the first. CREATE TABLE IF NOT EXISTS runs ( id INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT NOT NULL, name TEXT NOT NULL DEFAULT '', -- What to call it on screen: "Fetching what the feeds left out". A task is -- not a job and has no handler to carry a label, so it is written here -- with the name, by whoever minted the run. label TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL, cancelled_at TEXT NOT NULL DEFAULT '' ); -- What the queue told somebody, kept so it can be read again. One row per -- `(code, key)`: the same failure said again is this row with a higher count -- and a later `last_at`, which is how "already said" is a fact about the table -- rather than about whichever browser tab happened to be open. See -- `pipeline/notices.py` for why this exists at all. -- -- `place` is `Notice.where`, which is not a name SQL will take unquoted. CREATE TABLE IF NOT EXISTS notices ( id INTEGER PRIMARY KEY, code TEXT NOT NULL, key TEXT NOT NULL DEFAULT '', level TEXT NOT NULL DEFAULT 'warning', kind TEXT NOT NULL DEFAULT 'expected', text TEXT NOT NULL, fix TEXT NOT NULL DEFAULT '', detail TEXT NOT NULL DEFAULT '', place TEXT NOT NULL DEFAULT '', first_at TEXT NOT NULL, last_at TEXT NOT NULL, count INTEGER NOT NULL DEFAULT 1, UNIQUE (code, key) ); -- Standing instructions: enqueue this kind, with this payload, this often. -- Here rather than in a scheduler process of its own, which is the whole -- design: there is no second process to keep alive, a restart between two -- runs loses nothing, and two workers on this file read one schedule instead -- of each holding a timer for it. -- -- `name` is unique because it is the handle: a default declared in the app's -- jobs module is matched to its row by name, and a name that already has a row -- keeps whatever the user edited. That is what makes the declarations a -- starting point rather than a setting — turning one off in the UI survives -- the next restart. -- -- `next_run` and `last_run` are UTC ISO stamps like every other time here, and -- `next_run` is compared as text in `due_schedules` for the same reason -- `run_after` is in `claim`: `''` sorts before every stamp, so "never" and -- "not due" need no coalesce and no special case. A cadence worded in a local -- timezone is turned into one of these before it is stored — see -- `pipeline/schedule.py`, which is the only place a zone is read. -- -- `last_job_id` is what stops a schedule piling up behind itself: the firing -- checks whether that job is still queued or running before starting another. CREATE TABLE IF NOT EXISTS schedules ( id INTEGER PRIMARY KEY, name TEXT NOT NULL, kind TEXT NOT NULL, label TEXT NOT NULL DEFAULT '', payload TEXT NOT NULL DEFAULT '{}', cadence TEXT NOT NULL, enabled INTEGER NOT NULL DEFAULT 1, next_run TEXT NOT NULL DEFAULT '', last_run TEXT NOT NULL DEFAULT '', last_job_id INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL, UNIQUE (name) ); """ #: Made after `_reconcile`, not with the table. An index names columns, and on #: a file that predates one of them `CREATE INDEX` fails before the `ALTER` #: that would have added it — which is a database that will not open, over an #: index. Splitting them puts the two in the only order that always works. INDEXES = """ CREATE INDEX IF NOT EXISTS idx_jobs_state ON jobs(state, id); CREATE INDEX IF NOT EXISTS idx_jobs_cascade ON jobs(cascade_id, id); CREATE INDEX IF NOT EXISTS idx_jobs_task ON jobs(task_id, id); -- For the one question that asks about a moment rather than about an id: -- which cascades ended just now, which is how a finished run stays on screen -- for a few seconds. Without it that clause is a scan of every job kept, taken -- on every push. `purge` is what bounds "every job kept". CREATE INDEX IF NOT EXISTS idx_jobs_finished ON jobs(finished_at); -- Partial, so it constrains only jobs that have not finished yet. This is what -- makes "one pull at a time" a property of the system rather than of whichever -- client asked: a second tab and a command collide here, where a -- flag in one browser could not see the others. Enqueueing a duplicate hands -- back the live job instead of failing. CREATE UNIQUE INDEX IF NOT EXISTS idx_jobs_dedup ON jobs(dedup_key) WHERE dedup_key <> '' AND state IN ('queued', 'running'); """ LIVE = ("queued", "running") #: What holds a job back. Both halves read the dependency's own state rather #: than a count kept alongside it, so nothing has to be incremented as jobs #: land and nothing can be lost to a restart mid-run. #: And whether a worker of its resource is free. `workers` arrives as JSON so #: one parameter carries every count; a resource it does not mention gets one. _HAS_ROOM = """ (SELECT count(*) FROM jobs r WHERE r.state = 'running' AND r.resource = q.resource) < coalesce(json_extract(?2, '$."' || q.resource || '"'), 1) """ _READY = """ NOT EXISTS (SELECT 1 FROM json_each(q.depends_on) d JOIN jobs dep ON dep.id = d.value WHERE dep.state IN ('queued','running')) AND (q.after = 'done' OR NOT EXISTS ( SELECT 1 FROM json_each(q.depends_on) d JOIN jobs dep ON dep.id = d.value WHERE dep.state IN ('failed','cancelled'))) """ #: And whether its time has come. The second thing that holds a job back, and #: it is the same kind of thing as the first: a clause in the claim, not an #: object waiting anywhere. A job scheduled for an hour hence costs a row and #: nothing else — no worker, no thread, no lease, no timer to be lost to a #: restart — and the loop that looks for work every second is what notices #: when the hour is up. #: #: `?1` is the same `now` the claim stamps with, so a job due at exactly this #: instant is taken by the pass that finds it rather than the one after. _DUE = "q.run_after <= ?1" #: The lease of a job nobody is running any more. Older than every real #: stamp, so `reclaim_stale` takes the row on its next look rather than after #: a silence long enough to infer what the worker already knew. _LONG_AGO = datetime.fromtimestamp(0, UTC).isoformat() def _now() -> str: return datetime.now(UTC).isoformat() def _in(seconds: float) -> str: """A stamp that many seconds from now, or `''` for no wait at all. Written in the same UTC ISO form as every other time in this table, because `run_after` is compared as text in `claim` and a second format in the same column would sort by whichever digits it happened to start with. Empty for nothing to wait for, which is what makes the comparison free of a special case: `''` is before every stamp there is. """ if seconds <= 0: return "" return (datetime.now(UTC) + timedelta(seconds=seconds)).isoformat() def on_its_own(name: str) -> str: """What to call the cascade minted for a lone job or a lone task. Two different things would otherwise both be a cascade named `pull`: the chain that starts with a pull and goes on to five more tasks, and the one job somebody pressed by itself. `live_run` answers by name, so it would hand the first back for the second — a Fetch that starts nothing, greys its own button, and returns a run that is over a second later. A task is not renamed, only the cascade wrapped round it: the `pull` task of a lone pull really is the pull task, and the strip draws its box from that name. It is the chain that was never asked for. The space is the whole of the guarantee. A job kind and a task name are identifiers — `pull`, `body_batch` — so nothing that mints a cascade deliberately can ever collide with this. """ return f"{name} on its own" class RunCancelled(Exception): """Raised by `enqueue` for a job under a run that has been killed. The whole of how a kill stops what has not been queued yet. Nobody tells a cascade it is over: its step tries to queue the next task, and the table says no — across processes, after a restart, and when the kill landed in the middle of the run. The queue logs it and drops the job. """ #: The largest `result` a job may leave on its row, as JSON. A result is a #: report — counts, ids, an error — and the snapshot and every tally read it #: back, so data returned instead of written to the app's store would make #: every one of those queries slower for as long as the row is kept. #: #: Past this the job *fails*. It used to keep the keys and log a warning, which #: is losing data where nobody is looking: the next job in a chain would get a #: result with its payload missing and carry on. Raise the limit if a project #: really wants bigger results; do not make it quiet. MAX_RESULT = 64 * 1024 class ResultTooLarge(ValueError): """A handler returned more than `MAX_RESULT` bytes of JSON. Raised by `Jobs.finish` before anything is written, so the row is not touched. The worker ends the job `failed` with this message and does not retry it: the same handler would return the same result. """ def _result(job_id: int, result: dict[str, Any] | None) -> str: text = json.dumps(result or {}) if len(text) > MAX_RESULT: raise ResultTooLarge( f"returned {len(text) // 1024} KB, and a job's result is capped at " f"{MAX_RESULT // 1024} KB (pipeline.table.MAX_RESULT). Write the " f"data to the store and return its ids; keys returned: " f"{', '.join(sorted(str(k) for k in (result or {}))[:20])}") return text def _entry(row: sqlite3.Row) -> NoticeEntry: """A `notices` row as the model, with `place` given back its real name.""" fields = dict(row) fields["where"] = fields.pop("place") return NoticeEntry.model_validate(fields) class Jobs: """The jobs table on one file. Every statement against it is here.""" def __init__(self, path: str | Path) -> None: self.path = Path(path).expanduser() self.path.parent.mkdir(parents=True, exist_ok=True) with closing(self._connect()) as conn, conn: conn.execute("PRAGMA journal_mode=WAL") conn.executescript(SCHEMA) self._reconcile(conn) conn.executescript(INDEXES) @staticmethod def _reconcile(conn: sqlite3.Connection) -> None: """Add any column `SCHEMA` has grown since this file was created.""" runs = {r["name"] for r in conn.execute("PRAGMA table_info(runs)")} for column in ("cancelled_at", "label"): if runs and column not in runs: conn.execute( f"ALTER TABLE runs ADD COLUMN {column} TEXT NOT NULL DEFAULT ''") have = {r["name"] for r in conn.execute("PRAGMA table_info(jobs)")} # The agent handoff, on a file made while it existed. Dropped rather # than left: a column nobody writes is a question every reader of this # table has to answer for itself, and the answer is that a job the # model cannot do now fails and says why. for gone in ("handoff", "answer", "instructions", "skill"): if gone in have: conn.execute(f"ALTER TABLE jobs DROP COLUMN {gone}") for column, ddl in (("seen_at", "TEXT"), ("depends_on", "TEXT NOT NULL DEFAULT '[]'"), ("after", "TEXT NOT NULL DEFAULT 'ok'"), ("resource", "TEXT NOT NULL DEFAULT 'cpu'"), ("cascade_id", "INTEGER NOT NULL DEFAULT 0"), ("task_id", "INTEGER NOT NULL DEFAULT 0"), # The retry policy, arriving on a file that # predates it. Every default is what the file # already behaved as: one attempt, no wait, due # now — so a database migrated by this reads # exactly as it did before the column existed. ("attempt", "INTEGER NOT NULL DEFAULT 0"), ("max_attempts", "INTEGER NOT NULL DEFAULT 1"), ("retry_delay", "REAL NOT NULL DEFAULT 0"), ("retry_backoff", "REAL NOT NULL DEFAULT 1"), ("run_after", "TEXT NOT NULL DEFAULT ''")): if column not in have: conn.execute(f"ALTER TABLE jobs ADD COLUMN {column} {ddl}") def _connect(self) -> sqlite3.Connection: # Thirty seconds, and whoever else opens this file uses the same number # — see `newsdesk/storage/sql/store.py`, which says why the longer of # two timeouts on one file is the one to keep. conn = sqlite3.connect(self.path, timeout=30) conn.row_factory = sqlite3.Row conn.execute("PRAGMA busy_timeout=30000") return conn def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]: with closing(self._connect()) as conn: return conn.execute(sql, params).fetchall() def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None: with closing(self._connect()) as conn: return conn.execute(sql, params).fetchone() def _write(self, sql: str, params: Any = ()) -> int: """One statement in its own transaction. Returns the rowcount.""" with closing(self._connect()) as conn, conn: return conn.execute(sql, params).rowcount # --------------------------------------------------------------- queue def enqueue(self, kind: str, label: str = "", payload: dict[str, Any] | None = None, dedup_key: str = "", depends_on: Sequence[int] = (), after: str = "ok", resource: str = "cpu", cascade_id: int = 0, task_id: int = 0, delay: float = 0.0, max_attempts: int = 1, retry_delay: float = 0.0, retry_backoff: float = 1.0) -> Job: """Queue one job. A duplicate hands back the live one instead of adding. `dedup_key` is enforced by a partial unique index over the unfinished states, so "one pull at a time" holds across every client at once. An empty key opts out, for jobs where several at once is the point. Returning the existing job rather than raising is deliberate: the caller asked for that work to be underway, and it is. `depends_on` holds this job back until those jobs are over — see the column comments in `SCHEMA` for what `after` chooses between. A job under a killed run is refused with `RunCancelled`. That is where a kill reaches everything that had not been queued when the button was pressed: the next task of a killed cascade, the batches a callback was about to fan out. Checked in the job's own transaction, so a kill landing between the check and the insert loses the race by a statement rather than by a process. The refusal spares an `after='done'` job under a killed *task*, which is the same carve-out `cancel_run` and `stop_task` make and for the same reason: killing a task is "skip this one", and its final job is what says so. Without it a kill landing while a task was still fanning out its batches would refuse the final job too, and the task would sit with nothing queued, nothing running and nobody to report that it had been killed. A killed *cascade* still refuses it — there is nothing left to report to. `delay` queues it for later: the row goes in `queued` with a `run_after` that many seconds hence, and `claim` passes over it until then. It is the durable form of "in five minutes" — the row is written now, so the wait survives the process that asked for it, and nothing is alive in the meantime. `max_attempts` and the two retry numbers are the bound `retry` will enforce, stored on the row. The defaults are one attempt and no wait, which is every job that has ever been queued here. """ if after not in ("ok", "done"): raise ValueError(f"after must be 'ok' or 'done', not {after!r}") if max_attempts < 1: raise ValueError(f"max_attempts counts the first attempt, so it is " f"at least 1, not {max_attempts!r}") try: with closing(self._connect()) as conn, conn: killed = conn.execute( """SELECT id FROM runs WHERE cancelled_at <> '' AND (id = ?1 OR (id = ?2 AND ?3 <> 'done'))""", (int(cascade_id), int(task_id), after), ).fetchone() if killed is not None: raise RunCancelled( f"run {killed['id']} was killed; {kind!r} is not queued") # A job queued with neither begins both: something asked for # this on its own, so it is a cascade of one task of one job. # Minted in the job's own transaction, so a duplicate that # rolls back takes its ids with it — a `runs` row with no job # under it would be an identity for nothing. # # The cascade says it stands alone and the task does not: the # task is the `pull` task however it was started, but a chain # named `pull` is a different thing from this and has to stay # findable. See `on_its_own`. cascade_id = int(cascade_id) or self._mint( conn, "cascade", on_its_own(kind), label) task_id = int(task_id) or self._mint(conn, "task", kind, label) job_id = conn.execute( """INSERT INTO jobs (kind, label, payload, state, dedup_key, depends_on, after, resource, cascade_id, task_id, max_attempts, retry_delay, retry_backoff, run_after, created_at) VALUES (?,?,?,'queued',?,?,?,?,?,?,?,?,?,?,?)""", (kind, label, json.dumps(payload or {}), dedup_key, json.dumps([int(i) for i in depends_on]), after, resource or "cpu", int(cascade_id), int(task_id), int(max_attempts), float(retry_delay), float(retry_backoff), _in(delay), _now()), ).lastrowid except sqlite3.IntegrityError: existing = self._row( """SELECT * FROM jobs WHERE dedup_key=? AND state IN ('queued','running') ORDER BY id DESC LIMIT 1""", (dedup_key,), ) if existing is None: raise return Job.from_row(existing) job = self.get(job_id) if job_id is not None else None if job is None: raise RuntimeError(f"job {job_id} disappeared between write and read") return job def claim(self, workers: Mapping[str, int] | None = None) -> Job | None: """Take the oldest ready job and mark it running, atomically. One statement, so two workers racing cannot both take the same job — which matters the moment a worker runs outside the web process. Ready rather than merely queued: a job with unmet dependencies is passed over and stays where it is, costing nothing while it waits, because nothing is running it. That is the whole of "waiting" here — no timer, no listener, no object holding a place. `workers` is how many of each kind there are, and so how many jobs of that resource may run at once. Counted from the table rather than from anything this process is holding, so a worker in a second process shares the same four model slots instead of helping itself to four more. A resource nobody sized gets one. `seen_at` is stamped with `started_at`, so the job is leased from the moment it is taken: a worker that dies in its first second still leaves a row that goes stale rather than one that never had a heartbeat at all and has to be guessed about. A job whose `run_after` has not come round is passed over the same way a job with unmet dependencies is, and costs exactly as little. `attempt` goes up here, in the statement that takes the row, because that is the only place a count of runs cannot be lost or doubled — a worker that dies a second later has still started this job once, and the row says so without anyone having to write it down afterwards. """ with closing(self._connect()) as conn, conn: row = conn.execute( f"""UPDATE jobs SET state='running', started_at=?1, seen_at=?1, attempt = attempt + 1 WHERE id = (SELECT q.id FROM jobs q WHERE q.state='queued' AND {_DUE} AND {_READY} AND {_HAS_ROOM} ORDER BY q.id LIMIT 1) RETURNING *""", (_now(), json.dumps(dict(workers or {}))), ).fetchone() return Job.from_row(row) if row else None def running(self) -> list[Job]: """Everything under way, oldest first.""" return [Job.from_row(r) for r in self._rows( "SELECT * FROM jobs WHERE state='running' ORDER BY id")] @contextmanager def serialize_starts(self): """Serialize cascade discovery and creation across processes. A separate SQLite lock file leaves the jobs database writable while the task creates its batches. SQLite releases the lock on a crash. """ with closing(sqlite3.connect(str(self.path) + ".start-lock", timeout=60)) as conn: conn.execute("BEGIN IMMEDIATE") try: yield finally: conn.rollback() def new_run(self, kind: str, name: str = "", label: str = "") -> int: """Mint an id for a cascade or a task, before it has queued anything. Its own series, not a job's. A task is a thing before any of its jobs exist — it is what decides how many there will be — so taking the id of whichever job it happened to queue first would be describing the task by one of its own consequences. The same goes for what it is called: `name` is what it is, `label` is how to say it. """ with closing(self._connect()) as conn, conn: return self._mint(conn, kind, name, label) @staticmethod def _mint(conn: sqlite3.Connection, kind: str, name: str, label: str = "") -> int: """One `runs` row, on the caller's connection, so it can share a transaction with the job that will carry the id.""" return int(conn.execute( "INSERT INTO runs (kind, name, label, created_at) VALUES (?,?,?,?)", (kind, name, label or name, _now()), ).lastrowid or 0) def run(self, run_id: int) -> Run | None: """What a cascade or task id names: its kind, its name, when it began.""" row = self._row("SELECT * FROM runs WHERE id=?", (int(run_id),)) return Run.from_row(row) if row else None def of_cascade(self, cascade_id: int) -> list[Job]: """Every job of one cascade, oldest first.""" return [Job.from_row(r) for r in self._rows( "SELECT * FROM jobs WHERE cascade_id=? ORDER BY id", (int(cascade_id),))] def of_task(self, task_id: int) -> list[Job]: """Every job of one task: its batches, and the final job behind them.""" return [Job.from_row(r) for r in self._rows( "SELECT * FROM jobs WHERE task_id=? ORDER BY id", (int(task_id),))] def live_run(self, kind: str, name: str) -> Run | None: """The newest run of that kind and name with a job still going. What "is a pull already under way?" means, and what no job kind can answer: `solo` stops a second `pull` while the first runs and says nothing once the run has moved on to bodies. A run is live for as long as any of its jobs is, which is the span the Fetch button means. The name is the whole of the question, which is why the cascade minted for something standing alone is named apart — see `on_its_own`. A lone `pull` job is not the pull chain, and answering with it would be this saying yes to a question nobody asked. """ row = self._row( """SELECT * FROM runs r WHERE r.kind=? AND r.name=? AND r.cancelled_at='' AND EXISTS (SELECT 1 FROM jobs j WHERE j.cascade_id=r.id AND j.state IN ('queued','running')) ORDER BY r.id DESC LIMIT 1""", (kind, name), ) return Run.from_row(row) if row else None def outcomes(self, ids: Sequence[int]) -> list[Job]: """How each of those jobs ended, oldest first. What a final job is handed, and the only thing it needs: seven done and one failed is read off the jobs themselves, at the moment it is asked, rather than counted up as they landed. There is nothing to increment and so nothing to lose to a restart half way through. """ if not ids: return [] holes = ",".join("?" for _ in ids) return [Job.from_row(r) for r in self._rows( f"SELECT * FROM jobs WHERE id IN ({holes}) ORDER BY id", tuple(int(i) for i in ids))] def blocked(self) -> list[Job]: """Queued jobs that `claim` would pass over. For the UI and for tests. Both reasons, because from here they are the same fact: this row is in the queue and is not going to be taken yet. Which of the two it is — something it waits for, or a time it waits until — is on the row, in `depends_on` and `run_after`. """ return [Job.from_row(r) for r in self._rows( f"""SELECT * FROM jobs q WHERE q.state='queued' AND NOT ({_DUE} AND {_READY}) ORDER BY q.id""", (_now(),))] def progress(self, job_id: int, done: int = 0, total: int = 0, message: str = "") -> None: self._write("UPDATE jobs SET done=?, total=?, message=? WHERE id=?", (done, total, message, job_id)) def finish(self, job_id: int, state: str, result: dict[str, Any] | None = None, error: str = "") -> Job | None: """Move a job to one of the terminal states, once, and settle what waited. A job that ended badly is the moment anything depending on it can be answered, so it is answered here rather than swept for later: nothing should sit queued forever waiting on something that is never coming. """ self._write( """UPDATE jobs SET state=?, result=?, error=?, finished_at=? WHERE id=? AND state NOT IN ('done','failed','cancelled')""", (state, _result(job_id, result), error, _now(), job_id), ) if state in ("failed", "cancelled"): self._settle_dependents() return self.get(job_id) def retry(self, job_id: int, *, error: str = "", delay: float | None = None) -> Job | None: """End an attempt that went badly: another one later, or give up. The one place the bound is read, and so the one place a job stops being tried. Three endings, and which one it is depends on nothing the caller knows: asked to stop `cancelled`. A Kill that landed while the attempt ran must not be undone by the attempt failing — that would be a button that quietly did nothing. attempts left back to `queued`, with `run_after` set and the lease handed back. Nothing runs it, nothing holds a place for it, and whatever depends on it goes on waiting, because it is still live. none left `failed`, and `Job.exhausted` from then on. The wait is `retry_delay` multiplied by `retry_backoff` once for each attempt already made, unless `delay` overrides it — which is how a server's own answer to "come back in 120 seconds" is honoured instead of guessed at. A hint does not buy an extra attempt: the bound is ours, the wait is the world's. The error is kept on the row either way, so a job sitting in the queue with an hour to wait still says what went wrong last time rather than looking like ordinary work nobody has got to yet. Read then written, rather than one statement like everything else here: the arithmetic wants the row's own numbers, and this is the one method whose row is `running` and held by the worker calling it, so there is no second writer to lose a race with. """ job = self.get(job_id) if job is None: return None if job.cancel_requested: return self.finish(job_id, "cancelled", error=error) if job.attempt >= job.max_attempts: return self.finish(job_id, "failed", error=error) wait = delay if delay is not None else ( job.retry_delay * job.retry_backoff ** max(job.attempt - 1, 0)) self._write( """UPDATE jobs SET state='queued', error=?, run_after=?, message='', started_at=NULL, seen_at=NULL WHERE id=? AND state='running'""", (error, _in(wait), job_id), ) return self.get(job_id) def waiting(self) -> list[Job]: """Queued jobs whose next attempt is still to come, oldest first.""" return [Job.from_row(r) for r in self._rows( "SELECT * FROM jobs WHERE state='queued' AND run_after > ? ORDER BY id", (_now(),))] def _settle_dependents(self) -> None: """End queued jobs that can no longer run, and their dependents in turn. Only jobs that wanted success. Killing one job is not stopping the run it belongs to — it is dropping that batch, exactly as a batch that failed is dropped, and the work comes round next time because every task rediscovers its own backlog. So the job waiting to report on the batches still runs, counts this one among those that did not land, and the chain carries on: to a cascade a killed batch and a failed one are the same thing. Stopping a whole task or a whole cascade is `cancel_run`, which is a different button and says so. Looped because a dependent is itself depended on, and each pass ends one more generation. It terminates: every pass moves at least one job out of `queued`, and nothing here puts one back. """ while self._write( """UPDATE jobs SET state='failed', finished_at=?, message='', error='a job it waited for did not finish' WHERE jobs.state = 'queued' AND jobs.after = 'ok' AND EXISTS (SELECT 1 FROM json_each(jobs.depends_on) d JOIN jobs dep ON dep.id = d.value WHERE dep.state IN ('failed','cancelled'))""", (_now(),), ): pass def get(self, job_id: int) -> Job | None: row = self._row("SELECT * FROM jobs WHERE id=?", (job_id,)) return Job.from_row(row) if row else None def last(self, kind: str, *, states: tuple[str, ...] = ("done",)) -> Job | None: """The most recent job of a kind that reached one of `states`. Ordered by `id` rather than `finished_at`: the id is the order things were asked for, which is what "the last pull" means. """ placeholders = ",".join("?" for _ in states) row = self._row( f"""SELECT * FROM jobs WHERE kind=? AND state IN ({placeholders}) ORDER BY id DESC LIMIT 1""", (kind, *states), ) return Job.from_row(row) if row else None # -------------------------------------------------------- cancelling def request_cancel(self, job_id: int) -> Job | None: """Ask for a job to stop. A queued job is finished here and now — nothing is part-way through. A running one only gets the request recorded; the worker notices on its next `context.check()` and writes the terminal state itself, because only it knows when it has reached a point where stopping is safe. A job waiting for its next attempt is a queued job and ends here with the rest of them. That is the point of the wait being a column rather than a sleeping worker: there is a row to cancel, from any process, at any moment of the hour it is waiting. """ job = self.get(job_id) if job is None or not job.live: return job if job.state == "queued": return self.finish(job_id, "cancelled") self._write("UPDATE jobs SET cancel_requested=1 WHERE id=?", (job_id,)) return self.get(job_id) def cancel_run(self, run_id: int) -> int: """Kill a run — a task or a cascade, the same operation for both. Three things: flag the row, end the queued jobs carrying that id, and ask the running ones. Plus the fourth that is not here: `enqueue` refuses anything later queued under it. That is the whole of how a kill stops the pipeline, and the flag is what makes it hold across processes, after a restart, and when the kill lands mid-run. The one asymmetry is deliberate and is what the two buttons mean. A cascade's id takes everything in it. A task's id spares the job waiting on `after='done'` — its final job — because reporting is not work: killing a task is "skip this one", not "make it never end". So the final job still runs and still calls the step, and what it hands over says the *task* was killed rather than only how many batches ended cancelled — the two are indistinguishable in the counts and mean opposite things, and `Jobs.cancelled` is where that flag is read back. What to do about it is then the cascade's decision, not this call's: newsdesk's skips the stage and carries on. The step's attempt to queue the next task goes ahead if only the task was killed, and is refused if the cascade was killed too. """ self._write( "UPDATE runs SET cancelled_at=? WHERE id=? AND cancelled_at=''", (_now(), int(run_id))) # Queued jobs end here and now — nothing is part-way through. A running # one is asked and stops at its next `context.check()`, because only it # knows when it has reached a point where stopping is safe. where = "(cascade_id=?1 OR (task_id=?1 AND after<>'done'))" ended = self._write( f"""UPDATE jobs SET state='cancelled', finished_at=?2, message='' WHERE {where} AND state='queued'""", (int(run_id), _now()), ) asked = self._write( f"UPDATE jobs SET cancel_requested=1 WHERE {where} AND state='running'", (int(run_id),)) self._settle_dependents() return ended + asked def stop_task(self, task_id: int, error: str = "") -> int: """End a task's queued batches without killing the task. How many. What a batch calls when the failure it just met is the backend's answer to the whole run rather than to its own rows — a spend limit, a login that has expired, no model on the machine. The siblings behind it would each make the same call and get the same refusal, and each put the same notice on screen; twenty-three of those is the bug this exists to stop. Three differences from `cancel_run`, and each of them is the point: the run is not flagged nothing was killed. `Tally.killed` means a person pressed Kill and the step is to be skipped, and borrowing the flag to mean "the model said no" would make the two indistinguishable in the one place that can still tell them apart. running jobs are left their requests are in the air and are a bill either way. They end however they end — including the one that called this, which keeps its own `failed` rather than being cancelled by its own instruction. the reason is kept written to `error` on each row, so a batch that never ran says why on screen instead of looking like one somebody dropped. The final job is spared, as it is by a kill and for the same reason: reporting is not work, and a run that stopped early is exactly the one that has something to report. """ ended = self._write( """UPDATE jobs SET state='cancelled', finished_at=?, message='', error=? WHERE task_id=? AND after<>'done' AND state='queued'""", (_now(), error, int(task_id)), ) if ended: self._settle_dependents() return ended def cancelled(self, run_id: int) -> bool: """Whether this task or cascade has been killed.""" row = self._row("SELECT cancelled_at FROM runs WHERE id=?", (int(run_id),)) return bool(row and row[0]) def cancel_requested(self, job_id: int) -> bool: row = self._row("SELECT cancel_requested FROM jobs WHERE id=?", (job_id,)) return bool(row[0]) if row else False # ---------------------------------------------------------- schedules #: Every schedule, with how its last job ended. One statement, because the #: list and the socket push both want the join and neither wants a query #: per row — and because a row read in two halves can disagree with itself. _SCHEDULES = """ SELECT s.*, coalesce(j.state, '') AS last_state FROM schedules s LEFT JOIN jobs j ON j.id = s.last_job_id """ def schedules(self) -> list[Schedule]: """Every schedule, by name. The order the UI lists them in. By name rather than by id, because the name is what a person reads and an id is the order they happened to be added in — including the order the app's declared defaults were imported in, which is not a fact about anything. """ return [Schedule.from_row(r) for r in self._rows(self._SCHEDULES + " ORDER BY s.name")] def schedule(self, schedule_id: int) -> Schedule | None: row = self._row(self._SCHEDULES + " WHERE s.id=?", (int(schedule_id),)) return Schedule.from_row(row) if row else None def add_schedule(self, name: str, kind: str, cadence: str, *, payload: dict[str, Any] | None = None, label: str = "", enabled: bool = True, next_run: str = "", only_if_new: bool = False) -> Schedule | None: """Write one schedule. `None` when `only_if_new` and the name is taken. `only_if_new` is the whole of reconciliation: the queue offers every declared default on every start, and a name that already has a row is left exactly as the user edited it — cadence, payload, enabled and all. Which is what makes a declaration a starting point rather than a setting somebody has to fight. One statement, so two workers starting at once insert a declared default once between them rather than racing a read against a write. The conflict is the second one finding the first one's row. """ clash = "DO NOTHING" if only_if_new else """DO UPDATE SET kind=excluded.kind, label=excluded.label, payload=excluded.payload, cadence=excluded.cadence, enabled=excluded.enabled, next_run=excluded.next_run""" with closing(self._connect()) as conn, conn: row = conn.execute( f"""INSERT INTO schedules (name, kind, label, payload, cadence, enabled, next_run, created_at) VALUES (?,?,?,?,?,?,?,?) ON CONFLICT (name) {clash} RETURNING id""", (name, kind, label, json.dumps(payload or {}), cadence, int(bool(enabled)), next_run, _now()), ).fetchone() return self.schedule(row["id"]) if row else None #: What `update_schedule` will write, and the whole of it. A whitelist #: rather than trust, because the column names go into the statement: the #: values are parameters but the keys cannot be, so this is the boundary. #: `name` is not here — it is the handle a declared default is matched on, #: and renaming a row would hand its edits back to a freshly inserted one. EDITABLE = ("kind", "label", "payload", "cadence", "enabled", "next_run", "last_run", "last_job_id") def update_schedule(self, schedule_id: int, **fields: Any) -> Schedule | None: """Change some of one schedule's columns. The row as it now stands. Also how a firing that decided not to fire moves on: setting `next_run` alone is "skip this slot", which is what happens when the last job of this schedule is still going. Nothing else is touched, so `last_run` goes on saying when it last actually ran. """ wanted = {k: v for k, v in fields.items() if k in self.EDITABLE} if not wanted: return self.schedule(schedule_id) if "payload" in wanted and not isinstance(wanted["payload"], str): wanted["payload"] = json.dumps(wanted["payload"] or {}) if "enabled" in wanted: wanted["enabled"] = int(bool(wanted["enabled"])) sets = ", ".join(f"{k}=?" for k in wanted) self._write(f"UPDATE schedules SET {sets} WHERE id=?", (*wanted.values(), int(schedule_id))) return self.schedule(schedule_id) def delete_schedule(self, schedule_id: int) -> bool: """Forget one. Not an error when it has already gone — two tabs.""" return bool(self._write("DELETE FROM schedules WHERE id=?", (int(schedule_id),))) def due_schedules(self, now: str) -> list[Schedule]: """Enabled schedules whose time has come, oldest due first. Text comparison against a UTC ISO `now`, for the reason `claim` uses one: `''` sorts before every stamp, so a schedule with no next run — one that has never been given a time — is simply not due, with no coalesce and no special case. Oldest first so that a queue coming back after a long silence fires what was due longest ago before what was due a minute ago, which is the order a person would have run them in. """ return [Schedule.from_row(r) for r in self._rows( self._SCHEDULES + """ WHERE s.enabled=1 AND s.next_run <> '' AND s.next_run <= ? ORDER BY s.next_run""", (now,))] def mark_fired(self, schedule_id: int, job_id: int, next_run: str) -> Schedule | None: """Record that this schedule just queued that job, and when it is next. One statement, because the three move together: a `last_job_id` written without the `next_run` beside it would be a schedule that fires again on the following pass, every pass, for as long as that job runs. """ self._write( """UPDATE schedules SET last_run=?, last_job_id=?, next_run=? WHERE id=?""", (_now(), int(job_id), next_run, int(schedule_id))) return self.schedule(schedule_id) # ------------------------------------------------------------ repairs def heartbeat(self, ids: Sequence[int]) -> int: """Say those running jobs are still being run. Returns how many rows. The other half of `reclaim_stale`. Called on a timer by the worker that holds them, and against the table rather than any object, because the process that has to read it is a different one. Only `running` rows are touched: a job that finished between the beat being scheduled and the beat landing must not have its lease refreshed under its terminal state. """ if not ids: return 0 holes = ",".join("?" for _ in ids) return self._write( f"UPDATE jobs SET seen_at=? WHERE state='running' AND id IN ({holes})", (_now(), *(int(i) for i in ids))) def release(self, ids: Sequence[int]) -> int: """Hand back the lease on jobs this worker is walking away from. A process on its way out knows something `reclaim_stale` can only wait to infer: that these jobs have nobody running them as of now. Saying so is what makes a restart pick them up at once rather than a lease later. Only a crash, which by definition says nothing, pays the wait. The lease is expired rather than the job re-queued, and that is the point of doing it this way: the row stays `running`, so a handler thread that outlives the shutdown still writes its own ending into it rather than being overwritten by a second run of itself. That ending is the thread's to write — see `JobQueue._execute`, where the handler call and the terminal write are one function — which is what makes expiring the lease the safe half-measure it reads as. Only the next `reclaim_stale` moves the row, and only if that ending never came. """ if not ids: return 0 holes = ",".join("?" for _ in ids) return self._write( f"UPDATE jobs SET seen_at=? WHERE state='running' AND id IN ({holes})", (_LONG_AGO, *(int(i) for i in ids))) def purge(self, older_than: float) -> int: """Delete jobs that finished more than `older_than` seconds ago. Returns how many. Without this both tables grow for as long as the app runs. Three things are kept whatever their age: anything not finished a queued or running row is work, not history. the newest ending of each `last(kind)` answers "when did this last kind in each state succeed" from the table, and a status line saying "never pulled" because the last pull was five weeks ago is wrong. a run with jobs left a run is deleted only once nothing points at it and it is old itself — one minted a moment ago may have no jobs yet because its finder is still looking. One transaction, so a snapshot never sees jobs gone and their runs still there. """ cutoff = (datetime.now(UTC) - timedelta(seconds=older_than)).isoformat() with closing(self._connect()) as conn, conn: gone = conn.execute( """DELETE FROM jobs WHERE state IN ('done','failed','cancelled') AND finished_at IS NOT NULL AND finished_at < ? AND id NOT IN (SELECT MAX(id) FROM jobs WHERE state IN ('done','failed','cancelled') GROUP BY kind, state)""", (cutoff,)).rowcount conn.execute( """DELETE FROM runs WHERE created_at < ? AND id NOT IN (SELECT cascade_id FROM jobs) AND id NOT IN (SELECT task_id FROM jobs)""", (cutoff,)) return int(gone or 0) # ------------------------------------------------------------- notices def record_notice(self, notice: Notice) -> NoticeEntry: """Write one notice into the log, and say how often it has been said. One statement, for the reason every write here is one: two batches of one run meet the same expired login in the same second, and a read followed by a write would file it twice. The conflict is the second batch finding the first one's row. A repeat takes the newer wording as well as the count. The text is a sentence about now — the refused article's title, the feed that died this time — and a log showing last week's sentence beside today's count is describing two different moments. `count == 1` on what comes back is how a caller knows this is news. See `NoticeEntry`. """ now = _now() with closing(self._connect()) as conn, conn: row = conn.execute( """INSERT INTO notices (code, key, level, kind, text, fix, detail, place, first_at, last_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (code, key) DO UPDATE SET count = count + 1, last_at = excluded.last_at, level = excluded.level, kind = excluded.kind, text = excluded.text, fix = excluded.fix, detail = excluded.detail, place = excluded.place RETURNING *""", (notice.code, notice.key, notice.level, notice.kind, notice.text, notice.fix, notice.detail, notice.where, now, now)).fetchone() return _entry(row) def notices(self, limit: int = 200) -> list[NoticeEntry]: """The log, most recently said first.""" return [_entry(row) for row in self._rows( "SELECT * FROM notices ORDER BY last_at DESC, id DESC LIMIT ?", (int(limit),))] def clear_notice(self, notice_id: int) -> bool: """Done with this one. The same failure afterwards is news again.""" return bool(self._write("DELETE FROM notices WHERE id=?", (int(notice_id),))) def clear_notices(self) -> int: return self._write("DELETE FROM notices") def purge_notices(self, quiet_for: float) -> int: """Forget what has not been said again for `quiet_for` seconds. By `last_at`, not `first_at`: a feed that has failed every evening for a month is a row a month old that was true an hour ago, and deleting it would make tonight's failure look like the first. What goes is what stopped happening, which is the only kind of notice that can be dropped without anyone having read it — and dropping it is what lets the same failure, months on, interrupt you again. """ cutoff = (datetime.now(UTC) - timedelta(seconds=quiet_for)).isoformat() return self._write("DELETE FROM notices WHERE last_at < ?", (cutoff,)) def reclaim_stale(self, older_than: float = 30.0, *, except_ids: Sequence[int] = ()) -> int: """Put jobs whose worker stopped saying it was there back in the queue. A `running` row is a claim plus a lease: `claim` stamps `seen_at` and the worker holding the job restamps it every few seconds. So a row whose lease has run out is one whose worker went away mid-job, and re-queueing is the right reading of that — the work was asked for and never happened. Handlers have to tolerate being re-run. The lease is what makes this safe on a file two processes share. Left as "every `running` row", a second worker starting up would re-queue the first one's live jobs and both would run them; `older_than` is how long a worker has to have been silent before that reading is the only one left. Which is also why this runs on a timer and not only at start: whichever process is still alive is the one that cleans up after the one that is not. A row with no `seen_at` at all — claimed by a version of this file that predates the column — falls back to `started_at`, so an old orphan is still reclaimed once it is old enough. One of these does not come back: a job that had been asked to stop before its worker died ends `cancelled` instead of returning to the queue. Re-queueing it would be a Kill that a crash quietly undid, and the request is in the table precisely so that it outlives the process that was supposed to act on it. A worker excludes the ids it still holds. Their expired leases mean its loop stalled, not that its handler threads stopped; renew them and warn instead of running the same handler twice in this process. """ cutoff = (datetime.now(UTC) - timedelta(seconds=older_than)).isoformat() ids = tuple(int(i) for i in except_ids) holes = ",".join("?" for _ in ids) if ids: renewed = self._write( f"UPDATE jobs SET seen_at=? WHERE state='running' " f"AND coalesce(seen_at, started_at, created_at) < ? " f"AND id IN ({holes})", (_now(), cutoff, *ids)) if renewed: log.warning("worker loop stalled; renewed %d held job lease(s)", renewed) excluded = f" AND id NOT IN ({holes})" if ids else "" taken = self._write( """UPDATE jobs SET state = CASE WHEN cancel_requested THEN 'cancelled' ELSE 'queued' END, finished_at = CASE WHEN cancel_requested THEN ?1 ELSE NULL END, started_at=NULL, seen_at=NULL, cancel_requested=0 WHERE state='running' AND coalesce(seen_at, started_at, created_at) < ?2""" + excluded, (_now(), cutoff, *ids)) if taken: # Cheap and only on a reclaim, which is rare. A job that came back # cancelled is a job something may have been waiting for. self._settle_dependents() return taken # ------------------------------------------------------------ reading #: How long a finished cascade stays in `cascades`. Nothing depends on it #: — it is so that a run you watched is still on screen the moment after #: it lands, rather than vanishing on the push that says it is over, which #: is the one push you wanted to see. LINGER = timedelta(seconds=15) def cascades(self) -> list[CascadeProgress]: """Every cascade with something live in it, tallied task by task. One query and a grouping. The rows are every job of those cascades and not only the live ones, because a task's landed jobs are most of what a bar of it is and they are long gone from `snapshot`'s `recent`. `finished_at` is set only when a job reaches a terminal state, so the second half of that `IN` needs no state list — and without one it can use the index rather than reading every job that ever finished. Ordered by the id of the first job of each, which for a cascade whose tasks queue one another is the order they ran. """ rows = self._rows( """SELECT j.cascade_id, j.task_id, j.id, j.state, j.label, c.name AS cascade_name, t.name AS task_name, t.label AS task_label FROM jobs j LEFT JOIN runs c ON c.id = j.cascade_id LEFT JOIN runs t ON t.id = j.task_id WHERE j.cascade_id IN ( SELECT cascade_id FROM jobs WHERE state IN ('queued','running') UNION SELECT cascade_id FROM jobs WHERE finished_at > ?) ORDER BY j.id""", ((datetime.now(UTC) - self.LINGER).isoformat(),)) runs: dict[int, CascadeProgress] = {} tasks: dict[tuple[int, int], TaskProgress] = {} for row in rows: run = runs.get(row["cascade_id"]) if run is None: run = runs[row["cascade_id"]] = CascadeProgress( id=row["cascade_id"], # A job queued outside any cascade begins one named after # itself and for standing alone, so there is always a name # and no chain answers to it; the fallback is for a row # from before `runs` existed. name=row["cascade_name"] or "") task = tasks.get((row["cascade_id"], row["task_id"])) if task is None: # Both off the `runs` row, minted before any of these jobs # existed. The first job's label is the fallback, for a run # from before the column and for a job queued on its own. task = tasks[(row["cascade_id"], row["task_id"])] = TaskProgress( id=row["task_id"], name=row["task_name"] or "", label=row["task_label"] or row["label"] or "") run.tasks.append(task) # One counter per state, by name. A state the model has no field # for is skipped rather than raised: this is a drawing, and it # must not be what takes the queue's whole snapshot down. if row["state"] in TaskProgress.model_fields: setattr(task, row["state"], getattr(task, row["state"]) + 1) return list(runs.values()) def snapshot(self, recent: int = 30) -> JobsSnapshot: """The whole queue as a UI shows it, in one read. Live and finished jobs in one statement, so a job that finishes while this runs is in one list or the other and never in both. """ rows = [Job.from_row(r) for r in self._rows( """SELECT * FROM jobs WHERE state IN ('queued','running') UNION ALL SELECT * FROM (SELECT * FROM jobs WHERE state IN ('done','failed','cancelled') ORDER BY id DESC LIMIT ?)""", (recent,), )] live = sorted((j for j in rows if j.state in ("queued", "running")), key=lambda j: j.id) return JobsSnapshot( cascades=self.cascades(), running=[j for j in live if j.state == "running"], queued=[j for j in live if j.state == "queued"], recent=sorted((j for j in rows if j.state not in ("queued", "running")), key=lambda j: j.id, reverse=True), )