1349 lines
67 KiB
Python
1349 lines
67 KiB
Python
"""The jobs table: the queue's own storage, on whatever SQLite file it is given.
|
|
|
|
The pipeline used to keep its rows in the application's store, which meant
|
|
the application's schema knew what a job was and the queue could not exist
|
|
without it. Now the table is the pipeline's: it creates it on open, owns its
|
|
shape, and asks nothing of the file beyond being SQLite. Point it at the
|
|
app's database and the jobs sit next to the app's rows, one file to back up;
|
|
point it at a file of its own and a worker runs with no app at all.
|
|
|
|
Why a table rather than a list in memory is in `queue.py`. Why every method
|
|
here is one statement is the same reason: the file is shared by whoever
|
|
opens it, so a claim has to be atomic and a state change has to be a single
|
|
`UPDATE` that a second process cannot half-see.
|
|
|
|
The shape is `CREATE TABLE IF NOT EXISTS`, which leaves a file that already
|
|
has the table exactly as it is — so a column added here would never reach one.
|
|
`_reconcile` is how a column arrives instead: read the catalogue, add what is
|
|
missing. Idempotent, so it runs on every open and there is no version to keep
|
|
in step. There is deliberately no counter of our own — `PRAGMA user_version`
|
|
on this file belongs to newsdesk (`newsdesk/storage/sql/schema.py`), and a
|
|
second migration series numbering itself on the same pragma would fight it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import sqlite3
|
|
from collections.abc import Mapping, Sequence
|
|
from contextlib import closing, contextmanager
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from pipeline.models import (
|
|
CascadeProgress, Job, JobsSnapshot, Notice, NoticeEntry, Run, Schedule,
|
|
TaskProgress,
|
|
)
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
SCHEMA = """
|
|
CREATE TABLE IF NOT EXISTS jobs (
|
|
id INTEGER PRIMARY KEY,
|
|
kind TEXT NOT NULL,
|
|
label TEXT NOT NULL DEFAULT '',
|
|
payload TEXT NOT NULL DEFAULT '{}',
|
|
state TEXT NOT NULL DEFAULT 'queued',
|
|
done INTEGER NOT NULL DEFAULT 0,
|
|
total INTEGER NOT NULL DEFAULT 0,
|
|
message TEXT NOT NULL DEFAULT '',
|
|
result TEXT NOT NULL DEFAULT '{}',
|
|
error TEXT NOT NULL DEFAULT '',
|
|
dedup_key TEXT NOT NULL DEFAULT '',
|
|
-- Which jobs this one waits for, and what it wants of them. `after='ok'`
|
|
-- is one job needing another's work to have happened, and fails with it;
|
|
-- `after='done'` waits only for them to be over, however they ended, and
|
|
-- is what a job whose purpose is to report on them needs — success-only
|
|
-- would mean the report never exists when there is something to report.
|
|
-- An id that names nothing is satisfied: it finished before this was
|
|
-- asked, or was never queued.
|
|
depends_on TEXT NOT NULL DEFAULT '[]',
|
|
after TEXT NOT NULL DEFAULT 'ok',
|
|
-- What this job spends while it runs: `llm` for anything that asks a
|
|
-- model, `http` for a fetch, `cpu` for work that wants the machine to
|
|
-- itself. Named on the job rather than counted in the worker, because a
|
|
-- second process opening this file has to be counted too — four model
|
|
-- calls is a fact about the subscription, not about one program.
|
|
resource TEXT NOT NULL DEFAULT 'cpu',
|
|
-- Which cascade this job belongs to, and which of its tasks. Both name a
|
|
-- row in `runs`, and both are set before the job is written — a task
|
|
-- exists before any of its jobs do, so its id is minted rather than
|
|
-- scavenged from whichever job it happened to queue first.
|
|
--
|
|
-- Two of them rather than one, because they are two different groupings
|
|
-- and the buttons want both: stopping a task is not stopping the cascade
|
|
-- it is part of.
|
|
cascade_id INTEGER NOT NULL DEFAULT 0,
|
|
task_id INTEGER NOT NULL DEFAULT 0,
|
|
created_at TEXT NOT NULL,
|
|
started_at TEXT,
|
|
-- The lease. A worker with a job in hand stamps this every few seconds
|
|
-- from its event loop, so a `running` row says not only that somebody
|
|
-- claimed it but that somebody is still there. Without it the only way to
|
|
-- read a `running` row on a shared file is to assume its worker died, and
|
|
-- a second process starting up would re-queue the first one's live work
|
|
-- and run it twice. Written by `claim` as well, so a job is leased from
|
|
-- the moment it is taken rather than from the first beat after it.
|
|
seen_at TEXT,
|
|
finished_at TEXT,
|
|
-- Cancellation is a request in the table, not a flag in the process that
|
|
-- happens to be running the job. A worker polls it, so Kill works whether
|
|
-- the worker is the web app's own thread or a second process that
|
|
-- opened the same file.
|
|
cancel_requested INTEGER NOT NULL DEFAULT 0,
|
|
-- The retry policy, and where it has got to. All five are on the row
|
|
-- rather than read off the job kind when a failure happens, because the
|
|
-- attempt that has to decide may be in another process, after a restart,
|
|
-- running a version of the code where the kind's numbers are different or
|
|
-- the kind is gone. A job carries its own bound for the same reason it
|
|
-- carries its own payload.
|
|
--
|
|
-- `attempt` is how many times this has been *started*, stamped by `claim`
|
|
-- in the same statement that takes the row — so a worker dying under a job
|
|
-- costs it an attempt, which is the reading that stops a job that kills
|
|
-- workers from being fed them forever.
|
|
attempt INTEGER NOT NULL DEFAULT 0,
|
|
-- Counting the first attempt, so 1 is "run it once" and is the default:
|
|
-- every job that existed before this column behaves exactly as it did.
|
|
max_attempts INTEGER NOT NULL DEFAULT 1,
|
|
retry_delay REAL NOT NULL DEFAULT 0,
|
|
retry_backoff REAL NOT NULL DEFAULT 1,
|
|
-- Not before this. The whole of how a job waits without being run: it
|
|
-- stays `queued`, `claim` passes over it, and no worker, no thread and no
|
|
-- lease is spent on the wait. Durable because it is a column — a next
|
|
-- attempt an hour away outlives the process that scheduled it, which a
|
|
-- timer in a worker could not.
|
|
--
|
|
-- `''` rather than NULL, and compared as text: `''` sorts before every
|
|
-- stamp, so "due" is one comparison with no coalesce and no special case.
|
|
run_after TEXT NOT NULL DEFAULT ''
|
|
);
|
|
|
|
-- Where a cascade's id and a task's come from. A row is identity, and one
|
|
-- thing more: whether it was killed. No state and no progress — the jobs
|
|
-- carrying the id are the record of what happened and a second copy could
|
|
-- only go out of step. The kill is the one fact about a run its jobs cannot
|
|
-- tell you, and a cascade exists nowhere else, so it is stored here beside
|
|
-- the job's own `cancel_requested`.
|
|
--
|
|
-- It exists so that a number can be handed out *before* there is a job to
|
|
-- take it from, and so that "the pull you started at 09:04" has somewhere to
|
|
-- be written down.
|
|
--
|
|
-- `name` is what was asked for, and it is also how "is one already going?"
|
|
-- is answered — so the two ways a cascade row comes about have to be told
|
|
-- apart by it. One is a chain somebody named; the other is minted only
|
|
-- because a lone job or a lone task needed an id to carry. The second is
|
|
-- named by `on_its_own` and so never answers for the first.
|
|
CREATE TABLE IF NOT EXISTS runs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
kind TEXT NOT NULL,
|
|
name TEXT NOT NULL DEFAULT '',
|
|
-- What to call it on screen: "Fetching what the feeds left out". A task is
|
|
-- not a job and has no handler to carry a label, so it is written here
|
|
-- with the name, by whoever minted the run.
|
|
label TEXT NOT NULL DEFAULT '',
|
|
created_at TEXT NOT NULL,
|
|
cancelled_at TEXT NOT NULL DEFAULT ''
|
|
);
|
|
|
|
-- What the queue told somebody, kept so it can be read again. One row per
|
|
-- `(code, key)`: the same failure said again is this row with a higher count
|
|
-- and a later `last_at`, which is how "already said" is a fact about the table
|
|
-- rather than about whichever browser tab happened to be open. See
|
|
-- `pipeline/notices.py` for why this exists at all.
|
|
--
|
|
-- `place` is `Notice.where`, which is not a name SQL will take unquoted.
|
|
CREATE TABLE IF NOT EXISTS notices (
|
|
id INTEGER PRIMARY KEY,
|
|
code TEXT NOT NULL,
|
|
key TEXT NOT NULL DEFAULT '',
|
|
level TEXT NOT NULL DEFAULT 'warning',
|
|
kind TEXT NOT NULL DEFAULT 'expected',
|
|
text TEXT NOT NULL,
|
|
fix TEXT NOT NULL DEFAULT '',
|
|
detail TEXT NOT NULL DEFAULT '',
|
|
place TEXT NOT NULL DEFAULT '',
|
|
first_at TEXT NOT NULL,
|
|
last_at TEXT NOT NULL,
|
|
count INTEGER NOT NULL DEFAULT 1,
|
|
UNIQUE (code, key)
|
|
);
|
|
|
|
-- Standing instructions: enqueue this kind, with this payload, this often.
|
|
-- Here rather than in a scheduler process of its own, which is the whole
|
|
-- design: there is no second process to keep alive, a restart between two
|
|
-- runs loses nothing, and two workers on this file read one schedule instead
|
|
-- of each holding a timer for it.
|
|
--
|
|
-- `name` is unique because it is the handle: a default declared in the app's
|
|
-- jobs module is matched to its row by name, and a name that already has a row
|
|
-- keeps whatever the user edited. That is what makes the declarations a
|
|
-- starting point rather than a setting — turning one off in the UI survives
|
|
-- the next restart.
|
|
--
|
|
-- `next_run` and `last_run` are UTC ISO stamps like every other time here, and
|
|
-- `next_run` is compared as text in `due_schedules` for the same reason
|
|
-- `run_after` is in `claim`: `''` sorts before every stamp, so "never" and
|
|
-- "not due" need no coalesce and no special case. A cadence worded in a local
|
|
-- timezone is turned into one of these before it is stored — see
|
|
-- `pipeline/schedule.py`, which is the only place a zone is read.
|
|
--
|
|
-- `last_job_id` is what stops a schedule piling up behind itself: the firing
|
|
-- checks whether that job is still queued or running before starting another.
|
|
CREATE TABLE IF NOT EXISTS schedules (
|
|
id INTEGER PRIMARY KEY,
|
|
name TEXT NOT NULL,
|
|
kind TEXT NOT NULL,
|
|
label TEXT NOT NULL DEFAULT '',
|
|
payload TEXT NOT NULL DEFAULT '{}',
|
|
cadence TEXT NOT NULL,
|
|
enabled INTEGER NOT NULL DEFAULT 1,
|
|
next_run TEXT NOT NULL DEFAULT '',
|
|
last_run TEXT NOT NULL DEFAULT '',
|
|
last_job_id INTEGER NOT NULL DEFAULT 0,
|
|
created_at TEXT NOT NULL,
|
|
UNIQUE (name)
|
|
);
|
|
"""
|
|
|
|
#: Made after `_reconcile`, not with the table. An index names columns, and on
|
|
#: a file that predates one of them `CREATE INDEX` fails before the `ALTER`
|
|
#: that would have added it — which is a database that will not open, over an
|
|
#: index. Splitting them puts the two in the only order that always works.
|
|
INDEXES = """
|
|
CREATE INDEX IF NOT EXISTS idx_jobs_state ON jobs(state, id);
|
|
CREATE INDEX IF NOT EXISTS idx_jobs_cascade ON jobs(cascade_id, id);
|
|
CREATE INDEX IF NOT EXISTS idx_jobs_task ON jobs(task_id, id);
|
|
|
|
-- For the one question that asks about a moment rather than about an id:
|
|
-- which cascades ended just now, which is how a finished run stays on screen
|
|
-- for a few seconds. Without it that clause is a scan of every job kept, taken
|
|
-- on every push. `purge` is what bounds "every job kept".
|
|
CREATE INDEX IF NOT EXISTS idx_jobs_finished ON jobs(finished_at);
|
|
|
|
-- Partial, so it constrains only jobs that have not finished yet. This is what
|
|
-- makes "one pull at a time" a property of the system rather than of whichever
|
|
-- client asked: a second tab and a command collide here, where a
|
|
-- flag in one browser could not see the others. Enqueueing a duplicate hands
|
|
-- back the live job instead of failing.
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_jobs_dedup ON jobs(dedup_key)
|
|
WHERE dedup_key <> '' AND state IN ('queued', 'running');
|
|
"""
|
|
|
|
LIVE = ("queued", "running")
|
|
|
|
#: What holds a job back. Both halves read the dependency's own state rather
|
|
#: than a count kept alongside it, so nothing has to be incremented as jobs
|
|
#: land and nothing can be lost to a restart mid-run.
|
|
#: And whether a worker of its resource is free. `workers` arrives as JSON so
|
|
#: one parameter carries every count; a resource it does not mention gets one.
|
|
_HAS_ROOM = """
|
|
(SELECT count(*) FROM jobs r WHERE r.state = 'running' AND r.resource = q.resource)
|
|
< coalesce(json_extract(?2, '$."' || q.resource || '"'), 1)
|
|
"""
|
|
|
|
_READY = """
|
|
NOT EXISTS (SELECT 1 FROM json_each(q.depends_on) d
|
|
JOIN jobs dep ON dep.id = d.value
|
|
WHERE dep.state IN ('queued','running'))
|
|
AND (q.after = 'done' OR NOT EXISTS (
|
|
SELECT 1 FROM json_each(q.depends_on) d
|
|
JOIN jobs dep ON dep.id = d.value
|
|
WHERE dep.state IN ('failed','cancelled')))
|
|
"""
|
|
|
|
#: And whether its time has come. The second thing that holds a job back, and
|
|
#: it is the same kind of thing as the first: a clause in the claim, not an
|
|
#: object waiting anywhere. A job scheduled for an hour hence costs a row and
|
|
#: nothing else — no worker, no thread, no lease, no timer to be lost to a
|
|
#: restart — and the loop that looks for work every second is what notices
|
|
#: when the hour is up.
|
|
#:
|
|
#: `?1` is the same `now` the claim stamps with, so a job due at exactly this
|
|
#: instant is taken by the pass that finds it rather than the one after.
|
|
_DUE = "q.run_after <= ?1"
|
|
|
|
|
|
|
|
#: The lease of a job nobody is running any more. Older than every real
|
|
#: stamp, so `reclaim_stale` takes the row on its next look rather than after
|
|
#: a silence long enough to infer what the worker already knew.
|
|
_LONG_AGO = datetime.fromtimestamp(0, UTC).isoformat()
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now(UTC).isoformat()
|
|
|
|
|
|
def _in(seconds: float) -> str:
|
|
"""A stamp that many seconds from now, or `''` for no wait at all.
|
|
|
|
Written in the same UTC ISO form as every other time in this table,
|
|
because `run_after` is compared as text in `claim` and a second format in
|
|
the same column would sort by whichever digits it happened to start with.
|
|
Empty for nothing to wait for, which is what makes the comparison free of
|
|
a special case: `''` is before every stamp there is.
|
|
"""
|
|
if seconds <= 0:
|
|
return ""
|
|
return (datetime.now(UTC) + timedelta(seconds=seconds)).isoformat()
|
|
|
|
|
|
def on_its_own(name: str) -> str:
|
|
"""What to call the cascade minted for a lone job or a lone task.
|
|
|
|
Two different things would otherwise both be a cascade named `pull`: the
|
|
chain that starts with a pull and goes on to five more tasks, and the one
|
|
job somebody pressed by itself. `live_run` answers by name, so it would
|
|
hand the first back for the second — a Fetch that starts nothing, greys
|
|
its own button, and returns a run that is over a second later.
|
|
|
|
A task is not renamed, only the cascade wrapped round it: the `pull` task
|
|
of a lone pull really is the pull task, and the strip draws its box from
|
|
that name. It is the chain that was never asked for.
|
|
|
|
The space is the whole of the guarantee. A job kind and a task name are
|
|
identifiers — `pull`, `body_batch` — so nothing that mints a cascade
|
|
deliberately can ever collide with this.
|
|
"""
|
|
return f"{name} on its own"
|
|
|
|
|
|
class RunCancelled(Exception):
|
|
"""Raised by `enqueue` for a job under a run that has been killed.
|
|
|
|
The whole of how a kill stops what has not been queued yet. Nobody tells a
|
|
cascade it is over: its step tries to queue the next task, and the table
|
|
says no — across processes, after a restart, and when the kill landed in
|
|
the middle of the run. The queue logs it and drops the job.
|
|
"""
|
|
|
|
|
|
#: The largest `result` a job may leave on its row, as JSON. A result is a
|
|
#: report — counts, ids, an error — and the snapshot and every tally read it
|
|
#: back, so data returned instead of written to the app's store would make
|
|
#: every one of those queries slower for as long as the row is kept.
|
|
#:
|
|
#: Past this the job *fails*. It used to keep the keys and log a warning, which
|
|
#: is losing data where nobody is looking: the next job in a chain would get a
|
|
#: result with its payload missing and carry on. Raise the limit if a project
|
|
#: really wants bigger results; do not make it quiet.
|
|
MAX_RESULT = 64 * 1024
|
|
|
|
|
|
class ResultTooLarge(ValueError):
|
|
"""A handler returned more than `MAX_RESULT` bytes of JSON.
|
|
|
|
Raised by `Jobs.finish` before anything is written, so the row is not
|
|
touched. The worker ends the job `failed` with this message and does not
|
|
retry it: the same handler would return the same result.
|
|
"""
|
|
|
|
|
|
def _result(job_id: int, result: dict[str, Any] | None) -> str:
|
|
text = json.dumps(result or {})
|
|
if len(text) > MAX_RESULT:
|
|
raise ResultTooLarge(
|
|
f"returned {len(text) // 1024} KB, and a job's result is capped at "
|
|
f"{MAX_RESULT // 1024} KB (pipeline.table.MAX_RESULT). Write the "
|
|
f"data to the store and return its ids; keys returned: "
|
|
f"{', '.join(sorted(str(k) for k in (result or {}))[:20])}")
|
|
return text
|
|
|
|
|
|
def _entry(row: sqlite3.Row) -> NoticeEntry:
|
|
"""A `notices` row as the model, with `place` given back its real name."""
|
|
fields = dict(row)
|
|
fields["where"] = fields.pop("place")
|
|
return NoticeEntry.model_validate(fields)
|
|
|
|
|
|
class Jobs:
|
|
"""The jobs table on one file. Every statement against it is here."""
|
|
|
|
def __init__(self, path: str | Path) -> None:
|
|
self.path = Path(path).expanduser()
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
with closing(self._connect()) as conn, conn:
|
|
conn.execute("PRAGMA journal_mode=WAL")
|
|
conn.executescript(SCHEMA)
|
|
self._reconcile(conn)
|
|
conn.executescript(INDEXES)
|
|
|
|
@staticmethod
|
|
def _reconcile(conn: sqlite3.Connection) -> None:
|
|
"""Add any column `SCHEMA` has grown since this file was created."""
|
|
runs = {r["name"] for r in conn.execute("PRAGMA table_info(runs)")}
|
|
for column in ("cancelled_at", "label"):
|
|
if runs and column not in runs:
|
|
conn.execute(
|
|
f"ALTER TABLE runs ADD COLUMN {column} TEXT NOT NULL DEFAULT ''")
|
|
have = {r["name"] for r in conn.execute("PRAGMA table_info(jobs)")}
|
|
# The agent handoff, on a file made while it existed. Dropped rather
|
|
# than left: a column nobody writes is a question every reader of this
|
|
# table has to answer for itself, and the answer is that a job the
|
|
# model cannot do now fails and says why.
|
|
for gone in ("handoff", "answer", "instructions", "skill"):
|
|
if gone in have:
|
|
conn.execute(f"ALTER TABLE jobs DROP COLUMN {gone}")
|
|
for column, ddl in (("seen_at", "TEXT"),
|
|
("depends_on", "TEXT NOT NULL DEFAULT '[]'"),
|
|
("after", "TEXT NOT NULL DEFAULT 'ok'"),
|
|
("resource", "TEXT NOT NULL DEFAULT 'cpu'"),
|
|
("cascade_id", "INTEGER NOT NULL DEFAULT 0"),
|
|
("task_id", "INTEGER NOT NULL DEFAULT 0"),
|
|
# The retry policy, arriving on a file that
|
|
# predates it. Every default is what the file
|
|
# already behaved as: one attempt, no wait, due
|
|
# now — so a database migrated by this reads
|
|
# exactly as it did before the column existed.
|
|
("attempt", "INTEGER NOT NULL DEFAULT 0"),
|
|
("max_attempts", "INTEGER NOT NULL DEFAULT 1"),
|
|
("retry_delay", "REAL NOT NULL DEFAULT 0"),
|
|
("retry_backoff", "REAL NOT NULL DEFAULT 1"),
|
|
("run_after", "TEXT NOT NULL DEFAULT ''")):
|
|
if column not in have:
|
|
conn.execute(f"ALTER TABLE jobs ADD COLUMN {column} {ddl}")
|
|
|
|
def _connect(self) -> sqlite3.Connection:
|
|
# Thirty seconds, and whoever else opens this file uses the same number
|
|
# — see `newsdesk/storage/sql/store.py`, which says why the longer of
|
|
# two timeouts on one file is the one to keep.
|
|
conn = sqlite3.connect(self.path, timeout=30)
|
|
conn.row_factory = sqlite3.Row
|
|
conn.execute("PRAGMA busy_timeout=30000")
|
|
return conn
|
|
|
|
def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]:
|
|
with closing(self._connect()) as conn:
|
|
return conn.execute(sql, params).fetchall()
|
|
|
|
def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None:
|
|
with closing(self._connect()) as conn:
|
|
return conn.execute(sql, params).fetchone()
|
|
|
|
def _write(self, sql: str, params: Any = ()) -> int:
|
|
"""One statement in its own transaction. Returns the rowcount."""
|
|
with closing(self._connect()) as conn, conn:
|
|
return conn.execute(sql, params).rowcount
|
|
|
|
# --------------------------------------------------------------- queue
|
|
|
|
def enqueue(self, kind: str, label: str = "", payload: dict[str, Any] | None = None,
|
|
dedup_key: str = "", depends_on: Sequence[int] = (),
|
|
after: str = "ok", resource: str = "cpu",
|
|
cascade_id: int = 0, task_id: int = 0, delay: float = 0.0,
|
|
max_attempts: int = 1, retry_delay: float = 0.0,
|
|
retry_backoff: float = 1.0) -> Job:
|
|
"""Queue one job. A duplicate hands back the live one instead of adding.
|
|
|
|
`dedup_key` is enforced by a partial unique index over the unfinished
|
|
states, so "one pull at a time" holds across every client at once. An
|
|
empty key opts out, for jobs where several at once is the point.
|
|
|
|
Returning the existing job rather than raising is deliberate: the
|
|
caller asked for that work to be underway, and it is.
|
|
|
|
The lookup and the insert share one `BEGIN IMMEDIATE` transaction, so
|
|
a live job that finishes during an enqueue finishes either before it,
|
|
and a new job is queued, or after it, and the caller is handed the
|
|
live one. An `IntegrityError` on the key cannot reach the caller. It
|
|
used to, when the insert failed on the index and the job had finished
|
|
by the time a second statement looked for it. Waiting for the write
|
|
lock goes through the connection's busy timeout like any other write.
|
|
|
|
`depends_on` holds this job back until those jobs are over — see the
|
|
column comments in `SCHEMA` for what `after` chooses between.
|
|
|
|
A job under a killed run is refused with `RunCancelled`. That is where
|
|
a kill reaches everything that had not been queued when the button was
|
|
pressed: the next task of a killed cascade, the batches a callback was
|
|
about to fan out. Checked in the job's own transaction, under its write
|
|
lock, so a kill cannot land between the check and the insert: it lands
|
|
before, and the job is refused, or after, and `cancel_run` finds the
|
|
job queued and ends it.
|
|
|
|
The refusal spares an `after='done'` job under a killed *task*, which
|
|
is the same carve-out `cancel_run` and `stop_task` make and for the
|
|
same reason: killing a task is "skip this one", and its final job is
|
|
what says so. Without it a kill landing while a task was still fanning
|
|
out its batches would refuse the final job too, and the task would sit
|
|
with nothing queued, nothing running and nobody to report that it had
|
|
been killed. A killed *cascade* still refuses it — there is nothing
|
|
left to report to.
|
|
|
|
`delay` queues it for later: the row goes in `queued` with a
|
|
`run_after` that many seconds hence, and `claim` passes over it until
|
|
then. It is the durable form of "in five minutes" — the row is written
|
|
now, so the wait survives the process that asked for it, and nothing is
|
|
alive in the meantime.
|
|
|
|
`max_attempts` and the two retry numbers are the bound `retry` will
|
|
enforce, stored on the row. The defaults are one attempt and no wait,
|
|
which is every job that has ever been queued here.
|
|
"""
|
|
if after not in ("ok", "done"):
|
|
raise ValueError(f"after must be 'ok' or 'done', not {after!r}")
|
|
if max_attempts < 1:
|
|
raise ValueError(f"max_attempts counts the first attempt, so it is "
|
|
f"at least 1, not {max_attempts!r}")
|
|
with closing(self._connect()) as conn, conn:
|
|
# Taken before the first read, so no other writer, in this process
|
|
# or another, can finish, cancel or queue a job between the lookup
|
|
# below and the insert. Waits out a busy file like any write, under
|
|
# the connection's busy timeout: IMMEDIATE takes the write lock at
|
|
# BEGIN, where SQLite calls the busy handler, rather than upgrading
|
|
# a read snapshot later, where it fails at once.
|
|
conn.execute("BEGIN IMMEDIATE")
|
|
killed = conn.execute(
|
|
"""SELECT id FROM runs WHERE cancelled_at <> ''
|
|
AND (id = ?1 OR (id = ?2 AND ?3 <> 'done'))""",
|
|
(int(cascade_id), int(task_id), after),
|
|
).fetchone()
|
|
if killed is not None:
|
|
raise RunCancelled(
|
|
f"run {killed['id']} was killed; {kind!r} is not queued")
|
|
# The same predicate as `idx_jobs_dedup`, read under the write
|
|
# lock: what it finds is still live when this commits, and when it
|
|
# finds nothing the insert cannot collide.
|
|
if dedup_key:
|
|
existing = conn.execute(
|
|
"""SELECT * FROM jobs
|
|
WHERE dedup_key=? AND state IN ('queued','running')
|
|
ORDER BY id DESC LIMIT 1""",
|
|
(dedup_key,),
|
|
).fetchone()
|
|
if existing is not None:
|
|
return Job.from_row(existing)
|
|
# A job queued with neither begins both: something asked for
|
|
# this on its own, so it is a cascade of one task of one job.
|
|
# Minted in the job's own transaction, so a refusal that rolls
|
|
# back takes its ids with it — a `runs` row with no job under it
|
|
# would be an identity for nothing.
|
|
#
|
|
# The cascade says it stands alone and the task does not: the
|
|
# task is the `pull` task however it was started, but a chain
|
|
# named `pull` is a different thing from this and has to stay
|
|
# findable. See `on_its_own`.
|
|
cascade_id = int(cascade_id) or self._mint(
|
|
conn, "cascade", on_its_own(kind), label)
|
|
task_id = int(task_id) or self._mint(conn, "task", kind, label)
|
|
row = conn.execute(
|
|
"""INSERT INTO jobs (kind, label, payload, state, dedup_key,
|
|
depends_on, after, resource, cascade_id, task_id,
|
|
max_attempts, retry_delay, retry_backoff,
|
|
run_after, created_at)
|
|
VALUES (?,?,?,'queued',?,?,?,?,?,?,?,?,?,?,?)
|
|
RETURNING *""",
|
|
(kind, label, json.dumps(payload or {}), dedup_key,
|
|
json.dumps([int(i) for i in depends_on]), after,
|
|
resource or "cpu", int(cascade_id), int(task_id),
|
|
int(max_attempts), float(retry_delay), float(retry_backoff),
|
|
_in(delay), _now()),
|
|
).fetchone()
|
|
return Job.from_row(row)
|
|
|
|
def claim(self, workers: Mapping[str, int] | None = None) -> Job | None:
|
|
"""Take the oldest ready job and mark it running, atomically.
|
|
|
|
One statement, so two workers racing cannot both take the same job —
|
|
which matters the moment a worker runs outside the web process. Ready
|
|
rather than merely queued: a job with unmet dependencies is passed over
|
|
and stays where it is, costing nothing while it waits, because nothing
|
|
is running it. That is the whole of "waiting" here — no timer, no
|
|
listener, no object holding a place.
|
|
|
|
`workers` is how many of each kind there are, and so how many jobs of
|
|
that resource may run at once. Counted
|
|
from the table rather than from anything this process is holding, so a
|
|
worker in a second process shares the same four model slots instead of
|
|
helping itself to four more. A resource nobody sized gets one.
|
|
|
|
`seen_at` is stamped with `started_at`, so the job is leased from the
|
|
moment it is taken: a worker that dies in its first second still leaves
|
|
a row that goes stale rather than one that never had a heartbeat at
|
|
all and has to be guessed about.
|
|
|
|
A job whose `run_after` has not come round is passed over the same way
|
|
a job with unmet dependencies is, and costs exactly as little. `attempt`
|
|
goes up here, in the statement that takes the row, because that is the
|
|
only place a count of runs cannot be lost or doubled — a worker that
|
|
dies a second later has still started this job once, and the row says
|
|
so without anyone having to write it down afterwards.
|
|
"""
|
|
with closing(self._connect()) as conn, conn:
|
|
row = conn.execute(
|
|
f"""UPDATE jobs SET state='running', started_at=?1, seen_at=?1,
|
|
attempt = attempt + 1
|
|
WHERE id = (SELECT q.id FROM jobs q WHERE q.state='queued'
|
|
AND {_DUE} AND {_READY} AND {_HAS_ROOM}
|
|
ORDER BY q.id LIMIT 1)
|
|
RETURNING *""",
|
|
(_now(), json.dumps(dict(workers or {}))),
|
|
).fetchone()
|
|
return Job.from_row(row) if row else None
|
|
|
|
def running(self) -> list[Job]:
|
|
"""Everything under way, oldest first."""
|
|
return [Job.from_row(r) for r in self._rows(
|
|
"SELECT * FROM jobs WHERE state='running' ORDER BY id")]
|
|
|
|
@contextmanager
|
|
def serialize_starts(self):
|
|
"""Serialize cascade discovery and creation across processes.
|
|
|
|
A separate SQLite lock file leaves the jobs database writable while
|
|
the task creates its batches. SQLite releases the lock on a crash.
|
|
"""
|
|
with closing(sqlite3.connect(str(self.path) + ".start-lock", timeout=60)) as conn:
|
|
conn.execute("BEGIN IMMEDIATE")
|
|
try:
|
|
yield
|
|
finally:
|
|
conn.rollback()
|
|
|
|
def new_run(self, kind: str, name: str = "", label: str = "") -> int:
|
|
"""Mint an id for a cascade or a task, before it has queued anything.
|
|
|
|
Its own series, not a job's. A task is a thing before any of its jobs
|
|
exist — it is what decides how many there will be — so taking the id
|
|
of whichever job it happened to queue first would be describing the
|
|
task by one of its own consequences. The same goes for what it is
|
|
called: `name` is what it is, `label` is how to say it.
|
|
"""
|
|
with closing(self._connect()) as conn, conn:
|
|
return self._mint(conn, kind, name, label)
|
|
|
|
@staticmethod
|
|
def _mint(conn: sqlite3.Connection, kind: str, name: str,
|
|
label: str = "") -> int:
|
|
"""One `runs` row, on the caller's connection, so it can share a
|
|
transaction with the job that will carry the id."""
|
|
return int(conn.execute(
|
|
"INSERT INTO runs (kind, name, label, created_at) VALUES (?,?,?,?)",
|
|
(kind, name, label or name, _now()),
|
|
).lastrowid or 0)
|
|
|
|
def run(self, run_id: int) -> Run | None:
|
|
"""What a cascade or task id names: its kind, its name, when it began."""
|
|
row = self._row("SELECT * FROM runs WHERE id=?", (int(run_id),))
|
|
return Run.from_row(row) if row else None
|
|
|
|
def of_cascade(self, cascade_id: int) -> list[Job]:
|
|
"""Every job of one cascade, oldest first."""
|
|
return [Job.from_row(r) for r in self._rows(
|
|
"SELECT * FROM jobs WHERE cascade_id=? ORDER BY id", (int(cascade_id),))]
|
|
|
|
def of_task(self, task_id: int) -> list[Job]:
|
|
"""Every job of one task: its batches, and the final job behind them."""
|
|
return [Job.from_row(r) for r in self._rows(
|
|
"SELECT * FROM jobs WHERE task_id=? ORDER BY id", (int(task_id),))]
|
|
|
|
def live_run(self, kind: str, name: str) -> Run | None:
|
|
"""The newest run of that kind and name with a job still going.
|
|
|
|
What "is a pull already under way?" means, and what no job kind can
|
|
answer: `solo` stops a second `pull` while the first runs and says
|
|
nothing once the run has moved on to bodies. A run is live for as long
|
|
as any of its jobs is, which is the span the Fetch button means.
|
|
|
|
The name is the whole of the question, which is why the cascade minted
|
|
for something standing alone is named apart — see `on_its_own`. A lone
|
|
`pull` job is not the pull chain, and answering with it would be this
|
|
saying yes to a question nobody asked.
|
|
"""
|
|
row = self._row(
|
|
"""SELECT * FROM runs r WHERE r.kind=? AND r.name=? AND r.cancelled_at=''
|
|
AND EXISTS (SELECT 1 FROM jobs j WHERE j.cascade_id=r.id
|
|
AND j.state IN ('queued','running'))
|
|
ORDER BY r.id DESC LIMIT 1""",
|
|
(kind, name),
|
|
)
|
|
return Run.from_row(row) if row else None
|
|
|
|
def outcomes(self, ids: Sequence[int]) -> list[Job]:
|
|
"""How each of those jobs ended, oldest first.
|
|
|
|
What a final job is handed, and the only thing it needs: seven done
|
|
and one failed is read off the jobs themselves, at the moment it is
|
|
asked, rather than counted up as they landed. There is nothing to
|
|
increment and so nothing to lose to a restart half way through.
|
|
"""
|
|
if not ids:
|
|
return []
|
|
holes = ",".join("?" for _ in ids)
|
|
return [Job.from_row(r) for r in self._rows(
|
|
f"SELECT * FROM jobs WHERE id IN ({holes}) ORDER BY id",
|
|
tuple(int(i) for i in ids))]
|
|
|
|
def blocked(self) -> list[Job]:
|
|
"""Queued jobs that `claim` would pass over. For the UI and for tests.
|
|
|
|
Both reasons, because from here they are the same fact: this row is in
|
|
the queue and is not going to be taken yet. Which of the two it is —
|
|
something it waits for, or a time it waits until — is on the row, in
|
|
`depends_on` and `run_after`.
|
|
"""
|
|
return [Job.from_row(r) for r in self._rows(
|
|
f"""SELECT * FROM jobs q WHERE q.state='queued'
|
|
AND NOT ({_DUE} AND {_READY}) ORDER BY q.id""",
|
|
(_now(),))]
|
|
|
|
def progress(self, job_id: int, done: int = 0, total: int = 0,
|
|
message: str = "") -> None:
|
|
self._write("UPDATE jobs SET done=?, total=?, message=? WHERE id=?",
|
|
(done, total, message, job_id))
|
|
|
|
def finish(self, job_id: int, state: str, result: dict[str, Any] | None = None,
|
|
error: str = "") -> Job | None:
|
|
"""Move a job to one of the terminal states, once, and settle what waited.
|
|
|
|
A job that ended badly is the moment anything depending on it can be
|
|
answered, so it is answered here rather than swept for later: nothing
|
|
should sit queued forever waiting on something that is never coming.
|
|
"""
|
|
self._write(
|
|
"""UPDATE jobs SET state=?, result=?, error=?, finished_at=?
|
|
WHERE id=? AND state NOT IN ('done','failed','cancelled')""",
|
|
(state, _result(job_id, result), error, _now(), job_id),
|
|
)
|
|
if state in ("failed", "cancelled"):
|
|
self._settle_dependents()
|
|
return self.get(job_id)
|
|
|
|
def retry(self, job_id: int, *, error: str = "",
|
|
delay: float | None = None) -> Job | None:
|
|
"""End an attempt that went badly: another one later, or give up.
|
|
|
|
The one place the bound is read, and so the one place a job stops
|
|
being tried. Three endings, and which one it is depends on nothing the
|
|
caller knows:
|
|
|
|
asked to stop `cancelled`. A Kill that landed while the attempt
|
|
ran must not be undone by the attempt failing —
|
|
that would be a button that quietly did nothing.
|
|
attempts left back to `queued`, with `run_after` set and the
|
|
lease handed back. Nothing runs it, nothing holds a
|
|
place for it, and whatever depends on it goes on
|
|
waiting, because it is still live.
|
|
none left `failed`, and `Job.exhausted` from then on.
|
|
|
|
The wait is `retry_delay` multiplied by `retry_backoff` once for each
|
|
attempt already made, unless `delay` overrides it — which is how a
|
|
server's own answer to "come back in 120 seconds" is honoured instead
|
|
of guessed at. A hint does not buy an extra attempt: the bound is
|
|
ours, the wait is the world's.
|
|
|
|
The error is kept on the row either way, so a job sitting in the queue
|
|
with an hour to wait still says what went wrong last time rather than
|
|
looking like ordinary work nobody has got to yet.
|
|
|
|
Read then written, rather than one statement like everything else
|
|
here: the arithmetic wants the row's own numbers, and this is the one
|
|
method whose row is `running` and held by the worker calling it, so
|
|
there is no second writer to lose a race with.
|
|
"""
|
|
job = self.get(job_id)
|
|
if job is None:
|
|
return None
|
|
if job.cancel_requested:
|
|
return self.finish(job_id, "cancelled", error=error)
|
|
if job.attempt >= job.max_attempts:
|
|
return self.finish(job_id, "failed", error=error)
|
|
wait = delay if delay is not None else (
|
|
job.retry_delay * job.retry_backoff ** max(job.attempt - 1, 0))
|
|
self._write(
|
|
"""UPDATE jobs SET state='queued', error=?, run_after=?, message='',
|
|
started_at=NULL, seen_at=NULL
|
|
WHERE id=? AND state='running'""",
|
|
(error, _in(wait), job_id),
|
|
)
|
|
return self.get(job_id)
|
|
|
|
def waiting(self) -> list[Job]:
|
|
"""Queued jobs whose next attempt is still to come, oldest first."""
|
|
return [Job.from_row(r) for r in self._rows(
|
|
"SELECT * FROM jobs WHERE state='queued' AND run_after > ? ORDER BY id",
|
|
(_now(),))]
|
|
|
|
def _settle_dependents(self) -> None:
|
|
"""End queued jobs that can no longer run, and their dependents in turn.
|
|
|
|
Only jobs that wanted success. Killing one job is not stopping the run
|
|
it belongs to — it is dropping that batch, exactly as a batch that
|
|
failed is dropped, and the work comes round next time because every
|
|
task rediscovers its own backlog. So the job waiting to report on the
|
|
batches still runs, counts this one among those that did not land, and
|
|
the chain carries on: to a cascade a killed batch and a failed one are
|
|
the same thing. Stopping a whole task or a whole cascade is
|
|
`cancel_run`, which is a different button and says so.
|
|
|
|
Looped because a dependent is itself depended on, and each pass ends
|
|
one more generation. It terminates: every pass moves at least one job
|
|
out of `queued`, and nothing here puts one back.
|
|
"""
|
|
while self._write(
|
|
"""UPDATE jobs SET state='failed', finished_at=?, message='',
|
|
error='a job it waited for did not finish'
|
|
WHERE jobs.state = 'queued' AND jobs.after = 'ok'
|
|
AND EXISTS (SELECT 1 FROM json_each(jobs.depends_on) d
|
|
JOIN jobs dep ON dep.id = d.value
|
|
WHERE dep.state IN ('failed','cancelled'))""",
|
|
(_now(),),
|
|
):
|
|
pass
|
|
|
|
def get(self, job_id: int) -> Job | None:
|
|
row = self._row("SELECT * FROM jobs WHERE id=?", (job_id,))
|
|
return Job.from_row(row) if row else None
|
|
|
|
def last(self, kind: str, *, states: tuple[str, ...] = ("done",)) -> Job | None:
|
|
"""The most recent job of a kind that reached one of `states`.
|
|
|
|
Ordered by `id` rather than `finished_at`: the id is the order things
|
|
were asked for, which is what "the last pull" means.
|
|
"""
|
|
placeholders = ",".join("?" for _ in states)
|
|
row = self._row(
|
|
f"""SELECT * FROM jobs WHERE kind=? AND state IN ({placeholders})
|
|
ORDER BY id DESC LIMIT 1""",
|
|
(kind, *states),
|
|
)
|
|
return Job.from_row(row) if row else None
|
|
|
|
# -------------------------------------------------------- cancelling
|
|
|
|
def request_cancel(self, job_id: int) -> Job | None:
|
|
"""Ask for a job to stop.
|
|
|
|
A queued job is finished here and now — nothing is part-way through.
|
|
A running one only gets the request recorded; the worker notices on
|
|
its next `context.check()` and writes the terminal state itself,
|
|
because only it knows when it has reached a point where stopping is
|
|
safe.
|
|
|
|
A job waiting for its next attempt is a queued job and ends here with
|
|
the rest of them. That is the point of the wait being a column rather
|
|
than a sleeping worker: there is a row to cancel, from any process, at
|
|
any moment of the hour it is waiting.
|
|
"""
|
|
job = self.get(job_id)
|
|
if job is None or not job.live:
|
|
return job
|
|
if job.state == "queued":
|
|
return self.finish(job_id, "cancelled")
|
|
self._write("UPDATE jobs SET cancel_requested=1 WHERE id=?", (job_id,))
|
|
return self.get(job_id)
|
|
|
|
def cancel_run(self, run_id: int) -> int:
|
|
"""Kill a run — a task or a cascade, the same operation for both.
|
|
|
|
Three things: flag the row, end the queued jobs carrying that id, and
|
|
ask the running ones. Plus the fourth that is not here: `enqueue`
|
|
refuses anything later queued under it. That is the whole of how a kill
|
|
stops the pipeline, and the flag is what makes it hold across
|
|
processes, after a restart, and when the kill lands mid-run.
|
|
|
|
The one asymmetry is deliberate and is what the two buttons mean. A
|
|
cascade's id takes everything in it. A task's id spares the job waiting
|
|
on `after='done'` — its final job — because reporting is not work:
|
|
killing a task is "skip this one", not "make it never end". So the
|
|
final job still runs and still calls the step, and what it hands over
|
|
says the *task* was killed rather than only how many batches ended
|
|
cancelled — the two are indistinguishable in the counts and mean
|
|
opposite things, and `Jobs.cancelled` is where that flag is read back.
|
|
What to do about it is then the cascade's decision, not this call's:
|
|
newsdesk's skips the stage and carries on. The step's attempt to queue
|
|
the next task goes ahead if only the task was killed, and is refused if
|
|
the cascade was killed too.
|
|
"""
|
|
self._write(
|
|
"UPDATE runs SET cancelled_at=? WHERE id=? AND cancelled_at=''",
|
|
(_now(), int(run_id)))
|
|
# Queued jobs end here and now — nothing is part-way through. A running
|
|
# one is asked and stops at its next `context.check()`, because only it
|
|
# knows when it has reached a point where stopping is safe.
|
|
where = "(cascade_id=?1 OR (task_id=?1 AND after<>'done'))"
|
|
ended = self._write(
|
|
f"""UPDATE jobs SET state='cancelled', finished_at=?2, message=''
|
|
WHERE {where} AND state='queued'""",
|
|
(int(run_id), _now()),
|
|
)
|
|
asked = self._write(
|
|
f"UPDATE jobs SET cancel_requested=1 WHERE {where} AND state='running'",
|
|
(int(run_id),))
|
|
self._settle_dependents()
|
|
return ended + asked
|
|
|
|
def stop_task(self, task_id: int, error: str = "") -> int:
|
|
"""End a task's queued batches without killing the task. How many.
|
|
|
|
What a batch calls when the failure it just met is the backend's
|
|
answer to the whole run rather than to its own rows — a spend limit, a
|
|
login that has expired, no model on the machine. The siblings behind it
|
|
would each make the same call and get the same refusal, and each put
|
|
the same notice on screen; twenty-three of those is the bug this
|
|
exists to stop.
|
|
|
|
Three differences from `cancel_run`, and each of them is the point:
|
|
|
|
the run is not flagged nothing was killed. `Tally.killed` means a
|
|
person pressed Kill and the step is to be
|
|
skipped, and borrowing the flag to mean
|
|
"the model said no" would make the two
|
|
indistinguishable in the one place that
|
|
can still tell them apart.
|
|
running jobs are left their requests are in the air and are a
|
|
bill either way. They end however they end
|
|
— including the one that called this, which
|
|
keeps its own `failed` rather than being
|
|
cancelled by its own instruction.
|
|
the reason is kept written to `error` on each row, so a batch
|
|
that never ran says why on screen instead
|
|
of looking like one somebody dropped.
|
|
|
|
The final job is spared, as it is by a kill and for the same reason:
|
|
reporting is not work, and a run that stopped early is exactly the one
|
|
that has something to report.
|
|
"""
|
|
ended = self._write(
|
|
"""UPDATE jobs SET state='cancelled', finished_at=?, message='',
|
|
error=? WHERE task_id=? AND after<>'done' AND state='queued'""",
|
|
(_now(), error, int(task_id)),
|
|
)
|
|
if ended:
|
|
self._settle_dependents()
|
|
return ended
|
|
|
|
def cancelled(self, run_id: int) -> bool:
|
|
"""Whether this task or cascade has been killed."""
|
|
row = self._row("SELECT cancelled_at FROM runs WHERE id=?", (int(run_id),))
|
|
return bool(row and row[0])
|
|
|
|
def cancel_requested(self, job_id: int) -> bool:
|
|
row = self._row("SELECT cancel_requested FROM jobs WHERE id=?", (job_id,))
|
|
return bool(row[0]) if row else False
|
|
|
|
# ---------------------------------------------------------- schedules
|
|
|
|
#: Every schedule, with how its last job ended. One statement, because the
|
|
#: list and the socket push both want the join and neither wants a query
|
|
#: per row — and because a row read in two halves can disagree with itself.
|
|
_SCHEDULES = """
|
|
SELECT s.*, coalesce(j.state, '') AS last_state
|
|
FROM schedules s LEFT JOIN jobs j ON j.id = s.last_job_id
|
|
"""
|
|
|
|
def schedules(self) -> list[Schedule]:
|
|
"""Every schedule, by name. The order the UI lists them in.
|
|
|
|
By name rather than by id, because the name is what a person reads and
|
|
an id is the order they happened to be added in — including the order
|
|
the app's declared defaults were imported in, which is not a fact about
|
|
anything.
|
|
"""
|
|
return [Schedule.from_row(r) for r in
|
|
self._rows(self._SCHEDULES + " ORDER BY s.name")]
|
|
|
|
def schedule(self, schedule_id: int) -> Schedule | None:
|
|
row = self._row(self._SCHEDULES + " WHERE s.id=?", (int(schedule_id),))
|
|
return Schedule.from_row(row) if row else None
|
|
|
|
def add_schedule(self, name: str, kind: str, cadence: str, *,
|
|
payload: dict[str, Any] | None = None, label: str = "",
|
|
enabled: bool = True, next_run: str = "",
|
|
only_if_new: bool = False) -> Schedule | None:
|
|
"""Write one schedule. `None` when `only_if_new` and the name is taken.
|
|
|
|
`only_if_new` is most of reconciliation: the queue offers every
|
|
declared default on every start, and a name that already has a row is
|
|
left as the user edited it — cadence, payload, enabled and all. Which
|
|
is what makes a declaration a starting point rather than a setting
|
|
somebody has to fight. The label is the exception, and
|
|
`relabel_schedule` is where it is refreshed.
|
|
|
|
One statement, so two workers starting at once insert a declared
|
|
default once between them rather than racing a read against a write.
|
|
The conflict is the second one finding the first one's row.
|
|
"""
|
|
clash = "DO NOTHING" if only_if_new else """DO UPDATE SET
|
|
kind=excluded.kind, label=excluded.label,
|
|
payload=excluded.payload, cadence=excluded.cadence,
|
|
enabled=excluded.enabled, next_run=excluded.next_run"""
|
|
with closing(self._connect()) as conn, conn:
|
|
row = conn.execute(
|
|
f"""INSERT INTO schedules (name, kind, label, payload, cadence,
|
|
enabled, next_run, created_at)
|
|
VALUES (?,?,?,?,?,?,?,?)
|
|
ON CONFLICT (name) {clash}
|
|
RETURNING id""",
|
|
(name, kind, label, json.dumps(payload or {}), cadence,
|
|
int(bool(enabled)), next_run, _now()),
|
|
).fetchone()
|
|
return self.schedule(row["id"]) if row else None
|
|
|
|
def relabel_schedule(self, name: str, kind: str, label: str) -> bool:
|
|
"""Give a declared row the label its declaration now carries.
|
|
|
|
Only when the kind still matches: a row whose kind was changed is no
|
|
longer the declared schedule in anything but its name, and a label
|
|
written for another job would misname it. One statement, like
|
|
`add_schedule`, and a no-op when the label is already right, so a
|
|
restart writes nothing.
|
|
"""
|
|
return bool(self._write(
|
|
"UPDATE schedules SET label=? WHERE name=? AND kind=? AND label<>?",
|
|
(label, name, kind, label)))
|
|
|
|
#: What `update_schedule` will write, and the whole of it. A whitelist
|
|
#: rather than trust, because the column names go into the statement: the
|
|
#: values are parameters but the keys cannot be, so this is the boundary.
|
|
#: `name` is not here — it is the handle a declared default is matched on,
|
|
#: and renaming a row would hand its edits back to a freshly inserted one.
|
|
EDITABLE = ("kind", "label", "payload", "cadence", "enabled",
|
|
"next_run", "last_run", "last_job_id")
|
|
|
|
def update_schedule(self, schedule_id: int, **fields: Any) -> Schedule | None:
|
|
"""Change some of one schedule's columns. The row as it now stands.
|
|
|
|
Also how a firing that decided not to fire moves on: setting
|
|
`next_run` alone is "skip this slot", which is what happens when the
|
|
last job of this schedule is still going. Nothing else is touched, so
|
|
`last_run` goes on saying when it last actually ran.
|
|
"""
|
|
wanted = {k: v for k, v in fields.items() if k in self.EDITABLE}
|
|
if not wanted:
|
|
return self.schedule(schedule_id)
|
|
if "payload" in wanted and not isinstance(wanted["payload"], str):
|
|
wanted["payload"] = json.dumps(wanted["payload"] or {})
|
|
if "enabled" in wanted:
|
|
wanted["enabled"] = int(bool(wanted["enabled"]))
|
|
sets = ", ".join(f"{k}=?" for k in wanted)
|
|
self._write(f"UPDATE schedules SET {sets} WHERE id=?",
|
|
(*wanted.values(), int(schedule_id)))
|
|
return self.schedule(schedule_id)
|
|
|
|
def delete_schedule(self, schedule_id: int) -> bool:
|
|
"""Forget one. Not an error when it has already gone — two tabs."""
|
|
return bool(self._write("DELETE FROM schedules WHERE id=?",
|
|
(int(schedule_id),)))
|
|
|
|
def due_schedules(self, now: str) -> list[Schedule]:
|
|
"""Enabled schedules whose time has come, oldest due first.
|
|
|
|
Text comparison against a UTC ISO `now`, for the reason `claim` uses
|
|
one: `''` sorts before every stamp, so a schedule with no next run —
|
|
one that has never been given a time — is simply not due, with no
|
|
coalesce and no special case.
|
|
|
|
Oldest first so that a queue coming back after a long silence fires
|
|
what was due longest ago before what was due a minute ago, which is
|
|
the order a person would have run them in.
|
|
"""
|
|
return [Schedule.from_row(r) for r in self._rows(
|
|
self._SCHEDULES + """ WHERE s.enabled=1 AND s.next_run <> ''
|
|
AND s.next_run <= ? ORDER BY s.next_run""",
|
|
(now,))]
|
|
|
|
def mark_fired(self, schedule_id: int, job_id: int, next_run: str) -> Schedule | None:
|
|
"""Record that this schedule just queued that job, and when it is next.
|
|
|
|
One statement, because the three move together: a `last_job_id` written
|
|
without the `next_run` beside it would be a schedule that fires again
|
|
on the following pass, every pass, for as long as that job runs.
|
|
"""
|
|
self._write(
|
|
"""UPDATE schedules SET last_run=?, last_job_id=?, next_run=?
|
|
WHERE id=?""",
|
|
(_now(), int(job_id), next_run, int(schedule_id)))
|
|
return self.schedule(schedule_id)
|
|
|
|
# ------------------------------------------------------------ repairs
|
|
|
|
def heartbeat(self, ids: Sequence[int]) -> int:
|
|
"""Say those running jobs are still being run. Returns how many rows.
|
|
|
|
The other half of `reclaim_stale`. Called on a timer by the worker
|
|
that holds them, and against the table rather than any object, because
|
|
the process that has to read it is a different one.
|
|
|
|
Only `running` rows are touched: a job that finished between the beat
|
|
being scheduled and the beat landing must not have its lease refreshed
|
|
under its terminal state.
|
|
"""
|
|
if not ids:
|
|
return 0
|
|
holes = ",".join("?" for _ in ids)
|
|
return self._write(
|
|
f"UPDATE jobs SET seen_at=? WHERE state='running' AND id IN ({holes})",
|
|
(_now(), *(int(i) for i in ids)))
|
|
|
|
def release(self, ids: Sequence[int]) -> int:
|
|
"""Hand back the lease on jobs this worker is walking away from.
|
|
|
|
A process on its way out knows something `reclaim_stale` can only wait
|
|
to infer: that these jobs have nobody running them as of now. Saying so
|
|
is what makes a restart pick them up at once rather than a lease later.
|
|
Only a crash, which by definition says nothing, pays the wait.
|
|
|
|
The lease is expired rather than the job re-queued, and that is the
|
|
point of doing it this way: the row stays `running`, so a handler
|
|
thread that outlives the shutdown still writes its own ending into it
|
|
rather than being overwritten by a second run of itself. That ending
|
|
is the thread's to write — see `JobQueue._execute`, where the handler
|
|
call and the terminal write are one function — which is what makes
|
|
expiring the lease the safe half-measure it reads as. Only the next
|
|
`reclaim_stale` moves the row, and only if that ending never came.
|
|
"""
|
|
if not ids:
|
|
return 0
|
|
holes = ",".join("?" for _ in ids)
|
|
return self._write(
|
|
f"UPDATE jobs SET seen_at=? WHERE state='running' AND id IN ({holes})",
|
|
(_LONG_AGO, *(int(i) for i in ids)))
|
|
|
|
def purge(self, older_than: float) -> int:
|
|
"""Delete jobs that finished more than `older_than` seconds ago.
|
|
|
|
Returns how many. Without this both tables grow for as long as the app
|
|
runs. Three things are kept whatever their age:
|
|
|
|
anything not finished a queued or running row is work, not
|
|
history.
|
|
the newest ending of each `last(kind)` answers "when did this last
|
|
kind in each state succeed" from the table, and a status
|
|
line saying "never pulled" because the
|
|
last pull was five weeks ago is wrong.
|
|
a run with jobs left a run is deleted only once nothing
|
|
points at it and it is old itself — one
|
|
minted a moment ago may have no jobs yet
|
|
because its finder is still looking.
|
|
|
|
One transaction, so a snapshot never sees jobs gone and their runs
|
|
still there.
|
|
"""
|
|
cutoff = (datetime.now(UTC) - timedelta(seconds=older_than)).isoformat()
|
|
with closing(self._connect()) as conn, conn:
|
|
gone = conn.execute(
|
|
"""DELETE FROM jobs
|
|
WHERE state IN ('done','failed','cancelled')
|
|
AND finished_at IS NOT NULL AND finished_at < ?
|
|
AND id NOT IN (SELECT MAX(id) FROM jobs
|
|
WHERE state IN ('done','failed','cancelled')
|
|
GROUP BY kind, state)""",
|
|
(cutoff,)).rowcount
|
|
conn.execute(
|
|
"""DELETE FROM runs WHERE created_at < ?
|
|
AND id NOT IN (SELECT cascade_id FROM jobs)
|
|
AND id NOT IN (SELECT task_id FROM jobs)""",
|
|
(cutoff,))
|
|
return int(gone or 0)
|
|
|
|
# ------------------------------------------------------------- notices
|
|
|
|
def record_notice(self, notice: Notice) -> NoticeEntry:
|
|
"""Write one notice into the log, and say how often it has been said.
|
|
|
|
One statement, for the reason every write here is one: two batches of
|
|
one run meet the same expired login in the same second, and a read
|
|
followed by a write would file it twice. The conflict is the second
|
|
batch finding the first one's row.
|
|
|
|
A repeat takes the newer wording as well as the count. The text is a
|
|
sentence about now — the refused article's title, the feed that died
|
|
this time — and a log showing last week's sentence beside today's
|
|
count is describing two different moments.
|
|
|
|
`count == 1` on what comes back is how a caller knows this is news. See
|
|
`NoticeEntry`.
|
|
"""
|
|
now = _now()
|
|
with closing(self._connect()) as conn, conn:
|
|
row = conn.execute(
|
|
"""INSERT INTO notices (code, key, level, kind, text, fix,
|
|
detail, place, first_at, last_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT (code, key) DO UPDATE SET
|
|
count = count + 1, last_at = excluded.last_at,
|
|
level = excluded.level, kind = excluded.kind,
|
|
text = excluded.text, fix = excluded.fix,
|
|
detail = excluded.detail, place = excluded.place
|
|
RETURNING *""",
|
|
(notice.code, notice.key, notice.level, notice.kind, notice.text,
|
|
notice.fix, notice.detail, notice.where, now, now)).fetchone()
|
|
return _entry(row)
|
|
|
|
def notices(self, limit: int = 200) -> list[NoticeEntry]:
|
|
"""The log, most recently said first."""
|
|
return [_entry(row) for row in self._rows(
|
|
"SELECT * FROM notices ORDER BY last_at DESC, id DESC LIMIT ?",
|
|
(int(limit),))]
|
|
|
|
def clear_notice(self, notice_id: int) -> bool:
|
|
"""Done with this one. The same failure afterwards is news again."""
|
|
return bool(self._write("DELETE FROM notices WHERE id=?", (int(notice_id),)))
|
|
|
|
def clear_notices(self) -> int:
|
|
return self._write("DELETE FROM notices")
|
|
|
|
def purge_notices(self, quiet_for: float) -> int:
|
|
"""Forget what has not been said again for `quiet_for` seconds.
|
|
|
|
By `last_at`, not `first_at`: a feed that has failed every evening for
|
|
a month is a row a month old that was true an hour ago, and deleting
|
|
it would make tonight's failure look like the first. What goes is what
|
|
stopped happening, which is the only kind of notice that can be
|
|
dropped without anyone having read it — and dropping it is what lets
|
|
the same failure, months on, interrupt you again.
|
|
"""
|
|
cutoff = (datetime.now(UTC) - timedelta(seconds=quiet_for)).isoformat()
|
|
return self._write("DELETE FROM notices WHERE last_at < ?", (cutoff,))
|
|
|
|
def reclaim_stale(self, older_than: float = 30.0, *,
|
|
except_ids: Sequence[int] = ()) -> int:
|
|
"""Put jobs whose worker stopped saying it was there back in the queue.
|
|
|
|
A `running` row is a claim plus a lease: `claim` stamps `seen_at` and
|
|
the worker holding the job restamps it every few seconds. So a row
|
|
whose lease has run out is one whose worker went away mid-job, and
|
|
re-queueing is the right reading of that — the work was asked for and
|
|
never happened. Handlers have to tolerate being re-run.
|
|
|
|
The lease is what makes this safe on a file two processes share. Left
|
|
as "every `running` row", a second worker starting up would re-queue
|
|
the first one's live jobs and both would run them; `older_than` is how
|
|
long a worker has to have been silent before that reading is the only
|
|
one left. Which is also why this runs on a timer and not only at
|
|
start: whichever process is still alive is the one that cleans up
|
|
after the one that is not.
|
|
|
|
A row with no `seen_at` at all — claimed by a version of this file
|
|
that predates the column — falls back to `started_at`, so an old
|
|
orphan is still reclaimed once it is old enough.
|
|
|
|
One of these does not come back: a job that had been asked to stop
|
|
before its worker died ends `cancelled` instead of returning to the
|
|
queue. Re-queueing it would be a Kill that a crash quietly undid, and
|
|
the request is in the table precisely so that it outlives the process
|
|
that was supposed to act on it.
|
|
|
|
A worker excludes the ids it still holds. Their expired leases mean
|
|
its loop stalled, not that its handler threads stopped; renew them and
|
|
warn instead of running the same handler twice in this process.
|
|
"""
|
|
cutoff = (datetime.now(UTC) - timedelta(seconds=older_than)).isoformat()
|
|
ids = tuple(int(i) for i in except_ids)
|
|
holes = ",".join("?" for _ in ids)
|
|
if ids:
|
|
renewed = self._write(
|
|
f"UPDATE jobs SET seen_at=? WHERE state='running' "
|
|
f"AND coalesce(seen_at, started_at, created_at) < ? "
|
|
f"AND id IN ({holes})", (_now(), cutoff, *ids))
|
|
if renewed:
|
|
log.warning("worker loop stalled; renewed %d held job lease(s)", renewed)
|
|
excluded = f" AND id NOT IN ({holes})" if ids else ""
|
|
taken = self._write(
|
|
"""UPDATE jobs SET
|
|
state = CASE WHEN cancel_requested THEN 'cancelled' ELSE 'queued' END,
|
|
finished_at = CASE WHEN cancel_requested THEN ?1 ELSE NULL END,
|
|
started_at=NULL, seen_at=NULL, cancel_requested=0
|
|
WHERE state='running'
|
|
AND coalesce(seen_at, started_at, created_at) < ?2""" + excluded,
|
|
(_now(), cutoff, *ids))
|
|
if taken:
|
|
# Cheap and only on a reclaim, which is rare. A job that came back
|
|
# cancelled is a job something may have been waiting for.
|
|
self._settle_dependents()
|
|
return taken
|
|
|
|
# ------------------------------------------------------------ reading
|
|
|
|
#: How long a finished cascade stays in `cascades`. Nothing depends on it
|
|
#: — it is so that a run you watched is still on screen the moment after
|
|
#: it lands, rather than vanishing on the push that says it is over, which
|
|
#: is the one push you wanted to see.
|
|
LINGER = timedelta(seconds=15)
|
|
|
|
def cascades(self) -> list[CascadeProgress]:
|
|
"""Every cascade with something live in it, tallied task by task.
|
|
|
|
One query and a grouping. The rows are every job of those cascades and
|
|
not only the live ones, because a task's landed jobs are most of what
|
|
a bar of it is and they are long gone from `snapshot`'s `recent`.
|
|
|
|
`finished_at` is set only when a job reaches a terminal state, so the
|
|
second half of that `IN` needs no state list — and without one it can
|
|
use the index rather than reading every job that ever finished.
|
|
|
|
Ordered by the id of the first job of each, which for a cascade whose
|
|
tasks queue one another is the order they ran.
|
|
"""
|
|
rows = self._rows(
|
|
"""SELECT j.cascade_id, j.task_id, j.id, j.state, j.label,
|
|
c.name AS cascade_name, t.name AS task_name,
|
|
t.label AS task_label
|
|
FROM jobs j
|
|
LEFT JOIN runs c ON c.id = j.cascade_id
|
|
LEFT JOIN runs t ON t.id = j.task_id
|
|
WHERE j.cascade_id IN (
|
|
SELECT cascade_id FROM jobs WHERE state IN ('queued','running')
|
|
UNION
|
|
SELECT cascade_id FROM jobs WHERE finished_at > ?)
|
|
ORDER BY j.id""",
|
|
((datetime.now(UTC) - self.LINGER).isoformat(),))
|
|
|
|
runs: dict[int, CascadeProgress] = {}
|
|
tasks: dict[tuple[int, int], TaskProgress] = {}
|
|
for row in rows:
|
|
run = runs.get(row["cascade_id"])
|
|
if run is None:
|
|
run = runs[row["cascade_id"]] = CascadeProgress(
|
|
id=row["cascade_id"],
|
|
# A job queued outside any cascade begins one named after
|
|
# itself and for standing alone, so there is always a name
|
|
# and no chain answers to it; the fallback is for a row
|
|
# from before `runs` existed.
|
|
name=row["cascade_name"] or "")
|
|
task = tasks.get((row["cascade_id"], row["task_id"]))
|
|
if task is None:
|
|
# Both off the `runs` row, minted before any of these jobs
|
|
# existed. The first job's label is the fallback, for a run
|
|
# from before the column and for a job queued on its own.
|
|
task = tasks[(row["cascade_id"], row["task_id"])] = TaskProgress(
|
|
id=row["task_id"], name=row["task_name"] or "",
|
|
label=row["task_label"] or row["label"] or "")
|
|
run.tasks.append(task)
|
|
# One counter per state, by name. A state the model has no field
|
|
# for is skipped rather than raised: this is a drawing, and it
|
|
# must not be what takes the queue's whole snapshot down.
|
|
if row["state"] in TaskProgress.model_fields:
|
|
setattr(task, row["state"], getattr(task, row["state"]) + 1)
|
|
return list(runs.values())
|
|
|
|
def snapshot(self, recent: int = 30) -> JobsSnapshot:
|
|
"""The whole queue as a UI shows it, in one read.
|
|
|
|
Live and finished jobs in one statement, so a job that finishes while
|
|
this runs is in one list or the other and never in both.
|
|
"""
|
|
rows = [Job.from_row(r) for r in self._rows(
|
|
"""SELECT * FROM jobs WHERE state IN ('queued','running')
|
|
UNION ALL
|
|
SELECT * FROM (SELECT * FROM jobs
|
|
WHERE state IN ('done','failed','cancelled')
|
|
ORDER BY id DESC LIMIT ?)""",
|
|
(recent,),
|
|
)]
|
|
live = sorted((j for j in rows if j.state in ("queued", "running")),
|
|
key=lambda j: j.id)
|
|
return JobsSnapshot(
|
|
cascades=self.cascades(),
|
|
running=[j for j in live if j.state == "running"],
|
|
queued=[j for j in live if j.state == "queued"],
|
|
recent=sorted((j for j in rows if j.state not in ("queued", "running")),
|
|
key=lambda j: j.id, reverse=True),
|
|
)
|