Files
jobboard/pipeline/queue.py
T
WolfandClaude Opus 5.5 9d666d0937 Bring in pipeline, auth, deploy and docs from the boilerplate
roledesk is built on the boilerplate from here on: pipeline 2.0 (queue and
scheduler), auth (one password, sessions, passkeys), the deploy kit and its
docs, plus the coupling and packaging tests. Downstream copies, unedited.

Refreshed from boilerplate/d56949f7d0f6c0bfe5745478e09007bfad67ae87

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:16 +02:00

1136 lines
57 KiB
Python

"""The worker: one loop, draining the jobs table.
Deliberately not an in-memory queue. The table *is* the queue, which buys three
things that matter more here than the microseconds it costs:
it survives a restart a pull interrupted by a reload comes back queued
rather than vanishing half-done.
it is the history the list the UI shows is a `SELECT`, not a
separate ring buffer kept in step by hand.
the worker can move out nothing here is reachable only from the web
process. `Jobs.claim` is a single atomic statement,
so a second worker — one started from a terminal,
or by Claude Code — races correctly against this
one instead of double-running a job.
Handlers are synchronous (feedparser, trafilatura, sklearn, and soon whisper
all block), so each runs in a thread and the loop stays free to push WebSocket
updates while it works. That is also why cancellation is cooperative: see
`RunContext.check`.
"""
from __future__ import annotations
import asyncio
import contextlib
import json
import logging
import sys
import tempfile
import time
from collections.abc import Awaitable, Callable, Coroutine, Sequence
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, tzinfo
from pathlib import Path
from typing import Any, ClassVar
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from pipeline.child import CANCELLED
from pipeline.registry import JOBS, Spec
from pipeline.context import Cancelled, Retry, RunContext, spawn
from pipeline.models import Job, JobsSnapshot, Notice, Schedule
from pipeline.notices import already_said, job_failed, render
from pipeline.schedule import SCHEDULES, Cadence
from pipeline.table import Jobs, ResultTooLarge, RunCancelled
from pipeline.task import fan_out as fan_out_jobs
log = logging.getLogger(__name__)
class ChildFailed(Exception):
"""What an `isolate=True` job raises when its process came back badly.
An ordinary exception on purpose, so the worker's own `except Exception`
ends the job exactly as it ends a handler that raised in a thread: the
retry bound applies, the notice is the same, and nothing above has to know
which side of a process boundary the failure happened on.
"""
Listener = Callable[[JobsSnapshot], Awaitable[None]]
#: Where a notice goes. Separate from `Listener` because the two have
#: opposite lifetimes: a snapshot is state, replaced by the next one, and a
#: notice is an event that is gone once delivered. Folding an event into a
#: state payload is how you get a banner nobody can dismiss.
#:
#: A coroutine rather than any awaitable, because it is handed to
#: `run_coroutine_threadsafe` from the worker thread and that is what it takes.
NoticeListener = Callable[[Notice], Coroutine[Any, Any, None]]
#: Where the schedule list goes when it changes. State, like `Listener`, and
#: separate from it because the two change at different moments: the queue
#: pushes jobs on every claim and every ending, and schedules only when one
#: fires, is skipped, or is edited. Folding them together would mean sending
#: the schedules on every progress frame of a long job.
#:
#: A coroutine, for the same reason `NoticeListener` is one: it is handed to
#: `run_coroutine_threadsafe` from a worker thread or a request thread.
ScheduleListener = Callable[[list[Schedule]], Coroutine[Any, Any, None]]
def local_zone(name: str = "") -> tzinfo:
"""The zone a queue's clock cadences are read in.
A name when there is one, and otherwise the machine's own — which is what
a script, a test or a notebook gets for free, and is right often enough
that asking for it would be ceremony. A deployed app names it, because a
server runs on UTC and nobody reads their day in it.
A name the system has no zone for is a warning and the machine's zone, not
a refusal: a mistyped timezone is not a reason for the queue not to start,
and `daily at 06:00` an hour out is a great deal better than no worker.
"""
if not name:
return datetime.now().astimezone().tzinfo or UTC
try:
return ZoneInfo(name)
except (ZoneInfoNotFoundError, ValueError):
log.warning("no timezone called %r; using this machine's own for "
"daily and weekly schedules", name)
return datetime.now().astimezone().tzinfo or UTC
class JobQueue:
"""Workers per resource. Start it once, enqueue from anywhere, cancel by id.
`jobs` is the table — see `table.py` — and `config` and `store` are
whatever the app's handlers take as their two positional arguments. The
queue never reads them; it passes them to a handler it runs, and to a
`Task` it starts, and has no opinion about either.
"""
#: How long to sleep before looking for work again when the table was
#: empty. Only reached when nothing woke us — an enqueue in this process
#: is instant; this is what notices one made by another process.
IDLE_POLL = 1.0
#: Fastest a running job's progress is allowed to reach the browser. A
#: handler that reports per item would otherwise push hundreds of frames a
#: second, which no eye and no socket wants.
PUSH_EVERY = 0.25
#: How many jobs of each resource may run at once, when a caller does not
#: say. The app's own `workers` section is where a person changes it; this is
#: what a test or a notebook gets for free. A resource named in neither
#: gets one, which is the safe way to be wrong.
#:
#: The count belongs to the worker, not the job kind: two kinds that both
#: ask a model share the same four instead of each helping itself to four.
WORKERS: ClassVar[dict[str, int]] = {"llm": 4, "http": 8, "cpu": 1}
#: How often a worker says the jobs it holds are still being run, and how
#: long a job may go unsaid-about before another worker decides nobody is
#: running it and puts it back in the queue. Beaten from the event loop
#: rather than from the handler, so a job that blocks in a thread for an
#: hour still holds its lease — being slow is not being dead.
#:
#: Far apart on purpose: the gap is how many beats may be missed to a
#: stalled loop or a slow disk before live work is re-queued underneath a
#: worker that is still there, which is the one mistake this must not make.
HEARTBEAT_EVERY = 5.0
STALE_AFTER = 30.0
#: How long a stop waits for handlers already in a thread to come back
#: before walking away from them. A thread cannot be cut off, only waited
#: for or abandoned, and both are wrong as an absolute: waiting means a
#: reload held up by a model call, abandoning means a handler still
#: writing to a file the process thinks it has finished with. So: long
#: enough for a fetch to land, short enough not to be noticed, and
#: whatever is still going when it expires is left to its own thread,
#: which still writes the ending. The lease is what covers the case
#: where the process goes before that thread does.
STOP_GRACE = 5.0
#: How long a cancellation check is cached. `check()` reads the table so
#: that Kill works across processes; without this a handler calling it in a
#: loop would turn one button press into a query storm.
CANCEL_CACHE = 0.25
#: How often the loop asks the table to forget old jobs. Hourly: history
#: does not need to be exact to the minute, and the delete is one
#: statement over an indexed column.
PURGE_EVERY = 3600.0
#: How often the loop looks for a schedule whose time has come, when a
#: caller does not say. Half a minute, which is the resolution the
#: cadences promise: the finest of them is `every 1m`, and a schedule is a
#: standing instruction rather than an alarm. It is also why there is no
#: seconds unit in `pipeline/schedule.py` — a cadence this cannot keep is
#: one it should refuse rather than round.
SCHEDULE_TICK = 30.0
#: How long a notice nobody cleared stays in the log once it has stopped
#: being said. A fortnight: long enough that a failure you meant to look
#: at on the weekend is still there, short enough that the log is about
#: now. Counted from the last time it was said — see `Jobs.purge_notices`.
NOTICE_QUIET = 14 * 86400.0
def __init__(self, jobs: Jobs, config: Any, store: Any, *,
listener: Listener | None = None,
on_notice: NoticeListener | None = None,
on_schedules: ScheduleListener | None = None,
workers: dict[str, int] | None = None,
bootstrap: str = "", bootstrap_arg: str = "",
retain: float = 30 * 86400,
tz: str = "", schedules: bool = True,
schedule_tick: float | None = None) -> None:
self.jobs = jobs
self.config = config
self.store = store
#: `module:function` returning this app's `(config, store)`, for the
#: child process an `isolate=True` job runs in — see `_isolated`. Only
#: those kinds need it, so an app with none may leave it empty and a
#: queue that meets one without it says so rather than guessing.
self.bootstrap = bootstrap
#: Handed to the bootstrap in the child — for an app, the path of the
#: config file it was started with, which the child has no other way
#: to learn. Empty calls the bootstrap with no argument.
self.bootstrap_arg = bootstrap_arg
#: Seconds of finished history to keep, or 0 to keep it all. See
#: `Jobs.purge` for what survives regardless.
self.retain = retain
#: What `daily at 06:00` is 06:00 in. An IANA name, or empty for the
#: machine's own — see `local_zone`. Only the clock cadences read it;
#: `every 6h` is six elapsed hours wherever the machine thinks it is.
self.zone = local_zone(tz)
#: Whether this worker fires schedules at all. Off for a second worker
#: on the same file: they would both fire every due row, and two of
#: them would fire it twice — the pile-up guard reads `last_job_id`,
#: which neither has written yet at the moment the other looks. One
#: scheduler per file, as many workers as you like.
self.schedules = schedules
self.schedule_tick = (self.SCHEDULE_TICK if schedule_tick is None
else schedule_tick)
self._listener = listener
self._on_notice = on_notice
self._on_schedules = on_schedules
self.workers = dict(workers if workers is not None else self.WORKERS)
#: The jobs this process has in hand, by the task running each. A map
#: rather than a set because the ids are what the heartbeat writes.
self._running: dict[asyncio.Task[None], int] = {}
self._loop: asyncio.AbstractEventLoop | None = None
self._task: asyncio.Task[None] | None = None
self._pulse: asyncio.Task[None] | None = None
#: Threads for handlers, and for nothing else. See `start`.
self._pool: ThreadPoolExecutor | None = None
self._wake = asyncio.Event()
self._stopping = False
self._reclaimed_at = 0.0
self._purged_at = 0.0
self._scheduled_at = 0.0
#: Schedules this process has already complained about, so a row with
#: a cadence nothing can parse or a kind nothing registers is said
#: once rather than twice a minute for as long as the app is up. The
#: same restraint `notices.already_said` shows, at the size it needs
#: to be here.
self._said_about: set[tuple[int, str]] = set()
# ----------------------------------------------------------- lifecycle
async def run_until_idle(self, timeout: float | None = None) -> bool:
"""Start, work until nothing is queued or running, and stop.
Returns whether it got there, as against running out of `timeout`. The
batch shape — a script, a cron entry, `python -m pipeline.worker
--until-idle`, a test — where the web app's shape is `start` and never
stopping.
"Nothing queued" includes jobs waiting for a later attempt, which is
what a caller draining a queue means: a retry due in a minute is still
work this run was asked to finish. Pass `timeout` when that could be
a long wait.
"""
await self.start()
deadline = None if timeout is None else time.monotonic() + timeout
try:
while True:
snap = await asyncio.to_thread(self.jobs.snapshot)
if not snap.running and not snap.queued and not self._running:
return True
if deadline is not None and time.monotonic() > deadline:
return False
await asyncio.sleep(0.05)
finally:
await self.stop()
def drain(self, timeout: float | None = None) -> bool:
"""`run_until_idle` from code that has no event loop running."""
return asyncio.run(self.run_until_idle(timeout))
async def start(self) -> None:
self._loop = asyncio.get_running_loop()
self._stopping = False # a stopped queue can be started again
# Made here rather than in `__init__` because an `asyncio.Event` binds
# to the first loop that uses it and refuses every later one. A queue
# started, stopped and started again — a test, a reload — would
# otherwise be waiting on a loop that has been closed since.
self._wake = asyncio.Event()
# Handlers get threads of their own rather than the interpreter's
# shared pool. That pool is sized from the cpu count, so on a small
# machine it is smaller than `workers` and two things go wrong at
# once: fewer jobs run than the table admits, and the loop's own
# claim and heartbeat queue up behind the handlers. A worker starved
# of a thread stops renewing its leases and has its live jobs
# re-queued underneath it, which is the one mistake the lease exists
# to prevent. So: as many threads as there are workers, plus a few
# for resources the app never sized, which the table runs one at a
# time each.
self._pool = ThreadPoolExecutor(
max_workers=sum(self.workers.values()) + 4,
thread_name_prefix="desk-handler")
await asyncio.to_thread(self._reclaim)
if self.schedules:
# Both here rather than left to the first turn of the loop, and in
# this order. Reconciling first means a default declared today has
# a row before anything looks for due ones, so a new schedule fires
# on the start that introduced it rather than the one after. And
# firing here means an app that was down over 06:00 runs the 06:00
# job when it comes back, which is the reading of "overdue" that a
# person expects — once, not once per slot missed.
await asyncio.to_thread(self._reconcile_schedules)
await asyncio.to_thread(self._fire_due)
self._task = asyncio.create_task(self._drain(), name="desk-worker")
self._pulse = asyncio.create_task(self._heartbeat(), name="desk-heartbeat")
async def stop(self) -> None:
"""Stop taking new work and let what is running finish.
An in-flight handler is a thread and is not interrupted — killing it
mid-write is exactly what the jobs table exists to avoid. So they are
given `STOP_GRACE` to come back on their own, which most of them use,
and whatever is still going after that is walked away from.
Walked away from, and not taken from: the ending of a job is written
by the thread running it, so a handler that misses the window still
lands its own `done` or `failed` in the row whenever it gets there.
What is handed back here is the lease, because a shutdown is the one
moment a process can say outright that it is no longer watching these.
That saves the next start from waiting out the silence for the ones
whose thread this process did not outlive — and only those, since a
row its thread did finish is no longer `running` for a reclaim to
take.
"""
self._stopping = True
self._wake.set()
for task in (self._task, self._pulse):
if task is not None:
task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await task
# A moment to land in, and no more. Cancelling the task does not
# reach into the thread, so the ones that miss this window keep going
# regardless; what the wait buys is that the common case — a fetch, a
# write — finishes properly instead of being torn off mid-sentence.
if self._running:
await asyncio.wait(list(self._running), timeout=self.STOP_GRACE)
held = list(self._running.values())
for task in list(self._running):
task.cancel()
self._running.clear()
# Said out loud rather than left to the lease. Only what this process
# holds, so a shutdown here never speaks for a job another worker is
# part way through.
await asyncio.to_thread(self.jobs.release, held)
# Forget the loop as well as the task. It is about to be closed, and a
# later `wake()` or `notify()` holding a reference to a closed loop
# raises rather than being the no-op it should be.
if self._pool is not None:
# Not waited for, same as the tasks: a handler in a thread is not
# coming back on request, and the interpreter joins it at exit.
self._pool.shutdown(wait=False)
self._pool = None
self._loop = None
self._task = None
self._pulse = None
# -------------------------------------------------------------- public
def enqueue(self, kind: str, payload: dict[str, Any] | None = None, *,
label: str = "", depends_on: Sequence[int] = (), after: str = "ok",
cascade_id: int = 0, task_id: int = 0, delay: float = 0.0,
max_attempts: int | None = None, retry_delay: float | None = None,
retry_backoff: float | None = None,
dedup_key: str | None = None) -> Job | None:
"""Queue work by name. Returns the job — or the live one it deduped to.
`depends_on` holds it back until those jobs are over; `after` says
whether it needs them to have succeeded. Dedup is skipped for a job
that waits on something, because the point of one is that there will
be several — eight of a kind, each waiting on its own batch.
`delay` queues it for later without anything staying alive in the
meantime: the row carries the time and `claim` passes over it until
then, so "in ten minutes" survives a restart four minutes in.
`max_attempts` and the two waits override what the kind declared, for
a caller that knows more about this one piece of work than the handler
does — which is what a plugin's own retry policy is: the outlet, not
the job kind, is what knows how long to leave a publisher alone. They
go onto the row, so the attempt that reads them may be in another
process, after a restart, under a plugin file that has since changed.
`label` overrides the kind's, for one piece of work that can say more
about itself than its kind can. A queued job has no `message` — that is
written by a running handler — so the label is the whole of what the
queue list has to show, and the kind's is by construction the same
string on every job of that kind. A caller that knows how the work was
cut says so here. Empty means the kind's, which is every caller but
one.
`dedup_key` overrides the kind's own, which is how one *item's*
acquisition is made unrepeatable while the kind stays freely
repeatable: two presses of a reader's retry button collide on the key
and the second is handed the first one's job, across tabs, clients and
processes, because the collision happens in a unique index rather than
in whichever page asked.
`None` when the run it would belong to has been killed. Logged and
dropped rather than raised: whatever asked for this — a cascade's step,
a task fanning out its batches — has no way of knowing and nothing to
do about it. The kill has already happened; this is it arriving.
"""
spec = JOBS.get(kind)
if spec is None:
raise KeyError(
f"no job registered as {kind!r} (have: {', '.join(sorted(JOBS)) or 'none'})"
)
solo = spec.solo and not depends_on
try:
job = self.jobs.enqueue(
kind, label or spec.label, payload,
dedup_key=(kind if solo else "") if dedup_key is None
else dedup_key,
depends_on=depends_on, after=after, resource=spec.resource,
cascade_id=cascade_id, task_id=task_id, delay=delay,
max_attempts=spec.max_attempts if max_attempts is None
else max_attempts,
retry_delay=spec.retry_delay if retry_delay is None
else retry_delay,
retry_backoff=spec.retry_backoff if retry_backoff is None
else retry_backoff)
except RunCancelled as refused:
log.info("%s", refused)
return None
self.wake()
self.notify()
return job
def new_run(self, kind: str, name: str = "", label: str = "") -> int:
"""Mint an id for a cascade or a task. Half of what a `Task` needs of
a sink; `enqueue` and `cancelled` are the rest."""
return self.jobs.new_run(kind, name, label)
def cancelled(self, run_id: int) -> bool:
return self.jobs.cancelled(run_id)
def start_task(self, task: Any, *, cascade_id: int = 0) -> int:
"""Run a `Task` here and now, and return its run id.
Here and now because a task is plain code: it asks the store what there
is to do and queues it. That is a query and a handful of inserts, which
is cheap enough to do in the request that pressed the button or in the
worker thread that has just finished the job before.
"""
return task.start(self.config, self.store, self, cascade_id=cascade_id)
def cancel(self, job_id: int) -> Job | None:
"""Ask a job to stop. Queued ones end here; running ones on their next
`context.check()`. Returns the job as it now stands, or None if there
isn't one."""
job = self.jobs.request_cancel(job_id)
self.wake()
self.notify()
return job
def cancel_run(self, run_id: int) -> int:
"""Kill a task or a cascade — one operation for both. How many ended.
Killing a task is "skip this one": its batches end, its final job still
runs, and what it reports says the task was killed rather than only how
many batches ended cancelled. What happens next is then the cascade's
to decide, and newsdesk's pull cascade decides to carry on to the next
stage. Killing the cascade takes the final job with it, so nothing
reports and nothing follows — the one decision that is not the
cascade's, because there is nothing left of it to ask.
`Jobs.cancel_run` is where all of that is one statement.
"""
ended = self.jobs.cancel_run(run_id)
self.wake()
self.notify()
return ended
def stop_task(self, task_id: int, why: str = "") -> int:
"""End a task's queued batches, sparing what is running and its final
job. How many ended.
Not a kill and not written as one: the task's row stays as it is, so
nothing reports that a person stopped this. What a handler reaches
through `RunContext.stop`, for the failure that is the backend's
answer to the whole run — see `Jobs.stop_task`.
"""
ended = self.jobs.stop_task(task_id, error=why)
self.wake()
self.notify()
return ended
def snapshot(self) -> JobsSnapshot:
return self.jobs.snapshot()
def get(self, job_id: int) -> Job | None:
return self.jobs.get(job_id)
def wake(self) -> None:
"""Tell the loop to look for work now instead of at the next poll.
A no-op when no worker is running in this process — enqueueing without
one is legitimate (a command queueing work for the app to pick up), and
the row is in the table either way.
"""
if self._loop is None or self._loop.is_closed():
return
with contextlib.suppress(RuntimeError):
self._loop.call_soon_threadsafe(self._wake.set)
def run_schedule(self, schedule_id: int) -> Job | None:
"""Fire one schedule now, leaving when it is next due alone.
What Run now presses. `next_run` is untouched on purpose: a person
asking for this run is not asking to move the nightly one, and a button
that quietly rescheduled what it touched would be a button nobody could
use twice.
`last_run` and `last_job_id` are written, because they are the truth
about this schedule — it did just run, and that is the job it queued.
Which also means the next tick will not start a second one while this
is still going.
"""
row = self.jobs.schedule(schedule_id)
if row is None:
return None
job = self.enqueue(row.kind, row.payload, label=row.label)
self.jobs.update_schedule(row.id, last_run=datetime.now(UTC).isoformat(),
last_job_id=job.id if job else 0)
self.notify_schedules()
return job
def notify_schedules(self) -> None:
"""Push the schedule list to listeners. Safe from any thread.
Called by whatever changed one — a firing here, a route over there —
rather than polled, so a row edited in one tab redraws in the others
without either of them asking every few seconds.
"""
if self._loop is None or self._on_schedules is None or self._loop.is_closed():
return
with contextlib.suppress(RuntimeError):
asyncio.run_coroutine_threadsafe(self._push_schedules(), self._loop)
def announce(self, notice: Notice) -> None:
"""Write one notice down, then push it to whoever is watching. Safe
from any thread.
Written first, because the push is fire-and-forget and the usual
listener is nobody: a pull runs on a schedule and the phone that would
have shown the line is in a pocket. It used to stop there — "the log
already has it" — which was true of a file on a server and no use to
the person the notice was for. The table is what a tab opened
afterwards reads, and where a line that faded can be found again.
What is pushed is the row, count included, so a listener can tell news
from a repeat of something already in the log. That judgement is left
to it on purpose: every notice still goes out, and what interrupts a
person is the interface's business. See `NoticeEntry`.
Called from the worker thread, so the push hops the loop the same way
`notify` does, and delivery is still never waited for.
"""
# Logged as well, always. The socket is the fast path, the table is
# what is on screen tomorrow, and the log is the only one of the three
# with the traceback beside it.
log.warning("%s\n%s", notice.code, render(notice))
try:
notice = self.jobs.record_notice(notice)
except Exception:
# A notice is already the report of a failure. One that cannot be
# written is still worth pushing, and is never worth failing the
# job that raised it.
log.exception("could not record the %s notice; pushing it anyway",
notice.code)
if self._loop is None or self._on_notice is None or self._loop.is_closed():
return
with contextlib.suppress(RuntimeError):
asyncio.run_coroutine_threadsafe(self._on_notice(notice), self._loop)
def notify(self) -> None:
"""Push the queue's state to listeners. Safe from any thread."""
if self._loop is None or self._listener is None or self._loop.is_closed():
return
with contextlib.suppress(RuntimeError):
asyncio.run_coroutine_threadsafe(self._announce(), self._loop)
# --------------------------------------------------------------- guts
async def _drain(self) -> None:
"""Claim whatever there is a free worker for, then wait to be woken.
Claiming until it comes back empty is what lets the table decide how
many run rather than this loop: it only asks, and stops when told. It never counts anything itself, which is
the only way a second process sharing the file gets the same answer.
One bad turn does not end it. A locked database or a file that went
away for a moment would otherwise take this task down with an
exception nobody is waiting to receive, and the process would sit
there looking healthy while draining nothing at all.
"""
while not self._stopping:
try:
await self._turn()
except asyncio.CancelledError:
raise
except Exception:
log.exception("the drain loop hit an error; carrying on")
await asyncio.sleep(self.IDLE_POLL)
async def _turn(self) -> None:
"""One pass: take what there is room for, or wait to be woken.
`IDLE_POLL` is also what notices a job whose next attempt has come
round. Nothing wakes the loop for one — the row said it was due at a
moment nobody was listening for — so the poll that exists to notice
another process's enqueue notices this too, and a retry starts within a
second of its time. Which is why a scheduled job needs no timer here:
there was already something looking.
"""
# Before the claim, not after it with the other housekeeping. Reclaim
# and purge are reached only when there was nothing to take, which is
# right for both — they are repairs, and a busy queue has more pressing
# work. A schedule is the opposite: it is work, and a queue that is
# never idle would be one where 06:00 never came round. The check
# itself is a clock comparison, so paying it on every pass costs
# nothing worth measuring.
if self.schedules and (time.monotonic() - self._scheduled_at
>= self.schedule_tick):
await asyncio.to_thread(self._fire_due)
job = await asyncio.to_thread(self.jobs.claim, self.workers)
if job is not None:
self._start(job)
return
# Nothing to take, which is the moment to ask whether some other
# worker's `running` row has gone quiet. Here rather than only at
# start, so a process that dies is cleaned up by whichever one is
# still alive instead of waiting for somebody to be restarted.
if time.monotonic() - self._reclaimed_at >= self.STALE_AFTER:
await asyncio.to_thread(self._reclaim)
if self.retain and time.monotonic() - self._purged_at >= self.PURGE_EVERY:
await asyncio.to_thread(self._purge)
self._wake.clear()
with contextlib.suppress(asyncio.TimeoutError):
await asyncio.wait_for(self._wake.wait(), self.IDLE_POLL)
def _reclaim(self) -> None:
"""Recover silent workers' jobs while renewing our own stalled leases."""
self._reclaimed_at = time.monotonic()
stale = self.jobs.reclaim_stale(
self.STALE_AFTER, except_ids=list(self._running.values()))
if stale:
log.info("re-queued %d job(s) whose worker stopped reporting in", stale)
def _purge(self) -> None:
"""Forget finished jobs older than `retain`. Only when idle, like the
reclaim above it, so it never competes with work for the file."""
self._purged_at = time.monotonic()
try:
gone = self.jobs.purge(self.retain)
# The notices with them, on their own clock: a job is history once
# it has finished, a notice once it has stopped being said.
self.jobs.purge_notices(self.NOTICE_QUIET)
except Exception:
log.exception("could not purge old jobs; carrying on")
return
if gone:
log.info("forgot %d finished job(s) older than %g days",
gone, self.retain / 86400)
# ----------------------------------------------------------- schedules
def _reconcile_schedules(self) -> None:
"""Give every declared default a row, and leave every existing one be.
The whole of what a declaration buys: a name nothing has a row for gets
one, and a name that does have one is not touched — not its cadence,
not its payload, not whether it is switched on. So the declarations in
an app's jobs module are where a schedule *comes from*, and the table
is what it *is*; turning one off in the UI survives the next deploy,
and changing one in the code reaches only the installs that never had
it.
Which is also why nothing here deletes. A declaration that goes away
leaves its row behind, `unknown kind` against it if the handler went
too, for a person to delete deliberately — the alternative is a deploy
that silently forgets a schedule somebody was relying on.
"""
now = datetime.now(UTC)
for declared in SCHEDULES.values():
row = self.jobs.add_schedule(
declared.name, declared.kind, declared.cadence,
payload=declared.payload, label=declared.label,
next_run=Cadence.parse(declared.cadence)
.next_after(now, self.zone).isoformat(),
only_if_new=True)
if row is not None:
log.info("schedule %r declared: %s %s, first at %s",
row.name, row.kind, row.cadence, row.next_run)
def _fire_due(self) -> None:
"""Queue a job for every schedule whose time has come. One pass.
From the drain loop's own thread pool, not a timer and not a second
thread: there is one place that decides what runs, and adding a second
clock would mean two things racing to enqueue the same row.
Three ways a due schedule does not fire:
unknown kind the handler is gone — a plugin uninstalled, a
deploy that dropped a job. Said once and then left
exactly as it is, `next_run` included, so that
registering the handler again runs it rather than
waiting out another cadence.
`<desk-schedules>` marks the row `unknown kind`
and a person deletes it if it is not coming back.
bad cadence only reachable by editing the table by hand, since
every route parses before it writes. Same
treatment, and for the same reason: fixing the
wording is meant to be enough.
still going the last job this schedule queued is queued or
running, or the kind is `solo` and one is under
way. Here `next_run` *does* advance and nothing is
queued, which is the difference between a schedule
and a backlog: a job slower than its own cadence
must not grow a queue of itself.
"""
self._scheduled_at = time.monotonic()
now = datetime.now(UTC)
moved = False
for row in self.jobs.due_schedules(now.isoformat()):
spec = JOBS.get(row.kind)
try:
when = Cadence.parse(row.cadence).next_after(now, self.zone)
except ValueError as bad:
self._say_once(row, "cadence", "schedule %r will not run: %s",
row.name, bad)
continue
if spec is None:
self._say_once(row, "kind",
"schedule %r wants %r, which no handler is "
"registered for; it will not run until one is",
row.name, row.kind)
continue
moved = True
if self._still_going(row, spec):
log.info("schedule %r was due, but its last job is still "
"going; next at %s", row.name, when.isoformat())
self.jobs.update_schedule(row.id, next_run=when.isoformat())
continue
job = self.enqueue(row.kind, row.payload, label=row.label)
self.jobs.mark_fired(row.id, job.id if job else 0, when.isoformat())
log.info("schedule %r fired %s as job %s; next at %s", row.name,
row.kind, job.id if job else "nothing", when.isoformat())
if moved:
self.notify_schedules()
def _still_going(self, row: Schedule, spec: Spec) -> bool:
"""Whether firing this schedule would pile one job on another.
Two questions, because a schedule can collide with itself two ways.
Its own last job may still be going, which is the slow-job case. Or the
kind may be `solo` and something else may have started one — a button,
another schedule, a cascade — in which case `enqueue` would hand back
that job and the schedule would record somebody else's work as its own
firing.
"""
if row.last_job_id:
last = self.jobs.get(row.last_job_id)
if last is not None and last.live:
return True
return bool(spec.solo and self.jobs.last(row.kind, states=("queued", "running")))
def _say_once(self, row: Schedule, about: str, message: str, *args: Any) -> None:
"""Log a schedule's own defect the first time this process meets it.
A tick every thirty seconds would otherwise make one broken row into
two thousand log lines a day, which is how a log stops being read.
Keyed on the row and what is wrong with it, so fixing the cadence and
then breaking the kind still says both.
"""
if (row.id, about) in self._said_about:
return
self._said_about.add((row.id, about))
log.warning(message, *args)
async def _push_schedules(self) -> None:
"""Read the list off the loop, for the reason `_announce` does."""
if self._on_schedules is None:
return
try:
await self._on_schedules(await asyncio.to_thread(self.jobs.schedules))
except Exception:
log.exception("schedule listener raised; carrying on")
async def _heartbeat(self) -> None:
"""Say the jobs this process holds are still being run.
From the loop, not from the handler: a handler is arbitrary blocking
code in a thread and may not come back for an hour, and a lease it can
only renew between units of work would make "slow" indistinguishable
from "dead". The loop is the thing that is still there either way, and
if it is not, then this process really has stopped and its jobs really
should go back.
"""
while not self._stopping:
await asyncio.sleep(self.HEARTBEAT_EVERY)
ids = list(self._running.values())
if not ids:
continue
try:
await asyncio.to_thread(self.jobs.heartbeat, ids)
except Exception:
# A missed beat is survivable — the next one lands well inside
# the lease — and a pulse that dies on one is not, because
# every job this process holds would be re-queued under it.
log.exception("heartbeat failed; carrying on")
def _start(self, job: Job) -> None:
"""Run one job alongside whatever else is going.
The finished callback wakes the loop rather than the loop watching the
set: a worker has just come free and something queued may now fit in it,
and that is the same signal an enqueue sends.
"""
task = asyncio.create_task(self._execute(job), name=f"job-{job.id}")
self._running[task] = job.id
def done(finished: asyncio.Task[None]) -> None:
self._running.pop(finished, None)
with contextlib.suppress(asyncio.CancelledError):
finished.result()
self._wake.set()
task.add_done_callback(done)
async def _execute(self, job: Job) -> None:
spec = JOBS.get(job.kind)
await self._announce()
if spec is None:
# Reachable: a job queued by a newer version, or by a plugin that
# is no longer installed. Failing it says so and clears the row,
# rather than leaving it claimed forever.
await asyncio.to_thread(self.jobs.finish,
job.id, "failed", error=f"no handler registered for {job.kind!r}")
await self._announce()
return
tasks: list[Any] = []
said: list[Notice] = []
fn, label = spec.fn, spec.label
args = (self.config, self.store)
if spec.isolate:
# The row is the whole input, so nothing is sent — the child is
# told where to look. Its signature matches a handler's on
# purpose: everything below this line, the ending, the cancel
# check on return, the retry, is the same code either way.
fn = self._isolated(job, spec)
def run() -> None:
"""The handler and this job's ending, in the same thread.
Together because the thread is the half of this that survives a
shutdown. `stop()` cancels the coroutine of a job whose handler
missed the grace period, and a cancelled coroutine never reaches
the line after its `await` — so an ending written back there is
precisely the ending an abandoned job never gets. The handler runs
on to the end regardless, writes whatever it writes, and leaves a
`running` row for the next start to re-queue and run a second
time: work already done, done again, and for anything that asks a
model a second bill for it.
Nothing in here needs the loop. Everything that does — the push
saying the row moved — stays outside.
"""
try:
if "context" in job.payload:
raise TypeError("context is reserved for the worker")
context = self._context(job, tasks, said)
extra = dict(job.payload)
if spec.wants_context:
extra["context"] = context
result = fn(*args, **extra)
result = result if isinstance(result, dict) else {}
if self.jobs.cancel_requested(job.id):
# Asked to stop while it ran, and it never looked — a handler
# with no `check()`, or one whose last unit was already under
# way. The return is its last chance to look, so it is taken
# here: the job ends cancelled, keeping what it made, and
# queues nothing. Ending `done` would start the next task
# under a cascade that has just been killed — a Kill that did
# nothing. (The guard in `Jobs.enqueue` would refuse the jobs
# anyway; this is the same answer one step earlier, and without
# a run minted for nothing.)
self.jobs.finish(job.id, "cancelled", result=result)
return
self.jobs.finish(job.id, "done", result=result)
for task in tasks:
# After the job landed, never before: a task that outlived the
# failure of the job that described it would be work nobody
# asked for. Here rather than back on the loop for the same
# two reasons the ending is — a finder query and a handful of
# inserts are no more the loop's business than the handler
# was, and a job abandoned by a shutdown still asks for what
# it said should follow. Its own failure is logged and no
# more: the job it came from is already over and did what it
# was asked.
try:
self.start_task(task, cascade_id=job.cascade_id)
except Exception:
log.exception("job %d asked for a task that would not start",
job.id)
except Cancelled:
self.jobs.finish(job.id, "cancelled")
except Retry as again:
# The handler naming its own wait — a rate limit's answer, a
# `Retry-After`. Logged rather than tracebacked: this is the
# world saying "not yet", which is a fact about the service and
# not a defect here. Whether there is another attempt to give
# it is still the row's to decide.
log.info("job %d (%s) asked to come back: %s", job.id, job.kind, again)
self._ended_badly(job, f"{type(again).__name__}: {again}",
label, said, delay=again.after)
except ResultTooLarge as exc:
# Loud and final. Not `_ended_badly`, because a retry would run
# the same handler to the same oversized result; and not
# quiet, because the alternative to failing is dropping data.
log.error("job %d (%s) %s", job.id, job.kind, exc)
error = f"{type(exc).__name__}: {exc}"
self.jobs.finish(job.id, "failed", error=error)
self.announce(job_failed(job.kind, label, error))
except Exception as exc:
# Deliberately broad. A handler is arbitrary third-party-ish code
# (a source plugin, a model, a subprocess); anything it raises is
# this job's failure and nothing else's, and letting it escape
# would take the worker down and strand every job behind it.
log.exception("job %d (%s) failed", job.id, job.kind)
self._ended_badly(job, f"{type(exc).__name__}: {exc}", label, said)
try:
await asyncio.get_running_loop().run_in_executor(self._pool, run)
finally:
await self._announce()
def _ended_badly(self, job: Job, error: str, label: str,
said: list[Notice], *, delay: float | None = None) -> None:
"""An attempt that did not land: another one later, or the end of it.
Called from the handler's thread, like the ending it replaces, and for
the same reason — a shutdown that abandoned the thread must not lose
the row's move to whatever comes next. `Jobs.retry` decides which move
that is; here there is only the question of what to say about it.
A failure with another attempt coming says nothing to the screen. A
notice is for what a person has to act on, and there is nothing to act
on yet: the queue is already doing the thing they would have asked for.
The log has it, as it has everything. Only the attempt that runs out
announces, and it announces the failure that ended it.
"""
ended = self.jobs.retry(job.id, error=error, delay=delay)
if ended is not None and ended.live:
log.info("job %d (%s) goes back in the queue: attempt %d of %d",
job.id, job.kind, job.attempt, job.max_attempts)
return
# And a notice, so it reaches the screen and not only the log — unless
# this failure has already put one there. A handler that classified its
# own failure said the useful thing about it, which login expired or
# which switch is off; adding "failed, looks like a bug" underneath
# contradicts it on the same screen, and sends you to a traceback about
# a machine that is working correctly.
#
# `job_failed` keeps the case it was written for: a handler that raised
# something nobody caught and said nothing about it. That really is
# ours to fix, and it is still the only thing here that says `bug`
# without working it out.
if ended is not None and ended.state == "cancelled":
return # a Kill that landed mid-attempt; nothing broke
if not already_said(said, error):
self.announce(job_failed(job.kind, label, error))
def _isolated(self, job: Job, spec: Spec) -> Callable[..., dict[str, Any]]:
"""A stand-in handler that runs this job in a child process.
Returned rather than called so the worker's thread calls it, in the
place the real handler would have been called: the ending, the retry
and the cancel-on-return are then one piece of code for both paths.
`context` is accepted and dropped. The child builds its own — one that
writes to the same table, which is how its progress reaches the
browser without anything being forwarded. What cannot cross is the
pair the handlers take, hence `bootstrap`.
Failure is read off the exit code and the result file, not off stderr.
A child that died of something Python never saw — an OOM kill, a
segfault in a native library — leaves no file, and gets the only
honest report available: its exit code and whatever it managed to say.
"""
def run_in_child(*_args: Any, context: RunContext | None = None,
**_payload: Any) -> dict[str, Any]:
# Checked in here rather than above, where it reads more naturally
# and would be raised while the worker was still assembling the
# run — outside the `try` that turns a handler's exception into
# this job's ending, so the row would stay `running` for the lease
# to clean up and the reason would be in the log only.
if not self.bootstrap:
raise RuntimeError(
f"{job.kind!r} is isolate=True, but this queue has no "
"bootstrap. A child process cannot be handed the app's "
"config and store, so it has to rebuild them: pass "
"bootstrap='module:function' to JobQueue, returning "
"(config, store)."
)
with tempfile.TemporaryDirectory(prefix="pipeline-job-") as tmp:
out = Path(tmp) / "result.json"
argv = [sys.executable, "-m", "pipeline.child",
"--db", str(self.jobs.path), "--job", str(job.id),
"--bootstrap", self.bootstrap, "--out", str(out)]
if self.bootstrap_arg:
argv += ["--bootstrap-arg", self.bootstrap_arg]
done = spawn(argv, check=context.check if context else None,
timeout=spec.timeout or None)
said = json.loads(out.read_text(encoding="utf-8")) if out.exists() else {}
if done.returncode == CANCELLED:
raise Cancelled()
if done.returncode != 0 or "error" in said:
detail = said.get("error") or (
done.stderr.decode("utf-8", "replace").strip().splitlines()[-1:]
or [f"exited {done.returncode}"])[0]
if said.get("traceback"):
log.error("job %d (%s) failed in its own process:\n%s",
job.id, job.kind, said["traceback"])
raise ChildFailed(detail)
result = said.get("result")
return result if isinstance(result, dict) else {}
return run_in_child
def _context(self, job: Job, tasks: list[Any],
said: list[Notice]) -> RunContext:
"""The `RunContext` for one run. The callbacks are called from the worker
thread, so they are written to be safe there.
`said` collects what the handler announced while it ran, which is how
the failure above knows whether this ending has already been explained.
"""
pushed_at = 0.0
checked_at = 0.0
cancelled = False
def on_progress(done: int, total: int, message: str) -> None:
nonlocal pushed_at
self.jobs.progress(job.id, done, total, message)
moment = time.monotonic()
if moment - pushed_at >= self.PUSH_EVERY:
pushed_at = moment
self.notify()
def is_cancelled() -> bool:
# Reads the table rather than a flag on this object, which is what
# lets a job be killed from the UI while the worker runs somewhere
# else entirely.
nonlocal checked_at, cancelled
moment = time.monotonic()
if moment - checked_at >= self.CANCEL_CACHE:
checked_at = moment
cancelled = self.jobs.cancel_requested(job.id)
return cancelled
def is_killed() -> bool:
# The task's row, not this job's request. A batch asked to stop is
# one batch dropped; a killed task is the whole step skipped, and
# nothing in the counts tells those apart. Uncached, unlike the
# check above: the job that asks is the one reporting at the end
# of a task, and it asks once.
return bool(job.task_id) and self.jobs.cancelled(job.task_id)
def on_task(task: Any) -> bool:
# Collected rather than queued, and always taken: the worker starts
# what is in here once the job lands in `done`. See
# `RunContext.task` for why that is the promise being made.
tasks.append(task)
return True
def on_stop(why: str) -> int:
# A job with no task behind it is the whole run, and there is
# nothing queued under it to spare. Its own failure is the answer.
if not job.task_id:
return 0
ended = self.stop_task(job.task_id, why)
log.warning("job %d stopped the rest of task %d: %s (%d batch(es))",
job.id, job.task_id, why or "no reason given", ended)
return ended
def on_notice(notice: Notice) -> None:
# Kept as well as pushed. One job is one thread, so appending here
# needs no lock, and the list dies with the run.
said.append(notice)
self.announce(notice)
def on_fan_out(kind: str, payloads: list[dict[str, Any]], *,
then: str, settle: str, label: str,
carries: dict[str, Any],
max_attempts: int | None,
retry_delay: float | None,
retry_backoff: float | None) -> tuple[int, ...]:
"""`RunContext.fan_out`, from the worker thread.
Inside this job's cascade rather than its task: the children are a
step of their own, so killing them is not killing the step that
asked for them, and the callback fanning out again makes a chain
rather than a nest. A job that fans out with no cascade of its own
begins one, which is `fan_out`'s business and not this closure's.
"""
return fan_out_jobs(
self, kind, payloads, then=then, settle=settle, label=label,
carries=carries, cascade_id=job.cascade_id,
max_attempts=max_attempts, retry_delay=retry_delay,
retry_backoff=retry_backoff)
return RunContext(
on_progress=on_progress,
is_cancelled=is_cancelled,
is_killed=is_killed,
on_notice=on_notice,
on_task=on_task,
on_stop=on_stop,
on_fan_out=on_fan_out,
waited_for=tuple(self.jobs.outcomes(job.depends_on)),
)
async def _announce(self) -> None:
"""Read snapshots off the loop so slow SQLite cannot delay heartbeats."""
if self._listener is None:
return
try:
await self._listener(await asyncio.to_thread(self.jobs.snapshot))
except Exception:
log.exception("job listener raised; carrying on")