The schedule cadence is a picker, the add row is behind a button and off here, a running-jobs chip with Kill and Stop run sits in every header, and a declared schedule's label is refreshed at startup. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1144 lines
57 KiB
Python
1144 lines
57 KiB
Python
"""The worker: one loop, draining the jobs table.
|
|
|
|
Deliberately not an in-memory queue. The table *is* the queue, which buys three
|
|
things that matter more here than the microseconds it costs:
|
|
|
|
it survives a restart a pull interrupted by a reload comes back queued
|
|
rather than vanishing half-done.
|
|
it is the history the list the UI shows is a `SELECT`, not a
|
|
separate ring buffer kept in step by hand.
|
|
the worker can move out nothing here is reachable only from the web
|
|
process. `Jobs.claim` is a single atomic statement,
|
|
so a second worker — one started from a terminal,
|
|
or by Claude Code — races correctly against this
|
|
one instead of double-running a job.
|
|
|
|
Handlers are synchronous (feedparser, trafilatura, sklearn, and soon whisper
|
|
all block), so each runs in a thread and the loop stays free to push WebSocket
|
|
updates while it works. That is also why cancellation is cooperative: see
|
|
`RunContext.check`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import contextlib
|
|
import json
|
|
import logging
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from collections.abc import Awaitable, Callable, Coroutine, Sequence
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import UTC, datetime, tzinfo
|
|
from pathlib import Path
|
|
from typing import Any, ClassVar
|
|
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
|
|
|
from pipeline.child import CANCELLED
|
|
from pipeline.registry import JOBS, Spec
|
|
from pipeline.context import Cancelled, Retry, RunContext, spawn
|
|
from pipeline.models import Job, JobsSnapshot, Notice, Schedule
|
|
from pipeline.notices import already_said, job_failed, render
|
|
from pipeline.schedule import SCHEDULES, Cadence
|
|
from pipeline.table import Jobs, ResultTooLarge, RunCancelled
|
|
from pipeline.task import fan_out as fan_out_jobs
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class ChildFailed(Exception):
|
|
"""What an `isolate=True` job raises when its process came back badly.
|
|
|
|
An ordinary exception on purpose, so the worker's own `except Exception`
|
|
ends the job exactly as it ends a handler that raised in a thread: the
|
|
retry bound applies, the notice is the same, and nothing above has to know
|
|
which side of a process boundary the failure happened on.
|
|
"""
|
|
|
|
|
|
Listener = Callable[[JobsSnapshot], Awaitable[None]]
|
|
#: Where a notice goes. Separate from `Listener` because the two have
|
|
#: opposite lifetimes: a snapshot is state, replaced by the next one, and a
|
|
#: notice is an event that is gone once delivered. Folding an event into a
|
|
#: state payload is how you get a banner nobody can dismiss.
|
|
#:
|
|
#: A coroutine rather than any awaitable, because it is handed to
|
|
#: `run_coroutine_threadsafe` from the worker thread and that is what it takes.
|
|
NoticeListener = Callable[[Notice], Coroutine[Any, Any, None]]
|
|
|
|
#: Where the schedule list goes when it changes. State, like `Listener`, and
|
|
#: separate from it because the two change at different moments: the queue
|
|
#: pushes jobs on every claim and every ending, and schedules only when one
|
|
#: fires, is skipped, or is edited. Folding them together would mean sending
|
|
#: the schedules on every progress frame of a long job.
|
|
#:
|
|
#: A coroutine, for the same reason `NoticeListener` is one: it is handed to
|
|
#: `run_coroutine_threadsafe` from a worker thread or a request thread.
|
|
ScheduleListener = Callable[[list[Schedule]], Coroutine[Any, Any, None]]
|
|
|
|
|
|
def local_zone(name: str = "") -> tzinfo:
|
|
"""The zone a queue's clock cadences are read in.
|
|
|
|
A name when there is one, and otherwise the machine's own — which is what
|
|
a script, a test or a notebook gets for free, and is right often enough
|
|
that asking for it would be ceremony. A deployed app names it, because a
|
|
server runs on UTC and nobody reads their day in it.
|
|
|
|
A name the system has no zone for is a warning and the machine's zone, not
|
|
a refusal: a mistyped timezone is not a reason for the queue not to start,
|
|
and `daily at 06:00` an hour out is a great deal better than no worker.
|
|
"""
|
|
if not name:
|
|
return datetime.now().astimezone().tzinfo or UTC
|
|
try:
|
|
return ZoneInfo(name)
|
|
except (ZoneInfoNotFoundError, ValueError):
|
|
log.warning("no timezone called %r; using this machine's own for "
|
|
"daily and weekly schedules", name)
|
|
return datetime.now().astimezone().tzinfo or UTC
|
|
|
|
|
|
|
|
class JobQueue:
|
|
"""Workers per resource. Start it once, enqueue from anywhere, cancel by id.
|
|
|
|
`jobs` is the table — see `table.py` — and `config` and `store` are
|
|
whatever the app's handlers take as their two positional arguments. The
|
|
queue never reads them; it passes them to a handler it runs, and to a
|
|
`Task` it starts, and has no opinion about either.
|
|
"""
|
|
|
|
#: How long to sleep before looking for work again when the table was
|
|
#: empty. Only reached when nothing woke us — an enqueue in this process
|
|
#: is instant; this is what notices one made by another process.
|
|
IDLE_POLL = 1.0
|
|
|
|
#: Fastest a running job's progress is allowed to reach the browser. A
|
|
#: handler that reports per item would otherwise push hundreds of frames a
|
|
#: second, which no eye and no socket wants.
|
|
PUSH_EVERY = 0.25
|
|
|
|
#: How many jobs of each resource may run at once, when a caller does not
|
|
#: say. The app's own `workers` section is where a person changes it; this is
|
|
#: what a test or a notebook gets for free. A resource named in neither
|
|
#: gets one, which is the safe way to be wrong.
|
|
#:
|
|
#: The count belongs to the worker, not the job kind: two kinds that both
|
|
#: ask a model share the same four instead of each helping itself to four.
|
|
WORKERS: ClassVar[dict[str, int]] = {"llm": 4, "http": 8, "cpu": 1}
|
|
|
|
#: How often a worker says the jobs it holds are still being run, and how
|
|
#: long a job may go unsaid-about before another worker decides nobody is
|
|
#: running it and puts it back in the queue. Beaten from the event loop
|
|
#: rather than from the handler, so a job that blocks in a thread for an
|
|
#: hour still holds its lease — being slow is not being dead.
|
|
#:
|
|
#: Far apart on purpose: the gap is how many beats may be missed to a
|
|
#: stalled loop or a slow disk before live work is re-queued underneath a
|
|
#: worker that is still there, which is the one mistake this must not make.
|
|
HEARTBEAT_EVERY = 5.0
|
|
STALE_AFTER = 30.0
|
|
|
|
#: How long a stop waits for handlers already in a thread to come back
|
|
#: before walking away from them. A thread cannot be cut off, only waited
|
|
#: for or abandoned, and both are wrong as an absolute: waiting means a
|
|
#: reload held up by a model call, abandoning means a handler still
|
|
#: writing to a file the process thinks it has finished with. So: long
|
|
#: enough for a fetch to land, short enough not to be noticed, and
|
|
#: whatever is still going when it expires is left to its own thread,
|
|
#: which still writes the ending. The lease is what covers the case
|
|
#: where the process goes before that thread does.
|
|
STOP_GRACE = 5.0
|
|
|
|
#: How long a cancellation check is cached. `check()` reads the table so
|
|
#: that Kill works across processes; without this a handler calling it in a
|
|
#: loop would turn one button press into a query storm.
|
|
CANCEL_CACHE = 0.25
|
|
|
|
#: How often the loop asks the table to forget old jobs. Hourly: history
|
|
#: does not need to be exact to the minute, and the delete is one
|
|
#: statement over an indexed column.
|
|
PURGE_EVERY = 3600.0
|
|
|
|
#: How often the loop looks for a schedule whose time has come, when a
|
|
#: caller does not say. Half a minute, which is the resolution the
|
|
#: cadences promise: the finest of them is `every 1m`, and a schedule is a
|
|
#: standing instruction rather than an alarm. It is also why there is no
|
|
#: seconds unit in `pipeline/schedule.py` — a cadence this cannot keep is
|
|
#: one it should refuse rather than round.
|
|
SCHEDULE_TICK = 30.0
|
|
|
|
#: How long a notice nobody cleared stays in the log once it has stopped
|
|
#: being said. A fortnight: long enough that a failure you meant to look
|
|
#: at on the weekend is still there, short enough that the log is about
|
|
#: now. Counted from the last time it was said — see `Jobs.purge_notices`.
|
|
NOTICE_QUIET = 14 * 86400.0
|
|
|
|
def __init__(self, jobs: Jobs, config: Any, store: Any, *,
|
|
listener: Listener | None = None,
|
|
on_notice: NoticeListener | None = None,
|
|
on_schedules: ScheduleListener | None = None,
|
|
workers: dict[str, int] | None = None,
|
|
bootstrap: str = "", bootstrap_arg: str = "",
|
|
retain: float = 30 * 86400,
|
|
tz: str = "", schedules: bool = True,
|
|
schedule_tick: float | None = None) -> None:
|
|
self.jobs = jobs
|
|
self.config = config
|
|
self.store = store
|
|
#: `module:function` returning this app's `(config, store)`, for the
|
|
#: child process an `isolate=True` job runs in — see `_isolated`. Only
|
|
#: those kinds need it, so an app with none may leave it empty and a
|
|
#: queue that meets one without it says so rather than guessing.
|
|
self.bootstrap = bootstrap
|
|
#: Handed to the bootstrap in the child — for an app, the path of the
|
|
#: config file it was started with, which the child has no other way
|
|
#: to learn. Empty calls the bootstrap with no argument.
|
|
self.bootstrap_arg = bootstrap_arg
|
|
#: Seconds of finished history to keep, or 0 to keep it all. See
|
|
#: `Jobs.purge` for what survives regardless.
|
|
self.retain = retain
|
|
#: What `daily at 06:00` is 06:00 in. An IANA name, or empty for the
|
|
#: machine's own — see `local_zone`. Only the clock cadences read it;
|
|
#: `every 6h` is six elapsed hours wherever the machine thinks it is.
|
|
self.zone = local_zone(tz)
|
|
#: Whether this worker fires schedules at all. Off for a second worker
|
|
#: on the same file: they would both fire every due row, and two of
|
|
#: them would fire it twice — the pile-up guard reads `last_job_id`,
|
|
#: which neither has written yet at the moment the other looks. One
|
|
#: scheduler per file, as many workers as you like.
|
|
self.schedules = schedules
|
|
self.schedule_tick = (self.SCHEDULE_TICK if schedule_tick is None
|
|
else schedule_tick)
|
|
self._listener = listener
|
|
self._on_notice = on_notice
|
|
self._on_schedules = on_schedules
|
|
self.workers = dict(workers if workers is not None else self.WORKERS)
|
|
#: The jobs this process has in hand, by the task running each. A map
|
|
#: rather than a set because the ids are what the heartbeat writes.
|
|
self._running: dict[asyncio.Task[None], int] = {}
|
|
self._loop: asyncio.AbstractEventLoop | None = None
|
|
self._task: asyncio.Task[None] | None = None
|
|
self._pulse: asyncio.Task[None] | None = None
|
|
#: Threads for handlers, and for nothing else. See `start`.
|
|
self._pool: ThreadPoolExecutor | None = None
|
|
self._wake = asyncio.Event()
|
|
self._stopping = False
|
|
self._reclaimed_at = 0.0
|
|
self._purged_at = 0.0
|
|
self._scheduled_at = 0.0
|
|
#: Schedules this process has already complained about, so a row with
|
|
#: a cadence nothing can parse or a kind nothing registers is said
|
|
#: once rather than twice a minute for as long as the app is up. The
|
|
#: same restraint `notices.already_said` shows, at the size it needs
|
|
#: to be here.
|
|
self._said_about: set[tuple[int, str]] = set()
|
|
|
|
# ----------------------------------------------------------- lifecycle
|
|
|
|
async def run_until_idle(self, timeout: float | None = None) -> bool:
|
|
"""Start, work until nothing is queued or running, and stop.
|
|
|
|
Returns whether it got there, as against running out of `timeout`. The
|
|
batch shape — a script, a cron entry, `python -m pipeline.worker
|
|
--until-idle`, a test — where the web app's shape is `start` and never
|
|
stopping.
|
|
|
|
"Nothing queued" includes jobs waiting for a later attempt, which is
|
|
what a caller draining a queue means: a retry due in a minute is still
|
|
work this run was asked to finish. Pass `timeout` when that could be
|
|
a long wait.
|
|
"""
|
|
await self.start()
|
|
deadline = None if timeout is None else time.monotonic() + timeout
|
|
try:
|
|
while True:
|
|
snap = await asyncio.to_thread(self.jobs.snapshot)
|
|
if not snap.running and not snap.queued and not self._running:
|
|
return True
|
|
if deadline is not None and time.monotonic() > deadline:
|
|
return False
|
|
await asyncio.sleep(0.05)
|
|
finally:
|
|
await self.stop()
|
|
|
|
def drain(self, timeout: float | None = None) -> bool:
|
|
"""`run_until_idle` from code that has no event loop running."""
|
|
return asyncio.run(self.run_until_idle(timeout))
|
|
|
|
async def start(self) -> None:
|
|
self._loop = asyncio.get_running_loop()
|
|
self._stopping = False # a stopped queue can be started again
|
|
# Made here rather than in `__init__` because an `asyncio.Event` binds
|
|
# to the first loop that uses it and refuses every later one. A queue
|
|
# started, stopped and started again — a test, a reload — would
|
|
# otherwise be waiting on a loop that has been closed since.
|
|
self._wake = asyncio.Event()
|
|
# Handlers get threads of their own rather than the interpreter's
|
|
# shared pool. That pool is sized from the cpu count, so on a small
|
|
# machine it is smaller than `workers` and two things go wrong at
|
|
# once: fewer jobs run than the table admits, and the loop's own
|
|
# claim and heartbeat queue up behind the handlers. A worker starved
|
|
# of a thread stops renewing its leases and has its live jobs
|
|
# re-queued underneath it, which is the one mistake the lease exists
|
|
# to prevent. So: as many threads as there are workers, plus a few
|
|
# for resources the app never sized, which the table runs one at a
|
|
# time each.
|
|
self._pool = ThreadPoolExecutor(
|
|
max_workers=sum(self.workers.values()) + 4,
|
|
thread_name_prefix="desk-handler")
|
|
await asyncio.to_thread(self._reclaim)
|
|
if self.schedules:
|
|
# Both here rather than left to the first turn of the loop, and in
|
|
# this order. Reconciling first means a default declared today has
|
|
# a row before anything looks for due ones, so a new schedule fires
|
|
# on the start that introduced it rather than the one after. And
|
|
# firing here means an app that was down over 06:00 runs the 06:00
|
|
# job when it comes back, which is the reading of "overdue" that a
|
|
# person expects — once, not once per slot missed.
|
|
await asyncio.to_thread(self._reconcile_schedules)
|
|
await asyncio.to_thread(self._fire_due)
|
|
|
|
self._task = asyncio.create_task(self._drain(), name="desk-worker")
|
|
self._pulse = asyncio.create_task(self._heartbeat(), name="desk-heartbeat")
|
|
|
|
async def stop(self) -> None:
|
|
"""Stop taking new work and let what is running finish.
|
|
|
|
An in-flight handler is a thread and is not interrupted — killing it
|
|
mid-write is exactly what the jobs table exists to avoid. So they are
|
|
given `STOP_GRACE` to come back on their own, which most of them use,
|
|
and whatever is still going after that is walked away from.
|
|
|
|
Walked away from, and not taken from: the ending of a job is written
|
|
by the thread running it, so a handler that misses the window still
|
|
lands its own `done` or `failed` in the row whenever it gets there.
|
|
What is handed back here is the lease, because a shutdown is the one
|
|
moment a process can say outright that it is no longer watching these.
|
|
That saves the next start from waiting out the silence for the ones
|
|
whose thread this process did not outlive — and only those, since a
|
|
row its thread did finish is no longer `running` for a reclaim to
|
|
take.
|
|
"""
|
|
self._stopping = True
|
|
self._wake.set()
|
|
for task in (self._task, self._pulse):
|
|
if task is not None:
|
|
task.cancel()
|
|
with contextlib.suppress(asyncio.CancelledError):
|
|
await task
|
|
# A moment to land in, and no more. Cancelling the task does not
|
|
# reach into the thread, so the ones that miss this window keep going
|
|
# regardless; what the wait buys is that the common case — a fetch, a
|
|
# write — finishes properly instead of being torn off mid-sentence.
|
|
if self._running:
|
|
await asyncio.wait(list(self._running), timeout=self.STOP_GRACE)
|
|
held = list(self._running.values())
|
|
for task in list(self._running):
|
|
task.cancel()
|
|
self._running.clear()
|
|
# Said out loud rather than left to the lease. Only what this process
|
|
# holds, so a shutdown here never speaks for a job another worker is
|
|
# part way through.
|
|
await asyncio.to_thread(self.jobs.release, held)
|
|
# Forget the loop as well as the task. It is about to be closed, and a
|
|
# later `wake()` or `notify()` holding a reference to a closed loop
|
|
# raises rather than being the no-op it should be.
|
|
if self._pool is not None:
|
|
# Not waited for, same as the tasks: a handler in a thread is not
|
|
# coming back on request, and the interpreter joins it at exit.
|
|
self._pool.shutdown(wait=False)
|
|
self._pool = None
|
|
self._loop = None
|
|
self._task = None
|
|
self._pulse = None
|
|
|
|
# -------------------------------------------------------------- public
|
|
|
|
def enqueue(self, kind: str, payload: dict[str, Any] | None = None, *,
|
|
label: str = "", depends_on: Sequence[int] = (), after: str = "ok",
|
|
cascade_id: int = 0, task_id: int = 0, delay: float = 0.0,
|
|
max_attempts: int | None = None, retry_delay: float | None = None,
|
|
retry_backoff: float | None = None,
|
|
dedup_key: str | None = None) -> Job | None:
|
|
"""Queue work by name. Returns the job — or the live one it deduped to.
|
|
|
|
`depends_on` holds it back until those jobs are over; `after` says
|
|
whether it needs them to have succeeded. Dedup is skipped for a job
|
|
that waits on something, because the point of one is that there will
|
|
be several — eight of a kind, each waiting on its own batch.
|
|
|
|
`delay` queues it for later without anything staying alive in the
|
|
meantime: the row carries the time and `claim` passes over it until
|
|
then, so "in ten minutes" survives a restart four minutes in.
|
|
|
|
`max_attempts` and the two waits override what the kind declared, for
|
|
a caller that knows more about this one piece of work than the handler
|
|
does — which is what a plugin's own retry policy is: the outlet, not
|
|
the job kind, is what knows how long to leave a publisher alone. They
|
|
go onto the row, so the attempt that reads them may be in another
|
|
process, after a restart, under a plugin file that has since changed.
|
|
|
|
`label` overrides the kind's, for one piece of work that can say more
|
|
about itself than its kind can. A queued job has no `message` — that is
|
|
written by a running handler — so the label is the whole of what the
|
|
queue list has to show, and the kind's is by construction the same
|
|
string on every job of that kind. A caller that knows how the work was
|
|
cut says so here. Empty means the kind's, which is every caller but
|
|
one.
|
|
|
|
`dedup_key` overrides the kind's own, which is how one *item's*
|
|
acquisition is made unrepeatable while the kind stays freely
|
|
repeatable: two presses of a reader's retry button collide on the key
|
|
and the second is handed the first one's job, across tabs, clients and
|
|
processes, because the collision happens in a unique index rather than
|
|
in whichever page asked.
|
|
|
|
`None` when the run it would belong to has been killed. Logged and
|
|
dropped rather than raised: whatever asked for this — a cascade's step,
|
|
a task fanning out its batches — has no way of knowing and nothing to
|
|
do about it. The kill has already happened; this is it arriving.
|
|
"""
|
|
spec = JOBS.get(kind)
|
|
if spec is None:
|
|
raise KeyError(
|
|
f"no job registered as {kind!r} (have: {', '.join(sorted(JOBS)) or 'none'})"
|
|
)
|
|
solo = spec.solo and not depends_on
|
|
try:
|
|
job = self.jobs.enqueue(
|
|
kind, label or spec.label, payload,
|
|
dedup_key=(kind if solo else "") if dedup_key is None
|
|
else dedup_key,
|
|
depends_on=depends_on, after=after, resource=spec.resource,
|
|
cascade_id=cascade_id, task_id=task_id, delay=delay,
|
|
max_attempts=spec.max_attempts if max_attempts is None
|
|
else max_attempts,
|
|
retry_delay=spec.retry_delay if retry_delay is None
|
|
else retry_delay,
|
|
retry_backoff=spec.retry_backoff if retry_backoff is None
|
|
else retry_backoff)
|
|
except RunCancelled as refused:
|
|
log.info("%s", refused)
|
|
return None
|
|
self.wake()
|
|
self.notify()
|
|
return job
|
|
|
|
def new_run(self, kind: str, name: str = "", label: str = "") -> int:
|
|
"""Mint an id for a cascade or a task. Half of what a `Task` needs of
|
|
a sink; `enqueue` and `cancelled` are the rest."""
|
|
return self.jobs.new_run(kind, name, label)
|
|
|
|
def cancelled(self, run_id: int) -> bool:
|
|
return self.jobs.cancelled(run_id)
|
|
|
|
def start_task(self, task: Any, *, cascade_id: int = 0) -> int:
|
|
"""Run a `Task` here and now, and return its run id.
|
|
|
|
Here and now because a task is plain code: it asks the store what there
|
|
is to do and queues it. That is a query and a handful of inserts, which
|
|
is cheap enough to do in the request that pressed the button or in the
|
|
worker thread that has just finished the job before.
|
|
"""
|
|
return task.start(self.config, self.store, self, cascade_id=cascade_id)
|
|
|
|
def cancel(self, job_id: int) -> Job | None:
|
|
"""Ask a job to stop. Queued ones end here; running ones on their next
|
|
`context.check()`. Returns the job as it now stands, or None if there
|
|
isn't one."""
|
|
job = self.jobs.request_cancel(job_id)
|
|
self.wake()
|
|
self.notify()
|
|
return job
|
|
|
|
def cancel_run(self, run_id: int) -> int:
|
|
"""Kill a task or a cascade — one operation for both. How many ended.
|
|
|
|
Killing a task is "skip this one": its batches end, its final job still
|
|
runs, and what it reports says the task was killed rather than only how
|
|
many batches ended cancelled. What happens next is then the cascade's
|
|
to decide, and newsdesk's pull cascade decides to carry on to the next
|
|
stage. Killing the cascade takes the final job with it, so nothing
|
|
reports and nothing follows — the one decision that is not the
|
|
cascade's, because there is nothing left of it to ask.
|
|
`Jobs.cancel_run` is where all of that is one statement.
|
|
"""
|
|
ended = self.jobs.cancel_run(run_id)
|
|
self.wake()
|
|
self.notify()
|
|
return ended
|
|
|
|
def stop_task(self, task_id: int, why: str = "") -> int:
|
|
"""End a task's queued batches, sparing what is running and its final
|
|
job. How many ended.
|
|
|
|
Not a kill and not written as one: the task's row stays as it is, so
|
|
nothing reports that a person stopped this. What a handler reaches
|
|
through `RunContext.stop`, for the failure that is the backend's
|
|
answer to the whole run — see `Jobs.stop_task`.
|
|
"""
|
|
ended = self.jobs.stop_task(task_id, error=why)
|
|
self.wake()
|
|
self.notify()
|
|
return ended
|
|
|
|
def snapshot(self) -> JobsSnapshot:
|
|
return self.jobs.snapshot()
|
|
|
|
def get(self, job_id: int) -> Job | None:
|
|
return self.jobs.get(job_id)
|
|
|
|
def wake(self) -> None:
|
|
"""Tell the loop to look for work now instead of at the next poll.
|
|
|
|
A no-op when no worker is running in this process — enqueueing without
|
|
one is legitimate (a command queueing work for the app to pick up), and
|
|
the row is in the table either way.
|
|
"""
|
|
if self._loop is None or self._loop.is_closed():
|
|
return
|
|
with contextlib.suppress(RuntimeError):
|
|
self._loop.call_soon_threadsafe(self._wake.set)
|
|
|
|
def run_schedule(self, schedule_id: int) -> Job | None:
|
|
"""Fire one schedule now, leaving when it is next due alone.
|
|
|
|
What Run now presses. `next_run` is untouched on purpose: a person
|
|
asking for this run is not asking to move the nightly one, and a button
|
|
that quietly rescheduled what it touched would be a button nobody could
|
|
use twice.
|
|
|
|
`last_run` and `last_job_id` are written, because they are the truth
|
|
about this schedule — it did just run, and that is the job it queued.
|
|
Which also means the next tick will not start a second one while this
|
|
is still going.
|
|
"""
|
|
row = self.jobs.schedule(schedule_id)
|
|
if row is None:
|
|
return None
|
|
job = self.enqueue(row.kind, row.payload, label=row.label)
|
|
self.jobs.update_schedule(row.id, last_run=datetime.now(UTC).isoformat(),
|
|
last_job_id=job.id if job else 0)
|
|
self.notify_schedules()
|
|
return job
|
|
|
|
def notify_schedules(self) -> None:
|
|
"""Push the schedule list to listeners. Safe from any thread.
|
|
|
|
Called by whatever changed one — a firing here, a route over there —
|
|
rather than polled, so a row edited in one tab redraws in the others
|
|
without either of them asking every few seconds.
|
|
"""
|
|
if self._loop is None or self._on_schedules is None or self._loop.is_closed():
|
|
return
|
|
with contextlib.suppress(RuntimeError):
|
|
asyncio.run_coroutine_threadsafe(self._push_schedules(), self._loop)
|
|
|
|
def announce(self, notice: Notice) -> None:
|
|
"""Write one notice down, then push it to whoever is watching. Safe
|
|
from any thread.
|
|
|
|
Written first, because the push is fire-and-forget and the usual
|
|
listener is nobody: a pull runs on a schedule and the phone that would
|
|
have shown the line is in a pocket. It used to stop there — "the log
|
|
already has it" — which was true of a file on a server and no use to
|
|
the person the notice was for. The table is what a tab opened
|
|
afterwards reads, and where a line that faded can be found again.
|
|
|
|
What is pushed is the row, count included, so a listener can tell news
|
|
from a repeat of something already in the log. That judgement is left
|
|
to it on purpose: every notice still goes out, and what interrupts a
|
|
person is the interface's business. See `NoticeEntry`.
|
|
|
|
Called from the worker thread, so the push hops the loop the same way
|
|
`notify` does, and delivery is still never waited for.
|
|
"""
|
|
# Logged as well, always. The socket is the fast path, the table is
|
|
# what is on screen tomorrow, and the log is the only one of the three
|
|
# with the traceback beside it.
|
|
log.warning("%s\n%s", notice.code, render(notice))
|
|
try:
|
|
notice = self.jobs.record_notice(notice)
|
|
except Exception:
|
|
# A notice is already the report of a failure. One that cannot be
|
|
# written is still worth pushing, and is never worth failing the
|
|
# job that raised it.
|
|
log.exception("could not record the %s notice; pushing it anyway",
|
|
notice.code)
|
|
if self._loop is None or self._on_notice is None or self._loop.is_closed():
|
|
return
|
|
with contextlib.suppress(RuntimeError):
|
|
asyncio.run_coroutine_threadsafe(self._on_notice(notice), self._loop)
|
|
|
|
def notify(self) -> None:
|
|
"""Push the queue's state to listeners. Safe from any thread."""
|
|
if self._loop is None or self._listener is None or self._loop.is_closed():
|
|
return
|
|
with contextlib.suppress(RuntimeError):
|
|
asyncio.run_coroutine_threadsafe(self._announce(), self._loop)
|
|
|
|
# --------------------------------------------------------------- guts
|
|
|
|
async def _drain(self) -> None:
|
|
"""Claim whatever there is a free worker for, then wait to be woken.
|
|
|
|
Claiming until it comes back empty is what lets the table decide how
|
|
many run rather than this loop: it only asks, and stops when told. It never counts anything itself, which is
|
|
the only way a second process sharing the file gets the same answer.
|
|
|
|
One bad turn does not end it. A locked database or a file that went
|
|
away for a moment would otherwise take this task down with an
|
|
exception nobody is waiting to receive, and the process would sit
|
|
there looking healthy while draining nothing at all.
|
|
"""
|
|
while not self._stopping:
|
|
try:
|
|
await self._turn()
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception:
|
|
log.exception("the drain loop hit an error; carrying on")
|
|
await asyncio.sleep(self.IDLE_POLL)
|
|
|
|
async def _turn(self) -> None:
|
|
"""One pass: take what there is room for, or wait to be woken.
|
|
|
|
`IDLE_POLL` is also what notices a job whose next attempt has come
|
|
round. Nothing wakes the loop for one — the row said it was due at a
|
|
moment nobody was listening for — so the poll that exists to notice
|
|
another process's enqueue notices this too, and a retry starts within a
|
|
second of its time. Which is why a scheduled job needs no timer here:
|
|
there was already something looking.
|
|
"""
|
|
# Before the claim, not after it with the other housekeeping. Reclaim
|
|
# and purge are reached only when there was nothing to take, which is
|
|
# right for both — they are repairs, and a busy queue has more pressing
|
|
# work. A schedule is the opposite: it is work, and a queue that is
|
|
# never idle would be one where 06:00 never came round. The check
|
|
# itself is a clock comparison, so paying it on every pass costs
|
|
# nothing worth measuring.
|
|
if self.schedules and (time.monotonic() - self._scheduled_at
|
|
>= self.schedule_tick):
|
|
await asyncio.to_thread(self._fire_due)
|
|
job = await asyncio.to_thread(self.jobs.claim, self.workers)
|
|
if job is not None:
|
|
self._start(job)
|
|
return
|
|
# Nothing to take, which is the moment to ask whether some other
|
|
# worker's `running` row has gone quiet. Here rather than only at
|
|
# start, so a process that dies is cleaned up by whichever one is
|
|
# still alive instead of waiting for somebody to be restarted.
|
|
if time.monotonic() - self._reclaimed_at >= self.STALE_AFTER:
|
|
await asyncio.to_thread(self._reclaim)
|
|
if self.retain and time.monotonic() - self._purged_at >= self.PURGE_EVERY:
|
|
await asyncio.to_thread(self._purge)
|
|
self._wake.clear()
|
|
with contextlib.suppress(asyncio.TimeoutError):
|
|
await asyncio.wait_for(self._wake.wait(), self.IDLE_POLL)
|
|
|
|
def _reclaim(self) -> None:
|
|
"""Recover silent workers' jobs while renewing our own stalled leases."""
|
|
self._reclaimed_at = time.monotonic()
|
|
stale = self.jobs.reclaim_stale(
|
|
self.STALE_AFTER, except_ids=list(self._running.values()))
|
|
if stale:
|
|
log.info("re-queued %d job(s) whose worker stopped reporting in", stale)
|
|
|
|
def _purge(self) -> None:
|
|
"""Forget finished jobs older than `retain`. Only when idle, like the
|
|
reclaim above it, so it never competes with work for the file."""
|
|
self._purged_at = time.monotonic()
|
|
try:
|
|
gone = self.jobs.purge(self.retain)
|
|
# The notices with them, on their own clock: a job is history once
|
|
# it has finished, a notice once it has stopped being said.
|
|
self.jobs.purge_notices(self.NOTICE_QUIET)
|
|
except Exception:
|
|
log.exception("could not purge old jobs; carrying on")
|
|
return
|
|
if gone:
|
|
log.info("forgot %d finished job(s) older than %g days",
|
|
gone, self.retain / 86400)
|
|
|
|
# ----------------------------------------------------------- schedules
|
|
|
|
def _reconcile_schedules(self) -> None:
|
|
"""Give every declared default a row, and leave every existing one be.
|
|
|
|
The whole of what a declaration buys: a name nothing has a row for gets
|
|
one, and a name that does have one keeps its cadence, its payload and
|
|
whether it is switched on. So the declarations in an app's jobs module
|
|
are where a schedule *comes from*, and the table is what it *is*;
|
|
turning one off in the UI survives the next deploy, and changing a
|
|
cadence in the code reaches only the installs that never had it.
|
|
|
|
The label is the one exception. No screen edits it, so the row's copy
|
|
is only ever the declaration's, and a relabelled declaration that
|
|
reached no existing install would leave the old wording on screen
|
|
forever. It is refreshed while the row still has the declared kind.
|
|
|
|
Which is also why nothing here deletes. A declaration that goes away
|
|
leaves its row behind, `unknown kind` against it if the handler went
|
|
too, for a person to delete deliberately — the alternative is a deploy
|
|
that silently forgets a schedule somebody was relying on.
|
|
"""
|
|
now = datetime.now(UTC)
|
|
for declared in SCHEDULES.values():
|
|
row = self.jobs.add_schedule(
|
|
declared.name, declared.kind, declared.cadence,
|
|
payload=declared.payload, label=declared.label,
|
|
next_run=Cadence.parse(declared.cadence)
|
|
.next_after(now, self.zone).isoformat(),
|
|
only_if_new=True)
|
|
if row is not None:
|
|
log.info("schedule %r declared: %s %s, first at %s",
|
|
row.name, row.kind, row.cadence, row.next_run)
|
|
elif self.jobs.relabel_schedule(declared.name, declared.kind,
|
|
declared.label):
|
|
log.info("schedule %r relabelled %r", declared.name,
|
|
declared.label)
|
|
|
|
def _fire_due(self) -> None:
|
|
"""Queue a job for every schedule whose time has come. One pass.
|
|
|
|
From the drain loop's own thread pool, not a timer and not a second
|
|
thread: there is one place that decides what runs, and adding a second
|
|
clock would mean two things racing to enqueue the same row.
|
|
|
|
Three ways a due schedule does not fire:
|
|
|
|
unknown kind the handler is gone — a plugin uninstalled, a
|
|
deploy that dropped a job. Said once and then left
|
|
exactly as it is, `next_run` included, so that
|
|
registering the handler again runs it rather than
|
|
waiting out another cadence.
|
|
`<desk-schedules>` marks the row `unknown kind`
|
|
and a person deletes it if it is not coming back.
|
|
bad cadence only reachable by editing the table by hand, since
|
|
every route parses before it writes. Same
|
|
treatment, and for the same reason: fixing the
|
|
wording is meant to be enough.
|
|
still going the last job this schedule queued is queued or
|
|
running, or the kind is `solo` and one is under
|
|
way. Here `next_run` *does* advance and nothing is
|
|
queued, which is the difference between a schedule
|
|
and a backlog: a job slower than its own cadence
|
|
must not grow a queue of itself.
|
|
"""
|
|
self._scheduled_at = time.monotonic()
|
|
now = datetime.now(UTC)
|
|
moved = False
|
|
for row in self.jobs.due_schedules(now.isoformat()):
|
|
spec = JOBS.get(row.kind)
|
|
try:
|
|
when = Cadence.parse(row.cadence).next_after(now, self.zone)
|
|
except ValueError as bad:
|
|
self._say_once(row, "cadence", "schedule %r will not run: %s",
|
|
row.name, bad)
|
|
continue
|
|
if spec is None:
|
|
self._say_once(row, "kind",
|
|
"schedule %r wants %r, which no handler is "
|
|
"registered for; it will not run until one is",
|
|
row.name, row.kind)
|
|
continue
|
|
moved = True
|
|
if self._still_going(row, spec):
|
|
log.info("schedule %r was due, but its last job is still "
|
|
"going; next at %s", row.name, when.isoformat())
|
|
self.jobs.update_schedule(row.id, next_run=when.isoformat())
|
|
continue
|
|
job = self.enqueue(row.kind, row.payload, label=row.label)
|
|
self.jobs.mark_fired(row.id, job.id if job else 0, when.isoformat())
|
|
log.info("schedule %r fired %s as job %s; next at %s", row.name,
|
|
row.kind, job.id if job else "nothing", when.isoformat())
|
|
if moved:
|
|
self.notify_schedules()
|
|
|
|
def _still_going(self, row: Schedule, spec: Spec) -> bool:
|
|
"""Whether firing this schedule would pile one job on another.
|
|
|
|
Two questions, because a schedule can collide with itself two ways.
|
|
Its own last job may still be going, which is the slow-job case. Or the
|
|
kind may be `solo` and something else may have started one — a button,
|
|
another schedule, a cascade — in which case `enqueue` would hand back
|
|
that job and the schedule would record somebody else's work as its own
|
|
firing.
|
|
"""
|
|
if row.last_job_id:
|
|
last = self.jobs.get(row.last_job_id)
|
|
if last is not None and last.live:
|
|
return True
|
|
return bool(spec.solo and self.jobs.last(row.kind, states=("queued", "running")))
|
|
|
|
def _say_once(self, row: Schedule, about: str, message: str, *args: Any) -> None:
|
|
"""Log a schedule's own defect the first time this process meets it.
|
|
|
|
A tick every thirty seconds would otherwise make one broken row into
|
|
two thousand log lines a day, which is how a log stops being read.
|
|
Keyed on the row and what is wrong with it, so fixing the cadence and
|
|
then breaking the kind still says both.
|
|
"""
|
|
if (row.id, about) in self._said_about:
|
|
return
|
|
self._said_about.add((row.id, about))
|
|
log.warning(message, *args)
|
|
|
|
async def _push_schedules(self) -> None:
|
|
"""Read the list off the loop, for the reason `_announce` does."""
|
|
if self._on_schedules is None:
|
|
return
|
|
try:
|
|
await self._on_schedules(await asyncio.to_thread(self.jobs.schedules))
|
|
except Exception:
|
|
log.exception("schedule listener raised; carrying on")
|
|
|
|
async def _heartbeat(self) -> None:
|
|
"""Say the jobs this process holds are still being run.
|
|
|
|
From the loop, not from the handler: a handler is arbitrary blocking
|
|
code in a thread and may not come back for an hour, and a lease it can
|
|
only renew between units of work would make "slow" indistinguishable
|
|
from "dead". The loop is the thing that is still there either way, and
|
|
if it is not, then this process really has stopped and its jobs really
|
|
should go back.
|
|
"""
|
|
while not self._stopping:
|
|
await asyncio.sleep(self.HEARTBEAT_EVERY)
|
|
ids = list(self._running.values())
|
|
if not ids:
|
|
continue
|
|
try:
|
|
await asyncio.to_thread(self.jobs.heartbeat, ids)
|
|
except Exception:
|
|
# A missed beat is survivable — the next one lands well inside
|
|
# the lease — and a pulse that dies on one is not, because
|
|
# every job this process holds would be re-queued under it.
|
|
log.exception("heartbeat failed; carrying on")
|
|
|
|
def _start(self, job: Job) -> None:
|
|
"""Run one job alongside whatever else is going.
|
|
|
|
The finished callback wakes the loop rather than the loop watching the
|
|
set: a worker has just come free and something queued may now fit in it,
|
|
and that is the same signal an enqueue sends.
|
|
"""
|
|
task = asyncio.create_task(self._execute(job), name=f"job-{job.id}")
|
|
self._running[task] = job.id
|
|
|
|
def done(finished: asyncio.Task[None]) -> None:
|
|
self._running.pop(finished, None)
|
|
with contextlib.suppress(asyncio.CancelledError):
|
|
finished.result()
|
|
self._wake.set()
|
|
|
|
task.add_done_callback(done)
|
|
|
|
async def _execute(self, job: Job) -> None:
|
|
spec = JOBS.get(job.kind)
|
|
await self._announce()
|
|
|
|
if spec is None:
|
|
# Reachable: a job queued by a newer version, or by a plugin that
|
|
# is no longer installed. Failing it says so and clears the row,
|
|
# rather than leaving it claimed forever.
|
|
await asyncio.to_thread(self.jobs.finish,
|
|
job.id, "failed", error=f"no handler registered for {job.kind!r}")
|
|
await self._announce()
|
|
return
|
|
|
|
tasks: list[Any] = []
|
|
said: list[Notice] = []
|
|
|
|
fn, label = spec.fn, spec.label
|
|
args = (self.config, self.store)
|
|
if spec.isolate:
|
|
# The row is the whole input, so nothing is sent — the child is
|
|
# told where to look. Its signature matches a handler's on
|
|
# purpose: everything below this line, the ending, the cancel
|
|
# check on return, the retry, is the same code either way.
|
|
fn = self._isolated(job, spec)
|
|
|
|
def run() -> None:
|
|
"""The handler and this job's ending, in the same thread.
|
|
|
|
Together because the thread is the half of this that survives a
|
|
shutdown. `stop()` cancels the coroutine of a job whose handler
|
|
missed the grace period, and a cancelled coroutine never reaches
|
|
the line after its `await` — so an ending written back there is
|
|
precisely the ending an abandoned job never gets. The handler runs
|
|
on to the end regardless, writes whatever it writes, and leaves a
|
|
`running` row for the next start to re-queue and run a second
|
|
time: work already done, done again, and for anything that asks a
|
|
model a second bill for it.
|
|
|
|
Nothing in here needs the loop. Everything that does — the push
|
|
saying the row moved — stays outside.
|
|
"""
|
|
try:
|
|
if "context" in job.payload:
|
|
raise TypeError("context is reserved for the worker")
|
|
context = self._context(job, tasks, said)
|
|
extra = dict(job.payload)
|
|
if spec.wants_context:
|
|
extra["context"] = context
|
|
result = fn(*args, **extra)
|
|
result = result if isinstance(result, dict) else {}
|
|
if self.jobs.cancel_requested(job.id):
|
|
# Asked to stop while it ran, and it never looked — a handler
|
|
# with no `check()`, or one whose last unit was already under
|
|
# way. The return is its last chance to look, so it is taken
|
|
# here: the job ends cancelled, keeping what it made, and
|
|
# queues nothing. Ending `done` would start the next task
|
|
# under a cascade that has just been killed — a Kill that did
|
|
# nothing. (The guard in `Jobs.enqueue` would refuse the jobs
|
|
# anyway; this is the same answer one step earlier, and without
|
|
# a run minted for nothing.)
|
|
self.jobs.finish(job.id, "cancelled", result=result)
|
|
return
|
|
self.jobs.finish(job.id, "done", result=result)
|
|
for task in tasks:
|
|
# After the job landed, never before: a task that outlived the
|
|
# failure of the job that described it would be work nobody
|
|
# asked for. Here rather than back on the loop for the same
|
|
# two reasons the ending is — a finder query and a handful of
|
|
# inserts are no more the loop's business than the handler
|
|
# was, and a job abandoned by a shutdown still asks for what
|
|
# it said should follow. Its own failure is logged and no
|
|
# more: the job it came from is already over and did what it
|
|
# was asked.
|
|
try:
|
|
self.start_task(task, cascade_id=job.cascade_id)
|
|
except Exception:
|
|
log.exception("job %d asked for a task that would not start",
|
|
job.id)
|
|
except Cancelled:
|
|
self.jobs.finish(job.id, "cancelled")
|
|
except Retry as again:
|
|
# The handler naming its own wait — a rate limit's answer, a
|
|
# `Retry-After`. Logged rather than tracebacked: this is the
|
|
# world saying "not yet", which is a fact about the service and
|
|
# not a defect here. Whether there is another attempt to give
|
|
# it is still the row's to decide.
|
|
log.info("job %d (%s) asked to come back: %s", job.id, job.kind, again)
|
|
self._ended_badly(job, f"{type(again).__name__}: {again}",
|
|
label, said, delay=again.after)
|
|
except ResultTooLarge as exc:
|
|
# Loud and final. Not `_ended_badly`, because a retry would run
|
|
# the same handler to the same oversized result; and not
|
|
# quiet, because the alternative to failing is dropping data.
|
|
log.error("job %d (%s) %s", job.id, job.kind, exc)
|
|
error = f"{type(exc).__name__}: {exc}"
|
|
self.jobs.finish(job.id, "failed", error=error)
|
|
self.announce(job_failed(job.kind, label, error))
|
|
except Exception as exc:
|
|
# Deliberately broad. A handler is arbitrary third-party-ish code
|
|
# (a source plugin, a model, a subprocess); anything it raises is
|
|
# this job's failure and nothing else's, and letting it escape
|
|
# would take the worker down and strand every job behind it.
|
|
log.exception("job %d (%s) failed", job.id, job.kind)
|
|
self._ended_badly(job, f"{type(exc).__name__}: {exc}", label, said)
|
|
|
|
try:
|
|
await asyncio.get_running_loop().run_in_executor(self._pool, run)
|
|
finally:
|
|
await self._announce()
|
|
|
|
def _ended_badly(self, job: Job, error: str, label: str,
|
|
said: list[Notice], *, delay: float | None = None) -> None:
|
|
"""An attempt that did not land: another one later, or the end of it.
|
|
|
|
Called from the handler's thread, like the ending it replaces, and for
|
|
the same reason — a shutdown that abandoned the thread must not lose
|
|
the row's move to whatever comes next. `Jobs.retry` decides which move
|
|
that is; here there is only the question of what to say about it.
|
|
|
|
A failure with another attempt coming says nothing to the screen. A
|
|
notice is for what a person has to act on, and there is nothing to act
|
|
on yet: the queue is already doing the thing they would have asked for.
|
|
The log has it, as it has everything. Only the attempt that runs out
|
|
announces, and it announces the failure that ended it.
|
|
"""
|
|
ended = self.jobs.retry(job.id, error=error, delay=delay)
|
|
if ended is not None and ended.live:
|
|
log.info("job %d (%s) goes back in the queue: attempt %d of %d",
|
|
job.id, job.kind, job.attempt, job.max_attempts)
|
|
return
|
|
# And a notice, so it reaches the screen and not only the log — unless
|
|
# this failure has already put one there. A handler that classified its
|
|
# own failure said the useful thing about it, which login expired or
|
|
# which switch is off; adding "failed, looks like a bug" underneath
|
|
# contradicts it on the same screen, and sends you to a traceback about
|
|
# a machine that is working correctly.
|
|
#
|
|
# `job_failed` keeps the case it was written for: a handler that raised
|
|
# something nobody caught and said nothing about it. That really is
|
|
# ours to fix, and it is still the only thing here that says `bug`
|
|
# without working it out.
|
|
if ended is not None and ended.state == "cancelled":
|
|
return # a Kill that landed mid-attempt; nothing broke
|
|
if not already_said(said, error):
|
|
self.announce(job_failed(job.kind, label, error))
|
|
|
|
def _isolated(self, job: Job, spec: Spec) -> Callable[..., dict[str, Any]]:
|
|
"""A stand-in handler that runs this job in a child process.
|
|
|
|
Returned rather than called so the worker's thread calls it, in the
|
|
place the real handler would have been called: the ending, the retry
|
|
and the cancel-on-return are then one piece of code for both paths.
|
|
|
|
`context` is accepted and dropped. The child builds its own — one that
|
|
writes to the same table, which is how its progress reaches the
|
|
browser without anything being forwarded. What cannot cross is the
|
|
pair the handlers take, hence `bootstrap`.
|
|
|
|
Failure is read off the exit code and the result file, not off stderr.
|
|
A child that died of something Python never saw — an OOM kill, a
|
|
segfault in a native library — leaves no file, and gets the only
|
|
honest report available: its exit code and whatever it managed to say.
|
|
"""
|
|
def run_in_child(*_args: Any, context: RunContext | None = None,
|
|
**_payload: Any) -> dict[str, Any]:
|
|
# Checked in here rather than above, where it reads more naturally
|
|
# and would be raised while the worker was still assembling the
|
|
# run — outside the `try` that turns a handler's exception into
|
|
# this job's ending, so the row would stay `running` for the lease
|
|
# to clean up and the reason would be in the log only.
|
|
if not self.bootstrap:
|
|
raise RuntimeError(
|
|
f"{job.kind!r} is isolate=True, but this queue has no "
|
|
"bootstrap. A child process cannot be handed the app's "
|
|
"config and store, so it has to rebuild them: pass "
|
|
"bootstrap='module:function' to JobQueue, returning "
|
|
"(config, store)."
|
|
)
|
|
with tempfile.TemporaryDirectory(prefix="pipeline-job-") as tmp:
|
|
out = Path(tmp) / "result.json"
|
|
argv = [sys.executable, "-m", "pipeline.child",
|
|
"--db", str(self.jobs.path), "--job", str(job.id),
|
|
"--bootstrap", self.bootstrap, "--out", str(out)]
|
|
if self.bootstrap_arg:
|
|
argv += ["--bootstrap-arg", self.bootstrap_arg]
|
|
done = spawn(argv, check=context.check if context else None,
|
|
timeout=spec.timeout or None)
|
|
said = json.loads(out.read_text(encoding="utf-8")) if out.exists() else {}
|
|
if done.returncode == CANCELLED:
|
|
raise Cancelled()
|
|
if done.returncode != 0 or "error" in said:
|
|
detail = said.get("error") or (
|
|
done.stderr.decode("utf-8", "replace").strip().splitlines()[-1:]
|
|
or [f"exited {done.returncode}"])[0]
|
|
if said.get("traceback"):
|
|
log.error("job %d (%s) failed in its own process:\n%s",
|
|
job.id, job.kind, said["traceback"])
|
|
raise ChildFailed(detail)
|
|
result = said.get("result")
|
|
return result if isinstance(result, dict) else {}
|
|
|
|
return run_in_child
|
|
|
|
def _context(self, job: Job, tasks: list[Any],
|
|
said: list[Notice]) -> RunContext:
|
|
"""The `RunContext` for one run. The callbacks are called from the worker
|
|
thread, so they are written to be safe there.
|
|
|
|
`said` collects what the handler announced while it ran, which is how
|
|
the failure above knows whether this ending has already been explained.
|
|
"""
|
|
pushed_at = 0.0
|
|
checked_at = 0.0
|
|
cancelled = False
|
|
|
|
def on_progress(done: int, total: int, message: str) -> None:
|
|
nonlocal pushed_at
|
|
self.jobs.progress(job.id, done, total, message)
|
|
moment = time.monotonic()
|
|
if moment - pushed_at >= self.PUSH_EVERY:
|
|
pushed_at = moment
|
|
self.notify()
|
|
|
|
def is_cancelled() -> bool:
|
|
# Reads the table rather than a flag on this object, which is what
|
|
# lets a job be killed from the UI while the worker runs somewhere
|
|
# else entirely.
|
|
nonlocal checked_at, cancelled
|
|
moment = time.monotonic()
|
|
if moment - checked_at >= self.CANCEL_CACHE:
|
|
checked_at = moment
|
|
cancelled = self.jobs.cancel_requested(job.id)
|
|
return cancelled
|
|
|
|
def is_killed() -> bool:
|
|
# The task's row, not this job's request. A batch asked to stop is
|
|
# one batch dropped; a killed task is the whole step skipped, and
|
|
# nothing in the counts tells those apart. Uncached, unlike the
|
|
# check above: the job that asks is the one reporting at the end
|
|
# of a task, and it asks once.
|
|
return bool(job.task_id) and self.jobs.cancelled(job.task_id)
|
|
|
|
def on_task(task: Any) -> bool:
|
|
# Collected rather than queued, and always taken: the worker starts
|
|
# what is in here once the job lands in `done`. See
|
|
# `RunContext.task` for why that is the promise being made.
|
|
tasks.append(task)
|
|
return True
|
|
|
|
def on_stop(why: str) -> int:
|
|
# A job with no task behind it is the whole run, and there is
|
|
# nothing queued under it to spare. Its own failure is the answer.
|
|
if not job.task_id:
|
|
return 0
|
|
ended = self.stop_task(job.task_id, why)
|
|
log.warning("job %d stopped the rest of task %d: %s (%d batch(es))",
|
|
job.id, job.task_id, why or "no reason given", ended)
|
|
return ended
|
|
|
|
def on_notice(notice: Notice) -> None:
|
|
# Kept as well as pushed. One job is one thread, so appending here
|
|
# needs no lock, and the list dies with the run.
|
|
said.append(notice)
|
|
self.announce(notice)
|
|
|
|
def on_fan_out(kind: str, payloads: list[dict[str, Any]], *,
|
|
then: str, settle: str, label: str,
|
|
carries: dict[str, Any],
|
|
max_attempts: int | None,
|
|
retry_delay: float | None,
|
|
retry_backoff: float | None) -> tuple[int, ...]:
|
|
"""`RunContext.fan_out`, from the worker thread.
|
|
|
|
Inside this job's cascade rather than its task: the children are a
|
|
step of their own, so killing them is not killing the step that
|
|
asked for them, and the callback fanning out again makes a chain
|
|
rather than a nest. A job that fans out with no cascade of its own
|
|
begins one, which is `fan_out`'s business and not this closure's.
|
|
"""
|
|
return fan_out_jobs(
|
|
self, kind, payloads, then=then, settle=settle, label=label,
|
|
carries=carries, cascade_id=job.cascade_id,
|
|
max_attempts=max_attempts, retry_delay=retry_delay,
|
|
retry_backoff=retry_backoff)
|
|
|
|
return RunContext(
|
|
on_progress=on_progress,
|
|
is_cancelled=is_cancelled,
|
|
is_killed=is_killed,
|
|
on_notice=on_notice,
|
|
on_task=on_task,
|
|
on_stop=on_stop,
|
|
on_fan_out=on_fan_out,
|
|
waited_for=tuple(self.jobs.outcomes(job.depends_on)),
|
|
)
|
|
|
|
async def _announce(self) -> None:
|
|
"""Read snapshots off the loop so slow SQLite cannot delay heartbeats."""
|
|
if self._listener is None:
|
|
return
|
|
try:
|
|
await self._listener(await asyncio.to_thread(self.jobs.snapshot))
|
|
except Exception:
|
|
log.exception("job listener raised; carrying on")
|