roledesk is built on the boilerplate from here on: pipeline 2.0 (queue and scheduler), auth (one password, sessions, passkeys), the deploy kit and its docs, plus the coupling and packaging tests. Downstream copies, unedited. Refreshed from boilerplate/d56949f7d0f6c0bfe5745478e09007bfad67ae87 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
141 lines
5.9 KiB
Python
141 lines
5.9 KiB
Python
"""The other side of `isolate=True`: one job, one process, and then gone.
|
|
|
|
python -m pipeline.child --db news.db --job 412 \
|
|
--bootstrap newsdesk.jobs:bootstrap --out C:/Temp/tmp123.json
|
|
|
|
Run by the worker, never by a person, and the whole reason it can exist is
|
|
that **a job row is already a complete description of the work.** A kind, a
|
|
JSON payload, a database file. Nothing is pickled and nothing is sent: the
|
|
child is told where to look and reads the same row the worker read.
|
|
|
|
Which is also the division of labour that keeps the state machine intact.
|
|
The child writes *progress* — a non-terminal update, and the snapshot the UI
|
|
reads is a `SELECT`, so a child's progress reaches the browser for free. The
|
|
child does **not** write the ending. It hands its result back through a file
|
|
and exits, and the worker writes `done` or `failed` exactly as it does for a
|
|
job that ran in a thread. One writer of the terminal state, as before.
|
|
|
|
A file rather than stdout because a handler is arbitrary code and arbitrary
|
|
code prints. Parsing a result out of a stream something else is also writing
|
|
to is a bug waiting for the first `print` in a fetch loop.
|
|
|
|
The `bootstrap` is the one thing an app has to provide, and it has two jobs:
|
|
import the modules that fill `JOBS`, and return the `(config, store)` pair
|
|
the handlers take. Neither survives a process boundary, so neither is sent —
|
|
it is rebuilt here from the config file the app already knows how to read.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import importlib
|
|
import json
|
|
import logging
|
|
import sys
|
|
import traceback
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from pipeline.context import Cancelled, RunContext
|
|
from pipeline.registry import JOBS
|
|
from pipeline.table import Jobs
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: What the worker reads off the exit code. Anything else is the process
|
|
#: dying of something that was never ours to catch — an OOM kill, a segfault
|
|
#: in a native library — and the worker reports it as the failure it is.
|
|
OK = 0
|
|
FAILED = 1
|
|
CANCELLED = 2
|
|
|
|
|
|
def resolve(name: str) -> Any:
|
|
"""`module:attribute` to the attribute. The same spelling a callback uses."""
|
|
module, _, attr = name.partition(":")
|
|
if not module or not attr:
|
|
raise ValueError(f"a bootstrap is 'module:function', not {name!r}")
|
|
return getattr(importlib.import_module(module), attr)
|
|
|
|
|
|
def context_for(jobs: Jobs, job_id: int, task_id: int) -> RunContext:
|
|
"""A context that reports to the table and asks it about cancellation.
|
|
|
|
Every channel the in-process context has, except the two that cannot
|
|
cross a process boundary and are not wanted here anyway: `on_task` and
|
|
`on_fan_out` both queue work, and an isolated job is isolated precisely
|
|
because it is one long piece of work rather than an organiser of other
|
|
work. Both fall back to their no-ops, which report that nothing took it.
|
|
|
|
Cancellation is a read of the row, which is what makes Kill work at all
|
|
from here: the button writes `cancel_requested` in the web process, and
|
|
this is the only way this process would ever hear about it. The worker
|
|
also kills us outright — this is the polite half, so a handler that calls
|
|
`check()` stops at a line of its own choosing and gets to clean up.
|
|
"""
|
|
return RunContext(
|
|
on_progress=lambda done, total, message: jobs.progress(
|
|
job_id, done, total, message),
|
|
is_cancelled=lambda: jobs.cancel_requested(job_id),
|
|
is_killed=lambda: bool(task_id) and jobs.cancelled(task_id),
|
|
on_notice=lambda notice: print(
|
|
json.dumps({"notice": notice.model_dump()}), file=sys.stderr),
|
|
)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(prog="pipeline.child", description=__doc__)
|
|
parser.add_argument("--db", required=True, help="the SQLite file the jobs table is on")
|
|
parser.add_argument("--job", required=True, type=int, help="which row to run")
|
|
parser.add_argument("--bootstrap", required=True,
|
|
help="module:function returning (config, store)")
|
|
parser.add_argument("--bootstrap-arg", default=None,
|
|
help="one string handed to the bootstrap, e.g. a config path")
|
|
parser.add_argument("--out", required=True, help="where to write the result JSON")
|
|
args = parser.parse_args(argv)
|
|
|
|
out = Path(args.out)
|
|
jobs = Jobs(args.db)
|
|
job = jobs.get(args.job)
|
|
if job is None:
|
|
out.write_text(json.dumps({"error": f"no job {args.job}"}), encoding="utf-8")
|
|
return FAILED
|
|
|
|
try:
|
|
build = resolve(args.bootstrap)
|
|
config, store = (build() if args.bootstrap_arg is None
|
|
else build(args.bootstrap_arg))
|
|
spec = JOBS.get(job.kind)
|
|
if spec is None:
|
|
raise KeyError(
|
|
f"{job.kind!r} is not a registered job in this process — "
|
|
f"does {args.bootstrap} import the module that declares it?"
|
|
)
|
|
context = context_for(jobs, job.id, job.task_id)
|
|
kwargs = dict(job.payload)
|
|
if spec.wants_context:
|
|
kwargs["context"] = context
|
|
result = spec.fn(config, store, **kwargs)
|
|
except Cancelled:
|
|
out.write_text(json.dumps({}), encoding="utf-8")
|
|
return CANCELLED
|
|
except BaseException as exc:
|
|
# The traceback goes in the file as well as to stderr. stderr is the
|
|
# worker's fallback and is captured either way; the file is what
|
|
# survives a child that also wrote something confusing to the stream.
|
|
out.write_text(json.dumps({
|
|
"error": f"{type(exc).__name__}: {exc}",
|
|
"traceback": traceback.format_exc(),
|
|
}), encoding="utf-8")
|
|
traceback.print_exc()
|
|
return FAILED
|
|
|
|
out.write_text(json.dumps({"result": result or {}}, default=str), encoding="utf-8")
|
|
return OK
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logging.basicConfig(level=logging.INFO,
|
|
format="%(levelname)s %(name)s: %(message)s")
|
|
raise SystemExit(main())
|