Files
jobboard/pipeline/child.py
T
WolfandClaude Opus 5.5 9d666d0937 Bring in pipeline, auth, deploy and docs from the boilerplate
roledesk is built on the boilerplate from here on: pipeline 2.0 (queue and
scheduler), auth (one password, sessions, passkeys), the deploy kit and its
docs, plus the coupling and packaging tests. Downstream copies, unedited.

Refreshed from boilerplate/d56949f7d0f6c0bfe5745478e09007bfad67ae87

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:16 +02:00

141 lines
5.9 KiB
Python

"""The other side of `isolate=True`: one job, one process, and then gone.
python -m pipeline.child --db news.db --job 412 \
--bootstrap newsdesk.jobs:bootstrap --out C:/Temp/tmp123.json
Run by the worker, never by a person, and the whole reason it can exist is
that **a job row is already a complete description of the work.** A kind, a
JSON payload, a database file. Nothing is pickled and nothing is sent: the
child is told where to look and reads the same row the worker read.
Which is also the division of labour that keeps the state machine intact.
The child writes *progress* — a non-terminal update, and the snapshot the UI
reads is a `SELECT`, so a child's progress reaches the browser for free. The
child does **not** write the ending. It hands its result back through a file
and exits, and the worker writes `done` or `failed` exactly as it does for a
job that ran in a thread. One writer of the terminal state, as before.
A file rather than stdout because a handler is arbitrary code and arbitrary
code prints. Parsing a result out of a stream something else is also writing
to is a bug waiting for the first `print` in a fetch loop.
The `bootstrap` is the one thing an app has to provide, and it has two jobs:
import the modules that fill `JOBS`, and return the `(config, store)` pair
the handlers take. Neither survives a process boundary, so neither is sent —
it is rebuilt here from the config file the app already knows how to read.
"""
from __future__ import annotations
import argparse
import importlib
import json
import logging
import sys
import traceback
from pathlib import Path
from typing import Any
from pipeline.context import Cancelled, RunContext
from pipeline.registry import JOBS
from pipeline.table import Jobs
log = logging.getLogger(__name__)
#: What the worker reads off the exit code. Anything else is the process
#: dying of something that was never ours to catch — an OOM kill, a segfault
#: in a native library — and the worker reports it as the failure it is.
OK = 0
FAILED = 1
CANCELLED = 2
def resolve(name: str) -> Any:
"""`module:attribute` to the attribute. The same spelling a callback uses."""
module, _, attr = name.partition(":")
if not module or not attr:
raise ValueError(f"a bootstrap is 'module:function', not {name!r}")
return getattr(importlib.import_module(module), attr)
def context_for(jobs: Jobs, job_id: int, task_id: int) -> RunContext:
"""A context that reports to the table and asks it about cancellation.
Every channel the in-process context has, except the two that cannot
cross a process boundary and are not wanted here anyway: `on_task` and
`on_fan_out` both queue work, and an isolated job is isolated precisely
because it is one long piece of work rather than an organiser of other
work. Both fall back to their no-ops, which report that nothing took it.
Cancellation is a read of the row, which is what makes Kill work at all
from here: the button writes `cancel_requested` in the web process, and
this is the only way this process would ever hear about it. The worker
also kills us outright — this is the polite half, so a handler that calls
`check()` stops at a line of its own choosing and gets to clean up.
"""
return RunContext(
on_progress=lambda done, total, message: jobs.progress(
job_id, done, total, message),
is_cancelled=lambda: jobs.cancel_requested(job_id),
is_killed=lambda: bool(task_id) and jobs.cancelled(task_id),
on_notice=lambda notice: print(
json.dumps({"notice": notice.model_dump()}), file=sys.stderr),
)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(prog="pipeline.child", description=__doc__)
parser.add_argument("--db", required=True, help="the SQLite file the jobs table is on")
parser.add_argument("--job", required=True, type=int, help="which row to run")
parser.add_argument("--bootstrap", required=True,
help="module:function returning (config, store)")
parser.add_argument("--bootstrap-arg", default=None,
help="one string handed to the bootstrap, e.g. a config path")
parser.add_argument("--out", required=True, help="where to write the result JSON")
args = parser.parse_args(argv)
out = Path(args.out)
jobs = Jobs(args.db)
job = jobs.get(args.job)
if job is None:
out.write_text(json.dumps({"error": f"no job {args.job}"}), encoding="utf-8")
return FAILED
try:
build = resolve(args.bootstrap)
config, store = (build() if args.bootstrap_arg is None
else build(args.bootstrap_arg))
spec = JOBS.get(job.kind)
if spec is None:
raise KeyError(
f"{job.kind!r} is not a registered job in this process — "
f"does {args.bootstrap} import the module that declares it?"
)
context = context_for(jobs, job.id, job.task_id)
kwargs = dict(job.payload)
if spec.wants_context:
kwargs["context"] = context
result = spec.fn(config, store, **kwargs)
except Cancelled:
out.write_text(json.dumps({}), encoding="utf-8")
return CANCELLED
except BaseException as exc:
# The traceback goes in the file as well as to stderr. stderr is the
# worker's fallback and is captured either way; the file is what
# survives a child that also wrote something confusing to the stream.
out.write_text(json.dumps({
"error": f"{type(exc).__name__}: {exc}",
"traceback": traceback.format_exc(),
}), encoding="utf-8")
traceback.print_exc()
return FAILED
out.write_text(json.dumps({"result": result or {}}, default=str), encoding="utf-8")
return OK
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO,
format="%(levelname)s %(name)s: %(message)s")
raise SystemExit(main())