"""The other side of `isolate=True`: one job, one process, and then gone. python -m pipeline.child --db news.db --job 412 \ --bootstrap newsdesk.jobs:bootstrap --out C:/Temp/tmp123.json Run by the worker, never by a person, and the whole reason it can exist is that **a job row is already a complete description of the work.** A kind, a JSON payload, a database file. Nothing is pickled and nothing is sent: the child is told where to look and reads the same row the worker read. Which is also the division of labour that keeps the state machine intact. The child writes *progress* — a non-terminal update, and the snapshot the UI reads is a `SELECT`, so a child's progress reaches the browser for free. The child does **not** write the ending. It hands its result back through a file and exits, and the worker writes `done` or `failed` exactly as it does for a job that ran in a thread. One writer of the terminal state, as before. A file rather than stdout because a handler is arbitrary code and arbitrary code prints. Parsing a result out of a stream something else is also writing to is a bug waiting for the first `print` in a fetch loop. The `bootstrap` is the one thing an app has to provide, and it has two jobs: import the modules that fill `JOBS`, and return the `(config, store)` pair the handlers take. Neither survives a process boundary, so neither is sent — it is rebuilt here from the config file the app already knows how to read. """ from __future__ import annotations import argparse import importlib import json import logging import sys import traceback from pathlib import Path from typing import Any from pipeline.context import Cancelled, RunContext from pipeline.registry import JOBS from pipeline.table import Jobs log = logging.getLogger(__name__) #: What the worker reads off the exit code. Anything else is the process #: dying of something that was never ours to catch — an OOM kill, a segfault #: in a native library — and the worker reports it as the failure it is. OK = 0 FAILED = 1 CANCELLED = 2 def resolve(name: str) -> Any: """`module:attribute` to the attribute. The same spelling a callback uses.""" module, _, attr = name.partition(":") if not module or not attr: raise ValueError(f"a bootstrap is 'module:function', not {name!r}") return getattr(importlib.import_module(module), attr) def context_for(jobs: Jobs, job_id: int, task_id: int) -> RunContext: """A context that reports to the table and asks it about cancellation. Every channel the in-process context has, except the two that cannot cross a process boundary and are not wanted here anyway: `on_task` and `on_fan_out` both queue work, and an isolated job is isolated precisely because it is one long piece of work rather than an organiser of other work. Both fall back to their no-ops, which report that nothing took it. Cancellation is a read of the row, which is what makes Kill work at all from here: the button writes `cancel_requested` in the web process, and this is the only way this process would ever hear about it. The worker also kills us outright — this is the polite half, so a handler that calls `check()` stops at a line of its own choosing and gets to clean up. """ return RunContext( on_progress=lambda done, total, message: jobs.progress( job_id, done, total, message), is_cancelled=lambda: jobs.cancel_requested(job_id), is_killed=lambda: bool(task_id) and jobs.cancelled(task_id), on_notice=lambda notice: print( json.dumps({"notice": notice.model_dump()}), file=sys.stderr), ) def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="pipeline.child", description=__doc__) parser.add_argument("--db", required=True, help="the SQLite file the jobs table is on") parser.add_argument("--job", required=True, type=int, help="which row to run") parser.add_argument("--bootstrap", required=True, help="module:function returning (config, store)") parser.add_argument("--bootstrap-arg", default=None, help="one string handed to the bootstrap, e.g. a config path") parser.add_argument("--out", required=True, help="where to write the result JSON") args = parser.parse_args(argv) out = Path(args.out) jobs = Jobs(args.db) job = jobs.get(args.job) if job is None: out.write_text(json.dumps({"error": f"no job {args.job}"}), encoding="utf-8") return FAILED try: build = resolve(args.bootstrap) config, store = (build() if args.bootstrap_arg is None else build(args.bootstrap_arg)) spec = JOBS.get(job.kind) if spec is None: raise KeyError( f"{job.kind!r} is not a registered job in this process — " f"does {args.bootstrap} import the module that declares it?" ) context = context_for(jobs, job.id, job.task_id) kwargs = dict(job.payload) if spec.wants_context: kwargs["context"] = context result = spec.fn(config, store, **kwargs) except Cancelled: out.write_text(json.dumps({}), encoding="utf-8") return CANCELLED except BaseException as exc: # The traceback goes in the file as well as to stderr. stderr is the # worker's fallback and is captured either way; the file is what # survives a child that also wrote something confusing to the stream. out.write_text(json.dumps({ "error": f"{type(exc).__name__}: {exc}", "traceback": traceback.format_exc(), }), encoding="utf-8") traceback.print_exc() return FAILED out.write_text(json.dumps({"result": result or {}}, default=str), encoding="utf-8") return OK if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") raise SystemExit(main())