"""A worker with no web app around it. python -m pipeline.worker --db app.db --bootstrap myapp.jobs:bootstrap python -m pipeline.worker --db app.db --bootstrap myapp.jobs:bootstrap \\ --bootstrap-arg config.yaml --workers cpu=2,http=8 --until-idle What a second machine, a cron entry or a batch script runs. It is the same `JobQueue` the web app starts, on the same table, so it races correctly against any other worker on that file: `Jobs.claim` is one atomic statement and every claimed job is held on a lease. Start as many as the file will bear. The bootstrap is the one `isolate=True` already asks for — `module:function` returning `(config, store)` — and it is used for the same two reasons here: importing it fills `JOBS`, and calling it builds the pair the handlers take. An app that has written one for isolation has written this worker's too. `--until-idle` drains what is queued and exits, which is the batch shape. Without it the worker runs until it is interrupted, and a SIGINT or SIGTERM stops it the way the web app stops: handlers in flight get `JobQueue.STOP_GRACE` to land, and what is still running after that goes back to the queue on its lease. **This worker fires schedules too**, which is what makes it a complete replacement for the web app's own worker — a machine running only this still gets its 06:00 job. `--no-schedules` turns that off, and is what every worker after the first passes: they all share the jobs table safely, but the pile-up guard reads `last_job_id`, which neither of two simultaneous firings has written yet when the other looks. One scheduler per file, as many workers as you like. `--timezone` is what the clock cadences are read in; without it they are read in the machine's own zone, which on a server is UTC and is usually not what the person who wrote `daily at 06:00` meant. """ from __future__ import annotations import argparse import asyncio import contextlib import logging import signal from pipeline.child import resolve from pipeline.queue import JobQueue from pipeline.table import Jobs log = logging.getLogger(__name__) def workers(text: str) -> dict[str, int]: """`cpu=2,http=8` to `{"cpu": 2, "http": 8}`.""" counts: dict[str, int] = {} for part in filter(None, (p.strip() for p in text.split(","))): name, _, n = part.partition("=") if not name or not n.isdigit(): raise argparse.ArgumentTypeError(f"not resource=count: {part!r}") counts[name] = int(n) return counts async def serve(queue: JobQueue) -> None: """Run until SIGINT or SIGTERM.""" stop = asyncio.Event() loop = asyncio.get_running_loop() for sig in (signal.SIGINT, signal.SIGTERM): # Windows has no loop signal handlers; there Ctrl+C arrives as # KeyboardInterrupt instead, and `main` turns it into the same stop. with contextlib.suppress(NotImplementedError, RuntimeError): loop.add_signal_handler(sig, stop.set) await queue.start() try: await stop.wait() finally: await queue.stop() def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="pipeline.worker", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--db", required=True, help="the SQLite file the jobs table is on") parser.add_argument("--bootstrap", required=True, help="module:function returning (config, store)") parser.add_argument("--bootstrap-arg", default="", help="one string handed to the bootstrap, e.g. a config path") parser.add_argument("--workers", type=workers, default=None, help="resource=count,... (default: JobQueue.WORKERS)") parser.add_argument("--until-idle", action="store_true", help="drain what is queued, then exit") parser.add_argument("--no-schedules", action="store_true", help="do not fire schedules here; for a second worker " "on a file that already has a scheduler") parser.add_argument("--timezone", default="", help="what 'daily at 06:00' is 06:00 in " "(default: this machine's own zone)") args = parser.parse_args(argv) build = resolve(args.bootstrap) config, store = build(args.bootstrap_arg) if args.bootstrap_arg else build() queue = JobQueue(Jobs(args.db), config, store, workers=args.workers, bootstrap=args.bootstrap, bootstrap_arg=args.bootstrap_arg, tz=args.timezone, schedules=not args.no_schedules) try: if args.until_idle: queue.drain() else: asyncio.run(serve(queue)) except KeyboardInterrupt: pass return 0 if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") raise SystemExit(main())