Files
jobboard/pipeline/worker.py
T
WolfandClaude Opus 5.5 9d666d0937 Bring in pipeline, auth, deploy and docs from the boilerplate
roledesk is built on the boilerplate from here on: pipeline 2.0 (queue and
scheduler), auth (one password, sessions, passkeys), the deploy kit and its
docs, plus the coupling and packaging tests. Downstream copies, unedited.

Refreshed from boilerplate/d56949f7d0f6c0bfe5745478e09007bfad67ae87

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:16 +02:00

114 lines
4.9 KiB
Python

"""A worker with no web app around it.
python -m pipeline.worker --db app.db --bootstrap myapp.jobs:bootstrap
python -m pipeline.worker --db app.db --bootstrap myapp.jobs:bootstrap \\
--bootstrap-arg config.yaml --workers cpu=2,http=8 --until-idle
What a second machine, a cron entry or a batch script runs. It is the same
`JobQueue` the web app starts, on the same table, so it races correctly against
any other worker on that file: `Jobs.claim` is one atomic statement and every
claimed job is held on a lease. Start as many as the file will bear.
The bootstrap is the one `isolate=True` already asks for — `module:function`
returning `(config, store)` — and it is used for the same two reasons here:
importing it fills `JOBS`, and calling it builds the pair the handlers take.
An app that has written one for isolation has written this worker's too.
`--until-idle` drains what is queued and exits, which is the batch shape. Without
it the worker runs until it is interrupted, and a SIGINT or SIGTERM stops it the
way the web app stops: handlers in flight get `JobQueue.STOP_GRACE` to land,
and what is still running after that goes back to the queue on its lease.
**This worker fires schedules too**, which is what makes it a complete
replacement for the web app's own worker — a machine running only this still
gets its 06:00 job. `--no-schedules` turns that off, and is what every worker
after the first passes: they all share the jobs table safely, but the pile-up
guard reads `last_job_id`, which neither of two simultaneous firings has
written yet when the other looks. One scheduler per file, as many workers as
you like. `--timezone` is what the clock cadences are read in; without it they
are read in the machine's own zone, which on a server is UTC and is usually not
what the person who wrote `daily at 06:00` meant.
"""
from __future__ import annotations
import argparse
import asyncio
import contextlib
import logging
import signal
from pipeline.child import resolve
from pipeline.queue import JobQueue
from pipeline.table import Jobs
log = logging.getLogger(__name__)
def workers(text: str) -> dict[str, int]:
"""`cpu=2,http=8` to `{"cpu": 2, "http": 8}`."""
counts: dict[str, int] = {}
for part in filter(None, (p.strip() for p in text.split(","))):
name, _, n = part.partition("=")
if not name or not n.isdigit():
raise argparse.ArgumentTypeError(f"not resource=count: {part!r}")
counts[name] = int(n)
return counts
async def serve(queue: JobQueue) -> None:
"""Run until SIGINT or SIGTERM."""
stop = asyncio.Event()
loop = asyncio.get_running_loop()
for sig in (signal.SIGINT, signal.SIGTERM):
# Windows has no loop signal handlers; there Ctrl+C arrives as
# KeyboardInterrupt instead, and `main` turns it into the same stop.
with contextlib.suppress(NotImplementedError, RuntimeError):
loop.add_signal_handler(sig, stop.set)
await queue.start()
try:
await stop.wait()
finally:
await queue.stop()
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(prog="pipeline.worker", description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--db", required=True, help="the SQLite file the jobs table is on")
parser.add_argument("--bootstrap", required=True,
help="module:function returning (config, store)")
parser.add_argument("--bootstrap-arg", default="",
help="one string handed to the bootstrap, e.g. a config path")
parser.add_argument("--workers", type=workers, default=None,
help="resource=count,... (default: JobQueue.WORKERS)")
parser.add_argument("--until-idle", action="store_true",
help="drain what is queued, then exit")
parser.add_argument("--no-schedules", action="store_true",
help="do not fire schedules here; for a second worker "
"on a file that already has a scheduler")
parser.add_argument("--timezone", default="",
help="what 'daily at 06:00' is 06:00 in "
"(default: this machine's own zone)")
args = parser.parse_args(argv)
build = resolve(args.bootstrap)
config, store = build(args.bootstrap_arg) if args.bootstrap_arg else build()
queue = JobQueue(Jobs(args.db), config, store, workers=args.workers,
bootstrap=args.bootstrap, bootstrap_arg=args.bootstrap_arg,
tz=args.timezone, schedules=not args.no_schedules)
try:
if args.until_idle:
queue.drain()
else:
asyncio.run(serve(queue))
except KeyboardInterrupt:
pass
return 0
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s")
raise SystemExit(main())