roledesk is built on the boilerplate from here on: pipeline 2.0 (queue and scheduler), auth (one password, sessions, passkeys), the deploy kit and its docs, plus the coupling and packaging tests. Downstream copies, unedited. Refreshed from boilerplate/d56949f7d0f6c0bfe5745478e09007bfad67ae87 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
210 lines
7.9 KiB
Python
210 lines
7.9 KiB
Python
"""What keeps a queue healthy for months: retention, bounded results, whose
|
|
retry policy wins, draining, and two real processes on one file.
|
|
|
|
python -m pytest pipeline/tests/test_housekeeping.py -q
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from contextlib import closing
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
|
|
from pipeline import NULL, Batch, Bound, JobQueue, Jobs, Task, job
|
|
from pipeline.models import Job
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
@job("h_plain", "A job with a policy of its own", max_attempts=3, retry_delay=7)
|
|
def h_plain(config, store, *, context=NULL, i: int = 0) -> dict:
|
|
return {"i": i}
|
|
|
|
|
|
@job("h_big", "Returning far too much", max_attempts=3)
|
|
def h_big(config, store, *, context=NULL) -> dict:
|
|
return {"rows": ["x" * 100] * 2000, "count": 2000}
|
|
|
|
|
|
@pytest.fixture
|
|
def jobs():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
yield Jobs(Path(tmp) / "h.db")
|
|
|
|
|
|
def got(found: Job | None) -> Job:
|
|
assert found is not None
|
|
return found
|
|
|
|
|
|
def age(jobs: Jobs, job_id: int, days: float) -> None:
|
|
"""Pretend a job finished `days` ago, and its runs were minted then too."""
|
|
when = (datetime.now(UTC) - timedelta(days=days)).isoformat()
|
|
with closing(sqlite3.connect(jobs.path)) as conn, conn:
|
|
conn.execute("UPDATE jobs SET finished_at=? WHERE id=?", (when, job_id))
|
|
conn.execute("UPDATE runs SET created_at=? WHERE id IN "
|
|
"(SELECT cascade_id FROM jobs WHERE id=? UNION "
|
|
" SELECT task_id FROM jobs WHERE id=?)", (when, job_id, job_id))
|
|
|
|
|
|
def runs(jobs: Jobs) -> int:
|
|
with closing(sqlite3.connect(jobs.path)) as conn:
|
|
return conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0]
|
|
|
|
|
|
# ---------------------------------------------------------------- retention
|
|
|
|
def test_purge_forgets_old_history_but_not_the_last_of_each_kind(jobs):
|
|
"""Without this both tables grow for as long as the app runs.
|
|
|
|
What survives an old cutoff: anything unfinished, and the newest ending of
|
|
each kind in each state — because `last(kind)` is how an app says "last
|
|
pulled five weeks ago", and deleting it would make that "never".
|
|
"""
|
|
old = [got(jobs.enqueue("h_plain")) for _ in range(3)]
|
|
for queued in old:
|
|
jobs.finish(queued.id, "done")
|
|
age(jobs, queued.id, days=40)
|
|
waiting = got(jobs.enqueue("h_plain")) # never finished
|
|
|
|
gone = jobs.purge(30 * 86400)
|
|
|
|
assert gone == 2, "two of the three old ones went"
|
|
assert got(jobs.last("h_plain")).id == old[-1].id, "the newest stayed"
|
|
assert got(jobs.get(waiting.id)).state == "queued", "unfinished work is not history"
|
|
|
|
|
|
def test_purge_leaves_recent_history_alone(jobs):
|
|
recent = got(jobs.enqueue("h_plain"))
|
|
jobs.finish(recent.id, "done")
|
|
before = runs(jobs)
|
|
|
|
assert jobs.purge(30 * 86400) == 0
|
|
assert jobs.get(recent.id) is not None
|
|
assert runs(jobs) == before
|
|
|
|
|
|
def test_purge_drops_runs_nothing_points_at_any_more(jobs):
|
|
first = got(jobs.enqueue("h_plain"))
|
|
second = got(jobs.enqueue("h_plain"))
|
|
for queued in (first, second):
|
|
jobs.finish(queued.id, "done")
|
|
age(jobs, queued.id, days=40)
|
|
before = runs(jobs)
|
|
|
|
jobs.purge(30 * 86400)
|
|
|
|
assert runs(jobs) < before, "the purged job's cascade and task went with it"
|
|
assert jobs.run(second.task_id) is not None, "the kept job's runs did not"
|
|
|
|
|
|
# ----------------------------------------------------------- bounded results
|
|
|
|
def test_a_result_too_large_fails_the_job_loudly_and_once(jobs):
|
|
"""Past `MAX_RESULT` the job fails rather than losing data quietly: the row
|
|
says why and how to fix it, a notice goes to whoever is watching, and it is
|
|
not retried — the same handler would return the same result."""
|
|
said: list = []
|
|
|
|
async def on_notice(notice) -> None:
|
|
said.append(notice)
|
|
|
|
queued = got(jobs.enqueue("h_big", max_attempts=3))
|
|
JobQueue(jobs, None, None, on_notice=on_notice).drain(timeout=10)
|
|
|
|
ended = got(jobs.get(queued.id))
|
|
assert ended.state == "failed"
|
|
assert "ResultTooLarge" in ended.error and "store" in ended.error, ended.error
|
|
assert "count, rows" in ended.error, "it names what was returned"
|
|
assert ended.result == {}, "nothing of it was written"
|
|
assert ended.attempt == 1, "and it was not tried again"
|
|
assert [n.code for n in said] == ["job.failed"], "and somebody was told"
|
|
|
|
|
|
def test_a_result_at_the_limit_is_kept_whole(jobs):
|
|
"""The limit is a line, not a tax: everything under it is stored as is."""
|
|
queued = got(jobs.enqueue("h_plain", payload={"i": 1}))
|
|
JobQueue(jobs, None, None).drain(timeout=10)
|
|
assert got(jobs.get(queued.id)).result == {"i": 1}
|
|
|
|
|
|
# -------------------------------------------------------- whose policy wins
|
|
|
|
def test_a_task_with_plain_payloads_keeps_the_kinds_own_retry_policy(jobs):
|
|
"""`@job(max_attempts=3)` has to mean something when a task queues it.
|
|
|
|
It used not to: a plain payload was given the default bound explicitly,
|
|
which silently overrode whatever the kind declared.
|
|
"""
|
|
queue = JobQueue(jobs, None, None)
|
|
Task(kind="h_plain", finder=lambda c, s: [{"i": 1}]).start(None, None, queue)
|
|
|
|
(child,) = [j for j in jobs.snapshot().queued if j.kind == "h_plain"]
|
|
assert (child.max_attempts, child.retry_delay) == (3, 7.0)
|
|
|
|
|
|
def test_a_batch_with_its_own_bound_overrides_the_kind(jobs):
|
|
"""The other half: a batch cut from one plugin profile carries that
|
|
profile's policy, and that wins, because the outlet knows its publisher."""
|
|
queue = JobQueue(jobs, None, None)
|
|
Task(kind="h_plain", finder=lambda c, s: [
|
|
Batch({"i": 1}, Bound(max_attempts=5, delay=60))]).start(None, None, queue)
|
|
|
|
(child,) = [j for j in jobs.snapshot().queued if j.kind == "h_plain"]
|
|
assert (child.max_attempts, child.retry_delay) == (5, 60.0)
|
|
|
|
|
|
# ------------------------------------------------------------------ draining
|
|
|
|
def test_drain_runs_everything_queued_and_stops(jobs):
|
|
"""The batch shape: a script or a cron entry with no web app around it."""
|
|
ids = [got(jobs.enqueue("h_plain", payload={"i": i})).id for i in range(5)]
|
|
|
|
assert JobQueue(jobs, None, None).drain(timeout=10) is True
|
|
assert {got(jobs.get(i)).state for i in ids} == {"done"}
|
|
|
|
|
|
# ------------------------------------------------- two processes, one file
|
|
|
|
def test_two_worker_processes_never_run_the_same_job_twice(tmp_path):
|
|
"""The claim the table-as-queue design rests on, tested the only way that
|
|
means anything: two separate `python -m pipeline.worker` processes draining
|
|
one file at once. Every job leaves a marker file named after itself and the
|
|
process that ran it; each job must have exactly one.
|
|
"""
|
|
db = tmp_path / "shared.db"
|
|
marks = tmp_path / "marks"
|
|
marks.mkdir()
|
|
jobs = Jobs(db)
|
|
for i in range(40):
|
|
jobs.enqueue("mp_mark", payload={"i": i})
|
|
|
|
env = dict(os.environ, PYTHONPATH=str(ROOT))
|
|
argv = [sys.executable, "-m", "pipeline.worker", "--db", str(db),
|
|
"--bootstrap", "pipeline.tests.isolated_handlers:marking",
|
|
"--bootstrap-arg", str(marks), "--workers", "cpu=2", "--until-idle"]
|
|
workers = [subprocess.Popen(argv, env=env, cwd=ROOT,
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
for _ in range(2)]
|
|
for worker in workers:
|
|
out, _ = worker.communicate(timeout=120)
|
|
assert worker.returncode == 0, out.decode(errors="replace")[-2000:]
|
|
|
|
ran: dict[int, list[str]] = {}
|
|
for mark in marks.iterdir():
|
|
i, pid = mark.name.split("-")[:2]
|
|
ran.setdefault(int(i), []).append(pid)
|
|
assert sorted(ran) == list(range(40)), "every job ran"
|
|
doubled = {i: pids for i, pids in ran.items() if len(pids) > 1}
|
|
assert not doubled, f"and none ran twice: {doubled}"
|
|
assert {j.state for j in jobs.snapshot().recent} == {"done"}
|