Files
jobboard/pipeline/tests/test_housekeeping.py
WolfandClaude Opus 5.5 9d666d0937 Bring in pipeline, auth, deploy and docs from the boilerplate
roledesk is built on the boilerplate from here on: pipeline 2.0 (queue and
scheduler), auth (one password, sessions, passkeys), the deploy kit and its
docs, plus the coupling and packaging tests. Downstream copies, unedited.

Refreshed from boilerplate/d56949f7d0f6c0bfe5745478e09007bfad67ae87

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:16 +02:00

210 lines
7.9 KiB
Python

"""What keeps a queue healthy for months: retention, bounded results, whose
retry policy wins, draining, and two real processes on one file.
python -m pytest pipeline/tests/test_housekeeping.py -q
"""
from __future__ import annotations
import os
import sqlite3
import subprocess
import sys
import tempfile
from contextlib import closing
from datetime import UTC, datetime, timedelta
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from pipeline import NULL, Batch, Bound, JobQueue, Jobs, Task, job
from pipeline.models import Job
ROOT = Path(__file__).resolve().parents[2]
@job("h_plain", "A job with a policy of its own", max_attempts=3, retry_delay=7)
def h_plain(config, store, *, context=NULL, i: int = 0) -> dict:
return {"i": i}
@job("h_big", "Returning far too much", max_attempts=3)
def h_big(config, store, *, context=NULL) -> dict:
return {"rows": ["x" * 100] * 2000, "count": 2000}
@pytest.fixture
def jobs():
with tempfile.TemporaryDirectory() as tmp:
yield Jobs(Path(tmp) / "h.db")
def got(found: Job | None) -> Job:
assert found is not None
return found
def age(jobs: Jobs, job_id: int, days: float) -> None:
"""Pretend a job finished `days` ago, and its runs were minted then too."""
when = (datetime.now(UTC) - timedelta(days=days)).isoformat()
with closing(sqlite3.connect(jobs.path)) as conn, conn:
conn.execute("UPDATE jobs SET finished_at=? WHERE id=?", (when, job_id))
conn.execute("UPDATE runs SET created_at=? WHERE id IN "
"(SELECT cascade_id FROM jobs WHERE id=? UNION "
" SELECT task_id FROM jobs WHERE id=?)", (when, job_id, job_id))
def runs(jobs: Jobs) -> int:
with closing(sqlite3.connect(jobs.path)) as conn:
return conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0]
# ---------------------------------------------------------------- retention
def test_purge_forgets_old_history_but_not_the_last_of_each_kind(jobs):
"""Without this both tables grow for as long as the app runs.
What survives an old cutoff: anything unfinished, and the newest ending of
each kind in each state — because `last(kind)` is how an app says "last
pulled five weeks ago", and deleting it would make that "never".
"""
old = [got(jobs.enqueue("h_plain")) for _ in range(3)]
for queued in old:
jobs.finish(queued.id, "done")
age(jobs, queued.id, days=40)
waiting = got(jobs.enqueue("h_plain")) # never finished
gone = jobs.purge(30 * 86400)
assert gone == 2, "two of the three old ones went"
assert got(jobs.last("h_plain")).id == old[-1].id, "the newest stayed"
assert got(jobs.get(waiting.id)).state == "queued", "unfinished work is not history"
def test_purge_leaves_recent_history_alone(jobs):
recent = got(jobs.enqueue("h_plain"))
jobs.finish(recent.id, "done")
before = runs(jobs)
assert jobs.purge(30 * 86400) == 0
assert jobs.get(recent.id) is not None
assert runs(jobs) == before
def test_purge_drops_runs_nothing_points_at_any_more(jobs):
first = got(jobs.enqueue("h_plain"))
second = got(jobs.enqueue("h_plain"))
for queued in (first, second):
jobs.finish(queued.id, "done")
age(jobs, queued.id, days=40)
before = runs(jobs)
jobs.purge(30 * 86400)
assert runs(jobs) < before, "the purged job's cascade and task went with it"
assert jobs.run(second.task_id) is not None, "the kept job's runs did not"
# ----------------------------------------------------------- bounded results
def test_a_result_too_large_fails_the_job_loudly_and_once(jobs):
"""Past `MAX_RESULT` the job fails rather than losing data quietly: the row
says why and how to fix it, a notice goes to whoever is watching, and it is
not retried — the same handler would return the same result."""
said: list = []
async def on_notice(notice) -> None:
said.append(notice)
queued = got(jobs.enqueue("h_big", max_attempts=3))
JobQueue(jobs, None, None, on_notice=on_notice).drain(timeout=10)
ended = got(jobs.get(queued.id))
assert ended.state == "failed"
assert "ResultTooLarge" in ended.error and "store" in ended.error, ended.error
assert "count, rows" in ended.error, "it names what was returned"
assert ended.result == {}, "nothing of it was written"
assert ended.attempt == 1, "and it was not tried again"
assert [n.code for n in said] == ["job.failed"], "and somebody was told"
def test_a_result_at_the_limit_is_kept_whole(jobs):
"""The limit is a line, not a tax: everything under it is stored as is."""
queued = got(jobs.enqueue("h_plain", payload={"i": 1}))
JobQueue(jobs, None, None).drain(timeout=10)
assert got(jobs.get(queued.id)).result == {"i": 1}
# -------------------------------------------------------- whose policy wins
def test_a_task_with_plain_payloads_keeps_the_kinds_own_retry_policy(jobs):
"""`@job(max_attempts=3)` has to mean something when a task queues it.
It used not to: a plain payload was given the default bound explicitly,
which silently overrode whatever the kind declared.
"""
queue = JobQueue(jobs, None, None)
Task(kind="h_plain", finder=lambda c, s: [{"i": 1}]).start(None, None, queue)
(child,) = [j for j in jobs.snapshot().queued if j.kind == "h_plain"]
assert (child.max_attempts, child.retry_delay) == (3, 7.0)
def test_a_batch_with_its_own_bound_overrides_the_kind(jobs):
"""The other half: a batch cut from one plugin profile carries that
profile's policy, and that wins, because the outlet knows its publisher."""
queue = JobQueue(jobs, None, None)
Task(kind="h_plain", finder=lambda c, s: [
Batch({"i": 1}, Bound(max_attempts=5, delay=60))]).start(None, None, queue)
(child,) = [j for j in jobs.snapshot().queued if j.kind == "h_plain"]
assert (child.max_attempts, child.retry_delay) == (5, 60.0)
# ------------------------------------------------------------------ draining
def test_drain_runs_everything_queued_and_stops(jobs):
"""The batch shape: a script or a cron entry with no web app around it."""
ids = [got(jobs.enqueue("h_plain", payload={"i": i})).id for i in range(5)]
assert JobQueue(jobs, None, None).drain(timeout=10) is True
assert {got(jobs.get(i)).state for i in ids} == {"done"}
# ------------------------------------------------- two processes, one file
def test_two_worker_processes_never_run_the_same_job_twice(tmp_path):
"""The claim the table-as-queue design rests on, tested the only way that
means anything: two separate `python -m pipeline.worker` processes draining
one file at once. Every job leaves a marker file named after itself and the
process that ran it; each job must have exactly one.
"""
db = tmp_path / "shared.db"
marks = tmp_path / "marks"
marks.mkdir()
jobs = Jobs(db)
for i in range(40):
jobs.enqueue("mp_mark", payload={"i": i})
env = dict(os.environ, PYTHONPATH=str(ROOT))
argv = [sys.executable, "-m", "pipeline.worker", "--db", str(db),
"--bootstrap", "pipeline.tests.isolated_handlers:marking",
"--bootstrap-arg", str(marks), "--workers", "cpu=2", "--until-idle"]
workers = [subprocess.Popen(argv, env=env, cwd=ROOT,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
for _ in range(2)]
for worker in workers:
out, _ = worker.communicate(timeout=120)
assert worker.returncode == 0, out.decode(errors="replace")[-2000:]
ran: dict[int, list[str]] = {}
for mark in marks.iterdir():
i, pid = mark.name.split("-")[:2]
ran.setdefault(int(i), []).append(pid)
assert sorted(ran) == list(range(40)), "every job ran"
doubled = {i: pids for i, pids in ran.items() if len(pids) > 1}
assert not doubled, f"and none ran twice: {doubled}"
assert {j.state for j in jobs.snapshot().recent} == {"done"}