"""What keeps a queue healthy for months: retention, bounded results, whose retry policy wins, draining, and two real processes on one file. python -m pytest pipeline/tests/test_housekeeping.py -q """ from __future__ import annotations import os import sqlite3 import subprocess import sys import tempfile from contextlib import closing from datetime import UTC, datetime, timedelta from pathlib import Path import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from pipeline import NULL, Batch, Bound, JobQueue, Jobs, Task, job from pipeline.models import Job ROOT = Path(__file__).resolve().parents[2] @job("h_plain", "A job with a policy of its own", max_attempts=3, retry_delay=7) def h_plain(config, store, *, context=NULL, i: int = 0) -> dict: return {"i": i} @job("h_big", "Returning far too much", max_attempts=3) def h_big(config, store, *, context=NULL) -> dict: return {"rows": ["x" * 100] * 2000, "count": 2000} @pytest.fixture def jobs(): with tempfile.TemporaryDirectory() as tmp: yield Jobs(Path(tmp) / "h.db") def got(found: Job | None) -> Job: assert found is not None return found def age(jobs: Jobs, job_id: int, days: float) -> None: """Pretend a job finished `days` ago, and its runs were minted then too.""" when = (datetime.now(UTC) - timedelta(days=days)).isoformat() with closing(sqlite3.connect(jobs.path)) as conn, conn: conn.execute("UPDATE jobs SET finished_at=? WHERE id=?", (when, job_id)) conn.execute("UPDATE runs SET created_at=? WHERE id IN " "(SELECT cascade_id FROM jobs WHERE id=? UNION " " SELECT task_id FROM jobs WHERE id=?)", (when, job_id, job_id)) def runs(jobs: Jobs) -> int: with closing(sqlite3.connect(jobs.path)) as conn: return conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0] # ---------------------------------------------------------------- retention def test_purge_forgets_old_history_but_not_the_last_of_each_kind(jobs): """Without this both tables grow for as long as the app runs. What survives an old cutoff: anything unfinished, and the newest ending of each kind in each state — because `last(kind)` is how an app says "last pulled five weeks ago", and deleting it would make that "never". """ old = [got(jobs.enqueue("h_plain")) for _ in range(3)] for queued in old: jobs.finish(queued.id, "done") age(jobs, queued.id, days=40) waiting = got(jobs.enqueue("h_plain")) # never finished gone = jobs.purge(30 * 86400) assert gone == 2, "two of the three old ones went" assert got(jobs.last("h_plain")).id == old[-1].id, "the newest stayed" assert got(jobs.get(waiting.id)).state == "queued", "unfinished work is not history" def test_purge_leaves_recent_history_alone(jobs): recent = got(jobs.enqueue("h_plain")) jobs.finish(recent.id, "done") before = runs(jobs) assert jobs.purge(30 * 86400) == 0 assert jobs.get(recent.id) is not None assert runs(jobs) == before def test_purge_drops_runs_nothing_points_at_any_more(jobs): first = got(jobs.enqueue("h_plain")) second = got(jobs.enqueue("h_plain")) for queued in (first, second): jobs.finish(queued.id, "done") age(jobs, queued.id, days=40) before = runs(jobs) jobs.purge(30 * 86400) assert runs(jobs) < before, "the purged job's cascade and task went with it" assert jobs.run(second.task_id) is not None, "the kept job's runs did not" # ----------------------------------------------------------- bounded results def test_a_result_too_large_fails_the_job_loudly_and_once(jobs): """Past `MAX_RESULT` the job fails rather than losing data quietly: the row says why and how to fix it, a notice goes to whoever is watching, and it is not retried — the same handler would return the same result.""" said: list = [] async def on_notice(notice) -> None: said.append(notice) queued = got(jobs.enqueue("h_big", max_attempts=3)) JobQueue(jobs, None, None, on_notice=on_notice).drain(timeout=10) ended = got(jobs.get(queued.id)) assert ended.state == "failed" assert "ResultTooLarge" in ended.error and "store" in ended.error, ended.error assert "count, rows" in ended.error, "it names what was returned" assert ended.result == {}, "nothing of it was written" assert ended.attempt == 1, "and it was not tried again" assert [n.code for n in said] == ["job.failed"], "and somebody was told" def test_a_result_at_the_limit_is_kept_whole(jobs): """The limit is a line, not a tax: everything under it is stored as is.""" queued = got(jobs.enqueue("h_plain", payload={"i": 1})) JobQueue(jobs, None, None).drain(timeout=10) assert got(jobs.get(queued.id)).result == {"i": 1} # -------------------------------------------------------- whose policy wins def test_a_task_with_plain_payloads_keeps_the_kinds_own_retry_policy(jobs): """`@job(max_attempts=3)` has to mean something when a task queues it. It used not to: a plain payload was given the default bound explicitly, which silently overrode whatever the kind declared. """ queue = JobQueue(jobs, None, None) Task(kind="h_plain", finder=lambda c, s: [{"i": 1}]).start(None, None, queue) (child,) = [j for j in jobs.snapshot().queued if j.kind == "h_plain"] assert (child.max_attempts, child.retry_delay) == (3, 7.0) def test_a_batch_with_its_own_bound_overrides_the_kind(jobs): """The other half: a batch cut from one plugin profile carries that profile's policy, and that wins, because the outlet knows its publisher.""" queue = JobQueue(jobs, None, None) Task(kind="h_plain", finder=lambda c, s: [ Batch({"i": 1}, Bound(max_attempts=5, delay=60))]).start(None, None, queue) (child,) = [j for j in jobs.snapshot().queued if j.kind == "h_plain"] assert (child.max_attempts, child.retry_delay) == (5, 60.0) # ------------------------------------------------------------------ draining def test_drain_runs_everything_queued_and_stops(jobs): """The batch shape: a script or a cron entry with no web app around it.""" ids = [got(jobs.enqueue("h_plain", payload={"i": i})).id for i in range(5)] assert JobQueue(jobs, None, None).drain(timeout=10) is True assert {got(jobs.get(i)).state for i in ids} == {"done"} # ------------------------------------------------- two processes, one file def test_two_worker_processes_never_run_the_same_job_twice(tmp_path): """The claim the table-as-queue design rests on, tested the only way that means anything: two separate `python -m pipeline.worker` processes draining one file at once. Every job leaves a marker file named after itself and the process that ran it; each job must have exactly one. """ db = tmp_path / "shared.db" marks = tmp_path / "marks" marks.mkdir() jobs = Jobs(db) for i in range(40): jobs.enqueue("mp_mark", payload={"i": i}) env = dict(os.environ, PYTHONPATH=str(ROOT)) argv = [sys.executable, "-m", "pipeline.worker", "--db", str(db), "--bootstrap", "pipeline.tests.isolated_handlers:marking", "--bootstrap-arg", str(marks), "--workers", "cpu=2", "--until-idle"] workers = [subprocess.Popen(argv, env=env, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for _ in range(2)] for worker in workers: out, _ = worker.communicate(timeout=120) assert worker.returncode == 0, out.decode(errors="replace")[-2000:] ran: dict[int, list[str]] = {} for mark in marks.iterdir(): i, pid = mark.name.split("-")[:2] ran.setdefault(int(i), []).append(pid) assert sorted(ran) == list(range(40)), "every job ran" doubled = {i: pids for i, pids in ran.items() if len(pids) > 1} assert not doubled, f"and none ran twice: {doubled}" assert {j.state for j in jobs.snapshot().recent} == {"done"}