"""Waiting to be tried again, without anything being alive while it waits. The claim under test is that a job between attempts costs a row and nothing else. No worker, no thread, no lease, no timer — so the tests that matter here are the ones that ask what *else* the queue can do while a job waits, and what survives the process that scheduled the wait. Each of the four properties step 12 asks for is pinned on its own, because they fail independently: a bound that is off by one still waits correctly, a `run_after` that never reaches the table still retries in one process, and a Kill that cannot reach a waiting row is invisible until somebody presses it. python3 -m pytest tests/test_retries.py -q """ from __future__ import annotations import asyncio import sqlite3 import sys import tempfile import time from contextlib import closing from datetime import UTC, datetime, timedelta from pathlib import Path import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from pipeline.models import Job from pipeline import NULL, JobQueue, Jobs, Retry, job @pytest.fixture def jobs(): """The pipeline's table on a file of its own. No app, no store — nothing here needs one, which is the point of the table owning its own schema.""" with tempfile.TemporaryDirectory() as tmp: yield Jobs(Path(tmp) / "t.db") def made(queued: Job | None) -> Job: """The job an enqueue made. `enqueue` answers None for a killed run, which in these tests nothing is.""" assert queued is not None return queued def got(found: Job | None) -> Job: """The job a read must have found — a None here is a test failure.""" assert found is not None return found def lease(jobs: Jobs, job_id: int) -> tuple[str | None, str | None]: """The `started_at` and `seen_at` of one job: whether anybody has it.""" with closing(sqlite3.connect(jobs.path)) as conn: row = conn.execute( "SELECT started_at, seen_at FROM jobs WHERE id=?", (job_id,)).fetchone() return row[0], row[1] def bring_forward(jobs: Jobs, job_id: int) -> None: """Move a waiting job's next attempt into the past, as if its wait were over. What the clock does, done in one statement so a test of an hour's backoff does not take an hour.""" with closing(sqlite3.connect(jobs.path)) as conn, conn: conn.execute("UPDATE jobs SET run_after=? WHERE id=?", ((datetime.now(UTC) - timedelta(hours=1)).isoformat(), job_id)) def drain(queue: JobQueue, *, timeout: float = 5.0) -> None: """Run the worker until the table is empty and this worker holds nothing. Both halves, because a job's last acts belong to its handler thread and between them the table reads as idle while it is not. A job waiting for its next attempt keeps this going, since it is still `queued` — which is the behaviour the retry tests are leaning on. """ async def go() -> None: await queue.start() deadline = time.monotonic() + timeout while time.monotonic() < deadline: snap = queue.jobs.snapshot() if not snap.running and not snap.queued and not queue._running: break await asyncio.sleep(0.01) await queue.stop() asyncio.run(go()) # ------------------------------------------------------- not holding a place def test_a_failed_attempt_goes_back_to_the_queue_with_a_time_on_it(jobs): """The resting state of a retry, and everything that is true of it.""" queued = made(jobs.enqueue("r_wait", max_attempts=3, retry_delay=600)) assert got(jobs.claim({})).attempt == 1, "claim counts the attempt" again = got(jobs.retry(queued.id, error="RuntimeError: the feed was down")) assert again.state == "queued", "it is in the queue, not failed" assert again.waiting and not again.exhausted assert again.error == "RuntimeError: the feed was down", "it says why it waits" assert lease(jobs, queued.id) == (None, None), "nobody holds it" assert jobs.running() == [], "and nothing is running it" assert [j.id for j in jobs.waiting()] == [queued.id] assert [j.id for j in jobs.blocked()] == [queued.id] def test_a_waiting_job_does_not_occupy_a_worker(jobs): """The whole of why the wait is a column. One cpu worker, one job waiting ten minutes for its next attempt, and the worker is free the entire time — a handler that slept would have held it and stopped the queue dead.""" waiting = made(jobs.enqueue("r_slot", max_attempts=2, retry_delay=600)) assert jobs.claim({"cpu": 1}) is not None jobs.retry(waiting.id, error="not this time") other = made(jobs.enqueue("r_other")) assert got(jobs.claim({"cpu": 1})).id == other.id, "the slot was free" assert got(jobs.get(waiting.id)).state == "queued" def test_a_job_queued_for_later_is_not_claimed_before_its_time(jobs): """`delay` on the enqueue itself: the durable form of "in ten minutes".""" later = made(jobs.enqueue("r_later", delay=600)) assert later.waiting assert jobs.claim({}) is None bring_forward(jobs, later.id) assert got(jobs.claim({})).id == later.id def test_what_waits_on_a_retrying_job_goes_on_waiting(jobs): """A retry is still live work, so a dependent is held rather than settled. The alternative would be the worse half of both: a batch that is coming back in a minute counted among the ones that did not land, and a final job reporting on a task that is not over. """ batch = made(jobs.enqueue("r_batch", max_attempts=2, retry_delay=600)) final = made(jobs.enqueue("r_final", depends_on=[batch.id], after="done")) jobs.claim({}) jobs.retry(batch.id, error="again in a minute") assert [j.id for j in jobs.blocked()] == [batch.id, final.id] bring_forward(jobs, batch.id) assert got(jobs.claim({})).id == batch.id, "the batch goes first" jobs.finish(batch.id, "done") assert got(jobs.claim({})).id == final.id # --------------------------------------------------------------- the bounds def test_the_bound_counts_the_first_attempt(jobs): """`max_attempts=3` is three runs of the handler, not three retries.""" runs = [] @job("r_bounded", "Bounded", max_attempts=3) def handler(config, store, *, context=NULL): runs.append(1) raise RuntimeError("this was never going to work") queue = JobQueue(jobs, None, None) queued = made(queue.enqueue("r_bounded")) drain(queue) ended = got(jobs.get(queued.id)) assert len(runs) == 3, "it ran exactly as many times as it was allowed" assert ended.state == "failed" and ended.attempt == 3 assert ended.exhausted, "and says the policy ran out rather than merely failing" def test_a_job_with_no_policy_fails_at_the_first_exception(jobs): """The default is one attempt, which is every job queued before any of this existed. Exhausted is not what that is: nothing was used up.""" runs = [] @job("r_unbounded", "Unbounded") def handler(config, store, *, context=NULL): runs.append(1) raise RuntimeError("no") queue = JobQueue(jobs, None, None) queued = made(queue.enqueue("r_unbounded")) drain(queue) ended = got(jobs.get(queued.id)) assert len(runs) == 1 assert ended.state == "failed" and not ended.exhausted def test_an_attempt_that_lands_stops_the_retrying(jobs): """The case the bound exists for: the second go works, and the job is done rather than carrying a memory of having failed once.""" runs = [] @job("r_second_time", "Second time", max_attempts=4) def handler(config, store, *, context=NULL): runs.append(1) if len(runs) < 2: raise RuntimeError("the first one never works") return {"runs": len(runs)} queue = JobQueue(jobs, None, None) queued = made(queue.enqueue("r_second_time")) drain(queue) ended = got(jobs.get(queued.id)) assert ended.state == "done" and ended.result == {"runs": 2} assert ended.attempt == 2 and not ended.exhausted def test_the_wait_lengthens_with_the_backoff(jobs): """Doubling, read off the two stamps rather than waited out.""" queued = made(jobs.enqueue("r_backoff", max_attempts=4, retry_delay=100, retry_backoff=2.0)) waits = [] for _ in range(3): assert jobs.claim({}) is not None again = got(jobs.retry(queued.id, error="again")) waits.append( datetime.fromisoformat(again.run_after) - datetime.now(UTC)) bring_forward(jobs, queued.id) seconds = [w.total_seconds() for w in waits] assert [round(s / 100) for s in seconds] == [1, 2, 4] def test_a_server_hint_replaces_the_backoff(jobs): """Honouring what the other end said, rather than guessing at it. The bound is not part of the bargain: a hint moves the next attempt, it does not add one. """ queued = made(jobs.enqueue("r_hinted", max_attempts=3, retry_delay=600)) jobs.claim({}) again = got(jobs.retry(queued.id, error="429", delay=5)) waited = datetime.fromisoformat(again.run_after) - datetime.now(UTC) assert 0 < waited.total_seconds() <= 5 def test_a_handler_can_name_its_own_wait(jobs): """`Retry(after=...)` from inside the handler, which is where a `Retry-After` header is actually read.""" @job("r_asked", "Asked to come back", max_attempts=3, retry_delay=600) def handler(config, store, *, context=NULL): raise Retry("rate limited", after=7) queue = JobQueue(jobs, None, None) queued = made(queue.enqueue("r_asked")) async def one_pass() -> None: await queue.start() deadline = time.monotonic() + 5 while time.monotonic() < deadline: if got(jobs.get(queued.id)).waiting: break await asyncio.sleep(0.01) await queue.stop() asyncio.run(one_pass()) waiting = got(jobs.get(queued.id)) assert waiting.waiting and waiting.attempt == 1 assert "rate limited" in waiting.error waited = datetime.fromisoformat(waiting.run_after) - datetime.now(UTC) assert 0 < waited.total_seconds() <= 7, "the handler's number, not the policy's" def test_asking_to_come_back_does_not_extend_the_bound(jobs): """A handler cannot grant itself a policy nobody declared.""" runs = [] @job("r_asked_once", "Asked once") def handler(config, store, *, context=NULL): runs.append(1) raise Retry("come back in a second", after=0) queue = JobQueue(jobs, None, None) queued = made(queue.enqueue("r_asked_once")) drain(queue) assert len(runs) == 1 assert got(jobs.get(queued.id)).state == "failed" # ------------------------------------------------------------- across a stop def test_the_next_attempt_survives_the_process_that_scheduled_it(jobs): """The wait is a column, so it outlives everything that is not the file. A second `Jobs` on the same path is what another process sees, and what this one sees after a restart: a queued row it will not take yet, and takes the moment its time comes round. Nothing was handed over, because there was nothing alive to hand. """ queued = made(jobs.enqueue("r_restart", max_attempts=3, retry_delay=600)) jobs.claim({}) jobs.retry(queued.id, error="the feed was down") restarted = Jobs(jobs.path) assert restarted.claim({}) is None, "still waiting, on a table that never saw it fail" assert [j.id for j in restarted.waiting()] == [queued.id] bring_forward(restarted, queued.id) taken = got(restarted.claim({})) assert taken.id == queued.id assert taken.attempt == 2, "and it is the second attempt, not the first again" # ------------------------------------------------------------- cancellation def test_cancelling_a_job_that_is_waiting_to_be_retried_ends_it(jobs): """Kill on a waiting row. It is queued, so it ends here and now — there is no worker to ask and nothing part-way through.""" queued = made(jobs.enqueue("r_killable", max_attempts=3, retry_delay=600)) jobs.claim({}) jobs.retry(queued.id, error="not yet") ended = got(jobs.request_cancel(queued.id)) assert ended.state == "cancelled" assert jobs.waiting() == [] def test_killing_the_run_takes_a_job_waiting_to_be_retried_with_it(jobs): """The same for the task and cascade buttons: an hour's wait is not an hour during which a run cannot be stopped.""" cascade = jobs.new_run("cascade", "r_run") task = jobs.new_run("task", "r_task") queued = made(jobs.enqueue("r_in_a_run", max_attempts=3, retry_delay=600, cascade_id=cascade, task_id=task)) jobs.claim({}) jobs.retry(queued.id, error="not yet") assert jobs.cancel_run(task) == 1 assert got(jobs.get(queued.id)).state == "cancelled" def test_a_kill_during_the_attempt_is_not_undone_by_the_retry(jobs): """The one ordering that would make Kill a lie. Press it while the attempt runs, the attempt then fails, and a retry that only looked at the bound would put the job straight back in the queue — a button that stopped nothing and said it had. """ queued = made(jobs.enqueue("r_killed_mid", max_attempts=3, retry_delay=0)) jobs.claim({}) jobs.request_cancel(queued.id) ended = got(jobs.retry(queued.id, error="it failed as well")) assert ended.state == "cancelled" assert not ended.waiting