Files
WolfandClaude Opus 5.5 9d666d0937 Bring in pipeline, auth, deploy and docs from the boilerplate
roledesk is built on the boilerplate from here on: pipeline 2.0 (queue and
scheduler), auth (one password, sessions, passkeys), the deploy kit and its
docs, plus the coupling and packaging tests. Downstream copies, unedited.

Refreshed from boilerplate/d56949f7d0f6c0bfe5745478e09007bfad67ae87

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:16 +02:00

356 lines
13 KiB
Python

"""Waiting to be tried again, without anything being alive while it waits.
The claim under test is that a job between attempts costs a row and nothing
else. No worker, no thread, no lease, no timer — so the tests that matter here
are the ones that ask what *else* the queue can do while a job waits, and what
survives the process that scheduled the wait.
Each of the four properties step 12 asks for is pinned on its own, because
they fail independently: a bound that is off by one still waits correctly, a
`run_after` that never reaches the table still retries in one process, and a
Kill that cannot reach a waiting row is invisible until somebody presses it.
python3 -m pytest tests/test_retries.py -q
"""
from __future__ import annotations
import asyncio
import sqlite3
import sys
import tempfile
import time
from contextlib import closing
from datetime import UTC, datetime, timedelta
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from pipeline.models import Job
from pipeline import NULL, JobQueue, Jobs, Retry, job
@pytest.fixture
def jobs():
"""The pipeline's table on a file of its own. No app, no store — nothing
here needs one, which is the point of the table owning its own schema."""
with tempfile.TemporaryDirectory() as tmp:
yield Jobs(Path(tmp) / "t.db")
def made(queued: Job | None) -> Job:
"""The job an enqueue made. `enqueue` answers None for a killed run, which
in these tests nothing is."""
assert queued is not None
return queued
def got(found: Job | None) -> Job:
"""The job a read must have found — a None here is a test failure."""
assert found is not None
return found
def lease(jobs: Jobs, job_id: int) -> tuple[str | None, str | None]:
"""The `started_at` and `seen_at` of one job: whether anybody has it."""
with closing(sqlite3.connect(jobs.path)) as conn:
row = conn.execute(
"SELECT started_at, seen_at FROM jobs WHERE id=?", (job_id,)).fetchone()
return row[0], row[1]
def bring_forward(jobs: Jobs, job_id: int) -> None:
"""Move a waiting job's next attempt into the past, as if its wait were
over. What the clock does, done in one statement so a test of an hour's
backoff does not take an hour."""
with closing(sqlite3.connect(jobs.path)) as conn, conn:
conn.execute("UPDATE jobs SET run_after=? WHERE id=?",
((datetime.now(UTC) - timedelta(hours=1)).isoformat(), job_id))
def drain(queue: JobQueue, *, timeout: float = 5.0) -> None:
"""Run the worker until the table is empty and this worker holds nothing.
Both halves, because a job's last acts belong to its handler thread and
between them the table reads as idle while it is not. A job waiting for its
next attempt keeps this going, since it is still `queued` — which is the
behaviour the retry tests are leaning on.
"""
async def go() -> None:
await queue.start()
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
snap = queue.jobs.snapshot()
if not snap.running and not snap.queued and not queue._running:
break
await asyncio.sleep(0.01)
await queue.stop()
asyncio.run(go())
# ------------------------------------------------------- not holding a place
def test_a_failed_attempt_goes_back_to_the_queue_with_a_time_on_it(jobs):
"""The resting state of a retry, and everything that is true of it."""
queued = made(jobs.enqueue("r_wait", max_attempts=3, retry_delay=600))
assert got(jobs.claim({})).attempt == 1, "claim counts the attempt"
again = got(jobs.retry(queued.id, error="RuntimeError: the feed was down"))
assert again.state == "queued", "it is in the queue, not failed"
assert again.waiting and not again.exhausted
assert again.error == "RuntimeError: the feed was down", "it says why it waits"
assert lease(jobs, queued.id) == (None, None), "nobody holds it"
assert jobs.running() == [], "and nothing is running it"
assert [j.id for j in jobs.waiting()] == [queued.id]
assert [j.id for j in jobs.blocked()] == [queued.id]
def test_a_waiting_job_does_not_occupy_a_worker(jobs):
"""The whole of why the wait is a column. One cpu worker, one job waiting
ten minutes for its next attempt, and the worker is free the entire time —
a handler that slept would have held it and stopped the queue dead."""
waiting = made(jobs.enqueue("r_slot", max_attempts=2, retry_delay=600))
assert jobs.claim({"cpu": 1}) is not None
jobs.retry(waiting.id, error="not this time")
other = made(jobs.enqueue("r_other"))
assert got(jobs.claim({"cpu": 1})).id == other.id, "the slot was free"
assert got(jobs.get(waiting.id)).state == "queued"
def test_a_job_queued_for_later_is_not_claimed_before_its_time(jobs):
"""`delay` on the enqueue itself: the durable form of "in ten minutes"."""
later = made(jobs.enqueue("r_later", delay=600))
assert later.waiting
assert jobs.claim({}) is None
bring_forward(jobs, later.id)
assert got(jobs.claim({})).id == later.id
def test_what_waits_on_a_retrying_job_goes_on_waiting(jobs):
"""A retry is still live work, so a dependent is held rather than settled.
The alternative would be the worse half of both: a batch that is coming
back in a minute counted among the ones that did not land, and a final job
reporting on a task that is not over.
"""
batch = made(jobs.enqueue("r_batch", max_attempts=2, retry_delay=600))
final = made(jobs.enqueue("r_final", depends_on=[batch.id], after="done"))
jobs.claim({})
jobs.retry(batch.id, error="again in a minute")
assert [j.id for j in jobs.blocked()] == [batch.id, final.id]
bring_forward(jobs, batch.id)
assert got(jobs.claim({})).id == batch.id, "the batch goes first"
jobs.finish(batch.id, "done")
assert got(jobs.claim({})).id == final.id
# --------------------------------------------------------------- the bounds
def test_the_bound_counts_the_first_attempt(jobs):
"""`max_attempts=3` is three runs of the handler, not three retries."""
runs = []
@job("r_bounded", "Bounded", max_attempts=3)
def handler(config, store, *, context=NULL):
runs.append(1)
raise RuntimeError("this was never going to work")
queue = JobQueue(jobs, None, None)
queued = made(queue.enqueue("r_bounded"))
drain(queue)
ended = got(jobs.get(queued.id))
assert len(runs) == 3, "it ran exactly as many times as it was allowed"
assert ended.state == "failed" and ended.attempt == 3
assert ended.exhausted, "and says the policy ran out rather than merely failing"
def test_a_job_with_no_policy_fails_at_the_first_exception(jobs):
"""The default is one attempt, which is every job queued before any of
this existed. Exhausted is not what that is: nothing was used up."""
runs = []
@job("r_unbounded", "Unbounded")
def handler(config, store, *, context=NULL):
runs.append(1)
raise RuntimeError("no")
queue = JobQueue(jobs, None, None)
queued = made(queue.enqueue("r_unbounded"))
drain(queue)
ended = got(jobs.get(queued.id))
assert len(runs) == 1
assert ended.state == "failed" and not ended.exhausted
def test_an_attempt_that_lands_stops_the_retrying(jobs):
"""The case the bound exists for: the second go works, and the job is done
rather than carrying a memory of having failed once."""
runs = []
@job("r_second_time", "Second time", max_attempts=4)
def handler(config, store, *, context=NULL):
runs.append(1)
if len(runs) < 2:
raise RuntimeError("the first one never works")
return {"runs": len(runs)}
queue = JobQueue(jobs, None, None)
queued = made(queue.enqueue("r_second_time"))
drain(queue)
ended = got(jobs.get(queued.id))
assert ended.state == "done" and ended.result == {"runs": 2}
assert ended.attempt == 2 and not ended.exhausted
def test_the_wait_lengthens_with_the_backoff(jobs):
"""Doubling, read off the two stamps rather than waited out."""
queued = made(jobs.enqueue("r_backoff", max_attempts=4,
retry_delay=100, retry_backoff=2.0))
waits = []
for _ in range(3):
assert jobs.claim({}) is not None
again = got(jobs.retry(queued.id, error="again"))
waits.append(
datetime.fromisoformat(again.run_after) - datetime.now(UTC))
bring_forward(jobs, queued.id)
seconds = [w.total_seconds() for w in waits]
assert [round(s / 100) for s in seconds] == [1, 2, 4]
def test_a_server_hint_replaces_the_backoff(jobs):
"""Honouring what the other end said, rather than guessing at it.
The bound is not part of the bargain: a hint moves the next attempt, it
does not add one.
"""
queued = made(jobs.enqueue("r_hinted", max_attempts=3, retry_delay=600))
jobs.claim({})
again = got(jobs.retry(queued.id, error="429", delay=5))
waited = datetime.fromisoformat(again.run_after) - datetime.now(UTC)
assert 0 < waited.total_seconds() <= 5
def test_a_handler_can_name_its_own_wait(jobs):
"""`Retry(after=...)` from inside the handler, which is where a
`Retry-After` header is actually read."""
@job("r_asked", "Asked to come back", max_attempts=3, retry_delay=600)
def handler(config, store, *, context=NULL):
raise Retry("rate limited", after=7)
queue = JobQueue(jobs, None, None)
queued = made(queue.enqueue("r_asked"))
async def one_pass() -> None:
await queue.start()
deadline = time.monotonic() + 5
while time.monotonic() < deadline:
if got(jobs.get(queued.id)).waiting:
break
await asyncio.sleep(0.01)
await queue.stop()
asyncio.run(one_pass())
waiting = got(jobs.get(queued.id))
assert waiting.waiting and waiting.attempt == 1
assert "rate limited" in waiting.error
waited = datetime.fromisoformat(waiting.run_after) - datetime.now(UTC)
assert 0 < waited.total_seconds() <= 7, "the handler's number, not the policy's"
def test_asking_to_come_back_does_not_extend_the_bound(jobs):
"""A handler cannot grant itself a policy nobody declared."""
runs = []
@job("r_asked_once", "Asked once")
def handler(config, store, *, context=NULL):
runs.append(1)
raise Retry("come back in a second", after=0)
queue = JobQueue(jobs, None, None)
queued = made(queue.enqueue("r_asked_once"))
drain(queue)
assert len(runs) == 1
assert got(jobs.get(queued.id)).state == "failed"
# ------------------------------------------------------------- across a stop
def test_the_next_attempt_survives_the_process_that_scheduled_it(jobs):
"""The wait is a column, so it outlives everything that is not the file.
A second `Jobs` on the same path is what another process sees, and what
this one sees after a restart: a queued row it will not take yet, and takes
the moment its time comes round. Nothing was handed over, because there
was nothing alive to hand.
"""
queued = made(jobs.enqueue("r_restart", max_attempts=3, retry_delay=600))
jobs.claim({})
jobs.retry(queued.id, error="the feed was down")
restarted = Jobs(jobs.path)
assert restarted.claim({}) is None, "still waiting, on a table that never saw it fail"
assert [j.id for j in restarted.waiting()] == [queued.id]
bring_forward(restarted, queued.id)
taken = got(restarted.claim({}))
assert taken.id == queued.id
assert taken.attempt == 2, "and it is the second attempt, not the first again"
# ------------------------------------------------------------- cancellation
def test_cancelling_a_job_that_is_waiting_to_be_retried_ends_it(jobs):
"""Kill on a waiting row. It is queued, so it ends here and now — there is
no worker to ask and nothing part-way through."""
queued = made(jobs.enqueue("r_killable", max_attempts=3, retry_delay=600))
jobs.claim({})
jobs.retry(queued.id, error="not yet")
ended = got(jobs.request_cancel(queued.id))
assert ended.state == "cancelled"
assert jobs.waiting() == []
def test_killing_the_run_takes_a_job_waiting_to_be_retried_with_it(jobs):
"""The same for the task and cascade buttons: an hour's wait is not an
hour during which a run cannot be stopped."""
cascade = jobs.new_run("cascade", "r_run")
task = jobs.new_run("task", "r_task")
queued = made(jobs.enqueue("r_in_a_run", max_attempts=3, retry_delay=600,
cascade_id=cascade, task_id=task))
jobs.claim({})
jobs.retry(queued.id, error="not yet")
assert jobs.cancel_run(task) == 1
assert got(jobs.get(queued.id)).state == "cancelled"
def test_a_kill_during_the_attempt_is_not_undone_by_the_retry(jobs):
"""The one ordering that would make Kill a lie.
Press it while the attempt runs, the attempt then fails, and a retry that
only looked at the bound would put the job straight back in the queue —
a button that stopped nothing and said it had.
"""
queued = made(jobs.enqueue("r_killed_mid", max_attempts=3, retry_delay=0))
jobs.claim({})
jobs.request_cancel(queued.id)
ended = got(jobs.retry(queued.id, error="it failed as well"))
assert ended.state == "cancelled"
assert not ended.waiting