roledesk is built on the boilerplate from here on: pipeline 2.0 (queue and scheduler), auth (one password, sessions, passkeys), the deploy kit and its docs, plus the coupling and packaging tests. Downstream copies, unedited. Refreshed from boilerplate/d56949f7d0f6c0bfe5745478e09007bfad67ae87 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
356 lines
13 KiB
Python
356 lines
13 KiB
Python
"""Waiting to be tried again, without anything being alive while it waits.
|
|
|
|
The claim under test is that a job between attempts costs a row and nothing
|
|
else. No worker, no thread, no lease, no timer — so the tests that matter here
|
|
are the ones that ask what *else* the queue can do while a job waits, and what
|
|
survives the process that scheduled the wait.
|
|
|
|
Each of the four properties step 12 asks for is pinned on its own, because
|
|
they fail independently: a bound that is off by one still waits correctly, a
|
|
`run_after` that never reaches the table still retries in one process, and a
|
|
Kill that cannot reach a waiting row is invisible until somebody presses it.
|
|
|
|
python3 -m pytest tests/test_retries.py -q
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from contextlib import closing
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
|
|
from pipeline.models import Job
|
|
from pipeline import NULL, JobQueue, Jobs, Retry, job
|
|
|
|
|
|
@pytest.fixture
|
|
def jobs():
|
|
"""The pipeline's table on a file of its own. No app, no store — nothing
|
|
here needs one, which is the point of the table owning its own schema."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
yield Jobs(Path(tmp) / "t.db")
|
|
|
|
|
|
def made(queued: Job | None) -> Job:
|
|
"""The job an enqueue made. `enqueue` answers None for a killed run, which
|
|
in these tests nothing is."""
|
|
assert queued is not None
|
|
return queued
|
|
|
|
|
|
def got(found: Job | None) -> Job:
|
|
"""The job a read must have found — a None here is a test failure."""
|
|
assert found is not None
|
|
return found
|
|
|
|
|
|
def lease(jobs: Jobs, job_id: int) -> tuple[str | None, str | None]:
|
|
"""The `started_at` and `seen_at` of one job: whether anybody has it."""
|
|
with closing(sqlite3.connect(jobs.path)) as conn:
|
|
row = conn.execute(
|
|
"SELECT started_at, seen_at FROM jobs WHERE id=?", (job_id,)).fetchone()
|
|
return row[0], row[1]
|
|
|
|
|
|
def bring_forward(jobs: Jobs, job_id: int) -> None:
|
|
"""Move a waiting job's next attempt into the past, as if its wait were
|
|
over. What the clock does, done in one statement so a test of an hour's
|
|
backoff does not take an hour."""
|
|
with closing(sqlite3.connect(jobs.path)) as conn, conn:
|
|
conn.execute("UPDATE jobs SET run_after=? WHERE id=?",
|
|
((datetime.now(UTC) - timedelta(hours=1)).isoformat(), job_id))
|
|
|
|
|
|
def drain(queue: JobQueue, *, timeout: float = 5.0) -> None:
|
|
"""Run the worker until the table is empty and this worker holds nothing.
|
|
|
|
Both halves, because a job's last acts belong to its handler thread and
|
|
between them the table reads as idle while it is not. A job waiting for its
|
|
next attempt keeps this going, since it is still `queued` — which is the
|
|
behaviour the retry tests are leaning on.
|
|
"""
|
|
|
|
async def go() -> None:
|
|
await queue.start()
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
snap = queue.jobs.snapshot()
|
|
if not snap.running and not snap.queued and not queue._running:
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
await queue.stop()
|
|
|
|
asyncio.run(go())
|
|
|
|
|
|
# ------------------------------------------------------- not holding a place
|
|
|
|
def test_a_failed_attempt_goes_back_to_the_queue_with_a_time_on_it(jobs):
|
|
"""The resting state of a retry, and everything that is true of it."""
|
|
queued = made(jobs.enqueue("r_wait", max_attempts=3, retry_delay=600))
|
|
assert got(jobs.claim({})).attempt == 1, "claim counts the attempt"
|
|
|
|
again = got(jobs.retry(queued.id, error="RuntimeError: the feed was down"))
|
|
assert again.state == "queued", "it is in the queue, not failed"
|
|
assert again.waiting and not again.exhausted
|
|
assert again.error == "RuntimeError: the feed was down", "it says why it waits"
|
|
assert lease(jobs, queued.id) == (None, None), "nobody holds it"
|
|
assert jobs.running() == [], "and nothing is running it"
|
|
assert [j.id for j in jobs.waiting()] == [queued.id]
|
|
assert [j.id for j in jobs.blocked()] == [queued.id]
|
|
|
|
|
|
def test_a_waiting_job_does_not_occupy_a_worker(jobs):
|
|
"""The whole of why the wait is a column. One cpu worker, one job waiting
|
|
ten minutes for its next attempt, and the worker is free the entire time —
|
|
a handler that slept would have held it and stopped the queue dead."""
|
|
waiting = made(jobs.enqueue("r_slot", max_attempts=2, retry_delay=600))
|
|
assert jobs.claim({"cpu": 1}) is not None
|
|
jobs.retry(waiting.id, error="not this time")
|
|
|
|
other = made(jobs.enqueue("r_other"))
|
|
assert got(jobs.claim({"cpu": 1})).id == other.id, "the slot was free"
|
|
assert got(jobs.get(waiting.id)).state == "queued"
|
|
|
|
|
|
def test_a_job_queued_for_later_is_not_claimed_before_its_time(jobs):
|
|
"""`delay` on the enqueue itself: the durable form of "in ten minutes"."""
|
|
later = made(jobs.enqueue("r_later", delay=600))
|
|
assert later.waiting
|
|
assert jobs.claim({}) is None
|
|
bring_forward(jobs, later.id)
|
|
assert got(jobs.claim({})).id == later.id
|
|
|
|
|
|
def test_what_waits_on_a_retrying_job_goes_on_waiting(jobs):
|
|
"""A retry is still live work, so a dependent is held rather than settled.
|
|
|
|
The alternative would be the worse half of both: a batch that is coming
|
|
back in a minute counted among the ones that did not land, and a final job
|
|
reporting on a task that is not over.
|
|
"""
|
|
batch = made(jobs.enqueue("r_batch", max_attempts=2, retry_delay=600))
|
|
final = made(jobs.enqueue("r_final", depends_on=[batch.id], after="done"))
|
|
jobs.claim({})
|
|
jobs.retry(batch.id, error="again in a minute")
|
|
|
|
assert [j.id for j in jobs.blocked()] == [batch.id, final.id]
|
|
bring_forward(jobs, batch.id)
|
|
assert got(jobs.claim({})).id == batch.id, "the batch goes first"
|
|
jobs.finish(batch.id, "done")
|
|
assert got(jobs.claim({})).id == final.id
|
|
|
|
|
|
# --------------------------------------------------------------- the bounds
|
|
|
|
def test_the_bound_counts_the_first_attempt(jobs):
|
|
"""`max_attempts=3` is three runs of the handler, not three retries."""
|
|
runs = []
|
|
|
|
@job("r_bounded", "Bounded", max_attempts=3)
|
|
def handler(config, store, *, context=NULL):
|
|
runs.append(1)
|
|
raise RuntimeError("this was never going to work")
|
|
|
|
queue = JobQueue(jobs, None, None)
|
|
queued = made(queue.enqueue("r_bounded"))
|
|
drain(queue)
|
|
|
|
ended = got(jobs.get(queued.id))
|
|
assert len(runs) == 3, "it ran exactly as many times as it was allowed"
|
|
assert ended.state == "failed" and ended.attempt == 3
|
|
assert ended.exhausted, "and says the policy ran out rather than merely failing"
|
|
|
|
|
|
def test_a_job_with_no_policy_fails_at_the_first_exception(jobs):
|
|
"""The default is one attempt, which is every job queued before any of
|
|
this existed. Exhausted is not what that is: nothing was used up."""
|
|
runs = []
|
|
|
|
@job("r_unbounded", "Unbounded")
|
|
def handler(config, store, *, context=NULL):
|
|
runs.append(1)
|
|
raise RuntimeError("no")
|
|
|
|
queue = JobQueue(jobs, None, None)
|
|
queued = made(queue.enqueue("r_unbounded"))
|
|
drain(queue)
|
|
|
|
ended = got(jobs.get(queued.id))
|
|
assert len(runs) == 1
|
|
assert ended.state == "failed" and not ended.exhausted
|
|
|
|
|
|
def test_an_attempt_that_lands_stops_the_retrying(jobs):
|
|
"""The case the bound exists for: the second go works, and the job is done
|
|
rather than carrying a memory of having failed once."""
|
|
runs = []
|
|
|
|
@job("r_second_time", "Second time", max_attempts=4)
|
|
def handler(config, store, *, context=NULL):
|
|
runs.append(1)
|
|
if len(runs) < 2:
|
|
raise RuntimeError("the first one never works")
|
|
return {"runs": len(runs)}
|
|
|
|
queue = JobQueue(jobs, None, None)
|
|
queued = made(queue.enqueue("r_second_time"))
|
|
drain(queue)
|
|
|
|
ended = got(jobs.get(queued.id))
|
|
assert ended.state == "done" and ended.result == {"runs": 2}
|
|
assert ended.attempt == 2 and not ended.exhausted
|
|
|
|
|
|
def test_the_wait_lengthens_with_the_backoff(jobs):
|
|
"""Doubling, read off the two stamps rather than waited out."""
|
|
queued = made(jobs.enqueue("r_backoff", max_attempts=4,
|
|
retry_delay=100, retry_backoff=2.0))
|
|
|
|
waits = []
|
|
for _ in range(3):
|
|
assert jobs.claim({}) is not None
|
|
again = got(jobs.retry(queued.id, error="again"))
|
|
waits.append(
|
|
datetime.fromisoformat(again.run_after) - datetime.now(UTC))
|
|
bring_forward(jobs, queued.id)
|
|
|
|
seconds = [w.total_seconds() for w in waits]
|
|
assert [round(s / 100) for s in seconds] == [1, 2, 4]
|
|
|
|
|
|
def test_a_server_hint_replaces_the_backoff(jobs):
|
|
"""Honouring what the other end said, rather than guessing at it.
|
|
|
|
The bound is not part of the bargain: a hint moves the next attempt, it
|
|
does not add one.
|
|
"""
|
|
queued = made(jobs.enqueue("r_hinted", max_attempts=3, retry_delay=600))
|
|
jobs.claim({})
|
|
again = got(jobs.retry(queued.id, error="429", delay=5))
|
|
waited = datetime.fromisoformat(again.run_after) - datetime.now(UTC)
|
|
assert 0 < waited.total_seconds() <= 5
|
|
|
|
|
|
def test_a_handler_can_name_its_own_wait(jobs):
|
|
"""`Retry(after=...)` from inside the handler, which is where a
|
|
`Retry-After` header is actually read."""
|
|
|
|
@job("r_asked", "Asked to come back", max_attempts=3, retry_delay=600)
|
|
def handler(config, store, *, context=NULL):
|
|
raise Retry("rate limited", after=7)
|
|
|
|
queue = JobQueue(jobs, None, None)
|
|
queued = made(queue.enqueue("r_asked"))
|
|
|
|
async def one_pass() -> None:
|
|
await queue.start()
|
|
deadline = time.monotonic() + 5
|
|
while time.monotonic() < deadline:
|
|
if got(jobs.get(queued.id)).waiting:
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
await queue.stop()
|
|
|
|
asyncio.run(one_pass())
|
|
|
|
waiting = got(jobs.get(queued.id))
|
|
assert waiting.waiting and waiting.attempt == 1
|
|
assert "rate limited" in waiting.error
|
|
waited = datetime.fromisoformat(waiting.run_after) - datetime.now(UTC)
|
|
assert 0 < waited.total_seconds() <= 7, "the handler's number, not the policy's"
|
|
|
|
|
|
def test_asking_to_come_back_does_not_extend_the_bound(jobs):
|
|
"""A handler cannot grant itself a policy nobody declared."""
|
|
runs = []
|
|
|
|
@job("r_asked_once", "Asked once")
|
|
def handler(config, store, *, context=NULL):
|
|
runs.append(1)
|
|
raise Retry("come back in a second", after=0)
|
|
|
|
queue = JobQueue(jobs, None, None)
|
|
queued = made(queue.enqueue("r_asked_once"))
|
|
drain(queue)
|
|
|
|
assert len(runs) == 1
|
|
assert got(jobs.get(queued.id)).state == "failed"
|
|
|
|
|
|
# ------------------------------------------------------------- across a stop
|
|
|
|
def test_the_next_attempt_survives_the_process_that_scheduled_it(jobs):
|
|
"""The wait is a column, so it outlives everything that is not the file.
|
|
|
|
A second `Jobs` on the same path is what another process sees, and what
|
|
this one sees after a restart: a queued row it will not take yet, and takes
|
|
the moment its time comes round. Nothing was handed over, because there
|
|
was nothing alive to hand.
|
|
"""
|
|
queued = made(jobs.enqueue("r_restart", max_attempts=3, retry_delay=600))
|
|
jobs.claim({})
|
|
jobs.retry(queued.id, error="the feed was down")
|
|
|
|
restarted = Jobs(jobs.path)
|
|
assert restarted.claim({}) is None, "still waiting, on a table that never saw it fail"
|
|
assert [j.id for j in restarted.waiting()] == [queued.id]
|
|
|
|
bring_forward(restarted, queued.id)
|
|
taken = got(restarted.claim({}))
|
|
assert taken.id == queued.id
|
|
assert taken.attempt == 2, "and it is the second attempt, not the first again"
|
|
|
|
|
|
# ------------------------------------------------------------- cancellation
|
|
|
|
def test_cancelling_a_job_that_is_waiting_to_be_retried_ends_it(jobs):
|
|
"""Kill on a waiting row. It is queued, so it ends here and now — there is
|
|
no worker to ask and nothing part-way through."""
|
|
queued = made(jobs.enqueue("r_killable", max_attempts=3, retry_delay=600))
|
|
jobs.claim({})
|
|
jobs.retry(queued.id, error="not yet")
|
|
|
|
ended = got(jobs.request_cancel(queued.id))
|
|
assert ended.state == "cancelled"
|
|
assert jobs.waiting() == []
|
|
|
|
|
|
def test_killing_the_run_takes_a_job_waiting_to_be_retried_with_it(jobs):
|
|
"""The same for the task and cascade buttons: an hour's wait is not an
|
|
hour during which a run cannot be stopped."""
|
|
cascade = jobs.new_run("cascade", "r_run")
|
|
task = jobs.new_run("task", "r_task")
|
|
queued = made(jobs.enqueue("r_in_a_run", max_attempts=3, retry_delay=600,
|
|
cascade_id=cascade, task_id=task))
|
|
jobs.claim({})
|
|
jobs.retry(queued.id, error="not yet")
|
|
|
|
assert jobs.cancel_run(task) == 1
|
|
assert got(jobs.get(queued.id)).state == "cancelled"
|
|
|
|
|
|
def test_a_kill_during_the_attempt_is_not_undone_by_the_retry(jobs):
|
|
"""The one ordering that would make Kill a lie.
|
|
|
|
Press it while the attempt runs, the attempt then fails, and a retry that
|
|
only looked at the bound would put the job straight back in the queue —
|
|
a button that stopped nothing and said it had.
|
|
"""
|
|
queued = made(jobs.enqueue("r_killed_mid", max_attempts=3, retry_delay=0))
|
|
jobs.claim({})
|
|
jobs.request_cancel(queued.id)
|
|
|
|
ended = got(jobs.retry(queued.id, error="it failed as well"))
|
|
assert ended.state == "cancelled"
|
|
assert not ended.waiting
|