Bring in pipeline, auth, deploy and docs from the boilerplate

roledesk is built on the boilerplate from here on: pipeline 2.0 (queue and
scheduler), auth (one password, sessions, passkeys), the deploy kit and its
docs, plus the coupling and packaging tests. Downstream copies, unedited.

Refreshed from boilerplate/d56949f7d0f6c0bfe5745478e09007bfad67ae87

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-23 12:29:16 +02:00
1 parent 1f436104f3
commit 9d666d0937
76 files changed
+15235

No files matched your search

+2
View File
@@ -0,0 +1,2 @@
"""The pipeline's own tests. They travel with the folder, so a copy can
prove itself where it lands: `python -m pytest pipeline/tests`."""
+74
View File
@@ -0,0 +1,74 @@
"""Handlers for the isolation tests, in a module a child process can import.
Not a test file — pytest does not collect it — and deliberately separate from
the tests that use it, because the point of `isolate=True` is that the work
runs in a *different process*. That process gets nothing from this one: it
imports `pipeline.tests.isolated_handlers` itself, by the name in
`--bootstrap`, which is exactly what an app's own bootstrap has to do.
It also documents the whole of the contract, which is two lines: import the
modules that fill `JOBS`, and return the `(config, store)` pair the handlers
take. Here both are dicts, since nothing in the pipeline reads them.
"""
from __future__ import annotations
import os
import time
from typing import Any
from pipeline import NULL, RunContext, job
@job("iso_ok", "Working in its own process", isolate=True)
def iso_ok(config: Any, store: Any, *, context: RunContext = NULL,
n: int = 0) -> dict[str, Any]:
"""Reports which process it ran in, so a test can prove it was not ours."""
context.progress(n, n, f"counting to {n}")
return {"doubled": n * 2, "pid": os.getpid()}
@job("iso_raises", "Failing in its own process", isolate=True)
def iso_raises(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
raise RuntimeError("the model said no")
@job("iso_slow", "Taking too long on purpose", isolate=True, timeout=1.0)
def iso_slow(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
"""Uninterruptible on purpose: no `check()`, so only a kill ends it.
`time.sleep` stands in for `sklearn.fit` — the difference that matters is
not what it does but that nothing in it ever asks whether to stop, which
is the case a thread has no answer for.
"""
time.sleep(120)
return {"never": True}
def bootstrap() -> tuple[Any, Any]:
"""What `--bootstrap` names. Importing this module registered the jobs."""
return {"config": "rebuilt in the child"}, {"store": "rebuilt in the child"}
# ---------------------------------------------------- two workers, one file
@job("mp_mark", "Leaving a mark")
def mp_mark(config: Any, store: Any, *, context: RunContext = NULL,
i: int = 0) -> dict[str, Any]:
"""Leave a file named after this job and the process that ran it.
A file per run rather than a line in a shared one, because creating a
uniquely named file is atomic on every platform and appending from two
processes is not. A job run twice leaves two files with the same number.
"""
import uuid
from pathlib import Path
(Path(store) / f"{i}-{os.getpid()}-{uuid.uuid4().hex}").touch()
time.sleep(0.02) # long enough that both workers get a share
return {"i": i}
def marking(marks: str) -> tuple[Any, Any]:
"""The bootstrap `pipeline.worker` is given: the marks directory is the store."""
return {}, marks
+39
View File
@@ -0,0 +1,39 @@
"""A job and a declared schedule, in a module a worker process can import.
Not a test file — pytest does not collect it — and separate from the tests that
use it for the same reason `isolated_handlers.py` is: the process under test is
a *different* process, and it gets nothing from this one. It imports this by
the name in `--bootstrap`, which fills `JOBS` and `SCHEDULES` exactly as an
app's own jobs module does.
Separate from `isolated_handlers.py` as well, because importing a module that
calls `declare` fills a global: a test suite that imported this one would find
its own schedule registry non-empty before it had declared anything.
"""
from __future__ import annotations
from pathlib import Path
from typing import Any
from pipeline import NULL, RunContext, declare, job
@job("sw_mark", "Leaving a mark")
def sw_mark(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
"""Touch a file in the store directory, so the parent process can see from
outside whether this ran at all."""
(Path(store) / "fired").touch()
return {"fired": True}
#: The declaration itself is what one half of the worker test is about: a
#: worker with no web app around it reconciles declared defaults like any
#: other, so a machine running only `python -m pipeline.worker` still gets
#: them.
declare("marker", "sw_mark", "every 1h", label="Leaving a mark, hourly")
def marking(marks: str) -> tuple[Any, Any]:
"""The bootstrap `pipeline.worker` is given: the marks directory is the store."""
return {}, marks
+113
View File
@@ -0,0 +1,113 @@
"""The web face of the queue, from an app that has written nothing but `connect`.
python -m pytest pipeline/tests/test_web.py -q
"""
from __future__ import annotations
import sys
import tempfile
import time
from contextlib import asynccontextmanager
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
pytest.importorskip("fastapi")
from fastapi import FastAPI
from fastapi.testclient import TestClient
from pipeline import NULL, Jobs, RunContext, job
from pipeline.web import connect, jobs_router
@job("w_hello", "Saying hello")
def w_hello(config, store, *, context: RunContext = NULL, name: str = "") -> dict:
context.progress(1, 1, f"greeting {name}")
return {"said": f"hello {name}", "config": config["app"]}
@pytest.fixture
def jobs():
with tempfile.TemporaryDirectory() as tmp:
yield Jobs(Path(tmp) / "web.db")
def settled(client: TestClient, job_id: int, timeout: float = 10.0) -> dict:
"""The job as the HTTP API reports it, once it has finished."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
snap = client.get("/api/jobs").json()
for row in snap["recent"]:
if row["id"] == job_id:
return row
time.sleep(0.05)
raise AssertionError(f"job {job_id} did not finish")
def test_connect_is_the_whole_of_the_wiring(jobs):
"""Routes, socket and a running worker, from one call.
The app below has nothing else: no lifespan, no broadcaster, no router of
its own. A job enqueued over HTTP runs to `done` because the worker came up
with the app, and the socket hands a new tab the current state on connect.
"""
app = FastAPI()
queue = connect(app, jobs, {"app": "tiny"}, None)
assert app.state.queue is queue
with TestClient(app) as client:
made = client.post("/api/jobs", json={"kind": "w_hello",
"payload": {"name": "you"}})
assert made.status_code == 202, made.text
ended = settled(client, made.json()["id"])
assert ended["state"] == "done"
assert ended["result"] == {"said": "hello you", "config": "tiny"}
with client.websocket_connect("/ws/jobs") as socket:
hello = socket.receive_json()
assert hello["type"] == "jobs"
assert any(row["id"] == ended["id"] for row in hello["recent"])
assert queue._task is None or queue._task.done(), "and it stopped with the app"
def test_connect_keeps_the_apps_own_lifespan(jobs):
"""An app with startup code of its own keeps it, and the worker is already
up when that code runs — so startup code may enqueue."""
seen: list[str] = []
@asynccontextmanager
async def lifespan(app: FastAPI):
seen.append("app started")
app.state.queue.enqueue("w_hello", {"name": "from startup"})
yield
seen.append("app stopped")
app = FastAPI(lifespan=lifespan)
connect(app, jobs, {"app": "tiny"}, None)
with TestClient(app) as client:
# By id: the snapshot reads live and recent jobs separately, so a job
# that finishes in between is listed in both.
snap = client.get("/api/jobs").json()
(job_id,) = {r["id"] for r in snap["queued"] + snap["running"]
+ snap["recent"] if r["kind"] == "w_hello"}
assert settled(client, job_id)["state"] == "done"
assert seen == ["app started", "app stopped"]
def test_the_element_is_served_by_the_queues_own_router():
"""Two lines in a template and a new app has a job list. Served from a
route, because a router cannot mount a static directory and needing the
app to mount one is the setup the element exists to remove."""
app = FastAPI()
app.include_router(jobs_router)
answer = TestClient(app).get("/_jobs/jobs.js")
assert answer.status_code == 200
assert answer.headers["content-type"].startswith("text/javascript")
assert 'customElements.define("desk-jobs"' in answer.text
+209
View File
@@ -0,0 +1,209 @@
"""What keeps a queue healthy for months: retention, bounded results, whose
retry policy wins, draining, and two real processes on one file.
python -m pytest pipeline/tests/test_housekeeping.py -q
"""
from __future__ import annotations
import os
import sqlite3
import subprocess
import sys
import tempfile
from contextlib import closing
from datetime import UTC, datetime, timedelta
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from pipeline import NULL, Batch, Bound, JobQueue, Jobs, Task, job
from pipeline.models import Job
ROOT = Path(__file__).resolve().parents[2]
@job("h_plain", "A job with a policy of its own", max_attempts=3, retry_delay=7)
def h_plain(config, store, *, context=NULL, i: int = 0) -> dict:
return {"i": i}
@job("h_big", "Returning far too much", max_attempts=3)
def h_big(config, store, *, context=NULL) -> dict:
return {"rows": ["x" * 100] * 2000, "count": 2000}
@pytest.fixture
def jobs():
with tempfile.TemporaryDirectory() as tmp:
yield Jobs(Path(tmp) / "h.db")
def got(found: Job | None) -> Job:
assert found is not None
return found
def age(jobs: Jobs, job_id: int, days: float) -> None:
"""Pretend a job finished `days` ago, and its runs were minted then too."""
when = (datetime.now(UTC) - timedelta(days=days)).isoformat()
with closing(sqlite3.connect(jobs.path)) as conn, conn:
conn.execute("UPDATE jobs SET finished_at=? WHERE id=?", (when, job_id))
conn.execute("UPDATE runs SET created_at=? WHERE id IN "
"(SELECT cascade_id FROM jobs WHERE id=? UNION "
" SELECT task_id FROM jobs WHERE id=?)", (when, job_id, job_id))
def runs(jobs: Jobs) -> int:
with closing(sqlite3.connect(jobs.path)) as conn:
return conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0]
# ---------------------------------------------------------------- retention
def test_purge_forgets_old_history_but_not_the_last_of_each_kind(jobs):
"""Without this both tables grow for as long as the app runs.
What survives an old cutoff: anything unfinished, and the newest ending of
each kind in each state — because `last(kind)` is how an app says "last
pulled five weeks ago", and deleting it would make that "never".
"""
old = [got(jobs.enqueue("h_plain")) for _ in range(3)]
for queued in old:
jobs.finish(queued.id, "done")
age(jobs, queued.id, days=40)
waiting = got(jobs.enqueue("h_plain")) # never finished
gone = jobs.purge(30 * 86400)
assert gone == 2, "two of the three old ones went"
assert got(jobs.last("h_plain")).id == old[-1].id, "the newest stayed"
assert got(jobs.get(waiting.id)).state == "queued", "unfinished work is not history"
def test_purge_leaves_recent_history_alone(jobs):
recent = got(jobs.enqueue("h_plain"))
jobs.finish(recent.id, "done")
before = runs(jobs)
assert jobs.purge(30 * 86400) == 0
assert jobs.get(recent.id) is not None
assert runs(jobs) == before
def test_purge_drops_runs_nothing_points_at_any_more(jobs):
first = got(jobs.enqueue("h_plain"))
second = got(jobs.enqueue("h_plain"))
for queued in (first, second):
jobs.finish(queued.id, "done")
age(jobs, queued.id, days=40)
before = runs(jobs)
jobs.purge(30 * 86400)
assert runs(jobs) < before, "the purged job's cascade and task went with it"
assert jobs.run(second.task_id) is not None, "the kept job's runs did not"
# ----------------------------------------------------------- bounded results
def test_a_result_too_large_fails_the_job_loudly_and_once(jobs):
"""Past `MAX_RESULT` the job fails rather than losing data quietly: the row
says why and how to fix it, a notice goes to whoever is watching, and it is
not retried — the same handler would return the same result."""
said: list = []
async def on_notice(notice) -> None:
said.append(notice)
queued = got(jobs.enqueue("h_big", max_attempts=3))
JobQueue(jobs, None, None, on_notice=on_notice).drain(timeout=10)
ended = got(jobs.get(queued.id))
assert ended.state == "failed"
assert "ResultTooLarge" in ended.error and "store" in ended.error, ended.error
assert "count, rows" in ended.error, "it names what was returned"
assert ended.result == {}, "nothing of it was written"
assert ended.attempt == 1, "and it was not tried again"
assert [n.code for n in said] == ["job.failed"], "and somebody was told"
def test_a_result_at_the_limit_is_kept_whole(jobs):
"""The limit is a line, not a tax: everything under it is stored as is."""
queued = got(jobs.enqueue("h_plain", payload={"i": 1}))
JobQueue(jobs, None, None).drain(timeout=10)
assert got(jobs.get(queued.id)).result == {"i": 1}
# -------------------------------------------------------- whose policy wins
def test_a_task_with_plain_payloads_keeps_the_kinds_own_retry_policy(jobs):
"""`@job(max_attempts=3)` has to mean something when a task queues it.
It used not to: a plain payload was given the default bound explicitly,
which silently overrode whatever the kind declared.
"""
queue = JobQueue(jobs, None, None)
Task(kind="h_plain", finder=lambda c, s: [{"i": 1}]).start(None, None, queue)
(child,) = [j for j in jobs.snapshot().queued if j.kind == "h_plain"]
assert (child.max_attempts, child.retry_delay) == (3, 7.0)
def test_a_batch_with_its_own_bound_overrides_the_kind(jobs):
"""The other half: a batch cut from one plugin profile carries that
profile's policy, and that wins, because the outlet knows its publisher."""
queue = JobQueue(jobs, None, None)
Task(kind="h_plain", finder=lambda c, s: [
Batch({"i": 1}, Bound(max_attempts=5, delay=60))]).start(None, None, queue)
(child,) = [j for j in jobs.snapshot().queued if j.kind == "h_plain"]
assert (child.max_attempts, child.retry_delay) == (5, 60.0)
# ------------------------------------------------------------------ draining
def test_drain_runs_everything_queued_and_stops(jobs):
"""The batch shape: a script or a cron entry with no web app around it."""
ids = [got(jobs.enqueue("h_plain", payload={"i": i})).id for i in range(5)]
assert JobQueue(jobs, None, None).drain(timeout=10) is True
assert {got(jobs.get(i)).state for i in ids} == {"done"}
# ------------------------------------------------- two processes, one file
def test_two_worker_processes_never_run_the_same_job_twice(tmp_path):
"""The claim the table-as-queue design rests on, tested the only way that
means anything: two separate `python -m pipeline.worker` processes draining
one file at once. Every job leaves a marker file named after itself and the
process that ran it; each job must have exactly one.
"""
db = tmp_path / "shared.db"
marks = tmp_path / "marks"
marks.mkdir()
jobs = Jobs(db)
for i in range(40):
jobs.enqueue("mp_mark", payload={"i": i})
env = dict(os.environ, PYTHONPATH=str(ROOT))
argv = [sys.executable, "-m", "pipeline.worker", "--db", str(db),
"--bootstrap", "pipeline.tests.isolated_handlers:marking",
"--bootstrap-arg", str(marks), "--workers", "cpu=2", "--until-idle"]
workers = [subprocess.Popen(argv, env=env, cwd=ROOT,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
for _ in range(2)]
for worker in workers:
out, _ = worker.communicate(timeout=120)
assert worker.returncode == 0, out.decode(errors="replace")[-2000:]
ran: dict[int, list[str]] = {}
for mark in marks.iterdir():
i, pid = mark.name.split("-")[:2]
ran.setdefault(int(i), []).append(pid)
assert sorted(ran) == list(range(40)), "every job ran"
doubled = {i: pids for i, pids in ran.items() if len(pids) > 1}
assert not doubled, f"and none ran twice: {doubled}"
assert {j.state for j in jobs.snapshot().recent} == {"done"}
+320
View File
@@ -0,0 +1,320 @@
"""Ending work that will not end itself, and how data moves between jobs.
Four claims, and they fail independently:
a deadline is honest `timeout` without `isolate` is refused at
registration rather than promising a kill a
thread cannot deliver.
a process can be killed an isolated job that never calls `check()` is
still over when its deadline passes, and its
row says so.
a chain reads, never writes `upstream()` is how B gets A's output — off A's
row, so B's own row was complete when written.
a fan-out is one call `fan_out` queues the children and the job that
reports on them, and an app never writes the
second one.
python3 -m pytest tests/test_isolation.py -q
"""
from __future__ import annotations
import asyncio
import os
import sys
import tempfile
import time
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from pipeline import NULL, Cancelled, JobQueue, Jobs, RunContext, job, spawn
from pipeline.models import Job
# Imported for the `@job` registrations, which the *parent* needs as much as
# the child does: the worker reads `isolate` and `timeout` off the spec before
# it decides to start a process at all. An app imports its handlers in every
# entry point for this reason; here that import is this line.
from pipeline.tests import isolated_handlers # noqa: F401
BOOTSTRAP = "pipeline.tests.isolated_handlers:bootstrap"
@job("t_child", "A child of a fan-out")
def t_child(config, store, *, context=NULL, i: int = 0) -> dict:
"""Queued by `fan_out` and never run in these tests — what is under test is
the shape of the rows it made, which is the part an app never writes."""
return {"i": i}
@pytest.fixture
def jobs():
with tempfile.TemporaryDirectory() as tmp:
yield Jobs(Path(tmp) / "t.db")
@pytest.fixture
def child_path(monkeypatch):
"""Put the folder holding `pipeline` where a child process can import it.
What an installed app gets for free and a checkout does not: the child is a
fresh interpreter, and the checkout is on *our* path only because pytest
put it there.
"""
root = Path(__file__).resolve().parents[2]
monkeypatch.setenv("PYTHONPATH", str(root))
def drain(queue: JobQueue, *, timeout: float = 20.0) -> None:
"""Run the worker until nothing is queued, running, or held by this one."""
async def go() -> None:
await queue.start()
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
snap = queue.jobs.snapshot()
if not snap.running and not snap.queued and not queue._running:
break
await asyncio.sleep(0.02)
await queue.stop()
asyncio.run(go())
def got(found: Job | None) -> Job:
assert found is not None
return found
# --------------------------------------------------- a deadline is honest
def test_a_timeout_without_isolation_is_refused():
"""The pipeline will not offer a kill it cannot perform.
A deadline over a job in a thread could only mark the row failed and walk
away: the work would carry on holding the core it was killed for holding,
and two things would intend to write one row. So the pair is refused where
it is declared, with the two ways out named.
"""
with pytest.raises(ValueError) as refused:
@job("t_threaded_deadline", timeout=5.0)
def handler(config, store, *, context=NULL):
return {}
said = str(refused.value)
assert "isolate=True" in said, "it names the way to get a real deadline"
assert "RunContext.spawn" in said, "and the cheaper way, for a command"
def test_isolation_without_a_timeout_is_fine():
"""Only the one combination is refused. An isolated job with no deadline
runs until it is done or killed by hand, which is every job the pipeline
has ever had."""
@job("t_isolated_no_deadline", isolate=True)
def handler(config, store, *, context=NULL):
return {}
from pipeline import JOBS
assert JOBS["t_isolated_no_deadline"].isolate
assert JOBS["t_isolated_no_deadline"].timeout == 0.0
def test_a_queue_with_no_bootstrap_says_so(jobs):
"""An isolated job needs its app rebuilt in the child, and a queue that
cannot do it fails the job with the reason rather than with a TypeError
from somewhere inside a subprocess."""
@job("t_needs_bootstrap", isolate=True)
def handler(config, store, *, context=NULL):
return {}
queued = got(jobs.enqueue("t_needs_bootstrap"))
drain(JobQueue(jobs, None, None)) # no bootstrap=
ended = got(jobs.get(queued.id))
assert ended.state == "failed"
assert "bootstrap" in ended.error, f"it names what is missing: {ended.error}"
# ------------------------------------------------ a process can be killed
def test_an_isolated_job_runs_somewhere_else_and_its_result_comes_back(jobs, child_path):
"""The happy path, and the proof that it was a real process.
Nothing was pickled to get there: the child was handed a database and a
row id and rebuilt the rest. Which is also why its progress is on the row
— it wrote it there itself, to the same table the snapshot reads.
"""
queued = got(jobs.enqueue("iso_ok", payload={"n": 21}))
drain(JobQueue(jobs, None, None, bootstrap=BOOTSTRAP))
ended = got(jobs.get(queued.id))
assert ended.state == "done", ended.error
assert ended.result["doubled"] == 42, "the handler's return travelled back"
assert ended.result["pid"] != os.getpid(), "and it ran in another process"
assert ended.message == "counting to 21", "its progress reached our table"
def test_an_isolated_job_that_raises_fails_with_what_it_said(jobs, child_path):
"""A failure across a process boundary reads like any other failure: the
row is `failed`, the error is the exception, and the retry bound applies
as it would to a handler that raised in a thread."""
queued = got(jobs.enqueue("iso_raises"))
drain(JobQueue(jobs, None, None, bootstrap=BOOTSTRAP))
ended = got(jobs.get(queued.id))
assert ended.state == "failed"
assert "the model said no" in ended.error, ended.error
def test_a_deadline_ends_a_job_that_never_looks_up(jobs, child_path):
"""The claim the whole feature exists for.
`iso_slow` sleeps for two minutes and never calls `check()`. In a thread
that is unkillable and the worker could only lie about having stopped it.
In a process of its own it is over in about a second, and the row says
why — which is what a person reading the job list needs.
"""
queued = got(jobs.enqueue("iso_slow"))
started = time.monotonic()
drain(JobQueue(jobs, None, None, bootstrap=BOOTSTRAP))
took = time.monotonic() - started
ended = got(jobs.get(queued.id))
assert ended.state == "failed", f"it ended: {ended.state}"
assert took < 15, f"and on its deadline, not in its own time: {took:.1f}s"
# ------------------------------------------------------ spawn, on its own
def test_spawn_kills_the_process_when_the_job_is_cancelled():
"""Kill, for a handler whose work is a command.
Cooperative in name only from the handler's side: it is blocked in
`spawn`, not in a loop with a `check()` in it, and the child is killed
outright. What makes that safe is that the thread was only sleeping.
"""
asked = [0]
def is_cancelled() -> bool:
asked[0] += 1
return asked[0] > 2 # pressed, a moment in
context = RunContext(is_cancelled=is_cancelled)
started = time.monotonic()
with pytest.raises(Cancelled):
context.spawn([sys.executable, "-c", "import time; time.sleep(120)"])
assert time.monotonic() - started < 10, "it did not wait the two minutes"
def test_spawn_raises_on_its_deadline_and_leaves_nothing_running():
"""The deadline, and that the process is gone after it — not merely
abandoned, which is the failure mode the whole thing is against."""
started = time.monotonic()
with pytest.raises(TimeoutError):
spawn([sys.executable, "-c", "import time; time.sleep(120)"], timeout=0.5)
assert time.monotonic() - started < 10
def test_spawn_hands_back_the_exit_code_rather_than_raising():
"""A non-zero exit is not an error here. Plenty of commands say something
useful with one, and only the handler knows which."""
done = spawn([sys.executable, "-c", "import sys; sys.exit(3)"])
assert done.returncode == 3
def test_spawn_survives_a_command_that_writes_more_than_a_pipe_holds():
"""Output goes to a file, not a `PIPE`, and this is why.
A child that fills the pipe buffer blocks on the write while the poll loop
blocks on the read, and neither side is wrong — the job simply never ends.
A megabyte is far past any platform's buffer.
"""
done = spawn([sys.executable, "-c", "print('x' * 1000000)"], timeout=60)
assert done.returncode == 0
assert len(done.stdout) >= 1000000
# ---------------------------------------------- a chain reads, never writes
def test_upstream_merges_what_the_parents_returned():
"""How B gets A's output: off A's row, not out of B's payload.
Which is the property worth having. B's row was complete when it was
written, so it can carry a dedup key, a retry can re-run it from itself,
and no worker can claim it between an earlier job's write and its commit.
"""
a = Job(id=1, kind="a", created_at="", result={"html": "<p>hi</p>", "code": 200})
b = Job(id=2, kind="b", created_at="", result={"code": 204})
context = RunContext(waited_for=(a, b))
assert context.upstream() == {"html": "<p>hi</p>", "code": 204}, "later parents win"
def test_upstream_is_empty_under_null_so_a_chained_handler_is_still_callable():
"""The one line that keeps a chained handler an ordinary function.
Every other handler in the pipeline can be called from a terminal. One
that read `waited_for[0].result` directly could not — it would raise
IndexError under `NULL` — so `upstream()` answers `{}` and the handler
falls back to the keyword it declared.
"""
assert NULL.upstream() == {}
@job("t_chained")
def handler(config, store, *, context=NULL, html: str = "") -> dict:
return {"html": html or context.upstream().get("html", "")}
assert handler(None, None, html="passed by hand") == {"html": "passed by hand"}
assert handler(None, None) == {"html": ""}, "and it does not blow up with neither"
# ----------------------------------------------------- a fan-out is one call
def test_fan_out_queues_the_children_and_the_job_that_reports_on_them(jobs):
"""One call, and the reporting job is not the app's to write.
The children get a run of their own inside the caller's cascade, and the
report waits on all of them with `after='done'` — the mode that exists
precisely so a job reporting that one of three failed is not held back by
that failure.
"""
queue = JobQueue(jobs, None, None)
parent = got(jobs.enqueue("t_parent"))
context = queue._context(parent, [], [])
children = context.fan_out(
"t_child", [{"i": 1}, {"i": 2}, {"i": 3}], then="mod:fn")
assert len(children) == 3, "it answers with the children's ids"
queued = jobs.snapshot().queued
report = [j for j in queued if j.kind == "advance"]
assert len(report) == 1, "and queued exactly one job to report on them"
assert sorted(report[0].depends_on) == sorted(children)
assert report[0].after == "done"
assert report[0].payload["callback"] == "mod:fn"
assert {j.task_id for j in queued if j.id in children} == {report[0].task_id}, \
"children and report are one run"
def test_fan_out_queues_nothing_under_null_and_says_so():
"""A handler called directly is being run by somebody who is standing
there, and the useful answer is to do its own work and report that nothing
was queued behind it — the same answer `task` gives, for the same reason."""
assert NULL.fan_out("t_child", [{"i": 1}]) == ()
def test_a_fan_out_carries_what_the_caller_puts_on_it(jobs):
"""`carries` goes *around* the children to the callback: static input from
whoever asked, as against `Tally.results`, which is what the children
themselves returned."""
queue = JobQueue(jobs, None, None)
parent = got(jobs.enqueue("t_parent"))
context = queue._context(parent, [], [])
context.fan_out("t_child", [{"i": 1}], then="mod:fn", carries={"since": "monday"})
report = [j for j in jobs.snapshot().queued if j.kind == "advance"]
assert report[0].payload["payload"] == {"since": "monday"}
+134
View File
@@ -0,0 +1,134 @@
"""The notice log: said once, kept until cleared, forgotten when it stops.
A notice used to be a line that faded in twelve seconds in whichever tab was
open, and was raised as often as not while none was. These hold the three
things the table is for: a repeat is the same row and not news, a cleared row
makes the next one news again, and what nobody clears goes when it has stopped
being said rather than when it was first said.
python -m pytest pipeline/tests/test_notice_log.py -q
"""
from __future__ import annotations
import sqlite3
import sys
import tempfile
from contextlib import closing
from datetime import UTC, datetime, timedelta
from pathlib import Path
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from pipeline import NULL, JobQueue, Jobs, Notice, RunContext, job
from pipeline.web import connect
@job("n_refused", "Meeting the same refusal as last time")
def n_refused(config, store, *, context: RunContext = NULL) -> dict:
context.notice(Notice(code="agent.refused", key="Worm hits npm",
text="No abstract: Worm hits npm", fix="Nothing to do."))
return {}
@pytest.fixture
def jobs():
with tempfile.TemporaryDirectory() as tmp:
yield Jobs(Path(tmp) / "n.db")
def refusal(text: str = "No abstract: Worm hits npm") -> Notice:
return Notice(code="agent.refused", key="Worm hits npm", text=text,
where="the tagging lines in the server log")
def test_the_same_notice_again_is_the_same_row_with_a_count(jobs):
first = jobs.record_notice(refusal())
again = jobs.record_notice(refusal("No abstract, again: Worm hits npm"))
other = jobs.record_notice(Notice(code="agent.refused", key="Other", text="x"))
assert (first.count, again.count) == (1, 2), "one is news, two is a repeat"
assert again.id == first.id and other.id != first.id, "keyed by code and key"
assert again.first_at == first.first_at and again.last_at >= first.last_at
assert again.text.startswith("No abstract, again"), "the newer wording is kept"
assert again.where == first.where, "and `where` survives its SQL column name"
assert [n.key for n in jobs.notices()] == ["Other", "Worm hits npm"]
def test_clearing_one_makes_the_next_occurrence_news_again(jobs):
said = jobs.record_notice(refusal())
assert jobs.clear_notice(said.id)
assert not jobs.clear_notice(said.id), "gone already is an answer, not an error"
assert jobs.record_notice(refusal()).count == 1
def test_a_notice_is_forgotten_when_it_stops_being_said_not_when_it_began(jobs):
"""By `last_at`. A feed that has failed nightly for a month is an old row
that was true an hour ago, and dropping it would make tonight's failure
read as the first."""
chronic = jobs.record_notice(Notice(code="source.failed", key="rss", text="x"))
over = jobs.record_notice(Notice(code="source.failed", key="old", text="y"))
month = (datetime.now(UTC) - timedelta(days=30)).isoformat()
with closing(sqlite3.connect(jobs.path)) as conn, conn:
conn.execute("UPDATE notices SET first_at=? WHERE id=?", (month, chronic.id))
conn.execute("UPDATE notices SET first_at=?, last_at=? WHERE id=?",
(month, month, over.id))
assert jobs.purge_notices(14 * 86400) == 1
assert [n.key for n in jobs.notices()] == ["rss"]
def test_a_job_that_repeats_itself_is_heard_as_a_repeat(jobs):
"""The whole path. Every notice still reaches the listener — what
interrupts a person is the interface's business — and the second carries
the count that says it is nothing new."""
heard: list = []
async def on_notice(notice) -> None:
heard.append(notice)
for _ in range(2):
jobs.enqueue("n_refused")
JobQueue(jobs, None, None, on_notice=on_notice).drain(timeout=10)
assert [n.count for n in heard] == [1, 2]
assert len(jobs.notices()) == 1
def test_a_log_that_cannot_be_written_does_not_cost_the_notice(jobs, monkeypatch):
heard: list = []
async def on_notice(notice) -> None:
heard.append(notice)
def broken(notice):
raise sqlite3.OperationalError("database is locked")
monkeypatch.setattr(jobs, "record_notice", broken)
done = jobs.enqueue("n_refused")
JobQueue(jobs, None, None, on_notice=on_notice).drain(timeout=10)
assert [n.code for n in heard] == ["agent.refused"]
assert jobs.get(done.id).state == "done", "and the job that said it is unharmed"
def test_the_log_is_read_and_cleared_over_http(jobs):
app = FastAPI()
connect(app, jobs, None, None)
jobs.record_notice(refusal())
jobs.record_notice(Notice(code="source.failed", key="rss", text="rss failed"))
with TestClient(app) as client:
listed = client.get("/api/notices").json()
assert [n["code"] for n in listed] == ["source.failed", "agent.refused"]
assert listed[1]["where"] == "the tagging lines in the server log"
assert client.delete(f"/api/notices/{listed[0]['id']}").json() == {"cleared": True}
assert client.delete(f"/api/notices/{listed[0]['id']}").json() == {"cleared": False}
assert client.delete("/api/notices").json() == {"cleared": 1}
assert client.get("/api/notices").json() == []
+355
View File
@@ -0,0 +1,355 @@
"""Waiting to be tried again, without anything being alive while it waits.
The claim under test is that a job between attempts costs a row and nothing
else. No worker, no thread, no lease, no timer — so the tests that matter here
are the ones that ask what *else* the queue can do while a job waits, and what
survives the process that scheduled the wait.
Each of the four properties step 12 asks for is pinned on its own, because
they fail independently: a bound that is off by one still waits correctly, a
`run_after` that never reaches the table still retries in one process, and a
Kill that cannot reach a waiting row is invisible until somebody presses it.
python3 -m pytest tests/test_retries.py -q
"""
from __future__ import annotations
import asyncio
import sqlite3
import sys
import tempfile
import time
from contextlib import closing
from datetime import UTC, datetime, timedelta
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from pipeline.models import Job
from pipeline import NULL, JobQueue, Jobs, Retry, job
@pytest.fixture
def jobs():
"""The pipeline's table on a file of its own. No app, no store — nothing
here needs one, which is the point of the table owning its own schema."""
with tempfile.TemporaryDirectory() as tmp:
yield Jobs(Path(tmp) / "t.db")
def made(queued: Job | None) -> Job:
"""The job an enqueue made. `enqueue` answers None for a killed run, which
in these tests nothing is."""
assert queued is not None
return queued
def got(found: Job | None) -> Job:
"""The job a read must have found — a None here is a test failure."""
assert found is not None
return found
def lease(jobs: Jobs, job_id: int) -> tuple[str | None, str | None]:
"""The `started_at` and `seen_at` of one job: whether anybody has it."""
with closing(sqlite3.connect(jobs.path)) as conn:
row = conn.execute(
"SELECT started_at, seen_at FROM jobs WHERE id=?", (job_id,)).fetchone()
return row[0], row[1]
def bring_forward(jobs: Jobs, job_id: int) -> None:
"""Move a waiting job's next attempt into the past, as if its wait were
over. What the clock does, done in one statement so a test of an hour's
backoff does not take an hour."""
with closing(sqlite3.connect(jobs.path)) as conn, conn:
conn.execute("UPDATE jobs SET run_after=? WHERE id=?",
((datetime.now(UTC) - timedelta(hours=1)).isoformat(), job_id))
def drain(queue: JobQueue, *, timeout: float = 5.0) -> None:
"""Run the worker until the table is empty and this worker holds nothing.
Both halves, because a job's last acts belong to its handler thread and
between them the table reads as idle while it is not. A job waiting for its
next attempt keeps this going, since it is still `queued` — which is the
behaviour the retry tests are leaning on.
"""
async def go() -> None:
await queue.start()
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
snap = queue.jobs.snapshot()
if not snap.running and not snap.queued and not queue._running:
break
await asyncio.sleep(0.01)
await queue.stop()
asyncio.run(go())
# ------------------------------------------------------- not holding a place
def test_a_failed_attempt_goes_back_to_the_queue_with_a_time_on_it(jobs):
"""The resting state of a retry, and everything that is true of it."""
queued = made(jobs.enqueue("r_wait", max_attempts=3, retry_delay=600))
assert got(jobs.claim({})).attempt == 1, "claim counts the attempt"
again = got(jobs.retry(queued.id, error="RuntimeError: the feed was down"))
assert again.state == "queued", "it is in the queue, not failed"
assert again.waiting and not again.exhausted
assert again.error == "RuntimeError: the feed was down", "it says why it waits"
assert lease(jobs, queued.id) == (None, None), "nobody holds it"
assert jobs.running() == [], "and nothing is running it"
assert [j.id for j in jobs.waiting()] == [queued.id]
assert [j.id for j in jobs.blocked()] == [queued.id]
def test_a_waiting_job_does_not_occupy_a_worker(jobs):
"""The whole of why the wait is a column. One cpu worker, one job waiting
ten minutes for its next attempt, and the worker is free the entire time —
a handler that slept would have held it and stopped the queue dead."""
waiting = made(jobs.enqueue("r_slot", max_attempts=2, retry_delay=600))
assert jobs.claim({"cpu": 1}) is not None
jobs.retry(waiting.id, error="not this time")
other = made(jobs.enqueue("r_other"))
assert got(jobs.claim({"cpu": 1})).id == other.id, "the slot was free"
assert got(jobs.get(waiting.id)).state == "queued"
def test_a_job_queued_for_later_is_not_claimed_before_its_time(jobs):
"""`delay` on the enqueue itself: the durable form of "in ten minutes"."""
later = made(jobs.enqueue("r_later", delay=600))
assert later.waiting
assert jobs.claim({}) is None
bring_forward(jobs, later.id)
assert got(jobs.claim({})).id == later.id
def test_what_waits_on_a_retrying_job_goes_on_waiting(jobs):
"""A retry is still live work, so a dependent is held rather than settled.
The alternative would be the worse half of both: a batch that is coming
back in a minute counted among the ones that did not land, and a final job
reporting on a task that is not over.
"""
batch = made(jobs.enqueue("r_batch", max_attempts=2, retry_delay=600))
final = made(jobs.enqueue("r_final", depends_on=[batch.id], after="done"))
jobs.claim({})
jobs.retry(batch.id, error="again in a minute")
assert [j.id for j in jobs.blocked()] == [batch.id, final.id]
bring_forward(jobs, batch.id)
assert got(jobs.claim({})).id == batch.id, "the batch goes first"
jobs.finish(batch.id, "done")
assert got(jobs.claim({})).id == final.id
# --------------------------------------------------------------- the bounds
def test_the_bound_counts_the_first_attempt(jobs):
"""`max_attempts=3` is three runs of the handler, not three retries."""
runs = []
@job("r_bounded", "Bounded", max_attempts=3)
def handler(config, store, *, context=NULL):
runs.append(1)
raise RuntimeError("this was never going to work")
queue = JobQueue(jobs, None, None)
queued = made(queue.enqueue("r_bounded"))
drain(queue)
ended = got(jobs.get(queued.id))
assert len(runs) == 3, "it ran exactly as many times as it was allowed"
assert ended.state == "failed" and ended.attempt == 3
assert ended.exhausted, "and says the policy ran out rather than merely failing"
def test_a_job_with_no_policy_fails_at_the_first_exception(jobs):
"""The default is one attempt, which is every job queued before any of
this existed. Exhausted is not what that is: nothing was used up."""
runs = []
@job("r_unbounded", "Unbounded")
def handler(config, store, *, context=NULL):
runs.append(1)
raise RuntimeError("no")
queue = JobQueue(jobs, None, None)
queued = made(queue.enqueue("r_unbounded"))
drain(queue)
ended = got(jobs.get(queued.id))
assert len(runs) == 1
assert ended.state == "failed" and not ended.exhausted
def test_an_attempt_that_lands_stops_the_retrying(jobs):
"""The case the bound exists for: the second go works, and the job is done
rather than carrying a memory of having failed once."""
runs = []
@job("r_second_time", "Second time", max_attempts=4)
def handler(config, store, *, context=NULL):
runs.append(1)
if len(runs) < 2:
raise RuntimeError("the first one never works")
return {"runs": len(runs)}
queue = JobQueue(jobs, None, None)
queued = made(queue.enqueue("r_second_time"))
drain(queue)
ended = got(jobs.get(queued.id))
assert ended.state == "done" and ended.result == {"runs": 2}
assert ended.attempt == 2 and not ended.exhausted
def test_the_wait_lengthens_with_the_backoff(jobs):
"""Doubling, read off the two stamps rather than waited out."""
queued = made(jobs.enqueue("r_backoff", max_attempts=4,
retry_delay=100, retry_backoff=2.0))
waits = []
for _ in range(3):
assert jobs.claim({}) is not None
again = got(jobs.retry(queued.id, error="again"))
waits.append(
datetime.fromisoformat(again.run_after) - datetime.now(UTC))
bring_forward(jobs, queued.id)
seconds = [w.total_seconds() for w in waits]
assert [round(s / 100) for s in seconds] == [1, 2, 4]
def test_a_server_hint_replaces_the_backoff(jobs):
"""Honouring what the other end said, rather than guessing at it.
The bound is not part of the bargain: a hint moves the next attempt, it
does not add one.
"""
queued = made(jobs.enqueue("r_hinted", max_attempts=3, retry_delay=600))
jobs.claim({})
again = got(jobs.retry(queued.id, error="429", delay=5))
waited = datetime.fromisoformat(again.run_after) - datetime.now(UTC)
assert 0 < waited.total_seconds() <= 5
def test_a_handler_can_name_its_own_wait(jobs):
"""`Retry(after=...)` from inside the handler, which is where a
`Retry-After` header is actually read."""
@job("r_asked", "Asked to come back", max_attempts=3, retry_delay=600)
def handler(config, store, *, context=NULL):
raise Retry("rate limited", after=7)
queue = JobQueue(jobs, None, None)
queued = made(queue.enqueue("r_asked"))
async def one_pass() -> None:
await queue.start()
deadline = time.monotonic() + 5
while time.monotonic() < deadline:
if got(jobs.get(queued.id)).waiting:
break
await asyncio.sleep(0.01)
await queue.stop()
asyncio.run(one_pass())
waiting = got(jobs.get(queued.id))
assert waiting.waiting and waiting.attempt == 1
assert "rate limited" in waiting.error
waited = datetime.fromisoformat(waiting.run_after) - datetime.now(UTC)
assert 0 < waited.total_seconds() <= 7, "the handler's number, not the policy's"
def test_asking_to_come_back_does_not_extend_the_bound(jobs):
"""A handler cannot grant itself a policy nobody declared."""
runs = []
@job("r_asked_once", "Asked once")
def handler(config, store, *, context=NULL):
runs.append(1)
raise Retry("come back in a second", after=0)
queue = JobQueue(jobs, None, None)
queued = made(queue.enqueue("r_asked_once"))
drain(queue)
assert len(runs) == 1
assert got(jobs.get(queued.id)).state == "failed"
# ------------------------------------------------------------- across a stop
def test_the_next_attempt_survives_the_process_that_scheduled_it(jobs):
"""The wait is a column, so it outlives everything that is not the file.
A second `Jobs` on the same path is what another process sees, and what
this one sees after a restart: a queued row it will not take yet, and takes
the moment its time comes round. Nothing was handed over, because there
was nothing alive to hand.
"""
queued = made(jobs.enqueue("r_restart", max_attempts=3, retry_delay=600))
jobs.claim({})
jobs.retry(queued.id, error="the feed was down")
restarted = Jobs(jobs.path)
assert restarted.claim({}) is None, "still waiting, on a table that never saw it fail"
assert [j.id for j in restarted.waiting()] == [queued.id]
bring_forward(restarted, queued.id)
taken = got(restarted.claim({}))
assert taken.id == queued.id
assert taken.attempt == 2, "and it is the second attempt, not the first again"
# ------------------------------------------------------------- cancellation
def test_cancelling_a_job_that_is_waiting_to_be_retried_ends_it(jobs):
"""Kill on a waiting row. It is queued, so it ends here and now — there is
no worker to ask and nothing part-way through."""
queued = made(jobs.enqueue("r_killable", max_attempts=3, retry_delay=600))
jobs.claim({})
jobs.retry(queued.id, error="not yet")
ended = got(jobs.request_cancel(queued.id))
assert ended.state == "cancelled"
assert jobs.waiting() == []
def test_killing_the_run_takes_a_job_waiting_to_be_retried_with_it(jobs):
"""The same for the task and cascade buttons: an hour's wait is not an
hour during which a run cannot be stopped."""
cascade = jobs.new_run("cascade", "r_run")
task = jobs.new_run("task", "r_task")
queued = made(jobs.enqueue("r_in_a_run", max_attempts=3, retry_delay=600,
cascade_id=cascade, task_id=task))
jobs.claim({})
jobs.retry(queued.id, error="not yet")
assert jobs.cancel_run(task) == 1
assert got(jobs.get(queued.id)).state == "cancelled"
def test_a_kill_during_the_attempt_is_not_undone_by_the_retry(jobs):
"""The one ordering that would make Kill a lie.
Press it while the attempt runs, the attempt then fails, and a retry that
only looked at the bound would put the job straight back in the queue —
a button that stopped nothing and said it had.
"""
queued = made(jobs.enqueue("r_killed_mid", max_attempts=3, retry_delay=0))
jobs.claim({})
jobs.request_cancel(queued.id)
ended = got(jobs.retry(queued.id, error="it failed as well"))
assert ended.state == "cancelled"
assert not ended.waiting
+701
View File
@@ -0,0 +1,701 @@
"""The scheduler: the five wordings, the table, the firing, and the routes.
python -m pytest pipeline/tests/test_schedule.py -q
Three things here are worth knowing before changing anything.
The daylight-saving tests are the reason `next_after` takes a zone at all, and
they are written around a real transition rather than a synthetic one, because
the bug they guard is precisely that `+ timedelta(days=1)` looks correct.
The firing tests call `JobQueue._fire_due` directly. It is a private, and that
is deliberate: driving it through `start()` would mean asserting on a loop that
ticks on a clock, and a test that sleeps to make a scheduler fire is a test
that fails on a busy machine. One test does start a real queue, to prove the
tick is wired into the loop at all.
`SCHEDULES` is a module-level registry, like `JOBS`, so anything here that
declares into it does so through the `declared` fixture, which puts it back.
"""
from __future__ import annotations
import os
import subprocess
import sys
import tempfile
from datetime import UTC, datetime, timedelta
from pathlib import Path
from zoneinfo import ZoneInfo
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from pipeline import NULL, JobQueue, Jobs, RunContext, job
from pipeline.schedule import SCHEDULES, BadCadence, Cadence, declare
ROOT = Path(__file__).resolve().parents[2]
COPENHAGEN = ZoneInfo("Europe/Copenhagen")
@job("s_tick", "A scheduled tick")
def s_tick(config, store, *, context: RunContext = NULL, note: str = "") -> dict:
store.append(note)
return {"note": note}
@job("s_solo", "One at a time", solo=True)
def s_solo(config, store, *, context: RunContext = NULL) -> dict:
return {}
@job("s_needs", "Needing a payload")
def s_needs(config, store, *, context: RunContext = NULL, target: str) -> dict:
return {"target": target}
@pytest.fixture
def jobs():
with tempfile.TemporaryDirectory() as tmp:
yield Jobs(Path(tmp) / "s.db")
@pytest.fixture
def queue(jobs):
"""A queue that is never started: the tests below fire it by hand.
`tz="UTC"` so a cadence's next run is the same number wherever this runs.
"""
return JobQueue(jobs, {}, [], tz="UTC")
@pytest.fixture
def declared():
"""Borrow the declaration registry and give it back.
`SCHEDULES` is global, like `JOBS`, and a test that left something in it
would give every later test in the session a schedule it never asked for.
"""
saved = dict(SCHEDULES)
SCHEDULES.clear()
yield SCHEDULES
SCHEDULES.clear()
SCHEDULES.update(saved)
def ago(seconds: float) -> str:
return (datetime.now(UTC) - timedelta(seconds=seconds)).isoformat()
# ------------------------------------------------------------ the grammar
WORDINGS = [
("every 15m", "every 15m"),
("every 6h", "every 6h"),
("every 2d", "every 2d"),
("daily at 06:00", "daily at 06:00"),
("weekly mon 07:00", "weekly mon 07:00"),
# Written how people write them, stored how the table stores them.
("weekly monday 07:00", "weekly mon 07:00"),
("Weekly Sunday 23:59", "weekly sun 23:59"),
("EVERY 15 M", "every 15m"),
(" daily at 6:05 ", "daily at 06:05"),
("every 1d", "every 1d"),
]
@pytest.mark.parametrize("typed,canonical", WORDINGS)
def test_every_wording_parses_and_comes_back_canonical(typed, canonical):
"""Five forms, one spelling each.
The round trip is the point: the table stores `str(cadence)`, so two
people writing the same schedule two ways end up with one row that reads
the same to both of them.
"""
cadence = Cadence.parse(typed)
assert str(cadence) == canonical
assert str(Cadence.parse(canonical)) == canonical
assert Cadence.parse(typed) == Cadence.parse(canonical)
def test_the_units_are_what_they_say():
assert Cadence.parse("every 15m").seconds == 900
assert Cadence.parse("every 6h").seconds == 6 * 3600
assert Cadence.parse("every 2d").seconds == 2 * 86400
# A clock cadence has no fixed length, and says so rather than guessing.
assert Cadence.parse("daily at 06:00").seconds == 0
@pytest.mark.parametrize("wording", [
"sometimes", "", "cron 0 6 * * *", "every 15", "every m", "every 0h",
"every 15s", "daily 06:00", "daily at 25:00", "daily at 06:60",
"weekly 07:00", "weekly funday 07:00", "every 15m please",
])
def test_a_wording_outside_the_five_is_refused_with_all_five(wording):
"""And the refusal is the sentence a person is shown.
It goes out of the API as `{"detail": ...}` and into a toast, so it has to
say what to type instead rather than merely that this was wrong.
"""
with pytest.raises(ValueError) as refused:
Cadence.parse(wording)
said = str(refused.value)
assert isinstance(refused.value, BadCadence)
for form in ("every 15m", "every 6h", "every 2d",
"daily at 06:00", "weekly mon 07:00"):
assert form in said
# ------------------------------------------------------------- next_after
def test_next_after_is_strictly_after():
"""A schedule fired at its own time must not find itself due again on the
same instant, which would be one firing per tick for ever."""
at_six = datetime(2025, 6, 1, 6, 0, tzinfo=COPENHAGEN)
when = Cadence.parse("daily at 06:00").next_after(at_six, COPENHAGEN)
assert when.astimezone(COPENHAGEN) == at_six + timedelta(days=1)
def test_next_after_always_comes_back_in_utc():
"""Because it is written straight into the table, where every other time
is UTC and the column is compared as text."""
when = Cadence.parse("daily at 06:00").next_after(
datetime(2025, 6, 1, 12, 0, tzinfo=COPENHAGEN), COPENHAGEN)
assert when.tzinfo is UTC
assert when.isoformat().endswith("+00:00")
def test_weekly_lands_on_the_day_it_names():
monday = Cadence.parse("weekly mon 07:00")
# A Wednesday: the next Monday is five days away, not tomorrow.
from_wed = monday.next_after(datetime(2025, 6, 4, 9, 0, tzinfo=COPENHAGEN),
COPENHAGEN).astimezone(COPENHAGEN)
assert from_wed.weekday() == 0 and (from_wed.hour, from_wed.minute) == (7, 0)
assert from_wed.date() == datetime(2025, 6, 9).date()
# Monday morning before seven: today, not next week.
from_mon = monday.next_after(datetime(2025, 6, 9, 6, 0, tzinfo=COPENHAGEN),
COPENHAGEN).astimezone(COPENHAGEN)
assert from_mon.date() == datetime(2025, 6, 9).date()
def test_daily_keeps_its_wall_clock_across_both_dst_changes():
"""06:00 is 06:00 on both sides, which is the whole reason for the zone.
Spring forward is a 23-hour day and autumn back is a 25-hour one, so the
real elapsed time between two runs is not 24 hours — and it must not be,
because what the wording promised is a time of day.
"""
daily = Cadence.parse("daily at 06:00")
# Europe/Copenhagen, 2025: forward on 30 March, back on 26 October.
forward = daily.next_after(
datetime(2025, 3, 29, 6, 30, tzinfo=COPENHAGEN), COPENHAGEN)
local = forward.astimezone(COPENHAGEN)
assert (local.hour, local.minute) == (6, 0)
assert local.date() == datetime(2025, 3, 30).date()
# 05:30Z to 04:00Z: 22h30m of real time for a 23h30m step of the clock.
assert forward - datetime(2025, 3, 29, 6, 30, tzinfo=COPENHAGEN) \
== timedelta(hours=22, minutes=30)
back = daily.next_after(
datetime(2025, 10, 25, 6, 30, tzinfo=COPENHAGEN), COPENHAGEN)
local = back.astimezone(COPENHAGEN)
assert (local.hour, local.minute) == (6, 0)
assert local.date() == datetime(2025, 10, 26).date()
assert back - datetime(2025, 10, 25, 6, 30, tzinfo=COPENHAGEN) \
== timedelta(hours=24, minutes=30)
def test_weekly_keeps_its_wall_clock_across_a_dst_change():
when = Cadence.parse("weekly sun 07:00").next_after(
datetime(2025, 3, 26, 9, 0, tzinfo=COPENHAGEN), COPENHAGEN)
local = when.astimezone(COPENHAGEN)
assert local.date() == datetime(2025, 3, 30).date() # the change itself
assert (local.hour, local.minute) == (7, 0)
def test_an_interval_is_elapsed_time_and_ignores_the_change():
"""The other half of the promise, and the reason both wordings exist.
`every 24h` across a spring-forward is 24 real hours and therefore lands
an hour later on the clock. A person who wanted 06:00 wrote `daily`.
"""
start = datetime(2025, 3, 29, 6, 30, tzinfo=COPENHAGEN)
when = Cadence.parse("every 24h").next_after(start, COPENHAGEN)
assert when - start == timedelta(hours=24)
assert when.astimezone(COPENHAGEN).hour == 7
def test_a_naive_moment_is_read_as_utc():
"""Rather than raising. Every stamp in the table is UTC, so a caller that
lost the tzinfo on the way is very nearly certainly holding one."""
naive = datetime(2025, 6, 1, 12, 0)
assert Cadence.parse("every 1h").next_after(naive) \
== datetime(2025, 6, 1, 13, 0, tzinfo=UTC)
# ---------------------------------------------------------------- the table
def test_the_table_holds_a_schedule_through_its_life(jobs):
row = jobs.add_schedule("nightly", "s_tick", "daily at 06:00",
payload={"note": "hello"}, label="A nightly tick",
next_run="2030-01-01T06:00:00+00:00")
assert row is not None
assert (row.name, row.kind, row.cadence) == ("nightly", "s_tick", "daily at 06:00")
assert row.payload == {"note": "hello"} and row.enabled is True
assert row.last_job_id == 0 and row.last_run == "" and row.last_state == ""
assert [s.name for s in jobs.schedules()] == ["nightly"]
assert jobs.schedule(row.id) == row
edited = jobs.update_schedule(row.id, cadence="every 15m", enabled=False,
payload={"note": "other"})
assert edited.cadence == "every 15m" and edited.enabled is False
assert edited.payload == {"note": "other"}
# Untouched columns stay as they were.
assert edited.name == "nightly" and edited.label == "A nightly tick"
assert jobs.delete_schedule(row.id) is True
assert jobs.delete_schedule(row.id) is False # two tabs; not an error
assert jobs.schedules() == []
def test_a_name_is_taken_once(jobs):
assert jobs.add_schedule("one", "s_tick", "every 15m") is not None
assert jobs.add_schedule("one", "s_tick", "every 6h", only_if_new=True) is None
assert jobs.schedules()[0].cadence == "every 15m"
def test_schedules_carry_the_state_of_their_last_job(jobs):
row = jobs.add_schedule("one", "s_tick", "every 15m")
queued = jobs.enqueue("s_tick")
jobs.mark_fired(row.id, queued.id, "2030-01-01T00:00:00+00:00")
assert jobs.schedules()[0].last_state == "queued"
jobs.claim({"cpu": 1})
jobs.finish(queued.id, "failed", error="no")
assert jobs.schedules()[0].last_state == "failed"
# And the time it fired, which is not the time it is next due.
listed = jobs.schedules()[0]
assert listed.last_run and listed.next_run == "2030-01-01T00:00:00+00:00"
def test_due_schedules_passes_over_what_is_off_or_not_yet(jobs):
jobs.add_schedule("due", "s_tick", "every 15m", next_run=ago(60))
jobs.add_schedule("off", "s_tick", "every 15m", next_run=ago(60), enabled=False)
jobs.add_schedule("later", "s_tick", "every 15m",
next_run="2099-01-01T00:00:00+00:00")
jobs.add_schedule("never", "s_tick", "every 15m") # no time at all
due = jobs.due_schedules(datetime.now(UTC).isoformat())
assert [s.name for s in due] == ["due"]
# --------------------------------------------------------------- the firing
def test_the_tick_fires_a_due_row_and_moves_it_on(queue, jobs):
row = jobs.add_schedule("regular", "s_tick", "every 15m",
payload={"note": "scheduled"}, next_run=ago(30))
queue._fire_due()
queued = jobs.snapshot().queued
assert [j.kind for j in queued] == ["s_tick"]
assert queued[0].payload == {"note": "scheduled"}
after = jobs.schedule(row.id)
assert after.last_job_id == queued[0].id
assert after.last_run
# Fifteen minutes from now, not from the moment it was due.
assert after.next_run > datetime.now(UTC).isoformat()
assert after.next_run < (datetime.now(UTC) + timedelta(minutes=16)).isoformat()
def test_a_schedule_does_not_pile_up_behind_its_own_job(queue, jobs):
"""The job from the last firing is still queued, so this one is skipped.
Skipped and *moved on*, not left due: a schedule whose job takes longer
than its cadence would otherwise fire on every tick for as long as the job
ran, and the queue would fill with copies of work already under way.
"""
row = jobs.add_schedule("regular", "s_tick", "every 15m", next_run=ago(30))
queue._fire_due()
first = jobs.schedule(row.id).last_job_id
jobs.update_schedule(row.id, next_run=ago(1)) # due again
queue._fire_due()
assert len(jobs.snapshot().queued) == 1
again = jobs.schedule(row.id)
assert again.last_job_id == first # nothing new was queued
assert again.next_run > datetime.now(UTC).isoformat()
def test_a_solo_kind_already_underway_holds_its_schedule_back(queue, jobs):
"""The other way a schedule collides: not with itself, with somebody else.
A button, a second schedule or a cascade started one, and `enqueue` would
hand that job back — so the schedule would record work it did not ask for
as its own firing.
"""
pressed = jobs.enqueue("s_solo", dedup_key="s_solo")
row = jobs.add_schedule("solo", "s_solo", "every 15m", next_run=ago(30))
queue._fire_due()
after = jobs.schedule(row.id)
assert after.last_job_id == 0 # it never fired
assert after.next_run > datetime.now(UTC).isoformat()
assert [j.id for j in jobs.snapshot().queued] == [pressed.id]
def test_downtime_is_caught_up_once_and_not_once_per_slot(queue, jobs):
"""Three days off an hourly schedule is one job, not seventy-two.
The row carries one `next_run`, and a firing sets it from *now* rather
than stepping through what was missed — so there is nowhere for a backlog
to accumulate even in principle.
"""
row = jobs.add_schedule("hourly", "s_tick", "every 1h", next_run=ago(3 * 86400))
queue._fire_due()
assert len(jobs.snapshot().queued) == 1
after = jobs.schedule(row.id)
assert after.next_run > datetime.now(UTC).isoformat()
assert after.next_run < (datetime.now(UTC) + timedelta(minutes=61)).isoformat()
def test_an_unknown_kind_never_fires_and_keeps_its_place(queue, jobs):
"""A handler that went away in a deploy. The row stays exactly as it is —
time included — so reinstalling the plugin runs it rather than starting
another cadence from scratch."""
row = jobs.add_schedule("gone", "s_vanished", "every 15m", next_run=ago(30))
queue._fire_due()
assert jobs.snapshot().queued == []
assert jobs.schedule(row.id).next_run == row.next_run
queue._fire_due() # and it says so once
assert queue._said_about == {(row.id, "kind")}
def test_a_cadence_the_table_cannot_parse_never_fires(queue, jobs):
"""Only reachable by editing the file by hand, since every route parses
first. It must not take the tick down with it."""
jobs.add_schedule("bent", "s_tick", "every 15m", next_run=ago(30))
jobs._write("UPDATE schedules SET cadence='sometimes'")
queue._fire_due()
assert jobs.snapshot().queued == []
assert jobs.schedules()[0].cadence == "sometimes" # left to be fixed
def test_a_disabled_row_is_not_fired(queue, jobs):
jobs.add_schedule("off", "s_tick", "every 15m", next_run=ago(30), enabled=False)
queue._fire_due()
assert jobs.snapshot().queued == []
def test_run_now_fires_without_moving_the_next_run(queue, jobs):
row = jobs.add_schedule("nightly", "s_tick", "daily at 06:00",
next_run="2099-01-01T06:00:00+00:00")
started = queue.run_schedule(row.id)
assert started is not None and started.kind == "s_tick"
after = jobs.schedule(row.id)
assert after.next_run == "2099-01-01T06:00:00+00:00"
# But it did just run, and that is the job it queued — so the next tick
# will not start a second one while this is still going.
assert after.last_job_id == started.id and after.last_run
# ------------------------------------------------------------ reconciliation
def test_declared_defaults_get_a_row_on_start(declared, jobs):
declare("nightly tick", "s_tick", "daily at 06:00", {"note": "declared"},
label="A tick, nightly")
JobQueue(jobs, {}, [], tz="UTC")._reconcile_schedules()
row = jobs.schedules()[0]
assert (row.name, row.kind, row.cadence) == ("nightly tick", "s_tick",
"daily at 06:00")
assert row.payload == {"note": "declared"} and row.label == "A tick, nightly"
assert row.next_run # and it knows when it is next
def test_reconciliation_keeps_what_the_user_edited(declared, jobs):
"""The whole of what a declaration means: a starting point, not a setting.
Switching a default off, or slowing it down, has to survive the next
deploy — otherwise the only way to be rid of one is to edit the code and
redeploy, which is not a thing the person running the app can do.
"""
declare("nightly tick", "s_tick", "daily at 06:00")
queue = JobQueue(jobs, {}, [], tz="UTC")
queue._reconcile_schedules()
row = jobs.schedules()[0]
jobs.update_schedule(row.id, cadence="every 2d", enabled=False,
payload={"note": "mine"})
queue._reconcile_schedules() # a restart
again = jobs.schedules()[0]
assert again.id == row.id
assert again.cadence == "every 2d"
assert again.enabled is False
assert again.payload == {"note": "mine"}
def test_a_declaration_that_goes_away_leaves_its_row(declared, jobs):
"""Nothing here deletes. A deploy that silently forgot a schedule somebody
was relying on is the failure this avoids; an orphan says `unknown kind`
and waits to be deleted deliberately."""
declare("nightly tick", "s_tick", "daily at 06:00")
queue = JobQueue(jobs, {}, [], tz="UTC")
queue._reconcile_schedules()
declared.clear()
queue._reconcile_schedules()
assert [s.name for s in jobs.schedules()] == ["nightly tick"]
def test_declaring_one_name_twice_differently_is_refused(declared):
declare("nightly", "s_tick", "daily at 06:00")
declare("nightly", "s_tick", "daily at 06:00") # the same: fine
with pytest.raises(ValueError) as refused:
declare("nightly", "s_tick", "daily at 07:00")
assert "nightly" in str(refused.value)
def test_a_declared_cadence_is_parsed_where_it_is_written(declared):
"""At import, with a traceback pointing at the line — rather than at 06:00
in a log nobody is reading."""
with pytest.raises(ValueError):
declare("bad", "s_tick", "every so often")
assert "bad" not in declared
# ------------------------------------------------------------- a real queue
def test_a_started_queue_fires_what_is_overdue(jobs, declared):
"""The one test that drives the loop rather than the method.
Everything above proves the firing decides correctly; this proves it is
wired into `start()` at all, which is the wire nobody notices is missing
until a schedule silently stops running.
"""
ran: list[str] = []
jobs.add_schedule("regular", "s_tick", "every 15m",
payload={"note": "from the loop"}, next_run=ago(30))
queue = JobQueue(jobs, {}, ran, tz="UTC", workers={"cpu": 1})
assert queue.drain(timeout=10) is True
assert ran == ["from the loop"]
assert jobs.schedules()[0].last_state == "done"
def test_a_queue_with_schedules_off_fires_nothing(jobs, declared):
ran: list[str] = []
jobs.add_schedule("regular", "s_tick", "every 15m", next_run=ago(30))
queue = JobQueue(jobs, {}, ran, tz="UTC", schedules=False)
assert queue.drain(timeout=10) is True
assert ran == []
assert jobs.schedules()[0].last_job_id == 0
def test_an_unknown_timezone_is_a_warning_and_not_a_refusal(jobs, caplog):
"""A mistyped zone must not be the reason a worker will not start."""
queue = JobQueue(jobs, {}, [], tz="Mars/Olympus_Mons")
assert queue.zone is not None
assert Cadence.parse("daily at 06:00").next_after(datetime.now(UTC), queue.zone)
# ------------------------------------------------------------------ the web
@pytest.fixture
def client(jobs, declared):
pytest.importorskip("fastapi")
from fastapi import FastAPI
from fastapi.testclient import TestClient
from pipeline.web import connect
app = FastAPI()
connect(app, jobs, {}, [], tz="UTC", schedules=False)
with TestClient(app) as client:
yield client
def test_the_element_is_served(client):
answer = client.get("/_jobs/schedules.js")
assert answer.status_code == 200
assert "javascript" in answer.headers["content-type"]
assert "desk-schedules" in answer.text
def test_adding_listing_editing_and_deleting(client):
added = client.post("/api/schedules", json={
"kind": "s_tick", "cadence": "weekly monday 07:00",
"payload": {"note": "over http"}})
assert added.status_code == 201
row = added.json()
# Named after its kind, since none was given, and stored canonically.
assert row["name"] == "s_tick" and row["cadence"] == "weekly mon 07:00"
assert row["next_run"] and row["unknown"] is False
assert [s["id"] for s in client.get("/api/schedules").json()] == [row["id"]]
off = client.patch(f"/api/schedules/{row['id']}", json={"enabled": False})
assert off.status_code == 200 and off.json()["enabled"] is False
# And only that: a PATCH of one field does not resubmit the rest.
assert off.json()["cadence"] == "weekly mon 07:00"
assert off.json()["payload"] == {"note": "over http"}
faster = client.patch(f"/api/schedules/{row['id']}", json={"cadence": "every 15m"})
assert faster.json()["cadence"] == "every 15m"
# A new cadence moves the time with it, rather than leaving the old one.
assert faster.json()["next_run"] != row["next_run"]
gone = client.delete(f"/api/schedules/{row['id']}")
assert gone.status_code == 200 and gone.json() == {"deleted": True}
assert client.delete(f"/api/schedules/{row['id']}").json() == {"deleted": False}
def test_a_second_schedule_of_one_kind_is_named_by_counting(client):
first = client.post("/api/schedules",
json={"kind": "s_tick", "cadence": "every 15m"}).json()
second = client.post("/api/schedules",
json={"kind": "s_tick", "cadence": "every 6h"}).json()
assert (first["name"], second["name"]) == ("s_tick", "s_tick 2")
def test_a_name_somebody_typed_is_refused_rather_than_altered(client):
client.post("/api/schedules",
json={"name": "nightly", "kind": "s_tick", "cadence": "every 15m"})
clash = client.post("/api/schedules",
json={"name": "nightly", "kind": "s_tick", "cadence": "every 6h"})
assert clash.status_code == 409
assert "nightly" in clash.json()["detail"]
def test_an_unregistered_kind_is_a_404_naming_what_there_is(client):
refused = client.post("/api/schedules",
json={"kind": "s_nonesuch", "cadence": "every 15m"})
assert refused.status_code == 404
detail = refused.json()["detail"]
assert "s_nonesuch" in detail and "s_tick" in detail
def test_a_payload_missing_what_the_handler_needs_is_a_422_naming_it(client):
refused = client.post("/api/schedules",
json={"kind": "s_needs", "cadence": "every 15m"})
assert refused.status_code == 422
assert "target" in refused.json()["detail"]
# And with it, the same request is fine.
assert client.post("/api/schedules", json={
"kind": "s_needs", "cadence": "every 15m",
"payload": {"target": "somewhere"}}).status_code == 201
def test_a_bad_cadence_is_a_422_listing_the_accepted_wordings(client):
refused = client.post("/api/schedules",
json={"kind": "s_tick", "cadence": "sometimes"})
assert refused.status_code == 422
detail = refused.json()["detail"]
assert isinstance(detail, str)
for form in ("every 15m", "every 6h", "every 2d",
"daily at 06:00", "weekly mon 07:00"):
assert form in detail
def test_an_edit_is_validated_the_same_way(client):
row = client.post("/api/schedules",
json={"kind": "s_tick", "cadence": "every 15m"}).json()
assert client.patch(f"/api/schedules/{row['id']}",
json={"cadence": "now and then"}).status_code == 422
assert client.patch(f"/api/schedules/{row['id']}",
json={"kind": "s_needs"}).status_code == 422
assert client.patch("/api/schedules/9999",
json={"enabled": False}).status_code == 404
def test_run_now_queues_the_job(client, jobs):
row = client.post("/api/schedules",
json={"kind": "s_tick", "cadence": "daily at 06:00"}).json()
started = client.post(f"/api/schedules/{row['id']}/run")
assert started.status_code == 202
assert started.json()["kind"] == "s_tick"
after = client.get("/api/schedules").json()[0]
assert after["last_job_id"] == started.json()["id"]
assert after["next_run"] == row["next_run"]
assert client.post("/api/schedules/9999/run").status_code == 404
def test_a_row_whose_handler_went_away_is_flagged_not_hidden(client, jobs):
jobs.add_schedule("orphan", "s_vanished", "every 15m")
row = next(s for s in client.get("/api/schedules").json()
if s["name"] == "orphan")
assert row["unknown"] is True
# Listed and deletable, which is the whole of what can be done about it.
assert client.delete(f"/api/schedules/{row['id']}").json() == {"deleted": True}
def test_the_socket_pushes_the_schedule_list_on_every_change(client):
"""So a row edited in one tab redraws in the others without polling.
The opening message is the *job* list — the socket is shared — which is
why `<desk-schedules>` reads `/api/schedules` once on connect and follows
this afterwards.
"""
with client.websocket_connect("/ws/jobs") as socket:
assert socket.receive_json()["type"] == "jobs"
added = client.post("/api/schedules",
json={"kind": "s_tick", "cadence": "every 15m"}).json()
pushed = _next_of(socket, "schedules")
assert [s["id"] for s in pushed["schedules"]] == [added["id"]]
assert pushed["schedules"][0]["cadence"] == "every 15m"
client.patch(f"/api/schedules/{added['id']}", json={"cadence": "every 6h"})
assert _next_of(socket, "schedules")["schedules"][0]["cadence"] == "every 6h"
client.delete(f"/api/schedules/{added['id']}")
assert _next_of(socket, "schedules")["schedules"] == []
def _next_of(socket, type_: str, tries: int = 12) -> dict:
"""The next message of one type, stepping over the job pushes between."""
for _ in range(tries):
message = socket.receive_json()
if message["type"] == type_:
return message
raise AssertionError(f"no {type_} message arrived")
# --------------------------------------------------------------- the worker
def _worker(db: Path, marks: Path, *extra: str) -> subprocess.CompletedProcess:
argv = [sys.executable, "-m", "pipeline.worker", "--db", str(db),
"--bootstrap", "pipeline.tests.scheduled_handlers:marking",
"--bootstrap-arg", str(marks), "--until-idle", *extra]
return subprocess.run(argv, env=dict(os.environ, PYTHONPATH=str(ROOT)),
cwd=ROOT, capture_output=True, timeout=120)
def test_the_standalone_worker_fires_schedules(tmp_path):
"""A machine running only `python -m pipeline.worker` still gets its 06:00
job, which is what makes it a complete replacement for the web app's
worker rather than half of one."""
db, marks = tmp_path / "w.db", tmp_path / "marks"
marks.mkdir()
jobs = Jobs(db)
jobs.add_schedule("marker", "sw_mark", "every 1h", next_run=ago(60))
done = _worker(db, marks)
assert done.returncode == 0, done.stdout.decode("utf-8", "replace")
assert (marks / "fired").exists()
assert jobs.schedules()[0].last_job_id
def test_no_schedules_turns_it_off_for_a_second_worker(tmp_path):
"""One scheduler per file, as many workers as you like: the pile-up guard
reads `last_job_id`, which neither of two simultaneous firings has written
yet when the other looks."""
db, marks = tmp_path / "w.db", tmp_path / "marks"
marks.mkdir()
jobs = Jobs(db)
jobs.add_schedule("marker", "sw_mark", "every 1h", next_run=ago(60))
done = _worker(db, marks, "--no-schedules")
assert done.returncode == 0, done.stdout.decode("utf-8", "replace")
assert not (marks / "fired").exists()
assert jobs.schedules()[0].last_job_id == 0
# Not even the declared default was reconciled: nothing about schedules
# happens in a worker that was told not to do them.
assert [s.name for s in jobs.schedules()] == ["marker"]