Bring in pipeline, auth, deploy and docs from the boilerplate
roledesk is built on the boilerplate from here on: pipeline 2.0 (queue and scheduler), auth (one password, sessions, passkeys), the deploy kit and its docs, plus the coupling and packaging tests. Downstream copies, unedited. Refreshed from boilerplate/d56949f7d0f6c0bfe5745478e09007bfad67ae87 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1f436104f3
commit
9d666d0937
76 files changed
+15235
No files matched your search
@@ -0,0 +1,2 @@
|
||||
"""The pipeline's own tests. They travel with the folder, so a copy can
|
||||
prove itself where it lands: `python -m pytest pipeline/tests`."""
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Handlers for the isolation tests, in a module a child process can import.
|
||||
|
||||
Not a test file — pytest does not collect it — and deliberately separate from
|
||||
the tests that use it, because the point of `isolate=True` is that the work
|
||||
runs in a *different process*. That process gets nothing from this one: it
|
||||
imports `pipeline.tests.isolated_handlers` itself, by the name in
|
||||
`--bootstrap`, which is exactly what an app's own bootstrap has to do.
|
||||
|
||||
It also documents the whole of the contract, which is two lines: import the
|
||||
modules that fill `JOBS`, and return the `(config, store)` pair the handlers
|
||||
take. Here both are dicts, since nothing in the pipeline reads them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from pipeline import NULL, RunContext, job
|
||||
|
||||
|
||||
@job("iso_ok", "Working in its own process", isolate=True)
|
||||
def iso_ok(config: Any, store: Any, *, context: RunContext = NULL,
|
||||
n: int = 0) -> dict[str, Any]:
|
||||
"""Reports which process it ran in, so a test can prove it was not ours."""
|
||||
context.progress(n, n, f"counting to {n}")
|
||||
return {"doubled": n * 2, "pid": os.getpid()}
|
||||
|
||||
|
||||
@job("iso_raises", "Failing in its own process", isolate=True)
|
||||
def iso_raises(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
|
||||
raise RuntimeError("the model said no")
|
||||
|
||||
|
||||
@job("iso_slow", "Taking too long on purpose", isolate=True, timeout=1.0)
|
||||
def iso_slow(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""Uninterruptible on purpose: no `check()`, so only a kill ends it.
|
||||
|
||||
`time.sleep` stands in for `sklearn.fit` — the difference that matters is
|
||||
not what it does but that nothing in it ever asks whether to stop, which
|
||||
is the case a thread has no answer for.
|
||||
"""
|
||||
time.sleep(120)
|
||||
return {"never": True}
|
||||
|
||||
|
||||
def bootstrap() -> tuple[Any, Any]:
|
||||
"""What `--bootstrap` names. Importing this module registered the jobs."""
|
||||
return {"config": "rebuilt in the child"}, {"store": "rebuilt in the child"}
|
||||
|
||||
|
||||
# ---------------------------------------------------- two workers, one file
|
||||
|
||||
@job("mp_mark", "Leaving a mark")
|
||||
def mp_mark(config: Any, store: Any, *, context: RunContext = NULL,
|
||||
i: int = 0) -> dict[str, Any]:
|
||||
"""Leave a file named after this job and the process that ran it.
|
||||
|
||||
A file per run rather than a line in a shared one, because creating a
|
||||
uniquely named file is atomic on every platform and appending from two
|
||||
processes is not. A job run twice leaves two files with the same number.
|
||||
"""
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
(Path(store) / f"{i}-{os.getpid()}-{uuid.uuid4().hex}").touch()
|
||||
time.sleep(0.02) # long enough that both workers get a share
|
||||
return {"i": i}
|
||||
|
||||
|
||||
def marking(marks: str) -> tuple[Any, Any]:
|
||||
"""The bootstrap `pipeline.worker` is given: the marks directory is the store."""
|
||||
return {}, marks
|
||||
@@ -0,0 +1,39 @@
|
||||
"""A job and a declared schedule, in a module a worker process can import.
|
||||
|
||||
Not a test file — pytest does not collect it — and separate from the tests that
|
||||
use it for the same reason `isolated_handlers.py` is: the process under test is
|
||||
a *different* process, and it gets nothing from this one. It imports this by
|
||||
the name in `--bootstrap`, which fills `JOBS` and `SCHEDULES` exactly as an
|
||||
app's own jobs module does.
|
||||
|
||||
Separate from `isolated_handlers.py` as well, because importing a module that
|
||||
calls `declare` fills a global: a test suite that imported this one would find
|
||||
its own schedule registry non-empty before it had declared anything.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pipeline import NULL, RunContext, declare, job
|
||||
|
||||
|
||||
@job("sw_mark", "Leaving a mark")
|
||||
def sw_mark(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""Touch a file in the store directory, so the parent process can see from
|
||||
outside whether this ran at all."""
|
||||
(Path(store) / "fired").touch()
|
||||
return {"fired": True}
|
||||
|
||||
|
||||
#: The declaration itself is what one half of the worker test is about: a
|
||||
#: worker with no web app around it reconciles declared defaults like any
|
||||
#: other, so a machine running only `python -m pipeline.worker` still gets
|
||||
#: them.
|
||||
declare("marker", "sw_mark", "every 1h", label="Leaving a mark, hourly")
|
||||
|
||||
|
||||
def marking(marks: str) -> tuple[Any, Any]:
|
||||
"""The bootstrap `pipeline.worker` is given: the marks directory is the store."""
|
||||
return {}, marks
|
||||
@@ -0,0 +1,113 @@
|
||||
"""The web face of the queue, from an app that has written nothing but `connect`.
|
||||
|
||||
python -m pytest pipeline/tests/test_web.py -q
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from pipeline import NULL, Jobs, RunContext, job
|
||||
from pipeline.web import connect, jobs_router
|
||||
|
||||
|
||||
@job("w_hello", "Saying hello")
|
||||
def w_hello(config, store, *, context: RunContext = NULL, name: str = "") -> dict:
|
||||
context.progress(1, 1, f"greeting {name}")
|
||||
return {"said": f"hello {name}", "config": config["app"]}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def jobs():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
yield Jobs(Path(tmp) / "web.db")
|
||||
|
||||
|
||||
def settled(client: TestClient, job_id: int, timeout: float = 10.0) -> dict:
|
||||
"""The job as the HTTP API reports it, once it has finished."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
snap = client.get("/api/jobs").json()
|
||||
for row in snap["recent"]:
|
||||
if row["id"] == job_id:
|
||||
return row
|
||||
time.sleep(0.05)
|
||||
raise AssertionError(f"job {job_id} did not finish")
|
||||
|
||||
|
||||
def test_connect_is_the_whole_of_the_wiring(jobs):
|
||||
"""Routes, socket and a running worker, from one call.
|
||||
|
||||
The app below has nothing else: no lifespan, no broadcaster, no router of
|
||||
its own. A job enqueued over HTTP runs to `done` because the worker came up
|
||||
with the app, and the socket hands a new tab the current state on connect.
|
||||
"""
|
||||
app = FastAPI()
|
||||
queue = connect(app, jobs, {"app": "tiny"}, None)
|
||||
assert app.state.queue is queue
|
||||
|
||||
with TestClient(app) as client:
|
||||
made = client.post("/api/jobs", json={"kind": "w_hello",
|
||||
"payload": {"name": "you"}})
|
||||
assert made.status_code == 202, made.text
|
||||
ended = settled(client, made.json()["id"])
|
||||
assert ended["state"] == "done"
|
||||
assert ended["result"] == {"said": "hello you", "config": "tiny"}
|
||||
|
||||
with client.websocket_connect("/ws/jobs") as socket:
|
||||
hello = socket.receive_json()
|
||||
assert hello["type"] == "jobs"
|
||||
assert any(row["id"] == ended["id"] for row in hello["recent"])
|
||||
|
||||
assert queue._task is None or queue._task.done(), "and it stopped with the app"
|
||||
|
||||
|
||||
def test_connect_keeps_the_apps_own_lifespan(jobs):
|
||||
"""An app with startup code of its own keeps it, and the worker is already
|
||||
up when that code runs — so startup code may enqueue."""
|
||||
seen: list[str] = []
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
seen.append("app started")
|
||||
app.state.queue.enqueue("w_hello", {"name": "from startup"})
|
||||
yield
|
||||
seen.append("app stopped")
|
||||
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
connect(app, jobs, {"app": "tiny"}, None)
|
||||
|
||||
with TestClient(app) as client:
|
||||
# By id: the snapshot reads live and recent jobs separately, so a job
|
||||
# that finishes in between is listed in both.
|
||||
snap = client.get("/api/jobs").json()
|
||||
(job_id,) = {r["id"] for r in snap["queued"] + snap["running"]
|
||||
+ snap["recent"] if r["kind"] == "w_hello"}
|
||||
assert settled(client, job_id)["state"] == "done"
|
||||
assert seen == ["app started", "app stopped"]
|
||||
|
||||
|
||||
def test_the_element_is_served_by_the_queues_own_router():
|
||||
"""Two lines in a template and a new app has a job list. Served from a
|
||||
route, because a router cannot mount a static directory and needing the
|
||||
app to mount one is the setup the element exists to remove."""
|
||||
app = FastAPI()
|
||||
app.include_router(jobs_router)
|
||||
answer = TestClient(app).get("/_jobs/jobs.js")
|
||||
|
||||
assert answer.status_code == 200
|
||||
assert answer.headers["content-type"].startswith("text/javascript")
|
||||
assert 'customElements.define("desk-jobs"' in answer.text
|
||||
@@ -0,0 +1,209 @@
|
||||
"""What keeps a queue healthy for months: retention, bounded results, whose
|
||||
retry policy wins, draining, and two real processes on one file.
|
||||
|
||||
python -m pytest pipeline/tests/test_housekeeping.py -q
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from contextlib import closing
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from pipeline import NULL, Batch, Bound, JobQueue, Jobs, Task, job
|
||||
from pipeline.models import Job
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
@job("h_plain", "A job with a policy of its own", max_attempts=3, retry_delay=7)
|
||||
def h_plain(config, store, *, context=NULL, i: int = 0) -> dict:
|
||||
return {"i": i}
|
||||
|
||||
|
||||
@job("h_big", "Returning far too much", max_attempts=3)
|
||||
def h_big(config, store, *, context=NULL) -> dict:
|
||||
return {"rows": ["x" * 100] * 2000, "count": 2000}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def jobs():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
yield Jobs(Path(tmp) / "h.db")
|
||||
|
||||
|
||||
def got(found: Job | None) -> Job:
|
||||
assert found is not None
|
||||
return found
|
||||
|
||||
|
||||
def age(jobs: Jobs, job_id: int, days: float) -> None:
|
||||
"""Pretend a job finished `days` ago, and its runs were minted then too."""
|
||||
when = (datetime.now(UTC) - timedelta(days=days)).isoformat()
|
||||
with closing(sqlite3.connect(jobs.path)) as conn, conn:
|
||||
conn.execute("UPDATE jobs SET finished_at=? WHERE id=?", (when, job_id))
|
||||
conn.execute("UPDATE runs SET created_at=? WHERE id IN "
|
||||
"(SELECT cascade_id FROM jobs WHERE id=? UNION "
|
||||
" SELECT task_id FROM jobs WHERE id=?)", (when, job_id, job_id))
|
||||
|
||||
|
||||
def runs(jobs: Jobs) -> int:
|
||||
with closing(sqlite3.connect(jobs.path)) as conn:
|
||||
return conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- retention
|
||||
|
||||
def test_purge_forgets_old_history_but_not_the_last_of_each_kind(jobs):
|
||||
"""Without this both tables grow for as long as the app runs.
|
||||
|
||||
What survives an old cutoff: anything unfinished, and the newest ending of
|
||||
each kind in each state — because `last(kind)` is how an app says "last
|
||||
pulled five weeks ago", and deleting it would make that "never".
|
||||
"""
|
||||
old = [got(jobs.enqueue("h_plain")) for _ in range(3)]
|
||||
for queued in old:
|
||||
jobs.finish(queued.id, "done")
|
||||
age(jobs, queued.id, days=40)
|
||||
waiting = got(jobs.enqueue("h_plain")) # never finished
|
||||
|
||||
gone = jobs.purge(30 * 86400)
|
||||
|
||||
assert gone == 2, "two of the three old ones went"
|
||||
assert got(jobs.last("h_plain")).id == old[-1].id, "the newest stayed"
|
||||
assert got(jobs.get(waiting.id)).state == "queued", "unfinished work is not history"
|
||||
|
||||
|
||||
def test_purge_leaves_recent_history_alone(jobs):
|
||||
recent = got(jobs.enqueue("h_plain"))
|
||||
jobs.finish(recent.id, "done")
|
||||
before = runs(jobs)
|
||||
|
||||
assert jobs.purge(30 * 86400) == 0
|
||||
assert jobs.get(recent.id) is not None
|
||||
assert runs(jobs) == before
|
||||
|
||||
|
||||
def test_purge_drops_runs_nothing_points_at_any_more(jobs):
|
||||
first = got(jobs.enqueue("h_plain"))
|
||||
second = got(jobs.enqueue("h_plain"))
|
||||
for queued in (first, second):
|
||||
jobs.finish(queued.id, "done")
|
||||
age(jobs, queued.id, days=40)
|
||||
before = runs(jobs)
|
||||
|
||||
jobs.purge(30 * 86400)
|
||||
|
||||
assert runs(jobs) < before, "the purged job's cascade and task went with it"
|
||||
assert jobs.run(second.task_id) is not None, "the kept job's runs did not"
|
||||
|
||||
|
||||
# ----------------------------------------------------------- bounded results
|
||||
|
||||
def test_a_result_too_large_fails_the_job_loudly_and_once(jobs):
|
||||
"""Past `MAX_RESULT` the job fails rather than losing data quietly: the row
|
||||
says why and how to fix it, a notice goes to whoever is watching, and it is
|
||||
not retried — the same handler would return the same result."""
|
||||
said: list = []
|
||||
|
||||
async def on_notice(notice) -> None:
|
||||
said.append(notice)
|
||||
|
||||
queued = got(jobs.enqueue("h_big", max_attempts=3))
|
||||
JobQueue(jobs, None, None, on_notice=on_notice).drain(timeout=10)
|
||||
|
||||
ended = got(jobs.get(queued.id))
|
||||
assert ended.state == "failed"
|
||||
assert "ResultTooLarge" in ended.error and "store" in ended.error, ended.error
|
||||
assert "count, rows" in ended.error, "it names what was returned"
|
||||
assert ended.result == {}, "nothing of it was written"
|
||||
assert ended.attempt == 1, "and it was not tried again"
|
||||
assert [n.code for n in said] == ["job.failed"], "and somebody was told"
|
||||
|
||||
|
||||
def test_a_result_at_the_limit_is_kept_whole(jobs):
|
||||
"""The limit is a line, not a tax: everything under it is stored as is."""
|
||||
queued = got(jobs.enqueue("h_plain", payload={"i": 1}))
|
||||
JobQueue(jobs, None, None).drain(timeout=10)
|
||||
assert got(jobs.get(queued.id)).result == {"i": 1}
|
||||
|
||||
|
||||
# -------------------------------------------------------- whose policy wins
|
||||
|
||||
def test_a_task_with_plain_payloads_keeps_the_kinds_own_retry_policy(jobs):
|
||||
"""`@job(max_attempts=3)` has to mean something when a task queues it.
|
||||
|
||||
It used not to: a plain payload was given the default bound explicitly,
|
||||
which silently overrode whatever the kind declared.
|
||||
"""
|
||||
queue = JobQueue(jobs, None, None)
|
||||
Task(kind="h_plain", finder=lambda c, s: [{"i": 1}]).start(None, None, queue)
|
||||
|
||||
(child,) = [j for j in jobs.snapshot().queued if j.kind == "h_plain"]
|
||||
assert (child.max_attempts, child.retry_delay) == (3, 7.0)
|
||||
|
||||
|
||||
def test_a_batch_with_its_own_bound_overrides_the_kind(jobs):
|
||||
"""The other half: a batch cut from one plugin profile carries that
|
||||
profile's policy, and that wins, because the outlet knows its publisher."""
|
||||
queue = JobQueue(jobs, None, None)
|
||||
Task(kind="h_plain", finder=lambda c, s: [
|
||||
Batch({"i": 1}, Bound(max_attempts=5, delay=60))]).start(None, None, queue)
|
||||
|
||||
(child,) = [j for j in jobs.snapshot().queued if j.kind == "h_plain"]
|
||||
assert (child.max_attempts, child.retry_delay) == (5, 60.0)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ draining
|
||||
|
||||
def test_drain_runs_everything_queued_and_stops(jobs):
|
||||
"""The batch shape: a script or a cron entry with no web app around it."""
|
||||
ids = [got(jobs.enqueue("h_plain", payload={"i": i})).id for i in range(5)]
|
||||
|
||||
assert JobQueue(jobs, None, None).drain(timeout=10) is True
|
||||
assert {got(jobs.get(i)).state for i in ids} == {"done"}
|
||||
|
||||
|
||||
# ------------------------------------------------- two processes, one file
|
||||
|
||||
def test_two_worker_processes_never_run_the_same_job_twice(tmp_path):
|
||||
"""The claim the table-as-queue design rests on, tested the only way that
|
||||
means anything: two separate `python -m pipeline.worker` processes draining
|
||||
one file at once. Every job leaves a marker file named after itself and the
|
||||
process that ran it; each job must have exactly one.
|
||||
"""
|
||||
db = tmp_path / "shared.db"
|
||||
marks = tmp_path / "marks"
|
||||
marks.mkdir()
|
||||
jobs = Jobs(db)
|
||||
for i in range(40):
|
||||
jobs.enqueue("mp_mark", payload={"i": i})
|
||||
|
||||
env = dict(os.environ, PYTHONPATH=str(ROOT))
|
||||
argv = [sys.executable, "-m", "pipeline.worker", "--db", str(db),
|
||||
"--bootstrap", "pipeline.tests.isolated_handlers:marking",
|
||||
"--bootstrap-arg", str(marks), "--workers", "cpu=2", "--until-idle"]
|
||||
workers = [subprocess.Popen(argv, env=env, cwd=ROOT,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
for _ in range(2)]
|
||||
for worker in workers:
|
||||
out, _ = worker.communicate(timeout=120)
|
||||
assert worker.returncode == 0, out.decode(errors="replace")[-2000:]
|
||||
|
||||
ran: dict[int, list[str]] = {}
|
||||
for mark in marks.iterdir():
|
||||
i, pid = mark.name.split("-")[:2]
|
||||
ran.setdefault(int(i), []).append(pid)
|
||||
assert sorted(ran) == list(range(40)), "every job ran"
|
||||
doubled = {i: pids for i, pids in ran.items() if len(pids) > 1}
|
||||
assert not doubled, f"and none ran twice: {doubled}"
|
||||
assert {j.state for j in jobs.snapshot().recent} == {"done"}
|
||||
@@ -0,0 +1,320 @@
|
||||
"""Ending work that will not end itself, and how data moves between jobs.
|
||||
|
||||
Four claims, and they fail independently:
|
||||
|
||||
a deadline is honest `timeout` without `isolate` is refused at
|
||||
registration rather than promising a kill a
|
||||
thread cannot deliver.
|
||||
a process can be killed an isolated job that never calls `check()` is
|
||||
still over when its deadline passes, and its
|
||||
row says so.
|
||||
a chain reads, never writes `upstream()` is how B gets A's output — off A's
|
||||
row, so B's own row was complete when written.
|
||||
a fan-out is one call `fan_out` queues the children and the job that
|
||||
reports on them, and an app never writes the
|
||||
second one.
|
||||
|
||||
python3 -m pytest tests/test_isolation.py -q
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from pipeline import NULL, Cancelled, JobQueue, Jobs, RunContext, job, spawn
|
||||
from pipeline.models import Job
|
||||
|
||||
# Imported for the `@job` registrations, which the *parent* needs as much as
|
||||
# the child does: the worker reads `isolate` and `timeout` off the spec before
|
||||
# it decides to start a process at all. An app imports its handlers in every
|
||||
# entry point for this reason; here that import is this line.
|
||||
from pipeline.tests import isolated_handlers # noqa: F401
|
||||
|
||||
BOOTSTRAP = "pipeline.tests.isolated_handlers:bootstrap"
|
||||
|
||||
|
||||
@job("t_child", "A child of a fan-out")
|
||||
def t_child(config, store, *, context=NULL, i: int = 0) -> dict:
|
||||
"""Queued by `fan_out` and never run in these tests — what is under test is
|
||||
the shape of the rows it made, which is the part an app never writes."""
|
||||
return {"i": i}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def jobs():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
yield Jobs(Path(tmp) / "t.db")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def child_path(monkeypatch):
|
||||
"""Put the folder holding `pipeline` where a child process can import it.
|
||||
|
||||
What an installed app gets for free and a checkout does not: the child is a
|
||||
fresh interpreter, and the checkout is on *our* path only because pytest
|
||||
put it there.
|
||||
"""
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
monkeypatch.setenv("PYTHONPATH", str(root))
|
||||
|
||||
|
||||
def drain(queue: JobQueue, *, timeout: float = 20.0) -> None:
|
||||
"""Run the worker until nothing is queued, running, or held by this one."""
|
||||
|
||||
async def go() -> None:
|
||||
await queue.start()
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
snap = queue.jobs.snapshot()
|
||||
if not snap.running and not snap.queued and not queue._running:
|
||||
break
|
||||
await asyncio.sleep(0.02)
|
||||
await queue.stop()
|
||||
|
||||
asyncio.run(go())
|
||||
|
||||
|
||||
def got(found: Job | None) -> Job:
|
||||
assert found is not None
|
||||
return found
|
||||
|
||||
|
||||
# --------------------------------------------------- a deadline is honest
|
||||
|
||||
def test_a_timeout_without_isolation_is_refused():
|
||||
"""The pipeline will not offer a kill it cannot perform.
|
||||
|
||||
A deadline over a job in a thread could only mark the row failed and walk
|
||||
away: the work would carry on holding the core it was killed for holding,
|
||||
and two things would intend to write one row. So the pair is refused where
|
||||
it is declared, with the two ways out named.
|
||||
"""
|
||||
with pytest.raises(ValueError) as refused:
|
||||
@job("t_threaded_deadline", timeout=5.0)
|
||||
def handler(config, store, *, context=NULL):
|
||||
return {}
|
||||
|
||||
said = str(refused.value)
|
||||
assert "isolate=True" in said, "it names the way to get a real deadline"
|
||||
assert "RunContext.spawn" in said, "and the cheaper way, for a command"
|
||||
|
||||
|
||||
def test_isolation_without_a_timeout_is_fine():
|
||||
"""Only the one combination is refused. An isolated job with no deadline
|
||||
runs until it is done or killed by hand, which is every job the pipeline
|
||||
has ever had."""
|
||||
|
||||
@job("t_isolated_no_deadline", isolate=True)
|
||||
def handler(config, store, *, context=NULL):
|
||||
return {}
|
||||
|
||||
from pipeline import JOBS
|
||||
assert JOBS["t_isolated_no_deadline"].isolate
|
||||
assert JOBS["t_isolated_no_deadline"].timeout == 0.0
|
||||
|
||||
|
||||
def test_a_queue_with_no_bootstrap_says_so(jobs):
|
||||
"""An isolated job needs its app rebuilt in the child, and a queue that
|
||||
cannot do it fails the job with the reason rather than with a TypeError
|
||||
from somewhere inside a subprocess."""
|
||||
|
||||
@job("t_needs_bootstrap", isolate=True)
|
||||
def handler(config, store, *, context=NULL):
|
||||
return {}
|
||||
|
||||
queued = got(jobs.enqueue("t_needs_bootstrap"))
|
||||
drain(JobQueue(jobs, None, None)) # no bootstrap=
|
||||
ended = got(jobs.get(queued.id))
|
||||
assert ended.state == "failed"
|
||||
assert "bootstrap" in ended.error, f"it names what is missing: {ended.error}"
|
||||
|
||||
|
||||
# ------------------------------------------------ a process can be killed
|
||||
|
||||
def test_an_isolated_job_runs_somewhere_else_and_its_result_comes_back(jobs, child_path):
|
||||
"""The happy path, and the proof that it was a real process.
|
||||
|
||||
Nothing was pickled to get there: the child was handed a database and a
|
||||
row id and rebuilt the rest. Which is also why its progress is on the row
|
||||
— it wrote it there itself, to the same table the snapshot reads.
|
||||
"""
|
||||
queued = got(jobs.enqueue("iso_ok", payload={"n": 21}))
|
||||
drain(JobQueue(jobs, None, None, bootstrap=BOOTSTRAP))
|
||||
|
||||
ended = got(jobs.get(queued.id))
|
||||
assert ended.state == "done", ended.error
|
||||
assert ended.result["doubled"] == 42, "the handler's return travelled back"
|
||||
assert ended.result["pid"] != os.getpid(), "and it ran in another process"
|
||||
assert ended.message == "counting to 21", "its progress reached our table"
|
||||
|
||||
|
||||
def test_an_isolated_job_that_raises_fails_with_what_it_said(jobs, child_path):
|
||||
"""A failure across a process boundary reads like any other failure: the
|
||||
row is `failed`, the error is the exception, and the retry bound applies
|
||||
as it would to a handler that raised in a thread."""
|
||||
queued = got(jobs.enqueue("iso_raises"))
|
||||
drain(JobQueue(jobs, None, None, bootstrap=BOOTSTRAP))
|
||||
|
||||
ended = got(jobs.get(queued.id))
|
||||
assert ended.state == "failed"
|
||||
assert "the model said no" in ended.error, ended.error
|
||||
|
||||
|
||||
def test_a_deadline_ends_a_job_that_never_looks_up(jobs, child_path):
|
||||
"""The claim the whole feature exists for.
|
||||
|
||||
`iso_slow` sleeps for two minutes and never calls `check()`. In a thread
|
||||
that is unkillable and the worker could only lie about having stopped it.
|
||||
In a process of its own it is over in about a second, and the row says
|
||||
why — which is what a person reading the job list needs.
|
||||
"""
|
||||
queued = got(jobs.enqueue("iso_slow"))
|
||||
started = time.monotonic()
|
||||
drain(JobQueue(jobs, None, None, bootstrap=BOOTSTRAP))
|
||||
took = time.monotonic() - started
|
||||
|
||||
ended = got(jobs.get(queued.id))
|
||||
assert ended.state == "failed", f"it ended: {ended.state}"
|
||||
assert took < 15, f"and on its deadline, not in its own time: {took:.1f}s"
|
||||
|
||||
|
||||
# ------------------------------------------------------ spawn, on its own
|
||||
|
||||
def test_spawn_kills_the_process_when_the_job_is_cancelled():
|
||||
"""Kill, for a handler whose work is a command.
|
||||
|
||||
Cooperative in name only from the handler's side: it is blocked in
|
||||
`spawn`, not in a loop with a `check()` in it, and the child is killed
|
||||
outright. What makes that safe is that the thread was only sleeping.
|
||||
"""
|
||||
asked = [0]
|
||||
|
||||
def is_cancelled() -> bool:
|
||||
asked[0] += 1
|
||||
return asked[0] > 2 # pressed, a moment in
|
||||
|
||||
context = RunContext(is_cancelled=is_cancelled)
|
||||
started = time.monotonic()
|
||||
with pytest.raises(Cancelled):
|
||||
context.spawn([sys.executable, "-c", "import time; time.sleep(120)"])
|
||||
assert time.monotonic() - started < 10, "it did not wait the two minutes"
|
||||
|
||||
|
||||
def test_spawn_raises_on_its_deadline_and_leaves_nothing_running():
|
||||
"""The deadline, and that the process is gone after it — not merely
|
||||
abandoned, which is the failure mode the whole thing is against."""
|
||||
started = time.monotonic()
|
||||
with pytest.raises(TimeoutError):
|
||||
spawn([sys.executable, "-c", "import time; time.sleep(120)"], timeout=0.5)
|
||||
assert time.monotonic() - started < 10
|
||||
|
||||
|
||||
def test_spawn_hands_back_the_exit_code_rather_than_raising():
|
||||
"""A non-zero exit is not an error here. Plenty of commands say something
|
||||
useful with one, and only the handler knows which."""
|
||||
done = spawn([sys.executable, "-c", "import sys; sys.exit(3)"])
|
||||
assert done.returncode == 3
|
||||
|
||||
|
||||
def test_spawn_survives_a_command_that_writes_more_than_a_pipe_holds():
|
||||
"""Output goes to a file, not a `PIPE`, and this is why.
|
||||
|
||||
A child that fills the pipe buffer blocks on the write while the poll loop
|
||||
blocks on the read, and neither side is wrong — the job simply never ends.
|
||||
A megabyte is far past any platform's buffer.
|
||||
"""
|
||||
done = spawn([sys.executable, "-c", "print('x' * 1000000)"], timeout=60)
|
||||
assert done.returncode == 0
|
||||
assert len(done.stdout) >= 1000000
|
||||
|
||||
|
||||
# ---------------------------------------------- a chain reads, never writes
|
||||
|
||||
def test_upstream_merges_what_the_parents_returned():
|
||||
"""How B gets A's output: off A's row, not out of B's payload.
|
||||
|
||||
Which is the property worth having. B's row was complete when it was
|
||||
written, so it can carry a dedup key, a retry can re-run it from itself,
|
||||
and no worker can claim it between an earlier job's write and its commit.
|
||||
"""
|
||||
a = Job(id=1, kind="a", created_at="", result={"html": "<p>hi</p>", "code": 200})
|
||||
b = Job(id=2, kind="b", created_at="", result={"code": 204})
|
||||
context = RunContext(waited_for=(a, b))
|
||||
|
||||
assert context.upstream() == {"html": "<p>hi</p>", "code": 204}, "later parents win"
|
||||
|
||||
|
||||
def test_upstream_is_empty_under_null_so_a_chained_handler_is_still_callable():
|
||||
"""The one line that keeps a chained handler an ordinary function.
|
||||
|
||||
Every other handler in the pipeline can be called from a terminal. One
|
||||
that read `waited_for[0].result` directly could not — it would raise
|
||||
IndexError under `NULL` — so `upstream()` answers `{}` and the handler
|
||||
falls back to the keyword it declared.
|
||||
"""
|
||||
assert NULL.upstream() == {}
|
||||
|
||||
@job("t_chained")
|
||||
def handler(config, store, *, context=NULL, html: str = "") -> dict:
|
||||
return {"html": html or context.upstream().get("html", "")}
|
||||
|
||||
assert handler(None, None, html="passed by hand") == {"html": "passed by hand"}
|
||||
assert handler(None, None) == {"html": ""}, "and it does not blow up with neither"
|
||||
|
||||
|
||||
# ----------------------------------------------------- a fan-out is one call
|
||||
|
||||
def test_fan_out_queues_the_children_and_the_job_that_reports_on_them(jobs):
|
||||
"""One call, and the reporting job is not the app's to write.
|
||||
|
||||
The children get a run of their own inside the caller's cascade, and the
|
||||
report waits on all of them with `after='done'` — the mode that exists
|
||||
precisely so a job reporting that one of three failed is not held back by
|
||||
that failure.
|
||||
"""
|
||||
queue = JobQueue(jobs, None, None)
|
||||
parent = got(jobs.enqueue("t_parent"))
|
||||
context = queue._context(parent, [], [])
|
||||
|
||||
children = context.fan_out(
|
||||
"t_child", [{"i": 1}, {"i": 2}, {"i": 3}], then="mod:fn")
|
||||
|
||||
assert len(children) == 3, "it answers with the children's ids"
|
||||
queued = jobs.snapshot().queued
|
||||
report = [j for j in queued if j.kind == "advance"]
|
||||
assert len(report) == 1, "and queued exactly one job to report on them"
|
||||
assert sorted(report[0].depends_on) == sorted(children)
|
||||
assert report[0].after == "done"
|
||||
assert report[0].payload["callback"] == "mod:fn"
|
||||
assert {j.task_id for j in queued if j.id in children} == {report[0].task_id}, \
|
||||
"children and report are one run"
|
||||
|
||||
|
||||
def test_fan_out_queues_nothing_under_null_and_says_so():
|
||||
"""A handler called directly is being run by somebody who is standing
|
||||
there, and the useful answer is to do its own work and report that nothing
|
||||
was queued behind it — the same answer `task` gives, for the same reason."""
|
||||
assert NULL.fan_out("t_child", [{"i": 1}]) == ()
|
||||
|
||||
|
||||
def test_a_fan_out_carries_what_the_caller_puts_on_it(jobs):
|
||||
"""`carries` goes *around* the children to the callback: static input from
|
||||
whoever asked, as against `Tally.results`, which is what the children
|
||||
themselves returned."""
|
||||
queue = JobQueue(jobs, None, None)
|
||||
parent = got(jobs.enqueue("t_parent"))
|
||||
context = queue._context(parent, [], [])
|
||||
|
||||
context.fan_out("t_child", [{"i": 1}], then="mod:fn", carries={"since": "monday"})
|
||||
|
||||
report = [j for j in jobs.snapshot().queued if j.kind == "advance"]
|
||||
assert report[0].payload["payload"] == {"since": "monday"}
|
||||
@@ -0,0 +1,134 @@
|
||||
"""The notice log: said once, kept until cleared, forgotten when it stops.
|
||||
|
||||
A notice used to be a line that faded in twelve seconds in whichever tab was
|
||||
open, and was raised as often as not while none was. These hold the three
|
||||
things the table is for: a repeat is the same row and not news, a cleared row
|
||||
makes the next one news again, and what nobody clears goes when it has stopped
|
||||
being said rather than when it was first said.
|
||||
|
||||
python -m pytest pipeline/tests/test_notice_log.py -q
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
from contextlib import closing
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from pipeline import NULL, JobQueue, Jobs, Notice, RunContext, job
|
||||
from pipeline.web import connect
|
||||
|
||||
|
||||
@job("n_refused", "Meeting the same refusal as last time")
|
||||
def n_refused(config, store, *, context: RunContext = NULL) -> dict:
|
||||
context.notice(Notice(code="agent.refused", key="Worm hits npm",
|
||||
text="No abstract: Worm hits npm", fix="Nothing to do."))
|
||||
return {}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def jobs():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
yield Jobs(Path(tmp) / "n.db")
|
||||
|
||||
|
||||
def refusal(text: str = "No abstract: Worm hits npm") -> Notice:
|
||||
return Notice(code="agent.refused", key="Worm hits npm", text=text,
|
||||
where="the tagging lines in the server log")
|
||||
|
||||
|
||||
def test_the_same_notice_again_is_the_same_row_with_a_count(jobs):
|
||||
first = jobs.record_notice(refusal())
|
||||
again = jobs.record_notice(refusal("No abstract, again: Worm hits npm"))
|
||||
other = jobs.record_notice(Notice(code="agent.refused", key="Other", text="x"))
|
||||
|
||||
assert (first.count, again.count) == (1, 2), "one is news, two is a repeat"
|
||||
assert again.id == first.id and other.id != first.id, "keyed by code and key"
|
||||
assert again.first_at == first.first_at and again.last_at >= first.last_at
|
||||
assert again.text.startswith("No abstract, again"), "the newer wording is kept"
|
||||
assert again.where == first.where, "and `where` survives its SQL column name"
|
||||
assert [n.key for n in jobs.notices()] == ["Other", "Worm hits npm"]
|
||||
|
||||
|
||||
def test_clearing_one_makes_the_next_occurrence_news_again(jobs):
|
||||
said = jobs.record_notice(refusal())
|
||||
|
||||
assert jobs.clear_notice(said.id)
|
||||
assert not jobs.clear_notice(said.id), "gone already is an answer, not an error"
|
||||
assert jobs.record_notice(refusal()).count == 1
|
||||
|
||||
|
||||
def test_a_notice_is_forgotten_when_it_stops_being_said_not_when_it_began(jobs):
|
||||
"""By `last_at`. A feed that has failed nightly for a month is an old row
|
||||
that was true an hour ago, and dropping it would make tonight's failure
|
||||
read as the first."""
|
||||
chronic = jobs.record_notice(Notice(code="source.failed", key="rss", text="x"))
|
||||
over = jobs.record_notice(Notice(code="source.failed", key="old", text="y"))
|
||||
month = (datetime.now(UTC) - timedelta(days=30)).isoformat()
|
||||
with closing(sqlite3.connect(jobs.path)) as conn, conn:
|
||||
conn.execute("UPDATE notices SET first_at=? WHERE id=?", (month, chronic.id))
|
||||
conn.execute("UPDATE notices SET first_at=?, last_at=? WHERE id=?",
|
||||
(month, month, over.id))
|
||||
|
||||
assert jobs.purge_notices(14 * 86400) == 1
|
||||
assert [n.key for n in jobs.notices()] == ["rss"]
|
||||
|
||||
|
||||
def test_a_job_that_repeats_itself_is_heard_as_a_repeat(jobs):
|
||||
"""The whole path. Every notice still reaches the listener — what
|
||||
interrupts a person is the interface's business — and the second carries
|
||||
the count that says it is nothing new."""
|
||||
heard: list = []
|
||||
|
||||
async def on_notice(notice) -> None:
|
||||
heard.append(notice)
|
||||
|
||||
for _ in range(2):
|
||||
jobs.enqueue("n_refused")
|
||||
JobQueue(jobs, None, None, on_notice=on_notice).drain(timeout=10)
|
||||
|
||||
assert [n.count for n in heard] == [1, 2]
|
||||
assert len(jobs.notices()) == 1
|
||||
|
||||
|
||||
def test_a_log_that_cannot_be_written_does_not_cost_the_notice(jobs, monkeypatch):
|
||||
heard: list = []
|
||||
|
||||
async def on_notice(notice) -> None:
|
||||
heard.append(notice)
|
||||
|
||||
def broken(notice):
|
||||
raise sqlite3.OperationalError("database is locked")
|
||||
|
||||
monkeypatch.setattr(jobs, "record_notice", broken)
|
||||
done = jobs.enqueue("n_refused")
|
||||
JobQueue(jobs, None, None, on_notice=on_notice).drain(timeout=10)
|
||||
|
||||
assert [n.code for n in heard] == ["agent.refused"]
|
||||
assert jobs.get(done.id).state == "done", "and the job that said it is unharmed"
|
||||
|
||||
|
||||
def test_the_log_is_read_and_cleared_over_http(jobs):
|
||||
app = FastAPI()
|
||||
connect(app, jobs, None, None)
|
||||
jobs.record_notice(refusal())
|
||||
jobs.record_notice(Notice(code="source.failed", key="rss", text="rss failed"))
|
||||
|
||||
with TestClient(app) as client:
|
||||
listed = client.get("/api/notices").json()
|
||||
assert [n["code"] for n in listed] == ["source.failed", "agent.refused"]
|
||||
assert listed[1]["where"] == "the tagging lines in the server log"
|
||||
|
||||
assert client.delete(f"/api/notices/{listed[0]['id']}").json() == {"cleared": True}
|
||||
assert client.delete(f"/api/notices/{listed[0]['id']}").json() == {"cleared": False}
|
||||
assert client.delete("/api/notices").json() == {"cleared": 1}
|
||||
assert client.get("/api/notices").json() == []
|
||||
@@ -0,0 +1,355 @@
|
||||
"""Waiting to be tried again, without anything being alive while it waits.
|
||||
|
||||
The claim under test is that a job between attempts costs a row and nothing
|
||||
else. No worker, no thread, no lease, no timer — so the tests that matter here
|
||||
are the ones that ask what *else* the queue can do while a job waits, and what
|
||||
survives the process that scheduled the wait.
|
||||
|
||||
Each of the four properties step 12 asks for is pinned on its own, because
|
||||
they fail independently: a bound that is off by one still waits correctly, a
|
||||
`run_after` that never reaches the table still retries in one process, and a
|
||||
Kill that cannot reach a waiting row is invisible until somebody presses it.
|
||||
|
||||
python3 -m pytest tests/test_retries.py -q
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from contextlib import closing
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from pipeline.models import Job
|
||||
from pipeline import NULL, JobQueue, Jobs, Retry, job
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def jobs():
|
||||
"""The pipeline's table on a file of its own. No app, no store — nothing
|
||||
here needs one, which is the point of the table owning its own schema."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
yield Jobs(Path(tmp) / "t.db")
|
||||
|
||||
|
||||
def made(queued: Job | None) -> Job:
|
||||
"""The job an enqueue made. `enqueue` answers None for a killed run, which
|
||||
in these tests nothing is."""
|
||||
assert queued is not None
|
||||
return queued
|
||||
|
||||
|
||||
def got(found: Job | None) -> Job:
|
||||
"""The job a read must have found — a None here is a test failure."""
|
||||
assert found is not None
|
||||
return found
|
||||
|
||||
|
||||
def lease(jobs: Jobs, job_id: int) -> tuple[str | None, str | None]:
|
||||
"""The `started_at` and `seen_at` of one job: whether anybody has it."""
|
||||
with closing(sqlite3.connect(jobs.path)) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT started_at, seen_at FROM jobs WHERE id=?", (job_id,)).fetchone()
|
||||
return row[0], row[1]
|
||||
|
||||
|
||||
def bring_forward(jobs: Jobs, job_id: int) -> None:
|
||||
"""Move a waiting job's next attempt into the past, as if its wait were
|
||||
over. What the clock does, done in one statement so a test of an hour's
|
||||
backoff does not take an hour."""
|
||||
with closing(sqlite3.connect(jobs.path)) as conn, conn:
|
||||
conn.execute("UPDATE jobs SET run_after=? WHERE id=?",
|
||||
((datetime.now(UTC) - timedelta(hours=1)).isoformat(), job_id))
|
||||
|
||||
|
||||
def drain(queue: JobQueue, *, timeout: float = 5.0) -> None:
|
||||
"""Run the worker until the table is empty and this worker holds nothing.
|
||||
|
||||
Both halves, because a job's last acts belong to its handler thread and
|
||||
between them the table reads as idle while it is not. A job waiting for its
|
||||
next attempt keeps this going, since it is still `queued` — which is the
|
||||
behaviour the retry tests are leaning on.
|
||||
"""
|
||||
|
||||
async def go() -> None:
|
||||
await queue.start()
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
snap = queue.jobs.snapshot()
|
||||
if not snap.running and not snap.queued and not queue._running:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
await queue.stop()
|
||||
|
||||
asyncio.run(go())
|
||||
|
||||
|
||||
# ------------------------------------------------------- not holding a place
|
||||
|
||||
def test_a_failed_attempt_goes_back_to_the_queue_with_a_time_on_it(jobs):
|
||||
"""The resting state of a retry, and everything that is true of it."""
|
||||
queued = made(jobs.enqueue("r_wait", max_attempts=3, retry_delay=600))
|
||||
assert got(jobs.claim({})).attempt == 1, "claim counts the attempt"
|
||||
|
||||
again = got(jobs.retry(queued.id, error="RuntimeError: the feed was down"))
|
||||
assert again.state == "queued", "it is in the queue, not failed"
|
||||
assert again.waiting and not again.exhausted
|
||||
assert again.error == "RuntimeError: the feed was down", "it says why it waits"
|
||||
assert lease(jobs, queued.id) == (None, None), "nobody holds it"
|
||||
assert jobs.running() == [], "and nothing is running it"
|
||||
assert [j.id for j in jobs.waiting()] == [queued.id]
|
||||
assert [j.id for j in jobs.blocked()] == [queued.id]
|
||||
|
||||
|
||||
def test_a_waiting_job_does_not_occupy_a_worker(jobs):
|
||||
"""The whole of why the wait is a column. One cpu worker, one job waiting
|
||||
ten minutes for its next attempt, and the worker is free the entire time —
|
||||
a handler that slept would have held it and stopped the queue dead."""
|
||||
waiting = made(jobs.enqueue("r_slot", max_attempts=2, retry_delay=600))
|
||||
assert jobs.claim({"cpu": 1}) is not None
|
||||
jobs.retry(waiting.id, error="not this time")
|
||||
|
||||
other = made(jobs.enqueue("r_other"))
|
||||
assert got(jobs.claim({"cpu": 1})).id == other.id, "the slot was free"
|
||||
assert got(jobs.get(waiting.id)).state == "queued"
|
||||
|
||||
|
||||
def test_a_job_queued_for_later_is_not_claimed_before_its_time(jobs):
|
||||
"""`delay` on the enqueue itself: the durable form of "in ten minutes"."""
|
||||
later = made(jobs.enqueue("r_later", delay=600))
|
||||
assert later.waiting
|
||||
assert jobs.claim({}) is None
|
||||
bring_forward(jobs, later.id)
|
||||
assert got(jobs.claim({})).id == later.id
|
||||
|
||||
|
||||
def test_what_waits_on_a_retrying_job_goes_on_waiting(jobs):
|
||||
"""A retry is still live work, so a dependent is held rather than settled.
|
||||
|
||||
The alternative would be the worse half of both: a batch that is coming
|
||||
back in a minute counted among the ones that did not land, and a final job
|
||||
reporting on a task that is not over.
|
||||
"""
|
||||
batch = made(jobs.enqueue("r_batch", max_attempts=2, retry_delay=600))
|
||||
final = made(jobs.enqueue("r_final", depends_on=[batch.id], after="done"))
|
||||
jobs.claim({})
|
||||
jobs.retry(batch.id, error="again in a minute")
|
||||
|
||||
assert [j.id for j in jobs.blocked()] == [batch.id, final.id]
|
||||
bring_forward(jobs, batch.id)
|
||||
assert got(jobs.claim({})).id == batch.id, "the batch goes first"
|
||||
jobs.finish(batch.id, "done")
|
||||
assert got(jobs.claim({})).id == final.id
|
||||
|
||||
|
||||
# --------------------------------------------------------------- the bounds
|
||||
|
||||
def test_the_bound_counts_the_first_attempt(jobs):
|
||||
"""`max_attempts=3` is three runs of the handler, not three retries."""
|
||||
runs = []
|
||||
|
||||
@job("r_bounded", "Bounded", max_attempts=3)
|
||||
def handler(config, store, *, context=NULL):
|
||||
runs.append(1)
|
||||
raise RuntimeError("this was never going to work")
|
||||
|
||||
queue = JobQueue(jobs, None, None)
|
||||
queued = made(queue.enqueue("r_bounded"))
|
||||
drain(queue)
|
||||
|
||||
ended = got(jobs.get(queued.id))
|
||||
assert len(runs) == 3, "it ran exactly as many times as it was allowed"
|
||||
assert ended.state == "failed" and ended.attempt == 3
|
||||
assert ended.exhausted, "and says the policy ran out rather than merely failing"
|
||||
|
||||
|
||||
def test_a_job_with_no_policy_fails_at_the_first_exception(jobs):
|
||||
"""The default is one attempt, which is every job queued before any of
|
||||
this existed. Exhausted is not what that is: nothing was used up."""
|
||||
runs = []
|
||||
|
||||
@job("r_unbounded", "Unbounded")
|
||||
def handler(config, store, *, context=NULL):
|
||||
runs.append(1)
|
||||
raise RuntimeError("no")
|
||||
|
||||
queue = JobQueue(jobs, None, None)
|
||||
queued = made(queue.enqueue("r_unbounded"))
|
||||
drain(queue)
|
||||
|
||||
ended = got(jobs.get(queued.id))
|
||||
assert len(runs) == 1
|
||||
assert ended.state == "failed" and not ended.exhausted
|
||||
|
||||
|
||||
def test_an_attempt_that_lands_stops_the_retrying(jobs):
|
||||
"""The case the bound exists for: the second go works, and the job is done
|
||||
rather than carrying a memory of having failed once."""
|
||||
runs = []
|
||||
|
||||
@job("r_second_time", "Second time", max_attempts=4)
|
||||
def handler(config, store, *, context=NULL):
|
||||
runs.append(1)
|
||||
if len(runs) < 2:
|
||||
raise RuntimeError("the first one never works")
|
||||
return {"runs": len(runs)}
|
||||
|
||||
queue = JobQueue(jobs, None, None)
|
||||
queued = made(queue.enqueue("r_second_time"))
|
||||
drain(queue)
|
||||
|
||||
ended = got(jobs.get(queued.id))
|
||||
assert ended.state == "done" and ended.result == {"runs": 2}
|
||||
assert ended.attempt == 2 and not ended.exhausted
|
||||
|
||||
|
||||
def test_the_wait_lengthens_with_the_backoff(jobs):
|
||||
"""Doubling, read off the two stamps rather than waited out."""
|
||||
queued = made(jobs.enqueue("r_backoff", max_attempts=4,
|
||||
retry_delay=100, retry_backoff=2.0))
|
||||
|
||||
waits = []
|
||||
for _ in range(3):
|
||||
assert jobs.claim({}) is not None
|
||||
again = got(jobs.retry(queued.id, error="again"))
|
||||
waits.append(
|
||||
datetime.fromisoformat(again.run_after) - datetime.now(UTC))
|
||||
bring_forward(jobs, queued.id)
|
||||
|
||||
seconds = [w.total_seconds() for w in waits]
|
||||
assert [round(s / 100) for s in seconds] == [1, 2, 4]
|
||||
|
||||
|
||||
def test_a_server_hint_replaces_the_backoff(jobs):
|
||||
"""Honouring what the other end said, rather than guessing at it.
|
||||
|
||||
The bound is not part of the bargain: a hint moves the next attempt, it
|
||||
does not add one.
|
||||
"""
|
||||
queued = made(jobs.enqueue("r_hinted", max_attempts=3, retry_delay=600))
|
||||
jobs.claim({})
|
||||
again = got(jobs.retry(queued.id, error="429", delay=5))
|
||||
waited = datetime.fromisoformat(again.run_after) - datetime.now(UTC)
|
||||
assert 0 < waited.total_seconds() <= 5
|
||||
|
||||
|
||||
def test_a_handler_can_name_its_own_wait(jobs):
|
||||
"""`Retry(after=...)` from inside the handler, which is where a
|
||||
`Retry-After` header is actually read."""
|
||||
|
||||
@job("r_asked", "Asked to come back", max_attempts=3, retry_delay=600)
|
||||
def handler(config, store, *, context=NULL):
|
||||
raise Retry("rate limited", after=7)
|
||||
|
||||
queue = JobQueue(jobs, None, None)
|
||||
queued = made(queue.enqueue("r_asked"))
|
||||
|
||||
async def one_pass() -> None:
|
||||
await queue.start()
|
||||
deadline = time.monotonic() + 5
|
||||
while time.monotonic() < deadline:
|
||||
if got(jobs.get(queued.id)).waiting:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
await queue.stop()
|
||||
|
||||
asyncio.run(one_pass())
|
||||
|
||||
waiting = got(jobs.get(queued.id))
|
||||
assert waiting.waiting and waiting.attempt == 1
|
||||
assert "rate limited" in waiting.error
|
||||
waited = datetime.fromisoformat(waiting.run_after) - datetime.now(UTC)
|
||||
assert 0 < waited.total_seconds() <= 7, "the handler's number, not the policy's"
|
||||
|
||||
|
||||
def test_asking_to_come_back_does_not_extend_the_bound(jobs):
|
||||
"""A handler cannot grant itself a policy nobody declared."""
|
||||
runs = []
|
||||
|
||||
@job("r_asked_once", "Asked once")
|
||||
def handler(config, store, *, context=NULL):
|
||||
runs.append(1)
|
||||
raise Retry("come back in a second", after=0)
|
||||
|
||||
queue = JobQueue(jobs, None, None)
|
||||
queued = made(queue.enqueue("r_asked_once"))
|
||||
drain(queue)
|
||||
|
||||
assert len(runs) == 1
|
||||
assert got(jobs.get(queued.id)).state == "failed"
|
||||
|
||||
|
||||
# ------------------------------------------------------------- across a stop
|
||||
|
||||
def test_the_next_attempt_survives_the_process_that_scheduled_it(jobs):
|
||||
"""The wait is a column, so it outlives everything that is not the file.
|
||||
|
||||
A second `Jobs` on the same path is what another process sees, and what
|
||||
this one sees after a restart: a queued row it will not take yet, and takes
|
||||
the moment its time comes round. Nothing was handed over, because there
|
||||
was nothing alive to hand.
|
||||
"""
|
||||
queued = made(jobs.enqueue("r_restart", max_attempts=3, retry_delay=600))
|
||||
jobs.claim({})
|
||||
jobs.retry(queued.id, error="the feed was down")
|
||||
|
||||
restarted = Jobs(jobs.path)
|
||||
assert restarted.claim({}) is None, "still waiting, on a table that never saw it fail"
|
||||
assert [j.id for j in restarted.waiting()] == [queued.id]
|
||||
|
||||
bring_forward(restarted, queued.id)
|
||||
taken = got(restarted.claim({}))
|
||||
assert taken.id == queued.id
|
||||
assert taken.attempt == 2, "and it is the second attempt, not the first again"
|
||||
|
||||
|
||||
# ------------------------------------------------------------- cancellation
|
||||
|
||||
def test_cancelling_a_job_that_is_waiting_to_be_retried_ends_it(jobs):
|
||||
"""Kill on a waiting row. It is queued, so it ends here and now — there is
|
||||
no worker to ask and nothing part-way through."""
|
||||
queued = made(jobs.enqueue("r_killable", max_attempts=3, retry_delay=600))
|
||||
jobs.claim({})
|
||||
jobs.retry(queued.id, error="not yet")
|
||||
|
||||
ended = got(jobs.request_cancel(queued.id))
|
||||
assert ended.state == "cancelled"
|
||||
assert jobs.waiting() == []
|
||||
|
||||
|
||||
def test_killing_the_run_takes_a_job_waiting_to_be_retried_with_it(jobs):
|
||||
"""The same for the task and cascade buttons: an hour's wait is not an
|
||||
hour during which a run cannot be stopped."""
|
||||
cascade = jobs.new_run("cascade", "r_run")
|
||||
task = jobs.new_run("task", "r_task")
|
||||
queued = made(jobs.enqueue("r_in_a_run", max_attempts=3, retry_delay=600,
|
||||
cascade_id=cascade, task_id=task))
|
||||
jobs.claim({})
|
||||
jobs.retry(queued.id, error="not yet")
|
||||
|
||||
assert jobs.cancel_run(task) == 1
|
||||
assert got(jobs.get(queued.id)).state == "cancelled"
|
||||
|
||||
|
||||
def test_a_kill_during_the_attempt_is_not_undone_by_the_retry(jobs):
|
||||
"""The one ordering that would make Kill a lie.
|
||||
|
||||
Press it while the attempt runs, the attempt then fails, and a retry that
|
||||
only looked at the bound would put the job straight back in the queue —
|
||||
a button that stopped nothing and said it had.
|
||||
"""
|
||||
queued = made(jobs.enqueue("r_killed_mid", max_attempts=3, retry_delay=0))
|
||||
jobs.claim({})
|
||||
jobs.request_cancel(queued.id)
|
||||
|
||||
ended = got(jobs.retry(queued.id, error="it failed as well"))
|
||||
assert ended.state == "cancelled"
|
||||
assert not ended.waiting
|
||||
@@ -0,0 +1,701 @@
|
||||
"""The scheduler: the five wordings, the table, the firing, and the routes.
|
||||
|
||||
python -m pytest pipeline/tests/test_schedule.py -q
|
||||
|
||||
Three things here are worth knowing before changing anything.
|
||||
|
||||
The daylight-saving tests are the reason `next_after` takes a zone at all, and
|
||||
they are written around a real transition rather than a synthetic one, because
|
||||
the bug they guard is precisely that `+ timedelta(days=1)` looks correct.
|
||||
|
||||
The firing tests call `JobQueue._fire_due` directly. It is a private, and that
|
||||
is deliberate: driving it through `start()` would mean asserting on a loop that
|
||||
ticks on a clock, and a test that sleeps to make a scheduler fire is a test
|
||||
that fails on a busy machine. One test does start a real queue, to prove the
|
||||
tick is wired into the loop at all.
|
||||
|
||||
`SCHEDULES` is a module-level registry, like `JOBS`, so anything here that
|
||||
declares into it does so through the `declared` fixture, which puts it back.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from pipeline import NULL, JobQueue, Jobs, RunContext, job
|
||||
from pipeline.schedule import SCHEDULES, BadCadence, Cadence, declare
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
COPENHAGEN = ZoneInfo("Europe/Copenhagen")
|
||||
|
||||
|
||||
@job("s_tick", "A scheduled tick")
|
||||
def s_tick(config, store, *, context: RunContext = NULL, note: str = "") -> dict:
|
||||
store.append(note)
|
||||
return {"note": note}
|
||||
|
||||
|
||||
@job("s_solo", "One at a time", solo=True)
|
||||
def s_solo(config, store, *, context: RunContext = NULL) -> dict:
|
||||
return {}
|
||||
|
||||
|
||||
@job("s_needs", "Needing a payload")
|
||||
def s_needs(config, store, *, context: RunContext = NULL, target: str) -> dict:
|
||||
return {"target": target}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def jobs():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
yield Jobs(Path(tmp) / "s.db")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def queue(jobs):
|
||||
"""A queue that is never started: the tests below fire it by hand.
|
||||
|
||||
`tz="UTC"` so a cadence's next run is the same number wherever this runs.
|
||||
"""
|
||||
return JobQueue(jobs, {}, [], tz="UTC")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def declared():
|
||||
"""Borrow the declaration registry and give it back.
|
||||
|
||||
`SCHEDULES` is global, like `JOBS`, and a test that left something in it
|
||||
would give every later test in the session a schedule it never asked for.
|
||||
"""
|
||||
saved = dict(SCHEDULES)
|
||||
SCHEDULES.clear()
|
||||
yield SCHEDULES
|
||||
SCHEDULES.clear()
|
||||
SCHEDULES.update(saved)
|
||||
|
||||
|
||||
def ago(seconds: float) -> str:
|
||||
return (datetime.now(UTC) - timedelta(seconds=seconds)).isoformat()
|
||||
|
||||
|
||||
# ------------------------------------------------------------ the grammar
|
||||
|
||||
WORDINGS = [
|
||||
("every 15m", "every 15m"),
|
||||
("every 6h", "every 6h"),
|
||||
("every 2d", "every 2d"),
|
||||
("daily at 06:00", "daily at 06:00"),
|
||||
("weekly mon 07:00", "weekly mon 07:00"),
|
||||
# Written how people write them, stored how the table stores them.
|
||||
("weekly monday 07:00", "weekly mon 07:00"),
|
||||
("Weekly Sunday 23:59", "weekly sun 23:59"),
|
||||
("EVERY 15 M", "every 15m"),
|
||||
(" daily at 6:05 ", "daily at 06:05"),
|
||||
("every 1d", "every 1d"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("typed,canonical", WORDINGS)
|
||||
def test_every_wording_parses_and_comes_back_canonical(typed, canonical):
|
||||
"""Five forms, one spelling each.
|
||||
|
||||
The round trip is the point: the table stores `str(cadence)`, so two
|
||||
people writing the same schedule two ways end up with one row that reads
|
||||
the same to both of them.
|
||||
"""
|
||||
cadence = Cadence.parse(typed)
|
||||
assert str(cadence) == canonical
|
||||
assert str(Cadence.parse(canonical)) == canonical
|
||||
assert Cadence.parse(typed) == Cadence.parse(canonical)
|
||||
|
||||
|
||||
def test_the_units_are_what_they_say():
|
||||
assert Cadence.parse("every 15m").seconds == 900
|
||||
assert Cadence.parse("every 6h").seconds == 6 * 3600
|
||||
assert Cadence.parse("every 2d").seconds == 2 * 86400
|
||||
# A clock cadence has no fixed length, and says so rather than guessing.
|
||||
assert Cadence.parse("daily at 06:00").seconds == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("wording", [
|
||||
"sometimes", "", "cron 0 6 * * *", "every 15", "every m", "every 0h",
|
||||
"every 15s", "daily 06:00", "daily at 25:00", "daily at 06:60",
|
||||
"weekly 07:00", "weekly funday 07:00", "every 15m please",
|
||||
])
|
||||
def test_a_wording_outside_the_five_is_refused_with_all_five(wording):
|
||||
"""And the refusal is the sentence a person is shown.
|
||||
|
||||
It goes out of the API as `{"detail": ...}` and into a toast, so it has to
|
||||
say what to type instead rather than merely that this was wrong.
|
||||
"""
|
||||
with pytest.raises(ValueError) as refused:
|
||||
Cadence.parse(wording)
|
||||
said = str(refused.value)
|
||||
assert isinstance(refused.value, BadCadence)
|
||||
for form in ("every 15m", "every 6h", "every 2d",
|
||||
"daily at 06:00", "weekly mon 07:00"):
|
||||
assert form in said
|
||||
|
||||
|
||||
# ------------------------------------------------------------- next_after
|
||||
|
||||
def test_next_after_is_strictly_after():
|
||||
"""A schedule fired at its own time must not find itself due again on the
|
||||
same instant, which would be one firing per tick for ever."""
|
||||
at_six = datetime(2025, 6, 1, 6, 0, tzinfo=COPENHAGEN)
|
||||
when = Cadence.parse("daily at 06:00").next_after(at_six, COPENHAGEN)
|
||||
assert when.astimezone(COPENHAGEN) == at_six + timedelta(days=1)
|
||||
|
||||
|
||||
def test_next_after_always_comes_back_in_utc():
|
||||
"""Because it is written straight into the table, where every other time
|
||||
is UTC and the column is compared as text."""
|
||||
when = Cadence.parse("daily at 06:00").next_after(
|
||||
datetime(2025, 6, 1, 12, 0, tzinfo=COPENHAGEN), COPENHAGEN)
|
||||
assert when.tzinfo is UTC
|
||||
assert when.isoformat().endswith("+00:00")
|
||||
|
||||
|
||||
def test_weekly_lands_on_the_day_it_names():
|
||||
monday = Cadence.parse("weekly mon 07:00")
|
||||
# A Wednesday: the next Monday is five days away, not tomorrow.
|
||||
from_wed = monday.next_after(datetime(2025, 6, 4, 9, 0, tzinfo=COPENHAGEN),
|
||||
COPENHAGEN).astimezone(COPENHAGEN)
|
||||
assert from_wed.weekday() == 0 and (from_wed.hour, from_wed.minute) == (7, 0)
|
||||
assert from_wed.date() == datetime(2025, 6, 9).date()
|
||||
# Monday morning before seven: today, not next week.
|
||||
from_mon = monday.next_after(datetime(2025, 6, 9, 6, 0, tzinfo=COPENHAGEN),
|
||||
COPENHAGEN).astimezone(COPENHAGEN)
|
||||
assert from_mon.date() == datetime(2025, 6, 9).date()
|
||||
|
||||
|
||||
def test_daily_keeps_its_wall_clock_across_both_dst_changes():
|
||||
"""06:00 is 06:00 on both sides, which is the whole reason for the zone.
|
||||
|
||||
Spring forward is a 23-hour day and autumn back is a 25-hour one, so the
|
||||
real elapsed time between two runs is not 24 hours — and it must not be,
|
||||
because what the wording promised is a time of day.
|
||||
"""
|
||||
daily = Cadence.parse("daily at 06:00")
|
||||
|
||||
# Europe/Copenhagen, 2025: forward on 30 March, back on 26 October.
|
||||
forward = daily.next_after(
|
||||
datetime(2025, 3, 29, 6, 30, tzinfo=COPENHAGEN), COPENHAGEN)
|
||||
local = forward.astimezone(COPENHAGEN)
|
||||
assert (local.hour, local.minute) == (6, 0)
|
||||
assert local.date() == datetime(2025, 3, 30).date()
|
||||
# 05:30Z to 04:00Z: 22h30m of real time for a 23h30m step of the clock.
|
||||
assert forward - datetime(2025, 3, 29, 6, 30, tzinfo=COPENHAGEN) \
|
||||
== timedelta(hours=22, minutes=30)
|
||||
|
||||
back = daily.next_after(
|
||||
datetime(2025, 10, 25, 6, 30, tzinfo=COPENHAGEN), COPENHAGEN)
|
||||
local = back.astimezone(COPENHAGEN)
|
||||
assert (local.hour, local.minute) == (6, 0)
|
||||
assert local.date() == datetime(2025, 10, 26).date()
|
||||
assert back - datetime(2025, 10, 25, 6, 30, tzinfo=COPENHAGEN) \
|
||||
== timedelta(hours=24, minutes=30)
|
||||
|
||||
|
||||
def test_weekly_keeps_its_wall_clock_across_a_dst_change():
|
||||
when = Cadence.parse("weekly sun 07:00").next_after(
|
||||
datetime(2025, 3, 26, 9, 0, tzinfo=COPENHAGEN), COPENHAGEN)
|
||||
local = when.astimezone(COPENHAGEN)
|
||||
assert local.date() == datetime(2025, 3, 30).date() # the change itself
|
||||
assert (local.hour, local.minute) == (7, 0)
|
||||
|
||||
|
||||
def test_an_interval_is_elapsed_time_and_ignores_the_change():
|
||||
"""The other half of the promise, and the reason both wordings exist.
|
||||
|
||||
`every 24h` across a spring-forward is 24 real hours and therefore lands
|
||||
an hour later on the clock. A person who wanted 06:00 wrote `daily`.
|
||||
"""
|
||||
start = datetime(2025, 3, 29, 6, 30, tzinfo=COPENHAGEN)
|
||||
when = Cadence.parse("every 24h").next_after(start, COPENHAGEN)
|
||||
assert when - start == timedelta(hours=24)
|
||||
assert when.astimezone(COPENHAGEN).hour == 7
|
||||
|
||||
|
||||
def test_a_naive_moment_is_read_as_utc():
|
||||
"""Rather than raising. Every stamp in the table is UTC, so a caller that
|
||||
lost the tzinfo on the way is very nearly certainly holding one."""
|
||||
naive = datetime(2025, 6, 1, 12, 0)
|
||||
assert Cadence.parse("every 1h").next_after(naive) \
|
||||
== datetime(2025, 6, 1, 13, 0, tzinfo=UTC)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the table
|
||||
|
||||
def test_the_table_holds_a_schedule_through_its_life(jobs):
|
||||
row = jobs.add_schedule("nightly", "s_tick", "daily at 06:00",
|
||||
payload={"note": "hello"}, label="A nightly tick",
|
||||
next_run="2030-01-01T06:00:00+00:00")
|
||||
assert row is not None
|
||||
assert (row.name, row.kind, row.cadence) == ("nightly", "s_tick", "daily at 06:00")
|
||||
assert row.payload == {"note": "hello"} and row.enabled is True
|
||||
assert row.last_job_id == 0 and row.last_run == "" and row.last_state == ""
|
||||
|
||||
assert [s.name for s in jobs.schedules()] == ["nightly"]
|
||||
assert jobs.schedule(row.id) == row
|
||||
|
||||
edited = jobs.update_schedule(row.id, cadence="every 15m", enabled=False,
|
||||
payload={"note": "other"})
|
||||
assert edited.cadence == "every 15m" and edited.enabled is False
|
||||
assert edited.payload == {"note": "other"}
|
||||
# Untouched columns stay as they were.
|
||||
assert edited.name == "nightly" and edited.label == "A nightly tick"
|
||||
|
||||
assert jobs.delete_schedule(row.id) is True
|
||||
assert jobs.delete_schedule(row.id) is False # two tabs; not an error
|
||||
assert jobs.schedules() == []
|
||||
|
||||
|
||||
def test_a_name_is_taken_once(jobs):
|
||||
assert jobs.add_schedule("one", "s_tick", "every 15m") is not None
|
||||
assert jobs.add_schedule("one", "s_tick", "every 6h", only_if_new=True) is None
|
||||
assert jobs.schedules()[0].cadence == "every 15m"
|
||||
|
||||
|
||||
def test_schedules_carry_the_state_of_their_last_job(jobs):
|
||||
row = jobs.add_schedule("one", "s_tick", "every 15m")
|
||||
queued = jobs.enqueue("s_tick")
|
||||
jobs.mark_fired(row.id, queued.id, "2030-01-01T00:00:00+00:00")
|
||||
assert jobs.schedules()[0].last_state == "queued"
|
||||
jobs.claim({"cpu": 1})
|
||||
jobs.finish(queued.id, "failed", error="no")
|
||||
assert jobs.schedules()[0].last_state == "failed"
|
||||
# And the time it fired, which is not the time it is next due.
|
||||
listed = jobs.schedules()[0]
|
||||
assert listed.last_run and listed.next_run == "2030-01-01T00:00:00+00:00"
|
||||
|
||||
|
||||
def test_due_schedules_passes_over_what_is_off_or_not_yet(jobs):
|
||||
jobs.add_schedule("due", "s_tick", "every 15m", next_run=ago(60))
|
||||
jobs.add_schedule("off", "s_tick", "every 15m", next_run=ago(60), enabled=False)
|
||||
jobs.add_schedule("later", "s_tick", "every 15m",
|
||||
next_run="2099-01-01T00:00:00+00:00")
|
||||
jobs.add_schedule("never", "s_tick", "every 15m") # no time at all
|
||||
due = jobs.due_schedules(datetime.now(UTC).isoformat())
|
||||
assert [s.name for s in due] == ["due"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------- the firing
|
||||
|
||||
def test_the_tick_fires_a_due_row_and_moves_it_on(queue, jobs):
|
||||
row = jobs.add_schedule("regular", "s_tick", "every 15m",
|
||||
payload={"note": "scheduled"}, next_run=ago(30))
|
||||
queue._fire_due()
|
||||
|
||||
queued = jobs.snapshot().queued
|
||||
assert [j.kind for j in queued] == ["s_tick"]
|
||||
assert queued[0].payload == {"note": "scheduled"}
|
||||
|
||||
after = jobs.schedule(row.id)
|
||||
assert after.last_job_id == queued[0].id
|
||||
assert after.last_run
|
||||
# Fifteen minutes from now, not from the moment it was due.
|
||||
assert after.next_run > datetime.now(UTC).isoformat()
|
||||
assert after.next_run < (datetime.now(UTC) + timedelta(minutes=16)).isoformat()
|
||||
|
||||
|
||||
def test_a_schedule_does_not_pile_up_behind_its_own_job(queue, jobs):
|
||||
"""The job from the last firing is still queued, so this one is skipped.
|
||||
|
||||
Skipped and *moved on*, not left due: a schedule whose job takes longer
|
||||
than its cadence would otherwise fire on every tick for as long as the job
|
||||
ran, and the queue would fill with copies of work already under way.
|
||||
"""
|
||||
row = jobs.add_schedule("regular", "s_tick", "every 15m", next_run=ago(30))
|
||||
queue._fire_due()
|
||||
first = jobs.schedule(row.id).last_job_id
|
||||
|
||||
jobs.update_schedule(row.id, next_run=ago(1)) # due again
|
||||
queue._fire_due()
|
||||
|
||||
assert len(jobs.snapshot().queued) == 1
|
||||
again = jobs.schedule(row.id)
|
||||
assert again.last_job_id == first # nothing new was queued
|
||||
assert again.next_run > datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def test_a_solo_kind_already_underway_holds_its_schedule_back(queue, jobs):
|
||||
"""The other way a schedule collides: not with itself, with somebody else.
|
||||
|
||||
A button, a second schedule or a cascade started one, and `enqueue` would
|
||||
hand that job back — so the schedule would record work it did not ask for
|
||||
as its own firing.
|
||||
"""
|
||||
pressed = jobs.enqueue("s_solo", dedup_key="s_solo")
|
||||
row = jobs.add_schedule("solo", "s_solo", "every 15m", next_run=ago(30))
|
||||
queue._fire_due()
|
||||
after = jobs.schedule(row.id)
|
||||
assert after.last_job_id == 0 # it never fired
|
||||
assert after.next_run > datetime.now(UTC).isoformat()
|
||||
assert [j.id for j in jobs.snapshot().queued] == [pressed.id]
|
||||
|
||||
|
||||
def test_downtime_is_caught_up_once_and_not_once_per_slot(queue, jobs):
|
||||
"""Three days off an hourly schedule is one job, not seventy-two.
|
||||
|
||||
The row carries one `next_run`, and a firing sets it from *now* rather
|
||||
than stepping through what was missed — so there is nowhere for a backlog
|
||||
to accumulate even in principle.
|
||||
"""
|
||||
row = jobs.add_schedule("hourly", "s_tick", "every 1h", next_run=ago(3 * 86400))
|
||||
queue._fire_due()
|
||||
assert len(jobs.snapshot().queued) == 1
|
||||
after = jobs.schedule(row.id)
|
||||
assert after.next_run > datetime.now(UTC).isoformat()
|
||||
assert after.next_run < (datetime.now(UTC) + timedelta(minutes=61)).isoformat()
|
||||
|
||||
|
||||
def test_an_unknown_kind_never_fires_and_keeps_its_place(queue, jobs):
|
||||
"""A handler that went away in a deploy. The row stays exactly as it is —
|
||||
time included — so reinstalling the plugin runs it rather than starting
|
||||
another cadence from scratch."""
|
||||
row = jobs.add_schedule("gone", "s_vanished", "every 15m", next_run=ago(30))
|
||||
queue._fire_due()
|
||||
assert jobs.snapshot().queued == []
|
||||
assert jobs.schedule(row.id).next_run == row.next_run
|
||||
queue._fire_due() # and it says so once
|
||||
assert queue._said_about == {(row.id, "kind")}
|
||||
|
||||
|
||||
def test_a_cadence_the_table_cannot_parse_never_fires(queue, jobs):
|
||||
"""Only reachable by editing the file by hand, since every route parses
|
||||
first. It must not take the tick down with it."""
|
||||
jobs.add_schedule("bent", "s_tick", "every 15m", next_run=ago(30))
|
||||
jobs._write("UPDATE schedules SET cadence='sometimes'")
|
||||
queue._fire_due()
|
||||
assert jobs.snapshot().queued == []
|
||||
assert jobs.schedules()[0].cadence == "sometimes" # left to be fixed
|
||||
|
||||
|
||||
def test_a_disabled_row_is_not_fired(queue, jobs):
|
||||
jobs.add_schedule("off", "s_tick", "every 15m", next_run=ago(30), enabled=False)
|
||||
queue._fire_due()
|
||||
assert jobs.snapshot().queued == []
|
||||
|
||||
|
||||
def test_run_now_fires_without_moving_the_next_run(queue, jobs):
|
||||
row = jobs.add_schedule("nightly", "s_tick", "daily at 06:00",
|
||||
next_run="2099-01-01T06:00:00+00:00")
|
||||
started = queue.run_schedule(row.id)
|
||||
assert started is not None and started.kind == "s_tick"
|
||||
after = jobs.schedule(row.id)
|
||||
assert after.next_run == "2099-01-01T06:00:00+00:00"
|
||||
# But it did just run, and that is the job it queued — so the next tick
|
||||
# will not start a second one while this is still going.
|
||||
assert after.last_job_id == started.id and after.last_run
|
||||
|
||||
|
||||
# ------------------------------------------------------------ reconciliation
|
||||
|
||||
def test_declared_defaults_get_a_row_on_start(declared, jobs):
|
||||
declare("nightly tick", "s_tick", "daily at 06:00", {"note": "declared"},
|
||||
label="A tick, nightly")
|
||||
JobQueue(jobs, {}, [], tz="UTC")._reconcile_schedules()
|
||||
row = jobs.schedules()[0]
|
||||
assert (row.name, row.kind, row.cadence) == ("nightly tick", "s_tick",
|
||||
"daily at 06:00")
|
||||
assert row.payload == {"note": "declared"} and row.label == "A tick, nightly"
|
||||
assert row.next_run # and it knows when it is next
|
||||
|
||||
|
||||
def test_reconciliation_keeps_what_the_user_edited(declared, jobs):
|
||||
"""The whole of what a declaration means: a starting point, not a setting.
|
||||
|
||||
Switching a default off, or slowing it down, has to survive the next
|
||||
deploy — otherwise the only way to be rid of one is to edit the code and
|
||||
redeploy, which is not a thing the person running the app can do.
|
||||
"""
|
||||
declare("nightly tick", "s_tick", "daily at 06:00")
|
||||
queue = JobQueue(jobs, {}, [], tz="UTC")
|
||||
queue._reconcile_schedules()
|
||||
row = jobs.schedules()[0]
|
||||
jobs.update_schedule(row.id, cadence="every 2d", enabled=False,
|
||||
payload={"note": "mine"})
|
||||
|
||||
queue._reconcile_schedules() # a restart
|
||||
|
||||
again = jobs.schedules()[0]
|
||||
assert again.id == row.id
|
||||
assert again.cadence == "every 2d"
|
||||
assert again.enabled is False
|
||||
assert again.payload == {"note": "mine"}
|
||||
|
||||
|
||||
def test_a_declaration_that_goes_away_leaves_its_row(declared, jobs):
|
||||
"""Nothing here deletes. A deploy that silently forgot a schedule somebody
|
||||
was relying on is the failure this avoids; an orphan says `unknown kind`
|
||||
and waits to be deleted deliberately."""
|
||||
declare("nightly tick", "s_tick", "daily at 06:00")
|
||||
queue = JobQueue(jobs, {}, [], tz="UTC")
|
||||
queue._reconcile_schedules()
|
||||
declared.clear()
|
||||
queue._reconcile_schedules()
|
||||
assert [s.name for s in jobs.schedules()] == ["nightly tick"]
|
||||
|
||||
|
||||
def test_declaring_one_name_twice_differently_is_refused(declared):
|
||||
declare("nightly", "s_tick", "daily at 06:00")
|
||||
declare("nightly", "s_tick", "daily at 06:00") # the same: fine
|
||||
with pytest.raises(ValueError) as refused:
|
||||
declare("nightly", "s_tick", "daily at 07:00")
|
||||
assert "nightly" in str(refused.value)
|
||||
|
||||
|
||||
def test_a_declared_cadence_is_parsed_where_it_is_written(declared):
|
||||
"""At import, with a traceback pointing at the line — rather than at 06:00
|
||||
in a log nobody is reading."""
|
||||
with pytest.raises(ValueError):
|
||||
declare("bad", "s_tick", "every so often")
|
||||
assert "bad" not in declared
|
||||
|
||||
|
||||
# ------------------------------------------------------------- a real queue
|
||||
|
||||
def test_a_started_queue_fires_what_is_overdue(jobs, declared):
|
||||
"""The one test that drives the loop rather than the method.
|
||||
|
||||
Everything above proves the firing decides correctly; this proves it is
|
||||
wired into `start()` at all, which is the wire nobody notices is missing
|
||||
until a schedule silently stops running.
|
||||
"""
|
||||
ran: list[str] = []
|
||||
jobs.add_schedule("regular", "s_tick", "every 15m",
|
||||
payload={"note": "from the loop"}, next_run=ago(30))
|
||||
queue = JobQueue(jobs, {}, ran, tz="UTC", workers={"cpu": 1})
|
||||
assert queue.drain(timeout=10) is True
|
||||
assert ran == ["from the loop"]
|
||||
assert jobs.schedules()[0].last_state == "done"
|
||||
|
||||
|
||||
def test_a_queue_with_schedules_off_fires_nothing(jobs, declared):
|
||||
ran: list[str] = []
|
||||
jobs.add_schedule("regular", "s_tick", "every 15m", next_run=ago(30))
|
||||
queue = JobQueue(jobs, {}, ran, tz="UTC", schedules=False)
|
||||
assert queue.drain(timeout=10) is True
|
||||
assert ran == []
|
||||
assert jobs.schedules()[0].last_job_id == 0
|
||||
|
||||
|
||||
def test_an_unknown_timezone_is_a_warning_and_not_a_refusal(jobs, caplog):
|
||||
"""A mistyped zone must not be the reason a worker will not start."""
|
||||
queue = JobQueue(jobs, {}, [], tz="Mars/Olympus_Mons")
|
||||
assert queue.zone is not None
|
||||
assert Cadence.parse("daily at 06:00").next_after(datetime.now(UTC), queue.zone)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ the web
|
||||
|
||||
@pytest.fixture
|
||||
def client(jobs, declared):
|
||||
pytest.importorskip("fastapi")
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from pipeline.web import connect
|
||||
|
||||
app = FastAPI()
|
||||
connect(app, jobs, {}, [], tz="UTC", schedules=False)
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
|
||||
|
||||
def test_the_element_is_served(client):
|
||||
answer = client.get("/_jobs/schedules.js")
|
||||
assert answer.status_code == 200
|
||||
assert "javascript" in answer.headers["content-type"]
|
||||
assert "desk-schedules" in answer.text
|
||||
|
||||
|
||||
def test_adding_listing_editing_and_deleting(client):
|
||||
added = client.post("/api/schedules", json={
|
||||
"kind": "s_tick", "cadence": "weekly monday 07:00",
|
||||
"payload": {"note": "over http"}})
|
||||
assert added.status_code == 201
|
||||
row = added.json()
|
||||
# Named after its kind, since none was given, and stored canonically.
|
||||
assert row["name"] == "s_tick" and row["cadence"] == "weekly mon 07:00"
|
||||
assert row["next_run"] and row["unknown"] is False
|
||||
|
||||
assert [s["id"] for s in client.get("/api/schedules").json()] == [row["id"]]
|
||||
|
||||
off = client.patch(f"/api/schedules/{row['id']}", json={"enabled": False})
|
||||
assert off.status_code == 200 and off.json()["enabled"] is False
|
||||
# And only that: a PATCH of one field does not resubmit the rest.
|
||||
assert off.json()["cadence"] == "weekly mon 07:00"
|
||||
assert off.json()["payload"] == {"note": "over http"}
|
||||
|
||||
faster = client.patch(f"/api/schedules/{row['id']}", json={"cadence": "every 15m"})
|
||||
assert faster.json()["cadence"] == "every 15m"
|
||||
# A new cadence moves the time with it, rather than leaving the old one.
|
||||
assert faster.json()["next_run"] != row["next_run"]
|
||||
|
||||
gone = client.delete(f"/api/schedules/{row['id']}")
|
||||
assert gone.status_code == 200 and gone.json() == {"deleted": True}
|
||||
assert client.delete(f"/api/schedules/{row['id']}").json() == {"deleted": False}
|
||||
|
||||
|
||||
def test_a_second_schedule_of_one_kind_is_named_by_counting(client):
|
||||
first = client.post("/api/schedules",
|
||||
json={"kind": "s_tick", "cadence": "every 15m"}).json()
|
||||
second = client.post("/api/schedules",
|
||||
json={"kind": "s_tick", "cadence": "every 6h"}).json()
|
||||
assert (first["name"], second["name"]) == ("s_tick", "s_tick 2")
|
||||
|
||||
|
||||
def test_a_name_somebody_typed_is_refused_rather_than_altered(client):
|
||||
client.post("/api/schedules",
|
||||
json={"name": "nightly", "kind": "s_tick", "cadence": "every 15m"})
|
||||
clash = client.post("/api/schedules",
|
||||
json={"name": "nightly", "kind": "s_tick", "cadence": "every 6h"})
|
||||
assert clash.status_code == 409
|
||||
assert "nightly" in clash.json()["detail"]
|
||||
|
||||
|
||||
def test_an_unregistered_kind_is_a_404_naming_what_there_is(client):
|
||||
refused = client.post("/api/schedules",
|
||||
json={"kind": "s_nonesuch", "cadence": "every 15m"})
|
||||
assert refused.status_code == 404
|
||||
detail = refused.json()["detail"]
|
||||
assert "s_nonesuch" in detail and "s_tick" in detail
|
||||
|
||||
|
||||
def test_a_payload_missing_what_the_handler_needs_is_a_422_naming_it(client):
|
||||
refused = client.post("/api/schedules",
|
||||
json={"kind": "s_needs", "cadence": "every 15m"})
|
||||
assert refused.status_code == 422
|
||||
assert "target" in refused.json()["detail"]
|
||||
# And with it, the same request is fine.
|
||||
assert client.post("/api/schedules", json={
|
||||
"kind": "s_needs", "cadence": "every 15m",
|
||||
"payload": {"target": "somewhere"}}).status_code == 201
|
||||
|
||||
|
||||
def test_a_bad_cadence_is_a_422_listing_the_accepted_wordings(client):
|
||||
refused = client.post("/api/schedules",
|
||||
json={"kind": "s_tick", "cadence": "sometimes"})
|
||||
assert refused.status_code == 422
|
||||
detail = refused.json()["detail"]
|
||||
assert isinstance(detail, str)
|
||||
for form in ("every 15m", "every 6h", "every 2d",
|
||||
"daily at 06:00", "weekly mon 07:00"):
|
||||
assert form in detail
|
||||
|
||||
|
||||
def test_an_edit_is_validated_the_same_way(client):
|
||||
row = client.post("/api/schedules",
|
||||
json={"kind": "s_tick", "cadence": "every 15m"}).json()
|
||||
assert client.patch(f"/api/schedules/{row['id']}",
|
||||
json={"cadence": "now and then"}).status_code == 422
|
||||
assert client.patch(f"/api/schedules/{row['id']}",
|
||||
json={"kind": "s_needs"}).status_code == 422
|
||||
assert client.patch("/api/schedules/9999",
|
||||
json={"enabled": False}).status_code == 404
|
||||
|
||||
|
||||
def test_run_now_queues_the_job(client, jobs):
|
||||
row = client.post("/api/schedules",
|
||||
json={"kind": "s_tick", "cadence": "daily at 06:00"}).json()
|
||||
started = client.post(f"/api/schedules/{row['id']}/run")
|
||||
assert started.status_code == 202
|
||||
assert started.json()["kind"] == "s_tick"
|
||||
after = client.get("/api/schedules").json()[0]
|
||||
assert after["last_job_id"] == started.json()["id"]
|
||||
assert after["next_run"] == row["next_run"]
|
||||
assert client.post("/api/schedules/9999/run").status_code == 404
|
||||
|
||||
|
||||
def test_a_row_whose_handler_went_away_is_flagged_not_hidden(client, jobs):
|
||||
jobs.add_schedule("orphan", "s_vanished", "every 15m")
|
||||
row = next(s for s in client.get("/api/schedules").json()
|
||||
if s["name"] == "orphan")
|
||||
assert row["unknown"] is True
|
||||
# Listed and deletable, which is the whole of what can be done about it.
|
||||
assert client.delete(f"/api/schedules/{row['id']}").json() == {"deleted": True}
|
||||
|
||||
|
||||
def test_the_socket_pushes_the_schedule_list_on_every_change(client):
|
||||
"""So a row edited in one tab redraws in the others without polling.
|
||||
|
||||
The opening message is the *job* list — the socket is shared — which is
|
||||
why `<desk-schedules>` reads `/api/schedules` once on connect and follows
|
||||
this afterwards.
|
||||
"""
|
||||
with client.websocket_connect("/ws/jobs") as socket:
|
||||
assert socket.receive_json()["type"] == "jobs"
|
||||
added = client.post("/api/schedules",
|
||||
json={"kind": "s_tick", "cadence": "every 15m"}).json()
|
||||
pushed = _next_of(socket, "schedules")
|
||||
assert [s["id"] for s in pushed["schedules"]] == [added["id"]]
|
||||
assert pushed["schedules"][0]["cadence"] == "every 15m"
|
||||
|
||||
client.patch(f"/api/schedules/{added['id']}", json={"cadence": "every 6h"})
|
||||
assert _next_of(socket, "schedules")["schedules"][0]["cadence"] == "every 6h"
|
||||
|
||||
client.delete(f"/api/schedules/{added['id']}")
|
||||
assert _next_of(socket, "schedules")["schedules"] == []
|
||||
|
||||
|
||||
def _next_of(socket, type_: str, tries: int = 12) -> dict:
|
||||
"""The next message of one type, stepping over the job pushes between."""
|
||||
for _ in range(tries):
|
||||
message = socket.receive_json()
|
||||
if message["type"] == type_:
|
||||
return message
|
||||
raise AssertionError(f"no {type_} message arrived")
|
||||
|
||||
|
||||
# --------------------------------------------------------------- the worker
|
||||
|
||||
def _worker(db: Path, marks: Path, *extra: str) -> subprocess.CompletedProcess:
|
||||
argv = [sys.executable, "-m", "pipeline.worker", "--db", str(db),
|
||||
"--bootstrap", "pipeline.tests.scheduled_handlers:marking",
|
||||
"--bootstrap-arg", str(marks), "--until-idle", *extra]
|
||||
return subprocess.run(argv, env=dict(os.environ, PYTHONPATH=str(ROOT)),
|
||||
cwd=ROOT, capture_output=True, timeout=120)
|
||||
|
||||
|
||||
def test_the_standalone_worker_fires_schedules(tmp_path):
|
||||
"""A machine running only `python -m pipeline.worker` still gets its 06:00
|
||||
job, which is what makes it a complete replacement for the web app's
|
||||
worker rather than half of one."""
|
||||
db, marks = tmp_path / "w.db", tmp_path / "marks"
|
||||
marks.mkdir()
|
||||
jobs = Jobs(db)
|
||||
jobs.add_schedule("marker", "sw_mark", "every 1h", next_run=ago(60))
|
||||
|
||||
done = _worker(db, marks)
|
||||
assert done.returncode == 0, done.stdout.decode("utf-8", "replace")
|
||||
assert (marks / "fired").exists()
|
||||
assert jobs.schedules()[0].last_job_id
|
||||
|
||||
|
||||
def test_no_schedules_turns_it_off_for_a_second_worker(tmp_path):
|
||||
"""One scheduler per file, as many workers as you like: the pile-up guard
|
||||
reads `last_job_id`, which neither of two simultaneous firings has written
|
||||
yet when the other looks."""
|
||||
db, marks = tmp_path / "w.db", tmp_path / "marks"
|
||||
marks.mkdir()
|
||||
jobs = Jobs(db)
|
||||
jobs.add_schedule("marker", "sw_mark", "every 1h", next_run=ago(60))
|
||||
|
||||
done = _worker(db, marks, "--no-schedules")
|
||||
assert done.returncode == 0, done.stdout.decode("utf-8", "replace")
|
||||
assert not (marks / "fired").exists()
|
||||
assert jobs.schedules()[0].last_job_id == 0
|
||||
# Not even the declared default was reconciled: nothing about schedules
|
||||
# happens in a worker that was told not to do them.
|
||||
assert [s.name for s in jobs.schedules()] == ["marker"]
|
||||
Reference in new issue
Block a user