Files
jobboard/pipeline/tests/test_isolation.py
T
WolfandClaude Opus 5.5 9d666d0937 Bring in pipeline, auth, deploy and docs from the boilerplate
roledesk is built on the boilerplate from here on: pipeline 2.0 (queue and
scheduler), auth (one password, sessions, passkeys), the deploy kit and its
docs, plus the coupling and packaging tests. Downstream copies, unedited.

Refreshed from boilerplate/d56949f7d0f6c0bfe5745478e09007bfad67ae87

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:16 +02:00

321 lines
13 KiB
Python

"""Ending work that will not end itself, and how data moves between jobs.
Four claims, and they fail independently:
a deadline is honest `timeout` without `isolate` is refused at
registration rather than promising a kill a
thread cannot deliver.
a process can be killed an isolated job that never calls `check()` is
still over when its deadline passes, and its
row says so.
a chain reads, never writes `upstream()` is how B gets A's output — off A's
row, so B's own row was complete when written.
a fan-out is one call `fan_out` queues the children and the job that
reports on them, and an app never writes the
second one.
python3 -m pytest tests/test_isolation.py -q
"""
from __future__ import annotations
import asyncio
import os
import sys
import tempfile
import time
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from pipeline import NULL, Cancelled, JobQueue, Jobs, RunContext, job, spawn
from pipeline.models import Job
# Imported for the `@job` registrations, which the *parent* needs as much as
# the child does: the worker reads `isolate` and `timeout` off the spec before
# it decides to start a process at all. An app imports its handlers in every
# entry point for this reason; here that import is this line.
from pipeline.tests import isolated_handlers # noqa: F401
BOOTSTRAP = "pipeline.tests.isolated_handlers:bootstrap"
@job("t_child", "A child of a fan-out")
def t_child(config, store, *, context=NULL, i: int = 0) -> dict:
"""Queued by `fan_out` and never run in these tests — what is under test is
the shape of the rows it made, which is the part an app never writes."""
return {"i": i}
@pytest.fixture
def jobs():
with tempfile.TemporaryDirectory() as tmp:
yield Jobs(Path(tmp) / "t.db")
@pytest.fixture
def child_path(monkeypatch):
"""Put the folder holding `pipeline` where a child process can import it.
What an installed app gets for free and a checkout does not: the child is a
fresh interpreter, and the checkout is on *our* path only because pytest
put it there.
"""
root = Path(__file__).resolve().parents[2]
monkeypatch.setenv("PYTHONPATH", str(root))
def drain(queue: JobQueue, *, timeout: float = 20.0) -> None:
"""Run the worker until nothing is queued, running, or held by this one."""
async def go() -> None:
await queue.start()
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
snap = queue.jobs.snapshot()
if not snap.running and not snap.queued and not queue._running:
break
await asyncio.sleep(0.02)
await queue.stop()
asyncio.run(go())
def got(found: Job | None) -> Job:
assert found is not None
return found
# --------------------------------------------------- a deadline is honest
def test_a_timeout_without_isolation_is_refused():
"""The pipeline will not offer a kill it cannot perform.
A deadline over a job in a thread could only mark the row failed and walk
away: the work would carry on holding the core it was killed for holding,
and two things would intend to write one row. So the pair is refused where
it is declared, with the two ways out named.
"""
with pytest.raises(ValueError) as refused:
@job("t_threaded_deadline", timeout=5.0)
def handler(config, store, *, context=NULL):
return {}
said = str(refused.value)
assert "isolate=True" in said, "it names the way to get a real deadline"
assert "RunContext.spawn" in said, "and the cheaper way, for a command"
def test_isolation_without_a_timeout_is_fine():
"""Only the one combination is refused. An isolated job with no deadline
runs until it is done or killed by hand, which is every job the pipeline
has ever had."""
@job("t_isolated_no_deadline", isolate=True)
def handler(config, store, *, context=NULL):
return {}
from pipeline import JOBS
assert JOBS["t_isolated_no_deadline"].isolate
assert JOBS["t_isolated_no_deadline"].timeout == 0.0
def test_a_queue_with_no_bootstrap_says_so(jobs):
"""An isolated job needs its app rebuilt in the child, and a queue that
cannot do it fails the job with the reason rather than with a TypeError
from somewhere inside a subprocess."""
@job("t_needs_bootstrap", isolate=True)
def handler(config, store, *, context=NULL):
return {}
queued = got(jobs.enqueue("t_needs_bootstrap"))
drain(JobQueue(jobs, None, None)) # no bootstrap=
ended = got(jobs.get(queued.id))
assert ended.state == "failed"
assert "bootstrap" in ended.error, f"it names what is missing: {ended.error}"
# ------------------------------------------------ a process can be killed
def test_an_isolated_job_runs_somewhere_else_and_its_result_comes_back(jobs, child_path):
"""The happy path, and the proof that it was a real process.
Nothing was pickled to get there: the child was handed a database and a
row id and rebuilt the rest. Which is also why its progress is on the row
— it wrote it there itself, to the same table the snapshot reads.
"""
queued = got(jobs.enqueue("iso_ok", payload={"n": 21}))
drain(JobQueue(jobs, None, None, bootstrap=BOOTSTRAP))
ended = got(jobs.get(queued.id))
assert ended.state == "done", ended.error
assert ended.result["doubled"] == 42, "the handler's return travelled back"
assert ended.result["pid"] != os.getpid(), "and it ran in another process"
assert ended.message == "counting to 21", "its progress reached our table"
def test_an_isolated_job_that_raises_fails_with_what_it_said(jobs, child_path):
"""A failure across a process boundary reads like any other failure: the
row is `failed`, the error is the exception, and the retry bound applies
as it would to a handler that raised in a thread."""
queued = got(jobs.enqueue("iso_raises"))
drain(JobQueue(jobs, None, None, bootstrap=BOOTSTRAP))
ended = got(jobs.get(queued.id))
assert ended.state == "failed"
assert "the model said no" in ended.error, ended.error
def test_a_deadline_ends_a_job_that_never_looks_up(jobs, child_path):
"""The claim the whole feature exists for.
`iso_slow` sleeps for two minutes and never calls `check()`. In a thread
that is unkillable and the worker could only lie about having stopped it.
In a process of its own it is over in about a second, and the row says
why — which is what a person reading the job list needs.
"""
queued = got(jobs.enqueue("iso_slow"))
started = time.monotonic()
drain(JobQueue(jobs, None, None, bootstrap=BOOTSTRAP))
took = time.monotonic() - started
ended = got(jobs.get(queued.id))
assert ended.state == "failed", f"it ended: {ended.state}"
assert took < 15, f"and on its deadline, not in its own time: {took:.1f}s"
# ------------------------------------------------------ spawn, on its own
def test_spawn_kills_the_process_when_the_job_is_cancelled():
"""Kill, for a handler whose work is a command.
Cooperative in name only from the handler's side: it is blocked in
`spawn`, not in a loop with a `check()` in it, and the child is killed
outright. What makes that safe is that the thread was only sleeping.
"""
asked = [0]
def is_cancelled() -> bool:
asked[0] += 1
return asked[0] > 2 # pressed, a moment in
context = RunContext(is_cancelled=is_cancelled)
started = time.monotonic()
with pytest.raises(Cancelled):
context.spawn([sys.executable, "-c", "import time; time.sleep(120)"])
assert time.monotonic() - started < 10, "it did not wait the two minutes"
def test_spawn_raises_on_its_deadline_and_leaves_nothing_running():
"""The deadline, and that the process is gone after it — not merely
abandoned, which is the failure mode the whole thing is against."""
started = time.monotonic()
with pytest.raises(TimeoutError):
spawn([sys.executable, "-c", "import time; time.sleep(120)"], timeout=0.5)
assert time.monotonic() - started < 10
def test_spawn_hands_back_the_exit_code_rather_than_raising():
"""A non-zero exit is not an error here. Plenty of commands say something
useful with one, and only the handler knows which."""
done = spawn([sys.executable, "-c", "import sys; sys.exit(3)"])
assert done.returncode == 3
def test_spawn_survives_a_command_that_writes_more_than_a_pipe_holds():
"""Output goes to a file, not a `PIPE`, and this is why.
A child that fills the pipe buffer blocks on the write while the poll loop
blocks on the read, and neither side is wrong — the job simply never ends.
A megabyte is far past any platform's buffer.
"""
done = spawn([sys.executable, "-c", "print('x' * 1000000)"], timeout=60)
assert done.returncode == 0
assert len(done.stdout) >= 1000000
# ---------------------------------------------- a chain reads, never writes
def test_upstream_merges_what_the_parents_returned():
"""How B gets A's output: off A's row, not out of B's payload.
Which is the property worth having. B's row was complete when it was
written, so it can carry a dedup key, a retry can re-run it from itself,
and no worker can claim it between an earlier job's write and its commit.
"""
a = Job(id=1, kind="a", created_at="", result={"html": "<p>hi</p>", "code": 200})
b = Job(id=2, kind="b", created_at="", result={"code": 204})
context = RunContext(waited_for=(a, b))
assert context.upstream() == {"html": "<p>hi</p>", "code": 204}, "later parents win"
def test_upstream_is_empty_under_null_so_a_chained_handler_is_still_callable():
"""The one line that keeps a chained handler an ordinary function.
Every other handler in the pipeline can be called from a terminal. One
that read `waited_for[0].result` directly could not — it would raise
IndexError under `NULL` — so `upstream()` answers `{}` and the handler
falls back to the keyword it declared.
"""
assert NULL.upstream() == {}
@job("t_chained")
def handler(config, store, *, context=NULL, html: str = "") -> dict:
return {"html": html or context.upstream().get("html", "")}
assert handler(None, None, html="passed by hand") == {"html": "passed by hand"}
assert handler(None, None) == {"html": ""}, "and it does not blow up with neither"
# ----------------------------------------------------- a fan-out is one call
def test_fan_out_queues_the_children_and_the_job_that_reports_on_them(jobs):
"""One call, and the reporting job is not the app's to write.
The children get a run of their own inside the caller's cascade, and the
report waits on all of them with `after='done'` — the mode that exists
precisely so a job reporting that one of three failed is not held back by
that failure.
"""
queue = JobQueue(jobs, None, None)
parent = got(jobs.enqueue("t_parent"))
context = queue._context(parent, [], [])
children = context.fan_out(
"t_child", [{"i": 1}, {"i": 2}, {"i": 3}], then="mod:fn")
assert len(children) == 3, "it answers with the children's ids"
queued = jobs.snapshot().queued
report = [j for j in queued if j.kind == "advance"]
assert len(report) == 1, "and queued exactly one job to report on them"
assert sorted(report[0].depends_on) == sorted(children)
assert report[0].after == "done"
assert report[0].payload["callback"] == "mod:fn"
assert {j.task_id for j in queued if j.id in children} == {report[0].task_id}, \
"children and report are one run"
def test_fan_out_queues_nothing_under_null_and_says_so():
"""A handler called directly is being run by somebody who is standing
there, and the useful answer is to do its own work and report that nothing
was queued behind it — the same answer `task` gives, for the same reason."""
assert NULL.fan_out("t_child", [{"i": 1}]) == ()
def test_a_fan_out_carries_what_the_caller_puts_on_it(jobs):
"""`carries` goes *around* the children to the callback: static input from
whoever asked, as against `Tally.results`, which is what the children
themselves returned."""
queue = JobQueue(jobs, None, None)
parent = got(jobs.enqueue("t_parent"))
context = queue._context(parent, [], [])
context.fan_out("t_child", [{"i": 1}], then="mod:fn", carries={"since": "monday"})
report = [j for j in jobs.snapshot().queued if j.kind == "advance"]
assert report[0].payload["payload"] == {"since": "monday"}