roledesk is built on the boilerplate from here on: pipeline 2.0 (queue and scheduler), auth (one password, sessions, passkeys), the deploy kit and its docs, plus the coupling and packaging tests. Downstream copies, unedited. Refreshed from boilerplate/d56949f7d0f6c0bfe5745478e09007bfad67ae87 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
321 lines
13 KiB
Python
321 lines
13 KiB
Python
"""Ending work that will not end itself, and how data moves between jobs.
|
|
|
|
Four claims, and they fail independently:
|
|
|
|
a deadline is honest `timeout` without `isolate` is refused at
|
|
registration rather than promising a kill a
|
|
thread cannot deliver.
|
|
a process can be killed an isolated job that never calls `check()` is
|
|
still over when its deadline passes, and its
|
|
row says so.
|
|
a chain reads, never writes `upstream()` is how B gets A's output — off A's
|
|
row, so B's own row was complete when written.
|
|
a fan-out is one call `fan_out` queues the children and the job that
|
|
reports on them, and an app never writes the
|
|
second one.
|
|
|
|
python3 -m pytest tests/test_isolation.py -q
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
|
|
from pipeline import NULL, Cancelled, JobQueue, Jobs, RunContext, job, spawn
|
|
from pipeline.models import Job
|
|
|
|
# Imported for the `@job` registrations, which the *parent* needs as much as
|
|
# the child does: the worker reads `isolate` and `timeout` off the spec before
|
|
# it decides to start a process at all. An app imports its handlers in every
|
|
# entry point for this reason; here that import is this line.
|
|
from pipeline.tests import isolated_handlers # noqa: F401
|
|
|
|
BOOTSTRAP = "pipeline.tests.isolated_handlers:bootstrap"
|
|
|
|
|
|
@job("t_child", "A child of a fan-out")
|
|
def t_child(config, store, *, context=NULL, i: int = 0) -> dict:
|
|
"""Queued by `fan_out` and never run in these tests — what is under test is
|
|
the shape of the rows it made, which is the part an app never writes."""
|
|
return {"i": i}
|
|
|
|
|
|
@pytest.fixture
|
|
def jobs():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
yield Jobs(Path(tmp) / "t.db")
|
|
|
|
|
|
@pytest.fixture
|
|
def child_path(monkeypatch):
|
|
"""Put the folder holding `pipeline` where a child process can import it.
|
|
|
|
What an installed app gets for free and a checkout does not: the child is a
|
|
fresh interpreter, and the checkout is on *our* path only because pytest
|
|
put it there.
|
|
"""
|
|
root = Path(__file__).resolve().parents[2]
|
|
monkeypatch.setenv("PYTHONPATH", str(root))
|
|
|
|
|
|
def drain(queue: JobQueue, *, timeout: float = 20.0) -> None:
|
|
"""Run the worker until nothing is queued, running, or held by this one."""
|
|
|
|
async def go() -> None:
|
|
await queue.start()
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
snap = queue.jobs.snapshot()
|
|
if not snap.running and not snap.queued and not queue._running:
|
|
break
|
|
await asyncio.sleep(0.02)
|
|
await queue.stop()
|
|
|
|
asyncio.run(go())
|
|
|
|
|
|
def got(found: Job | None) -> Job:
|
|
assert found is not None
|
|
return found
|
|
|
|
|
|
# --------------------------------------------------- a deadline is honest
|
|
|
|
def test_a_timeout_without_isolation_is_refused():
|
|
"""The pipeline will not offer a kill it cannot perform.
|
|
|
|
A deadline over a job in a thread could only mark the row failed and walk
|
|
away: the work would carry on holding the core it was killed for holding,
|
|
and two things would intend to write one row. So the pair is refused where
|
|
it is declared, with the two ways out named.
|
|
"""
|
|
with pytest.raises(ValueError) as refused:
|
|
@job("t_threaded_deadline", timeout=5.0)
|
|
def handler(config, store, *, context=NULL):
|
|
return {}
|
|
|
|
said = str(refused.value)
|
|
assert "isolate=True" in said, "it names the way to get a real deadline"
|
|
assert "RunContext.spawn" in said, "and the cheaper way, for a command"
|
|
|
|
|
|
def test_isolation_without_a_timeout_is_fine():
|
|
"""Only the one combination is refused. An isolated job with no deadline
|
|
runs until it is done or killed by hand, which is every job the pipeline
|
|
has ever had."""
|
|
|
|
@job("t_isolated_no_deadline", isolate=True)
|
|
def handler(config, store, *, context=NULL):
|
|
return {}
|
|
|
|
from pipeline import JOBS
|
|
assert JOBS["t_isolated_no_deadline"].isolate
|
|
assert JOBS["t_isolated_no_deadline"].timeout == 0.0
|
|
|
|
|
|
def test_a_queue_with_no_bootstrap_says_so(jobs):
|
|
"""An isolated job needs its app rebuilt in the child, and a queue that
|
|
cannot do it fails the job with the reason rather than with a TypeError
|
|
from somewhere inside a subprocess."""
|
|
|
|
@job("t_needs_bootstrap", isolate=True)
|
|
def handler(config, store, *, context=NULL):
|
|
return {}
|
|
|
|
queued = got(jobs.enqueue("t_needs_bootstrap"))
|
|
drain(JobQueue(jobs, None, None)) # no bootstrap=
|
|
ended = got(jobs.get(queued.id))
|
|
assert ended.state == "failed"
|
|
assert "bootstrap" in ended.error, f"it names what is missing: {ended.error}"
|
|
|
|
|
|
# ------------------------------------------------ a process can be killed
|
|
|
|
def test_an_isolated_job_runs_somewhere_else_and_its_result_comes_back(jobs, child_path):
|
|
"""The happy path, and the proof that it was a real process.
|
|
|
|
Nothing was pickled to get there: the child was handed a database and a
|
|
row id and rebuilt the rest. Which is also why its progress is on the row
|
|
— it wrote it there itself, to the same table the snapshot reads.
|
|
"""
|
|
queued = got(jobs.enqueue("iso_ok", payload={"n": 21}))
|
|
drain(JobQueue(jobs, None, None, bootstrap=BOOTSTRAP))
|
|
|
|
ended = got(jobs.get(queued.id))
|
|
assert ended.state == "done", ended.error
|
|
assert ended.result["doubled"] == 42, "the handler's return travelled back"
|
|
assert ended.result["pid"] != os.getpid(), "and it ran in another process"
|
|
assert ended.message == "counting to 21", "its progress reached our table"
|
|
|
|
|
|
def test_an_isolated_job_that_raises_fails_with_what_it_said(jobs, child_path):
|
|
"""A failure across a process boundary reads like any other failure: the
|
|
row is `failed`, the error is the exception, and the retry bound applies
|
|
as it would to a handler that raised in a thread."""
|
|
queued = got(jobs.enqueue("iso_raises"))
|
|
drain(JobQueue(jobs, None, None, bootstrap=BOOTSTRAP))
|
|
|
|
ended = got(jobs.get(queued.id))
|
|
assert ended.state == "failed"
|
|
assert "the model said no" in ended.error, ended.error
|
|
|
|
|
|
def test_a_deadline_ends_a_job_that_never_looks_up(jobs, child_path):
|
|
"""The claim the whole feature exists for.
|
|
|
|
`iso_slow` sleeps for two minutes and never calls `check()`. In a thread
|
|
that is unkillable and the worker could only lie about having stopped it.
|
|
In a process of its own it is over in about a second, and the row says
|
|
why — which is what a person reading the job list needs.
|
|
"""
|
|
queued = got(jobs.enqueue("iso_slow"))
|
|
started = time.monotonic()
|
|
drain(JobQueue(jobs, None, None, bootstrap=BOOTSTRAP))
|
|
took = time.monotonic() - started
|
|
|
|
ended = got(jobs.get(queued.id))
|
|
assert ended.state == "failed", f"it ended: {ended.state}"
|
|
assert took < 15, f"and on its deadline, not in its own time: {took:.1f}s"
|
|
|
|
|
|
# ------------------------------------------------------ spawn, on its own
|
|
|
|
def test_spawn_kills_the_process_when_the_job_is_cancelled():
|
|
"""Kill, for a handler whose work is a command.
|
|
|
|
Cooperative in name only from the handler's side: it is blocked in
|
|
`spawn`, not in a loop with a `check()` in it, and the child is killed
|
|
outright. What makes that safe is that the thread was only sleeping.
|
|
"""
|
|
asked = [0]
|
|
|
|
def is_cancelled() -> bool:
|
|
asked[0] += 1
|
|
return asked[0] > 2 # pressed, a moment in
|
|
|
|
context = RunContext(is_cancelled=is_cancelled)
|
|
started = time.monotonic()
|
|
with pytest.raises(Cancelled):
|
|
context.spawn([sys.executable, "-c", "import time; time.sleep(120)"])
|
|
assert time.monotonic() - started < 10, "it did not wait the two minutes"
|
|
|
|
|
|
def test_spawn_raises_on_its_deadline_and_leaves_nothing_running():
|
|
"""The deadline, and that the process is gone after it — not merely
|
|
abandoned, which is the failure mode the whole thing is against."""
|
|
started = time.monotonic()
|
|
with pytest.raises(TimeoutError):
|
|
spawn([sys.executable, "-c", "import time; time.sleep(120)"], timeout=0.5)
|
|
assert time.monotonic() - started < 10
|
|
|
|
|
|
def test_spawn_hands_back_the_exit_code_rather_than_raising():
|
|
"""A non-zero exit is not an error here. Plenty of commands say something
|
|
useful with one, and only the handler knows which."""
|
|
done = spawn([sys.executable, "-c", "import sys; sys.exit(3)"])
|
|
assert done.returncode == 3
|
|
|
|
|
|
def test_spawn_survives_a_command_that_writes_more_than_a_pipe_holds():
|
|
"""Output goes to a file, not a `PIPE`, and this is why.
|
|
|
|
A child that fills the pipe buffer blocks on the write while the poll loop
|
|
blocks on the read, and neither side is wrong — the job simply never ends.
|
|
A megabyte is far past any platform's buffer.
|
|
"""
|
|
done = spawn([sys.executable, "-c", "print('x' * 1000000)"], timeout=60)
|
|
assert done.returncode == 0
|
|
assert len(done.stdout) >= 1000000
|
|
|
|
|
|
# ---------------------------------------------- a chain reads, never writes
|
|
|
|
def test_upstream_merges_what_the_parents_returned():
|
|
"""How B gets A's output: off A's row, not out of B's payload.
|
|
|
|
Which is the property worth having. B's row was complete when it was
|
|
written, so it can carry a dedup key, a retry can re-run it from itself,
|
|
and no worker can claim it between an earlier job's write and its commit.
|
|
"""
|
|
a = Job(id=1, kind="a", created_at="", result={"html": "<p>hi</p>", "code": 200})
|
|
b = Job(id=2, kind="b", created_at="", result={"code": 204})
|
|
context = RunContext(waited_for=(a, b))
|
|
|
|
assert context.upstream() == {"html": "<p>hi</p>", "code": 204}, "later parents win"
|
|
|
|
|
|
def test_upstream_is_empty_under_null_so_a_chained_handler_is_still_callable():
|
|
"""The one line that keeps a chained handler an ordinary function.
|
|
|
|
Every other handler in the pipeline can be called from a terminal. One
|
|
that read `waited_for[0].result` directly could not — it would raise
|
|
IndexError under `NULL` — so `upstream()` answers `{}` and the handler
|
|
falls back to the keyword it declared.
|
|
"""
|
|
assert NULL.upstream() == {}
|
|
|
|
@job("t_chained")
|
|
def handler(config, store, *, context=NULL, html: str = "") -> dict:
|
|
return {"html": html or context.upstream().get("html", "")}
|
|
|
|
assert handler(None, None, html="passed by hand") == {"html": "passed by hand"}
|
|
assert handler(None, None) == {"html": ""}, "and it does not blow up with neither"
|
|
|
|
|
|
# ----------------------------------------------------- a fan-out is one call
|
|
|
|
def test_fan_out_queues_the_children_and_the_job_that_reports_on_them(jobs):
|
|
"""One call, and the reporting job is not the app's to write.
|
|
|
|
The children get a run of their own inside the caller's cascade, and the
|
|
report waits on all of them with `after='done'` — the mode that exists
|
|
precisely so a job reporting that one of three failed is not held back by
|
|
that failure.
|
|
"""
|
|
queue = JobQueue(jobs, None, None)
|
|
parent = got(jobs.enqueue("t_parent"))
|
|
context = queue._context(parent, [], [])
|
|
|
|
children = context.fan_out(
|
|
"t_child", [{"i": 1}, {"i": 2}, {"i": 3}], then="mod:fn")
|
|
|
|
assert len(children) == 3, "it answers with the children's ids"
|
|
queued = jobs.snapshot().queued
|
|
report = [j for j in queued if j.kind == "advance"]
|
|
assert len(report) == 1, "and queued exactly one job to report on them"
|
|
assert sorted(report[0].depends_on) == sorted(children)
|
|
assert report[0].after == "done"
|
|
assert report[0].payload["callback"] == "mod:fn"
|
|
assert {j.task_id for j in queued if j.id in children} == {report[0].task_id}, \
|
|
"children and report are one run"
|
|
|
|
|
|
def test_fan_out_queues_nothing_under_null_and_says_so():
|
|
"""A handler called directly is being run by somebody who is standing
|
|
there, and the useful answer is to do its own work and report that nothing
|
|
was queued behind it — the same answer `task` gives, for the same reason."""
|
|
assert NULL.fan_out("t_child", [{"i": 1}]) == ()
|
|
|
|
|
|
def test_a_fan_out_carries_what_the_caller_puts_on_it(jobs):
|
|
"""`carries` goes *around* the children to the callback: static input from
|
|
whoever asked, as against `Tally.results`, which is what the children
|
|
themselves returned."""
|
|
queue = JobQueue(jobs, None, None)
|
|
parent = got(jobs.enqueue("t_parent"))
|
|
context = queue._context(parent, [], [])
|
|
|
|
context.fan_out("t_child", [{"i": 1}], then="mod:fn", carries={"since": "monday"})
|
|
|
|
report = [j for j in jobs.snapshot().queued if j.kind == "advance"]
|
|
assert report[0].payload["payload"] == {"since": "monday"}
|