The model batches are isolated with a deadline, so a Kill or a hang ends them

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-28 19:42:28 +02:00
1 parent c63f348dc9
commit b73a0fc2cb
6 files changed
+54 -9

No files matched your search

+11 -1
View File
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from desk.models import Item # noqa: E402
from pipeline import FINAL, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, Tally # noqa: E402
from pipeline import FINAL, JOBS, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, Tally # noqa: E402
from plugins.ats import ATSSource # noqa: E402
from app.config import AgentSeed, Config # noqa: E402
from app.jobs import cascade, embed, score # noqa: E402
@@ -487,6 +487,16 @@ def test_pull_is_a_job_on_a_six_hour_schedule_and_the_stages_are_tasks() -> None
assert SCHEDULES["pull"].cadence == "every 6h"
def test_the_model_batches_are_isolated_with_a_timeout() -> None:
"""A hung model call is otherwise unkillable short of restarting the
server: `context.check()` runs between calls, not inside one, so only a
process boundary can end a batch that never comes back."""
for kind in ("extract_batch", "summarise_batch"):
spec = JOBS[kind]
assert spec.isolate, f"{kind} must run in its own process to be killable"
assert spec.timeout > 0, f"{kind} must have a deadline now that it can"
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
"""By position, not by name: `score` is in the chain twice, once before
extract and once after, and the second must lead on to summarise."""
+12 -1
View File
@@ -958,10 +958,21 @@ def test_the_posting_text_is_fetched_once(
def test_summarise_queues_one_role_and_the_job_writes_it(
app: FastAPI, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""No lifespan, so no worker: the job waits in the table until drained."""
"""No lifespan, so no worker: the job waits in the table until drained.
`summarise_batch` is `isolate=True` in production, which runs it in a
child process that imports `desk.tools.agent` fresh and never sees the
stub below. What is under test here is the route and the queue, not
isolation itself — that has its own tests in `pipeline/tests` — so the
job runs in-thread for this one drain.
"""
from dataclasses import replace
from desk.tools import agent
from pipeline import JOBS
monkeypatch.setitem(JOBS, "summarise_batch",
replace(JOBS["summarise_batch"], isolate=False, timeout=0.0))
client = TestClient(app)
store.upsert_role(posting("1"))
store.upsert_role(posting("2"))