The model batches are isolated with a deadline, so a Kill or a hang ends them

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-28 19:42:28 +02:00
1 parent c63f348dc9
commit b73a0fc2cb
6 files changed
+54 -9

No files matched your search

+13 -2
View File
@@ -216,10 +216,21 @@ TASK: ModelTask[ExtractResult] = ModelTask(
)
@job(f"{TASK.kind}_batch", "Extracting a batch", resource="llm")
# One call is bounded at AgentConfig.timeout (300s by default), and a batch
# is one call, plus a few short ones when a refusal halves it down
# (desk/modeltask.py `_narrow`). A batch of 20 still running after 30
# minutes is stuck, not slow.
@job(f"{TASK.kind}_batch", "Extracting a batch", resource="llm",
isolate=True, timeout=1800)
def run_batch(config: Config, store: Store, *, context: RunContext = NULL,
uids: list[str] | None = None) -> dict[str, Any]:
"""One model call over one batch of the roles that were waiting."""
"""One model call over one batch of the roles that were waiting.
Isolated so a Kill lands even while the call itself is in flight: `one`
only checks between calls, and a hung request — the CLI backend spawns a
process of its own, the API one blocks on a socket — has no such gap for
a thread to be ended at.
"""
return modeltask.one(TASK, config, store, uids or [], context=context).model_dump()
+12 -2
View File
@@ -177,11 +177,21 @@ TASK: ModelTask[SummariseResult] = ModelTask(
)
@job(f"{TASK.kind}_batch", "Summarising a batch", resource="llm")
# One call is bounded at AgentConfig.timeout (300s by default), and a batch
# is one call, plus a few short ones when a refusal halves it down
# (desk/modeltask.py `_narrow`). A batch of 10 still running after 15
# minutes is stuck, not slow.
@job(f"{TASK.kind}_batch", "Summarising a batch", resource="llm",
isolate=True, timeout=900)
def run_batch(config: Config, store: Store, *, context: RunContext = NULL,
uids: list[str] | None = None) -> dict[str, Any]:
"""One model call over one batch. Queued by the task's finder, or with
one uid by the read page's Summarise button."""
one uid by the read page's Summarise button.
Isolated for the same reason as `extract.run_batch`: a Kill only reaches
`one` between calls, and a call that hangs — on either backend — is
otherwise unkillable short of restarting the server.
"""
return modeltask.one(TASK, config, store, uids or [], context=context).model_dump()
+4 -3
View File
@@ -213,14 +213,15 @@
fresh.slice(0, MAX_ON_LOAD).forEach((n) => show(fromNotice(n)));
if (fresh.length > MAX_ON_LOAD) {
toast.info(`${fresh.length - MAX_ON_LOAD} more messages since your last visit.`,
{ link: { label: "all messages", href: "/messages" } });
{ link: { label: "all messages", href: "/settings/sources#messages" } });
}
if (fresh.length) markSeen(fresh[0].last_at);
}
document.addEventListener("app:notice", (e) => onNotice(e.detail));
document.addEventListener("DOMContentLoaded", () => {
// Not on Messages: the table is right there, in full, already.
if (location.pathname !== "/login" && location.pathname !== "/messages") missed();
// Not on Sources: the messages table is right there, in full, already.
// `/messages` itself only redirects now, so the browser never lands there.
if (location.pathname !== "/login" && location.pathname !== "/settings/sources") missed();
});
})();
+2
View File
@@ -92,6 +92,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**Extract is bounded like summarise: the top roles of each agent, by that agent's own `limits`.** Extract has short answers but would otherwise touch every open role, and the seed boards alone produced 29,000; at Haiku's rates that is tens of dollars for a first pass over roles nobody will look at. Bounded to the ranking, a pull reads a few hundred. `extract_top` is the backstop (default 500) and `extract_min` the real stop (default 0.3, loose): a fixed count stays flat whatever the boards produce, where a percentage would not. The bounds are per agent rather than config, because one agent may be a wide monitoring search that should read a thousand and another a narrow hunt that wants fifty; one global number fits neither. A batch dealt named uids finishes them whatever their rank, so a running batch is not cut short by a rescore.
**`extract_batch` and `summarise_batch` are `isolate=True`, with a timeout.** `context.check()` only runs between the calls `modeltask.one` makes, not inside one, so a call that hangs on either backend was otherwise unkillable short of restarting the server; a process boundary is what lets a Kill, or the timeout itself, actually end it. One call is already bounded at `agent.timeout`, so the deadline is a few calls' worth (30 minutes for extract, 15 for summarise): a batch still running after that is stuck, not slow. The cost: an isolated job's context has no working `stop`, `task` or `fan_out` (see `pipeline/child.py`'s `context_for`), so a refusal or an account-level failure inside a batch can no longer stop the rest of that task's queued siblings — they run on to the same failure instead of being spared it.
## The pages
**Each roledesk tab renders an empty shell; `static/roles.js` fills it and drives every action.** Roles, Applications, Agents and Companies extend the boilerplate's `base.html`; the server renders only select options (from `app.models` and `plugins.ats.FUNCTIONS`, so they cannot drift from what the store accepts). One code path for "show this" beats a server-rendered first page plus a client re-render after every filter change.
+11 -1
View File
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from desk.models import Item # noqa: E402
from pipeline import FINAL, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, Tally # noqa: E402
from pipeline import FINAL, JOBS, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, Tally # noqa: E402
from plugins.ats import ATSSource # noqa: E402
from app.config import AgentSeed, Config # noqa: E402
from app.jobs import cascade, embed, score # noqa: E402
@@ -487,6 +487,16 @@ def test_pull_is_a_job_on_a_six_hour_schedule_and_the_stages_are_tasks() -> None
assert SCHEDULES["pull"].cadence == "every 6h"
def test_the_model_batches_are_isolated_with_a_timeout() -> None:
"""A hung model call is otherwise unkillable short of restarting the
server: `context.check()` runs between calls, not inside one, so only a
process boundary can end a batch that never comes back."""
for kind in ("extract_batch", "summarise_batch"):
spec = JOBS[kind]
assert spec.isolate, f"{kind} must run in its own process to be killable"
assert spec.timeout > 0, f"{kind} must have a deadline now that it can"
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
"""By position, not by name: `score` is in the chain twice, once before
extract and once after, and the second must lead on to summarise."""
+12 -1
View File
@@ -958,10 +958,21 @@ def test_the_posting_text_is_fetched_once(
def test_summarise_queues_one_role_and_the_job_writes_it(
app: FastAPI, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""No lifespan, so no worker: the job waits in the table until drained."""
"""No lifespan, so no worker: the job waits in the table until drained.
`summarise_batch` is `isolate=True` in production, which runs it in a
child process that imports `desk.tools.agent` fresh and never sees the
stub below. What is under test here is the route and the queue, not
isolation itself — that has its own tests in `pipeline/tests` — so the
job runs in-thread for this one drain.
"""
from dataclasses import replace
from desk.tools import agent
from pipeline import JOBS
monkeypatch.setitem(JOBS, "summarise_batch",
replace(JOBS["summarise_batch"], isolate=False, timeout=0.0))
client = TestClient(app)
store.upsert_role(posting("1"))
store.upsert_role(posting("2"))