diff --git a/app/jobs/extract.py b/app/jobs/extract.py index 91e16e5..d68e326 100644 --- a/app/jobs/extract.py +++ b/app/jobs/extract.py @@ -216,10 +216,21 @@ TASK: ModelTask[ExtractResult] = ModelTask( ) -@job(f"{TASK.kind}_batch", "Extracting a batch", resource="llm") +# One call is bounded at AgentConfig.timeout (300s by default), and a batch +# is one call, plus a few short ones when a refusal halves it down +# (desk/modeltask.py `_narrow`). A batch of 20 still running after 30 +# minutes is stuck, not slow. +@job(f"{TASK.kind}_batch", "Extracting a batch", resource="llm", + isolate=True, timeout=1800) def run_batch(config: Config, store: Store, *, context: RunContext = NULL, uids: list[str] | None = None) -> dict[str, Any]: - """One model call over one batch of the roles that were waiting.""" + """One model call over one batch of the roles that were waiting. + + Isolated so a Kill lands even while the call itself is in flight: `one` + only checks between calls, and a hung request — the CLI backend spawns a + process of its own, the API one blocks on a socket — has no such gap for + a thread to be ended at. + """ return modeltask.one(TASK, config, store, uids or [], context=context).model_dump() diff --git a/app/jobs/summarise.py b/app/jobs/summarise.py index 6090f6e..5c1b815 100644 --- a/app/jobs/summarise.py +++ b/app/jobs/summarise.py @@ -177,11 +177,21 @@ TASK: ModelTask[SummariseResult] = ModelTask( ) -@job(f"{TASK.kind}_batch", "Summarising a batch", resource="llm") +# One call is bounded at AgentConfig.timeout (300s by default), and a batch +# is one call, plus a few short ones when a refusal halves it down +# (desk/modeltask.py `_narrow`). A batch of 10 still running after 15 +# minutes is stuck, not slow. +@job(f"{TASK.kind}_batch", "Summarising a batch", resource="llm", + isolate=True, timeout=900) def run_batch(config: Config, store: Store, *, context: RunContext = NULL, uids: list[str] | None = None) -> dict[str, Any]: """One model call over one batch. Queued by the task's finder, or with - one uid by the read page's Summarise button.""" + one uid by the read page's Summarise button. + + Isolated for the same reason as `extract.run_batch`: a Kill only reaches + `one` between calls, and a call that hangs — on either backend — is + otherwise unkillable short of restarting the server. + """ return modeltask.one(TASK, config, store, uids or [], context=context).model_dump() diff --git a/app/static/toasts.js b/app/static/toasts.js index ccacb8a..ae6403d 100644 --- a/app/static/toasts.js +++ b/app/static/toasts.js @@ -213,14 +213,15 @@ fresh.slice(0, MAX_ON_LOAD).forEach((n) => show(fromNotice(n))); if (fresh.length > MAX_ON_LOAD) { toast.info(`${fresh.length - MAX_ON_LOAD} more messages since your last visit.`, - { link: { label: "all messages", href: "/messages" } }); + { link: { label: "all messages", href: "/settings/sources#messages" } }); } if (fresh.length) markSeen(fresh[0].last_at); } document.addEventListener("app:notice", (e) => onNotice(e.detail)); document.addEventListener("DOMContentLoaded", () => { - // Not on Messages: the table is right there, in full, already. - if (location.pathname !== "/login" && location.pathname !== "/messages") missed(); + // Not on Sources: the messages table is right there, in full, already. + // `/messages` itself only redirects now, so the browser never lands there. + if (location.pathname !== "/login" && location.pathname !== "/settings/sources") missed(); }); })(); diff --git a/docs/decisions.md b/docs/decisions.md index 4ee0dc5..cfc5fc4 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -92,6 +92,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning **Extract is bounded like summarise: the top roles of each agent, by that agent's own `limits`.** Extract has short answers but would otherwise touch every open role, and the seed boards alone produced 29,000; at Haiku's rates that is tens of dollars for a first pass over roles nobody will look at. Bounded to the ranking, a pull reads a few hundred. `extract_top` is the backstop (default 500) and `extract_min` the real stop (default 0.3, loose): a fixed count stays flat whatever the boards produce, where a percentage would not. The bounds are per agent rather than config, because one agent may be a wide monitoring search that should read a thousand and another a narrow hunt that wants fifty; one global number fits neither. A batch dealt named uids finishes them whatever their rank, so a running batch is not cut short by a rescore. +**`extract_batch` and `summarise_batch` are `isolate=True`, with a timeout.** `context.check()` only runs between the calls `modeltask.one` makes, not inside one, so a call that hangs on either backend was otherwise unkillable short of restarting the server; a process boundary is what lets a Kill, or the timeout itself, actually end it. One call is already bounded at `agent.timeout`, so the deadline is a few calls' worth (30 minutes for extract, 15 for summarise): a batch still running after that is stuck, not slow. The cost: an isolated job's context has no working `stop`, `task` or `fan_out` (see `pipeline/child.py`'s `context_for`), so a refusal or an account-level failure inside a batch can no longer stop the rest of that task's queued siblings — they run on to the same failure instead of being spared it. + ## The pages **Each roledesk tab renders an empty shell; `static/roles.js` fills it and drives every action.** Roles, Applications, Agents and Companies extend the boilerplate's `base.html`; the server renders only select options (from `app.models` and `plugins.ats.FUNCTIONS`, so they cannot drift from what the store accepts). One code path for "show this" beats a server-rendered first page plus a client re-render after every filter change. diff --git a/tests/test_score.py b/tests/test_score.py index 20efee3..50121f9 100644 --- a/tests/test_score.py +++ b/tests/test_score.py @@ -17,7 +17,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from desk.models import Item # noqa: E402 -from pipeline import FINAL, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, Tally # noqa: E402 +from pipeline import FINAL, JOBS, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, Tally # noqa: E402 from plugins.ats import ATSSource # noqa: E402 from app.config import AgentSeed, Config # noqa: E402 from app.jobs import cascade, embed, score # noqa: E402 @@ -487,6 +487,16 @@ def test_pull_is_a_job_on_a_six_hour_schedule_and_the_stages_are_tasks() -> None assert SCHEDULES["pull"].cadence == "every 6h" +def test_the_model_batches_are_isolated_with_a_timeout() -> None: + """A hung model call is otherwise unkillable short of restarting the + server: `context.check()` runs between calls, not inside one, so only a + process boundary can end a batch that never comes back.""" + for kind in ("extract_batch", "summarise_batch"): + spec = JOBS[kind] + assert spec.isolate, f"{kind} must run in its own process to be killable" + assert spec.timeout > 0, f"{kind} must have a deadline now that it can" + + def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None: """By position, not by name: `score` is in the chain twice, once before extract and once after, and the second must lead on to summarise.""" diff --git a/tests/test_web.py b/tests/test_web.py index f7ccacf..48108b9 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -958,10 +958,21 @@ def test_the_posting_text_is_fetched_once( def test_summarise_queues_one_role_and_the_job_writes_it( app: FastAPI, store: Store, monkeypatch: pytest.MonkeyPatch) -> None: - """No lifespan, so no worker: the job waits in the table until drained.""" + """No lifespan, so no worker: the job waits in the table until drained. + + `summarise_batch` is `isolate=True` in production, which runs it in a + child process that imports `desk.tools.agent` fresh and never sees the + stub below. What is under test here is the route and the queue, not + isolation itself — that has its own tests in `pipeline/tests` — so the + job runs in-thread for this one drain. + """ + from dataclasses import replace from desk.tools import agent + from pipeline import JOBS + monkeypatch.setitem(JOBS, "summarise_batch", + replace(JOBS["summarise_batch"], isolate=False, timeout=0.0)) client = TestClient(app) store.upsert_role(posting("1")) store.upsert_role(posting("2"))