From 91d15fcc7d835d133f39b08e035b256747fbe8eb Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 15:16:23 +0200 Subject: [PATCH] A change made while score runs gets one follow-up score, and the page waits on it score is solo, so a request during a run was deduped into the running job, which had already read the old state. _score_after_change returns a queued score when one has not started, else enqueues a plain score and, if that dedupes to a running job, one follow-up with depends_on it, after="done" and dedup_key score:after:. POST /api/score answers with that job and agents.js waits on it; the affinity route answers with score_job. The flaky affinity test was pipeline's race, not the app's: Jobs.enqueue re-raises the UNIQUE IntegrityError when the live job it collided with finishes before the SELECT that should hand it back. The app retries. Co-Authored-By: Claude Opus --- app/static/agents.js | 8 ++++-- app/web/api.py | 65 +++++++++++++++++++++++++++++++++++++++----- docs/decisions.md | 2 +- tests/test_web.py | 47 +++++++++++++++++++++++++++++++- 4 files changed, 110 insertions(+), 12 deletions(-) diff --git a/app/static/agents.js b/app/static/agents.js index 9274b49..5b85be5 100644 --- a/app/static/agents.js +++ b/app/static/agents.js @@ -289,14 +289,16 @@ render(); } - // Start the `score` job and wait for it, so "scored" means the Roles tab - // already orders by what was saved. `said` is what happened first. + // Ask for a `score` and wait for the job the server answers with, so + // "scored" means the Roles tab already orders by what was saved. While a + // score is running that is a follow-up queued after it, since the running + // one read the agent before the save. `said` is what happened first. async function rescore(status, agent, said) { const runs = () => link("/settings/sources#runs", "follow it under Runs"); status.replaceChildren(`${said}. Scoring every open role; `, runs(), "."); let job; try { - job = await api("/api/jobs", { method: "POST", body: { kind: "score", payload: {} } }); + job = await api("/api/score", { method: "POST" }); } catch (err) { status.textContent = `${said}, but scoring did not start.`; return; diff --git a/app/web/api.py b/app/web/api.py index 374ef7c..65915fd 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -9,6 +9,7 @@ This is roledesk's own API, over the same store. Every refusal is from __future__ import annotations +import sqlite3 from typing import Annotated, Any, Literal from fastapi import APIRouter, HTTPException, Query, Response @@ -349,18 +350,68 @@ def detach_company_board(key: str, store: StoreDep) -> dict[str, Any]: return {"closed": closed} +def _score_after_change(queue: Any) -> Job | None: + """The `score` job whose result will include a change just written. + + A `score` that is queued and has not started will read the change, so + it is the answer: the plain one, keyed `score`, or a follow-up. Else a + plain request, which the queue dedupes to a live job keyed `score` + because the kind is `solo`. When that job is already running it read + the old state, so it gets one follow-up that waits for it (`depends_on`, + `after="done"`), deduped on `score:after:`: ten requests during one + run are one extra pass. A pull's `score` stage is a plain `score` job + too, queued when its stage starts, so a request during a pull joins it. + + The retry is for a race in `pipeline.table.Jobs.enqueue`: when the + INSERT collides with a live job that finishes before the SELECT after + it, the SELECT finds nothing and the `IntegrityError` is re-raised. That + job has finished by then, so a second try succeeds.""" + for attempt in range(3): + try: + queued = queue.jobs.last("score", states=("queued",)) + if queued is not None and (queued.dedup_key == "score" + or queued.dedup_key.startswith("score:after:")): + return queued + job = queue.enqueue("score") + if job is not None and job.state == "running": + job = queue.enqueue("score", depends_on=[job.id], after="done", + dedup_key=f"score:after:{job.id}") + return job + except sqlite3.IntegrityError: + if attempt == 2: + raise + return None + + +@router.post("/score") +def request_score(queue: QueueDep) -> Job: + """Score every agent's roles after a change: now, or right after the + `score` that is already running (see `_score_after_change`). Returns the + job to wait for; the Search agents page waits on it after a save.""" + job = _score_after_change(queue) + if job is None: + raise HTTPException(409, "Scoring would not start.") + return job + + class AffinityBody(BaseModel): affinity: int #: Whose affinity; the default agent when absent. agent: int | None = None +class AffinityOut(Company): + #: The `score` job whose result will include this change; None when none + #: could be queued. + score_job: int | None = None + + @router.post("/companies/{key}/affinity") -def set_affinity(key: str, body: AffinityBody, store: StoreDep, queue: QueueDep) -> Company: +def set_affinity(key: str, body: AffinityBody, store: StoreDep, + queue: QueueDep) -> AffinityOut: """-1 hidden, 0 neutral, 1 liked, 2 favourite, for `body.agent` or the default agent. A change here is not seen in Best match until `score` has - run: the queue dedupes to a live job of that kind, so several clicks are - one run.""" + run, so it queues one (`_score_after_change`) and answers with its id.""" if body.affinity not in AFFINITY: raise HTTPException(422, "affinity is -1, 0, 1 or 2") owner = _agent_or_default(store, body.agent) @@ -368,8 +419,8 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep, queue: QueueDep) company = store.set_affinity(owner.id, key, body.affinity) except KeyError: raise HTTPException(404, f"no such company: {key}") from None - queue.enqueue("score") - return company + job = _score_after_change(queue) + return AffinityOut(**company.model_dump(), score_job=job.id if job else None) # ----------------------------------------------------------------- sources @@ -589,8 +640,8 @@ class AgentBody(BaseModel): @router.post("/agents") def save_agent(body: AgentBody, store: StoreDep) -> Agent: - """Scores are not recomputed here: that is the `score` task, which the - page starts after a save.""" + """Scores are not recomputed here: the page asks `POST /api/score` after + a save and waits on the job it answers with.""" try: return store.save_agent(body.name, body.intent, body.gates, body.weights, body.enabled, agent_id=body.id, limits=body.limits) diff --git a/docs/decisions.md b/docs/decisions.md index cb4878a..052d6d6 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -158,7 +158,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning **"Hidden" is the word the UI uses for affinity -1; the code keeps `mute` and `hide_muted`.** Renaming the columns would touch every reader of `agents` and `companies` for no behavioural gain, so only the toggle's title, the filter label and the dimmed row say "hidden" in the UI. -**A company affinity change enqueues `score`.** `POST /api/companies/{key}/affinity` queues the `score` job right after the update, deduped to a live run of the same kind so ten clicks are one pass. Otherwise an affinity change is invisible in Best match until the next 6-hourly pull, and an unmuted company's roles have no score rows at all until then, so they are missing from that order entirely. +**A company affinity change or an agent save queues a `score` that will see it.** `POST /api/companies/{key}/affinity` and `POST /api/score`, which the Search agents page calls after a save, answer with the id of that job, and the page waits on it. `score` is `solo`, so the queue dedupes a plain request into the live job keyed `score`; a queued one has not read anything yet and sees the change, but a running one read the old state, and deduping into it reported "Saved and scored" for a score that did not include the save. So a request during a run queues one follow-up with `depends_on` the running job, `after="done"` and the key `score:after:`, which makes ten clicks during one run one extra pass, and a queued follow-up is the answer until it starts. Without a score after it, an affinity change is invisible in Best match until the next 6-hourly pull, and an unmuted company's roles have no score rows until then. `pipeline`'s `Jobs.enqueue` has a race: when its INSERT collides with a live job that finishes before the SELECT after it, it re-raises the `IntegrityError` (`tests/test_web.py`'s affinity test failed on it about one run in ten). The app retries, which succeeds because that job has finished; the fix belongs upstream in `../boilerplate`. **A search role's posting text is fetched on demand, not in the pull.** `POST /api/roles/{uid}/text` scrapes `original_url` only when the read page asks for it, because following Jobindex's redirect for every role during a pull would count as a click in Jobindex's own statistics; one click for a posting somebody actually opened is fair. diff --git a/tests/test_web.py b/tests/test_web.py index 5d2fb69..4835b99 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -5,6 +5,7 @@ worker beyond starting and stopping it with the app.""" from __future__ import annotations import json +import sqlite3 from collections.abc import Iterator from datetime import UTC, datetime, timedelta from pathlib import Path @@ -18,6 +19,7 @@ from app.config import Config, ServerConfig from app.main import create_app from app.storage import Store from app.web import MOVED, SETTINGS, TABS +from app.web.api import _score_after_change from app.web.pages import SECTIONS from auth import AuthConfig from desk.models import Item @@ -444,7 +446,7 @@ def test_set_affinity_validates_updates_and_enqueues_a_score( assert res.json()["affinity"] == 2 assert store.company("acme", agent.id).affinity == 2 # type: ignore[union-attr] assert store.company("acme", other.id).affinity == 0 # type: ignore[union-attr] - assert queue.jobs.last("score", states=("queued", "running", "done")) is not None + assert queue.jobs.get(res.json()["score_job"]).kind == "score" res = client.post("/api/companies/acme/affinity", json={"affinity": -1, "agent": other.id}) assert res.status_code == 200 and res.json()["affinity"] == -1 @@ -458,6 +460,49 @@ def test_set_affinity_validates_updates_and_enqueues_a_score( assert client.get("/api/companies", params={"agent": 999}).status_code == 404 +def test_a_score_asked_for_during_a_run_is_one_follow_up_after_it(queue: Any) -> None: + """No worker runs here (no client), so the test moves the jobs itself.""" + first = _score_after_change(queue) + assert first is not None and first.dedup_key == "score" + assert _score_after_change(queue).id == first.id # queued: it will read the change + + assert queue.jobs.claim({"cpu": 1}).id == first.id + during = {_score_after_change(queue).id for _ in range(10)} + assert len(during) == 1 and first.id not in during + follow = queue.jobs.get(during.pop()) + assert (follow.depends_on, follow.after, follow.dedup_key) == ( + [first.id], "done", f"score:after:{first.id}") + + queue.jobs.finish(first.id, "done") + assert _score_after_change(queue).id == follow.id # queued, not started: still the one + + assert queue.jobs.claim({"cpu": 1}).id == follow.id + after = _score_after_change(queue) + assert after.id not in (first.id, follow.id) and after.dedup_key == "score" + + +def test_score_after_change_retries_the_queues_dedup_race( + queue: Any, monkeypatch: pytest.MonkeyPatch) -> None: + """`Jobs.enqueue` re-raises the UNIQUE failure when the live job it + collided with finishes before it reads it back.""" + real, calls = queue.enqueue, [] + + def racy(*args: Any, **kwargs: Any) -> Any: + calls.append(args) + if len(calls) == 1: + raise sqlite3.IntegrityError("UNIQUE constraint failed: jobs.dedup_key") + return real(*args, **kwargs) + + monkeypatch.setattr(queue, "enqueue", racy) + assert _score_after_change(queue) is not None and len(calls) == 2 + + +def test_post_score_answers_with_the_job_to_wait_for(client: TestClient, queue: Any) -> None: + res = client.post("/api/score") + assert res.status_code == 200 + assert queue.jobs.get(res.json()["id"]).kind == "score" + + # ----------------------------------------------------------- applications def test_save_creates_an_application_with_a_saved_event(