From 92466e2cd4654d4561536b1879c9eacddf5e714a Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 16:02:58 +0200 Subject: [PATCH] Drop the retry around the queue's dedup race, fixed upstream `_score_after_change` tried three times because `Jobs.enqueue` could re-raise an IntegrityError when the live job finished between its insert and its lookup. The queue now does both in one BEGIN IMMEDIATE transaction (boilerplate a1c6f0d), so the workaround and its test go. Co-Authored-By: Claude Fable 5.1 --- app/web/api.py | 32 ++++++++++---------------------- docs/decisions.md | 2 +- tests/test_web.py | 17 ----------------- 3 files changed, 11 insertions(+), 40 deletions(-) diff --git a/app/web/api.py b/app/web/api.py index 9aef1a9..0e34723 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -9,7 +9,6 @@ This is roledesk's own API, over the same store. Every refusal is from __future__ import annotations -import sqlite3 from typing import Annotated, Any, Literal from fastapi import APIRouter, HTTPException, Query, Response @@ -405,27 +404,16 @@ def _score_after_change(queue: Any) -> Job | None: the old state, so it gets one follow-up that waits for it (`depends_on`, `after="done"`), deduped on `score:after:`: ten requests during one run are one extra pass. A pull's `score` stage is a plain `score` job - too, queued when its stage starts, so a request during a pull joins it. - - The retry is for a race in `pipeline.table.Jobs.enqueue`: when the - INSERT collides with a live job that finishes before the SELECT after - it, the SELECT finds nothing and the `IntegrityError` is re-raised. That - job has finished by then, so a second try succeeds.""" - for attempt in range(3): - try: - queued = queue.jobs.last("score", states=("queued",)) - if queued is not None and (queued.dedup_key == "score" - or queued.dedup_key.startswith("score:after:")): - return queued - job = queue.enqueue("score") - if job is not None and job.state == "running": - job = queue.enqueue("score", depends_on=[job.id], after="done", - dedup_key=f"score:after:{job.id}") - return job - except sqlite3.IntegrityError: - if attempt == 2: - raise - return None + too, queued when its stage starts, so a request during a pull joins it.""" + queued = queue.jobs.last("score", states=("queued",)) + if queued is not None and (queued.dedup_key == "score" + or queued.dedup_key.startswith("score:after:")): + return queued + job: Job | None = queue.enqueue("score") + if job is not None and job.state == "running": + job = queue.enqueue("score", depends_on=[job.id], after="done", + dedup_key=f"score:after:{job.id}") + return job @router.post("/score") diff --git a/docs/decisions.md b/docs/decisions.md index 075b377..320d3d9 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -178,7 +178,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning **"Hidden" is the word the UI uses for affinity -1; the code keeps `mute` and `hide_muted`.** Renaming the columns would touch every reader of `agents` and `companies` for no behavioural gain, so only the toggle's title, the filter label and the dimmed row say "hidden" in the UI. -**A company affinity change or an agent save queues a `score` that will see it.** `POST /api/companies/{key}/affinity` and `POST /api/score`, which the Search agents page calls after a save, answer with the id of that job, and the page waits on it. `score` is `solo`, so the queue dedupes a plain request into the live job keyed `score`; a queued one has not read anything yet and sees the change, but a running one read the old state, and deduping into it reported "Saved and scored" for a score that did not include the save. So a request during a run queues one follow-up with `depends_on` the running job, `after="done"` and the key `score:after:`, which makes ten clicks during one run one extra pass, and a queued follow-up is the answer until it starts. Without a score after it, an affinity change is invisible in Best match until the next 6-hourly pull, and an unmuted company's roles have no score rows until then. `pipeline`'s `Jobs.enqueue` has a race: when its INSERT collides with a live job that finishes before the SELECT after it, it re-raises the `IntegrityError` (`tests/test_web.py`'s affinity test failed on it about one run in ten). The app retries, which succeeds because that job has finished; the fix belongs upstream in `../boilerplate`. +**A company affinity change or an agent save queues a `score` that will see it.** `POST /api/companies/{key}/affinity` and `POST /api/score`, which the Search agents page calls after a save, answer with the id of that job, and the page waits on it. `score` is `solo`, so the queue dedupes a plain request into the live job keyed `score`; a queued one has not read anything yet and sees the change, but a running one read the old state, and deduping into it reported "Saved and scored" for a score that did not include the save. So a request during a run queues one follow-up with `depends_on` the running job, `after="done"` and the key `score:after:`, which makes ten clicks during one run one extra pass, and a queued follow-up is the answer until it starts. Without a score after it, an affinity change is invisible in Best match until the next 6-hourly pull, and an unmuted company's roles have no score rows until then. **A search role's posting text is fetched on demand, not in the pull.** `POST /api/roles/{uid}/text` scrapes `original_url` only when the read page asks for it, because following Jobindex's redirect for every role during a pull would count as a click in Jobindex's own statistics; one click for a posting somebody actually opened is fair. diff --git a/tests/test_web.py b/tests/test_web.py index b056508..19287b6 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -5,7 +5,6 @@ worker beyond starting and stopping it with the app.""" from __future__ import annotations import json -import sqlite3 from collections.abc import Iterator from datetime import UTC, datetime, timedelta from pathlib import Path @@ -547,22 +546,6 @@ def test_a_score_asked_for_during_a_run_is_one_follow_up_after_it(queue: Any) -> assert after.id not in (first.id, follow.id) and after.dedup_key == "score" -def test_score_after_change_retries_the_queues_dedup_race( - queue: Any, monkeypatch: pytest.MonkeyPatch) -> None: - """`Jobs.enqueue` re-raises the UNIQUE failure when the live job it - collided with finishes before it reads it back.""" - real, calls = queue.enqueue, [] - - def racy(*args: Any, **kwargs: Any) -> Any: - calls.append(args) - if len(calls) == 1: - raise sqlite3.IntegrityError("UNIQUE constraint failed: jobs.dedup_key") - return real(*args, **kwargs) - - monkeypatch.setattr(queue, "enqueue", racy) - assert _score_after_change(queue) is not None and len(calls) == 2 - - def test_post_score_answers_with_the_job_to_wait_for(client: TestClient, queue: Any) -> None: res = client.post("/api/score") assert res.status_code == 200