"""A `dedup_key` enqueue racing the live job it collides with. `Jobs.enqueue` either hands back the live job holding the key or queues a new one. It must never raise `IntegrityError` because that job finished while the enqueue was between looking for it and inserting. The first test makes the finish land at that exact moment, by hooking the enqueue's own connection: when it starts the statement that looks the key up, a second thread tries to finish the live job on a connection of its own. The second test is the same race left to the scheduler, many times over. python3 -m pytest pipeline/tests/test_dedup.py -q """ from __future__ import annotations import sqlite3 import sys import tempfile import threading from contextlib import closing from pathlib import Path import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from pipeline import Jobs from pipeline.models import Job @pytest.fixture def jobs(): with tempfile.TemporaryDirectory() as tmp: yield Jobs(Path(tmp) / "t.db") def made(queued: Job | None) -> Job: assert queued is not None return queued def live(jobs: Jobs, key: str) -> list[int]: with closing(sqlite3.connect(jobs.path)) as conn: return [r[0] for r in conn.execute( "SELECT id FROM jobs WHERE dedup_key=? AND state IN ('queued','running')", (key,))] def finish_when_looked_up(jobs: Jobs, monkeypatch: pytest.MonkeyPatch, job_id: int) -> list[tuple[threading.Thread, bool]]: """Finish `job_id` when an enqueue starts the SELECT that looks up its key. A trace callback runs as a statement starts, before it takes its read snapshot, so a write committed from inside it is visible to that SELECT. The finish runs on its own thread and connection, and the callback waits for it up to half a second: without a write lock held by the enqueue it lands at once, and with one it waits until the enqueue commits. Either way the enqueue then goes on, so the outcome does not depend on timing. Returns the finishing thread, and whether it was still blocked when the callback stopped waiting for it. """ fired: list[tuple[threading.Thread, bool]] = [] enqueuer = threading.get_ident() connect = jobs._connect def trace(sql: str) -> None: if fired or threading.get_ident() != enqueuer: return text = " ".join(sql.split()).upper() if not (text.startswith("SELECT") and "DEDUP_KEY" in text): return worker = threading.Thread(target=jobs.finish, args=(job_id, "done")) worker.start() worker.join(timeout=0.5) fired.append((worker, worker.is_alive())) def hooked() -> sqlite3.Connection: conn = connect() conn.set_trace_callback(trace) return conn monkeypatch.setattr(jobs, "_connect", hooked) return fired def test_live_job_finishing_mid_enqueue_is_not_an_error(jobs, monkeypatch): first = made(jobs.enqueue("pull", dedup_key="pull")) assert made(jobs.claim()).id == first.id fired = finish_when_looked_up(jobs, monkeypatch, first.id) again = jobs.enqueue("pull", dedup_key="pull") assert fired, "the hook never saw the key being looked up" finisher, blocked = fired[0] finisher.join(timeout=35) assert not finisher.is_alive() # The finish could not come between the lookup and the insert: it waited # for the enqueue's write lock, so the enqueue saw the job live and was # handed it, and the finish landed after. assert blocked assert again is not None and again.id == first.id assert made(jobs.get(first.id)).state == "done" assert live(jobs, "pull") == [] # And with the key free, the next enqueue queues a job of its own. fresh = made(jobs.enqueue("pull", dedup_key="pull")) assert fresh.id != first.id and fresh.state == "queued" def test_enqueues_racing_a_worker_never_raise_or_double_up(jobs): """Several threads enqueue one key while another claims and finishes it.""" key, rounds, threads = "solo", 60, 6 errors: list[BaseException] = [] doubled: list[list[int]] = [] stop = threading.Event() def enqueue() -> None: try: for _ in range(rounds): job = made(jobs.enqueue("solo", dedup_key=key)) assert job.dedup_key == key except BaseException as exc: # noqa: BLE001 - reported below errors.append(exc) def work() -> None: try: while not stop.is_set(): job = jobs.claim() if job is not None: jobs.finish(job.id, "done") except BaseException as exc: # noqa: BLE001 errors.append(exc) def watch() -> None: while not stop.is_set(): ids = live(jobs, key) if len(ids) > 1: doubled.append(ids) helpers = [threading.Thread(target=work), threading.Thread(target=watch)] senders = [threading.Thread(target=enqueue) for _ in range(threads)] for t in helpers + senders: t.start() for t in senders: t.join(timeout=120) stop.set() for t in helpers: t.join(timeout=35) assert not errors, errors assert not doubled, doubled done = [j for j in jobs.snapshot(recent=10_000).recent if j.dedup_key == key] assert done, "the worker never finished a job, so nothing raced"