Pull pipeline/ from boilerplate a1c6f0d: a dedup key is looked up and inserted in one transaction
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
471ace703c
commit
2e4624cbc4
3 files changed
+222
-51
No files matched your search
@@ -0,0 +1,154 @@
|
||||
"""A `dedup_key` enqueue racing the live job it collides with.
|
||||
|
||||
`Jobs.enqueue` either hands back the live job holding the key or queues a new
|
||||
one. It must never raise `IntegrityError` because that job finished while the
|
||||
enqueue was between looking for it and inserting.
|
||||
|
||||
The first test makes the finish land at that exact moment, by hooking the
|
||||
enqueue's own connection: when it starts the statement that looks the key up,
|
||||
a second thread tries to finish the live job on a connection of its own. The
|
||||
second test is the same race left to the scheduler, many times over.
|
||||
|
||||
python3 -m pytest pipeline/tests/test_dedup.py -q
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from pipeline import Jobs
|
||||
from pipeline.models import Job
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def jobs():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
yield Jobs(Path(tmp) / "t.db")
|
||||
|
||||
|
||||
def made(queued: Job | None) -> Job:
|
||||
assert queued is not None
|
||||
return queued
|
||||
|
||||
|
||||
def live(jobs: Jobs, key: str) -> list[int]:
|
||||
with closing(sqlite3.connect(jobs.path)) as conn:
|
||||
return [r[0] for r in conn.execute(
|
||||
"SELECT id FROM jobs WHERE dedup_key=? AND state IN ('queued','running')",
|
||||
(key,))]
|
||||
|
||||
|
||||
def finish_when_looked_up(jobs: Jobs, monkeypatch: pytest.MonkeyPatch,
|
||||
job_id: int) -> list[tuple[threading.Thread, bool]]:
|
||||
"""Finish `job_id` when an enqueue starts the SELECT that looks up its key.
|
||||
|
||||
A trace callback runs as a statement starts, before it takes its read
|
||||
snapshot, so a write committed from inside it is visible to that SELECT.
|
||||
The finish runs on its own thread and connection, and the callback waits
|
||||
for it up to half a second: without a write lock held by the enqueue it
|
||||
lands at once, and with one it waits until the enqueue commits. Either way
|
||||
the enqueue then goes on, so the outcome does not depend on timing.
|
||||
|
||||
Returns the finishing thread, and whether it was still blocked when the
|
||||
callback stopped waiting for it.
|
||||
"""
|
||||
fired: list[tuple[threading.Thread, bool]] = []
|
||||
enqueuer = threading.get_ident()
|
||||
connect = jobs._connect
|
||||
|
||||
def trace(sql: str) -> None:
|
||||
if fired or threading.get_ident() != enqueuer:
|
||||
return
|
||||
text = " ".join(sql.split()).upper()
|
||||
if not (text.startswith("SELECT") and "DEDUP_KEY" in text):
|
||||
return
|
||||
worker = threading.Thread(target=jobs.finish, args=(job_id, "done"))
|
||||
worker.start()
|
||||
worker.join(timeout=0.5)
|
||||
fired.append((worker, worker.is_alive()))
|
||||
|
||||
def hooked() -> sqlite3.Connection:
|
||||
conn = connect()
|
||||
conn.set_trace_callback(trace)
|
||||
return conn
|
||||
|
||||
monkeypatch.setattr(jobs, "_connect", hooked)
|
||||
return fired
|
||||
|
||||
|
||||
def test_live_job_finishing_mid_enqueue_is_not_an_error(jobs, monkeypatch):
|
||||
first = made(jobs.enqueue("pull", dedup_key="pull"))
|
||||
assert made(jobs.claim()).id == first.id
|
||||
fired = finish_when_looked_up(jobs, monkeypatch, first.id)
|
||||
|
||||
again = jobs.enqueue("pull", dedup_key="pull")
|
||||
|
||||
assert fired, "the hook never saw the key being looked up"
|
||||
finisher, blocked = fired[0]
|
||||
finisher.join(timeout=35)
|
||||
assert not finisher.is_alive()
|
||||
# The finish could not come between the lookup and the insert: it waited
|
||||
# for the enqueue's write lock, so the enqueue saw the job live and was
|
||||
# handed it, and the finish landed after.
|
||||
assert blocked
|
||||
assert again is not None and again.id == first.id
|
||||
assert made(jobs.get(first.id)).state == "done"
|
||||
assert live(jobs, "pull") == []
|
||||
# And with the key free, the next enqueue queues a job of its own.
|
||||
fresh = made(jobs.enqueue("pull", dedup_key="pull"))
|
||||
assert fresh.id != first.id and fresh.state == "queued"
|
||||
|
||||
|
||||
def test_enqueues_racing_a_worker_never_raise_or_double_up(jobs):
|
||||
"""Several threads enqueue one key while another claims and finishes it."""
|
||||
key, rounds, threads = "solo", 60, 6
|
||||
errors: list[BaseException] = []
|
||||
doubled: list[list[int]] = []
|
||||
stop = threading.Event()
|
||||
|
||||
def enqueue() -> None:
|
||||
try:
|
||||
for _ in range(rounds):
|
||||
job = made(jobs.enqueue("solo", dedup_key=key))
|
||||
assert job.dedup_key == key
|
||||
except BaseException as exc: # noqa: BLE001 - reported below
|
||||
errors.append(exc)
|
||||
|
||||
def work() -> None:
|
||||
try:
|
||||
while not stop.is_set():
|
||||
job = jobs.claim()
|
||||
if job is not None:
|
||||
jobs.finish(job.id, "done")
|
||||
except BaseException as exc: # noqa: BLE001
|
||||
errors.append(exc)
|
||||
|
||||
def watch() -> None:
|
||||
while not stop.is_set():
|
||||
ids = live(jobs, key)
|
||||
if len(ids) > 1:
|
||||
doubled.append(ids)
|
||||
|
||||
helpers = [threading.Thread(target=work), threading.Thread(target=watch)]
|
||||
senders = [threading.Thread(target=enqueue) for _ in range(threads)]
|
||||
for t in helpers + senders:
|
||||
t.start()
|
||||
for t in senders:
|
||||
t.join(timeout=120)
|
||||
stop.set()
|
||||
for t in helpers:
|
||||
t.join(timeout=35)
|
||||
|
||||
assert not errors, errors
|
||||
assert not doubled, doubled
|
||||
done = [j for j in jobs.snapshot(recent=10_000).recent if j.dedup_key == key]
|
||||
assert done, "the worker never finished a job, so nothing raced"
|
||||
Reference in new issue
Block a user