155 lines
5.3 KiB
Python
155 lines
5.3 KiB
Python
"""A `dedup_key` enqueue racing the live job it collides with.
|
|
|
|
`Jobs.enqueue` either hands back the live job holding the key or queues a new
|
|
one. It must never raise `IntegrityError` because that job finished while the
|
|
enqueue was between looking for it and inserting.
|
|
|
|
The first test makes the finish land at that exact moment, by hooking the
|
|
enqueue's own connection: when it starts the statement that looks the key up,
|
|
a second thread tries to finish the live job on a connection of its own. The
|
|
second test is the same race left to the scheduler, many times over.
|
|
|
|
python3 -m pytest pipeline/tests/test_dedup.py -q
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
from contextlib import closing
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
|
|
from pipeline import Jobs
|
|
from pipeline.models import Job
|
|
|
|
|
|
@pytest.fixture
|
|
def jobs():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
yield Jobs(Path(tmp) / "t.db")
|
|
|
|
|
|
def made(queued: Job | None) -> Job:
|
|
assert queued is not None
|
|
return queued
|
|
|
|
|
|
def live(jobs: Jobs, key: str) -> list[int]:
|
|
with closing(sqlite3.connect(jobs.path)) as conn:
|
|
return [r[0] for r in conn.execute(
|
|
"SELECT id FROM jobs WHERE dedup_key=? AND state IN ('queued','running')",
|
|
(key,))]
|
|
|
|
|
|
def finish_when_looked_up(jobs: Jobs, monkeypatch: pytest.MonkeyPatch,
|
|
job_id: int) -> list[tuple[threading.Thread, bool]]:
|
|
"""Finish `job_id` when an enqueue starts the SELECT that looks up its key.
|
|
|
|
A trace callback runs as a statement starts, before it takes its read
|
|
snapshot, so a write committed from inside it is visible to that SELECT.
|
|
The finish runs on its own thread and connection, and the callback waits
|
|
for it up to half a second: without a write lock held by the enqueue it
|
|
lands at once, and with one it waits until the enqueue commits. Either way
|
|
the enqueue then goes on, so the outcome does not depend on timing.
|
|
|
|
Returns the finishing thread, and whether it was still blocked when the
|
|
callback stopped waiting for it.
|
|
"""
|
|
fired: list[tuple[threading.Thread, bool]] = []
|
|
enqueuer = threading.get_ident()
|
|
connect = jobs._connect
|
|
|
|
def trace(sql: str) -> None:
|
|
if fired or threading.get_ident() != enqueuer:
|
|
return
|
|
text = " ".join(sql.split()).upper()
|
|
if not (text.startswith("SELECT") and "DEDUP_KEY" in text):
|
|
return
|
|
worker = threading.Thread(target=jobs.finish, args=(job_id, "done"))
|
|
worker.start()
|
|
worker.join(timeout=0.5)
|
|
fired.append((worker, worker.is_alive()))
|
|
|
|
def hooked() -> sqlite3.Connection:
|
|
conn = connect()
|
|
conn.set_trace_callback(trace)
|
|
return conn
|
|
|
|
monkeypatch.setattr(jobs, "_connect", hooked)
|
|
return fired
|
|
|
|
|
|
def test_live_job_finishing_mid_enqueue_is_not_an_error(jobs, monkeypatch):
|
|
first = made(jobs.enqueue("pull", dedup_key="pull"))
|
|
assert made(jobs.claim()).id == first.id
|
|
fired = finish_when_looked_up(jobs, monkeypatch, first.id)
|
|
|
|
again = jobs.enqueue("pull", dedup_key="pull")
|
|
|
|
assert fired, "the hook never saw the key being looked up"
|
|
finisher, blocked = fired[0]
|
|
finisher.join(timeout=35)
|
|
assert not finisher.is_alive()
|
|
# The finish could not come between the lookup and the insert: it waited
|
|
# for the enqueue's write lock, so the enqueue saw the job live and was
|
|
# handed it, and the finish landed after.
|
|
assert blocked
|
|
assert again is not None and again.id == first.id
|
|
assert made(jobs.get(first.id)).state == "done"
|
|
assert live(jobs, "pull") == []
|
|
# And with the key free, the next enqueue queues a job of its own.
|
|
fresh = made(jobs.enqueue("pull", dedup_key="pull"))
|
|
assert fresh.id != first.id and fresh.state == "queued"
|
|
|
|
|
|
def test_enqueues_racing_a_worker_never_raise_or_double_up(jobs):
|
|
"""Several threads enqueue one key while another claims and finishes it."""
|
|
key, rounds, threads = "solo", 60, 6
|
|
errors: list[BaseException] = []
|
|
doubled: list[list[int]] = []
|
|
stop = threading.Event()
|
|
|
|
def enqueue() -> None:
|
|
try:
|
|
for _ in range(rounds):
|
|
job = made(jobs.enqueue("solo", dedup_key=key))
|
|
assert job.dedup_key == key
|
|
except BaseException as exc: # noqa: BLE001 - reported below
|
|
errors.append(exc)
|
|
|
|
def work() -> None:
|
|
try:
|
|
while not stop.is_set():
|
|
job = jobs.claim()
|
|
if job is not None:
|
|
jobs.finish(job.id, "done")
|
|
except BaseException as exc: # noqa: BLE001
|
|
errors.append(exc)
|
|
|
|
def watch() -> None:
|
|
while not stop.is_set():
|
|
ids = live(jobs, key)
|
|
if len(ids) > 1:
|
|
doubled.append(ids)
|
|
|
|
helpers = [threading.Thread(target=work), threading.Thread(target=watch)]
|
|
senders = [threading.Thread(target=enqueue) for _ in range(threads)]
|
|
for t in helpers + senders:
|
|
t.start()
|
|
for t in senders:
|
|
t.join(timeout=120)
|
|
stop.set()
|
|
for t in helpers:
|
|
t.join(timeout=35)
|
|
|
|
assert not errors, errors
|
|
assert not doubled, doubled
|
|
done = [j for j in jobs.snapshot(recent=10_000).recent if j.dedup_key == key]
|
|
assert done, "the worker never finished a job, so nothing raced"
|