Files
jobboard/pipeline/tests/test_dedup.py
T

155 lines
5.3 KiB
Python

"""A `dedup_key` enqueue racing the live job it collides with.
`Jobs.enqueue` either hands back the live job holding the key or queues a new
one. It must never raise `IntegrityError` because that job finished while the
enqueue was between looking for it and inserting.
The first test makes the finish land at that exact moment, by hooking the
enqueue's own connection: when it starts the statement that looks the key up,
a second thread tries to finish the live job on a connection of its own. The
second test is the same race left to the scheduler, many times over.
python3 -m pytest pipeline/tests/test_dedup.py -q
"""
from __future__ import annotations
import sqlite3
import sys
import tempfile
import threading
from contextlib import closing
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from pipeline import Jobs
from pipeline.models import Job
@pytest.fixture
def jobs():
with tempfile.TemporaryDirectory() as tmp:
yield Jobs(Path(tmp) / "t.db")
def made(queued: Job | None) -> Job:
assert queued is not None
return queued
def live(jobs: Jobs, key: str) -> list[int]:
with closing(sqlite3.connect(jobs.path)) as conn:
return [r[0] for r in conn.execute(
"SELECT id FROM jobs WHERE dedup_key=? AND state IN ('queued','running')",
(key,))]
def finish_when_looked_up(jobs: Jobs, monkeypatch: pytest.MonkeyPatch,
job_id: int) -> list[tuple[threading.Thread, bool]]:
"""Finish `job_id` when an enqueue starts the SELECT that looks up its key.
A trace callback runs as a statement starts, before it takes its read
snapshot, so a write committed from inside it is visible to that SELECT.
The finish runs on its own thread and connection, and the callback waits
for it up to half a second: without a write lock held by the enqueue it
lands at once, and with one it waits until the enqueue commits. Either way
the enqueue then goes on, so the outcome does not depend on timing.
Returns the finishing thread, and whether it was still blocked when the
callback stopped waiting for it.
"""
fired: list[tuple[threading.Thread, bool]] = []
enqueuer = threading.get_ident()
connect = jobs._connect
def trace(sql: str) -> None:
if fired or threading.get_ident() != enqueuer:
return
text = " ".join(sql.split()).upper()
if not (text.startswith("SELECT") and "DEDUP_KEY" in text):
return
worker = threading.Thread(target=jobs.finish, args=(job_id, "done"))
worker.start()
worker.join(timeout=0.5)
fired.append((worker, worker.is_alive()))
def hooked() -> sqlite3.Connection:
conn = connect()
conn.set_trace_callback(trace)
return conn
monkeypatch.setattr(jobs, "_connect", hooked)
return fired
def test_live_job_finishing_mid_enqueue_is_not_an_error(jobs, monkeypatch):
first = made(jobs.enqueue("pull", dedup_key="pull"))
assert made(jobs.claim()).id == first.id
fired = finish_when_looked_up(jobs, monkeypatch, first.id)
again = jobs.enqueue("pull", dedup_key="pull")
assert fired, "the hook never saw the key being looked up"
finisher, blocked = fired[0]
finisher.join(timeout=35)
assert not finisher.is_alive()
# The finish could not come between the lookup and the insert: it waited
# for the enqueue's write lock, so the enqueue saw the job live and was
# handed it, and the finish landed after.
assert blocked
assert again is not None and again.id == first.id
assert made(jobs.get(first.id)).state == "done"
assert live(jobs, "pull") == []
# And with the key free, the next enqueue queues a job of its own.
fresh = made(jobs.enqueue("pull", dedup_key="pull"))
assert fresh.id != first.id and fresh.state == "queued"
def test_enqueues_racing_a_worker_never_raise_or_double_up(jobs):
"""Several threads enqueue one key while another claims and finishes it."""
key, rounds, threads = "solo", 60, 6
errors: list[BaseException] = []
doubled: list[list[int]] = []
stop = threading.Event()
def enqueue() -> None:
try:
for _ in range(rounds):
job = made(jobs.enqueue("solo", dedup_key=key))
assert job.dedup_key == key
except BaseException as exc: # noqa: BLE001 - reported below
errors.append(exc)
def work() -> None:
try:
while not stop.is_set():
job = jobs.claim()
if job is not None:
jobs.finish(job.id, "done")
except BaseException as exc: # noqa: BLE001
errors.append(exc)
def watch() -> None:
while not stop.is_set():
ids = live(jobs, key)
if len(ids) > 1:
doubled.append(ids)
helpers = [threading.Thread(target=work), threading.Thread(target=watch)]
senders = [threading.Thread(target=enqueue) for _ in range(threads)]
for t in helpers + senders:
t.start()
for t in senders:
t.join(timeout=120)
stop.set()
for t in helpers:
t.join(timeout=35)
assert not errors, errors
assert not doubled, doubled
done = [j for j in jobs.snapshot(recent=10_000).recent if j.dedup_key == key]
assert done, "the worker never finished a job, so nothing raced"