Pull pipeline/ from boilerplate a1c6f0d: a dedup key is looked up and inserted in one transaction
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
471ace703c
commit
2e4624cbc4
3 files changed
+222
-51
No files matched your search
+66
-51
@@ -447,15 +447,24 @@ class Jobs:
|
||||
Returning the existing job rather than raising is deliberate: the
|
||||
caller asked for that work to be underway, and it is.
|
||||
|
||||
The lookup and the insert share one `BEGIN IMMEDIATE` transaction, so
|
||||
a live job that finishes during an enqueue finishes either before it,
|
||||
and a new job is queued, or after it, and the caller is handed the
|
||||
live one. An `IntegrityError` on the key cannot reach the caller. It
|
||||
used to, when the insert failed on the index and the job had finished
|
||||
by the time a second statement looked for it. Waiting for the write
|
||||
lock goes through the connection's busy timeout like any other write.
|
||||
|
||||
`depends_on` holds this job back until those jobs are over — see the
|
||||
column comments in `SCHEMA` for what `after` chooses between.
|
||||
|
||||
A job under a killed run is refused with `RunCancelled`. That is where
|
||||
a kill reaches everything that had not been queued when the button was
|
||||
pressed: the next task of a killed cascade, the batches a callback was
|
||||
about to fan out. Checked in the job's own transaction, so a kill
|
||||
landing between the check and the insert loses the race by a statement
|
||||
rather than by a process.
|
||||
about to fan out. Checked in the job's own transaction, under its write
|
||||
lock, so a kill cannot land between the check and the insert: it lands
|
||||
before, and the job is refused, or after, and `cancel_run` finds the
|
||||
job queued and ends it.
|
||||
|
||||
The refusal spares an `after='done'` job under a killed *task*, which
|
||||
is the same carve-out `cancel_run` and `stop_task` make and for the
|
||||
@@ -481,55 +490,61 @@ class Jobs:
|
||||
if max_attempts < 1:
|
||||
raise ValueError(f"max_attempts counts the first attempt, so it is "
|
||||
f"at least 1, not {max_attempts!r}")
|
||||
try:
|
||||
with closing(self._connect()) as conn, conn:
|
||||
killed = conn.execute(
|
||||
"""SELECT id FROM runs WHERE cancelled_at <> ''
|
||||
AND (id = ?1 OR (id = ?2 AND ?3 <> 'done'))""",
|
||||
(int(cascade_id), int(task_id), after),
|
||||
with closing(self._connect()) as conn, conn:
|
||||
# Taken before the first read, so no other writer, in this process
|
||||
# or another, can finish, cancel or queue a job between the lookup
|
||||
# below and the insert. Waits out a busy file like any write, under
|
||||
# the connection's busy timeout: IMMEDIATE takes the write lock at
|
||||
# BEGIN, where SQLite calls the busy handler, rather than upgrading
|
||||
# a read snapshot later, where it fails at once.
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
killed = conn.execute(
|
||||
"""SELECT id FROM runs WHERE cancelled_at <> ''
|
||||
AND (id = ?1 OR (id = ?2 AND ?3 <> 'done'))""",
|
||||
(int(cascade_id), int(task_id), after),
|
||||
).fetchone()
|
||||
if killed is not None:
|
||||
raise RunCancelled(
|
||||
f"run {killed['id']} was killed; {kind!r} is not queued")
|
||||
# The same predicate as `idx_jobs_dedup`, read under the write
|
||||
# lock: what it finds is still live when this commits, and when it
|
||||
# finds nothing the insert cannot collide.
|
||||
if dedup_key:
|
||||
existing = conn.execute(
|
||||
"""SELECT * FROM jobs
|
||||
WHERE dedup_key=? AND state IN ('queued','running')
|
||||
ORDER BY id DESC LIMIT 1""",
|
||||
(dedup_key,),
|
||||
).fetchone()
|
||||
if killed is not None:
|
||||
raise RunCancelled(
|
||||
f"run {killed['id']} was killed; {kind!r} is not queued")
|
||||
# A job queued with neither begins both: something asked for
|
||||
# this on its own, so it is a cascade of one task of one job.
|
||||
# Minted in the job's own transaction, so a duplicate that
|
||||
# rolls back takes its ids with it — a `runs` row with no job
|
||||
# under it would be an identity for nothing.
|
||||
#
|
||||
# The cascade says it stands alone and the task does not: the
|
||||
# task is the `pull` task however it was started, but a chain
|
||||
# named `pull` is a different thing from this and has to stay
|
||||
# findable. See `on_its_own`.
|
||||
cascade_id = int(cascade_id) or self._mint(
|
||||
conn, "cascade", on_its_own(kind), label)
|
||||
task_id = int(task_id) or self._mint(conn, "task", kind, label)
|
||||
job_id = conn.execute(
|
||||
"""INSERT INTO jobs (kind, label, payload, state, dedup_key,
|
||||
depends_on, after, resource, cascade_id, task_id,
|
||||
max_attempts, retry_delay, retry_backoff,
|
||||
run_after, created_at)
|
||||
VALUES (?,?,?,'queued',?,?,?,?,?,?,?,?,?,?,?)""",
|
||||
(kind, label, json.dumps(payload or {}), dedup_key,
|
||||
json.dumps([int(i) for i in depends_on]), after,
|
||||
resource or "cpu", int(cascade_id), int(task_id),
|
||||
int(max_attempts), float(retry_delay), float(retry_backoff),
|
||||
_in(delay), _now()),
|
||||
).lastrowid
|
||||
except sqlite3.IntegrityError:
|
||||
existing = self._row(
|
||||
"""SELECT * FROM jobs
|
||||
WHERE dedup_key=? AND state IN ('queued','running')
|
||||
ORDER BY id DESC LIMIT 1""",
|
||||
(dedup_key,),
|
||||
)
|
||||
if existing is None:
|
||||
raise
|
||||
return Job.from_row(existing)
|
||||
job = self.get(job_id) if job_id is not None else None
|
||||
if job is None:
|
||||
raise RuntimeError(f"job {job_id} disappeared between write and read")
|
||||
return job
|
||||
if existing is not None:
|
||||
return Job.from_row(existing)
|
||||
# A job queued with neither begins both: something asked for
|
||||
# this on its own, so it is a cascade of one task of one job.
|
||||
# Minted in the job's own transaction, so a refusal that rolls
|
||||
# back takes its ids with it — a `runs` row with no job under it
|
||||
# would be an identity for nothing.
|
||||
#
|
||||
# The cascade says it stands alone and the task does not: the
|
||||
# task is the `pull` task however it was started, but a chain
|
||||
# named `pull` is a different thing from this and has to stay
|
||||
# findable. See `on_its_own`.
|
||||
cascade_id = int(cascade_id) or self._mint(
|
||||
conn, "cascade", on_its_own(kind), label)
|
||||
task_id = int(task_id) or self._mint(conn, "task", kind, label)
|
||||
row = conn.execute(
|
||||
"""INSERT INTO jobs (kind, label, payload, state, dedup_key,
|
||||
depends_on, after, resource, cascade_id, task_id,
|
||||
max_attempts, retry_delay, retry_backoff,
|
||||
run_after, created_at)
|
||||
VALUES (?,?,?,'queued',?,?,?,?,?,?,?,?,?,?,?)
|
||||
RETURNING *""",
|
||||
(kind, label, json.dumps(payload or {}), dedup_key,
|
||||
json.dumps([int(i) for i in depends_on]), after,
|
||||
resource or "cpu", int(cascade_id), int(task_id),
|
||||
int(max_attempts), float(retry_delay), float(retry_backoff),
|
||||
_in(delay), _now()),
|
||||
).fetchone()
|
||||
return Job.from_row(row)
|
||||
|
||||
def claim(self, workers: Mapping[str, int] | None = None) -> Job | None:
|
||||
"""Take the oldest ready job and mark it running, atomically.
|
||||
|
||||
Reference in new issue
Block a user