86 lines
3.7 KiB
Python
86 lines
3.7 KiB
Python
"""Offline database replacement, run as the compose service's uid.
|
|
|
|
The app container must be stopped. Sent over stdin to the image on the
|
|
server, whose working directory is the app's: the database is the one the
|
|
app's config names (`from app.config` follows the package when it is renamed),
|
|
and the rest is stdlib, so the old image and the new one both run it.
|
|
|
|
`REQUIRED_TABLES` is the table check, which stops a right-shaped-but-wrong
|
|
database (last project's, or a half-downloaded one) from landing on top of a
|
|
good one. `tests/test_deploy.py` fails when the app stops creating one of them.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import gzip
|
|
import os
|
|
import shutil
|
|
import sqlite3
|
|
import tempfile
|
|
from contextlib import closing
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
INCOMING = Path("/data/incoming.db.gz")
|
|
BACKUPS = Path("/data/backups")
|
|
|
|
#: Any one of these missing means this is not our database. Name tables that
|
|
#: have existed since the first schema, not ones a recent migration added:
|
|
#: `jobs` is the queue's and `roles` is `app/storage/schema.py`'s first table
|
|
#: (the boilerplate has `ticks`). backup.sh's TABLES is the same set.
|
|
REQUIRED_TABLES = {"jobs", "roles"}
|
|
|
|
|
|
def replace_database(database: Path, archive: Path, backups: Path) -> Path | None:
|
|
"""Validate first, snapshot the stopped database, then atomically replace it."""
|
|
fd, name = tempfile.mkstemp(prefix=".incoming-", suffix=".db", dir=database.parent)
|
|
staged = Path(name)
|
|
backup = None
|
|
try:
|
|
with os.fdopen(fd, "wb") as output, gzip.open(archive, "rb") as source:
|
|
shutil.copyfileobj(source, output)
|
|
output.flush()
|
|
os.fsync(output.fileno())
|
|
with closing(sqlite3.connect(staged)) as candidate:
|
|
if candidate.execute("PRAGMA integrity_check").fetchall() != [("ok",)]:
|
|
raise ValueError("Incoming database failed integrity_check")
|
|
tables = {row[0] for row in candidate.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='table'")}
|
|
if not REQUIRED_TABLES <= tables:
|
|
raise ValueError("Incoming file is not this application's database")
|
|
|
|
if database.exists():
|
|
backups.mkdir(parents=True, exist_ok=True)
|
|
stamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S-%f")
|
|
snapshot = backups / f"{database.stem}-{stamp}.db"
|
|
backup = snapshot.with_suffix(".db.gz")
|
|
with closing(sqlite3.connect(database)) as current:
|
|
current.execute("VACUUM INTO ?", (str(snapshot),))
|
|
# Refuse a competing connection instead of removing a live WAL.
|
|
busy, _, _ = current.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()
|
|
if busy:
|
|
raise RuntimeError("Database is busy; stop every writer before importing")
|
|
# Keep the uncompressed recovery copy if compression fails.
|
|
with snapshot.open("rb") as source, gzip.open(backup, "xb") as output:
|
|
shutil.copyfileobj(source, output)
|
|
with backup.open("r+b") as saved:
|
|
os.fsync(saved.fileno())
|
|
snapshot.unlink()
|
|
|
|
for suffix in ("-wal", "-shm"):
|
|
Path(str(database) + suffix).unlink(missing_ok=True)
|
|
os.replace(staged, database)
|
|
return backup
|
|
finally:
|
|
staged.unlink(missing_ok=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
from app.config import load
|
|
|
|
# The snapshot is the whole database, so it is owner only whatever the
|
|
# container's umask is; the gzip of it is made under the same mask.
|
|
os.umask(0o077)
|
|
saved = replace_database(load().database.absolute(), INCOMING, BACKUPS)
|
|
print(f"Database replaced; previous snapshot: {saved or 'none (first import)'}")
|