Files
jobboard/deploy/server/import_database.py
T

86 lines
3.7 KiB
Python

"""Offline database replacement, run as the compose service's uid.
The app container must be stopped. Sent over stdin to the image on the
server, whose working directory is the app's: the database is the one the
app's config names (`from app.config` follows the package when it is renamed),
and the rest is stdlib, so the old image and the new one both run it.
`REQUIRED_TABLES` is the table check, which stops a right-shaped-but-wrong
database (last project's, or a half-downloaded one) from landing on top of a
good one. `tests/test_deploy.py` fails when the app stops creating one of them.
"""
from __future__ import annotations
import gzip
import os
import shutil
import sqlite3
import tempfile
from contextlib import closing
from datetime import UTC, datetime
from pathlib import Path
INCOMING = Path("/data/incoming.db.gz")
BACKUPS = Path("/data/backups")
#: Any one of these missing means this is not our database. Name tables that
#: have existed since the first schema, not ones a recent migration added:
#: `jobs` is the queue's and `roles` is `app/storage/schema.py`'s first table
#: (the boilerplate has `ticks`). backup.sh's TABLES is the same set.
REQUIRED_TABLES = {"jobs", "roles"}
def replace_database(database: Path, archive: Path, backups: Path) -> Path | None:
"""Validate first, snapshot the stopped database, then atomically replace it."""
fd, name = tempfile.mkstemp(prefix=".incoming-", suffix=".db", dir=database.parent)
staged = Path(name)
backup = None
try:
with os.fdopen(fd, "wb") as output, gzip.open(archive, "rb") as source:
shutil.copyfileobj(source, output)
output.flush()
os.fsync(output.fileno())
with closing(sqlite3.connect(staged)) as candidate:
if candidate.execute("PRAGMA integrity_check").fetchall() != [("ok",)]:
raise ValueError("Incoming database failed integrity_check")
tables = {row[0] for row in candidate.execute(
"SELECT name FROM sqlite_master WHERE type='table'")}
if not REQUIRED_TABLES <= tables:
raise ValueError("Incoming file is not this application's database")
if database.exists():
backups.mkdir(parents=True, exist_ok=True)
stamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S-%f")
snapshot = backups / f"{database.stem}-{stamp}.db"
backup = snapshot.with_suffix(".db.gz")
with closing(sqlite3.connect(database)) as current:
current.execute("VACUUM INTO ?", (str(snapshot),))
# Refuse a competing connection instead of removing a live WAL.
busy, _, _ = current.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()
if busy:
raise RuntimeError("Database is busy; stop every writer before importing")
# Keep the uncompressed recovery copy if compression fails.
with snapshot.open("rb") as source, gzip.open(backup, "xb") as output:
shutil.copyfileobj(source, output)
with backup.open("r+b") as saved:
os.fsync(saved.fileno())
snapshot.unlink()
for suffix in ("-wal", "-shm"):
Path(str(database) + suffix).unlink(missing_ok=True)
os.replace(staged, database)
return backup
finally:
staged.unlink(missing_ok=True)
if __name__ == "__main__":
from app.config import load
# The snapshot is the whole database, so it is owner only whatever the
# container's umask is; the gzip of it is made under the same mask.
os.umask(0o077)
saved = replace_database(load().database.absolute(), INCOMING, BACKUPS)
print(f"Database replaced; previous snapshot: {saved or 'none (first import)'}")