#!/usr/bin/env bash # The database, in both directions. # # bash deploy/db.sh push [--yes] this machine's database -> the server # bash deploy/db.sh pull a fresh snapshot from the server -> backups/ # # `push` is the one-off at the start: production begins knowing what you already # have, rather than starting empty. `pull` is the one you keep running, on a # schedule from the server itself (server/backup.sh under cron) and by hand from # here whenever you want the copy somewhere the server is not. # # `push` moves the database and nothing else. It does not stop the app before or # start it after: a deployment knows what else is moving and in what order, and # this does not. Stop the app on the server first; the import is a file swap # under SQLite's feet and is refused while the app is up. Start it again # afterwards. # # Both go through SQLite's own `VACUUM INTO`. The database is in WAL mode, so the # .db file on disk is always missing whatever is still in the -wal beside it, and # copying the three files with the app running is a race you lose quietly: a # database that opens fine and is short a day's writes. # # After a push the two copies diverge from the first thing written on either. # There is no merge for that: pick the one that is authoritative, and use the # other read-only or not at all. # # Settings from release.env: SSH_HOST, REMOTE_DIR and NAME (the compose service) # are required; REMOTE_DATA (default REMOTE_DIR/data), PYTHON and LOCAL_DB # (default the database the app's config names) are not. set -euo pipefail # shellcheck source=_common.sh . "$(dirname "${BASH_SOURCE[0]}")/_common.sh" action='' yes=0 for arg in "$@"; do case "$arg" in push|pull) action=$arg ;; --yes) yes=1 ;; -h|--help) sed -n '2,29p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; *) die "unknown option $arg" ;; esac done [[ -n $action ]] || die "say push or pull: bash deploy/db.sh push|pull" ssh_host=$(require_setting SSH_HOST) remote_dir=$(require_setting REMOTE_DIR) # Where the data actually is. Beside the compose file unless APP_DATA put it # under a shared root; then REMOTE_DATA has to say so. remote_data=$(setting REMOTE_DATA "$remote_dir/data") name=$(require_setting NAME) python=$(setting PYTHON "$root/.venv/bin/python") cd "$root" # ---------------------------------------------------------------------- push if [[ $action == push ]]; then [[ -x $python ]] || die "no interpreter at $python - run uv sync --all-extras, or set PYTHON in deploy/release.env" # The file the app writes: `database` in config.yaml (or $APP_CONFIG), # roledesk.db without one, asked of the app from the checkout's root. local_db=$(setting LOCAL_DB) if [[ -z $local_db ]]; then local_db=$("$python" -c 'from app.config import load; print(load().database.absolute())') \ || die "could not read the database path from the app's config; set LOCAL_DB in deploy/release.env" fi [[ -f $local_db ]] || die "no $local_db" snap=$root/$name-$(date +%Y%m%d-%H%M%S).db.gz trap 'rm -f -- "$snap"' EXIT echo "snapshotting $local_db" # One Python call for both halves: `VACUUM INTO` is the only way to get a # consistent single file out of a WAL database, and it writes the gzip the # server's `gunzip` expects. run_or_die snapshot "$python" -c ' import gzip, pathlib, shutil, sqlite3, sys, tempfile source, target = sys.argv[1], pathlib.Path(sys.argv[2]) with tempfile.TemporaryDirectory() as work: plain = pathlib.Path(work) / "snapshot.db" db = sqlite3.connect(source) try: db.execute("VACUUM INTO ?", (str(plain),)) finally: db.close() with plain.open("rb") as raw, gzip.open(target, "wb") as out: shutil.copyfileobj(raw, out) ' "$local_db" "$snap" printf ' %s\n' "$(du -h "$snap" | cut -f1)" echo echo "This replaces the database on $ssh_host with this one." echo 'Anything written there since is gone.' echo 'The app must already be stopped there, and stays stopped after.' if ((! yes)); then read -r -p 'go ahead? [y/N] ' reply || reply='' [[ $reply =~ ^[yY] ]] || { echo 'stopped.'; exit 0; } fi run_or_die 'scp database' scp -C "$snap" "$ssh_host:$remote_data/incoming.db.gz" run_or_die 'scp importer' scp "$here/server/import_database.py" "$ssh_host:$remote_dir/import_database.py" rm -f -- "$snap" # import.sh takes its directory as an argument, so it can be streamed. run_or_die import ssh "$ssh_host" "bash -s -- '$remote_dir'" < "$here/server/import.sh" echo 'pushed. The app is still stopped; start it when the rest of' echo 'the deployment is in place:' echo " ssh $ssh_host \"cd '$remote_dir' && docker compose up -d --wait $name\"" exit 0 fi # ---------------------------------------------------------------------- pull into=$root/backups mkdir -p -- "$into" # The server's own copy of backup.sh, not a streamed one: it is the same script # cron runs, and it locates its own directory through BASH_SOURCE, which a # script read from stdin does not have. echo "snapshotting on $ssh_host" run_or_die backup ssh "$ssh_host" "cd '$remote_dir' && bash backup.sh" # shellcheck disable=SC2029 # the remote command is meant to expand here newest=$(ssh "$ssh_host" "ls -1t '$remote_data'/backups/$name-*.db.gz | head -1") || die 'listing backups failed' [[ -n $newest ]] || die 'no backup appeared on the server' run_or_die scp scp -C "$ssh_host:$newest" "$into/" file=$(basename "$newest") echo "backups/$file" echo echo 'To read it here, gunzip it, or in Python:' echo " import gzip, shutil; shutil.copyfileobj(gzip.open('backups/$file'), open('read-only.db','wb'))"