131 lines
5.6 KiB
Bash
131 lines
5.6 KiB
Bash
#!/usr/bin/env bash
|
|
# The database, in both directions.
|
|
#
|
|
# bash deploy/db.sh push [--yes] this machine's database -> the server
|
|
# bash deploy/db.sh pull a fresh snapshot from the server -> backups/
|
|
#
|
|
# `push` is the one-off at the start: production begins knowing what you already
|
|
# have, rather than starting empty. `pull` is the one you keep running, on a
|
|
# schedule from the server itself (server/backup.sh under cron) and by hand from
|
|
# here whenever you want the copy somewhere the server is not.
|
|
#
|
|
# `push` moves the database and nothing else. It does not stop the app before or
|
|
# start it after: a deployment knows what else is moving and in what order, and
|
|
# this does not. Stop the app on the server first; the import is a file swap
|
|
# under SQLite's feet and is refused while the app is up. Start it again
|
|
# afterwards.
|
|
#
|
|
# Both go through SQLite's own `VACUUM INTO`. The database is in WAL mode, so the
|
|
# .db file on disk is always missing whatever is still in the -wal beside it, and
|
|
# copying the three files with the app running is a race you lose quietly: a
|
|
# database that opens fine and is short a day's writes.
|
|
#
|
|
# After a push the two copies diverge from the first thing written on either.
|
|
# There is no merge for that: pick the one that is authoritative, and use the
|
|
# other read-only or not at all.
|
|
#
|
|
# Settings from release.env: SSH_HOST, REMOTE_DIR and NAME (the compose service)
|
|
# are required; REMOTE_DATA (default REMOTE_DIR/data), PYTHON and LOCAL_DB
|
|
# (default the database the app's config names) are not.
|
|
set -euo pipefail
|
|
# shellcheck source=_common.sh
|
|
. "$(dirname "${BASH_SOURCE[0]}")/_common.sh"
|
|
|
|
action='' yes=0
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
push|pull) action=$arg ;;
|
|
--yes) yes=1 ;;
|
|
-h|--help) sed -n '2,29p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
|
*) die "unknown option $arg" ;;
|
|
esac
|
|
done
|
|
[[ -n $action ]] || die "say push or pull: bash deploy/db.sh push|pull"
|
|
|
|
ssh_host=$(require_setting SSH_HOST)
|
|
remote_dir=$(require_setting REMOTE_DIR)
|
|
# Where the data actually is. Beside the compose file unless APP_DATA put it
|
|
# under a shared root; then REMOTE_DATA has to say so.
|
|
remote_data=$(setting REMOTE_DATA "$remote_dir/data")
|
|
name=$(require_setting NAME)
|
|
python=$(setting PYTHON "$root/.venv/bin/python")
|
|
cd "$root"
|
|
|
|
# ---------------------------------------------------------------------- push
|
|
|
|
if [[ $action == push ]]; then
|
|
[[ -x $python ]] || die "no interpreter at $python - run uv sync --all-extras, or set PYTHON in deploy/release.env"
|
|
# The file the app writes: `database` in config.yaml (or $APP_CONFIG),
|
|
# roledesk.db without one, asked of the app from the checkout's root.
|
|
local_db=$(setting LOCAL_DB)
|
|
if [[ -z $local_db ]]; then
|
|
local_db=$("$python" -c 'from app.config import load; print(load().database.absolute())') \
|
|
|| die "could not read the database path from the app's config; set LOCAL_DB in deploy/release.env"
|
|
fi
|
|
[[ -f $local_db ]] || die "no $local_db"
|
|
|
|
snap=$root/$name-$(date +%Y%m%d-%H%M%S).db.gz
|
|
trap 'rm -f -- "$snap"' EXIT
|
|
echo "snapshotting $local_db"
|
|
|
|
# One Python call for both halves: `VACUUM INTO` is the only way to get a
|
|
# consistent single file out of a WAL database, and it writes the gzip the
|
|
# server's `gunzip` expects.
|
|
run_or_die snapshot "$python" -c '
|
|
import gzip, pathlib, shutil, sqlite3, sys, tempfile
|
|
source, target = sys.argv[1], pathlib.Path(sys.argv[2])
|
|
with tempfile.TemporaryDirectory() as work:
|
|
plain = pathlib.Path(work) / "snapshot.db"
|
|
db = sqlite3.connect(source)
|
|
try:
|
|
db.execute("VACUUM INTO ?", (str(plain),))
|
|
finally:
|
|
db.close()
|
|
with plain.open("rb") as raw, gzip.open(target, "wb") as out:
|
|
shutil.copyfileobj(raw, out)
|
|
' "$local_db" "$snap"
|
|
printf ' %s\n' "$(du -h "$snap" | cut -f1)"
|
|
|
|
echo
|
|
echo "This replaces the database on $ssh_host with this one."
|
|
echo 'Anything written there since is gone.'
|
|
echo 'The app must already be stopped there, and stays stopped after.'
|
|
if ((! yes)); then
|
|
read -r -p 'go ahead? [y/N] ' reply || reply=''
|
|
[[ $reply =~ ^[yY] ]] || { echo 'stopped.'; exit 0; }
|
|
fi
|
|
|
|
run_or_die 'scp database' scp -C "$snap" "$ssh_host:$remote_data/incoming.db.gz"
|
|
run_or_die 'scp importer' scp "$here/server/import_database.py" "$ssh_host:$remote_dir/import_database.py"
|
|
rm -f -- "$snap"
|
|
|
|
# import.sh takes its directory as an argument, so it can be streamed.
|
|
run_or_die import ssh "$ssh_host" "bash -s -- '$remote_dir'" < "$here/server/import.sh"
|
|
echo 'pushed. The app is still stopped; start it when the rest of'
|
|
echo 'the deployment is in place:'
|
|
echo " ssh $ssh_host \"cd '$remote_dir' && docker compose up -d --wait $name\""
|
|
exit 0
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------- pull
|
|
|
|
into=$root/backups
|
|
mkdir -p -- "$into"
|
|
|
|
# The server's own copy of backup.sh, not a streamed one: it is the same script
|
|
# cron runs, and it locates its own directory through BASH_SOURCE, which a
|
|
# script read from stdin does not have.
|
|
echo "snapshotting on $ssh_host"
|
|
run_or_die backup ssh "$ssh_host" "cd '$remote_dir' && bash backup.sh"
|
|
|
|
# shellcheck disable=SC2029 # the remote command is meant to expand here
|
|
newest=$(ssh "$ssh_host" "ls -1t '$remote_data'/backups/$name-*.db.gz | head -1") || die 'listing backups failed'
|
|
[[ -n $newest ]] || die 'no backup appeared on the server'
|
|
|
|
run_or_die scp scp -C "$ssh_host:$newest" "$into/"
|
|
file=$(basename "$newest")
|
|
echo "backups/$file"
|
|
echo
|
|
echo 'To read it here, gunzip it, or in Python:'
|
|
echo " import gzip, shutil; shutil.copyfileobj(gzip.open('backups/$file'), open('read-only.db','wb'))"
|