Refresh deploy/ and bin/ from the boilerplate at 09e144d (the release split between the distro and Windows, wsl-setup, the server config, USER 1000, python:3.15-slim) with roledesk's facts put back: no frontend stage, desk, plugins and seed copied, the extras passkeys, embeddings and llm, roles as the table the backup and the importer require, bin/pull.sh and bin/worker.sh with their profile commands, and Install Server.md and Install WSL.md as copies in place of the five old deployment docs
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
83e965dfeb
commit
a1f281a23a
40 files changed
+2226
-1287
No files matched your search
+130
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env bash
|
||||
# The database, in both directions.
|
||||
#
|
||||
# bash deploy/db.sh push [--yes] this machine's database -> the server
|
||||
# bash deploy/db.sh pull a fresh snapshot from the server -> backups/
|
||||
#
|
||||
# `push` is the one-off at the start: production begins knowing what you already
|
||||
# have, rather than starting empty. `pull` is the one you keep running, on a
|
||||
# schedule from the server itself (server/backup.sh under cron) and by hand from
|
||||
# here whenever you want the copy somewhere the server is not.
|
||||
#
|
||||
# `push` moves the database and nothing else. It does not stop the app before or
|
||||
# start it after: a deployment knows what else is moving and in what order, and
|
||||
# this does not. Stop the app on the server first; the import is a file swap
|
||||
# under SQLite's feet and is refused while the app is up. Start it again
|
||||
# afterwards.
|
||||
#
|
||||
# Both go through SQLite's own `VACUUM INTO`. The database is in WAL mode, so the
|
||||
# .db file on disk is always missing whatever is still in the -wal beside it, and
|
||||
# copying the three files with the app running is a race you lose quietly: a
|
||||
# database that opens fine and is short a day's writes.
|
||||
#
|
||||
# After a push the two copies diverge from the first thing written on either.
|
||||
# There is no merge for that: pick the one that is authoritative, and use the
|
||||
# other read-only or not at all.
|
||||
#
|
||||
# Settings from release.env: SSH_HOST, REMOTE_DIR and NAME (the compose service)
|
||||
# are required; REMOTE_DATA (default REMOTE_DIR/data), PYTHON and LOCAL_DB
|
||||
# (default the database the app's config names) are not.
|
||||
set -euo pipefail
|
||||
# shellcheck source=_common.sh
|
||||
. "$(dirname "${BASH_SOURCE[0]}")/_common.sh"
|
||||
|
||||
action='' yes=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
push|pull) action=$arg ;;
|
||||
--yes) yes=1 ;;
|
||||
-h|--help) sed -n '2,29p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
*) die "unknown option $arg" ;;
|
||||
esac
|
||||
done
|
||||
[[ -n $action ]] || die "say push or pull: bash deploy/db.sh push|pull"
|
||||
|
||||
ssh_host=$(require_setting SSH_HOST)
|
||||
remote_dir=$(require_setting REMOTE_DIR)
|
||||
# Where the data actually is. Beside the compose file unless APP_DATA put it
|
||||
# under a shared root; then REMOTE_DATA has to say so.
|
||||
remote_data=$(setting REMOTE_DATA "$remote_dir/data")
|
||||
name=$(require_setting NAME)
|
||||
python=$(setting PYTHON "$root/.venv/bin/python")
|
||||
cd "$root"
|
||||
|
||||
# ---------------------------------------------------------------------- push
|
||||
|
||||
if [[ $action == push ]]; then
|
||||
[[ -x $python ]] || die "no interpreter at $python - run uv sync --all-extras, or set PYTHON in deploy/release.env"
|
||||
# The file the app writes: `database` in config.yaml (or $APP_CONFIG),
|
||||
# roledesk.db without one, asked of the app from the checkout's root.
|
||||
local_db=$(setting LOCAL_DB)
|
||||
if [[ -z $local_db ]]; then
|
||||
local_db=$("$python" -c 'from app.config import load; print(load().database.absolute())') \
|
||||
|| die "could not read the database path from the app's config; set LOCAL_DB in deploy/release.env"
|
||||
fi
|
||||
[[ -f $local_db ]] || die "no $local_db"
|
||||
|
||||
snap=$root/$name-$(date +%Y%m%d-%H%M%S).db.gz
|
||||
trap 'rm -f -- "$snap"' EXIT
|
||||
echo "snapshotting $local_db"
|
||||
|
||||
# One Python call for both halves: `VACUUM INTO` is the only way to get a
|
||||
# consistent single file out of a WAL database, and it writes the gzip the
|
||||
# server's `gunzip` expects.
|
||||
run_or_die snapshot "$python" -c '
|
||||
import gzip, pathlib, shutil, sqlite3, sys, tempfile
|
||||
source, target = sys.argv[1], pathlib.Path(sys.argv[2])
|
||||
with tempfile.TemporaryDirectory() as work:
|
||||
plain = pathlib.Path(work) / "snapshot.db"
|
||||
db = sqlite3.connect(source)
|
||||
try:
|
||||
db.execute("VACUUM INTO ?", (str(plain),))
|
||||
finally:
|
||||
db.close()
|
||||
with plain.open("rb") as raw, gzip.open(target, "wb") as out:
|
||||
shutil.copyfileobj(raw, out)
|
||||
' "$local_db" "$snap"
|
||||
printf ' %s\n' "$(du -h "$snap" | cut -f1)"
|
||||
|
||||
echo
|
||||
echo "This replaces the database on $ssh_host with this one."
|
||||
echo 'Anything written there since is gone.'
|
||||
echo 'The app must already be stopped there, and stays stopped after.'
|
||||
if ((! yes)); then
|
||||
read -r -p 'go ahead? [y/N] ' reply || reply=''
|
||||
[[ $reply =~ ^[yY] ]] || { echo 'stopped.'; exit 0; }
|
||||
fi
|
||||
|
||||
run_or_die 'scp database' scp -C "$snap" "$ssh_host:$remote_data/incoming.db.gz"
|
||||
run_or_die 'scp importer' scp "$here/server/import_database.py" "$ssh_host:$remote_dir/import_database.py"
|
||||
rm -f -- "$snap"
|
||||
|
||||
# import.sh takes its directory as an argument, so it can be streamed.
|
||||
run_or_die import ssh "$ssh_host" "bash -s -- '$remote_dir'" < "$here/server/import.sh"
|
||||
echo 'pushed. The app is still stopped; start it when the rest of'
|
||||
echo 'the deployment is in place:'
|
||||
echo " ssh $ssh_host \"cd '$remote_dir' && docker compose up -d --wait $name\""
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------- pull
|
||||
|
||||
into=$root/backups
|
||||
mkdir -p -- "$into"
|
||||
|
||||
# The server's own copy of backup.sh, not a streamed one: it is the same script
|
||||
# cron runs, and it locates its own directory through BASH_SOURCE, which a
|
||||
# script read from stdin does not have.
|
||||
echo "snapshotting on $ssh_host"
|
||||
run_or_die backup ssh "$ssh_host" "cd '$remote_dir' && bash backup.sh"
|
||||
|
||||
# shellcheck disable=SC2029 # the remote command is meant to expand here
|
||||
newest=$(ssh "$ssh_host" "ls -1t '$remote_data'/backups/$name-*.db.gz | head -1") || die 'listing backups failed'
|
||||
[[ -n $newest ]] || die 'no backup appeared on the server'
|
||||
|
||||
run_or_die scp scp -C "$ssh_host:$newest" "$into/"
|
||||
file=$(basename "$newest")
|
||||
echo "backups/$file"
|
||||
echo
|
||||
echo 'To read it here, gunzip it, or in Python:'
|
||||
echo " import gzip, shutil; shutil.copyfileobj(gzip.open('backups/$file'), open('read-only.db','wb'))"
|
||||
Reference in new issue
Block a user