Refresh deploy/ and bin/ from the boilerplate at 09e144d (the release split between the distro and Windows, wsl-setup, the server config, USER 1000, python:3.15-slim) with roledesk's facts put back: no frontend stage, desk, plugins and seed copied, the extras passkeys, embeddings and llm, roles as the table the backup and the importer require, bin/pull.sh and bin/worker.sh with their profile commands, and Install Server.md and Install WSL.md as copies in place of the five old deployment docs
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
83e965dfeb
commit
a1f281a23a
40 files changed
+2226
-1287
No files matched your search
+43
-10
@@ -13,9 +13,26 @@
|
||||
# self-contained and compacted file, and never blocks a writer for long enough
|
||||
# to matter on a database of this size.
|
||||
#
|
||||
# Rename `app` and `APP_DATA` to your service's name.
|
||||
# The database is the one the app's own config names, asked of the app inside
|
||||
# the container, and it is opened read-only, so a path that names no file is
|
||||
# an error rather than a new empty database and a snapshot of nothing.
|
||||
#
|
||||
# SERVICE is the compose service (deploy.sh says why it is here); APP_DATA is
|
||||
# renamed with the other APP_ variables, and `from app.config` with the package.
|
||||
|
||||
SERVICE=app
|
||||
# Tables every database this app writes has: the same set as REQUIRED_TABLES in
|
||||
# import_database.py, which a test holds this line to. A snapshot without one
|
||||
# of them is deleted and the run fails.
|
||||
# Ours: `roles` is app/storage/schema.py's first table, where the boilerplate has `ticks`.
|
||||
TABLES="jobs roles"
|
||||
|
||||
set -euo pipefail
|
||||
# A snapshot is the whole database, so it is owner only whatever the caller's
|
||||
# umask is. This covers the folder made below; the umask inside the container
|
||||
# (set in the snippet) covers the .db, and gzip gives the .gz the mode of the
|
||||
# file it compresses.
|
||||
umask 077
|
||||
cd "${1:-$(dirname "$(readlink -f "$0")")}"
|
||||
|
||||
# Same source of truth as the compose file and deploy.sh.
|
||||
@@ -27,18 +44,34 @@ stamp=$(date +%Y%m%d-%H%M%S)
|
||||
|
||||
mkdir -p "$DATA/backups"
|
||||
|
||||
docker compose exec -T app python -c "
|
||||
import sqlite3
|
||||
db = sqlite3.connect('/data/app.db')
|
||||
db.execute('VACUUM INTO ?', ('/data/backups/app-$stamp.db',))
|
||||
db.close()
|
||||
"
|
||||
gzip -f "$DATA/backups/app-$stamp.db"
|
||||
echo "$DATA/backups/app-$stamp.db.gz $(du -h "$DATA/backups/app-$stamp.db.gz" | cut -f1)"
|
||||
# shellcheck disable=SC2086 # TABLES is a list of words on purpose
|
||||
docker compose exec -T "$SERVICE" python - "$SERVICE-$stamp" $TABLES <<'PY'
|
||||
import os, sqlite3, sys
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
|
||||
from app.config import load
|
||||
|
||||
os.umask(0o077)
|
||||
name, required = sys.argv[1], set(sys.argv[2:])
|
||||
database = load().database.absolute()
|
||||
snapshot = Path("/data/backups") / f"{name}.db"
|
||||
with closing(sqlite3.connect(f"{database.as_uri()}?mode=ro", uri=True)) as db:
|
||||
db.execute("VACUUM INTO ?", (str(snapshot),))
|
||||
with closing(sqlite3.connect(f"{snapshot.as_uri()}?mode=ro", uri=True)) as db:
|
||||
tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||||
missing = sorted(required - tables)
|
||||
if missing:
|
||||
snapshot.unlink()
|
||||
sys.exit(f"{database} has no table {', '.join(missing)}: not this app's database")
|
||||
print(f"snapshot of {database}")
|
||||
PY
|
||||
gzip -f "$DATA/backups/$SERVICE-$stamp.db"
|
||||
echo "$DATA/backups/$SERVICE-$stamp.db.gz $(du -h "$DATA/backups/$SERVICE-$stamp.db.gz" | cut -f1)"
|
||||
|
||||
# Oldest first, everything past KEEP goes. Named by timestamp, so sorting the
|
||||
# names sorts them by age.
|
||||
ls -1 "$DATA"/backups/app-*.db.gz 2>/dev/null \
|
||||
ls -1 "$DATA"/backups/"$SERVICE"-*.db.gz 2>/dev/null \
|
||||
| sort \
|
||||
| head -n -"$KEEP" \
|
||||
| while read -r old; do rm -f "$old"; echo "removed $old"; done
|
||||
@@ -9,7 +9,10 @@
|
||||
# `env.example` to the `.env` next to it, and it is `docker compose up -d app`
|
||||
# from then on — which recreates this container and no other.
|
||||
#
|
||||
# Rename `app` and `APP_` throughout to your service's name.
|
||||
# Rename `app` and `APP_` throughout to your service's name. The service and
|
||||
# `container_name` are the name `SERVICE=` in deploy.sh, backup.sh and
|
||||
# import.sh, `$upstream_app` in the proxy conf and `NAME=` in release.env
|
||||
# repeat; tests/test_deploy.py reads it from this file and holds them to it.
|
||||
|
||||
services:
|
||||
app:
|
||||
@@ -33,12 +36,12 @@ services:
|
||||
# Which model answers, decided here rather than in the image. With
|
||||
# `backend: auto` the API key wins when there is one and the
|
||||
# subscription CLI is the fallback, so filling in ANTHROPIC_API_KEY
|
||||
# later switches it over with no rebuild. See docs/llm-backends.md.
|
||||
# later switches it over with no rebuild. See llm/README.md.
|
||||
CLAUDE_CODE_OAUTH_TOKEN: ${CLAUDE_CODE_OAUTH_TOKEN:-}
|
||||
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
||||
# Normally empty. Set it to pin the password — the way back in when a
|
||||
# login is lost — then unset it and recreate to hand control of the
|
||||
# password back to the app. See docs/password-protection.md.
|
||||
# password back to the app. See auth/README.md.
|
||||
APP_PASSWORD: ${APP_PASSWORD:-}
|
||||
|
||||
# No `ports:`. Nothing on the host reaches it; SWAG does, by container
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
# The server's config.yaml. Copy it to $APP_DATA/config.yaml, which the
|
||||
# container reads as /data/config.yaml (APP_CONFIG in the Dockerfile), and set
|
||||
# the hostname. deploy.sh refuses a config without `database` or `auth.enabled`.
|
||||
# Every key left out keeps its default; ../../config.example.yaml lists them all.
|
||||
# Ours: the keys are roledesk's (app/config.py); an unknown one is an error.
|
||||
|
||||
server:
|
||||
host: 0.0.0.0 # the container's interface; SWAG is in front of it
|
||||
port: 8787 # the port the proxy conf and the HEALTHCHECK use
|
||||
|
||||
database: /data/roledesk.db # in the volume; /app, the working directory, is not writable
|
||||
|
||||
auth:
|
||||
enabled: always # the lock must not depend on how the process was started
|
||||
secure_cookie: always # auto needs uvicorn to trust X-Forwarded-Proto, which only `app serve` off loopback sets
|
||||
rp_id: app.example.com # the hostname passkeys are bound to; changing it voids every passkey
|
||||
origin: https://app.example.com # what the browser's address bar shows, scheme included
|
||||
|
||||
# Who answers when a stage needs a model (extract, summarise, judge). `auto`
|
||||
# takes ANTHROPIC_API_KEY when the compose file passes one and otherwise the
|
||||
# subscription CLI with CLAUDE_CODE_OAUTH_TOKEN, so switching is an .env edit.
|
||||
# The stages themselves are switched under Settings, Sources, Runs.
|
||||
agent:
|
||||
backend: auto
|
||||
|
||||
# The vectors behind prompt and CV closeness. The model is not in the image:
|
||||
# the first `embed` job downloads it (~460 MB) into HOME, the /data volume, so
|
||||
# it happens once per server and not once per release. Changing it re-embeds.
|
||||
embed:
|
||||
model: intfloat/multilingual-e5-small
|
||||
+27
-10
@@ -1,9 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run on the server, next to the compose file, or with its directory as $1.
|
||||
# No remote shell required: the release scripts on Windows publish an image
|
||||
# No remote shell required: the release scripts publish an image
|
||||
# and stop there, and this is the other half.
|
||||
#
|
||||
# Rename `app` and `APP_DATA` to your service's name.
|
||||
# SERVICE is the compose service, the one name a project renames in these
|
||||
# scripts (README, "Starting a project from it"); the tests read it from
|
||||
# compose.yaml and hold every script to it. APP_DATA is renamed with the other
|
||||
# APP_ variables.
|
||||
SERVICE=app
|
||||
|
||||
set -euo pipefail
|
||||
cd "${1:-$(dirname "$(readlink -f "$0")")}"
|
||||
|
||||
@@ -15,24 +20,36 @@ cd "${1:-$(dirname "$(readlink -f "$0")")}"
|
||||
DATA=$(sed -n 's/^[[:space:]]*APP_DATA=\([^#]*\).*/\1/p' .env | tail -1 | xargs || true)
|
||||
DATA=${DATA:-./data}
|
||||
|
||||
[[ -s "$DATA/config.yaml" ]] || {
|
||||
echo "Install $DATA/config.yaml before deploying." >&2
|
||||
config=$DATA/config.yaml
|
||||
[[ -s $config ]] || {
|
||||
echo "Install $config before deploying, from config.example.yaml beside this script." >&2
|
||||
exit 1
|
||||
}
|
||||
# The two keys whose defaults are wrong in a container: `auth.enabled: auto`
|
||||
# ties the lock to how the process was started, and the default
|
||||
# database is wherever the config file happens to be. Read with grep and awk,
|
||||
# because the host need not have a Python with a YAML parser.
|
||||
if ! grep -Eq '^database:[[:space:]]*[^[:space:]#]' "$config" ||
|
||||
! awk '/^auth:/ { a = 1; next } /^[^[:space:]#]/ { a = 0 }
|
||||
a && /^[[:space:]]+enabled:[[:space:]]*[^[:space:]#]/ { found = 1 }
|
||||
END { exit !found }' "$config"; then
|
||||
echo "$config must set database and auth.enabled; config.example.yaml beside this script has both." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Pull explicitly, here and nowhere else: the compose file has no pull policy,
|
||||
# so starting or importing never fetches an image on its own. A registry
|
||||
# failure must leave the running service alone, not deploy a cache.
|
||||
echo 'Pulling the configured image…'
|
||||
if ! docker compose pull app; then
|
||||
echo 'Image pull failed; app was not restarted.' >&2
|
||||
if ! docker compose pull "$SERVICE"; then
|
||||
echo "Image pull failed; $SERVICE was not restarted." >&2
|
||||
exit 1
|
||||
fi
|
||||
# Use the image just pulled. Do not start or recreate other services in the
|
||||
# file. `--wait` holds until the container's own HEALTHCHECK passes.
|
||||
if ! docker compose up -d --no-deps --pull never --wait --wait-timeout 240 app; then
|
||||
docker compose ps app >&2 || true
|
||||
echo 'Deployment failed. Check docker compose logs app.' >&2
|
||||
if ! docker compose up -d --no-deps --pull never --wait --wait-timeout 240 "$SERVICE"; then
|
||||
docker compose ps "$SERVICE" >&2 || true
|
||||
echo "Deployment failed. Check docker compose logs $SERVICE." >&2
|
||||
exit 1
|
||||
fi
|
||||
docker compose ps app
|
||||
docker compose ps "$SERVICE"
|
||||
@@ -4,7 +4,7 @@
|
||||
#
|
||||
# chmod 600. Rename APP_ to your service's name throughout.
|
||||
|
||||
APP_IMAGE=git.rittershofer.org/wolf/app
|
||||
APP_IMAGE=registry.example/you/app
|
||||
APP_TAG=latest # follows published images; pin a commit tag for rollback
|
||||
|
||||
# Where the app keeps its database, its config and its backups. Relative to
|
||||
@@ -20,7 +20,7 @@ PGID=1000
|
||||
TZ=Europe/Berlin
|
||||
|
||||
# Which model answers. Set one or the other, not both — see
|
||||
# docs/llm-backends.md for what each one means and what it costs.
|
||||
# llm/README.md for what each one means and what it costs.
|
||||
#
|
||||
# CLAUDE_CODE_OAUTH_TOKEN a year-long token for the Claude subscription, from
|
||||
# `claude setup-token` on a machine you are signed in
|
||||
@@ -34,7 +34,7 @@ ANTHROPIC_API_KEY=
|
||||
# `docker compose logs app`; after that it is the app's Account screen's to
|
||||
# change. Fill this in only to force one — it is the way back in — then empty
|
||||
# it again and recreate, or the Account screen refuses to change what the
|
||||
# environment pins. See docs/password-protection.md.
|
||||
# environment pins. See auth/README.md.
|
||||
APP_PASSWORD=
|
||||
|
||||
# Unlimited until the box has been watched through the heaviest thing the app
|
||||
|
||||
@@ -1,18 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
# Called on the server after uploading incoming.db.gz and import_database.py.
|
||||
# `db.ps1 push` does both and then runs this; by hand it is the same two files
|
||||
# `db.sh push` does both and then runs this; by hand it is the same two files
|
||||
# in the same two places.
|
||||
#
|
||||
# This replaces the database and does nothing else. It does not stop the app
|
||||
# and it does not start it: whoever is running a deployment knows what else is
|
||||
# moving and in what order, and a tool that helpfully restarts things underneath
|
||||
# that is a tool fighting its caller. By hand it is
|
||||
# `docker compose stop app` and `docker compose up -d --wait app` around this.
|
||||
# `docker compose stop app` and `docker compose up -d --wait app` around this,
|
||||
# with SERVICE (deploy.sh says why it is here) in place of app.
|
||||
#
|
||||
# What it will not do is import into a running app. The replacement is a file
|
||||
# swap under SQLite's feet, and the live database is checkpointed first, which
|
||||
# needs every writer gone — `import_database.py` refuses a busy database, and
|
||||
# this refuses an app that is up, because the clearer error is the earlier one.
|
||||
SERVICE=app
|
||||
|
||||
set -euo pipefail
|
||||
cd "${1:?server directory required}"
|
||||
|
||||
@@ -22,14 +25,14 @@ DATA=${DATA:-./data}
|
||||
|
||||
[[ -s "$DATA/incoming.db.gz" && -s import_database.py ]]
|
||||
|
||||
if [[ -n "$(docker compose ps -q --status running app 2>/dev/null)" ]]; then
|
||||
echo 'app is running. Stop it before importing:' >&2
|
||||
echo ' docker compose stop app' >&2
|
||||
if [[ -n "$(docker compose ps -q --status running "$SERVICE" 2>/dev/null)" ]]; then
|
||||
echo "$SERVICE is running. Stop it before importing:" >&2
|
||||
echo " docker compose stop $SERVICE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The one-off container uses compose's uid and its mounted /data. `--pull
|
||||
# never`: an import runs on the image already here, whatever the installed
|
||||
# compose file says about pulling. Getting a new image is deploy.sh's job.
|
||||
docker compose run --rm --no-deps --pull never -T --entrypoint python app - < import_database.py
|
||||
docker compose run --rm --no-deps --pull never -T --entrypoint python "$SERVICE" - < import_database.py
|
||||
rm -f "$DATA/incoming.db.gz"
|
||||
@@ -1,11 +1,13 @@
|
||||
"""Offline database replacement, run as the compose service's uid.
|
||||
|
||||
The app container must be stopped. Only stdlib is used so this can be sent
|
||||
over stdin to either the old or the new image during a deployment.
|
||||
The app container must be stopped. Sent over stdin to the image on the
|
||||
server, whose working directory is the app's: the database is the one the
|
||||
app's config names (`from app.config` follows the package when it is renamed),
|
||||
and the rest is stdlib, so the old image and the new one both run it.
|
||||
|
||||
Two things to edit for your service: `DATABASE` and `REQUIRED_TABLES`. The
|
||||
table check is what stops a right-shaped-but-wrong database — last project's,
|
||||
or a half-downloaded one — from landing on top of a good one.
|
||||
`REQUIRED_TABLES` is the table check, which stops a right-shaped-but-wrong
|
||||
database (last project's, or a half-downloaded one) from landing on top of a
|
||||
good one. `tests/test_deploy.py` fails when the app stops creating one of them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -19,13 +21,14 @@ from contextlib import closing
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
DATABASE = Path("/data/app.db")
|
||||
INCOMING = Path("/data/incoming.db.gz")
|
||||
BACKUPS = Path("/data/backups")
|
||||
|
||||
#: Any one of these missing means this is not our database. Name tables that
|
||||
#: have existed since the first schema, not ones a recent migration added.
|
||||
REQUIRED_TABLES = {"items"}
|
||||
#: have existed since the first schema, not ones a recent migration added:
|
||||
#: `jobs` is the queue's and `roles` is `app/storage/schema.py`'s first table
|
||||
#: (the boilerplate has `ticks`). backup.sh's TABLES is the same set.
|
||||
REQUIRED_TABLES = {"jobs", "roles"}
|
||||
|
||||
|
||||
def replace_database(database: Path, archive: Path, backups: Path) -> Path | None:
|
||||
@@ -73,5 +76,10 @@ def replace_database(database: Path, archive: Path, backups: Path) -> Path | Non
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
saved = replace_database(DATABASE, INCOMING, BACKUPS)
|
||||
from app.config import load
|
||||
|
||||
# The snapshot is the whole database, so it is owner only whatever the
|
||||
# container's umask is; the gzip of it is made under the same mask.
|
||||
os.umask(0o077)
|
||||
saved = replace_database(load().database.absolute(), INCOMING, BACKUPS)
|
||||
print(f"Database replaced; previous snapshot: {saved or 'none (first import)'}")
|
||||
@@ -3,7 +3,7 @@
|
||||
##
|
||||
## The subdomain has to exist before this does anything: add <app> to SWAG's
|
||||
## SUBDOMAINS (or run it with a wildcard cert) and point the DNS record at the
|
||||
## same address as the rest of the stack. See docs/hosting.md.
|
||||
## same address as the rest of the stack. See docs/Install Server.md.
|
||||
|
||||
## Per-IP throttle for the login endpoints, if the app has a password. Declared
|
||||
## here at `http` scope — SWAG includes proxy-confs outside any server block,
|
||||
@@ -15,23 +15,30 @@ server {
|
||||
listen 443 ssl;
|
||||
listen [::]:443 ssl;
|
||||
|
||||
server_name app.rittershofer.org;
|
||||
server_name app.example.com;
|
||||
|
||||
include /config/nginx/ssl.conf;
|
||||
# Put proxy defaults at server scope so location timeouts override them
|
||||
# through inheritance, rather than duplicate directives from proxy.conf.
|
||||
include /config/nginx/proxy.conf;
|
||||
|
||||
# 0 is no limit. Set a real one if strangers can post to this app; leave it
|
||||
# off when the only uploads are yours and can be large.
|
||||
client_max_body_size 0;
|
||||
# nginx buffers a request body to disk before it hands it on, and the login
|
||||
# takes one without a session, so no limit here lets anyone spool gigabytes.
|
||||
# This is small. A route that takes a large upload gets its own
|
||||
# `location = /path { client_max_body_size 0; proxy_request_buffering off;
|
||||
# ... }` above `location /`, with the same proxy lines as below; streaming
|
||||
# the body lets the app refuse a request without a session before nginx
|
||||
# has stored any of it.
|
||||
client_max_body_size 1m;
|
||||
|
||||
# Guessing the password is already pointless if the app locks out after a
|
||||
# handful of wrong answers (see docs/password-protection.md). This is the
|
||||
# handful of wrong answers (see auth/README.md). This is the
|
||||
# cheap half of the rest: attempts are slowed per address, at the proxy,
|
||||
# before they reach the app at all. `burst=5 nodelay` lets a person who
|
||||
# fat-fingers it a few times through immediately and then holds the rate.
|
||||
location ~ ^/login(/passkey)?$ {
|
||||
# `/login/passkey/options` is open and writes a challenge row per call, so
|
||||
# it is throttled with the two routes that check a credential.
|
||||
location ~ ^/login(/passkey(/options)?)?$ {
|
||||
limit_req zone=app_login burst=5 nodelay;
|
||||
limit_req_status 429;
|
||||
|
||||
|
||||
Reference in new issue
Block a user