Refresh deploy/ and bin/ from the boilerplate at 09e144d (the release split between the distro and Windows, wsl-setup, the server config, USER 1000, python:3.15-slim) with roledesk's facts put back: no frontend stage, desk, plugins and seed copied, the extras passkeys, embeddings and llm, roles as the table the backup and the importer require, bin/pull.sh and bin/worker.sh with their profile commands, and Install Server.md and Install WSL.md as copies in place of the five old deployment docs

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-10-27 11:11:19 +01:00
1 parent 83e965dfeb
commit a1f281a23a
40 files changed
+2226 -1287

No files matched your search

+43 -10
View File
@@ -13,9 +13,26 @@
# self-contained and compacted file, and never blocks a writer for long enough
# to matter on a database of this size.
#
# Rename `app` and `APP_DATA` to your service's name.
# The database is the one the app's own config names, asked of the app inside
# the container, and it is opened read-only, so a path that names no file is
# an error rather than a new empty database and a snapshot of nothing.
#
# SERVICE is the compose service (deploy.sh says why it is here); APP_DATA is
# renamed with the other APP_ variables, and `from app.config` with the package.
SERVICE=app
# Tables every database this app writes has: the same set as REQUIRED_TABLES in
# import_database.py, which a test holds this line to. A snapshot without one
# of them is deleted and the run fails.
# Ours: `roles` is app/storage/schema.py's first table, where the boilerplate has `ticks`.
TABLES="jobs roles"
set -euo pipefail
# A snapshot is the whole database, so it is owner only whatever the caller's
# umask is. This covers the folder made below; the umask inside the container
# (set in the snippet) covers the .db, and gzip gives the .gz the mode of the
# file it compresses.
umask 077
cd "${1:-$(dirname "$(readlink -f "$0")")}"
# Same source of truth as the compose file and deploy.sh.
@@ -27,18 +44,34 @@ stamp=$(date +%Y%m%d-%H%M%S)
mkdir -p "$DATA/backups"
docker compose exec -T app python -c "
import sqlite3
db = sqlite3.connect('/data/app.db')
db.execute('VACUUM INTO ?', ('/data/backups/app-$stamp.db',))
db.close()
"
gzip -f "$DATA/backups/app-$stamp.db"
echo "$DATA/backups/app-$stamp.db.gz $(du -h "$DATA/backups/app-$stamp.db.gz" | cut -f1)"
# shellcheck disable=SC2086 # TABLES is a list of words on purpose
docker compose exec -T "$SERVICE" python - "$SERVICE-$stamp" $TABLES <<'PY'
import os, sqlite3, sys
from contextlib import closing
from pathlib import Path
from app.config import load
os.umask(0o077)
name, required = sys.argv[1], set(sys.argv[2:])
database = load().database.absolute()
snapshot = Path("/data/backups") / f"{name}.db"
with closing(sqlite3.connect(f"{database.as_uri()}?mode=ro", uri=True)) as db:
db.execute("VACUUM INTO ?", (str(snapshot),))
with closing(sqlite3.connect(f"{snapshot.as_uri()}?mode=ro", uri=True)) as db:
tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
missing = sorted(required - tables)
if missing:
snapshot.unlink()
sys.exit(f"{database} has no table {', '.join(missing)}: not this app's database")
print(f"snapshot of {database}")
PY
gzip -f "$DATA/backups/$SERVICE-$stamp.db"
echo "$DATA/backups/$SERVICE-$stamp.db.gz $(du -h "$DATA/backups/$SERVICE-$stamp.db.gz" | cut -f1)"
# Oldest first, everything past KEEP goes. Named by timestamp, so sorting the
# names sorts them by age.
ls -1 "$DATA"/backups/app-*.db.gz 2>/dev/null \
ls -1 "$DATA"/backups/"$SERVICE"-*.db.gz 2>/dev/null \
| sort \
| head -n -"$KEEP" \
| while read -r old; do rm -f "$old"; echo "removed $old"; done
+6 -3
View File
@@ -9,7 +9,10 @@
# `env.example` to the `.env` next to it, and it is `docker compose up -d app`
# from then on — which recreates this container and no other.
#
# Rename `app` and `APP_` throughout to your service's name.
# Rename `app` and `APP_` throughout to your service's name. The service and
# `container_name` are the name `SERVICE=` in deploy.sh, backup.sh and
# import.sh, `$upstream_app` in the proxy conf and `NAME=` in release.env
# repeat; tests/test_deploy.py reads it from this file and holds them to it.
services:
app:
@@ -33,12 +36,12 @@ services:
# Which model answers, decided here rather than in the image. With
# `backend: auto` the API key wins when there is one and the
# subscription CLI is the fallback, so filling in ANTHROPIC_API_KEY
# later switches it over with no rebuild. See docs/llm-backends.md.
# later switches it over with no rebuild. See llm/README.md.
CLAUDE_CODE_OAUTH_TOKEN: ${CLAUDE_CODE_OAUTH_TOKEN:-}
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
# Normally empty. Set it to pin the password — the way back in when a
# login is lost — then unset it and recreate to hand control of the
# password back to the app. See docs/password-protection.md.
# password back to the app. See auth/README.md.
APP_PASSWORD: ${APP_PASSWORD:-}
# No `ports:`. Nothing on the host reaches it; SWAG does, by container
+30
View File
@@ -0,0 +1,30 @@
# The server's config.yaml. Copy it to $APP_DATA/config.yaml, which the
# container reads as /data/config.yaml (APP_CONFIG in the Dockerfile), and set
# the hostname. deploy.sh refuses a config without `database` or `auth.enabled`.
# Every key left out keeps its default; ../../config.example.yaml lists them all.
# Ours: the keys are roledesk's (app/config.py); an unknown one is an error.
server:
host: 0.0.0.0 # the container's interface; SWAG is in front of it
port: 8787 # the port the proxy conf and the HEALTHCHECK use
database: /data/roledesk.db # in the volume; /app, the working directory, is not writable
auth:
enabled: always # the lock must not depend on how the process was started
secure_cookie: always # auto needs uvicorn to trust X-Forwarded-Proto, which only `app serve` off loopback sets
rp_id: app.example.com # the hostname passkeys are bound to; changing it voids every passkey
origin: https://app.example.com # what the browser's address bar shows, scheme included
# Who answers when a stage needs a model (extract, summarise, judge). `auto`
# takes ANTHROPIC_API_KEY when the compose file passes one and otherwise the
# subscription CLI with CLAUDE_CODE_OAUTH_TOKEN, so switching is an .env edit.
# The stages themselves are switched under Settings, Sources, Runs.
agent:
backend: auto
# The vectors behind prompt and CV closeness. The model is not in the image:
# the first `embed` job downloads it (~460 MB) into HOME, the /data volume, so
# it happens once per server and not once per release. Changing it re-embeds.
embed:
model: intfloat/multilingual-e5-small
+27 -10
View File
@@ -1,9 +1,14 @@
#!/usr/bin/env bash
# Run on the server, next to the compose file, or with its directory as $1.
# No remote shell required: the release scripts on Windows publish an image
# No remote shell required: the release scripts publish an image
# and stop there, and this is the other half.
#
# Rename `app` and `APP_DATA` to your service's name.
# SERVICE is the compose service, the one name a project renames in these
# scripts (README, "Starting a project from it"); the tests read it from
# compose.yaml and hold every script to it. APP_DATA is renamed with the other
# APP_ variables.
SERVICE=app
set -euo pipefail
cd "${1:-$(dirname "$(readlink -f "$0")")}"
@@ -15,24 +20,36 @@ cd "${1:-$(dirname "$(readlink -f "$0")")}"
DATA=$(sed -n 's/^[[:space:]]*APP_DATA=\([^#]*\).*/\1/p' .env | tail -1 | xargs || true)
DATA=${DATA:-./data}
[[ -s "$DATA/config.yaml" ]] || {
echo "Install $DATA/config.yaml before deploying." >&2
config=$DATA/config.yaml
[[ -s $config ]] || {
echo "Install $config before deploying, from config.example.yaml beside this script." >&2
exit 1
}
# The two keys whose defaults are wrong in a container: `auth.enabled: auto`
# ties the lock to how the process was started, and the default
# database is wherever the config file happens to be. Read with grep and awk,
# because the host need not have a Python with a YAML parser.
if ! grep -Eq '^database:[[:space:]]*[^[:space:]#]' "$config" ||
! awk '/^auth:/ { a = 1; next } /^[^[:space:]#]/ { a = 0 }
a && /^[[:space:]]+enabled:[[:space:]]*[^[:space:]#]/ { found = 1 }
END { exit !found }' "$config"; then
echo "$config must set database and auth.enabled; config.example.yaml beside this script has both." >&2
exit 1
fi
# Pull explicitly, here and nowhere else: the compose file has no pull policy,
# so starting or importing never fetches an image on its own. A registry
# failure must leave the running service alone, not deploy a cache.
echo 'Pulling the configured image…'
if ! docker compose pull app; then
echo 'Image pull failed; app was not restarted.' >&2
if ! docker compose pull "$SERVICE"; then
echo "Image pull failed; $SERVICE was not restarted." >&2
exit 1
fi
# Use the image just pulled. Do not start or recreate other services in the
# file. `--wait` holds until the container's own HEALTHCHECK passes.
if ! docker compose up -d --no-deps --pull never --wait --wait-timeout 240 app; then
docker compose ps app >&2 || true
echo 'Deployment failed. Check docker compose logs app.' >&2
if ! docker compose up -d --no-deps --pull never --wait --wait-timeout 240 "$SERVICE"; then
docker compose ps "$SERVICE" >&2 || true
echo "Deployment failed. Check docker compose logs $SERVICE." >&2
exit 1
fi
docker compose ps app
docker compose ps "$SERVICE"
+3 -3
View File
@@ -4,7 +4,7 @@
#
# chmod 600. Rename APP_ to your service's name throughout.
APP_IMAGE=git.rittershofer.org/wolf/app
APP_IMAGE=registry.example/you/app
APP_TAG=latest # follows published images; pin a commit tag for rollback
# Where the app keeps its database, its config and its backups. Relative to
@@ -20,7 +20,7 @@ PGID=1000
TZ=Europe/Berlin
# Which model answers. Set one or the other, not both — see
# docs/llm-backends.md for what each one means and what it costs.
# llm/README.md for what each one means and what it costs.
#
# CLAUDE_CODE_OAUTH_TOKEN a year-long token for the Claude subscription, from
# `claude setup-token` on a machine you are signed in
@@ -34,7 +34,7 @@ ANTHROPIC_API_KEY=
# `docker compose logs app`; after that it is the app's Account screen's to
# change. Fill this in only to force one — it is the way back in — then empty
# it again and recreate, or the Account screen refuses to change what the
# environment pins. See docs/password-protection.md.
# environment pins. See auth/README.md.
APP_PASSWORD=
# Unlimited until the box has been watched through the heaviest thing the app
+9 -6
View File
@@ -1,18 +1,21 @@
#!/usr/bin/env bash
# Called on the server after uploading incoming.db.gz and import_database.py.
# `db.ps1 push` does both and then runs this; by hand it is the same two files
# `db.sh push` does both and then runs this; by hand it is the same two files
# in the same two places.
#
# This replaces the database and does nothing else. It does not stop the app
# and it does not start it: whoever is running a deployment knows what else is
# moving and in what order, and a tool that helpfully restarts things underneath
# that is a tool fighting its caller. By hand it is
# `docker compose stop app` and `docker compose up -d --wait app` around this.
# `docker compose stop app` and `docker compose up -d --wait app` around this,
# with SERVICE (deploy.sh says why it is here) in place of app.
#
# What it will not do is import into a running app. The replacement is a file
# swap under SQLite's feet, and the live database is checkpointed first, which
# needs every writer gone — `import_database.py` refuses a busy database, and
# this refuses an app that is up, because the clearer error is the earlier one.
SERVICE=app
set -euo pipefail
cd "${1:?server directory required}"
@@ -22,14 +25,14 @@ DATA=${DATA:-./data}
[[ -s "$DATA/incoming.db.gz" && -s import_database.py ]]
if [[ -n "$(docker compose ps -q --status running app 2>/dev/null)" ]]; then
echo 'app is running. Stop it before importing:' >&2
echo ' docker compose stop app' >&2
if [[ -n "$(docker compose ps -q --status running "$SERVICE" 2>/dev/null)" ]]; then
echo "$SERVICE is running. Stop it before importing:" >&2
echo " docker compose stop $SERVICE" >&2
exit 1
fi
# The one-off container uses compose's uid and its mounted /data. `--pull
# never`: an import runs on the image already here, whatever the installed
# compose file says about pulling. Getting a new image is deploy.sh's job.
docker compose run --rm --no-deps --pull never -T --entrypoint python app - < import_database.py
docker compose run --rm --no-deps --pull never -T --entrypoint python "$SERVICE" - < import_database.py
rm -f "$DATA/incoming.db.gz"
+17 -9
View File
@@ -1,11 +1,13 @@
"""Offline database replacement, run as the compose service's uid.
The app container must be stopped. Only stdlib is used so this can be sent
over stdin to either the old or the new image during a deployment.
The app container must be stopped. Sent over stdin to the image on the
server, whose working directory is the app's: the database is the one the
app's config names (`from app.config` follows the package when it is renamed),
and the rest is stdlib, so the old image and the new one both run it.
Two things to edit for your service: `DATABASE` and `REQUIRED_TABLES`. The
table check is what stops a right-shaped-but-wrong database — last project's,
or a half-downloaded one — from landing on top of a good one.
`REQUIRED_TABLES` is the table check, which stops a right-shaped-but-wrong
database (last project's, or a half-downloaded one) from landing on top of a
good one. `tests/test_deploy.py` fails when the app stops creating one of them.
"""
from __future__ import annotations
@@ -19,13 +21,14 @@ from contextlib import closing
from datetime import UTC, datetime
from pathlib import Path
DATABASE = Path("/data/app.db")
INCOMING = Path("/data/incoming.db.gz")
BACKUPS = Path("/data/backups")
#: Any one of these missing means this is not our database. Name tables that
#: have existed since the first schema, not ones a recent migration added.
REQUIRED_TABLES = {"items"}
#: have existed since the first schema, not ones a recent migration added:
#: `jobs` is the queue's and `roles` is `app/storage/schema.py`'s first table
#: (the boilerplate has `ticks`). backup.sh's TABLES is the same set.
REQUIRED_TABLES = {"jobs", "roles"}
def replace_database(database: Path, archive: Path, backups: Path) -> Path | None:
@@ -73,5 +76,10 @@ def replace_database(database: Path, archive: Path, backups: Path) -> Path | Non
if __name__ == "__main__":
saved = replace_database(DATABASE, INCOMING, BACKUPS)
from app.config import load
# The snapshot is the whole database, so it is owner only whatever the
# container's umask is; the gzip of it is made under the same mask.
os.umask(0o077)
saved = replace_database(load().database.absolute(), INCOMING, BACKUPS)
print(f"Database replaced; previous snapshot: {saved or 'none (first import)'}")
+14 -7
View File
@@ -3,7 +3,7 @@
##
## The subdomain has to exist before this does anything: add <app> to SWAG's
## SUBDOMAINS (or run it with a wildcard cert) and point the DNS record at the
## same address as the rest of the stack. See docs/hosting.md.
## same address as the rest of the stack. See docs/Install Server.md.
## Per-IP throttle for the login endpoints, if the app has a password. Declared
## here at `http` scope — SWAG includes proxy-confs outside any server block,
@@ -15,23 +15,30 @@ server {
listen 443 ssl;
listen [::]:443 ssl;
server_name app.rittershofer.org;
server_name app.example.com;
include /config/nginx/ssl.conf;
# Put proxy defaults at server scope so location timeouts override them
# through inheritance, rather than duplicate directives from proxy.conf.
include /config/nginx/proxy.conf;
# 0 is no limit. Set a real one if strangers can post to this app; leave it
# off when the only uploads are yours and can be large.
client_max_body_size 0;
# nginx buffers a request body to disk before it hands it on, and the login
# takes one without a session, so no limit here lets anyone spool gigabytes.
# This is small. A route that takes a large upload gets its own
# `location = /path { client_max_body_size 0; proxy_request_buffering off;
# ... }` above `location /`, with the same proxy lines as below; streaming
# the body lets the app refuse a request without a session before nginx
# has stored any of it.
client_max_body_size 1m;
# Guessing the password is already pointless if the app locks out after a
# handful of wrong answers (see docs/password-protection.md). This is the
# handful of wrong answers (see auth/README.md). This is the
# cheap half of the rest: attempts are slowed per address, at the proxy,
# before they reach the app at all. `burst=5 nodelay` lets a person who
# fat-fingers it a few times through immediately and then holds the rate.
location ~ ^/login(/passkey)?$ {
# `/login/passkey/options` is open and writes a challenge row per call, so
# it is throttled with the two routes that check a credential.
location ~ ^/login(/passkey(/options)?)?$ {
limit_req zone=app_login burst=5 nodelay;
limit_req_status 429;