Bring in pipeline, auth, deploy and docs from the boilerplate

roledesk is built on the boilerplate from here on: pipeline 2.0 (queue and
scheduler), auth (one password, sessions, passkeys), the deploy kit and its
docs, plus the coupling and packaging tests. Downstream copies, unedited.

Refreshed from boilerplate/d56949f7d0f6c0bfe5745478e09007bfad67ae87

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-23 12:29:16 +02:00
1 parent 1f436104f3
commit 9d666d0937
76 files changed
+15235

No files matched your search

+14
View File
@@ -0,0 +1,14 @@
<#
.SYNOPSIS
Run steps 1, 2 and 3: test/promote, build image, push image.
.DESCRIPTION
Stops on the first failure. Server deployment is a separate operation --
nothing here connects to the server. Git branch publication is optional
through -PushBranch.
#>
[CmdletBinding()]
param([switch] $PushBranch)
$ErrorActionPreference = 'Stop'
& "$PSScriptRoot/1-test-and-promote.ps1" -PushBranch:$PushBranch
& "$PSScriptRoot/2-build-image.ps1"
& "$PSScriptRoot/3-push-image.ps1"
+83
View File
@@ -0,0 +1,83 @@
<#
.SYNOPSIS
Test the latest local main commit and fast-forward production to it.
.DESCRIPTION
Uses a fixed commit and an isolated worktree. PushBranch optionally publishes
production to origin; without it promotion stays local. Does not run Docker.
#>
[CmdletBinding()]
param([switch] $PushBranch, [switch] $NoTests)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
. "$PSScriptRoot/_common.ps1"
$Source = Setting SOURCE 'main'
$Branch = Setting BRANCH 'production'
$Python = Setting PYTHON (Join-Path $root '.venv\Scripts\python.exe')
Push-Location $root
try {
$commit = git rev-parse --verify "$Source^{commit}"
Assert-Ok 'resolve source commit'
$commit = $commit.Trim()
# ------------------------------------------------------- test the commit
# In a throwaway worktree, not here. The tests then run against exactly the
# tree that is about to become production, with none of the working copy's
# uncommitted edits in the way. Worktrees have no .venv of their own, so the
# interpreter comes from the main checkout and the code from the worktree.
if (-not $NoTests) {
Write-Step "tests on $Source"
if (-not (Test-Path $Python)) {
throw "no interpreter at $Python - set PYTHON in deploy\release.env"
}
$base = Join-Path ([IO.Path]::GetTempPath()) ("$Name-test-" + [guid]::NewGuid().ToString('N').Substring(0, 8))
$tree = Join-Path $base $Name
try {
git worktree add --detach $tree $commit | Out-Null
Assert-Ok 'git worktree add'
Push-Location $tree
try {
# Your project's checks. A gate that is red for weeks is not a
# gate -- it is a flag you learn to type -- so take one out when
# it stops meaning anything, and put it back when it is green.
#
# The starter's dev group is pytest and httpx, so pytest is the
# only gate that can run out of the box. Add ruff and mypy to
# [dependency-groups] dev in pyproject.toml, then uncomment the
# two lines below -- a gate whose tool is not installed fails
# every release for a reason that has nothing to do with the code.
& $Python -m pytest -q; Assert-Ok 'pytest'
# & $Python -m ruff check .; Assert-Ok 'ruff'
# & $Python -m mypy --cache-dir (Join-Path $root '.mypy_cache')
# Assert-Ok 'mypy'
} finally { Pop-Location }
} finally {
git worktree remove --force $tree 2>$null | Out-Null
Remove-ReleaseTemp @($base)
}
}
# ------------------------------------------------------------- promote
# Fast-forward only. If production has commits main does not, that is a
# release made from somewhere else, and this stops rather than discarding it.
Write-Step "promote $Source -> $Branch"
git show-ref --verify --quiet "refs/heads/$Branch"
if ($LASTEXITCODE -eq 0) {
git merge-base --is-ancestor $Branch $commit
if ($LASTEXITCODE -ne 0) {
Write-Host "$Branch is not an ancestor of $Source - it has commits of its own." -ForegroundColor Red
throw "look at: git log --oneline $Source..$Branch"
}
}
# Git publication is optional and independent of the image.
git branch -f $Branch $commit; Assert-Ok 'git branch -f'
if ($PushBranch) {
git push origin "${commit}:refs/heads/$Branch"
Assert-Ok 'publish production branch'
}
Write-Host "Promoted $Branch to $commit"
} finally { Pop-Location }
+57
View File
@@ -0,0 +1,57 @@
<#
.SYNOPSIS
Build a local Docker image from the committed production branch.
.DESCRIPTION
Does not test, promote, push Git branches, use SSH, or start a container.
The image stays local. Use 3-push-image.ps1 to publish it separately.
#>
[CmdletBinding()]
param()
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
. "$PSScriptRoot/_common.ps1"
$Image = Setting IMAGE -Required
$Branch = Setting BRANCH 'production'
$Platform = Setting PLATFORM 'linux/amd64'
Push-Location $root
try {
$commit = git rev-parse --verify "$Branch^{commit}"
Assert-Ok 'resolve production commit'
$commit = $commit.Trim()
$sha = (git rev-parse --short $commit).Trim()
Assert-Ok 'resolve image tag'
$tagged = "${Image}:${sha}"
$latest = "${Image}:latest"
# --------------------------------------------------------------- build
# `git archive` to a file and unpack it, rather than piping it. A PowerShell
# pipeline carries text: it would re-encode a tar stream and hand docker a
# corrupted context. Writing the archive out and building from the directory
# keeps the same guarantee -- the context is the committed tree and nothing
# else -- without a byte going through a pipe.
#
# The Dockerfile comes out of that same tree, at deploy/Dockerfile, so the
# recipe is versioned with the code it builds. Docker reads the ignore rules
# from deploy/Dockerfile.dockerignore beside it.
Write-Step "build $tagged"
$ctx = Join-Path ([IO.Path]::GetTempPath()) "$Name-build-$sha"
$tar = "$ctx.tar"
Remove-ReleaseTemp @($ctx, $tar)
New-Item -ItemType Directory -Path $ctx | Out-Null
try {
git archive --format=tar -o $tar $commit; Assert-Ok 'git archive'
tar -xf $tar -C $ctx; Assert-Ok 'tar -xf'
$dockerfile = Join-Path $ctx 'deploy/Dockerfile'
docker build --platform $Platform -f $dockerfile -t $tagged -t $latest $ctx
Assert-Ok 'docker build'
docker image ls $Image --format ' {{.Tag}} {{.Size}}'
} finally {
Remove-ReleaseTemp @($ctx, $tar)
}
Write-Host "Image: $tagged"
} finally { Pop-Location }
+36
View File
@@ -0,0 +1,36 @@
<#
.SYNOPSIS
Publish the already-built production image to the registry.
.DESCRIPTION
Never rebuilds. If the push fails, fix access or connectivity and run this
again; the image is already on disk. `docker login` first, with a token
that has package-write access.
#>
[CmdletBinding()]
param()
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
. "$PSScriptRoot/_common.ps1"
$Image = Setting IMAGE -Required
$Branch = Setting BRANCH 'production'
Push-Location $root
try {
$commit = git rev-parse --verify "$Branch^{commit}"
Assert-Ok 'resolve production commit'
$sha = (git rev-parse --short $commit.Trim()).Trim()
Assert-Ok 'resolve image tag'
$tagged = "${Image}:${sha}"
$latest = "${Image}:latest"
docker image inspect $tagged --format '{{.Id}}' | Out-Null
Assert-Ok "find local $tagged (run step 2 first)"
# latest must name this candidate, even if another image was built since.
docker tag $tagged $latest
Assert-Ok 'tag latest image'
docker push $tagged
Assert-Ok 'push commit image'
docker push $latest
Assert-Ok 'push latest image'
Write-Host "Published $tagged and $latest"
} finally { Pop-Location }
+94
View File
@@ -0,0 +1,94 @@
# A Python app, for a server rather than a laptop.
#
# Two stages so the runtime carries no build tooling: `uv` resolves and
# installs into /app/.venv, and the second stage copies that venv and the
# source next to it. The install is deliberately editable, which matters if
# the app reads a file from beside its package (an example config, a template
# directory): that path is true in a checkout and true here, and false in
# site-packages.
#
# Already in place at deploy/Dockerfile, with its ignore file beside it.
# 2-build-image.ps1 reads it out of a `git archive` of the production commit,
# so the recipe is versioned with the code it builds.
#
# The package is `app` and stays `app` — a project started from this repo
# renames nothing. Edit the two `uv sync` lines when you add extras.
ARG PYTHON=3.14-slim
ARG UV=0.9.17
FROM ghcr.io/astral-sh/uv:${UV} AS uv
# ------------------------------------------------------------------ build
FROM python:${PYTHON} AS build
COPY --from=uv /uv /usr/local/bin/uv
ENV UV_LINK_MODE=copy \
UV_COMPILE_BYTECODE=1 \
UV_PYTHON_DOWNLOADS=never
WORKDIR /app
# Dependencies first and on their own layer, keyed on the lock file alone, so
# editing a Python file rebuilds in seconds rather than re-resolving the tree.
COPY pyproject.toml uv.lock README.md ./
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-dev --no-install-project
COPY app app
# The two module folders this repo ships beside the app, here and again in the
# runtime stage below. `pyproject.toml` lists `pipeline*` and `auth*` in its
# packages as well, or the install leaves them out.
COPY pipeline pipeline
# The door. A project that does not want one drops this line *and* the
# `auth.connect` call in `app/main.py` together: dropping the folder and
# leaving the call is a container that fails at startup, and dropping the call
# and leaving the folder is a container that starts with the door open.
COPY auth auth
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-dev
# ---------------------------------------------------------------- runtime
FROM python:${PYTHON}
# Nothing is built here, so nothing but the interpreter's own runtime is
# needed. `tini` is the one addition: a web server's shutdown and a worker
# pool's stop both hang off SIGTERM, and PID 1 without an init reaps no
# children and forwards no signals, so a `docker compose down` would wait the
# full ten seconds and then kill the workers mid-job.
RUN apt-get update \
&& apt-get install -y --no-install-recommends tini \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY --from=build /app/.venv /app/.venv
COPY app app
COPY pipeline pipeline
# The login page and the two scripts are files inside this folder, served from
# routes; see the build stage above.
COPY auth auth
ENV PATH=/app/.venv/bin:$PATH \
PYTHONUNBUFFERED=1 \
APP_CONFIG=/data/config.yaml \
HOME=/data/home
# In the volume, not the image: whatever uid the compose file runs as owns
# /data, and a CLI the app shells out to — the `claude` binary, for one —
# wants somewhere to write. An image-baked /home/app would be right for uid
# 1000 and unwritable for anyone else.
VOLUME /data
EXPOSE 8787
# A session-free probe that checks the application can still read its store.
# `docker compose up --wait` in deploy.sh waits on exactly this, so a probe
# that only proves the port is open buys a deployment that reports success
# while the database is unreachable.
HEALTHCHECK --interval=60s --timeout=5s --start-period=20s --retries=3 \
CMD python -c "import urllib.request as u; u.urlopen('http://127.0.0.1:8787/health', timeout=4)"
ENTRYPOINT ["/usr/bin/tini", "--"]
CMD ["app", "serve", "--host", "0.0.0.0", "--port", "8787"]
+42
View File
@@ -0,0 +1,42 @@
# Named for the Dockerfile beside it: Docker reads `<dockerfile>.dockerignore`
# before falling back to one at the root of the context, which is how the ignore
# rules get to live next to the recipe they belong to instead of at the top of
# the repository.
#
# 2-build-image.ps1 already hands docker a context extracted from `git archive`,
# so the untracked heavyweights below are not there to exclude. They are listed
# for the other way in — `docker build -f deploy/Dockerfile .` from a checkout —
# where .venv alone can be several hundred megabytes.
.git
.venv
.claude
.idea
.github
.gitea
.scratch
docs
# A module folder's tests travel with the folder, not into the image. One line
# per folder, so a fourth module is a fourth line here.
tests
pipeline/tests
auth/tests
__pycache__
*.py[cod]
*.egg-info
.pytest_cache
.ruff_cache
.mypy_cache
# Local state. The config and the database live in the server's volume, and
# baking either into the image would ship one machine's data to another and
# quietly make the image the source of truth for both.
config.yaml
*.db
*.db-shm
*.db-wal
deploy
Todo.md
CLAUDE.md
+52
View File
@@ -0,0 +1,52 @@
# Deploy
Everything needed to put this app on the rittershofer.org box behind SWAG: build an image on Windows, publish it to Gitea, run it as one more service in the compose file that already runs the proxy, back up its database, and give it a password that survives being on the open internet.
Extracted from newsdesk, which is the worked example — every decision here has been run in production at least once, and the comments say why rather than what.
**It is already in place.** Nothing has to be copied and nothing has to be renamed: the package is `app`, the environment variables are `APP_`, and the files below are where the scripts expect them. `<app>` in the docs is your *service's* name — the hostname, the data directory, the image tag, the compose service — and it is the one thing you choose.
## Where each file is
| Here | | |
|---|---|---|
| `deploy/*.ps1` | the release scripts, run on Windows | |
| `deploy/Dockerfile` | read out of the archived commit by step 2 | copies `app`, `pipeline` and `auth` |
| `deploy/Dockerfile.dockerignore` | Docker finds it by that name | |
| `deploy/server/*` | committed here, then installed on the server | |
| `deploy/server/compose.yaml` | to `/srv/swag/docker-compose.yml`, as one `services:` block | |
| `deploy/server/env.example` | to `/srv/swag/.env`, as appended variables, `chmod 600` | |
| `deploy/server/deploy.sh`, `deploy/server/backup.sh` | to `/srv/swag/`, next to the compose file | |
| `deploy/server/swag/app.subdomain.conf` | to SWAG's `proxy-confs/`, renamed to `<app>.subdomain.conf` | |
`db.ps1` expects `server/` beside it, which is why `deploy/server/` is committed here as well as installed on the server.
**Keep the `.ps1` files to ASCII.** Windows PowerShell 5.1 reads a script without a byte-order mark as Windows-1252, so an em dash becomes three characters ending in a closing quote: the string ends in the middle of itself and the script fails to parse before it runs a line. Write `--`. newsdesk hit this on a comment nobody thought was code, and `tests/test_packaging.py` here holds every script in this folder to it.
`.gitignore` already covers `deploy/release.env` and `deploy/backups/`.
## The docs
They are one directory up, in [../docs/](../docs), because they are about the hosting rather than about these scripts.
- **[../docs/hosting.md](../docs/hosting.md)** — what the box is, the seven steps to add a service, what the templates already decide for you, and the things that have already gone wrong on it. Start here for a service that is not on the box yet.
- **[../docs/release.md](../docs/release.md)** — the Windows side: `release.env`, the four numbered scripts, what each one refuses to do, and why the tests run in a throwaway worktree.
- **[../docs/server.md](../docs/server.md)** — installing, deploying, backups, moving a SQLite database in either direction, rolling back, and what to verify before relying on any of it.
- **[../docs/llm-backends.md](../docs/llm-backends.md)** — asking Claude from a deployed app: API key or subscription token, selected by which one is in the environment, with a self-contained reference implementation.
- **[../docs/password-protection.md](../docs/password-protection.md)** — one password on a public hostname: the lockout, sessions, passkeys, and how much throttling is actually worth building. `auth/` is the implementation of it.
## A new service, in order
1. DNS, then certificate, then registry login, then data directory — [../docs/hosting.md](../docs/hosting.md).
2. Copy `release.env.example` to `release.env` and fill it in. That is the only file this step touches: everything else is already committed.
3. `.\deploy\0-release.ps1` to test, promote, build and publish.
4. Install the server files, edit `.env`, `bash deploy.sh` — [../docs/server.md](../docs/server.md).
5. Put the model credential in `.env` if the app needs one — [../docs/llm-backends.md](../docs/llm-backends.md).
6. Get in, rotate the bootstrap password, register a passkey — [../docs/password-protection.md](../docs/password-protection.md).
7. Schedule `backup.sh`, then restore one of its backups somewhere else to prove it works.
Step 1 of `0-release.ps1` runs `pytest` in a throwaway worktree, with `ruff` and `mypy` commented out beside it: this repository's dev group is pytest and httpx, and a gate whose tool is not installed fails every release for a reason that has nothing to do with the code. Add the tools, then uncomment the gates.
## What this is not
Not a framework and not a generator: nothing here is imported or extended, it is copied and edited. That is the point — three services with three edited copies beats three services sharing one abstraction that has to be right for all of them. What this repository changes is only *where the copy comes from*: `boilerplate/deploy/` is the source of truth, and news and health hold copies of it.
+66
View File
@@ -0,0 +1,66 @@
<#
.SYNOPSIS
Shared plumbing for the release scripts. Dot-source it; do not run it.
.DESCRIPTION
Dot-sourcing runs in the caller's scope, so everything defined here --
$root, $settings, Setting, Assert-Ok, Write-Step, Remove-ReleaseTemp -- is
the caller's afterwards. One copy of the release.env reader, because three
copies is three places for it to drift.
#>
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
#: The repository root, worked out from this file rather than from the working
#: directory, so the scripts can be run from anywhere.
$root = Split-Path -Parent $PSScriptRoot
function Assert-Ok([string] $What) {
if ($LASTEXITCODE -ne 0) { throw "$What failed (exit $LASTEXITCODE)" }
}
function Write-Step([string] $Text) {
Write-Host ''
Write-Host "== $Text" -ForegroundColor Cyan
}
$envFile = Join-Path $PSScriptRoot 'release.env'
if (-not (Test-Path $envFile)) {
throw "no deploy\release.env - copy deploy\release.env.example and fill it in"
}
$settings = @{}
foreach ($line in Get-Content $envFile) {
if ($line -match '^\s*(#|$)') { continue }
# A line with no `=` is a typo, not a setting. Say so here rather than
# letting it surface later as a null reference three functions away.
if ($line -notmatch '=') { throw "release.env: not a KEY=VALUE line: $line" }
$key, $value = $line -split '=', 2
$settings[$key.Trim()] = $value.Trim().Trim('"')
}
function Setting([string] $Name, [string] $Default, [switch] $Required) {
if ($settings.ContainsKey($Name) -and $settings[$Name]) { return $settings[$Name] }
if ($Required) { throw "set $Name in deploy\release.env" }
return $Default
}
#: What temporary directories this release is allowed to delete. Everything
#: below is created by these scripts and named for it.
$Name = Setting NAME -Required
# Only ever deletes a <name>-* entry directly under the temp directory. A
# guard, not a convenience: these paths are built by string concatenation and
# handed to `Remove-Item -Recurse -Force`, and one empty variable in that
# sentence is a very bad afternoon.
function Remove-ReleaseTemp([string[]] $Paths) {
$tempRoot = [IO.Path]::GetFullPath([IO.Path]::GetTempPath()).TrimEnd('\') + '\'
foreach ($path in $Paths) {
$resolved = [IO.Path]::GetFullPath($path)
if (-not $resolved.StartsWith($tempRoot, [StringComparison]::OrdinalIgnoreCase) -or
-not ([IO.Path]::GetFileName($resolved)).StartsWith("$Name-")) {
throw "Unsafe release temporary path: $resolved"
}
if (Test-Path -LiteralPath $resolved) { Remove-Item -LiteralPath $resolved -Recurse -Force }
}
}
+142
View File
@@ -0,0 +1,142 @@
<#
.SYNOPSIS
The database, in both directions.
.DESCRIPTION
push this machine's database -> the server
pull a fresh snapshot from the server -> .\backups
`push` is the one-off at the start: production begins knowing what you
already have, rather than starting empty. `pull` is the one you keep
running -- on a schedule from the server itself (server\backup.sh under
cron) and by hand from here whenever you want the copy somewhere the
server is not.
`push` moves the database and nothing else. It does not stop the app before
or start it after: a deployment knows what else is moving and in what order,
and this does not. Stop the app on the server first -- the import is a file
swap under SQLite's feet and is refused while the app is up. Start it again
afterwards.
Both go through SQLite's own `VACUUM INTO`. The database is in WAL mode, so
the .db file on disk is always missing whatever is still in the -wal beside
it, and copying the three files with the app running is a race you lose
quietly -- a database that opens fine and is short a day's writes.
After a push the two copies diverge from the first thing written on either.
There is no merge for that and this does not pretend there is: pick the one
that is authoritative, and use the other read-only or not at all.
.EXAMPLE
.\deploy\db.ps1 push
.EXAMPLE
.\deploy\db.ps1 pull
#>
[CmdletBinding()]
param(
[Parameter(Mandatory)]
[ValidateSet('push', 'pull')]
[string] $Action,
# Skip the confirmation before replacing the server's database.
[switch] $Yes
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
. "$PSScriptRoot/_common.ps1"
Set-Location $root
$SshHost = Setting SSH_HOST -Required
$RemoteDir = Setting REMOTE_DIR -Required
# Where the data actually is. Beside the compose file unless APP_DATA put it
# under a shared root; then REMOTE_DATA has to say so.
$RemoteData = Setting REMOTE_DATA "$RemoteDir/data"
$Python = Setting PYTHON (Join-Path $root '.venv\Scripts\python.exe')
$LocalDb = Setting LOCAL_DB (Join-Path $root "$Name.db")
switch ($Action) {
# ---------------------------------------------------------------- push
'push' {
if (-not (Test-Path $LocalDb)) { throw "no $LocalDb" }
$stamp = Get-Date -Format 'yyyyMMdd-HHmmss'
$snap = Join-Path $root "$Name-$stamp.db.gz"
Write-Host "snapshotting $LocalDb"
# One Python call for both halves. `VACUUM INTO` is the only way to get a
# consistent single file out of a WAL database, and Python is already the
# thing that has to be here -- PowerShell has no gzip that writes the format
# the server's `gunzip` expects.
$snapshot = @'
import gzip, pathlib, shutil, sqlite3, sys, tempfile
source, target = sys.argv[1], pathlib.Path(sys.argv[2])
with tempfile.TemporaryDirectory() as work:
plain = pathlib.Path(work) / "snapshot.db"
db = sqlite3.connect(source)
try:
db.execute("VACUUM INTO ?", (str(plain),))
finally:
db.close()
with plain.open("rb") as raw, gzip.open(target, "wb") as out:
shutil.copyfileobj(raw, out)
'@
& $Python -c $snapshot $LocalDb $snap; Assert-Ok 'snapshot'
' {0:N1} MB' -f ((Get-Item $snap).Length / 1MB) | Write-Host
try {
Write-Host ''
Write-Host "This replaces the database on $SshHost with this one." -ForegroundColor Yellow
Write-Host 'Anything written there since is gone.' -ForegroundColor Yellow
Write-Host 'The app must already be stopped there, and stays stopped after.' -ForegroundColor Yellow
if (-not $Yes) {
$reply = Read-Host 'go ahead? [y/N]'
if ($reply -notmatch '^[yY]') { Write-Host 'stopped.'; return }
}
scp -C $snap "${SshHost}:$RemoteData/incoming.db.gz"; Assert-Ok 'scp database'
scp (Join-Path $PSScriptRoot 'server\import_database.py') "${SshHost}:$RemoteDir/import_database.py"
Assert-Ok 'scp importer'
} finally {
Remove-Item -Force $snap -ErrorAction SilentlyContinue
}
# Base64 rather than stdin: PowerShell has no `<` for a native command, and
# piping the text would rewrite the script's line endings on the way.
$b64 = [Convert]::ToBase64String([IO.File]::ReadAllBytes((Join-Path $PSScriptRoot 'server\import.sh')))
ssh $SshHost "echo $b64 | base64 -d | bash -s -- '$RemoteDir'"; Assert-Ok 'import'
Write-Host "pushed. The app is still stopped; start it when the rest of"
Write-Host 'the deployment is in place:'
Write-Host " ssh $SshHost `"cd '$RemoteDir' && docker compose up -d --wait $Name`""
}
# ---------------------------------------------------------------- pull
'pull' {
$into = Join-Path $root 'backups'
New-Item -ItemType Directory -Force -Path $into | Out-Null
# The server's own copy of backup.sh, not a streamed one. It is the same
# script cron runs, it is already there, and it locates its own directory
# through BASH_SOURCE -- which is exactly the thing a script read from stdin
# does not have. import.sh takes its directory as an argument and so can be
# streamed; this one cannot.
Write-Host "snapshotting on $SshHost"
ssh $SshHost "cd '$RemoteDir' && bash backup.sh"; Assert-Ok 'backup'
$newest = (ssh $SshHost "ls -1t '$RemoteData'/backups/$Name-*.db.gz | head -1").Trim()
Assert-Ok 'listing backups'
if (-not $newest) { throw 'no backup appeared on the server' }
scp -C "${SshHost}:$newest" $into; Assert-Ok 'scp'
$file = Split-Path -Leaf $newest
Write-Host "backups\$file"
Write-Host ''
Write-Host 'To read it here, gunzip it, or in Python:'
Write-Host " import gzip, shutil; shutil.copyfileobj(gzip.open(r'backups\$file'), open('read-only.db','wb'))"
}
}
+28
View File
@@ -0,0 +1,28 @@
# Copy to release.env beside these scripts and fill it in. Never committed:
# it names your registry and, for db.ps1, your server.
#
# Local main is the source. Fetching it first is your choice, not the script's.
# The service's name. Used for the image tag, for the temporary directories
# these scripts create, and for the database file db.ps1 moves. One word, no
# spaces — it ends up in paths.
NAME=app
IMAGE=git.rittershofer.org/wolf/app
PLATFORM=linux/amd64
BRANCH=production
SOURCE=main
# Defaults to the checkout's .venv/Scripts/python.exe. Only needed for the
# tests in step 1 and for db.ps1's snapshot.
# PYTHON=C:/Users/Coolkau/code/app/.venv/Scripts/python.exe
# db.ps1 uses SSH. The numbered release scripts never do. REMOTE_DIR is the
# directory holding the compose file; REMOTE_DATA is where APP_DATA points,
# and defaults to REMOTE_DIR/data when left blank.
SSH_HOST=
REMOTE_DIR=/srv/swag
REMOTE_DATA=/srv/data/app
# The database this machine pushes, if it is not <root>/<NAME>.db.
# LOCAL_DB=C:/Users/Coolkau/code/app/app.db
+44
View File
@@ -0,0 +1,44 @@
#!/usr/bin/env bash
#
# On the server, next to the compose file, or with its directory as $1. One
# snapshot of the database, taken through the running container, kept
# alongside the others.
#
# bash backup.sh # into $APP_DATA/backups
# 0 4 * * * bash /srv/swag/backup.sh >> /srv/swag/backup.log 2>&1
#
# `VACUUM INTO` rather than copying the file: SQLite is in WAL mode, so the
# .db on disk is missing whatever is still in the -wal, and `cp` of a live
# database is a coin toss. VACUUM INTO takes a read lock, writes one
# self-contained and compacted file, and never blocks a writer for long enough
# to matter on a database of this size.
#
# Rename `app` and `APP_DATA` to your service's name.
set -euo pipefail
cd "${1:-$(dirname "$(readlink -f "$0")")}"
# Same source of truth as the compose file and deploy.sh.
DATA=$(sed -n 's/^[[:space:]]*APP_DATA=\([^#]*\).*/\1/p' .env | tail -1 | xargs || true)
DATA=${DATA:-./data}
KEEP=${KEEP:-14}
stamp=$(date +%Y%m%d-%H%M%S)
mkdir -p "$DATA/backups"
docker compose exec -T app python -c "
import sqlite3
db = sqlite3.connect('/data/app.db')
db.execute('VACUUM INTO ?', ('/data/backups/app-$stamp.db',))
db.close()
"
gzip -f "$DATA/backups/app-$stamp.db"
echo "$DATA/backups/app-$stamp.db.gz $(du -h "$DATA/backups/app-$stamp.db.gz" | cut -f1)"
# Oldest first, everything past KEEP goes. Named by timestamp, so sorting the
# names sorts them by age.
ls -1 "$DATA"/backups/app-*.db.gz 2>/dev/null \
| sort \
| head -n -"$KEEP" \
| while read -r old; do rm -f "$old"; echo "removed $old"; done
+60
View File
@@ -0,0 +1,60 @@
# The service block, for the compose file that already runs your proxy.
#
# It needs to be on the same Docker network as SWAG, because that is how nginx
# reaches it — by container name, on the network Compose gives every project.
# Being in the same file is what puts it there; nothing else about it is
# related to the other services, and nothing else needs to be.
#
# Paste this `app:` block into that file's `services:`, add the variables from
# `env.example` to the `.env` next to it, and it is `docker compose up -d app`
# from then on — which recreates this container and no other.
#
# Rename `app` and `APP_` throughout to your service's name.
services:
app:
image: ${APP_IMAGE:?set APP_IMAGE in .env}:${APP_TAG:-latest}
container_name: app
restart: unless-stopped
# A plain directory on the host rather than a managed volume, so a backup
# is `tar` and it is owned by you rather than by root. Relative to this
# file by default; set APP_DATA to an absolute path to keep every
# container's data under one root, which is the thing a backup wants.
volumes:
- ${APP_DATA:-./data}:/data
# The container follows the host directory's uid instead of the other way
# round — the same PUID/PGID the rest of the stack already runs with.
user: "${PUID:-1000}:${PGID:-1000}"
environment:
TZ: ${TZ:-Europe/Berlin}
# Which model answers, decided here rather than in the image. With
# `backend: auto` the API key wins when there is one and the
# subscription CLI is the fallback, so filling in ANTHROPIC_API_KEY
# later switches it over with no rebuild. See docs/llm-backends.md.
CLAUDE_CODE_OAUTH_TOKEN: ${CLAUDE_CODE_OAUTH_TOKEN:-}
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
# Normally empty. Set it to pin the password — the way back in when a
# login is lost — then unset it and recreate to hand control of the
# password back to the app. See docs/password-protection.md.
APP_PASSWORD: ${APP_PASSWORD:-}
# No `ports:`. Nothing on the host reaches it; SWAG does, by container
# name, on the project's network. That is what keeps the app's port off
# the internet even if the firewall is wrong.
# `deploy: resources: limits:` is Swarm-only and silently ignored by
# `docker compose up`. This is the one that works. 0 is unlimited, which
# is the honest default until the box has been watched under real load.
mem_limit: ${APP_MEM:-0}
# The log is the record — nothing here writes a second copy of what went
# wrong — so it is worth keeping a few rotations of, and worth capping on
# a small disk.
logging:
driver: json-file
options:
max-size: "20m"
max-file: "5"
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env bash
# Run on the server, next to the compose file, or with its directory as $1.
# No remote shell required: the release scripts on Windows publish an image
# and stop there, and this is the other half.
#
# Rename `app` and `APP_DATA` to your service's name.
set -euo pipefail
cd "${1:-$(dirname "$(readlink -f "$0")")}"
[[ -s .env ]] || { echo 'Install .env before deploying.' >&2; exit 1; }
# Where the data directory is depends on APP_DATA, which the compose file also
# reads. Pulled out of .env rather than sourced, because Compose's .env is not
# a shell script and running it as one is how a stray quote becomes a surprise.
DATA=$(sed -n 's/^[[:space:]]*APP_DATA=\([^#]*\).*/\1/p' .env | tail -1 | xargs || true)
DATA=${DATA:-./data}
[[ -s "$DATA/config.yaml" ]] || {
echo "Install $DATA/config.yaml before deploying." >&2
exit 1
}
# Pull explicitly, here and nowhere else: the compose file has no pull policy,
# so starting or importing never fetches an image on its own. A registry
# failure must leave the running service alone, not deploy a cache.
echo 'Pulling the configured image…'
if ! docker compose pull app; then
echo 'Image pull failed; app was not restarted.' >&2
exit 1
fi
# Use the image just pulled. Do not start or recreate other services in the
# file. `--wait` holds until the container's own HEALTHCHECK passes.
if ! docker compose up -d --no-deps --pull never --wait --wait-timeout 240 app; then
docker compose ps app >&2 || true
echo 'Deployment failed. Check docker compose logs app.' >&2
exit 1
fi
docker compose ps app
+42
View File
@@ -0,0 +1,42 @@
# The variables this service adds to the `.env` next to your compose file.
# Written by hand once, on the server, and never synced from a repository — it
# holds the secrets and it is the record of which image is running.
#
# chmod 600. Rename APP_ to your service's name throughout.
APP_IMAGE=git.rittershofer.org/wolf/app
APP_TAG=latest # follows published images; pin a commit tag for rollback
# Where the app keeps its database, its config and its backups. Relative to
# the compose file if you leave it out. Set it to an absolute path to put it
# under the one root every container on the box writes to — the thing that
# makes a backup one target instead of a hunt.
APP_DATA=/srv/data/app
# The same PUID/PGID the rest of the stack runs as. The data directory must be
# owned by it.
PUID=1000
PGID=1000
TZ=Europe/Berlin
# Which model answers. Set one or the other, not both — see
# docs/llm-backends.md for what each one means and what it costs.
#
# CLAUDE_CODE_OAUTH_TOKEN a year-long token for the Claude subscription, from
# `claude setup-token` on a machine you are signed in
# on. Nothing billed per token.
# ANTHROPIC_API_KEY per-token billing, no local CLI, the unattended
# path. Preferred by `auto` when both are set.
CLAUDE_CODE_OAUTH_TOKEN=
ANTHROPIC_API_KEY=
# Leave empty. The first locked start generates a password and prints it to
# `docker compose logs app`; after that it is the app's Account screen's to
# change. Fill this in only to force one — it is the way back in — then empty
# it again and recreate, or the Account screen refuses to change what the
# environment pins. See docs/password-protection.md.
APP_PASSWORD=
# Unlimited until the box has been watched through the heaviest thing the app
# does. Then something like 1500m.
APP_MEM=0
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env bash
# Called on the server after uploading incoming.db.gz and import_database.py.
# `db.ps1 push` does both and then runs this; by hand it is the same two files
# in the same two places.
#
# This replaces the database and does nothing else. It does not stop the app
# and it does not start it: whoever is running a deployment knows what else is
# moving and in what order, and a tool that helpfully restarts things underneath
# that is a tool fighting its caller. By hand it is
# `docker compose stop app` and `docker compose up -d --wait app` around this.
#
# What it will not do is import into a running app. The replacement is a file
# swap under SQLite's feet, and the live database is checkpointed first, which
# needs every writer gone — `import_database.py` refuses a busy database, and
# this refuses an app that is up, because the clearer error is the earlier one.
set -euo pipefail
cd "${1:?server directory required}"
# Same source of truth as the compose file, deploy.sh and backup.sh.
DATA=$(sed -n 's/^[[:space:]]*APP_DATA=\([^#]*\).*/\1/p' .env | tail -1 | xargs || true)
DATA=${DATA:-./data}
[[ -s "$DATA/incoming.db.gz" && -s import_database.py ]]
if [[ -n "$(docker compose ps -q --status running app 2>/dev/null)" ]]; then
echo 'app is running. Stop it before importing:' >&2
echo ' docker compose stop app' >&2
exit 1
fi
# The one-off container uses compose's uid and its mounted /data. `--pull
# never`: an import runs on the image already here, whatever the installed
# compose file says about pulling. Getting a new image is deploy.sh's job.
docker compose run --rm --no-deps --pull never -T --entrypoint python app - < import_database.py
rm -f "$DATA/incoming.db.gz"
+77
View File
@@ -0,0 +1,77 @@
"""Offline database replacement, run as the compose service's uid.
The app container must be stopped. Only stdlib is used so this can be sent
over stdin to either the old or the new image during a deployment.
Two things to edit for your service: `DATABASE` and `REQUIRED_TABLES`. The
table check is what stops a right-shaped-but-wrong database — last project's,
or a half-downloaded one — from landing on top of a good one.
"""
from __future__ import annotations
import gzip
import os
import shutil
import sqlite3
import tempfile
from contextlib import closing
from datetime import UTC, datetime
from pathlib import Path
DATABASE = Path("/data/app.db")
INCOMING = Path("/data/incoming.db.gz")
BACKUPS = Path("/data/backups")
#: Any one of these missing means this is not our database. Name tables that
#: have existed since the first schema, not ones a recent migration added.
REQUIRED_TABLES = {"items"}
def replace_database(database: Path, archive: Path, backups: Path) -> Path | None:
"""Validate first, snapshot the stopped database, then atomically replace it."""
fd, name = tempfile.mkstemp(prefix=".incoming-", suffix=".db", dir=database.parent)
staged = Path(name)
backup = None
try:
with os.fdopen(fd, "wb") as output, gzip.open(archive, "rb") as source:
shutil.copyfileobj(source, output)
output.flush()
os.fsync(output.fileno())
with closing(sqlite3.connect(staged)) as candidate:
if candidate.execute("PRAGMA integrity_check").fetchall() != [("ok",)]:
raise ValueError("Incoming database failed integrity_check")
tables = {row[0] for row in candidate.execute(
"SELECT name FROM sqlite_master WHERE type='table'")}
if not REQUIRED_TABLES <= tables:
raise ValueError("Incoming file is not this application's database")
if database.exists():
backups.mkdir(parents=True, exist_ok=True)
stamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S-%f")
snapshot = backups / f"{database.stem}-{stamp}.db"
backup = snapshot.with_suffix(".db.gz")
with closing(sqlite3.connect(database)) as current:
current.execute("VACUUM INTO ?", (str(snapshot),))
# Refuse a competing connection instead of removing a live WAL.
busy, _, _ = current.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()
if busy:
raise RuntimeError("Database is busy; stop every writer before importing")
# Keep the uncompressed recovery copy if compression fails.
with snapshot.open("rb") as source, gzip.open(backup, "xb") as output:
shutil.copyfileobj(source, output)
with backup.open("r+b") as saved:
os.fsync(saved.fileno())
snapshot.unlink()
for suffix in ("-wal", "-shm"):
Path(str(database) + suffix).unlink(missing_ok=True)
os.replace(staged, database)
return backup
finally:
staged.unlink(missing_ok=True)
if __name__ == "__main__":
saved = replace_database(DATABASE, INCOMING, BACKUPS)
print(f"Database replaced; previous snapshot: {saved or 'none (first import)'}")
+64
View File
@@ -0,0 +1,64 @@
## Rename to <app>.subdomain.conf, copy to SWAG's /config/nginx/proxy-confs/,
## validate, reload.
##
## The subdomain has to exist before this does anything: add <app> to SWAG's
## SUBDOMAINS (or run it with a wildcard cert) and point the DNS record at the
## same address as the rest of the stack. See docs/hosting.md.
## Per-IP throttle for the login endpoints, if the app has a password. Declared
## here at `http` scope — SWAG includes proxy-confs outside any server block,
## which is what makes this legal in this file. 1 MB of zone holds about 16,000
## addresses. Delete both this and the `location` below if there is no login.
limit_req_zone $binary_remote_addr zone=app_login:1m rate=6r/m;
server {
listen 443 ssl;
listen [::]:443 ssl;
server_name app.rittershofer.org;
include /config/nginx/ssl.conf;
# Put proxy defaults at server scope so location timeouts override them
# through inheritance, rather than duplicate directives from proxy.conf.
include /config/nginx/proxy.conf;
# 0 is no limit. Set a real one if strangers can post to this app; leave it
# off when the only uploads are yours and can be large.
client_max_body_size 0;
# Guessing the password is already pointless if the app locks out after a
# handful of wrong answers (see docs/password-protection.md). This is the
# cheap half of the rest: attempts are slowed per address, at the proxy,
# before they reach the app at all. `burst=5 nodelay` lets a person who
# fat-fingers it a few times through immediately and then holds the rate.
location ~ ^/login(/passkey)?$ {
limit_req zone=app_login burst=5 nodelay;
limit_req_status 429;
include /config/nginx/resolver.conf;
set $upstream_app app;
set $upstream_port 8787;
set $upstream_proto http;
proxy_pass $upstream_proto://$upstream_app:$upstream_port;
}
location / {
include /config/nginx/resolver.conf;
set $upstream_app app;
set $upstream_port 8787;
set $upstream_proto http;
proxy_pass $upstream_proto://$upstream_app:$upstream_port;
# Only if the app holds a WebSocket open. SWAG's proxy.conf already
# sets the Upgrade headers; what it also sets is a read timeout of a
# couple of minutes, which would close every open tab's socket on the
# minute and leave the browser reconnecting for no reason.
proxy_read_timeout 86400s;
proxy_send_timeout 86400s;
}
}
## `listen 443 ssl` without `http2` is correct — `http2 on;` is set once at
## `http` scope in nginx.conf, and the `listen ... http2` parameter is
## deprecated in nginx 1.25 and later.