Refresh deploy/ and bin/ from the boilerplate at 09e144d (the release split between the distro and Windows, wsl-setup, the server config, USER 1000, python:3.15-slim) with roledesk's facts put back: no frontend stage, desk, plugins and seed copied, the extras passkeys, embeddings and llm, roles as the table the backup and the importer require, bin/pull.sh and bin/worker.sh with their profile commands, and Install Server.md and Install WSL.md as copies in place of the five old deployment docs
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
83e965dfeb
commit
a1f281a23a
40 files changed
+2226
-1287
No files matched your search
@@ -12,3 +12,5 @@
|
|||||||
# PowerShell reads LF happily; what it cannot read is a non-ASCII byte, which
|
# PowerShell reads LF happily; what it cannot read is a non-ASCII byte, which
|
||||||
# is a different rule and lives in `tests/test_packaging.py`.
|
# is a different rule and lives in `tests/test_packaging.py`.
|
||||||
deploy/** text eol=lf
|
deploy/** text eol=lf
|
||||||
|
# cmd.exe misparses LF-only batch files, so the double-click launcher stays CRLF.
|
||||||
|
*.cmd text eol=crlf
|
||||||
@@ -76,4 +76,4 @@ The reader, `/roles/{uid}`, shows the full posting: the same vote and Save contr
|
|||||||
- [`plugins/README.md`](plugins/README.md): every reader, its endpoints and what each states; [`docs/filter-survey.md`](docs/filter-survey.md): the filter parameters each host takes at the source.
|
- [`plugins/README.md`](plugins/README.md): every reader, its endpoints and what each states; [`docs/filter-survey.md`](docs/filter-survey.md): the filter parameters each host takes at the source.
|
||||||
- `docs/plan-*.md`: plans in progress, folded into `decisions.md` and deleted when their work lands.
|
- `docs/plan-*.md`: plans in progress, folded into `decisions.md` and deleted when their work lands.
|
||||||
- [`desk/README.md`](desk/README.md), [`pipeline/README.md`](pipeline/README.md), [`auth/README.md`](auth/README.md), [`deploy/README.md`](deploy/README.md): the copied modules, where each comes from and how to refresh it.
|
- [`desk/README.md`](desk/README.md), [`pipeline/README.md`](pipeline/README.md), [`auth/README.md`](auth/README.md), [`deploy/README.md`](deploy/README.md): the copied modules, where each comes from and how to refresh it.
|
||||||
- [`docs/hosting.md`](docs/hosting.md), [`docs/release.md`](docs/release.md), [`docs/server.md`](docs/server.md), [`docs/password-protection.md`](docs/password-protection.md), [`docs/llm-backends.md`](docs/llm-backends.md): the boilerplate's deployment docs.
|
- [`docs/Install Server.md`](<docs/Install Server.md>) and [`docs/Install WSL.md`](<docs/Install WSL.md>): the server and the Windows machine the release runs on, copies of the boilerplate's files; the release steps are in [`deploy/README.md`](deploy/README.md).
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# The backend in the foreground until Ctrl-C: uv run app serve, on :8790. The profile's <name>-backend runs it.
|
||||||
|
# Ours: `-v` and `--config` go before `serve`, where the app takes them, and every other argument
|
||||||
|
# after it (`--reload`, `--port`), so `.run/serve.run.xml`'s `serve --reload` is `<name>-backend --reload`.
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
||||||
|
# A shell started by wsl.exe has not read the rc file that puts uv on PATH.
|
||||||
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
|
before=() after=()
|
||||||
|
while (($#)); do
|
||||||
|
case $1 in
|
||||||
|
-v|--verbose) before+=("$1") ;;
|
||||||
|
-c|--config) before+=("$1" "${2:?$1 needs a file}"); shift ;;
|
||||||
|
--config=*) before+=("$1") ;;
|
||||||
|
*) after+=("$1") ;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
exec uv run app "${before[@]}" serve "${after[@]}"
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# The local gate on the working tree, uncommitted edits included, without promoting:
|
||||||
|
# the steps deploy/test-commit.sh runs on a commit; the profile's <name>-check runs it.
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
||||||
|
exec sh deploy/test-commit.sh --here
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# roledesk's own, beside the boilerplate's backend and check: one pull in the foreground, `app job pull`.
|
||||||
|
# The profile's <name>-pull runs it; `.run/pull.run.xml` is the same command from PyCharm.
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
||||||
|
# A shell started by wsl.exe has not read the rc file that puts uv on PATH.
|
||||||
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
|
exec uv run app job pull "$@"
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# roledesk's own, beside the boilerplate's backend and check: the queue drained with no web app, `app worker`.
|
||||||
|
# The profile's <name>-worker runs it; `.run/worker.run.xml` is the same command from PyCharm.
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
||||||
|
# A shell started by wsl.exe has not read the rc file that puts uv on PATH.
|
||||||
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
|
exec uv run app worker "$@"
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
<#
|
|
||||||
.SYNOPSIS
|
|
||||||
Run steps 1, 2 and 3: test/promote, build image, push image.
|
|
||||||
.DESCRIPTION
|
|
||||||
Stops on the first failure. Server deployment is a separate operation --
|
|
||||||
nothing here connects to the server. Git branch publication is optional
|
|
||||||
through -PushBranch.
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
|
||||||
param([switch] $PushBranch)
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
& "$PSScriptRoot/1-test-and-promote.ps1" -PushBranch:$PushBranch
|
|
||||||
& "$PSScriptRoot/2-build-image.ps1"
|
|
||||||
& "$PSScriptRoot/3-push-image.ps1"
|
|
||||||
@@ -1,90 +0,0 @@
|
|||||||
<#
|
|
||||||
.SYNOPSIS
|
|
||||||
Test the latest local main commit and fast-forward production to it.
|
|
||||||
.DESCRIPTION
|
|
||||||
Uses a fixed commit and an isolated worktree. PushBranch optionally publishes
|
|
||||||
production to origin; without it promotion stays local. Does not run Docker.
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
|
||||||
param([switch] $PushBranch, [switch] $NoTests)
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
. "$PSScriptRoot/_common.ps1"
|
|
||||||
|
|
||||||
$Source = Setting SOURCE 'main'
|
|
||||||
$Branch = Setting BRANCH 'production'
|
|
||||||
$Python = Setting PYTHON (Join-Path $root '.venv\Scripts\python.exe')
|
|
||||||
|
|
||||||
Push-Location $root
|
|
||||||
try {
|
|
||||||
$commit = git rev-parse --verify "$Source^{commit}"
|
|
||||||
Assert-Ok 'resolve source commit'
|
|
||||||
$commit = $commit.Trim()
|
|
||||||
|
|
||||||
# ------------------------------------------------------- test the commit
|
|
||||||
|
|
||||||
# In a throwaway worktree, not here. The tests then run against exactly the
|
|
||||||
# tree that is about to become production, with none of the working copy's
|
|
||||||
# uncommitted edits in the way. Worktrees have no .venv of their own, so the
|
|
||||||
# interpreter comes from the main checkout and the code from the worktree.
|
|
||||||
#
|
|
||||||
# A checkout inside WSL has a Linux .venv, so there the same steps run in
|
|
||||||
# the distro through deploy/test-commit.sh.
|
|
||||||
if (-not $NoTests -and $Wsl.Success) {
|
|
||||||
Write-Step "tests on $Source, in WSL $($Wsl.Groups[1].Value)"
|
|
||||||
wsl.exe -d $Wsl.Groups[1].Value --cd (ConvertTo-LinuxPath $root) -- sh deploy/test-commit.sh $commit
|
|
||||||
Assert-Ok 'tests'
|
|
||||||
} elseif (-not $NoTests) {
|
|
||||||
Write-Step "tests on $Source"
|
|
||||||
if (-not (Test-Path $Python)) {
|
|
||||||
throw "no interpreter at $Python - set PYTHON in deploy\release.env"
|
|
||||||
}
|
|
||||||
$base = Join-Path ([IO.Path]::GetTempPath()) ("$Name-test-" + [guid]::NewGuid().ToString('N').Substring(0, 8))
|
|
||||||
$tree = Join-Path $base $Name
|
|
||||||
try {
|
|
||||||
git worktree add --detach $tree $commit | Out-Null
|
|
||||||
Assert-Ok 'git worktree add'
|
|
||||||
Push-Location $tree
|
|
||||||
try {
|
|
||||||
# Your project's checks. A gate that is red for weeks is not a
|
|
||||||
# gate -- it is a flag you learn to type -- so take one out when
|
|
||||||
# it stops meaning anything, and put it back when it is green.
|
|
||||||
#
|
|
||||||
# The starter's dev group is pytest and httpx, so pytest is the
|
|
||||||
# only gate that can run out of the box. Add ruff and mypy to
|
|
||||||
# [dependency-groups] dev in pyproject.toml, then uncomment the
|
|
||||||
# two lines below -- a gate whose tool is not installed fails
|
|
||||||
# every release for a reason that has nothing to do with the code.
|
|
||||||
& $Python -m pytest -q; Assert-Ok 'pytest'
|
|
||||||
# & $Python -m ruff check .; Assert-Ok 'ruff'
|
|
||||||
# & $Python -m mypy --cache-dir (Join-Path $root '.mypy_cache')
|
|
||||||
# Assert-Ok 'mypy'
|
|
||||||
} finally { Pop-Location }
|
|
||||||
} finally {
|
|
||||||
git worktree remove --force $tree 2>$null | Out-Null
|
|
||||||
Remove-ReleaseTemp @($base)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- promote
|
|
||||||
|
|
||||||
# Fast-forward only. If production has commits main does not, that is a
|
|
||||||
# release made from somewhere else, and this stops rather than discarding it.
|
|
||||||
Write-Step "promote $Source -> $Branch"
|
|
||||||
git show-ref --verify --quiet "refs/heads/$Branch"
|
|
||||||
if ($LASTEXITCODE -eq 0) {
|
|
||||||
git merge-base --is-ancestor $Branch $commit
|
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
Write-Host "$Branch is not an ancestor of $Source - it has commits of its own." -ForegroundColor Red
|
|
||||||
throw "look at: git log --oneline $Source..$Branch"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
# Git publication is optional and independent of the image.
|
|
||||||
git branch -f $Branch $commit; Assert-Ok 'git branch -f'
|
|
||||||
|
|
||||||
if ($PushBranch) {
|
|
||||||
git push origin "${commit}:refs/heads/$Branch"
|
|
||||||
Assert-Ok 'publish production branch'
|
|
||||||
}
|
|
||||||
Write-Host "Promoted $Branch to $commit"
|
|
||||||
} finally { Pop-Location }
|
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Test the latest local main commit, fast-forward production to it, and export
|
||||||
|
# it for the image build on Windows.
|
||||||
|
#
|
||||||
|
# bash deploy/1-test-and-promote.sh [--push-branch] [--no-tests]
|
||||||
|
#
|
||||||
|
# Uses a fixed commit and an isolated worktree. --push-branch optionally
|
||||||
|
# publishes production to origin; without it promotion stays local. --no-tests
|
||||||
|
# skips the gates. The export is the production tree in <RELEASE_DIR>/<folder>/<sha>/
|
||||||
|
# and its sha in <RELEASE_DIR>/<folder>/production, RELEASE_DIR defaulting to
|
||||||
|
# release/ beside the checkout. Uncommitted work is never included.
|
||||||
|
set -euo pipefail
|
||||||
|
# shellcheck source=_common.sh
|
||||||
|
. "$(dirname "${BASH_SOURCE[0]}")/_common.sh"
|
||||||
|
|
||||||
|
push_branch=0 no_tests=0
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
--push-branch) push_branch=1 ;;
|
||||||
|
--no-tests) no_tests=1 ;;
|
||||||
|
-h|--help) sed -n '2,11p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||||
|
*) die "unknown option $arg" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
need_git
|
||||||
|
SOURCE=$(setting SOURCE main)
|
||||||
|
BRANCH=$(setting BRANCH production)
|
||||||
|
PYTHON=$(setting PYTHON "$root/.venv/bin/python")
|
||||||
|
|
||||||
|
cd "$root"
|
||||||
|
commit=$(resolve_commit "$SOURCE" source)
|
||||||
|
|
||||||
|
# ----------------------------------------------------------- test the commit
|
||||||
|
|
||||||
|
# In a throwaway worktree, not here. The tests then run against exactly the tree
|
||||||
|
# that is about to become production, with none of the working copy's
|
||||||
|
# uncommitted edits in the way. test-commit.sh makes the worktree, runs pytest,
|
||||||
|
# ruff, mypy and the client's check in it, and removes it. Worktrees have no
|
||||||
|
# .venv of their own, so the interpreter comes from this checkout.
|
||||||
|
if ((! no_tests)); then
|
||||||
|
step "tests on $SOURCE"
|
||||||
|
run_or_die tests env PYTHON="$PYTHON" sh "$here/test-commit.sh" "$commit"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------- promote
|
||||||
|
|
||||||
|
# Fast-forward only. If production has commits main does not, that is a release
|
||||||
|
# made from somewhere else, and this stops rather than discarding it.
|
||||||
|
step "promote $SOURCE -> $BRANCH"
|
||||||
|
if git show-ref --verify --quiet "refs/heads/$BRANCH"; then
|
||||||
|
if ! git merge-base --is-ancestor "$BRANCH" "$commit"; then
|
||||||
|
echo "$BRANCH is not an ancestor of $SOURCE - it has commits of its own." >&2
|
||||||
|
die "look at: git log --oneline $SOURCE..$BRANCH"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
# Git publication is optional and independent of the image.
|
||||||
|
run_or_die 'git branch -f' git branch -f "$BRANCH" "$commit"
|
||||||
|
|
||||||
|
if ((push_branch)); then
|
||||||
|
run_or_die 'publish production branch' git push origin "$commit:refs/heads/$BRANCH"
|
||||||
|
fi
|
||||||
|
echo "Promoted $BRANCH to $commit"
|
||||||
|
|
||||||
|
# -------------------------------------------------------------------- export
|
||||||
|
|
||||||
|
# The image is built on Windows by wslc (2-build-image.ps1), which reads this
|
||||||
|
# folder through \\wsl.localhost. A tree rather than a tar, so Windows has
|
||||||
|
# nothing to unpack. wslc hands the Dockerfile to the builder on stdin, so the
|
||||||
|
# builder only finds the ignore rules at the root of the context:
|
||||||
|
# Dockerfile.dockerignore is copied there as .dockerignore. A tree that exists
|
||||||
|
# for this sha is kept, not rewritten, because wslc can build from a stale view
|
||||||
|
# of a folder deleted and recreated at the same path (microsoft/WSL#41287).
|
||||||
|
# The promote has happened by now; when the export fails, rerun this step.
|
||||||
|
name=$(basename "$root")
|
||||||
|
release=${RELEASE_DIR:-$(dirname "$root")/release}/$name
|
||||||
|
sha=$(git rev-parse --short "$commit") || die 'resolve image tag failed'
|
||||||
|
tree=$release/$sha
|
||||||
|
step "export $BRANCH ($sha) to $tree"
|
||||||
|
mkdir -p -- "$release" 2>/dev/null && [[ -w $release ]] ||
|
||||||
|
die "cannot write $release; $BRANCH is promoted, rerun this step once it is writable"
|
||||||
|
if [[ ! -d $tree ]]; then
|
||||||
|
part=$release/.part-$sha
|
||||||
|
rm -rf -- "$part"
|
||||||
|
mkdir -- "$part"
|
||||||
|
git archive --format=tar "$commit" | tar -x -C "$part" || die 'export failed'
|
||||||
|
if [[ -f $part/deploy/Dockerfile.dockerignore ]]; then
|
||||||
|
cp -- "$part/deploy/Dockerfile.dockerignore" "$part/.dockerignore"
|
||||||
|
fi
|
||||||
|
mv -- "$part" "$tree"
|
||||||
|
fi
|
||||||
|
for old in "$release"/*/; do
|
||||||
|
[[ ${old%/} == "$tree" ]] || rm -rf -- "${old%/}"
|
||||||
|
done
|
||||||
|
printf '%s\n' "$sha" > "$release/production"
|
||||||
|
echo "Exported $tree"
|
||||||
+12
-56
@@ -1,57 +1,13 @@
|
|||||||
<#
|
# Step 2, on Windows: build the image from the tree step 1 exported, with wslc.
|
||||||
.SYNOPSIS
|
# Tags <IMAGE>:<sha> and <IMAGE>:latest. Never pushes; 3-push-image.ps1 does.
|
||||||
Build a local Docker image from the committed production branch.
|
|
||||||
.DESCRIPTION
|
|
||||||
Does not test, promote, push Git branches, use SSH, or start a container.
|
|
||||||
The image stays local. Use 3-push-image.ps1 to publish it separately.
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
|
||||||
param()
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Stop'
|
$ErrorActionPreference = 'Stop'
|
||||||
. "$PSScriptRoot/_common.ps1"
|
. "$PSScriptRoot\_common.ps1"
|
||||||
|
Assert-Wslc
|
||||||
$Image = Setting IMAGE -Required
|
$sha = Get-ReleaseSha
|
||||||
$Branch = Setting BRANCH 'production'
|
$context = Join-Path $Release $sha
|
||||||
$Platform = Setting PLATFORM 'linux/amd64'
|
Write-Host "== build ${Image}:$sha from $context" -ForegroundColor Cyan
|
||||||
|
# -f is a path wslc opens itself, not one inside the context.
|
||||||
Push-Location $root
|
wslc build -t "${Image}:$sha" -t "${Image}:latest" -f "$context\deploy\Dockerfile" $context
|
||||||
try {
|
if ($LASTEXITCODE -ne 0) { throw "wslc build failed (exit $LASTEXITCODE)" }
|
||||||
$commit = git rev-parse --verify "$Branch^{commit}"
|
wslc image list
|
||||||
Assert-Ok 'resolve production commit'
|
Write-Host "Image: ${Image}:$sha"
|
||||||
$commit = $commit.Trim()
|
|
||||||
$sha = (git rev-parse --short $commit).Trim()
|
|
||||||
Assert-Ok 'resolve image tag'
|
|
||||||
$tagged = "${Image}:${sha}"
|
|
||||||
$latest = "${Image}:latest"
|
|
||||||
|
|
||||||
# --------------------------------------------------------------- build
|
|
||||||
|
|
||||||
# `git archive` to a file and unpack it, rather than piping it. A PowerShell
|
|
||||||
# pipeline carries text: it would re-encode a tar stream and hand docker a
|
|
||||||
# corrupted context. Writing the archive out and building from the directory
|
|
||||||
# keeps the same guarantee -- the context is the committed tree and nothing
|
|
||||||
# else -- without a byte going through a pipe.
|
|
||||||
#
|
|
||||||
# The Dockerfile comes out of that same tree, at deploy/Dockerfile, so the
|
|
||||||
# recipe is versioned with the code it builds. Docker reads the ignore rules
|
|
||||||
# from deploy/Dockerfile.dockerignore beside it.
|
|
||||||
|
|
||||||
Write-Step "build $tagged"
|
|
||||||
$ctx = Join-Path ([IO.Path]::GetTempPath()) "$Name-build-$sha"
|
|
||||||
$tar = "$ctx.tar"
|
|
||||||
Remove-ReleaseTemp @($ctx, $tar)
|
|
||||||
New-Item -ItemType Directory -Path $ctx | Out-Null
|
|
||||||
try {
|
|
||||||
git archive --format=tar -o $tar $commit; Assert-Ok 'git archive'
|
|
||||||
tar -xf $tar -C $ctx; Assert-Ok 'tar -xf'
|
|
||||||
$dockerfile = Join-Path $ctx 'deploy/Dockerfile'
|
|
||||||
docker build --platform $Platform -f $dockerfile -t $tagged -t $latest $ctx
|
|
||||||
Assert-Ok 'docker build'
|
|
||||||
docker image ls $Image --format ' {{.Tag}} {{.Size}}'
|
|
||||||
} finally {
|
|
||||||
Remove-ReleaseTemp @($ctx, $tar)
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host "Image: $tagged"
|
|
||||||
} finally { Pop-Location }
|
|
||||||
+19
-35
@@ -1,36 +1,20 @@
|
|||||||
<#
|
# Step 3, on Windows: push the image step 2 built, <IMAGE>:<sha> and :latest,
|
||||||
.SYNOPSIS
|
# with wslc. Never rebuilds: after a failed push, fix the login and run it again.
|
||||||
Publish the already-built production image to the registry.
|
|
||||||
.DESCRIPTION
|
|
||||||
Never rebuilds. If the push fails, fix access or connectivity and run this
|
|
||||||
again; the image is already on disk. `docker login` first, with a token
|
|
||||||
that has package-write access.
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
|
||||||
param()
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Stop'
|
$ErrorActionPreference = 'Stop'
|
||||||
. "$PSScriptRoot/_common.ps1"
|
. "$PSScriptRoot\_common.ps1"
|
||||||
|
Assert-Wslc
|
||||||
$Image = Setting IMAGE -Required
|
$sha = Get-ReleaseSha
|
||||||
$Branch = Setting BRANCH 'production'
|
wslc image inspect "${Image}:$sha" | Out-Null
|
||||||
|
if ($LASTEXITCODE -ne 0) { throw "no local ${Image}:$sha - run step 2 first" }
|
||||||
Push-Location $root
|
# latest must name this candidate, even if another image was built since.
|
||||||
try {
|
wslc tag "${Image}:$sha" "${Image}:latest"
|
||||||
$commit = git rev-parse --verify "$Branch^{commit}"
|
if ($LASTEXITCODE -ne 0) { throw "wslc tag failed (exit $LASTEXITCODE)" }
|
||||||
Assert-Ok 'resolve production commit'
|
foreach ($tag in $sha, 'latest') {
|
||||||
$sha = (git rev-parse --short $commit.Trim()).Trim()
|
Write-Host "== push ${Image}:$tag" -ForegroundColor Cyan
|
||||||
Assert-Ok 'resolve image tag'
|
wslc push "${Image}:$tag"
|
||||||
$tagged = "${Image}:${sha}"
|
if ($LASTEXITCODE -ne 0) {
|
||||||
$latest = "${Image}:latest"
|
Write-Host "If the push was refused: wslc login $(($Image -split '/')[0]) -u <user>, with a token that has package-write access" -ForegroundColor Yellow
|
||||||
docker image inspect $tagged --format '{{.Id}}' | Out-Null
|
throw "wslc push ${Image}:$tag failed (exit $LASTEXITCODE)"
|
||||||
Assert-Ok "find local $tagged (run step 2 first)"
|
}
|
||||||
# latest must name this candidate, even if another image was built since.
|
}
|
||||||
docker tag $tagged $latest
|
Write-Host "Published ${Image}:$sha and ${Image}:latest"
|
||||||
Assert-Ok 'tag latest image'
|
|
||||||
docker push $tagged
|
|
||||||
Assert-Ok 'push commit image'
|
|
||||||
docker push $latest
|
|
||||||
Assert-Ok 'push latest image'
|
|
||||||
Write-Host "Published $tagged and $latest"
|
|
||||||
} finally { Pop-Location }
|
|
||||||
+50
-16
@@ -1,21 +1,28 @@
|
|||||||
# A Python app, for a server rather than a laptop.
|
# A Python app, for a server rather than a laptop.
|
||||||
#
|
#
|
||||||
# Two stages so the runtime carries no build tooling: `uv` resolves and
|
# Two stages so the runtime carries no build tooling: `uv` resolves and
|
||||||
# installs into /app/.venv, and the second stage copies that venv and the
|
# installs into /app/.venv, and the second stage copies the venv and the source
|
||||||
# source next to it. The install is deliberately editable, which matters if
|
# next to it. The install is deliberately editable, which matters because the
|
||||||
# the app reads a file from beside its package (an example config, a template
|
# app reads files from beside its package (templates, `app/static`, `seed/`):
|
||||||
# directory): that path is true in a checkout and true here, and false in
|
# that path is true in a checkout and true here, and false in site-packages.
|
||||||
# site-packages.
|
# Ours: roledesk has no `frontend/` and no Node stage, since the server renders
|
||||||
|
# Jinja templates and serves `app/static` as it is.
|
||||||
#
|
#
|
||||||
# Already in place at deploy/Dockerfile, with its ignore file beside it.
|
# Already in place at deploy/Dockerfile, with its ignore file beside it.
|
||||||
# 2-build-image.ps1 reads it out of a `git archive` of the production commit,
|
# 2-build-image.ps1 builds with it from the `git archive` of the production
|
||||||
# so the recipe is versioned with the code it builds.
|
# commit that step 1 exports, so the recipe is versioned with the code it builds.
|
||||||
#
|
#
|
||||||
# The package is `app` and stays `app` — a project started from this repo
|
# The package is `app` and stays `app` (the domain noun is `role`). Edit the
|
||||||
# renames nothing. Edit the two `uv sync` lines when you add extras.
|
# two `uv sync` lines when you add extras.
|
||||||
|
#
|
||||||
|
# Each base image is pinned by tag and digest: the tag says which release, the
|
||||||
|
# digest makes a rebuild next month install the same bytes. The digest is the
|
||||||
|
# multi-platform index's. To move on, look up the new tag's digest on its
|
||||||
|
# registry (Docker Hub's tag API for python, ghcr.io for uv) and
|
||||||
|
# replace both halves.
|
||||||
|
|
||||||
ARG PYTHON=3.14-slim
|
ARG PYTHON=3.15-slim@sha256:2da676046630d7bf14c6764ae1143dc7fa0377a53f94bf5a642f0b214aca5bce
|
||||||
ARG UV=0.9.17
|
ARG UV=0.13.1@sha256:4a1f3666816dafadd20ceb71eac9a7d56d7ef30279f7d64048519ab901ada336
|
||||||
|
|
||||||
FROM ghcr.io/astral-sh/uv:${UV} AS uv
|
FROM ghcr.io/astral-sh/uv:${UV} AS uv
|
||||||
|
|
||||||
@@ -32,22 +39,38 @@ WORKDIR /app
|
|||||||
|
|
||||||
# Dependencies first and on their own layer, keyed on the lock file alone, so
|
# Dependencies first and on their own layer, keyed on the lock file alone, so
|
||||||
# editing a Python file rebuilds in seconds rather than re-resolving the tree.
|
# editing a Python file rebuilds in seconds rather than re-resolving the tree.
|
||||||
|
# The extras are the same on both `uv sync` lines: `passkeys`, without which
|
||||||
|
# the passkey routes answer 501, `embeddings`, the default ranking backend
|
||||||
|
# (torch comes from the CPU index named in pyproject.toml, [tool.uv.sources],
|
||||||
|
# not the CUDA build; the model itself is downloaded by the first `embed` job
|
||||||
|
# into HOME, which is the /data volume), and `llm`, the SDKs the compose file
|
||||||
|
# passes keys for.
|
||||||
COPY pyproject.toml uv.lock README.md ./
|
COPY pyproject.toml uv.lock README.md ./
|
||||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||||
uv sync --frozen --no-dev --no-install-project
|
uv sync --frozen --no-dev --no-install-project \
|
||||||
|
--extra passkeys --extra embeddings --extra llm
|
||||||
|
|
||||||
COPY app app
|
COPY app app
|
||||||
# The two module folders this repo ships beside the app, here and again in the
|
# The module folders this repo ships beside the app, here and again in the
|
||||||
# runtime stage below. `pyproject.toml` lists `pipeline*` and `auth*` in its
|
# runtime stage below. `pyproject.toml` lists `pipeline*`, `auth*`, `llm*` and
|
||||||
# packages as well, or the install leaves them out.
|
# `desk*` in its packages as well, or the install leaves them out.
|
||||||
COPY pipeline pipeline
|
COPY pipeline pipeline
|
||||||
# The door. A project that does not want one drops this line *and* the
|
# The door. A project that does not want one drops this line *and* the
|
||||||
# `auth.connect` call in `app/main.py` together: dropping the folder and
|
# `auth.connect` call in `app/main.py` together: dropping the folder and
|
||||||
# leaving the call is a container that fails at startup, and dropping the call
|
# leaving the call is a container that fails at startup, and dropping the call
|
||||||
# and leaving the folder is a container that starts with the door open.
|
# and leaving the folder is a container that starts with the door open.
|
||||||
COPY auth auth
|
COPY auth auth
|
||||||
|
# The model call. Nothing imports it until an app adds its SDKs, and then the
|
||||||
|
# folder has to be here already, so that is not a Dockerfile edit.
|
||||||
|
COPY llm llm
|
||||||
|
# Ours: the source contract and the generic desk modules, the plugin folder the
|
||||||
|
# config's `plugin_paths` names, and the data a first run imports (`seed/`).
|
||||||
|
COPY desk desk
|
||||||
|
COPY plugins plugins
|
||||||
|
COPY seed seed
|
||||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||||
uv sync --frozen --no-dev
|
uv sync --frozen --no-dev \
|
||||||
|
--extra passkeys --extra embeddings --extra llm
|
||||||
|
|
||||||
# ---------------------------------------------------------------- runtime
|
# ---------------------------------------------------------------- runtime
|
||||||
|
|
||||||
@@ -69,6 +92,10 @@ COPY pipeline pipeline
|
|||||||
# The login page and the two scripts are files inside this folder, served from
|
# The login page and the two scripts are files inside this folder, served from
|
||||||
# routes; see the build stage above.
|
# routes; see the build stage above.
|
||||||
COPY auth auth
|
COPY auth auth
|
||||||
|
COPY llm llm
|
||||||
|
COPY desk desk
|
||||||
|
COPY plugins plugins
|
||||||
|
COPY seed seed
|
||||||
|
|
||||||
ENV PATH=/app/.venv/bin:$PATH \
|
ENV PATH=/app/.venv/bin:$PATH \
|
||||||
PYTHONUNBUFFERED=1 \
|
PYTHONUNBUFFERED=1 \
|
||||||
@@ -79,6 +106,13 @@ ENV PATH=/app/.venv/bin:$PATH \
|
|||||||
# /data, and a CLI the app shells out to — the `claude` binary, for one —
|
# /data, and a CLI the app shells out to — the `claude` binary, for one —
|
||||||
# wants somewhere to write. An image-baked /home/app would be right for uid
|
# wants somewhere to write. An image-baked /home/app would be right for uid
|
||||||
# 1000 and unwritable for anyone else.
|
# 1000 and unwritable for anyone else.
|
||||||
|
#
|
||||||
|
# The image runs as 1000:1000, the compose file's default PUID:PGID and the
|
||||||
|
# owner docs/Install Server.md gives the data folder, so a `docker run` without a
|
||||||
|
# `user:` does not write root-owned files into it. /data is made here, before
|
||||||
|
# VOLUME, so an anonymous volume starts out owned by that uid too.
|
||||||
|
RUN install -d -o 1000 -g 1000 /data /data/home
|
||||||
|
USER 1000:1000
|
||||||
VOLUME /data
|
VOLUME /data
|
||||||
|
|
||||||
EXPOSE 8787
|
EXPOSE 8787
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
# Named for the Dockerfile beside it: Docker reads `<dockerfile>.dockerignore`
|
# Named for the Dockerfile beside it. Docker reads `<dockerfile>.dockerignore`
|
||||||
# before falling back to one at the root of the context, which is how the ignore
|
# before falling back to one at the root of the context, but wslc hands the
|
||||||
# rules get to live next to the recipe they belong to instead of at the top of
|
# Dockerfile to the builder on stdin, which leaves only the root one: step 1
|
||||||
# the repository.
|
# (1-test-and-promote.sh) copies this file to `.dockerignore` at the root of
|
||||||
|
# the tree it exports, so the rules still live next to the recipe here.
|
||||||
#
|
#
|
||||||
# 2-build-image.ps1 already hands docker a context extracted from `git archive`,
|
# That tree is a `git archive` of the production commit, so the untracked
|
||||||
# so the untracked heavyweights below are not there to exclude. They are listed
|
# heavyweights below are not there to exclude. They are listed for the other
|
||||||
# for the other way in — `docker build -f deploy/Dockerfile .` from a checkout —
|
# way in, `docker build -f deploy/Dockerfile .` from a checkout, where .venv
|
||||||
# where .venv alone can be several hundred megabytes.
|
# alone can be several hundred megabytes.
|
||||||
.git
|
.git
|
||||||
.venv
|
.venv
|
||||||
.claude
|
.claude
|
||||||
@@ -19,6 +20,7 @@ docs
|
|||||||
# per folder, so a fourth module is a fourth line here.
|
# per folder, so a fourth module is a fourth line here.
|
||||||
tests
|
tests
|
||||||
pipeline/tests
|
pipeline/tests
|
||||||
|
llm/tests
|
||||||
auth/tests
|
auth/tests
|
||||||
|
|
||||||
__pycache__
|
__pycache__
|
||||||
|
|||||||
+45
-21
@@ -1,52 +1,76 @@
|
|||||||
# Deploy
|
# Deploy
|
||||||
|
|
||||||
Everything needed to put this app on the rittershofer.org box behind SWAG: build an image on Windows, publish it to Gitea, run it as one more service in the compose file that already runs the proxy, back up its database, and give it a password that survives being on the open internet.
|
Everything needed to put roledesk on a server behind SWAG ([../docs/Install Server.md](../docs/Install%20Server.md) is the box it was written for, a copy of the boilerplate's file): test it in WSL, build an image on Windows with `wslc`, publish it to Gitea, run it as one more service in the compose file that already runs the proxy, back up its database, and give it a password that survives being on the open internet.
|
||||||
|
|
||||||
Extracted from newsdesk, which is the worked example — every decision here has been run in production at least once, and the comments say why rather than what.
|
This folder is a downstream copy of `../boilerplate/deploy` (commit 09e144d), refreshed with `git archive`; a fix to a shared script lands upstream first. roledesk's own facts are put back after a refresh: the Dockerfile without a frontend stage and with `desk`, `plugins`, `llm` and `seed`, the extras `passkeys`, `embeddings` and `llm`, our image name in `release.env.example`, `server/config.example.yaml` for roledesk's keys, `TABLES` and `REQUIRED_TABLES` naming `roles`, and `bin/pull.sh` and `bin/worker.sh`. `tests/test_deploy.py` pins them.
|
||||||
|
|
||||||
**It is already in place.** Nothing has to be copied and nothing has to be renamed: the package is `app`, the environment variables are `APP_`, and the files below are where the scripts expect them. `<app>` in the docs is your *service's* name — the hostname, the data directory, the image tag, the compose service — and it is the one thing you choose.
|
The package is `app`, and the files below are where the scripts expect them: the `APP_` variables, the compose service `app` and the hostname `app.example.com` in the templates are placeholders for the server, and `<app>` in the docs is the service's name: the hostname, the data directory, the image tag, the compose service. The domain noun is `role`, so "job" only ever means queued work.
|
||||||
|
|
||||||
## Where each file is
|
## Where each file is
|
||||||
|
|
||||||
| Here | | |
|
| Here | | |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `deploy/*.ps1` | the release scripts, run on Windows | |
|
| `deploy/1-test-and-promote.sh`, `test-commit.sh`, `_common.sh`, `db.sh` | the gate, the export and the database moves, run in the WSL distro | |
|
||||||
| `deploy/Dockerfile` | read out of the archived commit by step 2 | copies `app`, `pipeline` and `auth` |
|
| `deploy/2-build-image.ps1`, `3-push-image.ps1`, `_common.ps1` | the image build and push, run on Windows with `wslc` | the distro is a sealed sandbox and does not hold a container socket |
|
||||||
| `deploy/Dockerfile.dockerignore` | Docker finds it by that name | |
|
| `deploy/wsl-setup.cmd`, `wsl-setup.ps1` | the profile commands, written once per checkout on Windows | |
|
||||||
|
| `bin/backend.sh`, `check.sh`, and ours `pull.sh`, `worker.sh` | the dev commands, run in the WSL distro: the backend, the gate on the working tree, a pull, the worker | |
|
||||||
|
| `.run/serve`, `pull`, `worker` | PyCharm run configurations, left as they are | `serve --reload` = `bin/backend.sh --reload` = `<folder>-backend --reload`; `pull` = `bin/pull.sh` = `<folder>-pull`; `worker` = `bin/worker.sh` = `<folder>-worker` |
|
||||||
|
| `deploy/Dockerfile` | read out of the tree step 1 exports, by step 2 | no frontend stage; copies `app`, `pipeline`, `auth`, `llm`, `desk`, `plugins` and `seed` |
|
||||||
|
| `deploy/Dockerfile.dockerignore` | copied to `.dockerignore` at the root of the exported tree | `wslc` streams the Dockerfile, so only a root ignore file applies |
|
||||||
| `deploy/server/*` | committed here, then installed on the server | |
|
| `deploy/server/*` | committed here, then installed on the server | |
|
||||||
| `deploy/server/compose.yaml` | to `/srv/swag/docker-compose.yml`, as one `services:` block | |
|
| `deploy/server/compose.yaml` | to `/srv/swag/docker-compose.yml`, as one `services:` block | |
|
||||||
| `deploy/server/env.example` | to `/srv/swag/.env`, as appended variables, `chmod 600` | |
|
| `deploy/server/env.example` | to `/srv/swag/.env`, as appended variables, `chmod 600` | |
|
||||||
|
| `deploy/server/config.example.yaml` | to `$APP_DATA/config.yaml`, with the hostname filled in | `deploy.sh` refuses a config without `database` and `auth.enabled` |
|
||||||
| `deploy/server/deploy.sh`, `deploy/server/backup.sh` | to `/srv/swag/`, next to the compose file | |
|
| `deploy/server/deploy.sh`, `deploy/server/backup.sh` | to `/srv/swag/`, next to the compose file | |
|
||||||
|
| `deploy/server/import.sh`, `deploy/server/import_database.py` | uploaded and run by `db.sh push` | |
|
||||||
| `deploy/server/swag/app.subdomain.conf` | to SWAG's `proxy-confs/`, renamed to `<app>.subdomain.conf` | |
|
| `deploy/server/swag/app.subdomain.conf` | to SWAG's `proxy-confs/`, renamed to `<app>.subdomain.conf` | |
|
||||||
|
|
||||||
`db.ps1` expects `server/` beside it, which is why `deploy/server/` is committed here as well as installed on the server.
|
`db.sh` expects `server/` beside it, which is why `deploy/server/` is committed here as well as installed on the server.
|
||||||
|
|
||||||
**Keep the `.ps1` files to ASCII.** Windows PowerShell 5.1 reads a script without a byte-order mark as Windows-1252, so an em dash becomes three characters ending in a closing quote: the string ends in the middle of itself and the script fails to parse before it runs a line. Write `--`. newsdesk hit this on a comment nobody thought was code, and `tests/test_packaging.py` here holds every script in this folder to it.
|
**Keep every `.ps1` to ASCII.** Windows PowerShell 5.1 reads a script without a byte-order mark as Windows-1252, so an em dash becomes three characters ending in a closing quote: the string ends in the middle of itself and the script fails to parse before it runs a line. Write `--`. newsdesk hit this on a comment nobody thought was code, and `tests/test_packaging.py` here holds every `.ps1` in this folder to it, the release scripts included.
|
||||||
|
|
||||||
|
`wsl-setup.sh` and `wsl-setup.ps1` are the two halves of setting up a checkout that lives in WSL. `wsl-setup.sh` runs in the distro ([../docs/Install WSL.md, Git](../docs/Install%20WSL.md#git)). `wsl-setup.ps1`, or a double-click on `deploy\wsl-setup.cmd`, writes the commands below into the PowerShell profile, with `<folder>` the checkout folder's name (`jobboard` here) or `-Name`. PowerShell 7, the execution policy, `wslc`, the hand-paste fallback and what to check when a command fails are in [../docs/Install WSL.md, PS Profile commands](../docs/Install%20WSL.md#ps-profile-commands).
|
||||||
|
|
||||||
|
| Command | Runs | Does |
|
||||||
|
|---|---|---|
|
||||||
|
| `<folder>-cd` | `Set-Location` | PowerShell into the checkout's folder |
|
||||||
|
| `<folder>-sh` | `wsl.exe` | a shell in the distro, in the checkout |
|
||||||
|
| `<folder>-backend [-v] [--reload]` | bin/backend.sh in WSL | `uv run app serve` on :8790 |
|
||||||
|
| `<folder>-pull [args]` | bin/pull.sh in WSL | `uv run app job pull`, one pull in the foreground (roledesk's) |
|
||||||
|
| `<folder>-worker [args]` | bin/worker.sh in WSL | `uv run app worker`, the queue drained with no web app (roledesk's) |
|
||||||
|
| `<folder>-check` | bin/check.sh in WSL | the gate on the working tree, without promoting |
|
||||||
|
| `<folder>-1test [--push-branch] [--no-tests]` | deploy/1-test-and-promote.sh in WSL | test local main, fast-forward `production`, export it |
|
||||||
|
| `<folder>-2build` | deploy\2-build-image.ps1 on Windows | build the exported tree with `wslc` |
|
||||||
|
| `<folder>-3push` | deploy\3-push-image.ps1 on Windows | push the built image with `wslc` |
|
||||||
|
| `<folder>-0release [--push-branch] [--no-tests]` | the three above | steps 1, 2 and 3 in order, stopping at the first failure |
|
||||||
|
| `<folder>-db pull\|push` | deploy/db.sh in WSL | the database, to and from the server |
|
||||||
|
|
||||||
|
Each command is one line in the profile: `wsl.exe -d <DISTRO> --cd <CHECKOUT> -- bash <script> @args` for the ones that run in the distro, and for steps 2 and 3 the `.ps1` with the execution policy lifted for that run. `DISTRO` and `CHECKOUT` are read from `deploy/release.env` when the setup runs, so rerun it after either changes. `-backend`, `-pull` and `-worker` stay in the foreground until Ctrl-C. The `-0release` to `-3push` names carry the number of the script's file name, so `0<Tab>` after `<folder>-` finds them.
|
||||||
|
|
||||||
`.gitignore` already covers `deploy/release.env` and `deploy/backups/`.
|
`.gitignore` already covers `deploy/release.env` and `deploy/backups/`.
|
||||||
|
|
||||||
## The docs
|
## The docs
|
||||||
|
|
||||||
They are one directory up, in [../docs/](../docs), because they are about the hosting rather than about these scripts.
|
They are one directory up, in [../docs/](../docs), because they are about the hosting rather than about these scripts. `Install Server.md` and `Install WSL.md` are copies of the boilerplate's `documentation/` files; the two proposals stay in the boilerplate checkout.
|
||||||
|
|
||||||
- **[../docs/hosting.md](../docs/hosting.md)** — what the box is, the seven steps to add a service, what the templates already decide for you, and the things that have already gone wrong on it. Start here for a service that is not on the box yet.
|
- **[../docs/Install Server.md](../docs/Install%20Server.md)** — what the box is, the seven steps to add a service, what the templates already decide for you, and the things that have already gone wrong on it. Start here for a service that is not on the box yet.
|
||||||
- **[../docs/release.md](../docs/release.md)** — the Windows side: `release.env`, the four numbered scripts, what each one refuses to do, and why the tests run in a throwaway worktree.
|
- **[../docs/Install Server.md, Publishing a release](../docs/Install%20Server.md#publishing-a-release)** — the release side: `release.env`, the numbered scripts and where each runs, what each one refuses to do, and why the tests run in a throwaway worktree.
|
||||||
- **[../docs/server.md](../docs/server.md)** — installing, deploying, backups, moving a SQLite database in either direction, rolling back, and what to verify before relying on any of it.
|
- **[../docs/Install Server.md, Operating it](../docs/Install%20Server.md#operating-it)** — installing, deploying, backups, moving a SQLite database in either direction, rolling back, and what to verify before relying on any of it.
|
||||||
- **[../docs/llm-backends.md](../docs/llm-backends.md)** — asking Claude from a deployed app: API key or subscription token, selected by which one is in the environment, with a self-contained reference implementation.
|
- **[Proposal LLM Backends.md](../../boilerplate/documentation/Proposal%20LLM%20Backends.md)** (boilerplate checkout) — asking Claude from a deployed app: API key or subscription token, selected by which one is in the environment. `llm/` is the implementation of it, see [../llm/README.md](../llm/README.md).
|
||||||
- **[../docs/password-protection.md](../docs/password-protection.md)** — one password on a public hostname: the lockout, sessions, passkeys, and how much throttling is actually worth building. `auth/` is the implementation of it.
|
- **[Proposal Password Protection.md](../../boilerplate/documentation/Proposal%20Password%20Protection.md)** (boilerplate checkout) — one password on a public hostname: the lockout, sessions, passkeys, and how much throttling is actually worth building. `auth/` is the implementation of it, see [../auth/README.md](../auth/README.md).
|
||||||
|
|
||||||
## A new service, in order
|
## A new service, in order
|
||||||
|
|
||||||
1. DNS, then certificate, then registry login, then data directory — [../docs/hosting.md](../docs/hosting.md).
|
1. DNS, then certificate, then registry login, then data directory — [../docs/Install Server.md](../docs/Install%20Server.md).
|
||||||
2. Copy `release.env.example` to `release.env` and fill it in. That is the only file this step touches: everything else is already committed.
|
2. Copy `release.env.example` to `release.env` and fill it in. That is the only file this step touches: everything else is already committed.
|
||||||
3. `.\deploy\0-release.ps1` to test, promote, build and publish.
|
3. `<folder>-0release` from Windows to test, promote, build and publish. `wslc` and its registry login are on Windows, [../docs/Install Server.md, Publishing a release](../docs/Install%20Server.md#publishing-a-release).
|
||||||
4. Install the server files, edit `.env`, `bash deploy.sh` — [../docs/server.md](../docs/server.md).
|
4. Install the server files and `config.yaml`, edit `.env`, `bash deploy.sh` — [../docs/Install Server.md, Operating it](../docs/Install%20Server.md#operating-it).
|
||||||
5. Put the model credential in `.env` if the app needs one — [../docs/llm-backends.md](../docs/llm-backends.md).
|
5. Put the model credential in `.env` if the app needs one — [../llm/README.md](../llm/README.md).
|
||||||
6. Get in, rotate the bootstrap password, register a passkey — [../docs/password-protection.md](../docs/password-protection.md).
|
6. Get in, rotate the bootstrap password, register a passkey — [../auth/README.md](../auth/README.md).
|
||||||
7. Schedule `backup.sh`, then restore one of its backups somewhere else to prove it works.
|
7. Schedule `backup.sh`, then restore one of its backups somewhere else to prove it works.
|
||||||
|
|
||||||
Step 1 of `0-release.ps1` runs `pytest` in a throwaway worktree, with `ruff` and `mypy` commented out beside it: this repository's dev group is pytest and httpx, and a gate whose tool is not installed fails every release for a reason that has nothing to do with the code. Add the tools, then uncomment the gates.
|
Step 1 runs `pytest` in a throwaway worktree through `test-commit.sh`, with `ruff` and `mypy` commented out beside it, as the release has always run. The dev group has both, so uncommenting them is the owner's call: the tree has to pass them first.
|
||||||
|
|
||||||
## What this is not
|
## What this is not
|
||||||
|
|
||||||
Not a framework and not a generator: nothing here is imported or extended, it is copied and edited. That is the point — three services with three edited copies beats three services sharing one abstraction that has to be right for all of them. What this repository changes is only *where the copy comes from*: `boilerplate/deploy/` is the source of truth, and news and health hold copies of it.
|
Not a framework: nothing here is imported or extended, it is copied and edited. `boilerplate/deploy/` is the source of truth, and news, health, jobboard and pocketmoney hold copies of it.
|
||||||
+22
-66
@@ -1,76 +1,32 @@
|
|||||||
<#
|
# Dot-sourced by 2-build-image.ps1 and 3-push-image.ps1. Reads deploy\release.env
|
||||||
.SYNOPSIS
|
# beside it (KEY=VALUE lines, # comments, never run as code) into $Settings,
|
||||||
Shared plumbing for the release scripts. Dot-source it; do not run it.
|
# and works out where step 1 exported the release: release\<folder> beside the
|
||||||
|
# checkout, in the distro, seen from Windows through \\wsl.localhost.
|
||||||
.DESCRIPTION
|
|
||||||
Dot-sourcing runs in the caller's scope, so everything defined here --
|
|
||||||
$root, $settings, Setting, Assert-Ok, Write-Step, Remove-ReleaseTemp -- is
|
|
||||||
the caller's afterwards. One copy of the release.env reader, because three
|
|
||||||
copies is three places for it to drift.
|
|
||||||
#>
|
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Stop'
|
$ErrorActionPreference = 'Stop'
|
||||||
|
|
||||||
#: The repository root, worked out from this file rather than from the working
|
|
||||||
#: directory, so the scripts can be run from anywhere.
|
|
||||||
$root = Split-Path -Parent $PSScriptRoot
|
|
||||||
|
|
||||||
#: A checkout inside WSL, run from Windows as \\wsl.localhost\<distro>\... Its
|
|
||||||
#: .venv is a Linux one, so the steps that run Python hand that part to the
|
|
||||||
#: distro. $Wsl.Groups[1] is the distro; ConvertTo-LinuxPath turns a path under
|
|
||||||
#: the checkout into the distro's own. See docs/wsl.md.
|
|
||||||
$WslPattern = '^\\\\wsl(?:\.localhost|\$)\\([^\\]+)(\\.*)$'
|
|
||||||
$Wsl = [regex]::Match($root, $WslPattern)
|
|
||||||
function ConvertTo-LinuxPath([string] $Path) {
|
|
||||||
[regex]::Match($Path, $WslPattern).Groups[2].Value -replace '\\', '/'
|
|
||||||
}
|
|
||||||
|
|
||||||
function Assert-Ok([string] $What) {
|
|
||||||
if ($LASTEXITCODE -ne 0) { throw "$What failed (exit $LASTEXITCODE)" }
|
|
||||||
}
|
|
||||||
|
|
||||||
function Write-Step([string] $Text) {
|
|
||||||
Write-Host ''
|
|
||||||
Write-Host "== $Text" -ForegroundColor Cyan
|
|
||||||
}
|
|
||||||
|
|
||||||
$envFile = Join-Path $PSScriptRoot 'release.env'
|
$envFile = Join-Path $PSScriptRoot 'release.env'
|
||||||
if (-not (Test-Path $envFile)) {
|
if (-not (Test-Path $envFile)) { throw "no $envFile - copy release.env.example and fill it in" }
|
||||||
throw "no deploy\release.env - copy deploy\release.env.example and fill it in"
|
$Settings = @{}
|
||||||
}
|
|
||||||
$settings = @{}
|
|
||||||
foreach ($line in Get-Content $envFile) {
|
foreach ($line in Get-Content $envFile) {
|
||||||
if ($line -match '^\s*(#|$)') { continue }
|
if ($line -match '^\s*(#|$)') { continue }
|
||||||
# A line with no `=` is a typo, not a setting. Say so here rather than
|
|
||||||
# letting it surface later as a null reference three functions away.
|
|
||||||
if ($line -notmatch '=') { throw "release.env: not a KEY=VALUE line: $line" }
|
|
||||||
$key, $value = $line -split '=', 2
|
$key, $value = $line -split '=', 2
|
||||||
$settings[$key.Trim()] = $value.Trim().Trim('"')
|
$Settings[$key.Trim()] = "$value".Trim().Trim('"')
|
||||||
}
|
}
|
||||||
|
foreach ($key in 'DISTRO', 'CHECKOUT', 'IMAGE') {
|
||||||
function Setting([string] $Name, [string] $Default, [switch] $Required) {
|
if (-not $Settings[$key]) { throw "set $key in $envFile" }
|
||||||
if ($settings.ContainsKey($Name) -and $settings[$Name]) { return $settings[$Name] }
|
|
||||||
if ($Required) { throw "set $Name in deploy\release.env" }
|
|
||||||
return $Default
|
|
||||||
}
|
}
|
||||||
|
$Image = $Settings.IMAGE
|
||||||
|
$Checkout = $Settings.CHECKOUT.TrimEnd('/')
|
||||||
|
$Release = "\\wsl.localhost\$($Settings.DISTRO)" + ($Checkout -replace '/[^/]+$', '' -replace '/', '\') + '\release\' + ($Checkout -replace '^.*/', '')
|
||||||
|
|
||||||
#: What temporary directories this release is allowed to delete. Everything
|
function Assert-Wslc {
|
||||||
#: below is created by these scripts and named for it.
|
if (-not (Get-Command wslc -ErrorAction SilentlyContinue)) {
|
||||||
$Name = Setting NAME -Required
|
throw 'wslc is not on PATH - run wsl --update (WSL 2.9.3 or later), see https://learn.microsoft.com/windows/wsl/tutorials/wsl-containers'
|
||||||
|
|
||||||
# Only ever deletes a <name>-* entry directly under the temp directory. A
|
|
||||||
# guard, not a convenience: these paths are built by string concatenation and
|
|
||||||
# handed to `Remove-Item -Recurse -Force`, and one empty variable in that
|
|
||||||
# sentence is a very bad afternoon.
|
|
||||||
function Remove-ReleaseTemp([string[]] $Paths) {
|
|
||||||
$tempRoot = [IO.Path]::GetFullPath([IO.Path]::GetTempPath()).TrimEnd('\') + '\'
|
|
||||||
foreach ($path in $Paths) {
|
|
||||||
$resolved = [IO.Path]::GetFullPath($path)
|
|
||||||
if (-not $resolved.StartsWith($tempRoot, [StringComparison]::OrdinalIgnoreCase) -or
|
|
||||||
-not ([IO.Path]::GetFileName($resolved)).StartsWith("$Name-")) {
|
|
||||||
throw "Unsafe release temporary path: $resolved"
|
|
||||||
}
|
|
||||||
if (Test-Path -LiteralPath $resolved) { Remove-Item -LiteralPath $resolved -Recurse -Force }
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# The tag step 1 exported, from release\<folder>\production.
|
||||||
|
function Get-ReleaseSha {
|
||||||
|
$file = Join-Path $Release 'production'
|
||||||
|
if (-not (Test-Path $file)) { throw "no $file - run step 1 first" }
|
||||||
|
(Get-Content $file -TotalCount 1).Trim()
|
||||||
|
}
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
# shellcheck shell=bash
|
||||||
|
# Shared plumbing for the release scripts that run in WSL. Source it; do not
|
||||||
|
# run it.
|
||||||
|
#
|
||||||
|
# Sourcing runs in the caller's shell, so everything defined here ($root,
|
||||||
|
# SETTINGS, setting, require_setting, die, step, run_or_die) is the caller's
|
||||||
|
# afterwards. The Windows half (steps 2 and 3) reads the same release.env with
|
||||||
|
# _common.ps1, which parses the same KEY=VALUE lines. The caller sets
|
||||||
|
# `set -euo pipefail` first.
|
||||||
|
|
||||||
|
#: The directory of these scripts and the repository root above it, worked out
|
||||||
|
#: from this file rather than from the working directory, so the scripts can be
|
||||||
|
#: run from anywhere.
|
||||||
|
here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||||
|
# shellcheck disable=SC2034 # read by the scripts that source this file
|
||||||
|
root=$(dirname "$here")
|
||||||
|
|
||||||
|
die() { printf 'error: %s\n' "$*" >&2; exit 1; }
|
||||||
|
|
||||||
|
step() { printf '\n== %s\n' "$*"; }
|
||||||
|
|
||||||
|
# Runs a command and, when it fails, stops with the step's name and its exit
|
||||||
|
# code: `run_or_die 'git archive' git archive ...`.
|
||||||
|
run_or_die() {
|
||||||
|
local what=$1 rc=0
|
||||||
|
shift
|
||||||
|
"$@" || rc=$?
|
||||||
|
((rc == 0)) || die "$what failed (exit $rc)"
|
||||||
|
}
|
||||||
|
|
||||||
|
# --------------------------------------------------------------- release.env
|
||||||
|
|
||||||
|
# Read, not sourced: sourcing would run whatever the file says and would break
|
||||||
|
# on a space in a value. A line is blank, a comment, or KEY=VALUE; anything else
|
||||||
|
# is a typo and is reported here rather than as an empty variable later. Keys
|
||||||
|
# and values are trimmed and surrounding double quotes are dropped. A CRLF file
|
||||||
|
# (saved from a Windows editor) reads the same as an LF one.
|
||||||
|
trim() {
|
||||||
|
local s=$1
|
||||||
|
s=${s#"${s%%[![:space:]]*}"}
|
||||||
|
s=${s%"${s##*[![:space:]]}"}
|
||||||
|
printf '%s' "$s"
|
||||||
|
}
|
||||||
|
|
||||||
|
declare -A SETTINGS=()
|
||||||
|
env_file=$here/release.env
|
||||||
|
[[ -f $env_file ]] || die "no deploy/release.env - copy deploy/release.env.example and fill it in"
|
||||||
|
while IFS= read -r line || [[ -n $line ]]; do
|
||||||
|
line=${line%$'\r'}
|
||||||
|
[[ $line =~ ^[[:space:]]*(#|$) ]] && continue
|
||||||
|
[[ $line == *=* ]] || die "release.env: not a KEY=VALUE line: $line"
|
||||||
|
key=$(trim "${line%%=*}")
|
||||||
|
value=$(trim "${line#*=}")
|
||||||
|
while [[ $value == \"* ]]; do value=${value#\"}; done
|
||||||
|
while [[ $value == *\" ]]; do value=${value%\"}; done
|
||||||
|
SETTINGS[$key]=$value
|
||||||
|
done < "$env_file"
|
||||||
|
unset line key value
|
||||||
|
|
||||||
|
# setting NAME [DEFAULT]: the value, or the default when unset or empty.
|
||||||
|
setting() {
|
||||||
|
local value=${SETTINGS[$1]-}
|
||||||
|
if [[ -n $value ]]; then printf '%s' "$value"; else printf '%s' "${2-}"; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# require_setting NAME: the value, or a stop naming the missing setting.
|
||||||
|
require_setting() {
|
||||||
|
local value
|
||||||
|
value=$(setting "$1")
|
||||||
|
[[ -n $value ]] || die "set $1 in deploy/release.env"
|
||||||
|
printf '%s' "$value"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- shared checks
|
||||||
|
|
||||||
|
need_git() {
|
||||||
|
command -v git >/dev/null 2>&1 || die "git is not on PATH"
|
||||||
|
}
|
||||||
|
|
||||||
|
# The commit a branch or ref points at, or a stop saying which one did not
|
||||||
|
# resolve.
|
||||||
|
resolve_commit() {
|
||||||
|
local commit
|
||||||
|
commit=$(git rev-parse --verify "$1^{commit}") || die "resolve $2 commit failed"
|
||||||
|
printf '%s' "$commit"
|
||||||
|
}
|
||||||
-150
@@ -1,150 +0,0 @@
|
|||||||
<#
|
|
||||||
.SYNOPSIS
|
|
||||||
The database, in both directions.
|
|
||||||
|
|
||||||
.DESCRIPTION
|
|
||||||
push this machine's database -> the server
|
|
||||||
pull a fresh snapshot from the server -> .\backups
|
|
||||||
|
|
||||||
`push` is the one-off at the start: production begins knowing what you
|
|
||||||
already have, rather than starting empty. `pull` is the one you keep
|
|
||||||
running -- on a schedule from the server itself (server\backup.sh under
|
|
||||||
cron) and by hand from here whenever you want the copy somewhere the
|
|
||||||
server is not.
|
|
||||||
|
|
||||||
`push` moves the database and nothing else. It does not stop the app before
|
|
||||||
or start it after: a deployment knows what else is moving and in what order,
|
|
||||||
and this does not. Stop the app on the server first -- the import is a file
|
|
||||||
swap under SQLite's feet and is refused while the app is up. Start it again
|
|
||||||
afterwards.
|
|
||||||
|
|
||||||
Both go through SQLite's own `VACUUM INTO`. The database is in WAL mode, so
|
|
||||||
the .db file on disk is always missing whatever is still in the -wal beside
|
|
||||||
it, and copying the three files with the app running is a race you lose
|
|
||||||
quietly -- a database that opens fine and is short a day's writes.
|
|
||||||
|
|
||||||
After a push the two copies diverge from the first thing written on either.
|
|
||||||
There is no merge for that and this does not pretend there is: pick the one
|
|
||||||
that is authoritative, and use the other read-only or not at all.
|
|
||||||
|
|
||||||
.EXAMPLE
|
|
||||||
.\deploy\db.ps1 push
|
|
||||||
|
|
||||||
.EXAMPLE
|
|
||||||
.\deploy\db.ps1 pull
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
|
||||||
param(
|
|
||||||
[Parameter(Mandatory)]
|
|
||||||
[ValidateSet('push', 'pull')]
|
|
||||||
[string] $Action,
|
|
||||||
|
|
||||||
# Skip the confirmation before replacing the server's database.
|
|
||||||
[switch] $Yes
|
|
||||||
)
|
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
. "$PSScriptRoot/_common.ps1"
|
|
||||||
Set-Location $root
|
|
||||||
|
|
||||||
$SshHost = Setting SSH_HOST -Required
|
|
||||||
$RemoteDir = Setting REMOTE_DIR -Required
|
|
||||||
# Where the data actually is. Beside the compose file unless APP_DATA put it
|
|
||||||
# under a shared root; then REMOTE_DATA has to say so.
|
|
||||||
$RemoteData = Setting REMOTE_DATA "$RemoteDir/data"
|
|
||||||
$Python = Setting PYTHON (Join-Path $root '.venv\Scripts\python.exe')
|
|
||||||
$LocalDb = Setting LOCAL_DB (Join-Path $root "$Name.db")
|
|
||||||
|
|
||||||
switch ($Action) {
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- push
|
|
||||||
|
|
||||||
'push' {
|
|
||||||
if (-not (Test-Path $LocalDb)) { throw "no $LocalDb" }
|
|
||||||
|
|
||||||
$stamp = Get-Date -Format 'yyyyMMdd-HHmmss'
|
|
||||||
$snap = Join-Path $root "$Name-$stamp.db.gz"
|
|
||||||
Write-Host "snapshotting $LocalDb"
|
|
||||||
|
|
||||||
# One Python call for both halves. `VACUUM INTO` is the only way to get a
|
|
||||||
# consistent single file out of a WAL database, and Python is already the
|
|
||||||
# thing that has to be here -- PowerShell has no gzip that writes the format
|
|
||||||
# the server's `gunzip` expects.
|
|
||||||
$snapshot = @'
|
|
||||||
import gzip, pathlib, shutil, sqlite3, sys, tempfile
|
|
||||||
source, target = sys.argv[1], pathlib.Path(sys.argv[2])
|
|
||||||
with tempfile.TemporaryDirectory() as work:
|
|
||||||
plain = pathlib.Path(work) / "snapshot.db"
|
|
||||||
db = sqlite3.connect(source)
|
|
||||||
try:
|
|
||||||
db.execute("VACUUM INTO ?", (str(plain),))
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
with plain.open("rb") as raw, gzip.open(target, "wb") as out:
|
|
||||||
shutil.copyfileobj(raw, out)
|
|
||||||
'@
|
|
||||||
# A checkout inside WSL has a Linux .venv, and SQLite cannot lock a WAL
|
|
||||||
# database through the \\wsl.localhost share, so the snapshot is taken in
|
|
||||||
# the distro. The script goes in on stdin, past both shells.
|
|
||||||
if ($Wsl.Success) {
|
|
||||||
$snapshot | wsl.exe -d $Wsl.Groups[1].Value --cd (ConvertTo-LinuxPath $root) --exec .venv/bin/python - (ConvertTo-LinuxPath $LocalDb) (ConvertTo-LinuxPath $snap)
|
|
||||||
} else {
|
|
||||||
& $Python -c $snapshot $LocalDb $snap
|
|
||||||
}
|
|
||||||
Assert-Ok 'snapshot'
|
|
||||||
' {0:N1} MB' -f ((Get-Item $snap).Length / 1MB) | Write-Host
|
|
||||||
|
|
||||||
try {
|
|
||||||
Write-Host ''
|
|
||||||
Write-Host "This replaces the database on $SshHost with this one." -ForegroundColor Yellow
|
|
||||||
Write-Host 'Anything written there since is gone.' -ForegroundColor Yellow
|
|
||||||
Write-Host 'The app must already be stopped there, and stays stopped after.' -ForegroundColor Yellow
|
|
||||||
if (-not $Yes) {
|
|
||||||
$reply = Read-Host 'go ahead? [y/N]'
|
|
||||||
if ($reply -notmatch '^[yY]') { Write-Host 'stopped.'; return }
|
|
||||||
}
|
|
||||||
|
|
||||||
scp -C $snap "${SshHost}:$RemoteData/incoming.db.gz"; Assert-Ok 'scp database'
|
|
||||||
scp (Join-Path $PSScriptRoot 'server\import_database.py') "${SshHost}:$RemoteDir/import_database.py"
|
|
||||||
Assert-Ok 'scp importer'
|
|
||||||
} finally {
|
|
||||||
Remove-Item -Force $snap -ErrorAction SilentlyContinue
|
|
||||||
}
|
|
||||||
|
|
||||||
# Base64 rather than stdin: PowerShell has no `<` for a native command, and
|
|
||||||
# piping the text would rewrite the script's line endings on the way.
|
|
||||||
$b64 = [Convert]::ToBase64String([IO.File]::ReadAllBytes((Join-Path $PSScriptRoot 'server\import.sh')))
|
|
||||||
ssh $SshHost "echo $b64 | base64 -d | bash -s -- '$RemoteDir'"; Assert-Ok 'import'
|
|
||||||
Write-Host "pushed. The app is still stopped; start it when the rest of"
|
|
||||||
Write-Host 'the deployment is in place:'
|
|
||||||
Write-Host " ssh $SshHost `"cd '$RemoteDir' && docker compose up -d --wait $Name`""
|
|
||||||
}
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- pull
|
|
||||||
|
|
||||||
'pull' {
|
|
||||||
$into = Join-Path $root 'backups'
|
|
||||||
New-Item -ItemType Directory -Force -Path $into | Out-Null
|
|
||||||
|
|
||||||
# The server's own copy of backup.sh, not a streamed one. It is the same
|
|
||||||
# script cron runs, it is already there, and it locates its own directory
|
|
||||||
# through BASH_SOURCE -- which is exactly the thing a script read from stdin
|
|
||||||
# does not have. import.sh takes its directory as an argument and so can be
|
|
||||||
# streamed; this one cannot.
|
|
||||||
Write-Host "snapshotting on $SshHost"
|
|
||||||
ssh $SshHost "cd '$RemoteDir' && bash backup.sh"; Assert-Ok 'backup'
|
|
||||||
|
|
||||||
$newest = (ssh $SshHost "ls -1t '$RemoteData'/backups/$Name-*.db.gz | head -1").Trim()
|
|
||||||
Assert-Ok 'listing backups'
|
|
||||||
if (-not $newest) { throw 'no backup appeared on the server' }
|
|
||||||
|
|
||||||
scp -C "${SshHost}:$newest" $into; Assert-Ok 'scp'
|
|
||||||
$file = Split-Path -Leaf $newest
|
|
||||||
Write-Host "backups\$file"
|
|
||||||
Write-Host ''
|
|
||||||
Write-Host 'To read it here, gunzip it, or in Python:'
|
|
||||||
Write-Host " import gzip, shutil; shutil.copyfileobj(gzip.open(r'backups\$file'), open('read-only.db','wb'))"
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
+130
@@ -0,0 +1,130 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# The database, in both directions.
|
||||||
|
#
|
||||||
|
# bash deploy/db.sh push [--yes] this machine's database -> the server
|
||||||
|
# bash deploy/db.sh pull a fresh snapshot from the server -> backups/
|
||||||
|
#
|
||||||
|
# `push` is the one-off at the start: production begins knowing what you already
|
||||||
|
# have, rather than starting empty. `pull` is the one you keep running, on a
|
||||||
|
# schedule from the server itself (server/backup.sh under cron) and by hand from
|
||||||
|
# here whenever you want the copy somewhere the server is not.
|
||||||
|
#
|
||||||
|
# `push` moves the database and nothing else. It does not stop the app before or
|
||||||
|
# start it after: a deployment knows what else is moving and in what order, and
|
||||||
|
# this does not. Stop the app on the server first; the import is a file swap
|
||||||
|
# under SQLite's feet and is refused while the app is up. Start it again
|
||||||
|
# afterwards.
|
||||||
|
#
|
||||||
|
# Both go through SQLite's own `VACUUM INTO`. The database is in WAL mode, so the
|
||||||
|
# .db file on disk is always missing whatever is still in the -wal beside it, and
|
||||||
|
# copying the three files with the app running is a race you lose quietly: a
|
||||||
|
# database that opens fine and is short a day's writes.
|
||||||
|
#
|
||||||
|
# After a push the two copies diverge from the first thing written on either.
|
||||||
|
# There is no merge for that: pick the one that is authoritative, and use the
|
||||||
|
# other read-only or not at all.
|
||||||
|
#
|
||||||
|
# Settings from release.env: SSH_HOST, REMOTE_DIR and NAME (the compose service)
|
||||||
|
# are required; REMOTE_DATA (default REMOTE_DIR/data), PYTHON and LOCAL_DB
|
||||||
|
# (default the database the app's config names) are not.
|
||||||
|
set -euo pipefail
|
||||||
|
# shellcheck source=_common.sh
|
||||||
|
. "$(dirname "${BASH_SOURCE[0]}")/_common.sh"
|
||||||
|
|
||||||
|
action='' yes=0
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
push|pull) action=$arg ;;
|
||||||
|
--yes) yes=1 ;;
|
||||||
|
-h|--help) sed -n '2,29p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||||
|
*) die "unknown option $arg" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
[[ -n $action ]] || die "say push or pull: bash deploy/db.sh push|pull"
|
||||||
|
|
||||||
|
ssh_host=$(require_setting SSH_HOST)
|
||||||
|
remote_dir=$(require_setting REMOTE_DIR)
|
||||||
|
# Where the data actually is. Beside the compose file unless APP_DATA put it
|
||||||
|
# under a shared root; then REMOTE_DATA has to say so.
|
||||||
|
remote_data=$(setting REMOTE_DATA "$remote_dir/data")
|
||||||
|
name=$(require_setting NAME)
|
||||||
|
python=$(setting PYTHON "$root/.venv/bin/python")
|
||||||
|
cd "$root"
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------- push
|
||||||
|
|
||||||
|
if [[ $action == push ]]; then
|
||||||
|
[[ -x $python ]] || die "no interpreter at $python - run uv sync --all-extras, or set PYTHON in deploy/release.env"
|
||||||
|
# The file the app writes: `database` in config.yaml (or $APP_CONFIG),
|
||||||
|
# roledesk.db without one, asked of the app from the checkout's root.
|
||||||
|
local_db=$(setting LOCAL_DB)
|
||||||
|
if [[ -z $local_db ]]; then
|
||||||
|
local_db=$("$python" -c 'from app.config import load; print(load().database.absolute())') \
|
||||||
|
|| die "could not read the database path from the app's config; set LOCAL_DB in deploy/release.env"
|
||||||
|
fi
|
||||||
|
[[ -f $local_db ]] || die "no $local_db"
|
||||||
|
|
||||||
|
snap=$root/$name-$(date +%Y%m%d-%H%M%S).db.gz
|
||||||
|
trap 'rm -f -- "$snap"' EXIT
|
||||||
|
echo "snapshotting $local_db"
|
||||||
|
|
||||||
|
# One Python call for both halves: `VACUUM INTO` is the only way to get a
|
||||||
|
# consistent single file out of a WAL database, and it writes the gzip the
|
||||||
|
# server's `gunzip` expects.
|
||||||
|
run_or_die snapshot "$python" -c '
|
||||||
|
import gzip, pathlib, shutil, sqlite3, sys, tempfile
|
||||||
|
source, target = sys.argv[1], pathlib.Path(sys.argv[2])
|
||||||
|
with tempfile.TemporaryDirectory() as work:
|
||||||
|
plain = pathlib.Path(work) / "snapshot.db"
|
||||||
|
db = sqlite3.connect(source)
|
||||||
|
try:
|
||||||
|
db.execute("VACUUM INTO ?", (str(plain),))
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
with plain.open("rb") as raw, gzip.open(target, "wb") as out:
|
||||||
|
shutil.copyfileobj(raw, out)
|
||||||
|
' "$local_db" "$snap"
|
||||||
|
printf ' %s\n' "$(du -h "$snap" | cut -f1)"
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "This replaces the database on $ssh_host with this one."
|
||||||
|
echo 'Anything written there since is gone.'
|
||||||
|
echo 'The app must already be stopped there, and stays stopped after.'
|
||||||
|
if ((! yes)); then
|
||||||
|
read -r -p 'go ahead? [y/N] ' reply || reply=''
|
||||||
|
[[ $reply =~ ^[yY] ]] || { echo 'stopped.'; exit 0; }
|
||||||
|
fi
|
||||||
|
|
||||||
|
run_or_die 'scp database' scp -C "$snap" "$ssh_host:$remote_data/incoming.db.gz"
|
||||||
|
run_or_die 'scp importer' scp "$here/server/import_database.py" "$ssh_host:$remote_dir/import_database.py"
|
||||||
|
rm -f -- "$snap"
|
||||||
|
|
||||||
|
# import.sh takes its directory as an argument, so it can be streamed.
|
||||||
|
run_or_die import ssh "$ssh_host" "bash -s -- '$remote_dir'" < "$here/server/import.sh"
|
||||||
|
echo 'pushed. The app is still stopped; start it when the rest of'
|
||||||
|
echo 'the deployment is in place:'
|
||||||
|
echo " ssh $ssh_host \"cd '$remote_dir' && docker compose up -d --wait $name\""
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------- pull
|
||||||
|
|
||||||
|
into=$root/backups
|
||||||
|
mkdir -p -- "$into"
|
||||||
|
|
||||||
|
# The server's own copy of backup.sh, not a streamed one: it is the same script
|
||||||
|
# cron runs, and it locates its own directory through BASH_SOURCE, which a
|
||||||
|
# script read from stdin does not have.
|
||||||
|
echo "snapshotting on $ssh_host"
|
||||||
|
run_or_die backup ssh "$ssh_host" "cd '$remote_dir' && bash backup.sh"
|
||||||
|
|
||||||
|
# shellcheck disable=SC2029 # the remote command is meant to expand here
|
||||||
|
newest=$(ssh "$ssh_host" "ls -1t '$remote_data'/backups/$name-*.db.gz | head -1") || die 'listing backups failed'
|
||||||
|
[[ -n $newest ]] || die 'no backup appeared on the server'
|
||||||
|
|
||||||
|
run_or_die scp scp -C "$ssh_host:$newest" "$into/"
|
||||||
|
file=$(basename "$newest")
|
||||||
|
echo "backups/$file"
|
||||||
|
echo
|
||||||
|
echo 'To read it here, gunzip it, or in Python:'
|
||||||
|
echo " import gzip, shutil; shutil.copyfileobj(gzip.open('backups/$file'), open('read-only.db','wb'))"
|
||||||
+26
-13
@@ -1,28 +1,41 @@
|
|||||||
# Copy to release.env beside these scripts and fill it in. Never committed:
|
# Copy to release.env. One file, read by both halves of the release: the bash
|
||||||
# it names your registry and, for db.ps1, your server.
|
# scripts (_common.sh), which run in the WSL distro that holds the checkout, and
|
||||||
|
# the PowerShell ones (_common.ps1, wsl-setup.ps1), which run on Windows. Both
|
||||||
|
# read KEY=VALUE lines and neither runs the file as shell, so $HOME and ~ are
|
||||||
|
# not expanded. Local main is the source; fetching is your choice. Never
|
||||||
|
# committed: it names your registry and, for db.sh, your server.
|
||||||
#
|
#
|
||||||
# Local main is the source. Fetching it first is your choice, not the script's.
|
# Where the checkout is: the distro's name and the path inside it, as
|
||||||
|
# docs/Install WSL.md lays them out, with the checkout folder's name last.
|
||||||
|
# wsl-setup.ps1 writes them into the profile commands, and steps 2 and 3 find
|
||||||
|
# step 1's export at \\wsl.localhost\<DISTRO>\<CHECKOUT's parent>\release\<folder>.
|
||||||
|
DISTRO=claudebox
|
||||||
|
CHECKOUT=/home/claude/cboxcode/jobboard
|
||||||
|
|
||||||
# The service's name. Used for the image tag, for the temporary directories
|
# The compose service, SERVICE in deploy/server/*.sh. Used by db.sh, on the
|
||||||
# these scripts create, and for the database file db.ps1 moves. One word, no
|
# Linux side, for the snapshot names (<NAME>-<stamp>.db.gz, as backup.sh writes
|
||||||
# spaces — it ends up in paths.
|
# them) and the start command it prints. One word, no spaces.
|
||||||
NAME=app
|
NAME=app
|
||||||
|
|
||||||
|
# Windows side, used by steps 2 and 3. wslc builds for the machine's own
|
||||||
|
# architecture (linux/amd64 here) and has no --platform.
|
||||||
IMAGE=git.rittershofer.org/wolf/app
|
IMAGE=git.rittershofer.org/wolf/app
|
||||||
PLATFORM=linux/amd64
|
|
||||||
|
# Linux side, used by step 1: which branch is tested, and which one it promotes.
|
||||||
BRANCH=production
|
BRANCH=production
|
||||||
SOURCE=main
|
SOURCE=main
|
||||||
|
|
||||||
# Defaults to the checkout's .venv/Scripts/python.exe. Only needed for the
|
# Linux side, used by step 1 and by db.sh's snapshot, in the distro. Defaults to
|
||||||
# tests in step 1 and for db.ps1's snapshot.
|
# the checkout's .venv/bin/python (a WSL path, /home/<user>/...).
|
||||||
# PYTHON=C:/Users/Coolkau/code/app/.venv/Scripts/python.exe
|
# PYTHON=/home/<user>/cboxcode/jobboard/.venv/bin/python
|
||||||
|
|
||||||
# db.ps1 uses SSH. The numbered release scripts never do. REMOTE_DIR is the
|
# db.sh uses SSH. The numbered release scripts never do. REMOTE_DIR is the
|
||||||
# directory holding the compose file; REMOTE_DATA is where APP_DATA points,
|
# directory holding the compose file; REMOTE_DATA is where APP_DATA points,
|
||||||
# and defaults to REMOTE_DIR/data when left blank.
|
# and defaults to REMOTE_DIR/data when left blank.
|
||||||
SSH_HOST=
|
SSH_HOST=
|
||||||
REMOTE_DIR=/srv/swag
|
REMOTE_DIR=/srv/swag
|
||||||
REMOTE_DATA=/srv/data/app
|
REMOTE_DATA=/srv/data/app
|
||||||
|
|
||||||
# The database this machine pushes, if it is not <root>/<NAME>.db.
|
# The database this machine pushes, if it is not the one the app's config names
|
||||||
# LOCAL_DB=C:/Users/Coolkau/code/app/app.db
|
# (`database` in config.yaml, roledesk.db in the checkout without one).
|
||||||
|
# LOCAL_DB=/home/<user>/cboxcode/jobboard/roledesk.db
|
||||||
+43
-10
@@ -13,9 +13,26 @@
|
|||||||
# self-contained and compacted file, and never blocks a writer for long enough
|
# self-contained and compacted file, and never blocks a writer for long enough
|
||||||
# to matter on a database of this size.
|
# to matter on a database of this size.
|
||||||
#
|
#
|
||||||
# Rename `app` and `APP_DATA` to your service's name.
|
# The database is the one the app's own config names, asked of the app inside
|
||||||
|
# the container, and it is opened read-only, so a path that names no file is
|
||||||
|
# an error rather than a new empty database and a snapshot of nothing.
|
||||||
|
#
|
||||||
|
# SERVICE is the compose service (deploy.sh says why it is here); APP_DATA is
|
||||||
|
# renamed with the other APP_ variables, and `from app.config` with the package.
|
||||||
|
|
||||||
|
SERVICE=app
|
||||||
|
# Tables every database this app writes has: the same set as REQUIRED_TABLES in
|
||||||
|
# import_database.py, which a test holds this line to. A snapshot without one
|
||||||
|
# of them is deleted and the run fails.
|
||||||
|
# Ours: `roles` is app/storage/schema.py's first table, where the boilerplate has `ticks`.
|
||||||
|
TABLES="jobs roles"
|
||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
# A snapshot is the whole database, so it is owner only whatever the caller's
|
||||||
|
# umask is. This covers the folder made below; the umask inside the container
|
||||||
|
# (set in the snippet) covers the .db, and gzip gives the .gz the mode of the
|
||||||
|
# file it compresses.
|
||||||
|
umask 077
|
||||||
cd "${1:-$(dirname "$(readlink -f "$0")")}"
|
cd "${1:-$(dirname "$(readlink -f "$0")")}"
|
||||||
|
|
||||||
# Same source of truth as the compose file and deploy.sh.
|
# Same source of truth as the compose file and deploy.sh.
|
||||||
@@ -27,18 +44,34 @@ stamp=$(date +%Y%m%d-%H%M%S)
|
|||||||
|
|
||||||
mkdir -p "$DATA/backups"
|
mkdir -p "$DATA/backups"
|
||||||
|
|
||||||
docker compose exec -T app python -c "
|
# shellcheck disable=SC2086 # TABLES is a list of words on purpose
|
||||||
import sqlite3
|
docker compose exec -T "$SERVICE" python - "$SERVICE-$stamp" $TABLES <<'PY'
|
||||||
db = sqlite3.connect('/data/app.db')
|
import os, sqlite3, sys
|
||||||
db.execute('VACUUM INTO ?', ('/data/backups/app-$stamp.db',))
|
from contextlib import closing
|
||||||
db.close()
|
from pathlib import Path
|
||||||
"
|
|
||||||
gzip -f "$DATA/backups/app-$stamp.db"
|
from app.config import load
|
||||||
echo "$DATA/backups/app-$stamp.db.gz $(du -h "$DATA/backups/app-$stamp.db.gz" | cut -f1)"
|
|
||||||
|
os.umask(0o077)
|
||||||
|
name, required = sys.argv[1], set(sys.argv[2:])
|
||||||
|
database = load().database.absolute()
|
||||||
|
snapshot = Path("/data/backups") / f"{name}.db"
|
||||||
|
with closing(sqlite3.connect(f"{database.as_uri()}?mode=ro", uri=True)) as db:
|
||||||
|
db.execute("VACUUM INTO ?", (str(snapshot),))
|
||||||
|
with closing(sqlite3.connect(f"{snapshot.as_uri()}?mode=ro", uri=True)) as db:
|
||||||
|
tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||||||
|
missing = sorted(required - tables)
|
||||||
|
if missing:
|
||||||
|
snapshot.unlink()
|
||||||
|
sys.exit(f"{database} has no table {', '.join(missing)}: not this app's database")
|
||||||
|
print(f"snapshot of {database}")
|
||||||
|
PY
|
||||||
|
gzip -f "$DATA/backups/$SERVICE-$stamp.db"
|
||||||
|
echo "$DATA/backups/$SERVICE-$stamp.db.gz $(du -h "$DATA/backups/$SERVICE-$stamp.db.gz" | cut -f1)"
|
||||||
|
|
||||||
# Oldest first, everything past KEEP goes. Named by timestamp, so sorting the
|
# Oldest first, everything past KEEP goes. Named by timestamp, so sorting the
|
||||||
# names sorts them by age.
|
# names sorts them by age.
|
||||||
ls -1 "$DATA"/backups/app-*.db.gz 2>/dev/null \
|
ls -1 "$DATA"/backups/"$SERVICE"-*.db.gz 2>/dev/null \
|
||||||
| sort \
|
| sort \
|
||||||
| head -n -"$KEEP" \
|
| head -n -"$KEEP" \
|
||||||
| while read -r old; do rm -f "$old"; echo "removed $old"; done
|
| while read -r old; do rm -f "$old"; echo "removed $old"; done
|
||||||
@@ -9,7 +9,10 @@
|
|||||||
# `env.example` to the `.env` next to it, and it is `docker compose up -d app`
|
# `env.example` to the `.env` next to it, and it is `docker compose up -d app`
|
||||||
# from then on — which recreates this container and no other.
|
# from then on — which recreates this container and no other.
|
||||||
#
|
#
|
||||||
# Rename `app` and `APP_` throughout to your service's name.
|
# Rename `app` and `APP_` throughout to your service's name. The service and
|
||||||
|
# `container_name` are the name `SERVICE=` in deploy.sh, backup.sh and
|
||||||
|
# import.sh, `$upstream_app` in the proxy conf and `NAME=` in release.env
|
||||||
|
# repeat; tests/test_deploy.py reads it from this file and holds them to it.
|
||||||
|
|
||||||
services:
|
services:
|
||||||
app:
|
app:
|
||||||
@@ -33,12 +36,12 @@ services:
|
|||||||
# Which model answers, decided here rather than in the image. With
|
# Which model answers, decided here rather than in the image. With
|
||||||
# `backend: auto` the API key wins when there is one and the
|
# `backend: auto` the API key wins when there is one and the
|
||||||
# subscription CLI is the fallback, so filling in ANTHROPIC_API_KEY
|
# subscription CLI is the fallback, so filling in ANTHROPIC_API_KEY
|
||||||
# later switches it over with no rebuild. See docs/llm-backends.md.
|
# later switches it over with no rebuild. See llm/README.md.
|
||||||
CLAUDE_CODE_OAUTH_TOKEN: ${CLAUDE_CODE_OAUTH_TOKEN:-}
|
CLAUDE_CODE_OAUTH_TOKEN: ${CLAUDE_CODE_OAUTH_TOKEN:-}
|
||||||
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
||||||
# Normally empty. Set it to pin the password — the way back in when a
|
# Normally empty. Set it to pin the password — the way back in when a
|
||||||
# login is lost — then unset it and recreate to hand control of the
|
# login is lost — then unset it and recreate to hand control of the
|
||||||
# password back to the app. See docs/password-protection.md.
|
# password back to the app. See auth/README.md.
|
||||||
APP_PASSWORD: ${APP_PASSWORD:-}
|
APP_PASSWORD: ${APP_PASSWORD:-}
|
||||||
|
|
||||||
# No `ports:`. Nothing on the host reaches it; SWAG does, by container
|
# No `ports:`. Nothing on the host reaches it; SWAG does, by container
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
# The server's config.yaml. Copy it to $APP_DATA/config.yaml, which the
|
||||||
|
# container reads as /data/config.yaml (APP_CONFIG in the Dockerfile), and set
|
||||||
|
# the hostname. deploy.sh refuses a config without `database` or `auth.enabled`.
|
||||||
|
# Every key left out keeps its default; ../../config.example.yaml lists them all.
|
||||||
|
# Ours: the keys are roledesk's (app/config.py); an unknown one is an error.
|
||||||
|
|
||||||
|
server:
|
||||||
|
host: 0.0.0.0 # the container's interface; SWAG is in front of it
|
||||||
|
port: 8787 # the port the proxy conf and the HEALTHCHECK use
|
||||||
|
|
||||||
|
database: /data/roledesk.db # in the volume; /app, the working directory, is not writable
|
||||||
|
|
||||||
|
auth:
|
||||||
|
enabled: always # the lock must not depend on how the process was started
|
||||||
|
secure_cookie: always # auto needs uvicorn to trust X-Forwarded-Proto, which only `app serve` off loopback sets
|
||||||
|
rp_id: app.example.com # the hostname passkeys are bound to; changing it voids every passkey
|
||||||
|
origin: https://app.example.com # what the browser's address bar shows, scheme included
|
||||||
|
|
||||||
|
# Who answers when a stage needs a model (extract, summarise, judge). `auto`
|
||||||
|
# takes ANTHROPIC_API_KEY when the compose file passes one and otherwise the
|
||||||
|
# subscription CLI with CLAUDE_CODE_OAUTH_TOKEN, so switching is an .env edit.
|
||||||
|
# The stages themselves are switched under Settings, Sources, Runs.
|
||||||
|
agent:
|
||||||
|
backend: auto
|
||||||
|
|
||||||
|
# The vectors behind prompt and CV closeness. The model is not in the image:
|
||||||
|
# the first `embed` job downloads it (~460 MB) into HOME, the /data volume, so
|
||||||
|
# it happens once per server and not once per release. Changing it re-embeds.
|
||||||
|
embed:
|
||||||
|
model: intfloat/multilingual-e5-small
|
||||||
+27
-10
@@ -1,9 +1,14 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Run on the server, next to the compose file, or with its directory as $1.
|
# Run on the server, next to the compose file, or with its directory as $1.
|
||||||
# No remote shell required: the release scripts on Windows publish an image
|
# No remote shell required: the release scripts publish an image
|
||||||
# and stop there, and this is the other half.
|
# and stop there, and this is the other half.
|
||||||
#
|
#
|
||||||
# Rename `app` and `APP_DATA` to your service's name.
|
# SERVICE is the compose service, the one name a project renames in these
|
||||||
|
# scripts (README, "Starting a project from it"); the tests read it from
|
||||||
|
# compose.yaml and hold every script to it. APP_DATA is renamed with the other
|
||||||
|
# APP_ variables.
|
||||||
|
SERVICE=app
|
||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
cd "${1:-$(dirname "$(readlink -f "$0")")}"
|
cd "${1:-$(dirname "$(readlink -f "$0")")}"
|
||||||
|
|
||||||
@@ -15,24 +20,36 @@ cd "${1:-$(dirname "$(readlink -f "$0")")}"
|
|||||||
DATA=$(sed -n 's/^[[:space:]]*APP_DATA=\([^#]*\).*/\1/p' .env | tail -1 | xargs || true)
|
DATA=$(sed -n 's/^[[:space:]]*APP_DATA=\([^#]*\).*/\1/p' .env | tail -1 | xargs || true)
|
||||||
DATA=${DATA:-./data}
|
DATA=${DATA:-./data}
|
||||||
|
|
||||||
[[ -s "$DATA/config.yaml" ]] || {
|
config=$DATA/config.yaml
|
||||||
echo "Install $DATA/config.yaml before deploying." >&2
|
[[ -s $config ]] || {
|
||||||
|
echo "Install $config before deploying, from config.example.yaml beside this script." >&2
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
# The two keys whose defaults are wrong in a container: `auth.enabled: auto`
|
||||||
|
# ties the lock to how the process was started, and the default
|
||||||
|
# database is wherever the config file happens to be. Read with grep and awk,
|
||||||
|
# because the host need not have a Python with a YAML parser.
|
||||||
|
if ! grep -Eq '^database:[[:space:]]*[^[:space:]#]' "$config" ||
|
||||||
|
! awk '/^auth:/ { a = 1; next } /^[^[:space:]#]/ { a = 0 }
|
||||||
|
a && /^[[:space:]]+enabled:[[:space:]]*[^[:space:]#]/ { found = 1 }
|
||||||
|
END { exit !found }' "$config"; then
|
||||||
|
echo "$config must set database and auth.enabled; config.example.yaml beside this script has both." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
# Pull explicitly, here and nowhere else: the compose file has no pull policy,
|
# Pull explicitly, here and nowhere else: the compose file has no pull policy,
|
||||||
# so starting or importing never fetches an image on its own. A registry
|
# so starting or importing never fetches an image on its own. A registry
|
||||||
# failure must leave the running service alone, not deploy a cache.
|
# failure must leave the running service alone, not deploy a cache.
|
||||||
echo 'Pulling the configured image…'
|
echo 'Pulling the configured image…'
|
||||||
if ! docker compose pull app; then
|
if ! docker compose pull "$SERVICE"; then
|
||||||
echo 'Image pull failed; app was not restarted.' >&2
|
echo "Image pull failed; $SERVICE was not restarted." >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
# Use the image just pulled. Do not start or recreate other services in the
|
# Use the image just pulled. Do not start or recreate other services in the
|
||||||
# file. `--wait` holds until the container's own HEALTHCHECK passes.
|
# file. `--wait` holds until the container's own HEALTHCHECK passes.
|
||||||
if ! docker compose up -d --no-deps --pull never --wait --wait-timeout 240 app; then
|
if ! docker compose up -d --no-deps --pull never --wait --wait-timeout 240 "$SERVICE"; then
|
||||||
docker compose ps app >&2 || true
|
docker compose ps "$SERVICE" >&2 || true
|
||||||
echo 'Deployment failed. Check docker compose logs app.' >&2
|
echo "Deployment failed. Check docker compose logs $SERVICE." >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
docker compose ps app
|
docker compose ps "$SERVICE"
|
||||||
@@ -4,7 +4,7 @@
|
|||||||
#
|
#
|
||||||
# chmod 600. Rename APP_ to your service's name throughout.
|
# chmod 600. Rename APP_ to your service's name throughout.
|
||||||
|
|
||||||
APP_IMAGE=git.rittershofer.org/wolf/app
|
APP_IMAGE=registry.example/you/app
|
||||||
APP_TAG=latest # follows published images; pin a commit tag for rollback
|
APP_TAG=latest # follows published images; pin a commit tag for rollback
|
||||||
|
|
||||||
# Where the app keeps its database, its config and its backups. Relative to
|
# Where the app keeps its database, its config and its backups. Relative to
|
||||||
@@ -20,7 +20,7 @@ PGID=1000
|
|||||||
TZ=Europe/Berlin
|
TZ=Europe/Berlin
|
||||||
|
|
||||||
# Which model answers. Set one or the other, not both — see
|
# Which model answers. Set one or the other, not both — see
|
||||||
# docs/llm-backends.md for what each one means and what it costs.
|
# llm/README.md for what each one means and what it costs.
|
||||||
#
|
#
|
||||||
# CLAUDE_CODE_OAUTH_TOKEN a year-long token for the Claude subscription, from
|
# CLAUDE_CODE_OAUTH_TOKEN a year-long token for the Claude subscription, from
|
||||||
# `claude setup-token` on a machine you are signed in
|
# `claude setup-token` on a machine you are signed in
|
||||||
@@ -34,7 +34,7 @@ ANTHROPIC_API_KEY=
|
|||||||
# `docker compose logs app`; after that it is the app's Account screen's to
|
# `docker compose logs app`; after that it is the app's Account screen's to
|
||||||
# change. Fill this in only to force one — it is the way back in — then empty
|
# change. Fill this in only to force one — it is the way back in — then empty
|
||||||
# it again and recreate, or the Account screen refuses to change what the
|
# it again and recreate, or the Account screen refuses to change what the
|
||||||
# environment pins. See docs/password-protection.md.
|
# environment pins. See auth/README.md.
|
||||||
APP_PASSWORD=
|
APP_PASSWORD=
|
||||||
|
|
||||||
# Unlimited until the box has been watched through the heaviest thing the app
|
# Unlimited until the box has been watched through the heaviest thing the app
|
||||||
|
|||||||
@@ -1,18 +1,21 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Called on the server after uploading incoming.db.gz and import_database.py.
|
# Called on the server after uploading incoming.db.gz and import_database.py.
|
||||||
# `db.ps1 push` does both and then runs this; by hand it is the same two files
|
# `db.sh push` does both and then runs this; by hand it is the same two files
|
||||||
# in the same two places.
|
# in the same two places.
|
||||||
#
|
#
|
||||||
# This replaces the database and does nothing else. It does not stop the app
|
# This replaces the database and does nothing else. It does not stop the app
|
||||||
# and it does not start it: whoever is running a deployment knows what else is
|
# and it does not start it: whoever is running a deployment knows what else is
|
||||||
# moving and in what order, and a tool that helpfully restarts things underneath
|
# moving and in what order, and a tool that helpfully restarts things underneath
|
||||||
# that is a tool fighting its caller. By hand it is
|
# that is a tool fighting its caller. By hand it is
|
||||||
# `docker compose stop app` and `docker compose up -d --wait app` around this.
|
# `docker compose stop app` and `docker compose up -d --wait app` around this,
|
||||||
|
# with SERVICE (deploy.sh says why it is here) in place of app.
|
||||||
#
|
#
|
||||||
# What it will not do is import into a running app. The replacement is a file
|
# What it will not do is import into a running app. The replacement is a file
|
||||||
# swap under SQLite's feet, and the live database is checkpointed first, which
|
# swap under SQLite's feet, and the live database is checkpointed first, which
|
||||||
# needs every writer gone — `import_database.py` refuses a busy database, and
|
# needs every writer gone — `import_database.py` refuses a busy database, and
|
||||||
# this refuses an app that is up, because the clearer error is the earlier one.
|
# this refuses an app that is up, because the clearer error is the earlier one.
|
||||||
|
SERVICE=app
|
||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
cd "${1:?server directory required}"
|
cd "${1:?server directory required}"
|
||||||
|
|
||||||
@@ -22,14 +25,14 @@ DATA=${DATA:-./data}
|
|||||||
|
|
||||||
[[ -s "$DATA/incoming.db.gz" && -s import_database.py ]]
|
[[ -s "$DATA/incoming.db.gz" && -s import_database.py ]]
|
||||||
|
|
||||||
if [[ -n "$(docker compose ps -q --status running app 2>/dev/null)" ]]; then
|
if [[ -n "$(docker compose ps -q --status running "$SERVICE" 2>/dev/null)" ]]; then
|
||||||
echo 'app is running. Stop it before importing:' >&2
|
echo "$SERVICE is running. Stop it before importing:" >&2
|
||||||
echo ' docker compose stop app' >&2
|
echo " docker compose stop $SERVICE" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# The one-off container uses compose's uid and its mounted /data. `--pull
|
# The one-off container uses compose's uid and its mounted /data. `--pull
|
||||||
# never`: an import runs on the image already here, whatever the installed
|
# never`: an import runs on the image already here, whatever the installed
|
||||||
# compose file says about pulling. Getting a new image is deploy.sh's job.
|
# compose file says about pulling. Getting a new image is deploy.sh's job.
|
||||||
docker compose run --rm --no-deps --pull never -T --entrypoint python app - < import_database.py
|
docker compose run --rm --no-deps --pull never -T --entrypoint python "$SERVICE" - < import_database.py
|
||||||
rm -f "$DATA/incoming.db.gz"
|
rm -f "$DATA/incoming.db.gz"
|
||||||
@@ -1,11 +1,13 @@
|
|||||||
"""Offline database replacement, run as the compose service's uid.
|
"""Offline database replacement, run as the compose service's uid.
|
||||||
|
|
||||||
The app container must be stopped. Only stdlib is used so this can be sent
|
The app container must be stopped. Sent over stdin to the image on the
|
||||||
over stdin to either the old or the new image during a deployment.
|
server, whose working directory is the app's: the database is the one the
|
||||||
|
app's config names (`from app.config` follows the package when it is renamed),
|
||||||
|
and the rest is stdlib, so the old image and the new one both run it.
|
||||||
|
|
||||||
Two things to edit for your service: `DATABASE` and `REQUIRED_TABLES`. The
|
`REQUIRED_TABLES` is the table check, which stops a right-shaped-but-wrong
|
||||||
table check is what stops a right-shaped-but-wrong database — last project's,
|
database (last project's, or a half-downloaded one) from landing on top of a
|
||||||
or a half-downloaded one — from landing on top of a good one.
|
good one. `tests/test_deploy.py` fails when the app stops creating one of them.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -19,13 +21,14 @@ from contextlib import closing
|
|||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
DATABASE = Path("/data/app.db")
|
|
||||||
INCOMING = Path("/data/incoming.db.gz")
|
INCOMING = Path("/data/incoming.db.gz")
|
||||||
BACKUPS = Path("/data/backups")
|
BACKUPS = Path("/data/backups")
|
||||||
|
|
||||||
#: Any one of these missing means this is not our database. Name tables that
|
#: Any one of these missing means this is not our database. Name tables that
|
||||||
#: have existed since the first schema, not ones a recent migration added.
|
#: have existed since the first schema, not ones a recent migration added:
|
||||||
REQUIRED_TABLES = {"items"}
|
#: `jobs` is the queue's and `roles` is `app/storage/schema.py`'s first table
|
||||||
|
#: (the boilerplate has `ticks`). backup.sh's TABLES is the same set.
|
||||||
|
REQUIRED_TABLES = {"jobs", "roles"}
|
||||||
|
|
||||||
|
|
||||||
def replace_database(database: Path, archive: Path, backups: Path) -> Path | None:
|
def replace_database(database: Path, archive: Path, backups: Path) -> Path | None:
|
||||||
@@ -73,5 +76,10 @@ def replace_database(database: Path, archive: Path, backups: Path) -> Path | Non
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
saved = replace_database(DATABASE, INCOMING, BACKUPS)
|
from app.config import load
|
||||||
|
|
||||||
|
# The snapshot is the whole database, so it is owner only whatever the
|
||||||
|
# container's umask is; the gzip of it is made under the same mask.
|
||||||
|
os.umask(0o077)
|
||||||
|
saved = replace_database(load().database.absolute(), INCOMING, BACKUPS)
|
||||||
print(f"Database replaced; previous snapshot: {saved or 'none (first import)'}")
|
print(f"Database replaced; previous snapshot: {saved or 'none (first import)'}")
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
##
|
##
|
||||||
## The subdomain has to exist before this does anything: add <app> to SWAG's
|
## The subdomain has to exist before this does anything: add <app> to SWAG's
|
||||||
## SUBDOMAINS (or run it with a wildcard cert) and point the DNS record at the
|
## SUBDOMAINS (or run it with a wildcard cert) and point the DNS record at the
|
||||||
## same address as the rest of the stack. See docs/hosting.md.
|
## same address as the rest of the stack. See docs/Install Server.md.
|
||||||
|
|
||||||
## Per-IP throttle for the login endpoints, if the app has a password. Declared
|
## Per-IP throttle for the login endpoints, if the app has a password. Declared
|
||||||
## here at `http` scope — SWAG includes proxy-confs outside any server block,
|
## here at `http` scope — SWAG includes proxy-confs outside any server block,
|
||||||
@@ -15,23 +15,30 @@ server {
|
|||||||
listen 443 ssl;
|
listen 443 ssl;
|
||||||
listen [::]:443 ssl;
|
listen [::]:443 ssl;
|
||||||
|
|
||||||
server_name app.rittershofer.org;
|
server_name app.example.com;
|
||||||
|
|
||||||
include /config/nginx/ssl.conf;
|
include /config/nginx/ssl.conf;
|
||||||
# Put proxy defaults at server scope so location timeouts override them
|
# Put proxy defaults at server scope so location timeouts override them
|
||||||
# through inheritance, rather than duplicate directives from proxy.conf.
|
# through inheritance, rather than duplicate directives from proxy.conf.
|
||||||
include /config/nginx/proxy.conf;
|
include /config/nginx/proxy.conf;
|
||||||
|
|
||||||
# 0 is no limit. Set a real one if strangers can post to this app; leave it
|
# nginx buffers a request body to disk before it hands it on, and the login
|
||||||
# off when the only uploads are yours and can be large.
|
# takes one without a session, so no limit here lets anyone spool gigabytes.
|
||||||
client_max_body_size 0;
|
# This is small. A route that takes a large upload gets its own
|
||||||
|
# `location = /path { client_max_body_size 0; proxy_request_buffering off;
|
||||||
|
# ... }` above `location /`, with the same proxy lines as below; streaming
|
||||||
|
# the body lets the app refuse a request without a session before nginx
|
||||||
|
# has stored any of it.
|
||||||
|
client_max_body_size 1m;
|
||||||
|
|
||||||
# Guessing the password is already pointless if the app locks out after a
|
# Guessing the password is already pointless if the app locks out after a
|
||||||
# handful of wrong answers (see docs/password-protection.md). This is the
|
# handful of wrong answers (see auth/README.md). This is the
|
||||||
# cheap half of the rest: attempts are slowed per address, at the proxy,
|
# cheap half of the rest: attempts are slowed per address, at the proxy,
|
||||||
# before they reach the app at all. `burst=5 nodelay` lets a person who
|
# before they reach the app at all. `burst=5 nodelay` lets a person who
|
||||||
# fat-fingers it a few times through immediately and then holds the rate.
|
# fat-fingers it a few times through immediately and then holds the rate.
|
||||||
location ~ ^/login(/passkey)?$ {
|
# `/login/passkey/options` is open and writes a challenge row per call, so
|
||||||
|
# it is throttled with the two routes that check a credential.
|
||||||
|
location ~ ^/login(/passkey(/options)?)?$ {
|
||||||
limit_req zone=app_login burst=5 nodelay;
|
limit_req zone=app_login burst=5 nodelay;
|
||||||
limit_req_status 429;
|
limit_req_status 429;
|
||||||
|
|
||||||
|
|||||||
+16
-9
@@ -1,17 +1,24 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# Test one commit in a throwaway worktree, with the checkout's Linux .venv.
|
# Test one commit in a throwaway worktree, with the checkout's Linux .venv.
|
||||||
# 1-test-and-promote.ps1 runs this through wsl.exe when the checkout lives
|
# 1-test-and-promote.sh runs this from the checkout root. PYTHON, when set (the
|
||||||
# inside WSL, where the .venv has no Windows python.exe to call. The gates are
|
# PYTHON line of release.env), names the interpreter instead of .venv/bin/python.
|
||||||
# the same as in that script; keep the two lists in step.
|
# The gates are this project's: roledesk has no `frontend/`, so there is no npm
|
||||||
|
# step, and ruff and mypy stay commented out as the release has run them so far.
|
||||||
# Usage: sh deploy/test-commit.sh <commit> (from the checkout root)
|
# Usage: sh deploy/test-commit.sh <commit> (from the checkout root)
|
||||||
|
# sh deploy/test-commit.sh --here (the working tree as it is, no worktree: bin/check.sh)
|
||||||
set -eu
|
set -eu
|
||||||
root=$(pwd)
|
root=$(pwd)
|
||||||
python="$root/.venv/bin/python"
|
python="${PYTHON:-$root/.venv/bin/python}"
|
||||||
[ -x "$python" ] || { echo "no interpreter at $python - run uv sync" >&2; exit 1; }
|
[ -x "$python" ] || { echo "no interpreter at $python - run uv sync --all-extras, or set PYTHON in deploy/release.env" >&2; exit 1; }
|
||||||
base=$(mktemp -d /tmp/test-commit-XXXXXXXX)
|
target="${1:?usage: sh deploy/test-commit.sh <commit>|--here}"
|
||||||
tree="$base/tree"
|
if [ "$target" = "--here" ]; then
|
||||||
trap 'git -C "$root" worktree remove --force "$tree" 2>/dev/null; rm -rf "$base"' EXIT
|
tree="$root"
|
||||||
git worktree add --detach "$tree" "$1" >/dev/null
|
else
|
||||||
|
base=$(mktemp -d /tmp/release-test-XXXXXXXX)
|
||||||
|
tree="$base/tree"
|
||||||
|
trap 'git -C "$root" worktree remove --force "$tree" 2>/dev/null; rm -rf "$base"' EXIT
|
||||||
|
git worktree add --detach "$tree" "$target" >/dev/null
|
||||||
|
fi
|
||||||
cd "$tree"
|
cd "$tree"
|
||||||
"$python" -m pytest -q
|
"$python" -m pytest -q
|
||||||
# "$python" -m ruff check .
|
# "$python" -m ruff check .
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
@echo off
|
||||||
|
rem Double-click to write this checkout's short commands, folder-cd to
|
||||||
|
rem folder-3push, into the PowerShell profile. It runs deploy\wsl-setup.ps1
|
||||||
|
rem beside it with PowerShell 7 (pwsh) when installed, else Windows PowerShell;
|
||||||
|
rem the profile it writes is that one's. cmd cannot sit in a \\wsl.localhost
|
||||||
|
rem folder, so the "UNC paths are not supported" line it prints is harmless.
|
||||||
|
rem No rem line here holds a redirection or pipe character: cmd still applies
|
||||||
|
rem them after rem, as dostips.com/forum/viewtopic.php?p=15252 shows.
|
||||||
|
where pwsh >nul 2>nul
|
||||||
|
if %errorlevel%==0 (set shell=pwsh) else (set shell=powershell)
|
||||||
|
%shell% -NoProfile -ExecutionPolicy Bypass -File "%~dp0wsl-setup.ps1" %*
|
||||||
|
echo.
|
||||||
|
echo Open a new PowerShell for the commands to appear.
|
||||||
|
pause
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
<#
|
||||||
|
.SYNOPSIS
|
||||||
|
Once per checkout, on Windows: writes this checkout's short commands into
|
||||||
|
the PowerShell profile. See docs/Install WSL.md.
|
||||||
|
.DESCRIPTION
|
||||||
|
Reads DISTRO and CHECKOUT from deploy\release.env beside this script. The
|
||||||
|
prefix is the checkout folder's name (news-, boilerplate-), or -Name. Writes
|
||||||
|
one marked block into $PROFILE and replaces it when run again.
|
||||||
|
#>
|
||||||
|
param([string] $Name)
|
||||||
|
$ErrorActionPreference = 'Stop'
|
||||||
|
|
||||||
|
$envFile = Join-Path $PSScriptRoot 'release.env'
|
||||||
|
if (-not (Test-Path $envFile)) { throw "no $envFile - copy release.env.example and fill it in" }
|
||||||
|
$settings = @{}
|
||||||
|
foreach ($line in Get-Content $envFile) {
|
||||||
|
if ($line -match '^\s*(#|$)') { continue }
|
||||||
|
$key, $value = $line -split '=', 2
|
||||||
|
$settings[$key.Trim()] = "$value".Trim().Trim('"')
|
||||||
|
}
|
||||||
|
$distro = $settings.DISTRO
|
||||||
|
$checkout = "$($settings.CHECKOUT)".TrimEnd('/')
|
||||||
|
if (-not $distro -or -not $checkout) { throw "set DISTRO and CHECKOUT in $envFile" }
|
||||||
|
if (-not $Name) { $Name = Split-Path -Leaf (Split-Path -Parent $PSScriptRoot) }
|
||||||
|
|
||||||
|
# Replaces the block between the two marker lines, or appends it.
|
||||||
|
function Set-ProfileBlock([string] $Key, [string] $Body) {
|
||||||
|
$path = $PROFILE.CurrentUserCurrentHost
|
||||||
|
$begin = "# >>> $Key >>>"
|
||||||
|
$end = "# <<< $Key <<<"
|
||||||
|
$block = "$begin`r`n$Body`r`n$end"
|
||||||
|
$text = if (Test-Path $path) { [IO.File]::ReadAllText($path) } else { '' }
|
||||||
|
$pattern = '(?s)' + [regex]::Escape($begin) + '.*?' + [regex]::Escape($end)
|
||||||
|
if ($text -match $pattern) {
|
||||||
|
$text = [regex]::Replace($text, $pattern, { $block })
|
||||||
|
} else {
|
||||||
|
$text = $text.TrimEnd() + "`r`n`r`n$block`r`n"
|
||||||
|
}
|
||||||
|
New-Item -ItemType Directory -Force -Path (Split-Path $path) | Out-Null
|
||||||
|
[IO.File]::WriteAllText($path, $text)
|
||||||
|
Write-Host "profile: wrote $Key to $path"
|
||||||
|
}
|
||||||
|
|
||||||
|
$unc = "\\wsl.localhost\$distro" + ($checkout -replace '/', '\')
|
||||||
|
$wsl = "wsl.exe -d $distro --cd $checkout"
|
||||||
|
$ps = "& (Get-Process -Id `$PID).Path -NoProfile -ExecutionPolicy Bypass -File"
|
||||||
|
$lines = @(
|
||||||
|
"function $Name-cd { Set-Location '$unc' }"
|
||||||
|
"function $Name-sh { $wsl }"
|
||||||
|
"function $Name-backend { $wsl -- bash bin/backend.sh @args }"
|
||||||
|
# roledesk's additions: the PyCharm run configurations in .run/, from PowerShell.
|
||||||
|
"function $Name-pull { $wsl -- bash bin/pull.sh @args }"
|
||||||
|
"function $Name-worker { $wsl -- bash bin/worker.sh @args }"
|
||||||
|
"function $Name-check { $wsl -- bash bin/check.sh @args }"
|
||||||
|
"function $Name-1test { $wsl -- bash deploy/1-test-and-promote.sh @args }"
|
||||||
|
"function $Name-2build { $ps '$unc\deploy\2-build-image.ps1' }"
|
||||||
|
"function $Name-3push { $ps '$unc\deploy\3-push-image.ps1' }"
|
||||||
|
"function $Name-0release { $Name-1test @args; if (`$LASTEXITCODE -ne 0) { return }; $Name-2build; if (`$LASTEXITCODE -ne 0) { return }; $Name-3push }"
|
||||||
|
)
|
||||||
|
if (Test-Path (Join-Path $PSScriptRoot 'db.sh')) {
|
||||||
|
$lines += "function $Name-db { $wsl -- bash deploy/db.sh @args }"
|
||||||
|
}
|
||||||
|
Set-ProfileBlock "$Name (deploy/wsl-setup.ps1)" ($lines -join "`r`n")
|
||||||
|
Write-Host 'Open a new PowerShell window to load the profile.'
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# Once, inside the distro, for a checkout that lives in WSL: line endings, the
|
||||||
|
# Linux .venv, and a test run to prove it works. deploy/wsl-setup.ps1 is the
|
||||||
|
# Windows half. See docs/Install WSL.md, section 9 and Working from Windows.
|
||||||
|
# Usage: sh deploy/wsl-setup.sh (from the checkout root)
|
||||||
|
set -eu
|
||||||
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
|
|
||||||
|
# Windows Git (PyCharm) has core.autocrlf=true, so it writes CRLF into the
|
||||||
|
# working tree, and Linux Git then reports every line of those files changed.
|
||||||
|
# The repository's own setting wins over the Windows one for both.
|
||||||
|
git config core.autocrlf false
|
||||||
|
if [ -n "$(git diff --ignore-cr-at-eol --name-only)" ]; then
|
||||||
|
echo "uncommitted changes beyond line endings - commit or stash them first:" >&2
|
||||||
|
git diff --ignore-cr-at-eol --stat >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
# Rewrites the CRLF copies with LF. Nothing else differs, checked above.
|
||||||
|
git checkout -- .
|
||||||
|
|
||||||
|
uv sync --all-extras
|
||||||
|
uv run pytest -q
|
||||||
@@ -0,0 +1,213 @@
|
|||||||
|
> A copy of the boilerplate's `documentation/Install Server.md` at commit 09e144d, unedited below this line; the fix lands upstream first.
|
||||||
|
|
||||||
|
# Deploying a service
|
||||||
|
|
||||||
|
`<app>` below is your *service's* name: the hostname, the data directory, the image tag, the compose service. It is the one thing you choose. The Python package stays `app` and the environment variables stay `APP_`, here and in every project started from this repository — nothing inside the checkout is renamed, and the files these steps need are already in place under `deploy/`.
|
||||||
|
|
||||||
|
## What the box is
|
||||||
|
|
||||||
|
A single Hetzner server running Ubuntu 24.04 on x86_64, with a 37 GB root filesystem and no separate data disk. One Docker Compose project holds everything.
|
||||||
|
|
||||||
|
/srv/swag/docker-compose.yml every service, one file, project name `swag`
|
||||||
|
/srv/swag/.env every secret, chmod 600, never in the compose file
|
||||||
|
/srv/data/<app>/ every service's data, owned 1000:1000
|
||||||
|
/etc/config/swag/ SWAG's /config: nginx confs, certificates, logs
|
||||||
|
|
||||||
|
A snapshot of that whole compose file, every neighbour included, is [example-docker-compose.yml](example-docker-compose.yml): what a new service block lands beside, and where the SWAG subdomain list lives.
|
||||||
|
|
||||||
|
`swag` is nginx, certbot and fail2ban in one container. It terminates TLS for every hostname and proxies to the others **by container name** over the project's default network. That shared network is the only reason all these services live in one compose file — nothing else about them is related.
|
||||||
|
|
||||||
|
Images are published to the Gitea instance on the same box, at `git.rittershofer.org/wolf/<app>`. Everything runs as uid/gid 1000 and `TZ=Europe/Brussels`.
|
||||||
|
|
||||||
|
Which compose file that is comes off the running container rather than out of memory:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker inspect swag --format '{{index .Config.Labels "com.docker.compose.project.config_files"}}'
|
||||||
|
```
|
||||||
|
|
||||||
|
Declaring the network `external` from a second Compose project would work too, and would hardcode the first project's name into the second — a rename away from a confusing failure.
|
||||||
|
|
||||||
|
## The seven steps
|
||||||
|
|
||||||
|
**1. DNS.** `<app>.rittershofer.org` must resolve to the server *before* anything else. Confirm with `getent hosts <app>.rittershofer.org`.
|
||||||
|
|
||||||
|
**2. Certificate.** Add `<app>` to `SUBDOMAINS` in the swag service's environment (docker compose yaml). That list separates on commas and nothing else. SWAG runs certbot on every container start and requests **one certificate covering every name in the list**. If SWAG already holds a wildcard certificate covering the name, reuse it and skip this step entirely.
|
||||||
|
|
||||||
|
**3. Registry login.** On the build machine a Gitea token with package **write**; on the server, package **read**. Same command both ends, token at the password prompt:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker login git.rittershofer.org -u wolf
|
||||||
|
```
|
||||||
|
|
||||||
|
Git credentials and Docker registry credentials are separate. A
|
||||||
|
|
||||||
|
**4. Data directory.** Root's `mkdir` produces a root-owned directory, and the container runs as 1000, so the chown is not optional:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo mkdir -p /srv/data/<app>/home && sudo chown -R 1000:1000 /srv/data/<app>
|
||||||
|
```
|
||||||
|
|
||||||
|
**5. Service block** from [../deploy/server/compose.yaml](../deploy/server/compose.yaml) into `/srv/swag/docker-compose.yml`, the variables from [../deploy/server/env.example](../deploy/server/env.example) appended to `/srv/swag/.env`, and [../deploy/server/backup.sh](../deploy/server/backup.sh) next to the compose file for the nightly cron. Copy [../deploy/server/config.example.yaml](../deploy/server/config.example.yaml) to `$APP_DATA/config.yaml` with `rp_id` and `origin` set to the hostname: it holds the keys whose defaults are wrong in a container (`server.host: 0.0.0.0`, `database: /data/app.db`, `auth.enabled: always`, `auth.secure_cookie: always`). Without it the app binds loopback inside the container and writes its database into the image's root-owned folder.
|
||||||
|
|
||||||
|
**6. Proxy conf** — [../deploy/server/swag/app.subdomain.conf](../deploy/server/swag/app.subdomain.conf) renamed to `<app>.subdomain.conf` in `/etc/config/swag/nginx/proxy-confs/`. Then validate, and reload only if validation passes:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec swag nginx -t && docker exec swag nginx -s reload
|
||||||
|
```
|
||||||
|
|
||||||
|
Confirm the other sites still respond afterwards.
|
||||||
|
|
||||||
|
**7. Deploy.** Pull first, since `up` alone keeps the `:latest` it already has, and name the service, so nothing else in the file is recreated:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /srv/swag && docker compose pull <app> && docker compose up -d <app>
|
||||||
|
```
|
||||||
|
|
||||||
|
[../deploy/server/deploy.sh](../deploy/server/deploy.sh) is the same two commands with a check that `config.yaml` exists and sets `database` and `auth.enabled`, and a wait for the health check before it returns.
|
||||||
|
|
||||||
|
## What the templates already decide for you
|
||||||
|
|
||||||
|
Both templates are in [../deploy/server/](../deploy/server), commented in place. The decisions worth knowing before you read them:
|
||||||
|
|
||||||
|
**No `ports:`.** SWAG reaches the container by name on the project network. That is what keeps the app's port off the internet even if the firewall is wrong.
|
||||||
|
|
||||||
|
**No `networks:` either.** Being in that compose file is what puts it on the right one.
|
||||||
|
|
||||||
|
**`user: "1000:1000"`.** Container uid is host uid — there is no translation layer. Without this the process runs as whatever the image declares, usually root, and everything it writes into `/srv/data` lands root-owned.
|
||||||
|
|
||||||
|
**`mem_limit:`, not `deploy: resources: limits:`.** The latter is Swarm-only and silently ignored by `docker compose up`. 0 is unlimited, which is the honest default until the box has been watched under real load.
|
||||||
|
|
||||||
|
**`logging:` with a size cap.** Without it, json-file logs grow until the disk does not.
|
||||||
|
|
||||||
|
**`listen 443 ssl` without `http2`.** `http2 on;` is set once at `http` scope in `nginx.conf`, and the `listen ... http2` parameter is deprecated in nginx 1.25 and later.
|
||||||
|
|
||||||
|
**`proxy_read_timeout 86400s`** only if the app holds a WebSocket open. SWAG's `proxy.conf` already sets the Upgrade headers; what it also sets is a read timeout of a couple of minutes, which closes an idle socket on the minute. Delete those two lines if there is no socket.
|
||||||
|
|
||||||
|
# Publishing a release
|
||||||
|
|
||||||
|
The scripts are in place in `deploy/`, with `deploy/server/` beside them. The Dockerfile at `deploy/Dockerfile`, which is where `2-build-image.ps1` looks for it in the tree step 1 exports.
|
||||||
|
|
||||||
|
The release is split where the machines are. The checkout, its `.venv` and the tests are Linux, so the gate, step 1, is bash and runs inside the WSL distro. The distro is a sealed sandbox (Windows interop off, no drive mounts, no container engine, see [Install WSL.md, 7. Cut the distro off from Windows](Install%20WSL.md#7-cut-the-distro-off-from-windows)) and stays that way, because a sandbox that can drive containers can mount the host. Step 1 ends by exporting the production tree to `release/<folder>/<sha>/` beside the checkout, and the image is built and pushed from Windows with `wslc`, the container CLI that ships with WSL: steps 2 and 3, two short PowerShell scripts that read that tree through `\\wsl.localhost`. The distro never holds a container socket.
|
||||||
|
|
||||||
|
## Every release
|
||||||
|
|
||||||
|
From Windows, in PowerShell, once `deploy\wsl-setup.cmd` has installed the profile commands:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
<folder>-0release
|
||||||
|
```
|
||||||
|
|
||||||
|
where `<folder>` is the checkout folder's name (`boilerplate` here). It runs `<folder>-1test`, `<folder>-2build` and `<folder>-3push` in order and stops at the first that fails; its arguments (`--push-branch`, `--no-tests`) go to step 1. Each step is a command of its own:
|
||||||
|
|
||||||
|
```text
|
||||||
|
<folder>-1test bash deploy/1-test-and-promote.sh in WSL: test local main, fast-forward local production, export it
|
||||||
|
<folder>-2build deploy\2-build-image.ps1 on Windows: build the exported tree with wslc
|
||||||
|
<folder>-3push deploy\3-push-image.ps1 on Windows: push the built image to the registry
|
||||||
|
```
|
||||||
|
|
||||||
|
The number is the order. Each script can be run on its own and works out its paths for itself, so the working directory does not matter.
|
||||||
|
|
||||||
|
**Step 1 never builds; step 2 never pushes; step 3 never rebuilds or deploys.** That separation is the point of the numbering: a failed push is `3` again, not the whole thing again.
|
||||||
|
|
||||||
|
## What each step is actually doing
|
||||||
|
|
||||||
|
**1: test and promote.** Resolves `main` to a commit, checks that commit out into a throwaway worktree, and runs the project's checks there. The worktree is what makes this honest: the tests run against exactly the tree that is about to become production, with none of the working copy's uncommitted edits in the way. Worktrees have no `.venv` of their own, so the interpreter comes from the main checkout and the code from the worktree.
|
||||||
|
|
||||||
|
The checks are pytest, then `npm ci` and `npm run check` in `frontend/`: lint, format, the generated types, the page's tests and a build. So the machine that runs step 1 needs Node, which `frontend/install.sh` installs once. Step 1 runs the gates through `deploy/test-commit.sh`, which looks for Node from nvm and from fnm, since a non-interactive shell has not read the rc file that puts it on `PATH`. `PYTHON` in `release.env` names the interpreter when it is not `.venv/bin/python`; `--no-tests` skips the gates.
|
||||||
|
|
||||||
|
Promotion is a fast-forward only. If `production` has commits `main` does not, that is a release made from somewhere else, and the script stops rather than discarding it.
|
||||||
|
|
||||||
|
Git publication is optional: run `git push origin production` yourself, or pass `--push-branch` to step 1 or `<folder>-0release`. `production` identifies the tested candidate; the server's running image identifies the deployed release.
|
||||||
|
|
||||||
|
Step 1 then exports the production commit: `git archive` unpacked into `release/<folder>/<sha>/` beside the checkout (the environment variable `RELEASE_DIR` names another folder for `release/`), with `deploy/Dockerfile.dockerignore` copied to `.dockerignore` at its root, and the short sha written to `release/<folder>/production`. Older trees are removed; a tree that already exists for this sha is kept, because `wslc` can build from a stale view of a folder deleted and recreated at the same path ([microsoft/WSL#41287](https://github.com/microsoft/WSL/issues/41287)). When the folder cannot be written, step 1 says so and fails after the promote; run it again.
|
||||||
|
|
||||||
|
**2: build.** Reads the sha from `production` and runs `wslc build -t <IMAGE>:<sha> -t <IMAGE>:latest -f <tree>\deploy\Dockerfile <tree>` with the tree's `\\wsl.localhost` path. `-f` is a path `wslc` opens on Windows and hands to the builder on stdin, which is why the ignore rules have to be at the root of the context. The image builds the React page itself, in a Node stage from `frontend/` and its lock file, so neither Node nor a built `app/web/dist/` is needed for this step.
|
||||||
|
|
||||||
|
`wslc` builds an **image**. No container starts or restarts on this machine.
|
||||||
|
|
||||||
|
**3: push.** Re-tags `latest` onto this candidate with `wslc tag`, then `wslc push`es both. It never rebuilds, so a push that failed on connectivity is just this step again. A refused push prints the `wslc login` line to use.
|
||||||
|
|
||||||
|
# Proposal - Operating it
|
||||||
|
|
||||||
|
[../deploy/server/backup.sh](../deploy/server/backup.sh), [../deploy/server/import.sh](../deploy/server/import.sh) and [../deploy/server/deploy.sh](../deploy/server/deploy.sh) name the compose service in `SERVICE=` at their top, and `NAME` in `deploy/release.env` names it for [../deploy/db.sh](../deploy/db.sh). Each has to be `<app>`. All of them read `APP_DATA` from `/srv/swag/.env`.
|
||||||
|
|
||||||
|
## The nightly backup
|
||||||
|
|
||||||
|
`backup.sh` beside the compose file takes one snapshot through the running container. Inside it, the app's own config names the database, which is opened read-only, so a path that names no file is an error rather than a new empty database. `VACUUM INTO` writes `/data/backups/<app>-<stamp>.db` as one consistent, compacted file without blocking a writer for long. The snapshot is reopened read-only and must hold every table in `TABLES` (`jobs ticks`; a test keeps it equal to `REQUIRED_TABLES` in [../deploy/server/import_database.py](../deploy/server/import_database.py)); otherwise it is deleted and the run fails. A project that replaces `ticks` with its own first table changes both. The snapshot is gzipped under `umask 077`, so every backup is owner-only, and everything past the newest `KEEP` (14 by default) is deleted.
|
||||||
|
|
||||||
|
Schedule it in a crontab that can run `docker compose`:
|
||||||
|
|
||||||
|
```
|
||||||
|
0 4 * * * bash /srv/swag/backup.sh >> /srv/swag/backup.log 2>&1
|
||||||
|
```
|
||||||
|
|
||||||
|
The container has to be running, since the snapshot is taken through `docker compose exec`. A backup on the same disk is not an off-server copy; `db.sh pull` below makes one.
|
||||||
|
|
||||||
|
## Moving the database between laptop and server
|
||||||
|
|
||||||
|
`deploy/db.sh` runs in the checkout, with the `ssh` and `scp` of that shell, and reads `SSH_HOST`, `REMOTE_DIR` (`/srv/swag`), `REMOTE_DATA` (`/srv/data/<app>`) and `NAME` from `deploy/release.env`.
|
||||||
|
|
||||||
|
`bash deploy/db.sh pull` runs the server's own `backup.sh` over ssh and downloads the newest `<app>-*.db.gz` into `backups/` in the checkout. Use it as a read-only copy. Pushing it back after the server has written discards those writes, and nothing merges two copies.
|
||||||
|
|
||||||
|
Pushing the laptop's database onto the server replaces the server's, including the password hash, sessions, passkeys and API tokens, because the `auth_*` tables live in the same file. Set the account up against the final database, not before the push.
|
||||||
|
|
||||||
|
**1. Stop the app.** The import is a file swap under SQLite and `import.sh` refuses a running service:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /srv/swag && docker compose stop <app>
|
||||||
|
```
|
||||||
|
|
||||||
|
**2. Push.** From the checkout:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bash deploy/db.sh push
|
||||||
|
```
|
||||||
|
|
||||||
|
It snapshots the local database (the one the checkout's config names, or `LOCAL_DB`) with `VACUUM INTO`, gzips it, asks before going on unless given `--yes`, uploads it as `$REMOTE_DATA/incoming.db.gz` and `import_database.py` into `$REMOTE_DIR`, and streams `import.sh` over ssh. On the server, `import.sh` runs `import_database.py` in a one-off container of the image already there (`--pull never`). The importer decompresses into a temporary file beside the database and requires `PRAGMA integrity_check` to answer `ok` and every table in `REQUIRED_TABLES` to exist. Only then does it snapshot the current database with `VACUUM INTO` into `/data/backups/<database name>-<stamp>.db.gz`, owner-only, refuse if `wal_checkpoint(TRUNCATE)` finds another connection, delete the `-wal` and `-shm` files and move the new file into place with one `os.replace`. It prints the snapshot's path. For `/data/app.db` the snapshot is `app-<stamp>.db.gz`, which `backup.sh` prunes with its own only while the service is also called `app`.
|
||||||
|
|
||||||
|
**3. Start the app.**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /srv/swag && docker compose up -d --wait <app>
|
||||||
|
```
|
||||||
|
|
||||||
|
## Restoring a backup on the server
|
||||||
|
|
||||||
|
The same import, by hand, with a file from `$APP_DATA/backups` in place of the pushed one. `import.sh` and `import_database.py` from `deploy/server/` have to be beside the compose file; `db.sh push` leaves the importer there but streams `import.sh`.
|
||||||
|
|
||||||
|
**1. Stop the app.** `cd /srv/swag && docker compose stop <app>`
|
||||||
|
|
||||||
|
**2. Stage the backup** as the incoming file:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo -u '#1000' cp /srv/data/<app>/backups/<app>-<stamp>.db.gz /srv/data/<app>/incoming.db.gz
|
||||||
|
```
|
||||||
|
|
||||||
|
The backups are owner-only and belong to uid 1000, which the importer runs as; a copy made as root is one it cannot read.
|
||||||
|
|
||||||
|
**3. Import.** `bash /srv/swag/import.sh /srv/swag` checks it, snapshots the current database and replaces it, as in step 2 above.
|
||||||
|
|
||||||
|
**4. Start the app.** `cd /srv/swag && docker compose up -d --wait <app>`
|
||||||
|
|
||||||
|
## Rolling back to a pinned image
|
||||||
|
|
||||||
|
Every release pushes `<IMAGE>:<sha>` and `<IMAGE>:latest`, where `<sha>` is the short hash of the promoted commit. `latest` moves on every release, so it cannot name an earlier one; the compose file runs `${APP_IMAGE}:${APP_TAG:-latest}`, and `APP_TAG` in `.env` pins a commit tag.
|
||||||
|
|
||||||
|
**1. Find the tag.** The image's package page in Gitea lists every pushed tag with its date. A tag removed by the package cleanup rule cannot be rolled back to.
|
||||||
|
|
||||||
|
**2. Pin it.** Set `APP_TAG=<sha>` in `/srv/swag/.env`.
|
||||||
|
|
||||||
|
**3. Deploy.** `bash /srv/swag/deploy.sh` pulls the pinned image, which the weekly prune removes from the box once no container runs it, and recreates `<app>` with it.
|
||||||
|
|
||||||
|
**4. Unpin** with `APP_TAG=latest` and deploy again once a fixed release is out.
|
||||||
|
|
||||||
|
A rollback replaces the code, not the database. The queue's tables only ever gain columns, each nullable or with a default; `pipeline/table.py` adds them and never drops one. A project whose migration changed the schema in a way the older code cannot read restores a backup taken before that release, with the app stopped, as above.
|
||||||
|
|
||||||
|
# Pitfalls
|
||||||
|
|
||||||
|
**Disk.** 37 GB, and it has been full. Four causes, all recurring. `:latest` orphans the previous image on every pull and nothing cleans them up; root's crontab runs `docker image prune -af` every Sunday at 04:00, and the `-a` also removes a pinned image that is not running, so a rollback tag must be pulled again. Gitea's package store under `/srv/data/gitea/gitea/packages` keeps every pushed `:sha` and `:latest` version of every image forever: it was 11 GB on 16 October 2026 after a week of newsdesk releases at 2.6 GB each, and the prune cron never touches it. A cleanup rule per owner at `/user/settings/packages/rules` (keep the most recent n, **container** type) is applied by Gitea's `cleanup_packages` cron at midnight, which also drops blobs unreferenced for 24 hours; `/-/admin/packages` lists every version with a delete button and a Cleanup expired data button runs the same job now. Journald defaults to 10% of the filesystem and is capped here at 200 MB. VS Code Remote installs a fresh server per version under `~/.vscode-server/cli/servers` forever. [disk_usage.sh](disk_usage.sh) prints all of it, read-only.
|
||||||
|
|
||||||
|
**SQLite is never safe to copy live.** Stop the container, or snapshot with `VACUUM INTO` through the running one. A `cp` of a WAL-mode database is a coin toss.
|
||||||
|
|
||||||
|
**Secrets in `.env`, never in the compose file.** It is easy to paste a compose file into a chat window; it is harder to do that by accident with a file you had to `cat` deliberately.
|
||||||
|
|
||||||
@@ -0,0 +1,408 @@
|
|||||||
|
> A copy of the boilerplate's `documentation/Install WSL.md` at commit 09e144d, unedited below this line; the fix lands upstream first.
|
||||||
|
|
||||||
|
# Abstract
|
||||||
|
|
||||||
|
The sandbox is a WSL distro. Claude Code runs inside it from the desktop app with all permissions, because the distro cannot reach anything on Windows. Interops is turned off, no drives are mounted, and to keep it this way admin permissions are not granted.
|
||||||
|
|
||||||
|
Root is reached only from Windows, with `wsl -d claudebox -u root`. That is how you install and update packages later.
|
||||||
|
|
||||||
|
The repos live inside the distro in `/home/claude/cboxcode`. [Working from Windows](#working-from-windows) covers working with them from Windows.
|
||||||
|
|
||||||
|
# Install
|
||||||
|
|
||||||
|
## 1. Check that WSL 2 runs
|
||||||
|
|
||||||
|
```
|
||||||
|
PS> wsl --status
|
||||||
|
```
|
||||||
|
|
||||||
|
It should say `Default Version: 2` and nothing about virtualisation.
|
||||||
|
|
||||||
|
If it says virtualisation is not enabled, run `wsl.exe --install --no-distribution` once in an admin PowerShell and reboot. If it still complains, switch on Intel VT-x or AMD-V in the BIOS. This is the only step that can need admin rights.
|
||||||
|
|
||||||
|
If a `claudebox` from an earlier attempt exists, `wsl -l -v` lists it. `wsl --unregister claudebox` deletes it with everything inside.
|
||||||
|
|
||||||
|
## 2. Install Ubuntu as `claudebox`
|
||||||
|
|
||||||
|
```
|
||||||
|
PS> wsl --install Ubuntu-24.04 --name claudebox --no-launch --web-download
|
||||||
|
```
|
||||||
|
|
||||||
|
- `--name claudebox` names the distro, so it does not collide with an ordinary Ubuntu.
|
||||||
|
- `--no-launch` skips Ubuntu's first-start wizard, which would create a user with sudo. We create our own user in step 5.
|
||||||
|
- `--web-download` downloads it directly instead of through the Microsoft Store.
|
||||||
|
|
||||||
|
## 3. Open a root shell
|
||||||
|
|
||||||
|
```
|
||||||
|
PS> wsl -d claudebox -u root
|
||||||
|
```
|
||||||
|
|
||||||
|
The prompt changes to something like `root@PC:/mnt/c/...#`. You are now inside Linux as root. `C:` is still visible at this point; step 7 takes it away.
|
||||||
|
|
||||||
|
## 4. Update and install packages
|
||||||
|
|
||||||
|
```
|
||||||
|
# apt update
|
||||||
|
# apt upgrade -y
|
||||||
|
# apt install -y build-essential git curl ripgrep sqlite3 jq unzip
|
||||||
|
```
|
||||||
|
|
||||||
|
`apt update` fetches the package lists, `apt upgrade` installs updates, and `apt install` adds the tools: compilers for Python packages that build from source, git, curl, ripgrep for Claude's searches, sqlite3 to look into the databases, jq for JSON, unzip.
|
||||||
|
|
||||||
|
## 5. Create the user `claude`, without sudo
|
||||||
|
|
||||||
|
```
|
||||||
|
# useradd --create-home --shell /bin/bash claude
|
||||||
|
# groups claude
|
||||||
|
```
|
||||||
|
|
||||||
|
`useradd` creates the user with a home directory and bash. It gets no password and no groups. `groups claude` must print only `claude : claude`. If `sudo` appears there, the user can become root, and `gpasswd -d claude sudo` removes it.
|
||||||
|
|
||||||
|
```
|
||||||
|
# passwd -l root
|
||||||
|
```
|
||||||
|
|
||||||
|
This locks root's password, so `su` cannot become root from inside either. Ubuntu ships it locked, so this only makes sure.
|
||||||
|
|
||||||
|
## 6. Install uv and Claude Code as `claude`
|
||||||
|
|
||||||
|
Switch to the user. Root can do that without a password:
|
||||||
|
|
||||||
|
```
|
||||||
|
# su - claude
|
||||||
|
```
|
||||||
|
|
||||||
|
The prompt ends in `$` now. Install the two tools into `~/.local/bin`, which belongs to `claude`, so they update without root:
|
||||||
|
|
||||||
|
```
|
||||||
|
$ curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||||
|
$ curl -fsSL https://claude.ai/install.sh | bash
|
||||||
|
$ echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc
|
||||||
|
```
|
||||||
|
|
||||||
|
The third line puts `~/.local/bin` on the path for every new shell.
|
||||||
|
|
||||||
|
Set the git identity, with your own name and email, and create the folder for the repos:
|
||||||
|
|
||||||
|
```
|
||||||
|
$ git config --global user.name "Your Name"
|
||||||
|
$ git config --global user.email "you@example.com"
|
||||||
|
$ git config --global init.defaultBranch main
|
||||||
|
$ mkdir ~/cboxcode
|
||||||
|
```
|
||||||
|
|
||||||
|
Back to root:
|
||||||
|
|
||||||
|
```
|
||||||
|
$ exit
|
||||||
|
```
|
||||||
|
|
||||||
|
## 7. Cut the distro off from Windows
|
||||||
|
|
||||||
|
`/etc/wsl.conf` is the file WSL reads when the distro starts. Open it in the editor:
|
||||||
|
|
||||||
|
```
|
||||||
|
# nano /etc/wsl.conf
|
||||||
|
```
|
||||||
|
|
||||||
|
Delete what is there and type this:
|
||||||
|
|
||||||
|
```
|
||||||
|
[boot]
|
||||||
|
systemd=true
|
||||||
|
|
||||||
|
[user]
|
||||||
|
default=claude
|
||||||
|
|
||||||
|
[automount]
|
||||||
|
enabled=false
|
||||||
|
mountFsTab=false
|
||||||
|
|
||||||
|
[interop]
|
||||||
|
enabled=false
|
||||||
|
appendWindowsPath=false
|
||||||
|
```
|
||||||
|
|
||||||
|
- `[user] default=claude` makes every normal start log in as `claude`, not root.
|
||||||
|
- `[automount] enabled=false` stops WSL from mounting `C:` and the other drives under `/mnt`. `mountFsTab=false` also ignores `/etc/fstab`, so no other way of mounting them applies.
|
||||||
|
- `[interop] enabled=false` stops Linux from starting Windows programs. That includes `wsl.exe`, which would otherwise be a way back to root. `appendWindowsPath=false` keeps the Windows folders off the Linux path.
|
||||||
|
- `[boot] systemd=true` is Ubuntu's default and stays.
|
||||||
|
|
||||||
|
Save with `Ctrl+O` then `Enter`, quit with `Ctrl+X`.
|
||||||
|
|
||||||
|
The first start in step 3 left an empty folder per Windows drive under `/mnt`. Nothing will be mounted on them any more, so remove them to avoid confusion. `rmdir` only removes empty folders, so it cannot delete anything:
|
||||||
|
|
||||||
|
```
|
||||||
|
# rmdir /mnt/c /mnt/d
|
||||||
|
```
|
||||||
|
|
||||||
|
Name every drive letter that `ls /mnt` shows. `wsl` and `wslg` belong to WSL and stay. Leave the distro:
|
||||||
|
|
||||||
|
```
|
||||||
|
# exit
|
||||||
|
```
|
||||||
|
|
||||||
|
WSL only reads the file on a fresh start, so stop the distro:
|
||||||
|
|
||||||
|
```
|
||||||
|
PS> wsl --terminate claudebox
|
||||||
|
```
|
||||||
|
|
||||||
|
## 8. Check the isolation
|
||||||
|
|
||||||
|
Start it the normal way, which now logs in as `claude`:
|
||||||
|
|
||||||
|
```
|
||||||
|
PS> wsl -d claudebox
|
||||||
|
```
|
||||||
|
|
||||||
|
Each check and what it must answer:
|
||||||
|
|
||||||
|
```
|
||||||
|
$ whoami
|
||||||
|
claude
|
||||||
|
$ groups
|
||||||
|
claude
|
||||||
|
$ passwd -S claude
|
||||||
|
claude L ...
|
||||||
|
$ sudo -n true
|
||||||
|
sudo: a password is required
|
||||||
|
$ ls /mnt
|
||||||
|
wsl wslg
|
||||||
|
$ uv --version
|
||||||
|
$ claude --version
|
||||||
|
```
|
||||||
|
|
||||||
|
- `groups` showing only `claude` means no admin group, so sudo refuses even with a password.
|
||||||
|
- `L` in `passwd -S` means the password is locked: no password exists, so nothing can be typed at a sudo or su prompt that works. A plain `sudo` still asks for one, and any answer fails.
|
||||||
|
- `ls /mnt` showing no drive letters means no Windows drive is there. `mount | grep -i drvfs` printing nothing confirms it.
|
||||||
|
- The last two print version numbers.
|
||||||
|
|
||||||
|
The real test that Windows programs cannot start needs one. In a second PowerShell window, copy a harmless one in from Windows, run it inside, and delete it again:
|
||||||
|
|
||||||
|
```
|
||||||
|
PS> Copy-Item C:\Windows\System32\whoami.exe \\wsl.localhost\claudebox\tmp\
|
||||||
|
PS> wsl -d claudebox -- /tmp/whoami.exe
|
||||||
|
PS> Remove-Item \\wsl.localhost\claudebox\tmp\whoami.exe
|
||||||
|
```
|
||||||
|
|
||||||
|
With interop off it prints no Windows user name, only a WSL error after a few seconds. If it prints your Windows name, step 7 did not take.
|
||||||
|
|
||||||
|
If any check answers differently, the step it belongs to did not take: 5 for `groups` and `passwd`, 7 for `/mnt` and `whoami.exe`.
|
||||||
|
|
||||||
|
```
|
||||||
|
$ exit
|
||||||
|
```
|
||||||
|
|
||||||
|
## PowerShell 7
|
||||||
|
|
||||||
|
Our scripts are running on powershell 7, which needs to be installed.
|
||||||
|
|
||||||
|
Then, set it to the default profile in the terminal app.
|
||||||
|
|
||||||
|
```
|
||||||
|
PS> winget install Microsoft.PowerShell
|
||||||
|
```
|
||||||
|
|
||||||
|
## The execution policy
|
||||||
|
|
||||||
|
Running commands in the WSL need extra permissions:
|
||||||
|
|
||||||
|
```
|
||||||
|
PS> pwsh -ExecutionPolicy Bypass -File <command>
|
||||||
|
```
|
||||||
|
|
||||||
|
This is one reason why we add all important commands into the profile
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## PyCharm
|
||||||
|
|
||||||
|
### Remote execution agent failure
|
||||||
|
|
||||||
|
PyCharm on Windows is not able to read folders on the cut-off WSL. Its bundled plugin Remote Execution Agent copies a helper called IJent into the distro from `/mnt/c/Program Files/JetBrains/...` and reads files and runs Git through it. With the drives unmounted the copy fails: opening a file reports "unable to locate", Git reports `execvpe(.../ttyfix) failed`, and `idea.log` shows `cp: cannot stat '/mnt/c/...ijent...'`.
|
||||||
|
|
||||||
|
So you need to disable the plugin: Settings > Plugins > Installed, search for `Remote Execution Agent`, untick it, restart PyCharm.
|
||||||
|
|
||||||
|
### Git
|
||||||
|
|
||||||
|
Set Settings > Version Control > Git > Path to Git executable to `C:\Program Files\Git\cmd\git.exe`, so it uses Windows Git instead of trying to start the distro's.
|
||||||
|
|
||||||
|
### Run configurations
|
||||||
|
|
||||||
|
We currently don't use them.
|
||||||
|
|
||||||
|
## Maintenance
|
||||||
|
|
||||||
|
### Update
|
||||||
|
|
||||||
|
Packages, as root from Windows:
|
||||||
|
|
||||||
|
```
|
||||||
|
PS> wsl -d claudebox -u root -- apt update
|
||||||
|
PS> wsl -d claudebox -u root -- apt upgrade -y
|
||||||
|
```
|
||||||
|
|
||||||
|
Claude Code updates itself. uv updates with the line below. The full path is needed because a command passed from PowerShell does not load `~/.bashrc`:
|
||||||
|
|
||||||
|
```
|
||||||
|
PS> wsl -d claudebox -- /home/claude/.local/bin/uv self update
|
||||||
|
```
|
||||||
|
|
||||||
|
### Remove WSL
|
||||||
|
|
||||||
|
Remove the distro with all repos in it, after pushing what matters:
|
||||||
|
|
||||||
|
```
|
||||||
|
PS> wsl --unregister claudebox
|
||||||
|
```
|
||||||
|
|
||||||
|
# Getting it ready for use
|
||||||
|
|
||||||
|
## Git
|
||||||
|
|
||||||
|
Tell Windows Git to trust that folder, since its files belong to the Linux user:
|
||||||
|
|
||||||
|
```
|
||||||
|
PS> git config --global --add safe.directory '%(prefix)///wsl.localhost/claudebox/home/claude/cboxcode/news'
|
||||||
|
```
|
||||||
|
|
||||||
|
`git config core.autocrlf false` stops Git from converting line endings in this repo. `git checkout -- .` rewrites the files Windows Git checked out with Windows line endings, so Linux Git does not report every line as changed.
|
||||||
|
|
||||||
|
## Working from Windows
|
||||||
|
|
||||||
|
The only direction that works is Windows into WSL: Windows reads and writes the checkout through `\\wsl.localhost\<distro>\...` and starts processes with `wsl.exe`. The distro never reaches back. Editing and the Git UI happen on Windows; the agents, the tests, `serve`, the jobs, `uv sync`, release step 1 and `db.sh` run in the distro, typed from Windows through the profile commands below; release steps 2 and 3 run on Windows with `wslc`, since the distro must not hold a container socket.
|
||||||
|
|
||||||
|
A `.venv` belongs to one OS: `bin/python` on Linux, `Scripts\python.exe` on Windows. The `.venv` in the checkout is the distro's. A Windows `uv sync` or `uv run` in that folder would replace it with a Windows one, and the distro's next `uv run` would swap it back, so run `uv` in the distro (`<folder>-sh` opens a shell there). A Windows interpreter for an editor's code insight belongs outside the checkout.
|
||||||
|
|
||||||
|
Run the app in the distro, not from Windows. Only one side may open a SQLite file at a time, since locking through the share is unreliable, and file change notifications through the share are unreliable too, so `--reload` started from Windows can miss edits.
|
||||||
|
|
||||||
|
## WSL Containers for the image
|
||||||
|
|
||||||
|
We don't need Docker Desktop but build images directly on WSL with `wslc`. It needs WSL 2.9.3 or later:
|
||||||
|
|
||||||
|
```
|
||||||
|
PS> wsl --update
|
||||||
|
PS> wsl --version
|
||||||
|
PS> wslc version
|
||||||
|
```
|
||||||
|
|
||||||
|
Log it in to the registry once, with a token that has package-write access, typed at its password prompt:
|
||||||
|
|
||||||
|
```
|
||||||
|
PS> wslc login git.rittershofer.org -u wolf
|
||||||
|
```
|
||||||
|
|
||||||
|
## Env files
|
||||||
|
|
||||||
|
Once per checkout, including every new repo, and again when `DISTRO` or `CHECKOUT` changes:
|
||||||
|
|
||||||
|
1. Copy `deploy/release.env.example` to `deploy/release.env` in the checkout (`cp` in the distro, or Explorer) and fill in `DISTRO` (`claudebox`), `CHECKOUT` (the path in the distro, `/home/claude/cboxcode/news`) and `IMAGE`. The file is gitignored.
|
||||||
|
|
||||||
|
## PS Profile commands
|
||||||
|
|
||||||
|
We want a simple way to run and deploy the app, `cd` into the repo. So we have script adding the relevant commands to the PS profile:
|
||||||
|
|
||||||
|
```
|
||||||
|
PS> pwsh -ExecutionPolicy Bypass -File \\wsl.localhost\claudebox\home\claude\cboxcode\news\deploy\wsl-setup.ps1
|
||||||
|
```
|
||||||
|
|
||||||
|
It creates commands into the profie
|
||||||
|
|
||||||
|
The prefix is the checkout folder's name, `news-` for `news` and `boilerplate-` for `boilerplate`; `-Name` on the `.ps1` sets another. The setup writes one block between `# >>> news (deploy/wsl-setup.ps1) >>>` and `# <<< news (deploy/wsl-setup.ps1) <<<` into `$PROFILE` and replaces it when run again. Blocks written by older versions of the setup, `newsdesk (deploy/wsl-setup.ps1)` and `wsl shared (deploy/wsl-setup.ps1)`, call scripts that no longer exist; delete them with `notepad $PROFILE`.
|
||||||
|
|
||||||
|
| Command | Runs | Does |
|
||||||
|
|---|---|---|
|
||||||
|
| `news-cd` | `Set-Location` | PowerShell into the checkout's folder |
|
||||||
|
| `news-sh` | `wsl.exe` | a shell in claudebox, in the checkout |
|
||||||
|
| `news-backend [-v]` | `bin/backend.sh` in claudebox | `uv run app serve` on :8787, until Ctrl-C |
|
||||||
|
| `news-frontend` | `bin/frontend.sh` in claudebox | Vite with hot reload on :5173, until Ctrl-C |
|
||||||
|
| `news-check` | `bin/check.sh` in claudebox | the gate on the working tree, without promoting |
|
||||||
|
| `news-1test [--push-branch] [--no-tests]` | `deploy/1-test-and-promote.sh` in claudebox | test local main, fast-forward `production`, export it |
|
||||||
|
| `news-2build` | `deploy\2-build-image.ps1` on Windows | build the exported tree with `wslc`, tags `<sha>` and `latest` |
|
||||||
|
| `news-3push` | `deploy\3-push-image.ps1` on Windows | push both tags with `wslc` |
|
||||||
|
| `news-0release [--push-branch] [--no-tests]` | the three above | `1test`, `2build`, `3push`, stopping at the first that fails |
|
||||||
|
| `<folder>-db pull\|push` | `deploy/db.sh` in claudebox | only where `deploy/db.sh` exists, as in the boilerplate |
|
||||||
|
|
||||||
|
If the setup does not run, paste the block by hand. `New-Item -Force $PROFILE` creates the file if `Test-Path $PROFILE` says it is missing; `notepad $PROFILE` opens it. For `news`, with `DISTRO=claudebox` and `CHECKOUT=/home/claude/cboxcode/news`, the setup writes exactly this:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
# >>> news (deploy/wsl-setup.ps1) >>>
|
||||||
|
function news-cd { Set-Location '\\wsl.localhost\claudebox\home\claude\cboxcode\news' }
|
||||||
|
function news-sh { wsl.exe -d claudebox --cd /home/claude/cboxcode/news }
|
||||||
|
function news-backend { wsl.exe -d claudebox --cd /home/claude/cboxcode/news -- bash bin/backend.sh @args }
|
||||||
|
function news-frontend { wsl.exe -d claudebox --cd /home/claude/cboxcode/news -- bash bin/frontend.sh @args }
|
||||||
|
function news-check { wsl.exe -d claudebox --cd /home/claude/cboxcode/news -- bash bin/check.sh @args }
|
||||||
|
function news-1test { wsl.exe -d claudebox --cd /home/claude/cboxcode/news -- bash deploy/1-test-and-promote.sh @args }
|
||||||
|
function news-2build { & (Get-Process -Id $PID).Path -NoProfile -ExecutionPolicy Bypass -File '\\wsl.localhost\claudebox\home\claude\cboxcode\news\deploy\2-build-image.ps1' }
|
||||||
|
function news-3push { & (Get-Process -Id $PID).Path -NoProfile -ExecutionPolicy Bypass -File '\\wsl.localhost\claudebox\home\claude\cboxcode\news\deploy\3-push-image.ps1' }
|
||||||
|
function news-0release { news-1test @args; if ($LASTEXITCODE -ne 0) { return }; news-2build; if ($LASTEXITCODE -ne 0) { return }; news-3push }
|
||||||
|
# <<< news (deploy/wsl-setup.ps1) <<<
|
||||||
|
```
|
||||||
|
|
||||||
|
For another checkout, replace `news` and the path.
|
||||||
|
|
||||||
|
### When a command fails
|
||||||
|
|
||||||
|
- `wsl -l -v` lists the distros. `DISTRO` must be one of the names exactly.
|
||||||
|
- `Test-Path \\wsl.localhost\claudebox\home\claude\cboxcode\news` must say `True`. `\\wsl$\claudebox\...` is the older spelling of the same share; if only that one answers, use it in the block.
|
||||||
|
- `$PROFILE` prints the profile file this PowerShell loads, and `Get-Command news-*` lists the commands it defined. Commands missing in one PowerShell and present in another means the setup ran in the other one.
|
||||||
|
- `Get-ExecutionPolicy -List` shows the policy per scope. `Restricted` or `AllSigned` where it applies means the profile does not load; a `MachinePolicy` or `UserPolicy` entry is group policy and overrides `CurrentUser`.
|
||||||
|
- `wslc version` failing means WSL is older than 2.9.3: `wsl --update`.
|
||||||
|
- `news-2build` saying there is no `production` file means step 1 has not exported yet: run `news-1test`.
|
||||||
|
- If `wslc build` refuses the `\\wsl.localhost` folder as its context, which Microsoft's documentation neither promises nor rules out, copy the tree to Windows and build there: `Copy-Item -Recurse \\wsl.localhost\claudebox\home\claude\cboxcode\release\news\<sha> $env:TEMP\release-<sha>`, then the `wslc build` line of `2-build-image.ps1` with that folder.
|
||||||
|
|
||||||
|
# Kill and restart
|
||||||
|
|
||||||
|
Run everything in an **admin PowerShell** on Windows.
|
||||||
|
|
||||||
|
### Normal shutdown
|
||||||
|
|
||||||
|
`wsl --shutdown`
|
||||||
|
|
||||||
|
### Killing it
|
||||||
|
|
||||||
|
If normal shutdown hangs:
|
||||||
|
|
||||||
|
```
|
||||||
|
taskkill /f /im wsl.exe
|
||||||
|
taskkill /f /im wslhost.exe
|
||||||
|
taskkill /f /im wslrelay.exe
|
||||||
|
```
|
||||||
|
|
||||||
|
### Check the service
|
||||||
|
|
||||||
|
`sc queryex WslService`
|
||||||
|
|
||||||
|
If it says `STOP_PENDING` or `START_PENDING`, take the PID from the output and run `taskkill /PID <PID> /F`.
|
||||||
|
|
||||||
|
If it says `STOPPED`, start it with `sc start WslService`, then run `wsl`.
|
||||||
|
|
||||||
|
### **Kill a lingering VM**
|
||||||
|
|
||||||
|
IF it still does not work check the VM
|
||||||
|
|
||||||
|
```
|
||||||
|
hcsdiag list
|
||||||
|
hcsdiag kill <ID>
|
||||||
|
```
|
||||||
|
|
||||||
|
If the old WSL VM is still listed, kill it with the ID shown. This was reported as the only fix short of a reboot in a complete-freeze case.
|
||||||
|
|
||||||
|
### **Restart the Hyper-V Host Compute Service**
|
||||||
|
|
||||||
|
```
|
||||||
|
Restart-Service vmcompute
|
||||||
|
```
|
||||||
|
|
||||||
|
A wedged vmcompute service is a common reason WSL stops working after it ran fine earlier in the same Windows session.
|
||||||
|
|
||||||
|
If it still hangs after all four, a reboot is the reliable fix.
|
||||||
|
|
||||||
|
Once it’s back up, running `wsl --update` is worth doing, since hangs like this are often fixed in newer versions.
|
||||||
|
|
||||||
|
# Pitfalls
|
||||||
|
|
||||||
|
- **Only open the distro from a normal PowerShell.** Opened from an admin one, WSL mounts the Windows drives despite `automount` being off ([microsoft/WSL#12443](https://github.com/microsoft/WSL/issues/12443)). `wsl --terminate claudebox` and a normal shell undo it.
|
||||||
|
- **Changes to `/etc/wsl.conf` apply only after `wsl --terminate claudebox`.**
|
||||||
@@ -1,175 +0,0 @@
|
|||||||
services:
|
|
||||||
|
|
||||||
mealie:
|
|
||||||
image: ghcr.io/mealie-recipes/mealie:latest
|
|
||||||
container_name: mealie
|
|
||||||
restart: unless-stopped
|
|
||||||
|
|
||||||
# `deploy: resources: limits:` is Swarm-only and was silently ignored,
|
|
||||||
# so this container has never actually had a memory limit. This one works.
|
|
||||||
mem_limit: 1000m
|
|
||||||
|
|
||||||
# Was 9925:9000 on every interface — the internet, if the firewall is ever
|
|
||||||
# wrong. SWAG reaches it by container name on the shared network; this is
|
|
||||||
# only the fallback for when you are on the box itself.
|
|
||||||
ports:
|
|
||||||
- "127.0.0.1:9925:9000"
|
|
||||||
|
|
||||||
# Was a Docker-managed volume under /var/lib/docker, root-owned and
|
|
||||||
# invisible. A plain directory next to this file, so a backup is `tar`.
|
|
||||||
volumes:
|
|
||||||
- /srv/data/mealie:/app/data
|
|
||||||
|
|
||||||
environment:
|
|
||||||
ALLOW_SIGNUP: "false"
|
|
||||||
PUID: 1000
|
|
||||||
PGID: 1000
|
|
||||||
TZ: Europe/Brussels
|
|
||||||
MAX_WORKERS: 1
|
|
||||||
WEB_CONCURRENCY: 1
|
|
||||||
BASE_URL: https://mealie.rittershofer.org
|
|
||||||
# From .env, chmod 600. Never in this file.
|
|
||||||
OPENAI_API_KEY: ${OPENAI_API_KEY:?set OPENAI_API_KEY in .env}
|
|
||||||
# A base, not an endpoint. The old /v1/models made it request
|
|
||||||
# /v1/models/chat/completions.
|
|
||||||
OPENAI_BASE_URL: https://api.openai.com/v1
|
|
||||||
|
|
||||||
logging:
|
|
||||||
driver: json-file
|
|
||||||
options:
|
|
||||||
max-size: "20m"
|
|
||||||
max-file: "5"
|
|
||||||
|
|
||||||
gitea:
|
|
||||||
image: gitea/gitea:latest
|
|
||||||
container_name: gitea
|
|
||||||
restart: unless-stopped
|
|
||||||
environment:
|
|
||||||
- USER_UID=1000
|
|
||||||
- USER_GID=1000
|
|
||||||
volumes:
|
|
||||||
- /srv/data/gitea:/data
|
|
||||||
- /etc/timezone:/etc/timezone:ro
|
|
||||||
- /etc/localtime:/etc/localtime:ro
|
|
||||||
logging:
|
|
||||||
driver: json-file
|
|
||||||
options:
|
|
||||||
max-size: "20m"
|
|
||||||
max-file: "5"
|
|
||||||
|
|
||||||
newsdesk:
|
|
||||||
image: ${NEWSDESK_IMAGE:?set NEWSDESK_IMAGE in .env}:${NEWSDESK_TAG:-latest}
|
|
||||||
container_name: newsdesk
|
|
||||||
restart: unless-stopped
|
|
||||||
user: "${PUID:-1000}:${PGID:-1000}"
|
|
||||||
volumes:
|
|
||||||
- /srv/data/newsdesk:/data
|
|
||||||
environment:
|
|
||||||
TZ: ${TZ:-Europe/Berlin}
|
|
||||||
CLAUDE_CODE_OAUTH_TOKEN: ${CLAUDE_CODE_OAUTH_TOKEN:-}
|
|
||||||
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
|
||||||
NEWSDESK_PASSWORD: ${NEWSDESK_PASSWORD:-}
|
|
||||||
mem_limit: ${NEWSDESK_MEM:-0}
|
|
||||||
logging:
|
|
||||||
driver: json-file
|
|
||||||
options:
|
|
||||||
max-size: "20m"
|
|
||||||
max-file: "5"
|
|
||||||
|
|
||||||
bookly:
|
|
||||||
image: git.rittershofer.org/wolf/bookly:${BOOKLY_TAG:-latest}
|
|
||||||
container_name: bookly
|
|
||||||
restart: unless-stopped
|
|
||||||
user: "1000:1000"
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
- /srv/data/bookly:/data
|
|
||||||
|
|
||||||
environment:
|
|
||||||
TZ: Europe/Brussels
|
|
||||||
# Required: without it the demo would be open to anyone who finds the hostname, so Compose refuses to start.
|
|
||||||
BOOKLY_PASSWORD: ${BOOKLY_PASSWORD:?set BOOKLY_PASSWORD in .env}
|
|
||||||
# subscription: the Claude CLI with CLAUDE_CODE_OAUTH_TOKEN (same variable newsdesk reads).
|
|
||||||
# api: ANTHROPIC_API_KEY, billed per token, lower latency.
|
|
||||||
BOOKLY_BACKEND: ${BOOKLY_BACKEND:-subscription}
|
|
||||||
CLAUDE_CODE_OAUTH_TOKEN: ${CLAUDE_CODE_OAUTH_TOKEN:-}
|
|
||||||
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
|
||||||
|
|
||||||
# No `ports:`. SWAG reaches :8000 by container name; nothing else can.
|
|
||||||
|
|
||||||
mem_limit: 0
|
|
||||||
|
|
||||||
logging:
|
|
||||||
driver: json-file
|
|
||||||
options:
|
|
||||||
max-size: "20m"
|
|
||||||
max-file: "5"
|
|
||||||
|
|
||||||
health:
|
|
||||||
image: ${HEALTH_IMAGE:?set HEALTH_IMAGE in .env}:${HEALTH_TAG:-latest}
|
|
||||||
container_name: health
|
|
||||||
restart: unless-stopped
|
|
||||||
|
|
||||||
# Warehouse, journal, job table, speech models, tiles, backups: all of it
|
|
||||||
# under one host directory, so a backup is one target.
|
|
||||||
volumes:
|
|
||||||
- ${HEALTH_DATA:-./data/health}:/data
|
|
||||||
|
|
||||||
user: "${PUID:-1000}:${PGID:-1000}"
|
|
||||||
|
|
||||||
environment:
|
|
||||||
TZ: ${TZ:-Europe/Berlin}
|
|
||||||
HEALTH_TIMEZONE: ${TZ:-Europe/Berlin}
|
|
||||||
# Which model writes the evening entry. The subscription token now; add
|
|
||||||
# ANTHROPIC_API_KEY and `docker compose up -d health` (not `restart`)
|
|
||||||
# to move to the API. Shared with the other services in this file.
|
|
||||||
CLAUDE_CODE_OAUTH_TOKEN: ${CLAUDE_CODE_OAUTH_TOKEN:-}
|
|
||||||
ANTHROPIC_API_KEY: ${HEALTH_ANTHROPIC_API_KEY:-}
|
|
||||||
# Normally empty. Set it to pin the password -- the way back in when a
|
|
||||||
# login is lost -- then empty it and recreate to hand the password back
|
|
||||||
# to the Settings page.
|
|
||||||
HEALTH_PASSWORD: ${HEALTH_PASSWORD:-}
|
|
||||||
# The speech model. `small` (0.7 GB) if turbo (1.9 GB while it runs)
|
|
||||||
# does not fit beside everything else on the box.
|
|
||||||
HEALTH_WHISPER_MODEL: ${HEALTH_WHISPER_MODEL:-large-v3-turbo}
|
|
||||||
|
|
||||||
# No `ports:` and no `networks:`: SWAG reaches it by name on this file's
|
|
||||||
# network, and nothing else reaches it at all.
|
|
||||||
|
|
||||||
# `deploy: resources: limits:` is Swarm-only and ignored here. 0 is
|
|
||||||
# unlimited until a transcription has been watched on the box.
|
|
||||||
mem_limit: ${HEALTH_MEM:-0}
|
|
||||||
|
|
||||||
logging:
|
|
||||||
driver: json-file
|
|
||||||
options:
|
|
||||||
max-size: "20m"
|
|
||||||
max-file: "5"
|
|
||||||
|
|
||||||
swag:
|
|
||||||
image: lscr.io/linuxserver/swag:latest
|
|
||||||
container_name: swag
|
|
||||||
cap_add:
|
|
||||||
- NET_ADMIN
|
|
||||||
environment:
|
|
||||||
- PUID=1000
|
|
||||||
- PGID=1000
|
|
||||||
- TZ=Europe/Brussels
|
|
||||||
- URL=rittershofer.org
|
|
||||||
- VALIDATION=http
|
|
||||||
# Commas and nothing else — SWAG splits on the comma alone, so a space
|
|
||||||
# asks for a hostname with a leading space. `news` added for newsdesk;
|
|
||||||
# `collabora` dropped, since nothing serves it any more.
|
|
||||||
- SUBDOMAINS=www,mealie,git,news,play,body
|
|
||||||
- ONLY_SUBDOMAINS=false
|
|
||||||
volumes:
|
|
||||||
- /etc/config/swag:/config
|
|
||||||
ports:
|
|
||||||
- 443:443
|
|
||||||
- 80:80
|
|
||||||
logging:
|
|
||||||
driver: json-file
|
|
||||||
options:
|
|
||||||
max-size: "20m"
|
|
||||||
max-file: "5"
|
|
||||||
restart: unless-stopped
|
|
||||||
-112
@@ -1,112 +0,0 @@
|
|||||||
# Deploying a service to rittershofer.org
|
|
||||||
|
|
||||||
The ground every service on this box shares: what the box is, the seven steps to add a service to it, the templates those steps need, and the things that have already gone wrong. [release.md](release.md) is how an image gets built and published; [server.md](server.md) is what happens once it has been.
|
|
||||||
|
|
||||||
`<app>` below is your *service's* name: the hostname, the data directory, the image tag, the compose service. It is the one thing you choose. The Python package stays `app` and the environment variables stay `APP_`, here and in every project started from this repository — nothing inside the checkout is renamed, and the files these steps need are already in place under `deploy/`.
|
|
||||||
|
|
||||||
## What the box is
|
|
||||||
|
|
||||||
A single Hetzner server running Ubuntu 24.04 on x86_64, with a 37 GB root filesystem and no separate data disk. One Docker Compose project holds everything.
|
|
||||||
|
|
||||||
/srv/swag/docker-compose.yml every service, one file, project name `swag`
|
|
||||||
/srv/swag/.env every secret, chmod 600, never in the compose file
|
|
||||||
/srv/data/<app>/ every service's data, owned 1000:1000
|
|
||||||
/etc/config/swag/ SWAG's /config: nginx confs, certificates, logs
|
|
||||||
|
|
||||||
A snapshot of that whole compose file, every neighbour included, is [box-docker-compose.yml](box-docker-compose.yml): what a new service block lands beside, and where the SWAG subdomain list lives.
|
|
||||||
|
|
||||||
`swag` is nginx, certbot and fail2ban in one container. It terminates TLS for every hostname and proxies to the others **by container name** over the project's default network. That shared network is the only reason all these services live in one compose file — nothing else about them is related.
|
|
||||||
|
|
||||||
Images are published to the Gitea instance on the same box, at `git.rittershofer.org/wolf/<app>`. Everything runs as uid/gid 1000 and `TZ=Europe/Brussels`.
|
|
||||||
|
|
||||||
Which compose file that is comes off the running container rather than out of memory:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker inspect swag --format '{{index .Config.Labels "com.docker.compose.project.config_files"}}'
|
|
||||||
```
|
|
||||||
|
|
||||||
Declaring the network `external` from a second Compose project would work too, and would hardcode the first project's name into the second — a rename away from a confusing failure.
|
|
||||||
|
|
||||||
## The seven steps
|
|
||||||
|
|
||||||
**1. DNS.** `<app>.rittershofer.org` must resolve to the server *before* anything else. Confirm with `getent hosts <app>.rittershofer.org`.
|
|
||||||
|
|
||||||
**2. Certificate.** Add `<app>` to `SUBDOMAINS` in the swag service's environment. That list separates on commas and nothing else — a space after one asks for a hostname with a leading space. SWAG runs certbot on every container start and requests **one certificate covering every name in the list**, so a name that does not answer on port 80 fails the whole request and leaves the existing names unrenewed. This is why step 1 comes first. Restart swag, then check `docker logs swag` for the issuance. If SWAG already holds a wildcard certificate covering the name, reuse it and skip this step entirely.
|
|
||||||
|
|
||||||
**3. Registry login.** On the build machine a Gitea token with package **write**; on the server, package **read**. Same command both ends, token at the password prompt:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker login git.rittershofer.org -u wolf
|
|
||||||
```
|
|
||||||
|
|
||||||
Git credentials and Docker registry credentials are separate. A successful Git push does not log Docker in.
|
|
||||||
|
|
||||||
**4. Data directory.** Root's `mkdir` produces a root-owned directory, and the container runs as 1000, so the chown is not optional:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
sudo mkdir -p /srv/data/<app> && sudo chown -R 1000:1000 /srv/data/<app>
|
|
||||||
```
|
|
||||||
|
|
||||||
**5. Service block** from [../deploy/server/compose.yaml](../deploy/server/compose.yaml) into `/srv/swag/docker-compose.yml`, and the variables from [../deploy/server/env.example](../deploy/server/env.example) appended to `/srv/swag/.env`.
|
|
||||||
|
|
||||||
**6. Proxy conf** — [../deploy/server/swag/app.subdomain.conf](../deploy/server/swag/app.subdomain.conf) renamed to `<app>.subdomain.conf` in `/etc/config/swag/nginx/proxy-confs/`. SWAG's `/config` is a host directory bind-mounted into the container; read the host side off the running container rather than assuming where it is:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker inspect swag --format '{{range .Mounts}}{{if eq .Destination "/config"}}{{.Source}}{{end}}{{end}}'
|
|
||||||
```
|
|
||||||
|
|
||||||
<config>/nginx/proxy-confs/ the per-site confs, where yours goes
|
|
||||||
<config>/nginx/proxy.conf the shared defaults every site includes
|
|
||||||
<config>/nginx/ssl.conf certificate paths and TLS settings
|
|
||||||
<config>/nginx/resolver.conf Docker's internal DNS
|
|
||||||
<config>/etc/letsencrypt/live/ the certificates themselves
|
|
||||||
|
|
||||||
Then validate, and reload only if validation passes:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker exec swag nginx -t && docker exec swag nginx -s reload
|
|
||||||
```
|
|
||||||
|
|
||||||
Confirm the other sites still respond afterwards.
|
|
||||||
|
|
||||||
**7. Deploy.** Name the service, so nothing else in the file is recreated:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /srv/swag && docker compose up -d <app>
|
|
||||||
```
|
|
||||||
|
|
||||||
## What the templates already decide for you
|
|
||||||
|
|
||||||
Both templates are in [../deploy/server/](../deploy/server), commented in place. The decisions worth knowing before you read them:
|
|
||||||
|
|
||||||
**No `ports:`.** SWAG reaches the container by name on the project network. That is what keeps the app's port off the internet even if the firewall is wrong.
|
|
||||||
|
|
||||||
**No `networks:` either.** Being in that compose file is what puts it on the right one.
|
|
||||||
|
|
||||||
**`user: "1000:1000"`.** Container uid is host uid — there is no translation layer. Without this the process runs as whatever the image declares, usually root, and everything it writes into `/srv/data` lands root-owned.
|
|
||||||
|
|
||||||
**`mem_limit:`, not `deploy: resources: limits:`.** The latter is Swarm-only and silently ignored by `docker compose up`. 0 is unlimited, which is the honest default until the box has been watched under real load.
|
|
||||||
|
|
||||||
**`logging:` with a size cap.** Without it, json-file logs grow until the disk does not.
|
|
||||||
|
|
||||||
**`listen 443 ssl` without `http2`.** `http2 on;` is set once at `http` scope in `nginx.conf`, and the `listen ... http2` parameter is deprecated in nginx 1.25 and later.
|
|
||||||
|
|
||||||
**`proxy_read_timeout 86400s`** only if the app holds a WebSocket open. SWAG's `proxy.conf` already sets the Upgrade headers; what it also sets is a read timeout of a couple of minutes, which closes an idle socket on the minute. Delete those two lines if there is no socket.
|
|
||||||
|
|
||||||
## Things that have already bitten
|
|
||||||
|
|
||||||
**Disk.** 37 GB, and it has been full. Three causes, all recurring: `:latest` orphans the previous image on every pull and nothing cleans them up (a weekly `docker image prune -f` cron handles it — no `-a`, so pinned images survive); journald defaults to 10% of the filesystem and is capped here at 200 MB; and VS Code Remote installs a fresh server per version under `~/.vscode-server` forever.
|
|
||||||
|
|
||||||
**`cp -a "$SRC/."` with an unset `SRC`** copies the root filesystem. Use `"${SRC:?}"` and echo the value before running anything destructive with it.
|
|
||||||
|
|
||||||
**SWAG never touches your live confs**, only the `.sample` files beside them. That is deliberate and it means they drift for years. Diff them occasionally:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /etc/config/swag/nginx && for f in *.conf site-confs/*.conf proxy-confs/*.subdomain.conf; do echo "=== $f"; diff "$f" "$f.sample"; done
|
|
||||||
```
|
|
||||||
|
|
||||||
Take the samples wholesale and re-apply your own edits. `nginx.conf` and the site/proxy confs must be updated **together** — older ones include `ssl.conf` at `http` scope, newer ones per server block, and mixing the two produces `no "ssl_certificate" is defined`.
|
|
||||||
|
|
||||||
**SQLite is never safe to copy live.** Stop the container, or snapshot with `VACUUM INTO` through the running one. A `cp` of a WAL-mode database is a coin toss. See [server.md](server.md#the-database).
|
|
||||||
|
|
||||||
**Secrets in `.env`, never in the compose file.** It is easy to paste a compose file into a chat window; it is harder to do that by accident with a file you had to `cat` deliberately.
|
|
||||||
@@ -1,241 +0,0 @@
|
|||||||
# Asking Claude, from a deployed app
|
|
||||||
|
|
||||||
One question — *ask Claude this, get JSON back* — and two ways to answer it, chosen by which credential is present in the environment. The caller asks without knowing which one ran.
|
|
||||||
|
|
||||||
api One call to the Claude API. Wants ANTHROPIC_API_KEY. No local
|
|
||||||
install, no subprocess, per-token billing. The unattended path.
|
|
||||||
claude_code The Claude Agent SDK drives the `claude` CLI, on the
|
|
||||||
subscription it is signed in with, authenticated by
|
|
||||||
CLAUDE_CODE_OAUTH_TOKEN. Nothing billed per token.
|
|
||||||
|
|
||||||
This is the reusable part of the deployment: the compose file passes both variables through, the app prefers the key when there is one, and switching from the subscription to the API is a line in `.env` and `docker compose up -d` — no rebuild, no code change.
|
|
||||||
|
|
||||||
## The two credentials
|
|
||||||
|
|
||||||
**`CLAUDE_CODE_OAUTH_TOKEN`** comes from `claude setup-token`, run interactively on a machine you are already signed in on. It lasts a year. Generating it needs browser authorization, so no deployment script can automate it — you produce it on your laptop and paste it into the server's `.env` with an editor. Then `docker compose up -d app`, because a `restart` will not pick it up.
|
|
||||||
|
|
||||||
**`ANTHROPIC_API_KEY`** is per-token billing from the Console. Nothing to renew, nothing to sign in to, and it is what production should run on once the thing is real.
|
|
||||||
|
|
||||||
Set one. If both are set, `auto` takes the key. Neither, and every call fails with a notice naming the reason, which is the correct outcome — better a batch that fails saying "no credentials" than one that waits for a human who is not there.
|
|
||||||
|
|
||||||
The API key and the OAuth token are not interchangeable and do not substitute for each other. `ANTHROPIC_API_KEY` in the environment does not make the CLI work, and an OAuth token does not authenticate an API call.
|
|
||||||
|
|
||||||
**Neither belongs in the image or in the compose file.** They live in the `.env` beside it, `chmod 600`, injected as environment variables at container creation. A compose file gets pasted into chat windows; a file you had to `cat` deliberately does not.
|
|
||||||
|
|
||||||
## Why there is a `backend` setting and not just detection
|
|
||||||
|
|
||||||
Detection answers "what can run here". The setting answers "what should". They differ in two cases worth naming: forcing `api` on a machine that also has the CLI, so a benchmark measures the thing you meant; and `off`, which makes everything needing a model fail on purpose, with a notice that says which switch did it. An app that silently degrades to no-model is worse than one that stops.
|
|
||||||
|
|
||||||
## What the `claude_code` path deliberately does not get
|
|
||||||
|
|
||||||
No tools (`tools=[]`, `allowed_tools=[]`), no filesystem settings (`setting_sources=None`, so no CLAUDE.md, no skills, no MCP servers leak into a classification call), no session to resume. One text-in, JSON-out turn — the same shape as the API call it stands in for. The CLI is built for agent work, and left to its defaults it will bring all of that with it.
|
|
||||||
|
|
||||||
## Thinking has to be stated, not defaulted
|
|
||||||
|
|
||||||
The CLI thinks unprompted; the Messages API thinks only when asked. So "say nothing about thinking" means opposite things on the two sides, and the two backends stop being interchangeable. State it explicitly to both. It is not a small difference: on a batch of twenty-five short classifications, the CLI's default cost 128 s against 17 s for the same answers.
|
|
||||||
|
|
||||||
Some models support manual extended thinking only and reject `adaptive`. Check before spending a call, not after.
|
|
||||||
|
|
||||||
## Both take the same JSON schema
|
|
||||||
|
|
||||||
The Agent SDK accepts the Messages API's `json_schema` output format and hands the parsed object back on `ResultMessage.structured_output`. That is what makes this two backends rather than two implementations: neither path is the degraded one, and moving between them is a config line.
|
|
||||||
|
|
||||||
## Reference implementation
|
|
||||||
|
|
||||||
Self-contained — no imports from your project, both SDKs optional. It is the shape, not the whole thing: the real one in newsdesk also streams partial answers out for a progress bar, retries, and classifies API errors by status code so a 401 and a 429 tell you different things.
|
|
||||||
|
|
||||||
```python
|
|
||||||
"""Ask Claude a question, constrained to a JSON schema, get a dict back.
|
|
||||||
|
|
||||||
Two backends, one seam. `ANTHROPIC_API_KEY` selects the API; otherwise the
|
|
||||||
Claude Agent SDK drives the `claude` CLI on a subscription, authenticated by
|
|
||||||
`CLAUDE_CODE_OAUTH_TOKEN`. Both are optional dependencies:
|
|
||||||
|
|
||||||
uv pip install anthropic # the api backend
|
|
||||||
uv pip install claude-agent-sdk # the claude_code backend
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import importlib.util
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
|
|
||||||
class AgentError(Exception):
|
|
||||||
"""The backend was there and the call still did not produce an answer."""
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(slots=True)
|
|
||||||
class AgentConfig:
|
|
||||||
#: "auto", "api", "claude_code", or "off" to make every call fail.
|
|
||||||
backend: str = "auto"
|
|
||||||
model: str = "claude-haiku-4-5-20251001"
|
|
||||||
timeout: float = 300.0
|
|
||||||
max_tokens: int = 8192
|
|
||||||
#: "off" or "adaptive". Stated to both backends, never defaulted.
|
|
||||||
thinking: str = "off"
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(slots=True)
|
|
||||||
class Reply:
|
|
||||||
data: dict[str, Any]
|
|
||||||
backend: str
|
|
||||||
model: str
|
|
||||||
usage: dict[str, Any]
|
|
||||||
|
|
||||||
|
|
||||||
def backend(settings: AgentConfig) -> str:
|
|
||||||
"""Which backend will answer, or "" for nobody."""
|
|
||||||
if settings.backend == "off":
|
|
||||||
return ""
|
|
||||||
if settings.backend in ("api", "claude_code"):
|
|
||||||
return settings.backend
|
|
||||||
if any(os.environ.get(name) for name in
|
|
||||||
("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_PROFILE")):
|
|
||||||
return "api"
|
|
||||||
# The SDK bundles its own `claude`, so having the package is enough. Not
|
|
||||||
# proof the call will work — the CLI may be signed out — but that surfaces
|
|
||||||
# as an AgentError on the call, which the caller handles anyway.
|
|
||||||
if importlib.util.find_spec("claude_agent_sdk") is not None:
|
|
||||||
return "claude_code"
|
|
||||||
return ""
|
|
||||||
|
|
||||||
|
|
||||||
def _thinking(setting: str) -> dict[str, str]:
|
|
||||||
return {"type": "adaptive"} if setting == "adaptive" else {"type": "disabled"}
|
|
||||||
|
|
||||||
|
|
||||||
def ask_json(settings: AgentConfig, *, system: str, question: dict[str, Any],
|
|
||||||
schema: dict[str, Any]) -> Reply:
|
|
||||||
"""Ask `question`, constrained to `schema`, and return the parsed answer.
|
|
||||||
|
|
||||||
`system` should be byte-identical across the calls in a run — it is what
|
|
||||||
the API path caches. `question` is the part that changes.
|
|
||||||
"""
|
|
||||||
name = backend(settings)
|
|
||||||
if not name:
|
|
||||||
raise AgentError("no model backend is available: set ANTHROPIC_API_KEY "
|
|
||||||
"or install claude-agent-sdk with CLAUDE_CODE_OAUTH_TOKEN")
|
|
||||||
if name == "api":
|
|
||||||
return _ask_api(settings, system, question, schema)
|
|
||||||
return _ask_claude_code(settings, system, question, schema)
|
|
||||||
|
|
||||||
|
|
||||||
def _ask_api(settings: AgentConfig, system: str, question: dict[str, Any],
|
|
||||||
schema: dict[str, Any]) -> Reply:
|
|
||||||
try:
|
|
||||||
import anthropic
|
|
||||||
except ImportError as exc:
|
|
||||||
raise AgentError("the anthropic SDK isn't installed") from exc
|
|
||||||
|
|
||||||
try:
|
|
||||||
client = anthropic.Anthropic(timeout=settings.timeout)
|
|
||||||
response = client.messages.create(
|
|
||||||
model=settings.model,
|
|
||||||
max_tokens=settings.max_tokens,
|
|
||||||
# The instructions are byte-identical across every call; only the
|
|
||||||
# payload below changes. Caching them is most of the saving.
|
|
||||||
system=[{"type": "text", "text": system,
|
|
||||||
"cache_control": {"type": "ephemeral"}}],
|
|
||||||
messages=[{"role": "user", "content": json.dumps(question)}],
|
|
||||||
thinking=_thinking(settings.thinking),
|
|
||||||
output_config={"format": {"type": "json_schema", "schema": schema}},
|
|
||||||
)
|
|
||||||
except anthropic.APIError as exc:
|
|
||||||
# Carry the status through in the text: 401 ("your key is wrong") and
|
|
||||||
# 429 ("slow down") want different reactions, and the message body
|
|
||||||
# alone does not always separate them.
|
|
||||||
status = getattr(exc, "status_code", "")
|
|
||||||
raise AgentError(f"the API call failed: {status} {exc}".strip()) from exc
|
|
||||||
|
|
||||||
text = next((b.text for b in response.content if b.type == "text"), "")
|
|
||||||
return Reply(data=json.loads(text), backend="api",
|
|
||||||
model=response.model or settings.model,
|
|
||||||
usage=response.usage.model_dump() if response.usage else {})
|
|
||||||
|
|
||||||
|
|
||||||
def _ask_claude_code(settings: AgentConfig, system: str, question: dict[str, Any],
|
|
||||||
schema: dict[str, Any]) -> Reply:
|
|
||||||
try:
|
|
||||||
from claude_agent_sdk import (
|
|
||||||
ClaudeAgentOptions, ClaudeSDKError, ResultMessage, query,
|
|
||||||
)
|
|
||||||
except ImportError as exc:
|
|
||||||
raise AgentError("the Claude Agent SDK isn't installed") from exc
|
|
||||||
|
|
||||||
options = ClaudeAgentOptions(
|
|
||||||
system_prompt=system,
|
|
||||||
model=settings.model or None,
|
|
||||||
# A classification call, not an agent: nothing to run, nothing to read,
|
|
||||||
# and no project settings — so no CLAUDE.md, no skills and no MCP
|
|
||||||
# servers get in front of the model on the way to an answer.
|
|
||||||
tools=[],
|
|
||||||
allowed_tools=[],
|
|
||||||
permission_mode="dontAsk",
|
|
||||||
setting_sources=None,
|
|
||||||
thinking=_thinking(settings.thinking),
|
|
||||||
output_format={"type": "json_schema", "schema": schema},
|
|
||||||
# The API takes max_tokens as a parameter; the CLI takes it only as an
|
|
||||||
# environment variable. Without this the setting is quietly API-only.
|
|
||||||
# The SDK merges this into the inherited environment rather than
|
|
||||||
# replacing it, so naming one variable does not cost the CLI its PATH.
|
|
||||||
env={"CLAUDE_CODE_MAX_OUTPUT_TOKENS": str(settings.max_tokens)},
|
|
||||||
)
|
|
||||||
|
|
||||||
async def ask() -> Reply:
|
|
||||||
answer: Reply | None = None
|
|
||||||
async for message in query(prompt=json.dumps(question), options=options):
|
|
||||||
if not isinstance(message, ResultMessage):
|
|
||||||
continue
|
|
||||||
if message.is_error:
|
|
||||||
raise AgentError("; ".join(message.errors or []) or message.result
|
|
||||||
or f"the agent ended as {message.subtype}")
|
|
||||||
data = message.structured_output
|
|
||||||
if not isinstance(data, dict):
|
|
||||||
# Ended without structured output — a limit, an interruption.
|
|
||||||
# `result` is the raw text if there is any, worth one attempt.
|
|
||||||
data = json.loads(message.result or "")
|
|
||||||
answer = Reply(data=data, backend="claude_code",
|
|
||||||
model=settings.model, usage=message.usage or {})
|
|
||||||
if answer is None:
|
|
||||||
raise AgentError("the agent returned no result")
|
|
||||||
return answer
|
|
||||||
|
|
||||||
try:
|
|
||||||
# asyncio.run, so this is a synchronous call like the other backend.
|
|
||||||
# From inside a running loop, await `ask()` directly instead.
|
|
||||||
return asyncio.run(asyncio.wait_for(ask(), settings.timeout))
|
|
||||||
except TimeoutError as exc:
|
|
||||||
raise AgentError(f"no answer within {settings.timeout:.0f}s") from exc
|
|
||||||
except ClaudeSDKError as exc:
|
|
||||||
# The SDK has its own exception hierarchy — a CLI that exited, a line
|
|
||||||
# it could not parse, an answer that overran max_tokens — and none of
|
|
||||||
# it is an AgentError. Left alone, those escape a caller that catches
|
|
||||||
# AgentError per item and take down a whole run. Every failure of this
|
|
||||||
# backend is an AgentError or none of them are.
|
|
||||||
raise AgentError(str(exc)) from exc
|
|
||||||
```
|
|
||||||
|
|
||||||
Using it:
|
|
||||||
|
|
||||||
```python
|
|
||||||
settings = AgentConfig()
|
|
||||||
schema = {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {"tags": {"type": "array", "items": {"type": "string"}}},
|
|
||||||
"required": ["tags"],
|
|
||||||
}
|
|
||||||
reply = ask_json(settings, system="Tag this article. Answer with the schema.",
|
|
||||||
question={"title": title, "body": body}, schema=schema)
|
|
||||||
print(reply.data["tags"], reply.backend)
|
|
||||||
```
|
|
||||||
|
|
||||||
## The two failures that will actually happen
|
|
||||||
|
|
||||||
**A changed token in `.env` appears to do nothing.** `docker compose restart` does not update environment variables. `docker compose up -d app` recreates the container, which does.
|
|
||||||
|
|
||||||
**The token expired and nothing said so.** The setup token lasts a year, and the failure looks like a model call failing rather than a credential problem. Read the error text — this is why the SDK's own message is passed through intact rather than summarised into "model call failed".
|
|
||||||
@@ -1,106 +0,0 @@
|
|||||||
# One password, on the open internet
|
|
||||||
|
|
||||||
The shape used by a single-user app that has to sit on a public hostname because that is where the browser is. No user management, no usernames, no registration: one password and the sessions it opens. Everything below is about making that enough.
|
|
||||||
|
|
||||||
This is the reasoning; `auth/` in this repository is the code, already written and already wired into `app/main.py`. `auth/README.md` is the ten-minute version of how to take the folder somewhere else.
|
|
||||||
|
|
||||||
## Locked by what it is bound to, not by a flag
|
|
||||||
|
|
||||||
The switch is the bind address. On `127.0.0.1` the door is already the machine's own login, and asking for a password again is theatre. On `0.0.0.0` it is reachable from elsewhere and locks itself.
|
|
||||||
|
|
||||||
Deriving it rather than configuring it removes the failure that looks fine: "deployed it, forgot to switch auth on". The production config then says `enabled: always` anyway — redundant with the bind address and worth saying out loud.
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
auth:
|
|
||||||
enabled: always # auto | always | never
|
|
||||||
session_days: 30
|
|
||||||
# `auto` marks the cookie Secure when the request arrived over HTTPS. Behind
|
|
||||||
# a TLS-terminating proxy the app only ever sees plain HTTP and cannot work
|
|
||||||
# that out, so behind SWAG this must be `always`.
|
|
||||||
secure_cookie: always
|
|
||||||
```
|
|
||||||
|
|
||||||
## The password itself
|
|
||||||
|
|
||||||
Stored as an scrypt hash, never in the config file. The first locked start either takes `APP_PASSWORD` from the environment or generates one and prints it to the log once. After that it is the app's own Account screen that changes it.
|
|
||||||
|
|
||||||
`APP_PASSWORD` is the way back in when a login is lost, and it is authoritative while it is set — the Account screen refuses to change what the environment pins. Set it, get in, empty it, recreate the container, then change the password in the app.
|
|
||||||
|
|
||||||
Two consequences worth writing down: a generated password is printed to the container log, so the retained log holds it until it rotates — rotate the password once you are in. And a password the *user* chooses is the weak spot, not the lockout: a minimum length of 8 is a very different thing from the 16 random characters the app generates.
|
|
||||||
|
|
||||||
## The lockout
|
|
||||||
|
|
||||||
Five wrong answers are free — past a typo and a stale password manager entry, short of anything systematic. After that each attempt waits, starting at 30 s and doubling to a ceiling of 15 minutes. That is about a hundred guesses a day against a secret with far more than a hundred possibilities, which is what "guessing is pointless" means in practice.
|
|
||||||
|
|
||||||
Three details that matter more than the numbers:
|
|
||||||
|
|
||||||
**Read the lockout before checking the password, not after.** Otherwise a fast attacker extends their own lockout forever and never notices, and — more to the point — an honest person hammering the button digs a deeper hole with every press.
|
|
||||||
|
|
||||||
**A wrong answer still costs a full scrypt derivation.** The delay an attacker meets is the one the honest user meets, and it is what lets the free-attempt count be as generous as it is.
|
|
||||||
|
|
||||||
**Cap the doubling.** `30s * 2**failures` with an uncapped counter eventually overflows whatever holds it, and a bot that has sent ten thousand guesses turns every future login attempt into a 500 — a denial of service delivered by arithmetic.
|
|
||||||
|
|
||||||
## One counter, not one per address
|
|
||||||
|
|
||||||
The obvious improvement is to lock out the address that is guessing, so a stranger cannot keep the owner waiting at their own door. Behind a proxy it cannot be built honestly: the only address the app sees is the one the proxy puts in a header, the peer is a container on the same Docker network, and a header is set by whoever is talking. Keyed on that, an attacker gets a fresh, empty counter per request by editing one line of their own request, while the owner still gets locked out by their own typos. Strictly worse, and it would look like protection.
|
|
||||||
|
|
||||||
So the address is **logged, not decided from** — at the layer that knows who connected, the decision can be made properly. See the throttling section below.
|
|
||||||
|
|
||||||
The honest cost of one global counter: an attacker hammering the door keeps password login locked for the owner too. A passkey login that skips the lockout is what makes that survivable.
|
|
||||||
|
|
||||||
## Sessions
|
|
||||||
|
|
||||||
One kind of session whichever door opened it, so revoking and expiry have one implementation. A row per session with its user agent and last-seen time, a screen listing them, a button to revoke one and a button to close all the others. `purge()` on startup drops the expired ones — otherwise that table grows forever.
|
|
||||||
|
|
||||||
The cookie: `HttpOnly` so an XSS cannot read it, `Secure` per the config above, `SameSite=Lax`, `path=/`.
|
|
||||||
|
|
||||||
**No CSRF token, and that is a decision rather than an omission.** A CSRF token defends a cookie-authenticated app against another origin submitting a form on your behalf. `SameSite=Lax` already refuses to send the cookie on any cross-site request that is not a top-level navigation, and if no `GET` in the app changes anything, the attack the token would stop cannot be mounted. Adding one anyway means a second secret, a second thing to rotate, and a second way for a request to fail that the UI has to explain. Where that reasoning stops holding is a browser old enough to ignore `SameSite` entirely.
|
|
||||||
|
|
||||||
## Passkeys
|
|
||||||
|
|
||||||
Here already, behind the `passkeys` extra: `uv sync --all-extras`, and without it the routes answer 501 and the Account tab simply does not draw the section.
|
|
||||||
|
|
||||||
A passkey and a security key are one feature: a YubiKey registers through the same flow as a phone, because to WebAuthn they are the same thing.
|
|
||||||
|
|
||||||
A passkey is bound to an origin, and that binding is what makes it unphishable — the browser only hands the credential back to the origin that created it. So it needs HTTPS and a real hostname, with `localhost` the one exception, which is what lets it be tried before deploying. Behind a proxy, state both explicitly:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
auth:
|
|
||||||
rp_id: app.rittershofer.org
|
|
||||||
rp_name: app
|
|
||||||
origin: https://app.rittershofer.org
|
|
||||||
```
|
|
||||||
|
|
||||||
Left blank they are derived from the request, which behind a TLS-terminating proxy derives `http://` and rejects every ceremony as an origin mismatch. Changing `rp_id` later invalidates every key already registered against the old one.
|
|
||||||
|
|
||||||
**The password does not go away**, deliberately: it is the way back in if a key is lost, which is why revoking a passkey can never lock you out.
|
|
||||||
|
|
||||||
## Throttling a stranger who is guessing
|
|
||||||
|
|
||||||
The lockout already makes the secret safe. What is left is keeping the door usable for you, and keeping the traffic off the app. Two layers, in the order worth adding them:
|
|
||||||
|
|
||||||
**1. Rate limit in nginx.** Five lines in the proxy conf, included in [../deploy/server/swag/app.subdomain.conf](../deploy/server/swag/app.subdomain.conf). Per-address, no state to manage, no capabilities, and it cannot lock you out — it just makes requests wait. This is the part that earns its place.
|
|
||||||
|
|
||||||
**2. fail2ban, only if a real attack turns up.** SWAG ships it. It reads nginx's access log, where the address is the one that actually connected rather than a header, and bans in the firewall across every service on the box. A filter matching the login routes with a 401/403/429 status, and a jail pointing at `/config/log/nginx/access.log`, is about ten lines.
|
|
||||||
|
|
||||||
What fail2ban adds over the throttle is narrow and worth being honest about: your own password login stays usable during an attack, the ban covers the whole box rather than one site, and you get a list of who has been banned. It adds no protection of the password — that is already done. Against that it costs `NET_ADMIN` on the SWAG container, a regex that breaks silently if the log format changes, and an unban procedure for when it catches you.
|
|
||||||
|
|
||||||
Add it with a real log sample to test the regex against:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker exec swag fail2ban-regex /config/log/nginx/access.log /config/fail2ban/filter.d/app.conf
|
|
||||||
```
|
|
||||||
|
|
||||||
If the domain is proxied through Cloudflare, nginx logs Cloudflare's addresses and a ban hits Cloudflare rather than the attacker. Check that before building anything on top of the log.
|
|
||||||
|
|
||||||
## What to log
|
|
||||||
|
|
||||||
One WARNING line per refusal, with the address first after `from`, so a filter is one regex:
|
|
||||||
|
|
||||||
```
|
|
||||||
WARNING auth.router: wrong password from 203.0.113.9
|
|
||||||
```
|
|
||||||
|
|
||||||
The log is the record. A second table of attempts in the database would only ever disagree with it. If the app is run at `log_level="warning"` there is no access log of its own, which makes this the whole record of who is knocking.
|
|
||||||
|
|
||||||
The forwarded address is the one piece of a stranger's own writing that ends up in that log, so strip it to the characters an address is made of before writing it. A line in this log is something a filter reads and a person believes, and neither should be forgeable by whoever is knocking.
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
# Building and publishing a release
|
|
||||||
|
|
||||||
Build on Windows, run on the server. Nothing in this half connects to the server: it tests a commit, promotes it, builds an image and pushes it to the registry. [server.md](server.md) is the other half.
|
|
||||||
|
|
||||||
The scripts are already in place in `deploy/`, with `deploy/server/` beside them — `db.ps1` expects that — and the Dockerfile at `deploy/Dockerfile`, which is where `2-build-image.ps1` looks for it inside the archived commit. The package is `app`; there is nothing to rename.
|
|
||||||
|
|
||||||
## Once
|
|
||||||
|
|
||||||
Copy `deploy/release.env.example` to `deploy/release.env` and fill it in. `.gitignore` already covers `deploy/release.env` and `deploy/backups/`.
|
|
||||||
|
|
||||||
Log Docker in with a token that has package-write access, before the first push:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker login git.rittershofer.org -u wolf
|
|
||||||
```
|
|
||||||
|
|
||||||
Enter the token at Docker's password prompt, not in source files or chat. Git credentials and Docker registry credentials are separate: a successful Git push does not log Docker in.
|
|
||||||
|
|
||||||
## Every release
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
.\deploy\1-test-and-promote.ps1 # test local main, fast-forward local production
|
|
||||||
.\deploy\2-build-image.ps1 # build production's image locally
|
|
||||||
.\deploy\3-push-image.ps1 # push the built image to the registry
|
|
||||||
```
|
|
||||||
|
|
||||||
The number is the order. Each script stops on failure, can be run on its own, and works out the repository root for itself, so the working directory does not matter. To run all three, use `.\deploy\0-release.ps1` — numbered zero so it sorts above the steps it runs.
|
|
||||||
|
|
||||||
**Step 1 never builds; step 2 never pushes; step 3 never rebuilds or deploys.** That separation is the point of the numbering: a failed push is `3` again, not the whole thing again.
|
|
||||||
|
|
||||||
## What each step is actually doing
|
|
||||||
|
|
||||||
**1 — test and promote.** Resolves `main` to a commit, checks that commit out into a throwaway worktree, and runs the project's checks there. The worktree is what makes this honest: the tests run against exactly the tree that is about to become production, with none of the working copy's uncommitted edits in the way. Worktrees have no `.venv` of their own, so the interpreter comes from the main checkout and the code from the worktree.
|
|
||||||
|
|
||||||
Promotion is a fast-forward only. If `production` has commits `main` does not, that is a release made from somewhere else, and the script stops rather than discarding it.
|
|
||||||
|
|
||||||
Git publication is optional: run `git push origin production` yourself, or pass `-PushBranch`. `production` identifies the tested candidate; the server's running image identifies the deployed release.
|
|
||||||
|
|
||||||
**2 — build.** `git archive` of the production commit, unpacked to a temporary directory, built from there. Writing the archive out rather than piping it is deliberate — a PowerShell pipeline carries text, so it would re-encode the tar stream and hand Docker a corrupted context. The build publishes two tags locally: the short commit sha and `latest`.
|
|
||||||
|
|
||||||
Docker builds an **image**. No container starts or restarts on this machine.
|
|
||||||
|
|
||||||
**3 — push.** Re-tags `latest` onto this candidate, then pushes both. It never rebuilds, so a push that failed on connectivity is just this step again.
|
|
||||||
|
|
||||||
## Uncommitted work is never included
|
|
||||||
|
|
||||||
Every step reads from git, not from the working copy. An edit you have not committed is not in the tests, not in the image, and not in the release. That is worth knowing in both directions: it cannot bite you by accident, and it will not pick up the fix you forgot to commit.
|
|
||||||
|
|
||||||
## Version pinning and rollback
|
|
||||||
|
|
||||||
`latest` moves on every publication, so it cannot identify a previous release. Commit tags can: keep them in the registry, and set `APP_TAG=<sha>` in the server's `.env` to pin or roll back. See [server.md](server.md#roll-back) for what else a rollback has to check.
|
|
||||||
@@ -1,75 +0,0 @@
|
|||||||
# Server setup and operation
|
|
||||||
|
|
||||||
What happens once an image has been published. The release half is [release.md](release.md); the box itself and the seven steps for putting a new service on it are [hosting.md](hosting.md).
|
|
||||||
|
|
||||||
Deployment is server-local. No Git checkout, build tooling or SSH from the development machine is needed for it — `db.ps1` is the one exception and it moves a database, not a release.
|
|
||||||
|
|
||||||
## Install the files
|
|
||||||
|
|
||||||
`APP_DATA` decides where the data directory goes. Left out, it is `./data` beside the compose file. Set to an absolute path, it can sit under the one root every container on the box writes to — which is what makes a backup a single target rather than a hunt.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
sudo mkdir -p /srv/data/app/home /srv/data/app/backups
|
|
||||||
sudo chown -R 1000:1000 /srv/data/app
|
|
||||||
```
|
|
||||||
|
|
||||||
`home/` is there because `HOME=/data/home` in the Dockerfile: a CLI the app shells out to needs somewhere writable, and the image is not it.
|
|
||||||
|
|
||||||
Then:
|
|
||||||
|
|
||||||
- the `app:` block from [../deploy/server/compose.yaml](../deploy/server/compose.yaml) into the compose file that runs the proxy
|
|
||||||
- the variables from [../deploy/server/env.example](../deploy/server/env.example) appended to the `.env` beside it, `chmod 600`
|
|
||||||
- [../deploy/server/deploy.sh](../deploy/server/deploy.sh) and [../deploy/server/backup.sh](../deploy/server/backup.sh) next to that compose file
|
|
||||||
- the app's own `config.yaml` into `$APP_DATA/config.yaml`
|
|
||||||
- [../deploy/server/swag/app.subdomain.conf](../deploy/server/swag/app.subdomain.conf) into SWAG's `proxy-confs/`, renamed
|
|
||||||
|
|
||||||
Edit `.env` for the actual data path and UID/GID. Keep `APP_TAG=latest` for ordinary deployments or choose a published commit tag to pin.
|
|
||||||
|
|
||||||
## Deploy
|
|
||||||
|
|
||||||
```bash
|
|
||||||
bash /srv/swag/deploy.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
It runs `docker compose pull app` first, and only on success runs `docker compose up -d --no-deps --pull never --wait --wait-timeout 240 app`, using that image and waiting for its `HEALTHCHECK`. Other services in the file are left alone. A failed pull stops before startup; an unhealthy startup exits nonzero. Inspect `docker compose ps` and `docker compose logs app` on failure.
|
|
||||||
|
|
||||||
**After changing credentials in `.env`, use `up -d`, not `restart`.** A restart does not update environment variables — the same container starts again with the environment it was created with. This is the single most common way an updated token appears not to have worked.
|
|
||||||
|
|
||||||
## The database
|
|
||||||
|
|
||||||
Never overwrite a live SQLite database with a plain file copy. It is in WAL mode, so the `.db` on disk is always missing whatever is still in the `-wal` beside it, and copying the three files while the app is running is a race you lose quietly: a database that opens fine and is short a day's writes.
|
|
||||||
|
|
||||||
**Backups.** `bash /srv/swag/backup.sh` takes a `VACUUM INTO` snapshot through the running container, gzips it into `$APP_DATA/backups`, and deletes everything past `KEEP` (14 by default). Schedule it with cron:
|
|
||||||
|
|
||||||
```
|
|
||||||
0 4 * * * bash /srv/swag/backup.sh >> /srv/swag/backup.log 2>&1
|
|
||||||
```
|
|
||||||
|
|
||||||
Keep an off-server copy, and test restoring it. A backup nobody has restored is a hypothesis.
|
|
||||||
|
|
||||||
**Moving a database onto the server.** `.\deploy\db.ps1 push` takes a snapshot of this machine's database, uploads it with the importer, and runs the import. The server validates the incoming file — `integrity_check` plus the tables it must have — and snapshots the live database into `/data/backups` before replacing it, atomically.
|
|
||||||
|
|
||||||
It does not stop the app and does not start it: **stop it yourself first** and start it afterwards. The import is refused while the app is up. Get the image the database needs into the registry and onto the server first, because older code refuses to open a newer schema.
|
|
||||||
|
|
||||||
If the app has accounts, importing replaces them too, so do the account setup against the final database rather than before the import.
|
|
||||||
|
|
||||||
**Taking one off.** `.\deploy\db.ps1 pull` runs the server's own `backup.sh` and downloads the newest snapshot into `backups\`. Work on a copy of it, never on a file you intend to push back: after a push the two copies diverge from the first write on either, and there is no merge for that.
|
|
||||||
|
|
||||||
## Roll back
|
|
||||||
|
|
||||||
Record the current image tag or digest before an update. Then set that commit tag in `.env` and run `deploy.sh` again. `latest` moves on publication, so it cannot name a previous release — this only works if commit-tagged images stay in the registry.
|
|
||||||
|
|
||||||
Check schema compatibility before rolling code back. If a release migrated the database incompatibly, restore a matching backup while the app is stopped. No automated rollback overwrites the database, and that is deliberate.
|
|
||||||
|
|
||||||
## Before relying on it
|
|
||||||
|
|
||||||
Verify, in this order, and write down what you saw rather than closing it from local reasoning:
|
|
||||||
|
|
||||||
1. HTTPS from outside, on the real hostname.
|
|
||||||
2. Login, and a passkey if the app has them — passkeys need `rp_id`/`origin` to match the hostname exactly.
|
|
||||||
3. The app's heaviest real operation, once, end to end.
|
|
||||||
4. Persistence: `docker compose up -d --force-recreate app`, then check the data is still there.
|
|
||||||
5. A restore of an off-server backup copy.
|
|
||||||
6. Memory under that heaviest operation, before choosing `APP_MEM`.
|
|
||||||
|
|
||||||
The health endpoint checks the web app and its store. It does not check model credentials, and a green container says nothing about whether the API key works.
|
|
||||||
@@ -0,0 +1,800 @@
|
|||||||
|
"""Release and deployment scripts without a live server, a registry or a network.
|
||||||
|
|
||||||
|
Ported from the boilerplate's tests/test_deploy.py. Ours: the database is
|
||||||
|
`roledesk.db` with `roles` as its first table, and the tests at the end hold
|
||||||
|
the facts put back after a refresh (the Dockerfile, the server config, `bin/`
|
||||||
|
and the profile commands).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
import shutil
|
||||||
|
import sqlite3
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
SERVER = ROOT / "deploy" / "server"
|
||||||
|
|
||||||
|
#: The compose service, read from the one block a project pastes into its
|
||||||
|
#: compose file. Every script and the proxy conf repeat it; a rename that
|
||||||
|
#: misses one fails `test_every_file_names_the_same_service`, not this whole
|
||||||
|
#: file.
|
||||||
|
SERVICE = next(iter(yaml.safe_load((SERVER / "compose.yaml").read_text(encoding="utf-8"))
|
||||||
|
["services"]))
|
||||||
|
|
||||||
|
|
||||||
|
def run_shell(tmp_path: Path, script: str, *args: str, fail: str = ""):
|
||||||
|
git_bash = Path("C:/Program Files/Git/bin/bash.exe")
|
||||||
|
bash = str(git_bash) if git_bash.exists() else shutil.which("bash")
|
||||||
|
if bash is None:
|
||||||
|
pytest.skip("bash is required for deployment script tests")
|
||||||
|
log = tmp_path / "docker.log"
|
||||||
|
wrapper = '''
|
||||||
|
docker() {
|
||||||
|
printf '%s\n' "$*" >> "$DEPLOY_TEST_LOG"
|
||||||
|
if [[ -n $DEPLOY_TEST_FAIL && "$*" == *"$DEPLOY_TEST_FAIL"* ]]; then return 1; fi
|
||||||
|
}
|
||||||
|
export -f docker
|
||||||
|
bash "$@"
|
||||||
|
'''
|
||||||
|
result = subprocess.run(
|
||||||
|
[bash, "-c", wrapper, "test", (ROOT / "deploy" / "server" / script).as_posix(),
|
||||||
|
tmp_path.as_posix(), *args], capture_output=True, text=True,
|
||||||
|
env={**os.environ, "DEPLOY_TEST_LOG": log.as_posix(), "DEPLOY_TEST_FAIL": fail})
|
||||||
|
return result, log.read_text() if log.exists() else ""
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("fail", ["", f"pull {SERVICE}", "up -d"])
|
||||||
|
def test_deploy_waits_for_health_and_reports_failure(tmp_path, fail):
|
||||||
|
(tmp_path / "data").mkdir()
|
||||||
|
shutil.copy(SERVER / "config.example.yaml", tmp_path / "data/config.yaml")
|
||||||
|
original = "APP_TAG=latest\nSECRET=do-not-print\n"
|
||||||
|
(tmp_path / ".env").write_text(original)
|
||||||
|
result, log = run_shell(tmp_path, "deploy.sh", fail=fail)
|
||||||
|
assert (result.returncode != 0) == bool(fail)
|
||||||
|
assert (tmp_path / ".env").read_text() == original
|
||||||
|
assert "do-not-print" not in result.stdout + result.stderr
|
||||||
|
assert f"compose pull {SERVICE}" in log
|
||||||
|
if fail.startswith("pull"):
|
||||||
|
assert "up -d" not in log
|
||||||
|
assert "not restarted" in result.stderr
|
||||||
|
else:
|
||||||
|
assert f"up -d --no-deps --pull never --wait --wait-timeout 240 {SERVICE}" in log
|
||||||
|
assert log.index("compose pull") < log.index("compose up")
|
||||||
|
if fail:
|
||||||
|
assert "Deployment failed" in result.stderr
|
||||||
|
|
||||||
|
|
||||||
|
def test_deploy_missing_config_does_not_call_docker(tmp_path):
|
||||||
|
(tmp_path / ".env").write_text("APP_TAG=latest\n")
|
||||||
|
result, log = run_shell(tmp_path, "deploy.sh")
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert not log
|
||||||
|
assert (tmp_path / ".env").read_text() == "APP_TAG=latest\n"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("text", [
|
||||||
|
"auth:\n enabled: always\n",
|
||||||
|
"database: /data/app.db\n",
|
||||||
|
"database: /data/app.db\nauth:\n rp_id: app.example.com\nenabled: always\n",
|
||||||
|
"database: /data/app.db\n# auth:\n# enabled: always\n",
|
||||||
|
"database:\nauth:\n enabled: always\n",
|
||||||
|
"# database: /data/app.db\nauth:\n enabled: always\n",
|
||||||
|
])
|
||||||
|
def test_deploy_refuses_a_config_without_database_and_auth_enabled(tmp_path, text):
|
||||||
|
"""The two keys whose defaults are wrong in a container. Read on the host
|
||||||
|
without a YAML parser, so a key commented out or outside its section is
|
||||||
|
the same as a missing one."""
|
||||||
|
(tmp_path / "data").mkdir()
|
||||||
|
(tmp_path / "data/config.yaml").write_text(text)
|
||||||
|
(tmp_path / ".env").write_text("APP_TAG=latest\n")
|
||||||
|
result, log = run_shell(tmp_path, "deploy.sh")
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert "must set database and auth.enabled" in result.stderr
|
||||||
|
assert not log
|
||||||
|
|
||||||
|
|
||||||
|
def test_deploy_reads_auth_enabled_past_comments_and_other_keys(tmp_path):
|
||||||
|
(tmp_path / "data").mkdir()
|
||||||
|
(tmp_path / "data/config.yaml").write_text(
|
||||||
|
"auth:\n # locked\n\n rp_id: x\n enabled: always # behind SWAG\n"
|
||||||
|
"database: /data/app.db\n")
|
||||||
|
(tmp_path / ".env").write_text("APP_TAG=latest\n")
|
||||||
|
result, log = run_shell(tmp_path, "deploy.sh")
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert f"compose pull {SERVICE}" in log
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# The release scripts that run in bash: _common.sh (the release.env reader) and
|
||||||
|
# 1-test-and-promote.sh. Each test copies them into a scratch git repository,
|
||||||
|
# which is the checkout they find their root from. test-commit.sh, the gates,
|
||||||
|
# is replaced by a stub that passes or fails on request; nothing runs the
|
||||||
|
# network or the real test suite. Steps 2 and 3 are PowerShell running wslc on
|
||||||
|
# Windows, and are not run here: there is no PowerShell and no wslc in the
|
||||||
|
# sandbox.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
IMAGE = "registry.example/team/app"
|
||||||
|
DEPLOY = ROOT / "deploy"
|
||||||
|
RELEASE_SCRIPTS = ["_common.sh", "1-test-and-promote.sh"]
|
||||||
|
|
||||||
|
TOOLS = ["bash", "sh", "env", "git", "tar", "dirname", "basename", "realpath", "rm", "mkdir",
|
||||||
|
"mktemp", "sed", "cat", "uname", "date", "head", "tr", "sleep", "readlink", "chmod",
|
||||||
|
"ls", "wc", "grep", "cut", "sort", "touch", "ln", "cp", "mv", "id", "hostname"]
|
||||||
|
|
||||||
|
|
||||||
|
def git(repo: Path, *args: str) -> str:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "-C", repo.as_posix(), "-c", "user.name=t", "-c", "user.email=t@example.org",
|
||||||
|
*args], capture_output=True, text=True, check=True)
|
||||||
|
return result.stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def release(tmp_path):
|
||||||
|
"""A scratch repository on main holding the release scripts and a stub gate."""
|
||||||
|
found = shutil.which("bash")
|
||||||
|
if found is None or shutil.which("git") is None:
|
||||||
|
pytest.skip("bash and git are required for release script tests")
|
||||||
|
bash: str = found
|
||||||
|
repo = tmp_path / "repo"
|
||||||
|
(repo / "deploy").mkdir(parents=True)
|
||||||
|
for name in RELEASE_SCRIPTS:
|
||||||
|
shutil.copy(DEPLOY / name, repo / "deploy" / name)
|
||||||
|
(repo / "deploy/test-commit.sh").write_text(
|
||||||
|
'#!/bin/sh\necho "$1" >> "$DEPLOY_TEST_GATE"\n[ -z "$DEPLOY_TEST_RED" ]\n')
|
||||||
|
(repo / "deploy/release.env").write_text(f"IMAGE={IMAGE}\n")
|
||||||
|
(repo / "deploy/Dockerfile.dockerignore").write_text("tests\n")
|
||||||
|
(repo / "README").write_text("one\n")
|
||||||
|
subprocess.run(["git", "init", "-q", "-b", "main", repo.as_posix()], check=True)
|
||||||
|
git(repo, "add", "-A", "-f")
|
||||||
|
git(repo, "commit", "-qm", "one")
|
||||||
|
# A PATH of only what the scripts need.
|
||||||
|
bare = tmp_path / "bare"
|
||||||
|
bare.mkdir()
|
||||||
|
for tool in TOOLS:
|
||||||
|
found = shutil.which(tool)
|
||||||
|
if found:
|
||||||
|
(bare / tool).symlink_to(found)
|
||||||
|
(tmp_path / "tmp").mkdir()
|
||||||
|
|
||||||
|
class Release:
|
||||||
|
def __init__(self):
|
||||||
|
self.repo, self.tmp = repo, tmp_path
|
||||||
|
self.gate = tmp_path / "gate.log"
|
||||||
|
self.commit = git(repo, "rev-parse", "HEAD")
|
||||||
|
|
||||||
|
def run(self, script, *args, **extra):
|
||||||
|
env = {**os.environ, "PATH": str(bare), "DEPLOY_TEST_GATE": self.gate.as_posix(),
|
||||||
|
"TMPDIR": (tmp_path / "tmp").as_posix(), "DEPLOY_TEST_RED": ""}
|
||||||
|
env.pop("RELEASE_DIR", None)
|
||||||
|
env.update({k: str(v) for k, v in extra.items()})
|
||||||
|
return subprocess.run([bash, (repo / "deploy" / script).as_posix(), *args],
|
||||||
|
cwd=tmp_path, capture_output=True, text=True, env=env)
|
||||||
|
|
||||||
|
def gate_log(self):
|
||||||
|
return self.gate.read_text().split() if self.gate.exists() else []
|
||||||
|
|
||||||
|
def branches(self):
|
||||||
|
return git(repo, "branch", "--format=%(refname:short)").split()
|
||||||
|
|
||||||
|
return Release()
|
||||||
|
|
||||||
|
|
||||||
|
def read_settings(release, text: bytes) -> subprocess.CompletedProcess:
|
||||||
|
"""Source _common.sh over a release.env with this text and print what it read."""
|
||||||
|
(release.repo / "deploy/release.env").write_bytes(text)
|
||||||
|
probe = release.repo / "deploy/probe.sh"
|
||||||
|
probe.write_text(
|
||||||
|
'set -euo pipefail\n. "$(dirname "${BASH_SOURCE[0]}")/_common.sh"\n'
|
||||||
|
'echo "IMAGE=[$(setting IMAGE)] BRANCH=[$(setting BRANCH production)] '
|
||||||
|
'NOTE=[$(setting NOTE)] SOURCE=[$(setting SOURCE main)]"\n'
|
||||||
|
'required=$(require_setting IMAGE)\necho "required=[$required]"\n')
|
||||||
|
return release.run("probe.sh")
|
||||||
|
|
||||||
|
|
||||||
|
def test_common_reads_release_env_without_sourcing_it(release):
|
||||||
|
marker = release.tmp / "ran"
|
||||||
|
text = (f'# a comment\n\n IMAGE = "{IMAGE}" \nNOTE=has a space # not a comment\n'
|
||||||
|
f'BRANCH=\nX=$(touch {marker})\n').encode()
|
||||||
|
result = read_settings(release, text)
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert f"IMAGE=[{IMAGE}] BRANCH=[production] NOTE=[has a space # not a comment]" in result.stdout
|
||||||
|
assert "SOURCE=[main]" in result.stdout
|
||||||
|
assert not marker.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_common_reads_a_crlf_release_env(release):
|
||||||
|
result = read_settings(release, f"IMAGE={IMAGE}\r\nNOTE=x\r\n".encode())
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert f"required=[{IMAGE}]" in result.stdout
|
||||||
|
assert "NOTE=[x]" in result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("text, message", [
|
||||||
|
(b"IMAGE=" + IMAGE.encode() + b"\nnot a setting\n", "not a KEY=VALUE line: not a setting"),
|
||||||
|
(b"NOTE=x\n", "set IMAGE in deploy/release.env"),
|
||||||
|
(b'IMAGE=""\n', "set IMAGE in deploy/release.env"),
|
||||||
|
])
|
||||||
|
def test_common_refuses_a_bad_release_env(release, text, message):
|
||||||
|
result = read_settings(release, text)
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert message in result.stderr
|
||||||
|
|
||||||
|
|
||||||
|
def test_common_refuses_a_missing_release_env(release):
|
||||||
|
(release.repo / "deploy/release.env").unlink()
|
||||||
|
result = release.run("1-test-and-promote.sh")
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert "no deploy/release.env" in result.stderr
|
||||||
|
assert not release.gate_log()
|
||||||
|
|
||||||
|
|
||||||
|
def test_test_and_promote_tests_the_commit_then_promotes_it(release):
|
||||||
|
result = release.run("1-test-and-promote.sh")
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert release.gate_log() == [release.commit]
|
||||||
|
assert git(release.repo, "rev-parse", "production") == release.commit
|
||||||
|
assert f"Promoted production to {release.commit}" in result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def test_test_and_promote_leaves_uncommitted_work_out(release):
|
||||||
|
(release.repo / "README").write_text("edited\n")
|
||||||
|
(release.repo / "untracked.txt").write_text("new\n")
|
||||||
|
result = release.run("1-test-and-promote.sh")
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert release.gate_log() == [release.commit]
|
||||||
|
assert git(release.repo, "rev-parse", "production") == release.commit
|
||||||
|
assert git(release.repo, "show", "production:README") == "one"
|
||||||
|
assert (release.repo / "README").read_text() == "edited\n"
|
||||||
|
|
||||||
|
|
||||||
|
def test_test_and_promote_red_tests_promote_nothing(release):
|
||||||
|
result = release.run("1-test-and-promote.sh", DEPLOY_TEST_RED="1")
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert "tests failed (exit 1)" in result.stderr
|
||||||
|
assert "production" not in release.branches()
|
||||||
|
|
||||||
|
|
||||||
|
def test_test_and_promote_no_tests_skips_the_gate(release):
|
||||||
|
result = release.run("1-test-and-promote.sh", "--no-tests", DEPLOY_TEST_RED="1")
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert not release.gate_log()
|
||||||
|
assert "production" in release.branches()
|
||||||
|
|
||||||
|
|
||||||
|
def test_test_and_promote_refuses_a_production_with_commits_of_its_own(release):
|
||||||
|
git(release.repo, "switch", "-qc", "production")
|
||||||
|
(release.repo / "hotfix").write_text("x\n")
|
||||||
|
git(release.repo, "add", "hotfix")
|
||||||
|
git(release.repo, "commit", "-qm", "hotfix")
|
||||||
|
own = git(release.repo, "rev-parse", "production")
|
||||||
|
git(release.repo, "switch", "-q", "main")
|
||||||
|
result = release.run("1-test-and-promote.sh")
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert "production is not an ancestor of main" in result.stderr
|
||||||
|
assert "git log --oneline main..production" in result.stderr
|
||||||
|
assert git(release.repo, "rev-parse", "production") == own
|
||||||
|
|
||||||
|
|
||||||
|
def test_test_and_promote_fast_forwards_an_older_production(release):
|
||||||
|
git(release.repo, "branch", "production")
|
||||||
|
(release.repo / "README").write_text("two\n")
|
||||||
|
git(release.repo, "commit", "-qam", "two")
|
||||||
|
newer = git(release.repo, "rev-parse", "HEAD")
|
||||||
|
result = release.run("1-test-and-promote.sh")
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert git(release.repo, "rev-parse", "production") == newer
|
||||||
|
|
||||||
|
|
||||||
|
def test_test_and_promote_publishes_the_branch_only_on_request(release):
|
||||||
|
origin = release.tmp / "origin.git"
|
||||||
|
subprocess.run(["git", "init", "-q", "--bare", origin.as_posix()], check=True)
|
||||||
|
git(release.repo, "remote", "add", "origin", origin.as_posix())
|
||||||
|
assert release.run("1-test-and-promote.sh").returncode == 0
|
||||||
|
assert subprocess.run(["git", "-C", origin.as_posix(), "branch"], capture_output=True,
|
||||||
|
text=True).stdout.strip() == ""
|
||||||
|
result = release.run("1-test-and-promote.sh", "--push-branch")
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert git(origin, "rev-parse", "production") == release.commit
|
||||||
|
|
||||||
|
|
||||||
|
def test_test_and_promote_honours_source_and_branch_settings(release):
|
||||||
|
git(release.repo, "branch", "next")
|
||||||
|
(release.repo / "deploy/release.env").write_text(f"IMAGE={IMAGE}\nSOURCE=next\nBRANCH=prod\n")
|
||||||
|
result = release.run("1-test-and-promote.sh")
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert git(release.repo, "rev-parse", "prod") == release.commit
|
||||||
|
|
||||||
|
|
||||||
|
def test_test_and_promote_refuses_an_unknown_option(release):
|
||||||
|
result = release.run("1-test-and-promote.sh", "--force")
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert "unknown option --force" in result.stderr
|
||||||
|
assert not release.gate_log()
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_example_release_env_is_read_by_the_bash_reader_and_names_the_checkout(release):
|
||||||
|
"""One file serves both sides; the profile and steps 2 and 3 need DISTRO and
|
||||||
|
CHECKOUT. wslc has no --platform, so there is no PLATFORM."""
|
||||||
|
example = (DEPLOY / "release.env.example").read_bytes()
|
||||||
|
result = read_settings(release, example)
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
keys = [line.split("=")[0].strip() for line in example.decode().splitlines()
|
||||||
|
if "=" in line and not line.lstrip().startswith("#")]
|
||||||
|
assert {"IMAGE", "DISTRO", "CHECKOUT"} <= set(keys)
|
||||||
|
assert "PLATFORM" not in keys
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Step 1's export: the production tree and its sha, in release/<folder>/ beside
|
||||||
|
# the checkout (or RELEASE_DIR/<folder>/), where 2-build-image.ps1 reads them
|
||||||
|
# through \\wsl.localhost.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def short(release) -> str:
|
||||||
|
return git(release.repo, "rev-parse", "--short", release.commit)
|
||||||
|
|
||||||
|
|
||||||
|
def test_test_and_promote_exports_the_tree_beside_the_checkout(release):
|
||||||
|
(release.repo / "README").write_text("edited\n")
|
||||||
|
result = release.run("1-test-and-promote.sh")
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
out = release.tmp / "release" / "repo"
|
||||||
|
sha = short(release)
|
||||||
|
assert (out / "production").read_text() == f"{sha}\n"
|
||||||
|
assert (out / sha / "README").read_text() == "one\n"
|
||||||
|
assert (out / sha / "deploy/test-commit.sh").is_file()
|
||||||
|
# wslc streams the Dockerfile, so the ignore rules go to the context's root.
|
||||||
|
assert (out / sha / ".dockerignore").read_text() == "tests\n"
|
||||||
|
assert sorted(p.name for p in out.iterdir()) == sorted([sha, "production"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_test_and_promote_honours_release_dir(release):
|
||||||
|
share = release.tmp / "share"
|
||||||
|
result = release.run("1-test-and-promote.sh", RELEASE_DIR=share)
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert (share / "repo/production").read_text() == f"{short(release)}\n"
|
||||||
|
assert not (release.tmp / "release").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_test_and_promote_keeps_this_shas_tree_and_drops_older_ones(release):
|
||||||
|
share = release.tmp / "share"
|
||||||
|
assert release.run("1-test-and-promote.sh", RELEASE_DIR=share).returncode == 0
|
||||||
|
first = short(release)
|
||||||
|
marker = share / "repo" / first / "marker"
|
||||||
|
marker.write_text("kept\n")
|
||||||
|
assert release.run("1-test-and-promote.sh", RELEASE_DIR=share).returncode == 0
|
||||||
|
assert marker.read_text() == "kept\n"
|
||||||
|
(release.repo / "README").write_text("two\n")
|
||||||
|
git(release.repo, "commit", "-qam", "two")
|
||||||
|
result = release.run("1-test-and-promote.sh", RELEASE_DIR=share)
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
second = git(release.repo, "rev-parse", "--short", "HEAD")
|
||||||
|
assert sorted(p.name for p in (share / "repo").iterdir()) == sorted([second, "production"])
|
||||||
|
assert (share / "repo" / second / "README").read_text() == "two\n"
|
||||||
|
|
||||||
|
|
||||||
|
def test_test_and_promote_says_so_when_the_export_cannot_be_written(release):
|
||||||
|
blocked = release.tmp / "blocked"
|
||||||
|
blocked.write_text("a file, not a folder\n")
|
||||||
|
result = release.run("1-test-and-promote.sh", RELEASE_DIR=blocked)
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert f"cannot write {blocked.as_posix()}/repo" in result.stderr
|
||||||
|
assert "production is promoted" in result.stderr
|
||||||
|
assert git(release.repo, "rev-parse", "production") == release.commit
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# deploy/db.sh. `ssh` and `scp` are stubs that log their arguments (and, for
|
||||||
|
# scp, copy nothing); the snapshot is taken for real, from a real SQLite file.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
SSH_STUB = '''#!/bin/sh
|
||||||
|
printf 'ssh %s\\n' "$*" >> "$DEPLOY_TEST_LOG"
|
||||||
|
case "$*" in
|
||||||
|
*"ls -1t"*) echo /srv/data/app/backups/app-20261015-040000.db.gz ;;
|
||||||
|
*"bash -s"*) cat > "$DEPLOY_TEST_STDIN" ;;
|
||||||
|
esac
|
||||||
|
'''
|
||||||
|
SCP_STUB = '''#!/bin/sh
|
||||||
|
printf 'scp %s\\n' "$*" >> "$DEPLOY_TEST_LOG"
|
||||||
|
for last; do :; done
|
||||||
|
case "$last" in */) : > "${last}app-20261015-040000.db.gz" ;; esac
|
||||||
|
exit 0
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def database(tmp_path):
|
||||||
|
"""A scratch repository folder holding db.sh, a database, and stub ssh and scp."""
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
|
||||||
|
found = shutil.which("bash")
|
||||||
|
if found is None:
|
||||||
|
pytest.skip("bash is required for deployment script tests")
|
||||||
|
bash: str = found
|
||||||
|
repo = tmp_path / "repo"
|
||||||
|
(repo / "deploy/server").mkdir(parents=True)
|
||||||
|
for name in ("_common.sh", "db.sh", "server/import.sh", "server/import_database.py"):
|
||||||
|
shutil.copy(DEPLOY / name, repo / "deploy" / name)
|
||||||
|
(repo / "deploy/release.env").write_text(
|
||||||
|
f"SSH_HOST=box\nREMOTE_DIR=/srv/swag\nREMOTE_DATA=/srv/data/app\nNAME={SERVICE}\n"
|
||||||
|
f"PYTHON={sys.executable}\n")
|
||||||
|
# Where the app writes without a config.yaml (`database: roledesk.db`),
|
||||||
|
# which is where db.sh looks when release.env names no LOCAL_DB.
|
||||||
|
with sqlite3.connect(repo / "roledesk.db") as db:
|
||||||
|
db.execute("CREATE TABLE t (n)")
|
||||||
|
db.execute("INSERT INTO t VALUES (1)")
|
||||||
|
stubs = tmp_path / "stubs"
|
||||||
|
stubs.mkdir()
|
||||||
|
for name, text in (("ssh", SSH_STUB), ("scp", SCP_STUB)):
|
||||||
|
(stubs / name).write_text(text)
|
||||||
|
(stubs / name).chmod(0o755)
|
||||||
|
log, stdin = tmp_path / "net.log", tmp_path / "stdin.txt"
|
||||||
|
|
||||||
|
def run(*args: str, input: str | None = None):
|
||||||
|
env = {**os.environ, "PATH": f"{stubs}:{os.environ['PATH']}",
|
||||||
|
"DEPLOY_TEST_LOG": log.as_posix(), "DEPLOY_TEST_STDIN": stdin.as_posix()}
|
||||||
|
env.pop("APP_CONFIG", None)
|
||||||
|
result = subprocess.run([bash, (repo / "deploy/db.sh").as_posix(), *args], cwd=tmp_path,
|
||||||
|
capture_output=True, text=True, env=env, input=input)
|
||||||
|
return result, (log.read_text() if log.exists() else ""), stdin
|
||||||
|
|
||||||
|
run.repo = repo # type: ignore[attr-defined]
|
||||||
|
return run
|
||||||
|
|
||||||
|
|
||||||
|
def test_db_push_asks_first_and_sends_nothing_on_no(database):
|
||||||
|
result, log, _ = database("push", input="n\n")
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert "stopped." in result.stdout
|
||||||
|
assert log == ""
|
||||||
|
assert not list(database.repo.glob("app-*.db.gz"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_db_push_uploads_a_snapshot_and_streams_import_sh(database):
|
||||||
|
result, log, stdin = database("push", "--yes")
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
lines = log.splitlines()
|
||||||
|
assert lines[0].startswith("scp -C ") and lines[0].endswith("box:/srv/data/app/incoming.db.gz")
|
||||||
|
assert lines[1].endswith("box:/srv/swag/import_database.py")
|
||||||
|
assert lines[2] == "ssh box bash -s -- '/srv/swag'"
|
||||||
|
assert stdin.read_text() == (DEPLOY / "server/import.sh").read_text()
|
||||||
|
assert f"docker compose up -d --wait {SERVICE}" in result.stdout
|
||||||
|
assert f"snapshotting {(database.repo / 'roledesk.db').as_posix()}" in result.stdout
|
||||||
|
assert not list(database.repo.glob("app-*.db.gz"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_db_push_takes_the_database_the_config_names(database):
|
||||||
|
(database.repo / "config.yaml").write_text("database: elsewhere/mine.db\n")
|
||||||
|
result, _, _ = database("push", input="n\n")
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert f"no {(database.repo / 'elsewhere/mine.db').as_posix()}" in result.stderr
|
||||||
|
(database.repo / "elsewhere").mkdir()
|
||||||
|
shutil.copy(database.repo / "roledesk.db", database.repo / "elsewhere/mine.db")
|
||||||
|
result, _, _ = database("push", input="n\n")
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert f"snapshotting {(database.repo / 'elsewhere/mine.db').as_posix()}" in result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def test_db_push_takes_local_db_over_the_config(database):
|
||||||
|
with (database.repo / "deploy/release.env").open("a") as env:
|
||||||
|
env.write(f"LOCAL_DB={(database.repo / 'roledesk.db').as_posix()}\n")
|
||||||
|
(database.repo / "config.yaml").write_text("database: nowhere.db\n")
|
||||||
|
result, _, _ = database("push", input="n\n")
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert f"snapshotting {(database.repo / 'roledesk.db').as_posix()}" in result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def test_db_pull_runs_the_servers_backup_and_downloads_the_newest(database):
|
||||||
|
result, log, _ = database("pull")
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert log.splitlines()[0] == "ssh box cd '/srv/swag' && bash backup.sh"
|
||||||
|
assert log.splitlines()[-1].endswith("box:/srv/data/app/backups/app-20261015-040000.db.gz " +
|
||||||
|
(database.repo / "backups").as_posix() + "/")
|
||||||
|
assert "backups/app-20261015-040000.db.gz" in result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def test_db_needs_an_action_and_a_server(database):
|
||||||
|
result, _, _ = database()
|
||||||
|
assert result.returncode != 0 and "push or pull" in result.stderr
|
||||||
|
(database.repo / "deploy/release.env").write_text("REMOTE_DIR=/srv/swag\n")
|
||||||
|
result, log, _ = database("pull")
|
||||||
|
assert result.returncode != 0 and "set SSH_HOST in deploy/release.env" in result.stderr
|
||||||
|
assert log == ""
|
||||||
|
# The compose service has no default here: a renamed project would
|
||||||
|
# otherwise pull and print the start command for `app`.
|
||||||
|
(database.repo / "deploy/release.env").write_text("SSH_HOST=box\nREMOTE_DIR=/srv/swag\n")
|
||||||
|
result, log, _ = database("pull")
|
||||||
|
assert result.returncode != 0 and "set NAME in deploy/release.env" in result.stderr
|
||||||
|
assert log == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_backup_script_writes_owner_only():
|
||||||
|
"""A snapshot is the whole database. The host umask is set before anything
|
||||||
|
is made; the container's before the `.db` is written, because gzip copies
|
||||||
|
that file's mode to the `.gz`."""
|
||||||
|
text = (DEPLOY / "server" / "backup.sh").read_text(encoding="utf-8")
|
||||||
|
assert "\numask 077\n" in text
|
||||||
|
assert text.index("\numask 077\n") < text.index("mkdir -p")
|
||||||
|
assert text.index("os.umask(0o077)") < text.index('db.execute("VACUUM INTO')
|
||||||
|
|
||||||
|
|
||||||
|
def backup_snippet() -> str:
|
||||||
|
"""The Python backup.sh runs inside the container, from its heredoc."""
|
||||||
|
text = (SERVER / "backup.sh").read_text(encoding="utf-8")
|
||||||
|
return text.split("<<'PY'\n", 1)[1].split("\nPY\n", 1)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def import_database():
|
||||||
|
spec = importlib.util.spec_from_file_location("import_database", SERVER / "import_database.py")
|
||||||
|
assert spec and spec.loader
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_backup_asks_the_app_for_the_database():
|
||||||
|
"""No path to the database in the script: the app's config names it, so a
|
||||||
|
server config with another `database` is backed up and not skipped."""
|
||||||
|
text = (SERVER / "backup.sh").read_text(encoding="utf-8")
|
||||||
|
assert "/data/app.db" not in text and "/data/roledesk.db" not in text
|
||||||
|
assert "from app.config import load" in backup_snippet()
|
||||||
|
tables = re.search(r'^TABLES="([^"]*)"$', text, re.M)
|
||||||
|
assert tables and set(tables[1].split()) == import_database().REQUIRED_TABLES
|
||||||
|
|
||||||
|
|
||||||
|
def run_backup(tmp_path: Path, database: Path):
|
||||||
|
(tmp_path / "config.yaml").write_text(f"database: {database.as_posix()}\n")
|
||||||
|
backups = tmp_path / "backups"
|
||||||
|
backups.mkdir(exist_ok=True)
|
||||||
|
code = backup_snippet().replace('"/data/backups"', repr(backups.as_posix()))
|
||||||
|
env = {**os.environ, "APP_CONFIG": (tmp_path / "config.yaml").as_posix()}
|
||||||
|
tables = sorted(import_database().REQUIRED_TABLES)
|
||||||
|
result = subprocess.run([sys.executable, "-", "app-1", *tables], input=code, cwd=ROOT,
|
||||||
|
capture_output=True, text=True, env=env)
|
||||||
|
return result, backups / "app-1.db"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_backup_of_a_database_that_is_not_there_fails(tmp_path):
|
||||||
|
"""Opened read-only, so a wrong path is an error and not a new empty file
|
||||||
|
whose snapshot looks like a backup."""
|
||||||
|
missing = tmp_path / "data/app.db"
|
||||||
|
result, snapshot = run_backup(tmp_path, missing)
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert "unable to open database file" in result.stderr
|
||||||
|
assert not missing.exists() and not snapshot.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_backup_of_a_database_without_the_tables_fails(tmp_path):
|
||||||
|
with sqlite3.connect(tmp_path / "other.db") as db:
|
||||||
|
db.execute("CREATE TABLE items (n)")
|
||||||
|
result, snapshot = run_backup(tmp_path, tmp_path / "other.db")
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert "not this app's database" in result.stderr
|
||||||
|
assert not snapshot.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_backup_snapshots_the_apps_database_owner_only(tmp_path):
|
||||||
|
from app.config import Config
|
||||||
|
from app.main import create_app
|
||||||
|
|
||||||
|
create_app(Config(database=tmp_path / "app.db"))
|
||||||
|
result, snapshot = run_backup(tmp_path, tmp_path / "app.db")
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert snapshot.exists()
|
||||||
|
if os.name == "posix":
|
||||||
|
assert snapshot.stat().st_mode & 0o777 == 0o600
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_importer_knows_this_apps_tables(tmp_path):
|
||||||
|
"""A fresh database of this app passes the importer's table check, so a
|
||||||
|
project that replaces its schema is told here to name its own table."""
|
||||||
|
from app.config import Config
|
||||||
|
from app.main import create_app
|
||||||
|
|
||||||
|
create_app(Config(database=tmp_path / "app.db"))
|
||||||
|
with sqlite3.connect(tmp_path / "app.db") as db:
|
||||||
|
tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||||||
|
assert import_database().REQUIRED_TABLES <= tables
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_importer_replaces_this_apps_database_and_refuses_another(tmp_path):
|
||||||
|
import gzip
|
||||||
|
from contextlib import closing
|
||||||
|
|
||||||
|
from app.config import Config
|
||||||
|
from app.main import create_app
|
||||||
|
|
||||||
|
def archive(source: Path, target: Path) -> None:
|
||||||
|
"""What db.sh uploads: a VACUUM INTO of the WAL database, gzipped."""
|
||||||
|
plain = source.with_suffix(".snapshot")
|
||||||
|
with closing(sqlite3.connect(source)) as db:
|
||||||
|
db.execute("VACUUM INTO ?", (str(plain),))
|
||||||
|
with plain.open("rb") as raw, gzip.open(target, "wb") as out:
|
||||||
|
shutil.copyfileobj(raw, out)
|
||||||
|
|
||||||
|
module = import_database()
|
||||||
|
create_app(Config(database=tmp_path / "incoming.db"))
|
||||||
|
with closing(sqlite3.connect(tmp_path / "incoming.db")) as db, db:
|
||||||
|
db.execute("INSERT INTO state (key, value) VALUES ('at', 'then')")
|
||||||
|
archive(tmp_path / "incoming.db", tmp_path / "in.db.gz")
|
||||||
|
with closing(sqlite3.connect(tmp_path / "wrong.db")) as db, db:
|
||||||
|
db.execute("CREATE TABLE items (n)")
|
||||||
|
archive(tmp_path / "wrong.db", tmp_path / "wrong.db.gz")
|
||||||
|
(tmp_path / "live").mkdir()
|
||||||
|
live = tmp_path / "live/app.db"
|
||||||
|
create_app(Config(database=live))
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="not this application's database"):
|
||||||
|
module.replace_database(live, tmp_path / "wrong.db.gz", tmp_path / "backups")
|
||||||
|
saved = module.replace_database(live, tmp_path / "in.db.gz", tmp_path / "backups")
|
||||||
|
assert saved is not None and saved.exists()
|
||||||
|
with closing(sqlite3.connect(live)) as db:
|
||||||
|
assert db.execute("SELECT value FROM state WHERE key = 'at'").fetchall() == [("then",)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_importer_snapshots_owner_only():
|
||||||
|
"""The snapshot before the swap is the whole database, written by the
|
||||||
|
container's uid under whatever umask the image has."""
|
||||||
|
text = (SERVER / "import_database.py").read_text(encoding="utf-8")
|
||||||
|
main = text.split('if __name__ == "__main__":', 1)[1]
|
||||||
|
assert main.index("os.umask(0o077)") < main.index("replace_database(")
|
||||||
|
assert "/data/app.db" not in text and "/data/roledesk.db" not in text
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_file_names_the_same_service():
|
||||||
|
"""The compose service is repeated where a script or nginx has to name it.
|
||||||
|
A rename that misses one deploys the image and then stops, backs up or
|
||||||
|
proxies to a container that does not exist."""
|
||||||
|
compose = yaml.safe_load((SERVER / "compose.yaml").read_text(encoding="utf-8"))
|
||||||
|
assert list(compose["services"]) == [SERVICE]
|
||||||
|
assert compose["services"][SERVICE]["container_name"] == SERVICE
|
||||||
|
for script in ("deploy.sh", "backup.sh", "import.sh"):
|
||||||
|
text = (SERVER / script).read_text(encoding="utf-8")
|
||||||
|
assert re.findall(r"^SERVICE=(\S+)$", text, re.M) == [SERVICE], script
|
||||||
|
conf = (SERVER / "swag" / "app.subdomain.conf").read_text(encoding="utf-8")
|
||||||
|
assert set(re.findall(r"set \$upstream_app (\S+);", conf)) == {SERVICE}
|
||||||
|
example = (DEPLOY / "release.env.example").read_text(encoding="utf-8")
|
||||||
|
assert re.findall(r"^NAME=(\S+)$", example, re.M) == [SERVICE]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_examples_carry_no_real_host():
|
||||||
|
"""The server templates start from placeholders; the real hostnames are in
|
||||||
|
docs/Install Server.md only. Ours: release.env.example is not held to it,
|
||||||
|
it names our registry's image, which the task keeps."""
|
||||||
|
for path in [*SERVER.rglob("*"), ROOT / "config.example.yaml"]:
|
||||||
|
if path.is_file() and "__pycache__" not in path.parts:
|
||||||
|
assert "rittershofer" not in path.read_text(encoding="utf-8"), path
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_proxy_throttles_every_open_login_route():
|
||||||
|
"""`/login/passkey/options` is open and writes a challenge row per call."""
|
||||||
|
conf = (SERVER / "swag" / "app.subdomain.conf").read_text(encoding="utf-8")
|
||||||
|
throttled = [m for m in re.finditer(r"location ~ (\S+) \{([^}]*)\}", conf)
|
||||||
|
if "limit_req zone=" in m[2]]
|
||||||
|
assert len(throttled) == 1
|
||||||
|
pattern = re.compile(throttled[0][1])
|
||||||
|
for path in ("/login", "/login/passkey", "/login/passkey/options"):
|
||||||
|
assert pattern.search(path), path
|
||||||
|
for path in ("/login/other", "/api/login", "/login/passkey/register"):
|
||||||
|
assert not pattern.search(path), path
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_proxy_conf_does_not_spool_unlimited_bodies():
|
||||||
|
"""nginx writes a body to disk before the app can refuse it, and the login
|
||||||
|
takes one without a session. 0 belongs only on a location that is a
|
||||||
|
deliberate upload route."""
|
||||||
|
conf = (DEPLOY / "server" / "swag" / "app.subdomain.conf").read_text(encoding="utf-8")
|
||||||
|
stack: list[str] = []
|
||||||
|
limits: dict[str, str] = {}
|
||||||
|
for line in conf.splitlines():
|
||||||
|
code = line.split("#", 1)[0].strip()
|
||||||
|
if code.endswith("{"):
|
||||||
|
stack.append(code[:-1].strip())
|
||||||
|
elif code == "}":
|
||||||
|
stack.pop()
|
||||||
|
elif code.startswith("client_max_body_size"):
|
||||||
|
limits[stack[-1]] = code.rstrip(";").split()[1]
|
||||||
|
assert not stack, "unbalanced braces"
|
||||||
|
assert limits.get("server") not in (None, "0")
|
||||||
|
assert "location /" not in limits
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# roledesk's own facts, put back after a refresh of deploy/ from the boilerplate.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_server_config_is_one_the_app_accepts():
|
||||||
|
"""An unknown key is an error, so a stale example would start a container
|
||||||
|
that dies at load. The values are the ones a container needs."""
|
||||||
|
from app.config import load
|
||||||
|
|
||||||
|
config = load(SERVER / "config.example.yaml")
|
||||||
|
assert config.server.host == "0.0.0.0" and config.server.port == 8787
|
||||||
|
assert config.database.as_posix() == "/data/roledesk.db"
|
||||||
|
assert config.auth.enabled == "always" and config.auth.secure_cookie == "always"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_dockerfile_is_roledesks():
|
||||||
|
text = (DEPLOY / "Dockerfile").read_text(encoding="utf-8")
|
||||||
|
lines = [line.strip() for line in text.splitlines() if not line.startswith("#")]
|
||||||
|
assert "frontend" not in "\n".join(lines) and "node" not in "\n".join(lines).lower()
|
||||||
|
for folder in ("desk", "plugins", "seed", "llm"):
|
||||||
|
assert lines.count(f"COPY {folder} {folder}") == 2, folder
|
||||||
|
syncs = [i for i, line in enumerate(lines) if line.startswith("uv sync")]
|
||||||
|
assert len(syncs) == 2
|
||||||
|
for i in syncs:
|
||||||
|
block = " ".join(lines[i:i + 2])
|
||||||
|
for extra in ("passkeys", "embeddings", "llm"):
|
||||||
|
assert f"--extra {extra}" in block, (i, extra)
|
||||||
|
assert "USER 1000:1000" in lines
|
||||||
|
assert any("127.0.0.1:8787/health" in line for line in lines)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_release_env_example_names_our_checkout_and_image():
|
||||||
|
example = (DEPLOY / "release.env.example").read_text(encoding="utf-8")
|
||||||
|
assert re.findall(r"^CHECKOUT=(\S+)$", example, re.M) == ["/home/claude/cboxcode/jobboard"]
|
||||||
|
assert re.findall(r"^IMAGE=(\S+)$", example, re.M) == ["git.rittershofer.org/wolf/app"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_dev_commands_are_the_ones_the_run_configurations_run():
|
||||||
|
"""`.run/` stays; each of its commands has a bin script and a profile command."""
|
||||||
|
assert sorted(p.name for p in (ROOT / "bin").iterdir()) == [
|
||||||
|
"backend.sh", "check.sh", "pull.sh", "worker.sh"]
|
||||||
|
for script, command in (("pull.sh", "uv run app job pull"), ("worker.sh", "uv run app worker")):
|
||||||
|
text = (ROOT / "bin" / script).read_text(encoding="utf-8")
|
||||||
|
assert f'exec {command} "$@"' in text, script
|
||||||
|
profile = (DEPLOY / "wsl-setup.ps1").read_text(encoding="utf-8")
|
||||||
|
for name in ("backend", "pull", "worker", "check"):
|
||||||
|
assert f"function $Name-{name} {{ $wsl -- bash bin/{name}.sh @args }}" in profile, name
|
||||||
|
assert "frontend" not in profile
|
||||||
|
|
||||||
|
|
||||||
|
def test_backend_sends_each_argument_to_the_command_that_takes_it(tmp_path):
|
||||||
|
"""`-v` and `--config` belong before `serve`, `--reload` and `--port` after."""
|
||||||
|
bash = shutil.which("bash")
|
||||||
|
if bash is None:
|
||||||
|
pytest.skip("bash is required for deployment script tests")
|
||||||
|
(tmp_path / ".local/bin").mkdir(parents=True)
|
||||||
|
uv = tmp_path / ".local/bin/uv"
|
||||||
|
uv.write_text("#!/bin/sh\necho \"$*\"\n")
|
||||||
|
uv.chmod(0o755)
|
||||||
|
env = {**os.environ, "HOME": tmp_path.as_posix()}
|
||||||
|
|
||||||
|
def run(*args: str) -> str:
|
||||||
|
return subprocess.run([bash, (ROOT / "bin/backend.sh").as_posix(), *args],
|
||||||
|
capture_output=True, text=True, env=env, check=True).stdout.strip()
|
||||||
|
|
||||||
|
assert run() == "run app serve"
|
||||||
|
assert run("--reload") == "run app serve --reload"
|
||||||
|
assert run("-v", "--reload", "--port", "9") == "run app -v serve --reload --port 9"
|
||||||
|
assert run("-c", "x.yaml") == "run app -c x.yaml serve"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_batch_launcher_stays_crlf_and_the_kit_lf():
|
||||||
|
attributes = (ROOT / ".gitattributes").read_text(encoding="utf-8")
|
||||||
|
assert re.search(r"^\*\.cmd\s+text eol=crlf$", attributes, re.M)
|
||||||
|
assert re.search(r"^deploy/\*\*\s+text eol=lf$", attributes, re.M)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_deployment_docs_are_the_boilerplates_copies():
|
||||||
|
for name in ("Install Server.md", "Install WSL.md"):
|
||||||
|
first = (ROOT / "docs" / name).read_text(encoding="utf-8").splitlines()[0]
|
||||||
|
assert first.startswith(f"> A copy of the boilerplate's `documentation/{name}` at commit "), name
|
||||||
|
for gone in ("hosting", "release", "server", "password-protection", "llm-backends"):
|
||||||
|
assert not (ROOT / "docs" / f"{gone}.md").exists(), gone
|
||||||
|
assert not (ROOT / "docs" / "box-docker-compose.yml").exists()
|
||||||
Reference in new issue
Block a user