Refresh deploy/ and bin/ from the boilerplate at 09e144d (the release split between the distro and Windows, wsl-setup, the server config, USER 1000, python:3.15-slim) with roledesk's facts put back: no frontend stage, desk, plugins and seed copied, the extras passkeys, embeddings and llm, roles as the table the backup and the importer require, bin/pull.sh and bin/worker.sh with their profile commands, and Install Server.md and Install WSL.md as copies in place of the five old deployment docs
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
83e965dfeb
commit
a1f281a23a
40 files changed
+2226
-1287
No files matched your search
@@ -0,0 +1,800 @@
|
||||
"""Release and deployment scripts without a live server, a registry or a network.
|
||||
|
||||
Ported from the boilerplate's tests/test_deploy.py. Ours: the database is
|
||||
`roledesk.db` with `roles` as its first table, and the tests at the end hold
|
||||
the facts put back after a refresh (the Dockerfile, the server config, `bin/`
|
||||
and the profile commands).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SERVER = ROOT / "deploy" / "server"
|
||||
|
||||
#: The compose service, read from the one block a project pastes into its
|
||||
#: compose file. Every script and the proxy conf repeat it; a rename that
|
||||
#: misses one fails `test_every_file_names_the_same_service`, not this whole
|
||||
#: file.
|
||||
SERVICE = next(iter(yaml.safe_load((SERVER / "compose.yaml").read_text(encoding="utf-8"))
|
||||
["services"]))
|
||||
|
||||
|
||||
def run_shell(tmp_path: Path, script: str, *args: str, fail: str = ""):
|
||||
git_bash = Path("C:/Program Files/Git/bin/bash.exe")
|
||||
bash = str(git_bash) if git_bash.exists() else shutil.which("bash")
|
||||
if bash is None:
|
||||
pytest.skip("bash is required for deployment script tests")
|
||||
log = tmp_path / "docker.log"
|
||||
wrapper = '''
|
||||
docker() {
|
||||
printf '%s\n' "$*" >> "$DEPLOY_TEST_LOG"
|
||||
if [[ -n $DEPLOY_TEST_FAIL && "$*" == *"$DEPLOY_TEST_FAIL"* ]]; then return 1; fi
|
||||
}
|
||||
export -f docker
|
||||
bash "$@"
|
||||
'''
|
||||
result = subprocess.run(
|
||||
[bash, "-c", wrapper, "test", (ROOT / "deploy" / "server" / script).as_posix(),
|
||||
tmp_path.as_posix(), *args], capture_output=True, text=True,
|
||||
env={**os.environ, "DEPLOY_TEST_LOG": log.as_posix(), "DEPLOY_TEST_FAIL": fail})
|
||||
return result, log.read_text() if log.exists() else ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fail", ["", f"pull {SERVICE}", "up -d"])
|
||||
def test_deploy_waits_for_health_and_reports_failure(tmp_path, fail):
|
||||
(tmp_path / "data").mkdir()
|
||||
shutil.copy(SERVER / "config.example.yaml", tmp_path / "data/config.yaml")
|
||||
original = "APP_TAG=latest\nSECRET=do-not-print\n"
|
||||
(tmp_path / ".env").write_text(original)
|
||||
result, log = run_shell(tmp_path, "deploy.sh", fail=fail)
|
||||
assert (result.returncode != 0) == bool(fail)
|
||||
assert (tmp_path / ".env").read_text() == original
|
||||
assert "do-not-print" not in result.stdout + result.stderr
|
||||
assert f"compose pull {SERVICE}" in log
|
||||
if fail.startswith("pull"):
|
||||
assert "up -d" not in log
|
||||
assert "not restarted" in result.stderr
|
||||
else:
|
||||
assert f"up -d --no-deps --pull never --wait --wait-timeout 240 {SERVICE}" in log
|
||||
assert log.index("compose pull") < log.index("compose up")
|
||||
if fail:
|
||||
assert "Deployment failed" in result.stderr
|
||||
|
||||
|
||||
def test_deploy_missing_config_does_not_call_docker(tmp_path):
|
||||
(tmp_path / ".env").write_text("APP_TAG=latest\n")
|
||||
result, log = run_shell(tmp_path, "deploy.sh")
|
||||
assert result.returncode != 0
|
||||
assert not log
|
||||
assert (tmp_path / ".env").read_text() == "APP_TAG=latest\n"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", [
|
||||
"auth:\n enabled: always\n",
|
||||
"database: /data/app.db\n",
|
||||
"database: /data/app.db\nauth:\n rp_id: app.example.com\nenabled: always\n",
|
||||
"database: /data/app.db\n# auth:\n# enabled: always\n",
|
||||
"database:\nauth:\n enabled: always\n",
|
||||
"# database: /data/app.db\nauth:\n enabled: always\n",
|
||||
])
|
||||
def test_deploy_refuses_a_config_without_database_and_auth_enabled(tmp_path, text):
|
||||
"""The two keys whose defaults are wrong in a container. Read on the host
|
||||
without a YAML parser, so a key commented out or outside its section is
|
||||
the same as a missing one."""
|
||||
(tmp_path / "data").mkdir()
|
||||
(tmp_path / "data/config.yaml").write_text(text)
|
||||
(tmp_path / ".env").write_text("APP_TAG=latest\n")
|
||||
result, log = run_shell(tmp_path, "deploy.sh")
|
||||
assert result.returncode != 0
|
||||
assert "must set database and auth.enabled" in result.stderr
|
||||
assert not log
|
||||
|
||||
|
||||
def test_deploy_reads_auth_enabled_past_comments_and_other_keys(tmp_path):
|
||||
(tmp_path / "data").mkdir()
|
||||
(tmp_path / "data/config.yaml").write_text(
|
||||
"auth:\n # locked\n\n rp_id: x\n enabled: always # behind SWAG\n"
|
||||
"database: /data/app.db\n")
|
||||
(tmp_path / ".env").write_text("APP_TAG=latest\n")
|
||||
result, log = run_shell(tmp_path, "deploy.sh")
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert f"compose pull {SERVICE}" in log
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The release scripts that run in bash: _common.sh (the release.env reader) and
|
||||
# 1-test-and-promote.sh. Each test copies them into a scratch git repository,
|
||||
# which is the checkout they find their root from. test-commit.sh, the gates,
|
||||
# is replaced by a stub that passes or fails on request; nothing runs the
|
||||
# network or the real test suite. Steps 2 and 3 are PowerShell running wslc on
|
||||
# Windows, and are not run here: there is no PowerShell and no wslc in the
|
||||
# sandbox.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
IMAGE = "registry.example/team/app"
|
||||
DEPLOY = ROOT / "deploy"
|
||||
RELEASE_SCRIPTS = ["_common.sh", "1-test-and-promote.sh"]
|
||||
|
||||
TOOLS = ["bash", "sh", "env", "git", "tar", "dirname", "basename", "realpath", "rm", "mkdir",
|
||||
"mktemp", "sed", "cat", "uname", "date", "head", "tr", "sleep", "readlink", "chmod",
|
||||
"ls", "wc", "grep", "cut", "sort", "touch", "ln", "cp", "mv", "id", "hostname"]
|
||||
|
||||
|
||||
def git(repo: Path, *args: str) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", repo.as_posix(), "-c", "user.name=t", "-c", "user.email=t@example.org",
|
||||
*args], capture_output=True, text=True, check=True)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def release(tmp_path):
|
||||
"""A scratch repository on main holding the release scripts and a stub gate."""
|
||||
found = shutil.which("bash")
|
||||
if found is None or shutil.which("git") is None:
|
||||
pytest.skip("bash and git are required for release script tests")
|
||||
bash: str = found
|
||||
repo = tmp_path / "repo"
|
||||
(repo / "deploy").mkdir(parents=True)
|
||||
for name in RELEASE_SCRIPTS:
|
||||
shutil.copy(DEPLOY / name, repo / "deploy" / name)
|
||||
(repo / "deploy/test-commit.sh").write_text(
|
||||
'#!/bin/sh\necho "$1" >> "$DEPLOY_TEST_GATE"\n[ -z "$DEPLOY_TEST_RED" ]\n')
|
||||
(repo / "deploy/release.env").write_text(f"IMAGE={IMAGE}\n")
|
||||
(repo / "deploy/Dockerfile.dockerignore").write_text("tests\n")
|
||||
(repo / "README").write_text("one\n")
|
||||
subprocess.run(["git", "init", "-q", "-b", "main", repo.as_posix()], check=True)
|
||||
git(repo, "add", "-A", "-f")
|
||||
git(repo, "commit", "-qm", "one")
|
||||
# A PATH of only what the scripts need.
|
||||
bare = tmp_path / "bare"
|
||||
bare.mkdir()
|
||||
for tool in TOOLS:
|
||||
found = shutil.which(tool)
|
||||
if found:
|
||||
(bare / tool).symlink_to(found)
|
||||
(tmp_path / "tmp").mkdir()
|
||||
|
||||
class Release:
|
||||
def __init__(self):
|
||||
self.repo, self.tmp = repo, tmp_path
|
||||
self.gate = tmp_path / "gate.log"
|
||||
self.commit = git(repo, "rev-parse", "HEAD")
|
||||
|
||||
def run(self, script, *args, **extra):
|
||||
env = {**os.environ, "PATH": str(bare), "DEPLOY_TEST_GATE": self.gate.as_posix(),
|
||||
"TMPDIR": (tmp_path / "tmp").as_posix(), "DEPLOY_TEST_RED": ""}
|
||||
env.pop("RELEASE_DIR", None)
|
||||
env.update({k: str(v) for k, v in extra.items()})
|
||||
return subprocess.run([bash, (repo / "deploy" / script).as_posix(), *args],
|
||||
cwd=tmp_path, capture_output=True, text=True, env=env)
|
||||
|
||||
def gate_log(self):
|
||||
return self.gate.read_text().split() if self.gate.exists() else []
|
||||
|
||||
def branches(self):
|
||||
return git(repo, "branch", "--format=%(refname:short)").split()
|
||||
|
||||
return Release()
|
||||
|
||||
|
||||
def read_settings(release, text: bytes) -> subprocess.CompletedProcess:
|
||||
"""Source _common.sh over a release.env with this text and print what it read."""
|
||||
(release.repo / "deploy/release.env").write_bytes(text)
|
||||
probe = release.repo / "deploy/probe.sh"
|
||||
probe.write_text(
|
||||
'set -euo pipefail\n. "$(dirname "${BASH_SOURCE[0]}")/_common.sh"\n'
|
||||
'echo "IMAGE=[$(setting IMAGE)] BRANCH=[$(setting BRANCH production)] '
|
||||
'NOTE=[$(setting NOTE)] SOURCE=[$(setting SOURCE main)]"\n'
|
||||
'required=$(require_setting IMAGE)\necho "required=[$required]"\n')
|
||||
return release.run("probe.sh")
|
||||
|
||||
|
||||
def test_common_reads_release_env_without_sourcing_it(release):
|
||||
marker = release.tmp / "ran"
|
||||
text = (f'# a comment\n\n IMAGE = "{IMAGE}" \nNOTE=has a space # not a comment\n'
|
||||
f'BRANCH=\nX=$(touch {marker})\n').encode()
|
||||
result = read_settings(release, text)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert f"IMAGE=[{IMAGE}] BRANCH=[production] NOTE=[has a space # not a comment]" in result.stdout
|
||||
assert "SOURCE=[main]" in result.stdout
|
||||
assert not marker.exists()
|
||||
|
||||
|
||||
def test_common_reads_a_crlf_release_env(release):
|
||||
result = read_settings(release, f"IMAGE={IMAGE}\r\nNOTE=x\r\n".encode())
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert f"required=[{IMAGE}]" in result.stdout
|
||||
assert "NOTE=[x]" in result.stdout
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, message", [
|
||||
(b"IMAGE=" + IMAGE.encode() + b"\nnot a setting\n", "not a KEY=VALUE line: not a setting"),
|
||||
(b"NOTE=x\n", "set IMAGE in deploy/release.env"),
|
||||
(b'IMAGE=""\n', "set IMAGE in deploy/release.env"),
|
||||
])
|
||||
def test_common_refuses_a_bad_release_env(release, text, message):
|
||||
result = read_settings(release, text)
|
||||
assert result.returncode != 0
|
||||
assert message in result.stderr
|
||||
|
||||
|
||||
def test_common_refuses_a_missing_release_env(release):
|
||||
(release.repo / "deploy/release.env").unlink()
|
||||
result = release.run("1-test-and-promote.sh")
|
||||
assert result.returncode != 0
|
||||
assert "no deploy/release.env" in result.stderr
|
||||
assert not release.gate_log()
|
||||
|
||||
|
||||
def test_test_and_promote_tests_the_commit_then_promotes_it(release):
|
||||
result = release.run("1-test-and-promote.sh")
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert release.gate_log() == [release.commit]
|
||||
assert git(release.repo, "rev-parse", "production") == release.commit
|
||||
assert f"Promoted production to {release.commit}" in result.stdout
|
||||
|
||||
|
||||
def test_test_and_promote_leaves_uncommitted_work_out(release):
|
||||
(release.repo / "README").write_text("edited\n")
|
||||
(release.repo / "untracked.txt").write_text("new\n")
|
||||
result = release.run("1-test-and-promote.sh")
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert release.gate_log() == [release.commit]
|
||||
assert git(release.repo, "rev-parse", "production") == release.commit
|
||||
assert git(release.repo, "show", "production:README") == "one"
|
||||
assert (release.repo / "README").read_text() == "edited\n"
|
||||
|
||||
|
||||
def test_test_and_promote_red_tests_promote_nothing(release):
|
||||
result = release.run("1-test-and-promote.sh", DEPLOY_TEST_RED="1")
|
||||
assert result.returncode != 0
|
||||
assert "tests failed (exit 1)" in result.stderr
|
||||
assert "production" not in release.branches()
|
||||
|
||||
|
||||
def test_test_and_promote_no_tests_skips_the_gate(release):
|
||||
result = release.run("1-test-and-promote.sh", "--no-tests", DEPLOY_TEST_RED="1")
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert not release.gate_log()
|
||||
assert "production" in release.branches()
|
||||
|
||||
|
||||
def test_test_and_promote_refuses_a_production_with_commits_of_its_own(release):
|
||||
git(release.repo, "switch", "-qc", "production")
|
||||
(release.repo / "hotfix").write_text("x\n")
|
||||
git(release.repo, "add", "hotfix")
|
||||
git(release.repo, "commit", "-qm", "hotfix")
|
||||
own = git(release.repo, "rev-parse", "production")
|
||||
git(release.repo, "switch", "-q", "main")
|
||||
result = release.run("1-test-and-promote.sh")
|
||||
assert result.returncode != 0
|
||||
assert "production is not an ancestor of main" in result.stderr
|
||||
assert "git log --oneline main..production" in result.stderr
|
||||
assert git(release.repo, "rev-parse", "production") == own
|
||||
|
||||
|
||||
def test_test_and_promote_fast_forwards_an_older_production(release):
|
||||
git(release.repo, "branch", "production")
|
||||
(release.repo / "README").write_text("two\n")
|
||||
git(release.repo, "commit", "-qam", "two")
|
||||
newer = git(release.repo, "rev-parse", "HEAD")
|
||||
result = release.run("1-test-and-promote.sh")
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert git(release.repo, "rev-parse", "production") == newer
|
||||
|
||||
|
||||
def test_test_and_promote_publishes_the_branch_only_on_request(release):
|
||||
origin = release.tmp / "origin.git"
|
||||
subprocess.run(["git", "init", "-q", "--bare", origin.as_posix()], check=True)
|
||||
git(release.repo, "remote", "add", "origin", origin.as_posix())
|
||||
assert release.run("1-test-and-promote.sh").returncode == 0
|
||||
assert subprocess.run(["git", "-C", origin.as_posix(), "branch"], capture_output=True,
|
||||
text=True).stdout.strip() == ""
|
||||
result = release.run("1-test-and-promote.sh", "--push-branch")
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert git(origin, "rev-parse", "production") == release.commit
|
||||
|
||||
|
||||
def test_test_and_promote_honours_source_and_branch_settings(release):
|
||||
git(release.repo, "branch", "next")
|
||||
(release.repo / "deploy/release.env").write_text(f"IMAGE={IMAGE}\nSOURCE=next\nBRANCH=prod\n")
|
||||
result = release.run("1-test-and-promote.sh")
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert git(release.repo, "rev-parse", "prod") == release.commit
|
||||
|
||||
|
||||
def test_test_and_promote_refuses_an_unknown_option(release):
|
||||
result = release.run("1-test-and-promote.sh", "--force")
|
||||
assert result.returncode != 0
|
||||
assert "unknown option --force" in result.stderr
|
||||
assert not release.gate_log()
|
||||
|
||||
|
||||
def test_the_example_release_env_is_read_by_the_bash_reader_and_names_the_checkout(release):
|
||||
"""One file serves both sides; the profile and steps 2 and 3 need DISTRO and
|
||||
CHECKOUT. wslc has no --platform, so there is no PLATFORM."""
|
||||
example = (DEPLOY / "release.env.example").read_bytes()
|
||||
result = read_settings(release, example)
|
||||
assert result.returncode == 0, result.stderr
|
||||
keys = [line.split("=")[0].strip() for line in example.decode().splitlines()
|
||||
if "=" in line and not line.lstrip().startswith("#")]
|
||||
assert {"IMAGE", "DISTRO", "CHECKOUT"} <= set(keys)
|
||||
assert "PLATFORM" not in keys
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 1's export: the production tree and its sha, in release/<folder>/ beside
|
||||
# the checkout (or RELEASE_DIR/<folder>/), where 2-build-image.ps1 reads them
|
||||
# through \\wsl.localhost.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def short(release) -> str:
|
||||
return git(release.repo, "rev-parse", "--short", release.commit)
|
||||
|
||||
|
||||
def test_test_and_promote_exports_the_tree_beside_the_checkout(release):
|
||||
(release.repo / "README").write_text("edited\n")
|
||||
result = release.run("1-test-and-promote.sh")
|
||||
assert result.returncode == 0, result.stderr
|
||||
out = release.tmp / "release" / "repo"
|
||||
sha = short(release)
|
||||
assert (out / "production").read_text() == f"{sha}\n"
|
||||
assert (out / sha / "README").read_text() == "one\n"
|
||||
assert (out / sha / "deploy/test-commit.sh").is_file()
|
||||
# wslc streams the Dockerfile, so the ignore rules go to the context's root.
|
||||
assert (out / sha / ".dockerignore").read_text() == "tests\n"
|
||||
assert sorted(p.name for p in out.iterdir()) == sorted([sha, "production"])
|
||||
|
||||
|
||||
def test_test_and_promote_honours_release_dir(release):
|
||||
share = release.tmp / "share"
|
||||
result = release.run("1-test-and-promote.sh", RELEASE_DIR=share)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert (share / "repo/production").read_text() == f"{short(release)}\n"
|
||||
assert not (release.tmp / "release").exists()
|
||||
|
||||
|
||||
def test_test_and_promote_keeps_this_shas_tree_and_drops_older_ones(release):
|
||||
share = release.tmp / "share"
|
||||
assert release.run("1-test-and-promote.sh", RELEASE_DIR=share).returncode == 0
|
||||
first = short(release)
|
||||
marker = share / "repo" / first / "marker"
|
||||
marker.write_text("kept\n")
|
||||
assert release.run("1-test-and-promote.sh", RELEASE_DIR=share).returncode == 0
|
||||
assert marker.read_text() == "kept\n"
|
||||
(release.repo / "README").write_text("two\n")
|
||||
git(release.repo, "commit", "-qam", "two")
|
||||
result = release.run("1-test-and-promote.sh", RELEASE_DIR=share)
|
||||
assert result.returncode == 0, result.stderr
|
||||
second = git(release.repo, "rev-parse", "--short", "HEAD")
|
||||
assert sorted(p.name for p in (share / "repo").iterdir()) == sorted([second, "production"])
|
||||
assert (share / "repo" / second / "README").read_text() == "two\n"
|
||||
|
||||
|
||||
def test_test_and_promote_says_so_when_the_export_cannot_be_written(release):
|
||||
blocked = release.tmp / "blocked"
|
||||
blocked.write_text("a file, not a folder\n")
|
||||
result = release.run("1-test-and-promote.sh", RELEASE_DIR=blocked)
|
||||
assert result.returncode != 0
|
||||
assert f"cannot write {blocked.as_posix()}/repo" in result.stderr
|
||||
assert "production is promoted" in result.stderr
|
||||
assert git(release.repo, "rev-parse", "production") == release.commit
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# deploy/db.sh. `ssh` and `scp` are stubs that log their arguments (and, for
|
||||
# scp, copy nothing); the snapshot is taken for real, from a real SQLite file.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SSH_STUB = '''#!/bin/sh
|
||||
printf 'ssh %s\\n' "$*" >> "$DEPLOY_TEST_LOG"
|
||||
case "$*" in
|
||||
*"ls -1t"*) echo /srv/data/app/backups/app-20261015-040000.db.gz ;;
|
||||
*"bash -s"*) cat > "$DEPLOY_TEST_STDIN" ;;
|
||||
esac
|
||||
'''
|
||||
SCP_STUB = '''#!/bin/sh
|
||||
printf 'scp %s\\n' "$*" >> "$DEPLOY_TEST_LOG"
|
||||
for last; do :; done
|
||||
case "$last" in */) : > "${last}app-20261015-040000.db.gz" ;; esac
|
||||
exit 0
|
||||
'''
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def database(tmp_path):
|
||||
"""A scratch repository folder holding db.sh, a database, and stub ssh and scp."""
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
found = shutil.which("bash")
|
||||
if found is None:
|
||||
pytest.skip("bash is required for deployment script tests")
|
||||
bash: str = found
|
||||
repo = tmp_path / "repo"
|
||||
(repo / "deploy/server").mkdir(parents=True)
|
||||
for name in ("_common.sh", "db.sh", "server/import.sh", "server/import_database.py"):
|
||||
shutil.copy(DEPLOY / name, repo / "deploy" / name)
|
||||
(repo / "deploy/release.env").write_text(
|
||||
f"SSH_HOST=box\nREMOTE_DIR=/srv/swag\nREMOTE_DATA=/srv/data/app\nNAME={SERVICE}\n"
|
||||
f"PYTHON={sys.executable}\n")
|
||||
# Where the app writes without a config.yaml (`database: roledesk.db`),
|
||||
# which is where db.sh looks when release.env names no LOCAL_DB.
|
||||
with sqlite3.connect(repo / "roledesk.db") as db:
|
||||
db.execute("CREATE TABLE t (n)")
|
||||
db.execute("INSERT INTO t VALUES (1)")
|
||||
stubs = tmp_path / "stubs"
|
||||
stubs.mkdir()
|
||||
for name, text in (("ssh", SSH_STUB), ("scp", SCP_STUB)):
|
||||
(stubs / name).write_text(text)
|
||||
(stubs / name).chmod(0o755)
|
||||
log, stdin = tmp_path / "net.log", tmp_path / "stdin.txt"
|
||||
|
||||
def run(*args: str, input: str | None = None):
|
||||
env = {**os.environ, "PATH": f"{stubs}:{os.environ['PATH']}",
|
||||
"DEPLOY_TEST_LOG": log.as_posix(), "DEPLOY_TEST_STDIN": stdin.as_posix()}
|
||||
env.pop("APP_CONFIG", None)
|
||||
result = subprocess.run([bash, (repo / "deploy/db.sh").as_posix(), *args], cwd=tmp_path,
|
||||
capture_output=True, text=True, env=env, input=input)
|
||||
return result, (log.read_text() if log.exists() else ""), stdin
|
||||
|
||||
run.repo = repo # type: ignore[attr-defined]
|
||||
return run
|
||||
|
||||
|
||||
def test_db_push_asks_first_and_sends_nothing_on_no(database):
|
||||
result, log, _ = database("push", input="n\n")
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "stopped." in result.stdout
|
||||
assert log == ""
|
||||
assert not list(database.repo.glob("app-*.db.gz"))
|
||||
|
||||
|
||||
def test_db_push_uploads_a_snapshot_and_streams_import_sh(database):
|
||||
result, log, stdin = database("push", "--yes")
|
||||
assert result.returncode == 0, result.stderr
|
||||
lines = log.splitlines()
|
||||
assert lines[0].startswith("scp -C ") and lines[0].endswith("box:/srv/data/app/incoming.db.gz")
|
||||
assert lines[1].endswith("box:/srv/swag/import_database.py")
|
||||
assert lines[2] == "ssh box bash -s -- '/srv/swag'"
|
||||
assert stdin.read_text() == (DEPLOY / "server/import.sh").read_text()
|
||||
assert f"docker compose up -d --wait {SERVICE}" in result.stdout
|
||||
assert f"snapshotting {(database.repo / 'roledesk.db').as_posix()}" in result.stdout
|
||||
assert not list(database.repo.glob("app-*.db.gz"))
|
||||
|
||||
|
||||
def test_db_push_takes_the_database_the_config_names(database):
|
||||
(database.repo / "config.yaml").write_text("database: elsewhere/mine.db\n")
|
||||
result, _, _ = database("push", input="n\n")
|
||||
assert result.returncode != 0
|
||||
assert f"no {(database.repo / 'elsewhere/mine.db').as_posix()}" in result.stderr
|
||||
(database.repo / "elsewhere").mkdir()
|
||||
shutil.copy(database.repo / "roledesk.db", database.repo / "elsewhere/mine.db")
|
||||
result, _, _ = database("push", input="n\n")
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert f"snapshotting {(database.repo / 'elsewhere/mine.db').as_posix()}" in result.stdout
|
||||
|
||||
|
||||
def test_db_push_takes_local_db_over_the_config(database):
|
||||
with (database.repo / "deploy/release.env").open("a") as env:
|
||||
env.write(f"LOCAL_DB={(database.repo / 'roledesk.db').as_posix()}\n")
|
||||
(database.repo / "config.yaml").write_text("database: nowhere.db\n")
|
||||
result, _, _ = database("push", input="n\n")
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert f"snapshotting {(database.repo / 'roledesk.db').as_posix()}" in result.stdout
|
||||
|
||||
|
||||
def test_db_pull_runs_the_servers_backup_and_downloads_the_newest(database):
|
||||
result, log, _ = database("pull")
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert log.splitlines()[0] == "ssh box cd '/srv/swag' && bash backup.sh"
|
||||
assert log.splitlines()[-1].endswith("box:/srv/data/app/backups/app-20261015-040000.db.gz " +
|
||||
(database.repo / "backups").as_posix() + "/")
|
||||
assert "backups/app-20261015-040000.db.gz" in result.stdout
|
||||
|
||||
|
||||
def test_db_needs_an_action_and_a_server(database):
|
||||
result, _, _ = database()
|
||||
assert result.returncode != 0 and "push or pull" in result.stderr
|
||||
(database.repo / "deploy/release.env").write_text("REMOTE_DIR=/srv/swag\n")
|
||||
result, log, _ = database("pull")
|
||||
assert result.returncode != 0 and "set SSH_HOST in deploy/release.env" in result.stderr
|
||||
assert log == ""
|
||||
# The compose service has no default here: a renamed project would
|
||||
# otherwise pull and print the start command for `app`.
|
||||
(database.repo / "deploy/release.env").write_text("SSH_HOST=box\nREMOTE_DIR=/srv/swag\n")
|
||||
result, log, _ = database("pull")
|
||||
assert result.returncode != 0 and "set NAME in deploy/release.env" in result.stderr
|
||||
assert log == ""
|
||||
|
||||
|
||||
def test_the_backup_script_writes_owner_only():
|
||||
"""A snapshot is the whole database. The host umask is set before anything
|
||||
is made; the container's before the `.db` is written, because gzip copies
|
||||
that file's mode to the `.gz`."""
|
||||
text = (DEPLOY / "server" / "backup.sh").read_text(encoding="utf-8")
|
||||
assert "\numask 077\n" in text
|
||||
assert text.index("\numask 077\n") < text.index("mkdir -p")
|
||||
assert text.index("os.umask(0o077)") < text.index('db.execute("VACUUM INTO')
|
||||
|
||||
|
||||
def backup_snippet() -> str:
|
||||
"""The Python backup.sh runs inside the container, from its heredoc."""
|
||||
text = (SERVER / "backup.sh").read_text(encoding="utf-8")
|
||||
return text.split("<<'PY'\n", 1)[1].split("\nPY\n", 1)[0]
|
||||
|
||||
|
||||
def import_database():
|
||||
spec = importlib.util.spec_from_file_location("import_database", SERVER / "import_database.py")
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_the_backup_asks_the_app_for_the_database():
|
||||
"""No path to the database in the script: the app's config names it, so a
|
||||
server config with another `database` is backed up and not skipped."""
|
||||
text = (SERVER / "backup.sh").read_text(encoding="utf-8")
|
||||
assert "/data/app.db" not in text and "/data/roledesk.db" not in text
|
||||
assert "from app.config import load" in backup_snippet()
|
||||
tables = re.search(r'^TABLES="([^"]*)"$', text, re.M)
|
||||
assert tables and set(tables[1].split()) == import_database().REQUIRED_TABLES
|
||||
|
||||
|
||||
def run_backup(tmp_path: Path, database: Path):
|
||||
(tmp_path / "config.yaml").write_text(f"database: {database.as_posix()}\n")
|
||||
backups = tmp_path / "backups"
|
||||
backups.mkdir(exist_ok=True)
|
||||
code = backup_snippet().replace('"/data/backups"', repr(backups.as_posix()))
|
||||
env = {**os.environ, "APP_CONFIG": (tmp_path / "config.yaml").as_posix()}
|
||||
tables = sorted(import_database().REQUIRED_TABLES)
|
||||
result = subprocess.run([sys.executable, "-", "app-1", *tables], input=code, cwd=ROOT,
|
||||
capture_output=True, text=True, env=env)
|
||||
return result, backups / "app-1.db"
|
||||
|
||||
|
||||
def test_the_backup_of_a_database_that_is_not_there_fails(tmp_path):
|
||||
"""Opened read-only, so a wrong path is an error and not a new empty file
|
||||
whose snapshot looks like a backup."""
|
||||
missing = tmp_path / "data/app.db"
|
||||
result, snapshot = run_backup(tmp_path, missing)
|
||||
assert result.returncode != 0
|
||||
assert "unable to open database file" in result.stderr
|
||||
assert not missing.exists() and not snapshot.exists()
|
||||
|
||||
|
||||
def test_the_backup_of_a_database_without_the_tables_fails(tmp_path):
|
||||
with sqlite3.connect(tmp_path / "other.db") as db:
|
||||
db.execute("CREATE TABLE items (n)")
|
||||
result, snapshot = run_backup(tmp_path, tmp_path / "other.db")
|
||||
assert result.returncode != 0
|
||||
assert "not this app's database" in result.stderr
|
||||
assert not snapshot.exists()
|
||||
|
||||
|
||||
def test_the_backup_snapshots_the_apps_database_owner_only(tmp_path):
|
||||
from app.config import Config
|
||||
from app.main import create_app
|
||||
|
||||
create_app(Config(database=tmp_path / "app.db"))
|
||||
result, snapshot = run_backup(tmp_path, tmp_path / "app.db")
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert snapshot.exists()
|
||||
if os.name == "posix":
|
||||
assert snapshot.stat().st_mode & 0o777 == 0o600
|
||||
|
||||
|
||||
def test_the_importer_knows_this_apps_tables(tmp_path):
|
||||
"""A fresh database of this app passes the importer's table check, so a
|
||||
project that replaces its schema is told here to name its own table."""
|
||||
from app.config import Config
|
||||
from app.main import create_app
|
||||
|
||||
create_app(Config(database=tmp_path / "app.db"))
|
||||
with sqlite3.connect(tmp_path / "app.db") as db:
|
||||
tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||||
assert import_database().REQUIRED_TABLES <= tables
|
||||
|
||||
|
||||
def test_the_importer_replaces_this_apps_database_and_refuses_another(tmp_path):
|
||||
import gzip
|
||||
from contextlib import closing
|
||||
|
||||
from app.config import Config
|
||||
from app.main import create_app
|
||||
|
||||
def archive(source: Path, target: Path) -> None:
|
||||
"""What db.sh uploads: a VACUUM INTO of the WAL database, gzipped."""
|
||||
plain = source.with_suffix(".snapshot")
|
||||
with closing(sqlite3.connect(source)) as db:
|
||||
db.execute("VACUUM INTO ?", (str(plain),))
|
||||
with plain.open("rb") as raw, gzip.open(target, "wb") as out:
|
||||
shutil.copyfileobj(raw, out)
|
||||
|
||||
module = import_database()
|
||||
create_app(Config(database=tmp_path / "incoming.db"))
|
||||
with closing(sqlite3.connect(tmp_path / "incoming.db")) as db, db:
|
||||
db.execute("INSERT INTO state (key, value) VALUES ('at', 'then')")
|
||||
archive(tmp_path / "incoming.db", tmp_path / "in.db.gz")
|
||||
with closing(sqlite3.connect(tmp_path / "wrong.db")) as db, db:
|
||||
db.execute("CREATE TABLE items (n)")
|
||||
archive(tmp_path / "wrong.db", tmp_path / "wrong.db.gz")
|
||||
(tmp_path / "live").mkdir()
|
||||
live = tmp_path / "live/app.db"
|
||||
create_app(Config(database=live))
|
||||
|
||||
with pytest.raises(ValueError, match="not this application's database"):
|
||||
module.replace_database(live, tmp_path / "wrong.db.gz", tmp_path / "backups")
|
||||
saved = module.replace_database(live, tmp_path / "in.db.gz", tmp_path / "backups")
|
||||
assert saved is not None and saved.exists()
|
||||
with closing(sqlite3.connect(live)) as db:
|
||||
assert db.execute("SELECT value FROM state WHERE key = 'at'").fetchall() == [("then",)]
|
||||
|
||||
|
||||
def test_the_importer_snapshots_owner_only():
|
||||
"""The snapshot before the swap is the whole database, written by the
|
||||
container's uid under whatever umask the image has."""
|
||||
text = (SERVER / "import_database.py").read_text(encoding="utf-8")
|
||||
main = text.split('if __name__ == "__main__":', 1)[1]
|
||||
assert main.index("os.umask(0o077)") < main.index("replace_database(")
|
||||
assert "/data/app.db" not in text and "/data/roledesk.db" not in text
|
||||
|
||||
|
||||
def test_every_file_names_the_same_service():
|
||||
"""The compose service is repeated where a script or nginx has to name it.
|
||||
A rename that misses one deploys the image and then stops, backs up or
|
||||
proxies to a container that does not exist."""
|
||||
compose = yaml.safe_load((SERVER / "compose.yaml").read_text(encoding="utf-8"))
|
||||
assert list(compose["services"]) == [SERVICE]
|
||||
assert compose["services"][SERVICE]["container_name"] == SERVICE
|
||||
for script in ("deploy.sh", "backup.sh", "import.sh"):
|
||||
text = (SERVER / script).read_text(encoding="utf-8")
|
||||
assert re.findall(r"^SERVICE=(\S+)$", text, re.M) == [SERVICE], script
|
||||
conf = (SERVER / "swag" / "app.subdomain.conf").read_text(encoding="utf-8")
|
||||
assert set(re.findall(r"set \$upstream_app (\S+);", conf)) == {SERVICE}
|
||||
example = (DEPLOY / "release.env.example").read_text(encoding="utf-8")
|
||||
assert re.findall(r"^NAME=(\S+)$", example, re.M) == [SERVICE]
|
||||
|
||||
|
||||
def test_the_examples_carry_no_real_host():
|
||||
"""The server templates start from placeholders; the real hostnames are in
|
||||
docs/Install Server.md only. Ours: release.env.example is not held to it,
|
||||
it names our registry's image, which the task keeps."""
|
||||
for path in [*SERVER.rglob("*"), ROOT / "config.example.yaml"]:
|
||||
if path.is_file() and "__pycache__" not in path.parts:
|
||||
assert "rittershofer" not in path.read_text(encoding="utf-8"), path
|
||||
|
||||
|
||||
def test_the_proxy_throttles_every_open_login_route():
|
||||
"""`/login/passkey/options` is open and writes a challenge row per call."""
|
||||
conf = (SERVER / "swag" / "app.subdomain.conf").read_text(encoding="utf-8")
|
||||
throttled = [m for m in re.finditer(r"location ~ (\S+) \{([^}]*)\}", conf)
|
||||
if "limit_req zone=" in m[2]]
|
||||
assert len(throttled) == 1
|
||||
pattern = re.compile(throttled[0][1])
|
||||
for path in ("/login", "/login/passkey", "/login/passkey/options"):
|
||||
assert pattern.search(path), path
|
||||
for path in ("/login/other", "/api/login", "/login/passkey/register"):
|
||||
assert not pattern.search(path), path
|
||||
|
||||
|
||||
def test_the_proxy_conf_does_not_spool_unlimited_bodies():
|
||||
"""nginx writes a body to disk before the app can refuse it, and the login
|
||||
takes one without a session. 0 belongs only on a location that is a
|
||||
deliberate upload route."""
|
||||
conf = (DEPLOY / "server" / "swag" / "app.subdomain.conf").read_text(encoding="utf-8")
|
||||
stack: list[str] = []
|
||||
limits: dict[str, str] = {}
|
||||
for line in conf.splitlines():
|
||||
code = line.split("#", 1)[0].strip()
|
||||
if code.endswith("{"):
|
||||
stack.append(code[:-1].strip())
|
||||
elif code == "}":
|
||||
stack.pop()
|
||||
elif code.startswith("client_max_body_size"):
|
||||
limits[stack[-1]] = code.rstrip(";").split()[1]
|
||||
assert not stack, "unbalanced braces"
|
||||
assert limits.get("server") not in (None, "0")
|
||||
assert "location /" not in limits
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# roledesk's own facts, put back after a refresh of deploy/ from the boilerplate.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_server_config_is_one_the_app_accepts():
|
||||
"""An unknown key is an error, so a stale example would start a container
|
||||
that dies at load. The values are the ones a container needs."""
|
||||
from app.config import load
|
||||
|
||||
config = load(SERVER / "config.example.yaml")
|
||||
assert config.server.host == "0.0.0.0" and config.server.port == 8787
|
||||
assert config.database.as_posix() == "/data/roledesk.db"
|
||||
assert config.auth.enabled == "always" and config.auth.secure_cookie == "always"
|
||||
|
||||
|
||||
def test_the_dockerfile_is_roledesks():
|
||||
text = (DEPLOY / "Dockerfile").read_text(encoding="utf-8")
|
||||
lines = [line.strip() for line in text.splitlines() if not line.startswith("#")]
|
||||
assert "frontend" not in "\n".join(lines) and "node" not in "\n".join(lines).lower()
|
||||
for folder in ("desk", "plugins", "seed", "llm"):
|
||||
assert lines.count(f"COPY {folder} {folder}") == 2, folder
|
||||
syncs = [i for i, line in enumerate(lines) if line.startswith("uv sync")]
|
||||
assert len(syncs) == 2
|
||||
for i in syncs:
|
||||
block = " ".join(lines[i:i + 2])
|
||||
for extra in ("passkeys", "embeddings", "llm"):
|
||||
assert f"--extra {extra}" in block, (i, extra)
|
||||
assert "USER 1000:1000" in lines
|
||||
assert any("127.0.0.1:8787/health" in line for line in lines)
|
||||
|
||||
|
||||
def test_the_release_env_example_names_our_checkout_and_image():
|
||||
example = (DEPLOY / "release.env.example").read_text(encoding="utf-8")
|
||||
assert re.findall(r"^CHECKOUT=(\S+)$", example, re.M) == ["/home/claude/cboxcode/jobboard"]
|
||||
assert re.findall(r"^IMAGE=(\S+)$", example, re.M) == ["git.rittershofer.org/wolf/app"]
|
||||
|
||||
|
||||
def test_the_dev_commands_are_the_ones_the_run_configurations_run():
|
||||
"""`.run/` stays; each of its commands has a bin script and a profile command."""
|
||||
assert sorted(p.name for p in (ROOT / "bin").iterdir()) == [
|
||||
"backend.sh", "check.sh", "pull.sh", "worker.sh"]
|
||||
for script, command in (("pull.sh", "uv run app job pull"), ("worker.sh", "uv run app worker")):
|
||||
text = (ROOT / "bin" / script).read_text(encoding="utf-8")
|
||||
assert f'exec {command} "$@"' in text, script
|
||||
profile = (DEPLOY / "wsl-setup.ps1").read_text(encoding="utf-8")
|
||||
for name in ("backend", "pull", "worker", "check"):
|
||||
assert f"function $Name-{name} {{ $wsl -- bash bin/{name}.sh @args }}" in profile, name
|
||||
assert "frontend" not in profile
|
||||
|
||||
|
||||
def test_backend_sends_each_argument_to_the_command_that_takes_it(tmp_path):
|
||||
"""`-v` and `--config` belong before `serve`, `--reload` and `--port` after."""
|
||||
bash = shutil.which("bash")
|
||||
if bash is None:
|
||||
pytest.skip("bash is required for deployment script tests")
|
||||
(tmp_path / ".local/bin").mkdir(parents=True)
|
||||
uv = tmp_path / ".local/bin/uv"
|
||||
uv.write_text("#!/bin/sh\necho \"$*\"\n")
|
||||
uv.chmod(0o755)
|
||||
env = {**os.environ, "HOME": tmp_path.as_posix()}
|
||||
|
||||
def run(*args: str) -> str:
|
||||
return subprocess.run([bash, (ROOT / "bin/backend.sh").as_posix(), *args],
|
||||
capture_output=True, text=True, env=env, check=True).stdout.strip()
|
||||
|
||||
assert run() == "run app serve"
|
||||
assert run("--reload") == "run app serve --reload"
|
||||
assert run("-v", "--reload", "--port", "9") == "run app -v serve --reload --port 9"
|
||||
assert run("-c", "x.yaml") == "run app -c x.yaml serve"
|
||||
|
||||
|
||||
def test_the_batch_launcher_stays_crlf_and_the_kit_lf():
|
||||
attributes = (ROOT / ".gitattributes").read_text(encoding="utf-8")
|
||||
assert re.search(r"^\*\.cmd\s+text eol=crlf$", attributes, re.M)
|
||||
assert re.search(r"^deploy/\*\*\s+text eol=lf$", attributes, re.M)
|
||||
|
||||
|
||||
def test_the_deployment_docs_are_the_boilerplates_copies():
|
||||
for name in ("Install Server.md", "Install WSL.md"):
|
||||
first = (ROOT / "docs" / name).read_text(encoding="utf-8").splitlines()[0]
|
||||
assert first.startswith(f"> A copy of the boilerplate's `documentation/{name}` at commit "), name
|
||||
for gone in ("hosting", "release", "server", "password-protection", "llm-backends"):
|
||||
assert not (ROOT / "docs" / f"{gone}.md").exists(), gone
|
||||
assert not (ROOT / "docs" / "box-docker-compose.yml").exists()
|
||||
Reference in new issue
Block a user