diff --git a/.gitattributes b/.gitattributes index 3fa7e18..c13b76b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -12,3 +12,5 @@ # PowerShell reads LF happily; what it cannot read is a non-ASCII byte, which # is a different rule and lives in `tests/test_packaging.py`. deploy/** text eol=lf +# cmd.exe misparses LF-only batch files, so the double-click launcher stays CRLF. +*.cmd text eol=crlf diff --git a/README.md b/README.md index 9106adf..42498ea 100644 --- a/README.md +++ b/README.md @@ -76,4 +76,4 @@ The reader, `/roles/{uid}`, shows the full posting: the same vote and Save contr - [`plugins/README.md`](plugins/README.md): every reader, its endpoints and what each states; [`docs/filter-survey.md`](docs/filter-survey.md): the filter parameters each host takes at the source. - `docs/plan-*.md`: plans in progress, folded into `decisions.md` and deleted when their work lands. - [`desk/README.md`](desk/README.md), [`pipeline/README.md`](pipeline/README.md), [`auth/README.md`](auth/README.md), [`deploy/README.md`](deploy/README.md): the copied modules, where each comes from and how to refresh it. -- [`docs/hosting.md`](docs/hosting.md), [`docs/release.md`](docs/release.md), [`docs/server.md`](docs/server.md), [`docs/password-protection.md`](docs/password-protection.md), [`docs/llm-backends.md`](docs/llm-backends.md): the boilerplate's deployment docs. +- [`docs/Install Server.md`]() and [`docs/Install WSL.md`](): the server and the Windows machine the release runs on, copies of the boilerplate's files; the release steps are in [`deploy/README.md`](deploy/README.md). diff --git a/bin/backend.sh b/bin/backend.sh new file mode 100644 index 0000000..d3d5fc5 --- /dev/null +++ b/bin/backend.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# The backend in the foreground until Ctrl-C: uv run app serve, on :8790. The profile's -backend runs it. +# Ours: `-v` and `--config` go before `serve`, where the app takes them, and every other argument +# after it (`--reload`, `--port`), so `.run/serve.run.xml`'s `serve --reload` is `-backend --reload`. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." +# A shell started by wsl.exe has not read the rc file that puts uv on PATH. +export PATH="$HOME/.local/bin:$PATH" +before=() after=() +while (($#)); do + case $1 in + -v|--verbose) before+=("$1") ;; + -c|--config) before+=("$1" "${2:?$1 needs a file}"); shift ;; + --config=*) before+=("$1") ;; + *) after+=("$1") ;; + esac + shift +done +exec uv run app "${before[@]}" serve "${after[@]}" diff --git a/bin/check.sh b/bin/check.sh new file mode 100644 index 0000000..b88bf52 --- /dev/null +++ b/bin/check.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# The local gate on the working tree, uncommitted edits included, without promoting: +# the steps deploy/test-commit.sh runs on a commit; the profile's -check runs it. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." +exec sh deploy/test-commit.sh --here diff --git a/bin/pull.sh b/bin/pull.sh new file mode 100644 index 0000000..22ece43 --- /dev/null +++ b/bin/pull.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# roledesk's own, beside the boilerplate's backend and check: one pull in the foreground, `app job pull`. +# The profile's -pull runs it; `.run/pull.run.xml` is the same command from PyCharm. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." +# A shell started by wsl.exe has not read the rc file that puts uv on PATH. +export PATH="$HOME/.local/bin:$PATH" +exec uv run app job pull "$@" diff --git a/bin/worker.sh b/bin/worker.sh new file mode 100644 index 0000000..1b08ac4 --- /dev/null +++ b/bin/worker.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# roledesk's own, beside the boilerplate's backend and check: the queue drained with no web app, `app worker`. +# The profile's -worker runs it; `.run/worker.run.xml` is the same command from PyCharm. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." +# A shell started by wsl.exe has not read the rc file that puts uv on PATH. +export PATH="$HOME/.local/bin:$PATH" +exec uv run app worker "$@" diff --git a/deploy/0-release.ps1 b/deploy/0-release.ps1 deleted file mode 100644 index a84e96b..0000000 --- a/deploy/0-release.ps1 +++ /dev/null @@ -1,14 +0,0 @@ -<# -.SYNOPSIS - Run steps 1, 2 and 3: test/promote, build image, push image. -.DESCRIPTION - Stops on the first failure. Server deployment is a separate operation -- - nothing here connects to the server. Git branch publication is optional - through -PushBranch. -#> -[CmdletBinding()] -param([switch] $PushBranch) -$ErrorActionPreference = 'Stop' -& "$PSScriptRoot/1-test-and-promote.ps1" -PushBranch:$PushBranch -& "$PSScriptRoot/2-build-image.ps1" -& "$PSScriptRoot/3-push-image.ps1" diff --git a/deploy/1-test-and-promote.ps1 b/deploy/1-test-and-promote.ps1 deleted file mode 100644 index b31f868..0000000 --- a/deploy/1-test-and-promote.ps1 +++ /dev/null @@ -1,90 +0,0 @@ -<# -.SYNOPSIS - Test the latest local main commit and fast-forward production to it. -.DESCRIPTION - Uses a fixed commit and an isolated worktree. PushBranch optionally publishes - production to origin; without it promotion stays local. Does not run Docker. -#> -[CmdletBinding()] -param([switch] $PushBranch, [switch] $NoTests) -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' -. "$PSScriptRoot/_common.ps1" - -$Source = Setting SOURCE 'main' -$Branch = Setting BRANCH 'production' -$Python = Setting PYTHON (Join-Path $root '.venv\Scripts\python.exe') - -Push-Location $root -try { - $commit = git rev-parse --verify "$Source^{commit}" - Assert-Ok 'resolve source commit' - $commit = $commit.Trim() - - # ------------------------------------------------------- test the commit - - # In a throwaway worktree, not here. The tests then run against exactly the - # tree that is about to become production, with none of the working copy's - # uncommitted edits in the way. Worktrees have no .venv of their own, so the - # interpreter comes from the main checkout and the code from the worktree. - # - # A checkout inside WSL has a Linux .venv, so there the same steps run in - # the distro through deploy/test-commit.sh. - if (-not $NoTests -and $Wsl.Success) { - Write-Step "tests on $Source, in WSL $($Wsl.Groups[1].Value)" - wsl.exe -d $Wsl.Groups[1].Value --cd (ConvertTo-LinuxPath $root) -- sh deploy/test-commit.sh $commit - Assert-Ok 'tests' - } elseif (-not $NoTests) { - Write-Step "tests on $Source" - if (-not (Test-Path $Python)) { - throw "no interpreter at $Python - set PYTHON in deploy\release.env" - } - $base = Join-Path ([IO.Path]::GetTempPath()) ("$Name-test-" + [guid]::NewGuid().ToString('N').Substring(0, 8)) - $tree = Join-Path $base $Name - try { - git worktree add --detach $tree $commit | Out-Null - Assert-Ok 'git worktree add' - Push-Location $tree - try { - # Your project's checks. A gate that is red for weeks is not a - # gate -- it is a flag you learn to type -- so take one out when - # it stops meaning anything, and put it back when it is green. - # - # The starter's dev group is pytest and httpx, so pytest is the - # only gate that can run out of the box. Add ruff and mypy to - # [dependency-groups] dev in pyproject.toml, then uncomment the - # two lines below -- a gate whose tool is not installed fails - # every release for a reason that has nothing to do with the code. - & $Python -m pytest -q; Assert-Ok 'pytest' - # & $Python -m ruff check .; Assert-Ok 'ruff' - # & $Python -m mypy --cache-dir (Join-Path $root '.mypy_cache') - # Assert-Ok 'mypy' - } finally { Pop-Location } - } finally { - git worktree remove --force $tree 2>$null | Out-Null - Remove-ReleaseTemp @($base) - } - } - - # ------------------------------------------------------------- promote - - # Fast-forward only. If production has commits main does not, that is a - # release made from somewhere else, and this stops rather than discarding it. - Write-Step "promote $Source -> $Branch" - git show-ref --verify --quiet "refs/heads/$Branch" - if ($LASTEXITCODE -eq 0) { - git merge-base --is-ancestor $Branch $commit - if ($LASTEXITCODE -ne 0) { - Write-Host "$Branch is not an ancestor of $Source - it has commits of its own." -ForegroundColor Red - throw "look at: git log --oneline $Source..$Branch" - } - } - # Git publication is optional and independent of the image. - git branch -f $Branch $commit; Assert-Ok 'git branch -f' - - if ($PushBranch) { - git push origin "${commit}:refs/heads/$Branch" - Assert-Ok 'publish production branch' - } - Write-Host "Promoted $Branch to $commit" -} finally { Pop-Location } diff --git a/deploy/1-test-and-promote.sh b/deploy/1-test-and-promote.sh new file mode 100644 index 0000000..2ed2c38 --- /dev/null +++ b/deploy/1-test-and-promote.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Test the latest local main commit, fast-forward production to it, and export +# it for the image build on Windows. +# +# bash deploy/1-test-and-promote.sh [--push-branch] [--no-tests] +# +# Uses a fixed commit and an isolated worktree. --push-branch optionally +# publishes production to origin; without it promotion stays local. --no-tests +# skips the gates. The export is the production tree in /// +# and its sha in //production, RELEASE_DIR defaulting to +# release/ beside the checkout. Uncommitted work is never included. +set -euo pipefail +# shellcheck source=_common.sh +. "$(dirname "${BASH_SOURCE[0]}")/_common.sh" + +push_branch=0 no_tests=0 +for arg in "$@"; do + case "$arg" in + --push-branch) push_branch=1 ;; + --no-tests) no_tests=1 ;; + -h|--help) sed -n '2,11p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) die "unknown option $arg" ;; + esac +done + +need_git +SOURCE=$(setting SOURCE main) +BRANCH=$(setting BRANCH production) +PYTHON=$(setting PYTHON "$root/.venv/bin/python") + +cd "$root" +commit=$(resolve_commit "$SOURCE" source) + +# ----------------------------------------------------------- test the commit + +# In a throwaway worktree, not here. The tests then run against exactly the tree +# that is about to become production, with none of the working copy's +# uncommitted edits in the way. test-commit.sh makes the worktree, runs pytest, +# ruff, mypy and the client's check in it, and removes it. Worktrees have no +# .venv of their own, so the interpreter comes from this checkout. +if ((! no_tests)); then + step "tests on $SOURCE" + run_or_die tests env PYTHON="$PYTHON" sh "$here/test-commit.sh" "$commit" +fi + +# ------------------------------------------------------------------- promote + +# Fast-forward only. If production has commits main does not, that is a release +# made from somewhere else, and this stops rather than discarding it. +step "promote $SOURCE -> $BRANCH" +if git show-ref --verify --quiet "refs/heads/$BRANCH"; then + if ! git merge-base --is-ancestor "$BRANCH" "$commit"; then + echo "$BRANCH is not an ancestor of $SOURCE - it has commits of its own." >&2 + die "look at: git log --oneline $SOURCE..$BRANCH" + fi +fi +# Git publication is optional and independent of the image. +run_or_die 'git branch -f' git branch -f "$BRANCH" "$commit" + +if ((push_branch)); then + run_or_die 'publish production branch' git push origin "$commit:refs/heads/$BRANCH" +fi +echo "Promoted $BRANCH to $commit" + +# -------------------------------------------------------------------- export + +# The image is built on Windows by wslc (2-build-image.ps1), which reads this +# folder through \\wsl.localhost. A tree rather than a tar, so Windows has +# nothing to unpack. wslc hands the Dockerfile to the builder on stdin, so the +# builder only finds the ignore rules at the root of the context: +# Dockerfile.dockerignore is copied there as .dockerignore. A tree that exists +# for this sha is kept, not rewritten, because wslc can build from a stale view +# of a folder deleted and recreated at the same path (microsoft/WSL#41287). +# The promote has happened by now; when the export fails, rerun this step. +name=$(basename "$root") +release=${RELEASE_DIR:-$(dirname "$root")/release}/$name +sha=$(git rev-parse --short "$commit") || die 'resolve image tag failed' +tree=$release/$sha +step "export $BRANCH ($sha) to $tree" +mkdir -p -- "$release" 2>/dev/null && [[ -w $release ]] || + die "cannot write $release; $BRANCH is promoted, rerun this step once it is writable" +if [[ ! -d $tree ]]; then + part=$release/.part-$sha + rm -rf -- "$part" + mkdir -- "$part" + git archive --format=tar "$commit" | tar -x -C "$part" || die 'export failed' + if [[ -f $part/deploy/Dockerfile.dockerignore ]]; then + cp -- "$part/deploy/Dockerfile.dockerignore" "$part/.dockerignore" + fi + mv -- "$part" "$tree" +fi +for old in "$release"/*/; do + [[ ${old%/} == "$tree" ]] || rm -rf -- "${old%/}" +done +printf '%s\n' "$sha" > "$release/production" +echo "Exported $tree" diff --git a/deploy/2-build-image.ps1 b/deploy/2-build-image.ps1 index 43143a6..5a6186c 100644 --- a/deploy/2-build-image.ps1 +++ b/deploy/2-build-image.ps1 @@ -1,57 +1,13 @@ -<# -.SYNOPSIS - Build a local Docker image from the committed production branch. -.DESCRIPTION - Does not test, promote, push Git branches, use SSH, or start a container. - The image stays local. Use 3-push-image.ps1 to publish it separately. -#> -[CmdletBinding()] -param() -Set-StrictMode -Version Latest +# Step 2, on Windows: build the image from the tree step 1 exported, with wslc. +# Tags : and :latest. Never pushes; 3-push-image.ps1 does. $ErrorActionPreference = 'Stop' -. "$PSScriptRoot/_common.ps1" - -$Image = Setting IMAGE -Required -$Branch = Setting BRANCH 'production' -$Platform = Setting PLATFORM 'linux/amd64' - -Push-Location $root -try { - $commit = git rev-parse --verify "$Branch^{commit}" - Assert-Ok 'resolve production commit' - $commit = $commit.Trim() - $sha = (git rev-parse --short $commit).Trim() - Assert-Ok 'resolve image tag' - $tagged = "${Image}:${sha}" - $latest = "${Image}:latest" - - # --------------------------------------------------------------- build - - # `git archive` to a file and unpack it, rather than piping it. A PowerShell - # pipeline carries text: it would re-encode a tar stream and hand docker a - # corrupted context. Writing the archive out and building from the directory - # keeps the same guarantee -- the context is the committed tree and nothing - # else -- without a byte going through a pipe. - # - # The Dockerfile comes out of that same tree, at deploy/Dockerfile, so the - # recipe is versioned with the code it builds. Docker reads the ignore rules - # from deploy/Dockerfile.dockerignore beside it. - - Write-Step "build $tagged" - $ctx = Join-Path ([IO.Path]::GetTempPath()) "$Name-build-$sha" - $tar = "$ctx.tar" - Remove-ReleaseTemp @($ctx, $tar) - New-Item -ItemType Directory -Path $ctx | Out-Null - try { - git archive --format=tar -o $tar $commit; Assert-Ok 'git archive' - tar -xf $tar -C $ctx; Assert-Ok 'tar -xf' - $dockerfile = Join-Path $ctx 'deploy/Dockerfile' - docker build --platform $Platform -f $dockerfile -t $tagged -t $latest $ctx - Assert-Ok 'docker build' - docker image ls $Image --format ' {{.Tag}} {{.Size}}' - } finally { - Remove-ReleaseTemp @($ctx, $tar) - } - - Write-Host "Image: $tagged" -} finally { Pop-Location } +. "$PSScriptRoot\_common.ps1" +Assert-Wslc +$sha = Get-ReleaseSha +$context = Join-Path $Release $sha +Write-Host "== build ${Image}:$sha from $context" -ForegroundColor Cyan +# -f is a path wslc opens itself, not one inside the context. +wslc build -t "${Image}:$sha" -t "${Image}:latest" -f "$context\deploy\Dockerfile" $context +if ($LASTEXITCODE -ne 0) { throw "wslc build failed (exit $LASTEXITCODE)" } +wslc image list +Write-Host "Image: ${Image}:$sha" diff --git a/deploy/3-push-image.ps1 b/deploy/3-push-image.ps1 index 40294a4..a139909 100644 --- a/deploy/3-push-image.ps1 +++ b/deploy/3-push-image.ps1 @@ -1,36 +1,20 @@ -<# -.SYNOPSIS - Publish the already-built production image to the registry. -.DESCRIPTION - Never rebuilds. If the push fails, fix access or connectivity and run this - again; the image is already on disk. `docker login` first, with a token - that has package-write access. -#> -[CmdletBinding()] -param() -Set-StrictMode -Version Latest +# Step 3, on Windows: push the image step 2 built, : and :latest, +# with wslc. Never rebuilds: after a failed push, fix the login and run it again. $ErrorActionPreference = 'Stop' -. "$PSScriptRoot/_common.ps1" - -$Image = Setting IMAGE -Required -$Branch = Setting BRANCH 'production' - -Push-Location $root -try { - $commit = git rev-parse --verify "$Branch^{commit}" - Assert-Ok 'resolve production commit' - $sha = (git rev-parse --short $commit.Trim()).Trim() - Assert-Ok 'resolve image tag' - $tagged = "${Image}:${sha}" - $latest = "${Image}:latest" - docker image inspect $tagged --format '{{.Id}}' | Out-Null - Assert-Ok "find local $tagged (run step 2 first)" - # latest must name this candidate, even if another image was built since. - docker tag $tagged $latest - Assert-Ok 'tag latest image' - docker push $tagged - Assert-Ok 'push commit image' - docker push $latest - Assert-Ok 'push latest image' - Write-Host "Published $tagged and $latest" -} finally { Pop-Location } +. "$PSScriptRoot\_common.ps1" +Assert-Wslc +$sha = Get-ReleaseSha +wslc image inspect "${Image}:$sha" | Out-Null +if ($LASTEXITCODE -ne 0) { throw "no local ${Image}:$sha - run step 2 first" } +# latest must name this candidate, even if another image was built since. +wslc tag "${Image}:$sha" "${Image}:latest" +if ($LASTEXITCODE -ne 0) { throw "wslc tag failed (exit $LASTEXITCODE)" } +foreach ($tag in $sha, 'latest') { + Write-Host "== push ${Image}:$tag" -ForegroundColor Cyan + wslc push "${Image}:$tag" + if ($LASTEXITCODE -ne 0) { + Write-Host "If the push was refused: wslc login $(($Image -split '/')[0]) -u , with a token that has package-write access" -ForegroundColor Yellow + throw "wslc push ${Image}:$tag failed (exit $LASTEXITCODE)" + } +} +Write-Host "Published ${Image}:$sha and ${Image}:latest" diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 7fd1c9b..f33f829 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,21 +1,28 @@ # A Python app, for a server rather than a laptop. # # Two stages so the runtime carries no build tooling: `uv` resolves and -# installs into /app/.venv, and the second stage copies that venv and the -# source next to it. The install is deliberately editable, which matters if -# the app reads a file from beside its package (an example config, a template -# directory): that path is true in a checkout and true here, and false in -# site-packages. +# installs into /app/.venv, and the second stage copies the venv and the source +# next to it. The install is deliberately editable, which matters because the +# app reads files from beside its package (templates, `app/static`, `seed/`): +# that path is true in a checkout and true here, and false in site-packages. +# Ours: roledesk has no `frontend/` and no Node stage, since the server renders +# Jinja templates and serves `app/static` as it is. # # Already in place at deploy/Dockerfile, with its ignore file beside it. -# 2-build-image.ps1 reads it out of a `git archive` of the production commit, -# so the recipe is versioned with the code it builds. +# 2-build-image.ps1 builds with it from the `git archive` of the production +# commit that step 1 exports, so the recipe is versioned with the code it builds. # -# The package is `app` and stays `app` — a project started from this repo -# renames nothing. Edit the two `uv sync` lines when you add extras. +# The package is `app` and stays `app` (the domain noun is `role`). Edit the +# two `uv sync` lines when you add extras. +# +# Each base image is pinned by tag and digest: the tag says which release, the +# digest makes a rebuild next month install the same bytes. The digest is the +# multi-platform index's. To move on, look up the new tag's digest on its +# registry (Docker Hub's tag API for python, ghcr.io for uv) and +# replace both halves. -ARG PYTHON=3.14-slim -ARG UV=0.9.17 +ARG PYTHON=3.15-slim@sha256:2da676046630d7bf14c6764ae1143dc7fa0377a53f94bf5a642f0b214aca5bce +ARG UV=0.13.1@sha256:4a1f3666816dafadd20ceb71eac9a7d56d7ef30279f7d64048519ab901ada336 FROM ghcr.io/astral-sh/uv:${UV} AS uv @@ -32,22 +39,38 @@ WORKDIR /app # Dependencies first and on their own layer, keyed on the lock file alone, so # editing a Python file rebuilds in seconds rather than re-resolving the tree. +# The extras are the same on both `uv sync` lines: `passkeys`, without which +# the passkey routes answer 501, `embeddings`, the default ranking backend +# (torch comes from the CPU index named in pyproject.toml, [tool.uv.sources], +# not the CUDA build; the model itself is downloaded by the first `embed` job +# into HOME, which is the /data volume), and `llm`, the SDKs the compose file +# passes keys for. COPY pyproject.toml uv.lock README.md ./ RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-dev --no-install-project + uv sync --frozen --no-dev --no-install-project \ + --extra passkeys --extra embeddings --extra llm COPY app app -# The two module folders this repo ships beside the app, here and again in the -# runtime stage below. `pyproject.toml` lists `pipeline*` and `auth*` in its -# packages as well, or the install leaves them out. +# The module folders this repo ships beside the app, here and again in the +# runtime stage below. `pyproject.toml` lists `pipeline*`, `auth*`, `llm*` and +# `desk*` in its packages as well, or the install leaves them out. COPY pipeline pipeline # The door. A project that does not want one drops this line *and* the # `auth.connect` call in `app/main.py` together: dropping the folder and # leaving the call is a container that fails at startup, and dropping the call # and leaving the folder is a container that starts with the door open. COPY auth auth +# The model call. Nothing imports it until an app adds its SDKs, and then the +# folder has to be here already, so that is not a Dockerfile edit. +COPY llm llm +# Ours: the source contract and the generic desk modules, the plugin folder the +# config's `plugin_paths` names, and the data a first run imports (`seed/`). +COPY desk desk +COPY plugins plugins +COPY seed seed RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-dev + uv sync --frozen --no-dev \ + --extra passkeys --extra embeddings --extra llm # ---------------------------------------------------------------- runtime @@ -69,6 +92,10 @@ COPY pipeline pipeline # The login page and the two scripts are files inside this folder, served from # routes; see the build stage above. COPY auth auth +COPY llm llm +COPY desk desk +COPY plugins plugins +COPY seed seed ENV PATH=/app/.venv/bin:$PATH \ PYTHONUNBUFFERED=1 \ @@ -79,6 +106,13 @@ ENV PATH=/app/.venv/bin:$PATH \ # /data, and a CLI the app shells out to — the `claude` binary, for one — # wants somewhere to write. An image-baked /home/app would be right for uid # 1000 and unwritable for anyone else. +# +# The image runs as 1000:1000, the compose file's default PUID:PGID and the +# owner docs/Install Server.md gives the data folder, so a `docker run` without a +# `user:` does not write root-owned files into it. /data is made here, before +# VOLUME, so an anonymous volume starts out owned by that uid too. +RUN install -d -o 1000 -g 1000 /data /data/home +USER 1000:1000 VOLUME /data EXPOSE 8787 diff --git a/deploy/Dockerfile.dockerignore b/deploy/Dockerfile.dockerignore index ca28962..5303ca3 100644 --- a/deploy/Dockerfile.dockerignore +++ b/deploy/Dockerfile.dockerignore @@ -1,12 +1,13 @@ -# Named for the Dockerfile beside it: Docker reads `.dockerignore` -# before falling back to one at the root of the context, which is how the ignore -# rules get to live next to the recipe they belong to instead of at the top of -# the repository. +# Named for the Dockerfile beside it. Docker reads `.dockerignore` +# before falling back to one at the root of the context, but wslc hands the +# Dockerfile to the builder on stdin, which leaves only the root one: step 1 +# (1-test-and-promote.sh) copies this file to `.dockerignore` at the root of +# the tree it exports, so the rules still live next to the recipe here. # -# 2-build-image.ps1 already hands docker a context extracted from `git archive`, -# so the untracked heavyweights below are not there to exclude. They are listed -# for the other way in — `docker build -f deploy/Dockerfile .` from a checkout — -# where .venv alone can be several hundred megabytes. +# That tree is a `git archive` of the production commit, so the untracked +# heavyweights below are not there to exclude. They are listed for the other +# way in, `docker build -f deploy/Dockerfile .` from a checkout, where .venv +# alone can be several hundred megabytes. .git .venv .claude @@ -19,6 +20,7 @@ docs # per folder, so a fourth module is a fourth line here. tests pipeline/tests +llm/tests auth/tests __pycache__ diff --git a/deploy/README.md b/deploy/README.md index 3165626..fe3cb24 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,52 +1,76 @@ # Deploy -Everything needed to put this app on the rittershofer.org box behind SWAG: build an image on Windows, publish it to Gitea, run it as one more service in the compose file that already runs the proxy, back up its database, and give it a password that survives being on the open internet. +Everything needed to put roledesk on a server behind SWAG ([../docs/Install Server.md](../docs/Install%20Server.md) is the box it was written for, a copy of the boilerplate's file): test it in WSL, build an image on Windows with `wslc`, publish it to Gitea, run it as one more service in the compose file that already runs the proxy, back up its database, and give it a password that survives being on the open internet. -Extracted from newsdesk, which is the worked example — every decision here has been run in production at least once, and the comments say why rather than what. +This folder is a downstream copy of `../boilerplate/deploy` (commit 09e144d), refreshed with `git archive`; a fix to a shared script lands upstream first. roledesk's own facts are put back after a refresh: the Dockerfile without a frontend stage and with `desk`, `plugins`, `llm` and `seed`, the extras `passkeys`, `embeddings` and `llm`, our image name in `release.env.example`, `server/config.example.yaml` for roledesk's keys, `TABLES` and `REQUIRED_TABLES` naming `roles`, and `bin/pull.sh` and `bin/worker.sh`. `tests/test_deploy.py` pins them. -**It is already in place.** Nothing has to be copied and nothing has to be renamed: the package is `app`, the environment variables are `APP_`, and the files below are where the scripts expect them. `` in the docs is your *service's* name — the hostname, the data directory, the image tag, the compose service — and it is the one thing you choose. +The package is `app`, and the files below are where the scripts expect them: the `APP_` variables, the compose service `app` and the hostname `app.example.com` in the templates are placeholders for the server, and `` in the docs is the service's name: the hostname, the data directory, the image tag, the compose service. The domain noun is `role`, so "job" only ever means queued work. ## Where each file is | Here | | | |---|---|---| -| `deploy/*.ps1` | the release scripts, run on Windows | | -| `deploy/Dockerfile` | read out of the archived commit by step 2 | copies `app`, `pipeline` and `auth` | -| `deploy/Dockerfile.dockerignore` | Docker finds it by that name | | +| `deploy/1-test-and-promote.sh`, `test-commit.sh`, `_common.sh`, `db.sh` | the gate, the export and the database moves, run in the WSL distro | | +| `deploy/2-build-image.ps1`, `3-push-image.ps1`, `_common.ps1` | the image build and push, run on Windows with `wslc` | the distro is a sealed sandbox and does not hold a container socket | +| `deploy/wsl-setup.cmd`, `wsl-setup.ps1` | the profile commands, written once per checkout on Windows | | +| `bin/backend.sh`, `check.sh`, and ours `pull.sh`, `worker.sh` | the dev commands, run in the WSL distro: the backend, the gate on the working tree, a pull, the worker | | +| `.run/serve`, `pull`, `worker` | PyCharm run configurations, left as they are | `serve --reload` = `bin/backend.sh --reload` = `-backend --reload`; `pull` = `bin/pull.sh` = `-pull`; `worker` = `bin/worker.sh` = `-worker` | +| `deploy/Dockerfile` | read out of the tree step 1 exports, by step 2 | no frontend stage; copies `app`, `pipeline`, `auth`, `llm`, `desk`, `plugins` and `seed` | +| `deploy/Dockerfile.dockerignore` | copied to `.dockerignore` at the root of the exported tree | `wslc` streams the Dockerfile, so only a root ignore file applies | | `deploy/server/*` | committed here, then installed on the server | | | `deploy/server/compose.yaml` | to `/srv/swag/docker-compose.yml`, as one `services:` block | | | `deploy/server/env.example` | to `/srv/swag/.env`, as appended variables, `chmod 600` | | +| `deploy/server/config.example.yaml` | to `$APP_DATA/config.yaml`, with the hostname filled in | `deploy.sh` refuses a config without `database` and `auth.enabled` | | `deploy/server/deploy.sh`, `deploy/server/backup.sh` | to `/srv/swag/`, next to the compose file | | +| `deploy/server/import.sh`, `deploy/server/import_database.py` | uploaded and run by `db.sh push` | | | `deploy/server/swag/app.subdomain.conf` | to SWAG's `proxy-confs/`, renamed to `.subdomain.conf` | | -`db.ps1` expects `server/` beside it, which is why `deploy/server/` is committed here as well as installed on the server. +`db.sh` expects `server/` beside it, which is why `deploy/server/` is committed here as well as installed on the server. -**Keep the `.ps1` files to ASCII.** Windows PowerShell 5.1 reads a script without a byte-order mark as Windows-1252, so an em dash becomes three characters ending in a closing quote: the string ends in the middle of itself and the script fails to parse before it runs a line. Write `--`. newsdesk hit this on a comment nobody thought was code, and `tests/test_packaging.py` here holds every script in this folder to it. +**Keep every `.ps1` to ASCII.** Windows PowerShell 5.1 reads a script without a byte-order mark as Windows-1252, so an em dash becomes three characters ending in a closing quote: the string ends in the middle of itself and the script fails to parse before it runs a line. Write `--`. newsdesk hit this on a comment nobody thought was code, and `tests/test_packaging.py` here holds every `.ps1` in this folder to it, the release scripts included. + +`wsl-setup.sh` and `wsl-setup.ps1` are the two halves of setting up a checkout that lives in WSL. `wsl-setup.sh` runs in the distro ([../docs/Install WSL.md, Git](../docs/Install%20WSL.md#git)). `wsl-setup.ps1`, or a double-click on `deploy\wsl-setup.cmd`, writes the commands below into the PowerShell profile, with `` the checkout folder's name (`jobboard` here) or `-Name`. PowerShell 7, the execution policy, `wslc`, the hand-paste fallback and what to check when a command fails are in [../docs/Install WSL.md, PS Profile commands](../docs/Install%20WSL.md#ps-profile-commands). + +| Command | Runs | Does | +|---|---|---| +| `-cd` | `Set-Location` | PowerShell into the checkout's folder | +| `-sh` | `wsl.exe` | a shell in the distro, in the checkout | +| `-backend [-v] [--reload]` | bin/backend.sh in WSL | `uv run app serve` on :8790 | +| `-pull [args]` | bin/pull.sh in WSL | `uv run app job pull`, one pull in the foreground (roledesk's) | +| `-worker [args]` | bin/worker.sh in WSL | `uv run app worker`, the queue drained with no web app (roledesk's) | +| `-check` | bin/check.sh in WSL | the gate on the working tree, without promoting | +| `-1test [--push-branch] [--no-tests]` | deploy/1-test-and-promote.sh in WSL | test local main, fast-forward `production`, export it | +| `-2build` | deploy\2-build-image.ps1 on Windows | build the exported tree with `wslc` | +| `-3push` | deploy\3-push-image.ps1 on Windows | push the built image with `wslc` | +| `-0release [--push-branch] [--no-tests]` | the three above | steps 1, 2 and 3 in order, stopping at the first failure | +| `-db pull\|push` | deploy/db.sh in WSL | the database, to and from the server | + +Each command is one line in the profile: `wsl.exe -d --cd -- bash