From 9d666d093787bedb58e7e0f352fd0d90672aac10 Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 23 Sep 2026 12:29:16 +0200 Subject: [PATCH] Bring in pipeline, auth, deploy and docs from the boilerplate roledesk is built on the boilerplate from here on: pipeline 2.0 (queue and scheduler), auth (one password, sessions, passkeys), the deploy kit and its docs, plus the coupling and packaging tests. Downstream copies, unedited. Refreshed from boilerplate/d56949f7d0f6c0bfe5745478e09007bfad67ae87 Co-Authored-By: Claude Opus 5.5 --- auth/README.md | 113 +++ auth/__init__.py | 160 +++ auth/config.py | 104 ++ auth/middleware.py | 246 +++++ auth/models.py | 81 ++ auth/passkeys.py | 398 ++++++++ auth/router.py | 489 +++++++++ auth/schemas.py | 56 ++ auth/static/account.js | 505 ++++++++++ auth/static/passkey.js | 100 ++ auth/store.py | 505 ++++++++++ auth/templates/login.html | 174 ++++ auth/tests/__init__.py | 20 + auth/tests/conftest.py | 79 ++ auth/tests/harness.py | 68 ++ auth/tests/softkey.py | 169 ++++ auth/tests/test_door.py | 489 +++++++++ auth/tests/test_passkeys.py | 373 +++++++ auth/tests/test_store.py | 227 +++++ auth/tests/test_without_passkeys.py | 100 ++ deploy/0-release.ps1 | 14 + deploy/1-test-and-promote.ps1 | 83 ++ deploy/2-build-image.ps1 | 57 ++ deploy/3-push-image.ps1 | 36 + deploy/Dockerfile | 94 ++ deploy/Dockerfile.dockerignore | 42 + deploy/README.md | 52 + deploy/_common.ps1 | 66 ++ deploy/db.ps1 | 142 +++ deploy/release.env.example | 28 + deploy/server/backup.sh | 44 + deploy/server/compose.yaml | 60 ++ deploy/server/deploy.sh | 38 + deploy/server/env.example | 42 + deploy/server/import.sh | 35 + deploy/server/import_database.py | 77 ++ deploy/server/swag/app.subdomain.conf | 64 ++ docs/box-docker-compose.yml | 175 ++++ docs/hosting.md | 112 +++ docs/llm-backends.md | 241 +++++ docs/password-protection.md | 106 ++ docs/release.md | 51 + docs/server.md | 75 ++ pipeline/README.md | 172 ++++ pipeline/__init__.py | 127 +++ pipeline/child.py | 140 +++ pipeline/context.py | 446 +++++++++ pipeline/models.py | 432 ++++++++ pipeline/notices.py | 136 +++ pipeline/pipeline.md | 395 ++++++++ pipeline/queue.py | 1135 +++++++++++++++++++++ pipeline/registry.py | 195 ++++ pipeline/schedule.py | 273 +++++ pipeline/table.py | 1319 +++++++++++++++++++++++++ pipeline/task.py | 495 ++++++++++ pipeline/tests/__init__.py | 2 + pipeline/tests/isolated_handlers.py | 74 ++ pipeline/tests/scheduled_handlers.py | 39 + pipeline/tests/test_connect.py | 113 +++ pipeline/tests/test_housekeeping.py | 209 ++++ pipeline/tests/test_isolation.py | 320 ++++++ pipeline/tests/test_notice_log.py | 134 +++ pipeline/tests/test_retries.py | 355 +++++++ pipeline/tests/test_schedule.py | 701 +++++++++++++ pipeline/web/__init__.py | 59 ++ pipeline/web/app.py | 86 ++ pipeline/web/broadcaster.py | 56 ++ pipeline/web/deps.py | 34 + pipeline/web/jobs.py | 219 ++++ pipeline/web/schedules.py | 218 ++++ pipeline/web/schemas.py | 75 ++ pipeline/web/static/jobs.js | 280 ++++++ pipeline/web/static/schedules.js | 543 ++++++++++ pipeline/worker.py | 113 +++ tests/test_coupling.py | 75 ++ tests/test_packaging.py | 75 ++ 76 files changed, 15235 insertions(+) create mode 100644 auth/README.md create mode 100644 auth/__init__.py create mode 100644 auth/config.py create mode 100644 auth/middleware.py create mode 100644 auth/models.py create mode 100644 auth/passkeys.py create mode 100644 auth/router.py create mode 100644 auth/schemas.py create mode 100644 auth/static/account.js create mode 100644 auth/static/passkey.js create mode 100644 auth/store.py create mode 100644 auth/templates/login.html create mode 100644 auth/tests/__init__.py create mode 100644 auth/tests/conftest.py create mode 100644 auth/tests/harness.py create mode 100644 auth/tests/softkey.py create mode 100644 auth/tests/test_door.py create mode 100644 auth/tests/test_passkeys.py create mode 100644 auth/tests/test_store.py create mode 100644 auth/tests/test_without_passkeys.py create mode 100644 deploy/0-release.ps1 create mode 100644 deploy/1-test-and-promote.ps1 create mode 100644 deploy/2-build-image.ps1 create mode 100644 deploy/3-push-image.ps1 create mode 100644 deploy/Dockerfile create mode 100644 deploy/Dockerfile.dockerignore create mode 100644 deploy/README.md create mode 100644 deploy/_common.ps1 create mode 100644 deploy/db.ps1 create mode 100644 deploy/release.env.example create mode 100644 deploy/server/backup.sh create mode 100644 deploy/server/compose.yaml create mode 100644 deploy/server/deploy.sh create mode 100644 deploy/server/env.example create mode 100644 deploy/server/import.sh create mode 100644 deploy/server/import_database.py create mode 100644 deploy/server/swag/app.subdomain.conf create mode 100644 docs/box-docker-compose.yml create mode 100644 docs/hosting.md create mode 100644 docs/llm-backends.md create mode 100644 docs/password-protection.md create mode 100644 docs/release.md create mode 100644 docs/server.md create mode 100644 pipeline/README.md create mode 100644 pipeline/__init__.py create mode 100644 pipeline/child.py create mode 100644 pipeline/context.py create mode 100644 pipeline/models.py create mode 100644 pipeline/notices.py create mode 100644 pipeline/pipeline.md create mode 100644 pipeline/queue.py create mode 100644 pipeline/registry.py create mode 100644 pipeline/schedule.py create mode 100644 pipeline/table.py create mode 100644 pipeline/task.py create mode 100644 pipeline/tests/__init__.py create mode 100644 pipeline/tests/isolated_handlers.py create mode 100644 pipeline/tests/scheduled_handlers.py create mode 100644 pipeline/tests/test_connect.py create mode 100644 pipeline/tests/test_housekeeping.py create mode 100644 pipeline/tests/test_isolation.py create mode 100644 pipeline/tests/test_notice_log.py create mode 100644 pipeline/tests/test_retries.py create mode 100644 pipeline/tests/test_schedule.py create mode 100644 pipeline/web/__init__.py create mode 100644 pipeline/web/app.py create mode 100644 pipeline/web/broadcaster.py create mode 100644 pipeline/web/deps.py create mode 100644 pipeline/web/jobs.py create mode 100644 pipeline/web/schedules.py create mode 100644 pipeline/web/schemas.py create mode 100644 pipeline/web/static/jobs.js create mode 100644 pipeline/web/static/schedules.js create mode 100644 pipeline/worker.py create mode 100644 tests/test_coupling.py create mode 100644 tests/test_packaging.py diff --git a/auth/README.md b/auth/README.md new file mode 100644 index 0000000..520a207 --- /dev/null +++ b/auth/README.md @@ -0,0 +1,113 @@ +# auth + +One password on a public hostname. scrypt, a lockout counted in the database, sessions you can revoke, passkeys, and one middleware in front of everything — the WebSocket included. No user management: there is one user, they are the person who deployed it, and the only questions are whether the door is locked and what opens it. + +Needs Python 3.12+, `fastapi` and `pydantic`. `webauthn` is optional and only for passkeys; every import of it is inside a function, and the passkey routes answer 501 with a sentence saying what to install when it is missing. + +The source of truth is `boilerplate/auth/`. Copies elsewhere are copies: take the whole folder, and run its tests where it lands. + +```bash +cp -r boilerplate/auth yourproject/auth +python -m pytest yourproject/auth/tests +``` + +Then add `auth*` to that project's `pyproject.toml` packages, `"auth" = ["templates/*.html", "static/*"]` to its package data, and `auth/tests` to its testpaths. `docs/password-protection.md` in this repository is the reasoning behind every decision below, at length. + +## Five lines + +```python +from fastapi import FastAPI +import auth + +app = FastAPI() +auth.connect(app, auth.AuthStore(config.database), config.auth, + host=config.server.host, open=("/static/",), + stylesheet="/static/app.css") +``` + +Call it **first**, before the app's own routers. Starlette wraps the whole router in its middleware whenever the routes were added, so the socket and every page are covered either way — but written first it is also read first, and nobody reviewing the file has to work out whether the socket is behind the password. + +That gives you `/login` (the page and the POST), `/logout`, `/api/account`, `/api/account/password`, `/api/account/sessions/{id}` and `/api/account/sessions/close`, the four passkey routes, `/_auth/passkey.js` and `/_auth/account.js` — and a startup bootstrap that makes sure there is a password at all. + +| `connect(app, store, config, ...)` | | +|---|---| +| `store` | an `AuthStore` on whatever database the app keeps. Handing it in rather than building it here is what lets the app decide there is one file. | +| `config` | an `AuthConfig`. The app's own config model carries it as a field; see below. | +| `host=` | the address the server binds — `config.server.host`. What `enabled: auto` decides from, and deliberately not taken from a request, since a `Host` header is written by whoever is talking. | +| `open=` | extra paths reachable without a session, added to `middleware.OPEN`. A name ending in `/` is a prefix; any other is matched whole. `("/static/",)` is what an app with a stylesheet passes. | +| `stylesheet=` | a URL the login page links, so it looks like the rest of the app. Optional: the page carries enough inline CSS to be presentable on its own. | + +The account screen is two lines in a template, the same deal `` makes: + +```html + + +``` + +It draws the lot — locked or not, change the password, the sessions with revoke and close-others, the passkeys with register, rename and revoke, and sign out — and it is themed from outside by the same `--desk-jobs-*` custom properties the queue's elements read, so `desk-jobs, desk-schedules, auth-account { ... }` in one stylesheet is all three. + +## The config block + +`AuthConfig` is pydantic and imports nothing from the app, so an app's own config model carries it as a field: + +```python +from auth import AuthConfig + +class Config(Section): + auth: AuthConfig = Field(default_factory=AuthConfig) +``` + +```yaml +auth: + enabled: auto # auto | always | never + session_days: 30 + secure_cookie: auto # `always` behind a proxy that terminates TLS + rp_id: "" # app.example.com -- passkeys are bound to a domain + rp_name: app # what the authenticator shows you + origin: "" # https://app.example.com + password_env: APP_PASSWORD + cookie: app_session +``` + +`enabled: auto` is the setting that matters: locked unless the server binds a loopback address. Deciding from the bind address rather than defaulting to off prevents the failure that actually happens — you deploy, it works, and nothing anywhere tells you the door is open. Production says `always` anyway, because behind a TLS-terminating proxy every request looks local to the app and is not. + +`password_env` and `cookie` are settings because this folder is copied into projects that are not called `app`, and two of those on one hostname would otherwise share a cookie and sign each other out. + +## Getting in the first time + +The password is never in the config file — only ever an scrypt hash in the database. The first locked start takes `$APP_PASSWORD` if it is set, and otherwise invents one and prints it to the log once, in a plain ASCII banner you can read back off `docker compose logs`. + +`APP_PASSWORD` is authoritative while it is set: the Account screen says so and declines to change it. That is the way back in when a login is lost — set it, get in, empty it, recreate the container, then change the password in the app. A variable that silently stopped applying the moment you changed the password in the browser would be worse than either rule. + +## What the app has to provide + +Almost nothing, and that is the point. + +- A SQLite path. `AuthStore` creates four `auth_*` tables in whatever file it is given, so it can share the app's own. +- The bind host, for `enabled: auto`. +- Nothing for the open list, as a rule. `/login`, `/login/`, `/_auth/` and `/health` are opened by this package; the app adds `/static/` if it serves one, or the login page arrives unstyled. Everything else stays shut — a module's own scripts and sockets included, so in this repository `/_jobs/jobs.js` and `/ws/jobs` need a session like the pages that load them. +- An error handler that keeps `{"detail": "..."}` for `/api`, `/ws` and `/_auth`, if it has one. Every refusal here is already that shape. + +What it must **not** do is protect its own routes. Everything is shut except the open list, which is four entries long and asserted in a test — so a router added next month is safe because it exists, not because somebody remembered. + +## The decisions, in one paragraph each + +**scrypt, not SHA-256.** Memory-hard rather than merely slow: one guess costs 16 MB, which is what takes a stolen database file out of range of a GPU. A wrong answer costs a full derivation too, so the delay an attacker meets is the one the honest user meets. + +**A session is 32 random bytes, and the table holds only its SHA-256.** No signing key to generate, store, keep out of backups and rotate; signing out everywhere is one `DELETE`, which no self-contained token can offer without keeping exactly this state. + +**The lockout is counted in the file and read before the check.** A counter on the process is a free reset for whoever is guessing, on a server that reloads. Read after instead of before, and hammering the button extends your own lockout — which punishes the owner far harder than the attacker who is not watching. Five wrong answers are free, then 30 s doubling to a quarter of an hour, with the exponent capped so a bot cannot turn arithmetic into a 500. + +**One counter, not one per address.** Behind a proxy the only address the app sees is one the caller wrote. Keyed on that, an attacker gets a fresh counter per request while the owner still gets locked out by their own typos. So the address is logged — one WARNING per refusal, `auth.router: wrong password from `, stripped to the characters an address is made of — and decided from at the layer that knows who connected. `docs/password-protection.md` has the nginx and fail2ban half. + +**No CSRF token.** `SameSite=Lax` refuses to send this cookie on any cross-site request that is not a top-level navigation, and no `GET` here changes anything, so the attack a token would stop cannot be mounted. A second secret to rotate, and a second way for a request to fail, buys nothing. + +**Passkeys need a real hostname.** A passkey is bound to an origin, and that binding is what makes it unphishable — so it needs HTTPS, with `localhost` the one exception, which is what lets it be tried before deploying. Behind a proxy, set `rp_id` and `origin` explicitly or every ceremony fails as an origin mismatch. Changing `rp_id` later invalidates every key registered against the old one. The password never goes away, which is why revoking a key can never lock you out. + +## Tests + +```bash +uv run pytest auth/tests +``` + +The passkey ceremonies are tested against `auth/tests/softkey.py`, a real WebAuthn authenticator that happens to be in software: it produces the same CBOR, the same authenticator data and the same ECDSA signatures a hardware key does, so a wrong origin fails because the origin is signed over and a clone is caught because the counter went backwards. Those tests skip without the extra; what the app does *without* it is `test_without_passkeys.py`, which always runs. diff --git a/auth/__init__.py b/auth/__init__.py new file mode 100644 index 0000000..9d6c934 --- /dev/null +++ b/auth/__init__.py @@ -0,0 +1,160 @@ +"""One password on a public hostname, in one call. + + app = FastAPI() + store = connect(app, AuthStore("data/app.db"), config.auth, + host=config.server.host, open=("/static/",), + stylesheet="/static/app.css") + +That is the middleware in front of everything (the WebSocket included), the +login page, the account API, the `` element, and the startup +bootstrap that makes sure there is a password at all. No user management: there +is one user, they are the person who deployed it, and the only questions are +whether the door is locked and what opens it. + +| module | | +|---|---| +| `config.py` | `AuthConfig`: the `auth:` block of the app's `config.yaml` | +| `store.py` | `AuthStore`: scrypt, the lockout, sessions, passkeys, four `auth_*` tables in whatever SQLite file it is given | +| `models.py` | `Account`, `SessionInfo`, `Passkey` — what the account screen is told, which carries no credential | +| `middleware.py` | `RequireLogin`, `locked()`, the open list, the cookie | +| `passkeys.py` | the two WebAuthn ceremonies; `webauthn` is an optional extra, imported lazily | +| `schemas.py` | the five request bodies | +| `router.py` | `/login`, `/logout`, `/api/account/*`, the passkey routes, `/_auth/*.js` | +| `templates/login.html` | the one page served without a session | +| `static/` | `passkey.js` (the browser half of WebAuthn) and `account.js` (``) | + +The reasoning is `docs/password-protection.md`, and this package is that +document in code. `auth/README.md` is how to copy the folder into a project +that has never heard of this one. +""" + +from __future__ import annotations + +import logging +import os +import secrets +from collections.abc import AsyncIterator, Iterable +from contextlib import asynccontextmanager +from typing import Any + +from fastapi import FastAPI + +from auth import passkeys +from auth.config import AuthConfig +from auth.middleware import OPEN, RequireLogin, client_address, locked, token_from +from auth.models import Account, Passkey, SessionInfo +from auth.router import router +from auth.store import AuthStore + +log = logging.getLogger(__name__) + +__all__ = [ + "OPEN", "Account", "AuthConfig", "AuthStore", "Passkey", "RequireLogin", + "SessionInfo", "bootstrap", "client_address", "connect", "locked", + "passkeys", "router", "token_from", +] + + +def connect(app: FastAPI, store: AuthStore, config: AuthConfig, *, + open: Iterable[str] = (), stylesheet: str | None = None, + host: str = "") -> AuthStore: + """Put `app` behind a password: middleware, routes, element, bootstrap. + + Call it first, before the app's own routers. Starlette wraps the whole + router in its middleware whenever the routes were added, so the socket and + every page are covered either way — but written first it is also *read* + first, and nobody reviewing the file has to work out whether the socket is + behind the password. + + `store` is an `AuthStore` on whatever database the app keeps; handing it in + rather than building it here is what lets the app decide there is one file. + + | keyword | | + |---|---| + | `open` | extra paths reachable without a session, added to `middleware.OPEN`. A name ending in `/` is a prefix; any other is matched whole. `("/static/",)` is what an app with a stylesheet passes. | + | `stylesheet` | a URL the login page links, so it looks like the rest of the app. Optional: the page carries enough inline CSS to be presentable on its own. | + | `host` | the address the server binds — `config.server.host`. What `enabled: auto` decides from, and deliberately not taken from a request, since a `Host` header is written by whoever is talking. | + + The lifespan is wrapped rather than replaced, the same way + `pipeline.web.connect` does it, so an app's own lifespan still runs inside + this one: the password exists before the app's startup code, which may want + to say something about it. + + Returns the store, for the app to keep if it wants one. + """ + app.state.auth = store + app.state.auth_config = config + #: Whether a session is required. Decided once, here, because it depends on + #: the bind address, which only the caller knows. + app.state.auth_locked = locked(config, host) + app.state.auth_stylesheet = stylesheet or "" + + app.include_router(router) + app.add_middleware(RequireLogin, store=store, config=config, + open=tuple(open), host=host) + + inner = app.router.lifespan_context + + @asynccontextmanager + async def lifespan(this: FastAPI) -> AsyncIterator[Any]: + if app.state.auth_locked: + bootstrap(store, config) + store.purge() + # Outside the `locked` check on purpose: a passkey can be registered on + # a loopback app before it is ever deployed, which is the order this + # encourages, so an unlocked one can still have keys it cannot use. + passkeys.warn_if_missing(store) + async with inner(this) as state: + yield state + + app.router.lifespan_context = lifespan + return store + + +def bootstrap(store: AuthStore, config: AuthConfig) -> None: + """Make sure there is a password, and say so once if we invented one. + + The variable named by `config.password_env` — `APP_PASSWORD` by default — + is authoritative while it is set: if it is in the environment, it *is* the + password, and the account screen says so and declines to change it. That is + a deliberate choice over seeding from it once — a variable that silently + stopped applying the moment you changed the password in the browser would + be worse than either rule, and this way round the environment doubles as + the way back in when you are locked out of your own server. + + Re-derived only when the stored hash does not already match, so a restart is + not a write and "password last changed" keeps meaning something. + """ + pinned = os.environ.get(config.password_env, "") + if pinned: + if not store.check_password(pinned): + store.set_password(pinned) + log.info("password set from %s", config.password_env) + return + + if store.has_password(): + return + + password = secrets.token_urlsafe(12) + store.set_password(password) + # At WARNING so it survives the default log level, and banner-shaped because + # it is printed exactly once in the life of a database and being scrolled + # past is the same as not being printed at all. + # + # ASCII, deliberately. This is the one log line whose *contents* someone has + # to read back character by character, and it is read wherever the server + # happens to be running -- a Windows console under cp1252, a `docker logs` + # pipe with no locale set. Box-drawing characters come out as `?` in both, + # which turns the frame around the password into noise around the password. + log.warning( + "\n" + " ------------------------------------------------------------\n" + " This app is locked and had no password, so here is one:\n" + "\n" + " %s\n" + "\n" + " This is the only time it is shown. Change it under Account,\n" + " or set %s to pin one.\n" + " ------------------------------------------------------------", + password, config.password_env, + ) diff --git a/auth/config.py b/auth/config.py new file mode 100644 index 0000000..9fadaf7 --- /dev/null +++ b/auth/config.py @@ -0,0 +1,104 @@ +"""What the app is allowed to say about the door. + +Pydantic, and nothing imported from the app around it: a config section is the +one piece of a module folder that a project has to merge into its own settings, +and one that dragged the app's `Config` in with it could not be merged at all. +The app writes `auth: AuthConfig = Field(default_factory=AuthConfig)` and is +done. + +There is one user, so there is no user management here — only whether the door +is locked, how long a session lasts, and the three strings a passkey is bound +to. The password is not in this file and never will be: it lives in the +database as an scrypt hash, is changed from the Account tab, and is seeded on +first run either from the environment variable named by `password_env` or by a +generated one printed to the log once. +""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class AuthConfig(BaseModel): + """The `auth:` block of `config.yaml`. See docs/password-protection.md.""" + + #: Unknown keys are refused rather than ignored, the same rule the app's + #: own sections keep: a setting you believe you have changed and have not + #: is the expensive kind of typo. + model_config = ConfigDict(extra="forbid") + + #: `auto` is the setting that matters, and it means: locked unless the + #: server is bound to loopback. An app on 127.0.0.1 is reachable by whoever + #: is already sitting at the machine, so a password would be a second lock + #: on an open door; the same app on 0.0.0.0 is reachable by the network, + #: and that is a different application with the same code. + #: + #: Deciding from the bind address rather than defaulting to off is the + #: whole point. The failure this prevents is the one that actually happens: + #: you deploy, it works, and nothing anywhere tells you the door is open. + #: `always` locks it on localhost too and is what a deployment behind a + #: TLS-terminating proxy needs, since there every request looks local to + #: the app and is not. `never` is for putting some other authentication in + #: front of this one, and means exactly what it says. + enabled: Literal["auto", "always", "never"] = "auto" + + #: How long a login lasts before it has to be done again. Long, because + #: this is one person on their own devices, and the security value of a + #: short session here is close to zero while the cost -- typing a password + #: into a phone on the bus -- is not. + session_days: int = Field(30, ge=1) + + #: Whether the session cookie is marked `Secure`, so a browser will only + #: ever send it back over HTTPS. `auto` sets it when the request itself + #: arrived over HTTPS, which is right when the app terminates TLS and wrong + #: behind a reverse proxy that terminates it for you -- there the app sees + #: plain HTTP and `auto` reads as "no". Set `always` when something else is + #: doing TLS. `never` exists for testing over http on a LAN and is the + #: setting to be uncomfortable with. + secure_cookie: Literal["auto", "always", "never"] = "auto" + + # --------------------------------------------------------------- passkeys + # + # A passkey is bound to a *domain*, not to a server, and that binding is the + # entire security property: the browser will only offer a credential back to + # the origin it was created for, which is why a passkey cannot be phished + # the way a password can. Two settings below name that domain, and both are + # blank by default because on a laptop the request itself is the honest + # answer and on a server it is not. + + #: The domain passkeys are registered against -- `app.example.com`. Blank + #: derives it from the request's `Host`, which is right in development and + #: is the setting to pin in production: `Host` is client-supplied, and a + #: proxy that passed a forged one through would otherwise move the RP ID. + #: + #: **Changing this invalidates every registered passkey**, because a + #: credential for one domain is not a credential for another. It is not a + #: setting to tune; it is the name of the thing, set once. + rp_id: str = "" + + #: What the authenticator shows when it asks you to confirm. Cosmetic, and + #: the one string here a person actually reads. + rp_name: str = "app" + + #: The full origin the browser must be at -- `https://app.example.com`. + #: Blank derives it from the request, which behind a TLS-terminating proxy + #: derives `http://` and then rejects every real ceremony as a mismatch. + #: If `secure_cookie: always` applies to you, so does this. + origin: str = "" + + # ---------------------------------------------------------- the two names + # + # Named rather than hard-coded because this folder is copied into projects + # that are not called `app`, and two of them on one hostname would otherwise + # share a cookie and overwrite each other's sessions. + + #: The environment variable that pins the password. Authoritative while it + #: is set: the Account tab says so and declines to change it. See + #: `auth.bootstrap`. + password_env: str = "APP_PASSWORD" + + #: The session cookie's name. `__Host-` is prefixed to it when the cookie + #: will be Secure -- see `auth.middleware.cookie_name`. + cookie: str = "app_session" diff --git a/auth/middleware.py b/auth/middleware.py new file mode 100644 index 0000000..5c707d4 --- /dev/null +++ b/auth/middleware.py @@ -0,0 +1,246 @@ +"""The lock on the door: is there a session, and what happens when there isn't. + +Three things live here — when the door is locked at all, who gets past it, and +the cookie that says so. The password itself is `auth/store.py`; this module +never sees a hash. + +## Why a middleware and not a dependency + +The FastAPI-shaped answer is `Depends(current_session)` on every route, and it +is the wrong one here for two reasons. + +The first is the WebSocket. `/ws/jobs` is a real socket, and route dependencies +on a WebSocket endpoint are awkward at best — but a cookie rides the handshake +like any other request, so one ASGI middleware in front of everything covers +the socket and the JSON with the same six lines. + +The second is the failure mode. A dependency is opt-in: protection is applied +by remembering to apply it, and the cost of forgetting is not a broken page but +a quiet hole — an API served to the internet, and nothing on screen different. +An app adds routers regularly, and the next one should be safe because it +exists, not because someone remembered. So the rule is inverted: everything is +shut, and the few things that must be open are named in one list you can read +at a glance. + +## What being locked means + +`auto` decides from the bind address, which is the only setting that ever +mattered: bound to loopback, the door is already the machine's own login, and a +password is a second lock on it; bound to anything else, the app is on a +network and needs its own. The alternative — default off, switch it on when you +deploy — fails silently in the exact case it exists for, because an app serving +the internet with no password looks and behaves precisely like one that is fine. +""" + +from __future__ import annotations + +import asyncio +import ipaddress +import logging +from http.cookies import SimpleCookie +from typing import Any +from urllib.parse import quote + +from starlette.requests import HTTPConnection +from starlette.responses import JSONResponse, RedirectResponse, Response +from starlette.types import ASGIApp, Receive, Scope, Send + +from auth.config import AuthConfig +from auth.store import AuthStore + +log = logging.getLogger(__name__) + +#: Reachable without a session, and nothing else is. `/login` has to be, or +#: there is no way in, and the passkey ceremony under `/login/` with it; +#: `/_auth/` carries the two scripts the login page and the account element +#: are; `/health` is for a proxy probe that has no session to offer. The app +#: adds its own — `/static/`, normally, or the login page arrives unstyled. +#: +#: A name ending in `/` is a prefix, any other is matched whole, so `/health` +#: does not open `/healthz` and `/login` does not open `/loginx`. Matched +#: against the path only — a query string cannot widen this. +OPEN: tuple[str, ...] = ("/login", "/login/", "/_auth/", "/health") + + +def locked(config: AuthConfig, host: str) -> bool: + """Whether this app should ask for a password. See `AuthConfig.enabled`. + + `host` is the address the server binds, which is the app's own + `server.host` setting and not anything a request carries: a `Host` header + is written by whoever is talking, and a door that could be talked out of + being a door is not one. + """ + if config.enabled == "always": + return True + if config.enabled == "never": + return False + return not _loopback(host) + + +def _loopback(host: str) -> bool: + """Whether binding here means "this machine only". + + `localhost` by name, and anything in 127.0.0.0/8 or ::1 by number. An empty + host, or `0.0.0.0`, or a LAN address is not loopback — and an unparseable + one is treated as not loopback too, because the failure to be careful about + is guessing "private" and being wrong. + """ + if host in ("localhost", "::1"): + return True + try: + return ipaddress.ip_address(host).is_loopback + except ValueError: + return False + + +def cookie_name(config: AuthConfig, scope_secure: bool) -> str: + """`__Host-` when the cookie will be Secure, plain otherwise. + + That prefix is a promise to the browser that the cookie is Secure, + path-scoped to `/` and carries no Domain, and a browser will refuse to + store one that is not — which presents as a login that appears to succeed + and lands you back on the login page, with nothing in any log. So the name + is chosen per deployment rather than fixed. + """ + return f"__Host-{config.cookie}" if is_secure(config, scope_secure) else config.cookie + + +def cookie_names(config: AuthConfig) -> tuple[str, str]: + """Both names, secure first. Read and cleared together, because which one + was set depends on `secure_cookie` and on the scheme the login happened + over — and clearing only one leaves a cookie the browser goes on offering.""" + return f"__Host-{config.cookie}", config.cookie + + +def is_secure(config: AuthConfig, scope_secure: bool) -> bool: + setting = config.secure_cookie + if setting == "always": + return True + if setting == "never": + return False + return scope_secure + + +def token_from(conn: HTTPConnection | Scope, config: AuthConfig) -> str: + """The session token this request carries, or "". + + Both cookie names are looked for, so flipping `secure_cookie` or moving + behind TLS does not strand a live session under the other name. + """ + scope: Any = conn.scope if isinstance(conn, HTTPConnection) else conn + for key, value in scope.get("headers", ()): + if key == b"cookie": + jar = SimpleCookie() + try: + jar.load(value.decode("latin-1")) + except Exception: # a cookie header that is not one + return "" + for name in cookie_names(config): + if name in jar: + return jar[name].value + return "" + + +def client_address(conn: HTTPConnection | Scope) -> str: + """Who is asking, as well as it can be known, for the log to say so. + + Behind a reverse proxy the peer is a container on the same Docker network + and says nothing about who is at the other end; the address worth writing + down is the first hop of `X-Forwarded-For`, which the proxy sets. Both are + reported when they differ, because the header is only as trustworthy as + whoever sent it — which is why nothing is *decided* from this, only logged. + See `AuthStore.check_password` for that argument in full. + + That header is also the one piece of a stranger's own writing that ends up + in the log, so only the characters an address is made of survive the trip: + a line in this log is something fail2ban reads and a person believes, and + neither should be forgeable by whoever is knocking. + """ + scope: Any = conn.scope if isinstance(conn, HTTPConnection) else conn + peer = (scope.get("client") or ("", 0))[0] or "unknown" + for key, value in scope.get("headers", ()): + if key == b"x-forwarded-for": + first = value.decode("latin-1").split(",")[0] + claimed = "".join(c for c in first if c.isalnum() or c in ".:%-")[:45] + if claimed and claimed != peer: + return f"{claimed} via {peer}" + return peer + + +#: Prefixes whose refusal is JSON rather than a redirect. A caller that asked +#: for data and is sent a login page reads it as a 200 full of HTML where its +#: JSON should be; `{"detail": ...}` with a 401 is something it can act on. +JSON_PREFIXES = ("/api", "/ws", "/_jobs", "/_auth") + + +class RequireLogin: + """Everything is shut except the open list. Raw ASGI, so it covers sockets. + + Deliberately not `BaseHTTPMiddleware`, which only sees `http` scopes — a + WebSocket would pass straight under it, and `/ws/jobs` streams the state of + every job in the app. + + Whether the door is locked at all is decided once, at construction, from + the config and the bind address: neither changes while the process runs, + and deciding per request would mean re-deriving the same answer a thousand + times a minute and having no single place to read it off. + """ + + def __init__(self, app: ASGIApp, store: AuthStore, config: AuthConfig, + open: tuple[str, ...] = (), host: str = "") -> None: + self.app = app + self.store = store + self.config = config + self.open = OPEN + tuple(open) + self.locked = locked(config, host) + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] not in ("http", "websocket") or not self.locked: + return await self.app(scope, receive, send) + + path = scope.get("path", "") + if any(path == p.rstrip("/") or (p.endswith("/") and path.startswith(p)) + for p in self.open): + return await self.app(scope, receive, send) + + token = token_from(scope, self.config) + # `to_thread`, because the store is SQLite and this is the event loop. + session = await asyncio.to_thread(self.store.touch, token) if token else None + if session is not None: + # Parked on the scope so `/api/account` can flag which row in the + # sessions list is the browser asking, and `/logout` can end the + # right one. Not a header and not a dependency: the middleware is + # the only thing that has verified it. + scope["auth_token"] = token + return await self.app(scope, receive, send) + + await self._refuse(scope, receive, send, path) + + async def _refuse(self, scope: Scope, receive: Receive, send: Send, + path: str) -> None: + """No session. Say so in whatever the caller can understand.""" + if scope["type"] == "websocket": + # Sent before accepting, which the ASGI server turns into an HTTP + # 403 on the handshake. The reconnect backoff in `` then + # does the right thing by itself. + await send({"type": "websocket.close", "code": 1008}) + return + + response: Response + # A GET of anything that is not an API path is a page being asked for, + # whatever the caller put in `Accept` — curl and a browser want the + # same thing there, and a 401 in place of the login page is a worse + # answer to both. Everything else is JSON, including a POST to a page + # path: a form submitted without a session has nowhere useful to land. + if scope.get("method") == "GET" and not path.startswith(JSON_PREFIXES): + # Carry where they were going, so a bookmarked deep link survives + # the login. Only ever a path from this app's own scope — but it is + # being written into a query string, and the scope has already + # decoded it, so a path holding a `&` or a `%` would arrive at the + # login page as a different path or as nothing. `quote` puts it back + # the way `URLSearchParams` will read it. + target = "/login" + (f"?next={quote(path)}" if path not in ("/", "") else "") + response = RedirectResponse(target, status_code=303) + else: + response = JSONResponse({"detail": "Not signed in"}, status_code=401) + await response(scope, receive, send) diff --git a/auth/models.py b/auth/models.py new file mode 100644 index 0000000..2512073 --- /dev/null +++ b/auth/models.py @@ -0,0 +1,81 @@ +"""What the Account tab is told, and nothing more. + +Three models, and the thing worth noticing about all three is what they cannot +carry. No token, no hash, no key material — not even the public half of a +passkey. The list of ways into an app is the one page most worth reading and +least worth leaking, so the shape it is rendered from is one that could not +leak a credential if it were logged whole. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class SessionInfo(BaseModel): + """One live login, for the account screen to list and revoke. + + `id` is the row's own integer, which is what the revoke button sends back. + The screen therefore holds nothing that could be replayed as a credential. + """ + + id: int + #: The browser that opened it, as it described itself. Advisory — a client + #: writes its own user agent — but it is what tells the laptop from the + #: phone, and an unexpected row here is the signal worth having. + label: str = "" + created_at: str = "" + last_seen_at: str = "" + expires_at: str = "" + #: The session making this request, which the UI must not offer to revoke + #: as though it were one of the others. + current: bool = False + + +class Passkey(BaseModel): + """One registered authenticator, as the account screen sees it.""" + + id: int + #: What you called it. A passkey you cannot name is one you will never dare + #: revoke, because you will not know which key it is. + label: str = "" + created_at: str = "" + last_used_at: str = "" + #: How the authenticator says it can be reached — `usb`, `nfc`, `internal`, + #: `hybrid`. Advisory and self-reported, but it is what tells "the YubiKey" + #: from "this laptop" in a list where both are just names. + transports: list[str] = Field(default_factory=list) + + +class Account(BaseModel): + """What the account screen knows. Deliberately almost nothing. + + There is one user and they are already signed in, so there is no name, no + email and no id to show — only whether the door is locked, when the password + last changed, and what is currently signed in. + """ + + #: Whether the door is locked at all. False on a loopback-bound app under + #: the default `auto`, and the screen says so rather than offering a + #: password change that would not be asked for. + locked: bool = False + #: The environment pins the password, so the browser cannot change it. The + #: variable is named by `AuthConfig.password_env`; see `auth.bootstrap`. + pinned: bool = False + #: Which variable that is, so the screen can name it rather than saying + #: "an environment variable" to somebody who has to go and find it. + pinned_by: str = "" + password_set_at: str = "" + sessions: list[SessionInfo] = Field(default_factory=list) + #: Registered authenticators. Always listed, even when the door is open, + #: because registering one on your laptop before you deploy is the sensible + #: order to do this in. + passkeys: list[Passkey] = Field(default_factory=list) + #: Whether this install can do WebAuthn at all — the `passkeys` extra is + #: optional, and a screen offering a button that 501s is worse than one + #: that says what to install. + passkeys_available: bool = False + #: Why passkeys cannot be *registered* right now, or "" if they can. The + #: honest common case is an insecure origin: WebAuthn refuses anything but + #: HTTPS, localhost excepted, so this reads "you need TLS first". + passkeys_blocked: str = "" diff --git a/auth/passkeys.py b/auth/passkeys.py new file mode 100644 index 0000000..c3b6bbf --- /dev/null +++ b/auth/passkeys.py @@ -0,0 +1,398 @@ +"""WebAuthn: the two ceremonies, and the three things that decide whether they +can run at all. + +A passkey and a YubiKey are the same feature. Both are WebAuthn authenticators; +the difference is only where the private key lives — in the phone's secure +enclave, in the browser profile, or on a stick in a USB port — and none of that +reaches this module. One registration flow covers all of them, which is why +there is no "YubiKey" anywhere in this package. + +## What this is worth having + +A password can be typed into the wrong site. A passkey cannot: the browser will +only hand a credential back to the origin it was created for, and it checks +that itself, before anything of ours runs. That property is the whole reason +this exists, and it is also the reason for every constraint below — the binding +to an origin is not a detail to be relaxed when it is inconvenient. + +## The three things that can stop it + +**The package.** `webauthn` is an optional extra, so every import of it is +inside a function and `available()` is what the account screen asks before +offering a button. A missing extra is a sentence about `uv sync`, not a 500. + +**The origin.** WebAuthn refuses to run on an insecure origin — `localhost` is +specifically exempted, everything else needs HTTPS. So an app served over plain +HTTP on a LAN address can hold passkeys and verify them, and cannot register +one, and the account screen says so rather than showing a button that fails +inside the browser with an error nobody surfaces. + +**The RP ID.** Derived from the request's `Host` when `rp_id` is blank, which +is right on a laptop and wrong the moment anything can forge that header. See +`AuthConfig.rp_id` for why production pins it. + +## The password does not go away + +Registering a passkey does not remove the password, and this module offers no +way to. A lost key with no fallback is an app nobody can reach — and the +recovery path for that would have to be at least as strong as the password it +replaced, which is to say: the password. +""" + +from __future__ import annotations + +import json +import logging +from typing import Any +from urllib.parse import urlsplit + +from starlette.requests import Request + +from auth.config import AuthConfig +from auth.store import AuthStore + +log = logging.getLogger(__name__) + +#: The one user's handle. A constant, and deliberately so: WebAuthn wants a +#: user id to tell accounts apart, and there is one account. Making it random +#: per install would buy nothing — two apps cannot share an RP ID, since the RP +#: ID *is* the domain — and would cost a stable value to store and migrate. It +#: is also not PII, which the spec asks of this field. +#: +#: One real consequence, and it is the one you want: re-registering on the same +#: authenticator replaces the resident credential rather than adding a second, +#: because (rp_id, user_handle) is what a discoverable credential is keyed on. +USER_HANDLE = b"app" + +#: What the authenticator shows in its own list, beside `rp_name`. Not a login +#: — there is no username here — but the field is required and something has to +#: be in it. +USER_NAME = "app" + +#: How to get the extra, said once and reused by every message that says it. +#: +#: The parenthetical is not padding. `uv sync --extra passkeys` *replaces* the +#: installed extras rather than adding to them, so running exactly what the +#: obvious instruction says will silently uninstall whatever else the project +#: uses — and the symptom of that is not an error at install time, it is +#: something unrelated failing later on. Anyone following this instruction is by +#: definition not thinking about their other extras, which is exactly when they +#: need to be told. +INSTALL_HINT = ("Run `uv sync --extra passkeys`, listing any other extras you " + "use in the same command — uv replaces the set rather than " + "adding to it — then restart.") + + +def available() -> bool: + """Whether the `passkeys` extra is installed.""" + try: + import webauthn # noqa: F401 + except ImportError: + return False + return True + + +def warn_if_missing(store: AuthStore) -> None: + """Say so at startup if there are passkeys here that cannot be used. + + The case this exists for: `uv sync` without `--extra passkeys` quietly + removes the package, and the next start has registered keys it cannot + verify. Without this the first sign anything is wrong is a login page with + no passkey button on it — which looks like the passkeys were lost, when they + are sitting in the database intact. + + Silent when nothing is registered, because then there is nothing to lose and + nothing to warn about: an install with no passkeys and no `webauthn` is just + an install that does not use passkeys. + """ + if available(): + return + registered = store.passkey_ids() + if not registered: + return + log.warning( + "%d passkey(s) are registered but the `passkeys` extra is not " + "installed, so they cannot be used. They are still stored and nothing " + "is lost. %s Until then, sign in with the password.", + len(registered), INSTALL_HINT, + ) + + +def rp_id(config: AuthConfig, request: Request) -> str: + """The domain to bind credentials to. + + The configured value wins. Falling back to the request's host is what makes + development work without configuration — and is why the config docstring + says to pin it in production, since `Host` arrives from the client. + """ + if config.rp_id: + return config.rp_id + return request.url.hostname or "localhost" + + +def origin(config: AuthConfig, request: Request) -> str: + """The exact origin the browser must be at, scheme and port included. + + Derived from the request when unset, which is right when the app is the + thing the browser connected to and wrong behind a proxy that terminated TLS + — there this derives `http://` and every ceremony fails verification with an + origin mismatch. That is the failure `AuthConfig.origin` exists for. + """ + if config.origin: + return config.origin.rstrip("/") + url = request.url + port = f":{url.port}" if url.port not in (None, 80, 443) else "" + return f"{url.scheme}://{url.hostname}{port}" + + +def blocked(config: AuthConfig, request: Request) -> str: + """Why a passkey cannot be *registered* from this request, or "". + + Registration is the half that needs a secure context; verification does not, + because by then the browser has already done its own checks. So this is + asked when the account screen renders, not when a login is attempted. + """ + if not available(): + return f"Passkeys need the `passkeys` extra, which is not installed. {INSTALL_HINT}" + where = origin(config, request) + host = urlsplit(where).hostname or "" + # The exact rule browsers apply: HTTPS anywhere, or HTTP on loopback. + # `.localhost` is in the same boat by RFC 6761 and Chrome honours it. + secure = (where.startswith("https://") or host == "localhost" + or host.endswith(".localhost") or host == "127.0.0.1" or host == "::1") + if not secure: + return (f"Passkeys need HTTPS. This app is being served at {where}, and " + f"a browser will not create one on an insecure origin. Put TLS " + f"in front of it, then set `auth.origin` in config.yaml.") + return "" + + +# -------------------------------------------------------------- registration + +def register_options(config: AuthConfig, request: Request, + store: AuthStore) -> dict[str, Any]: + """The challenge for adding a passkey, as the browser's `create()` wants it. + + `exclude_credentials` is every key already registered, which is what makes + an authenticator say "you already have one of these" instead of quietly + creating a second credential that `add_passkey` would then reject on the + unique index — an error at the end of a ceremony you had already touched the + key for. + """ + from webauthn import generate_registration_options, options_to_json + from webauthn.helpers.structs import ( + AuthenticatorSelectionCriteria, PublicKeyCredentialDescriptor, + ResidentKeyRequirement, UserVerificationRequirement, + ) + + options = generate_registration_options( + rp_id=rp_id(config, request), + rp_name=config.rp_name, + user_id=USER_HANDLE, + user_name=USER_NAME, + user_display_name=config.rp_name, + exclude_credentials=[PublicKeyCredentialDescriptor(id=cid) + for cid in store.passkey_ids()], + authenticator_selection=AuthenticatorSelectionCriteria( + # `preferred`, not `required`: a resident (discoverable) credential + # is what lets the login page offer "use a passkey" with nothing + # typed first, and it is worth asking for. Requiring it would turn + # away older security keys that have no room to store one, and a + # key that works as a second factor is still worth having. + resident_key=ResidentKeyRequirement.PREFERRED, + # Likewise: ask for a PIN or a fingerprint, do not insist. Insisting + # locks out keys with no PIN set, for a property the password behind + # this does not have either. + user_verification=UserVerificationRequirement.PREFERRED, + ), + ) + store.remember_challenge(options.challenge, "register") + return json.loads(options_to_json(options)) + + +def register_verify(config: AuthConfig, request: Request, store: AuthStore, + credential: dict[str, Any], label: str) -> int: + """Check what the authenticator sent back and keep it. Returns the row id. + + Raises `PasskeyError` with a sentence fit to show, because every failure + here is either a stale ceremony or a misconfigured origin, and both have a + specific thing to do about them. + """ + from webauthn import verify_registration_response + from webauthn.helpers.exceptions import ( + InvalidRegistrationResponse, WebAuthnException, + ) + + challenge = _challenge_from(credential) + if not store.take_challenge(challenge, "register"): + raise PasskeyError("That registration took too long, or was already " + "used. Press Add a passkey and try again.") + try: + verified = verify_registration_response( + credential=credential, + expected_challenge=challenge, + expected_rp_id=rp_id(config, request), + expected_origin=origin(config, request), + ) + except InvalidRegistrationResponse as exc: + raise PasskeyError(_explain(exc, config, request)) from None + except (WebAuthnException, ValueError) as exc: + raise PasskeyMalformed(_malformed(exc)) from None + + transports = ",".join( + t if isinstance(t, str) else t.value + for t in (credential.get("response", {}).get("transports") or [])) + return store.add_passkey( + credential_id=verified.credential_id, + public_key=verified.credential_public_key, + sign_count=verified.sign_count, + transports=transports, + label=label or "Passkey", + ) + + +# ---------------------------------------------------------------- signing in + +def login_options(config: AuthConfig, request: Request, + store: AuthStore) -> dict[str, Any]: + """The challenge for signing in. + + `allow_credentials` lists what is registered, which is what makes a security + key light up rather than the browser asking the user to pick from nothing. + For a discoverable credential it is redundant; for a key registered without + one it is the only thing that works. + """ + from webauthn import generate_authentication_options, options_to_json + from webauthn.helpers.structs import ( + PublicKeyCredentialDescriptor, UserVerificationRequirement, + ) + + options = generate_authentication_options( + rp_id=rp_id(config, request), + allow_credentials=[PublicKeyCredentialDescriptor(id=cid) + for cid in store.passkey_ids()], + user_verification=UserVerificationRequirement.PREFERRED, + ) + store.remember_challenge(options.challenge, "login") + return json.loads(options_to_json(options)) + + +def login_verify(config: AuthConfig, request: Request, store: AuthStore, + credential: dict[str, Any]) -> int: + """Check a signature against a registered key. Returns the passkey's id. + + Raises `PasskeyError` on anything that is not a good signature from a key + this app knows. + """ + from webauthn import verify_authentication_response + from webauthn.helpers import base64url_to_bytes + from webauthn.helpers.exceptions import ( + InvalidAuthenticationResponse, WebAuthnException, + ) + + challenge = _challenge_from(credential) + if not store.take_challenge(challenge, "login"): + raise PasskeyError("That sign-in took too long, or was already used. " + "Try again.") + + known = store.passkey_for(base64url_to_bytes(credential.get("rawId", ""))) + if known is None: + # Deliberately the same sentence as a bad signature. There is one user, + # so this leaks little, but "that key is not registered" is an oracle + # for enumerating keys and costs nothing to withhold. + raise PasskeyError("That passkey is not registered here.") + passkey_id, public_key, stored_count = known + + try: + verified = verify_authentication_response( + credential=credential, + expected_challenge=challenge, + expected_rp_id=rp_id(config, request), + expected_origin=origin(config, request), + credential_public_key=public_key, + credential_current_sign_count=stored_count, + ) + except InvalidAuthenticationResponse as exc: + raise PasskeyError(_explain(exc, config, request)) from None + except (WebAuthnException, ValueError) as exc: + raise PasskeyMalformed(_malformed(exc)) from None + + # The clone check — a counter that goes backwards means two authenticators + # are answering for one credential — is the library's, not ours: it is + # inside `verify_authentication_response`, which is handed the stored count + # above and raises before returning. All that is left to do is translate its + # message; see `_explain`. + store.record_passkey_use(passkey_id, verified.new_sign_count) + return passkey_id + + +# -------------------------------------------------------------------- helpers + +class PasskeyError(Exception): + """A ceremony that did not complete, with a sentence worth showing.""" + + +class PasskeyMalformed(PasskeyError): + """A body that was never a ceremony — the caller's mistake, not a refusal. + + The distinction is only worth making at the login route, where a + `PasskeyError` is a 401 because something was wrong with the *credential*. + Nothing about a request missing half its fields says a credential was + refused, and answering 401 to it invites a retry of a request that cannot + ever work. It is a subclass so that anywhere the difference does not matter + — registering, behind a session — one `except` still covers both. + """ + + +def _challenge_from(credential: dict[str, Any]) -> bytes: + """The challenge the browser echoed back inside `clientDataJSON`. + + Read from the response rather than remembered per session, because the login + ceremony has no session to remember it in. That is safe *only* because + `take_challenge` then checks it against what was actually handed out, for + the right purpose, unexpired, and deletes it — a challenge this server never + issued matches nothing, and one it did issue works once. + """ + from webauthn.helpers import base64url_to_bytes + + try: + raw = credential["response"]["clientDataJSON"] + data = json.loads(base64url_to_bytes(raw)) + return base64url_to_bytes(data["challenge"]) + except (KeyError, TypeError, ValueError) as exc: + raise PasskeyMalformed("That response was not a passkey.") from exc + + +def _malformed(exc: Exception) -> str: + """A body the library could not read at all, said without a stack trace. + + `verify_*_response` raises a family of these — `InvalidJSONStructure` for a + missing field, and its siblings for CBOR and key material — and they are not + `Invalid{Registration,Authentication}Response`, so catching only that one + leaves the rest to come out of an unauthenticated endpoint as a 500 with a + traceback. The type is worth keeping in the message; the traceback is not, + because there is no bug here to find. + """ + return f"That was not a passkey response: {type(exc).__name__}: {exc}" + + +def _explain(exc: Exception, config: AuthConfig, request: Request) -> str: + """A verification failure, said in terms of what to change. + + The library's own message names the mismatch precisely and means nothing to + anyone who has not read the spec; the useful half is which of the two + settings is wrong, which is almost always the origin behind a proxy. + """ + detail = str(exc) + if "sign count" in detail.lower(): + # The one failure here that is not a misconfiguration. Two devices are + # answering for one credential and one of them is a copy — or, far more + # often, a key was restored from a backup that predates its own counter. + return ("That passkey looks cloned — its counter went backwards. " + "Revoke it under Account and register a new one.") + if "origin" in detail.lower() or "rp id" in detail.lower(): + return (f"This passkey was not created for {origin(config, request)} " + f"(domain {rp_id(config, request)}). Behind a proxy that " + f"terminates TLS, set `auth.origin` and `auth.rp_id` in " + f"config.yaml to the address you actually use.") + return f"That passkey could not be verified: {detail}" diff --git a/auth/router.py b/auth/router.py new file mode 100644 index 0000000..1450067 --- /dev/null +++ b/auth/router.py @@ -0,0 +1,489 @@ +"""Signing in, signing out, and the one account there is. + +The middleware in `auth/middleware.py` decides who gets past the door; this is +the door itself, plus the small screen for changing the key. Everything here is +deliberately about *this* user's own credential — there is no user to create, +none to list, and none to delete, because there is one. + +Nothing is read from a module-level global: the store, the config and whether +the door is locked all come off `app.state`, where `connect` put them. That is +what lets two apps in one test process each have their own, and what keeps this +router importable by an app that never calls `connect` at all. + +## Why the cookie is `SameSite=Lax` and there is no CSRF token + +A CSRF token defends a cookie-authenticated app against another origin +submitting a form on your behalf. `SameSite=Lax` already refuses to send this +cookie on any cross-site request that is not a top-level navigation, and no +`GET` in this package changes anything — so the attack the token would stop +cannot be mounted. Adding one anyway would mean a second secret, a second thing +to rotate, and a second way for a request to fail for reasons the UI has to +explain. Where that reasoning stops being true is a browser old enough to +ignore `SameSite` entirely, which is not one that will run the login page. +""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path +from urllib.parse import unquote, urlsplit + +from fastapi import APIRouter, HTTPException, Request, Response +from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse +from starlette.requests import HTTPConnection + +from auth import passkeys as pk +from auth.config import AuthConfig +from auth.middleware import ( + client_address, cookie_name, cookie_names, is_secure, token_from, +) +from auth.models import Account +from auth.schemas import ( + LabelBody, LoginBody, PasskeyBody, PasskeyLoginBody, PasswordBody, +) +from auth.store import AuthStore + +log = logging.getLogger(__name__) + +router = APIRouter(tags=["auth"]) + +#: The page and the two scripts, shipped with the package. Served from routes +#: rather than mounted as a static directory because a router cannot mount one +#: — only an app can — and needing the app to mount something is exactly the +#: line of setup `connect` exists to remove. The same deal `/_jobs/jobs.js` +#: makes in `pipeline/web/jobs.py`. +TEMPLATES = Path(__file__).resolve().parent / "templates" +STATIC = Path(__file__).resolve().parent / "static" + +#: The login page must never be cached: a proxy or a browser holding on to it +#: is a form posting to a door that has since been opened, or a page shown to +#: the next person at this machine. +NO_STORE = {"Cache-Control": "no-store"} + +#: The shortest new password the account screen will take. Low, deliberately: +#: it is a floor against a typo, not a password policy. The honest advice is in +#: docs/password-protection.md — the generated password is sixteen random +#: characters and anything you choose yourself is the weak half. +MIN_LENGTH = 8 + + +# ------------------------------------------------------- what connect left us + +def _store(conn: HTTPConnection) -> AuthStore: + return conn.app.state.auth + + +def _config(conn: HTTPConnection) -> AuthConfig: + return conn.app.state.auth_config + + +def _locked(conn: HTTPConnection) -> bool: + """Whether this app is running with the door locked. + + Read off `app.state`, where `connect` put it, rather than recomputed: the + answer depends on the bind address, which the config alone does not know + and a request cannot be asked for. + """ + return bool(conn.app.state.auth_locked) + + +def _pinned(conn: HTTPConnection) -> str: + """The password from the environment, or "". See `auth.bootstrap`.""" + return os.environ.get(_config(conn).password_env, "") + + +# ---------------------------------------------------------------- the scripts + +@router.get("/_auth/passkey.js", include_in_schema=False) +def passkey_js() -> FileResponse: + """The browser half of WebAuthn, shared by the login page and the element.""" + return _script("passkey.js") + + +@router.get("/_auth/account.js", include_in_schema=False) +def account_js() -> FileResponse: + """``: the account screen, for an app with no UI for it yet. + + + + """ + return _script("account.js") + + +def _script(name: str) -> FileResponse: + """Cached for an hour rather than forever. These change when the package is + copied into a project and then hardly ever, and an app that wants them + fingerprinted can serve the files itself from `STATIC`.""" + return FileResponse(STATIC / name, media_type="text/javascript", + headers={"Cache-Control": "public, max-age=3600"}) + + +# ------------------------------------------------------------------- the door + +@router.get("/login", response_class=HTMLResponse, include_in_schema=False) +def login_page(request: Request) -> Response: + """The one page served without a session. + + Sent only if you ask for it directly with no session cookie; with one, + there is nothing here to do, so it bounces to the front page rather than + offering a form that would replace a session you have. + """ + if not _locked(request): + return RedirectResponse("/", status_code=303) + token = token_from(request, _config(request)) + if token and _store(request).touch(token): + return RedirectResponse("/", status_code=303) + + config = _config(request) + store = _store(request) + # Whether to draw the passkey button at all. Decided here rather than by + # the page asking, because a button that appears and then discovers there + # is nothing to press is worse than one that was never drawn — and the page + # has no session, so it cannot ask anything else about the account. + offer = "true" if (pk.available() and store.passkey_ids()) else "false" + sheet = request.app.state.auth_stylesheet + html = (TEMPLATES / "login.html").read_text(encoding="utf-8") + html = (html + .replace("{stylesheet}", + f'' if sheet else "") + .replace("{name}", config.rp_name) + .replace("{passkeys}", offer)) + return HTMLResponse(html, headers=NO_STORE) + + +@router.post("/login", include_in_schema=False) +def sign_in(request: Request, response: Response, body: LoginBody) -> dict[str, str]: + """Check the password, and if it is right, open a session. + + The lockout is read *before* the password is checked, so hammering the + button while locked out does not extend the lockout — see + `AuthStore.locked_for`. A wrong password and an unknown one are the same + answer here for the usual reason, which is thin cover with one account but + costs nothing to keep. + """ + if not _locked(request): + raise HTTPException(400, "This app is not password-protected.") + + _refuse_while_locked(request) + + if not _store(request).check_password(body.password): + _warn(request, "wrong password") + raise HTTPException(401, "That is not the password.") + + _open_session(request, response) + return {"next": _safe_next(body.next)} + + +@router.post("/logout", include_in_schema=False) +def sign_out(request: Request, response: Response) -> dict[str, bool]: + """End this session and clear the cookie. Other devices are untouched.""" + token = token_from(request, _config(request)) + if token: + _store(request).close_session(token) + # Both names, because which one was set depends on `secure_cookie` and on + # the scheme the login happened over — and clearing only one of them leaves + # a cookie the browser will keep offering. + for name in cookie_names(_config(request)): + response.delete_cookie(name, path="/") + return {"ok": True} + + +# ---------------------------------------------------------------- the account + +@router.get("/api/account") +def account(request: Request) -> Account: + """What there is to know and what can be changed.""" + store = _store(request) + config = _config(request) + return Account( + locked=_locked(request), + pinned=bool(_pinned(request)), + pinned_by=config.password_env, + password_set_at=store.changed_at(), + sessions=store.sessions(token_from(request, config)), + passkeys=store.passkeys(), + passkeys_available=pk.available(), + passkeys_blocked=pk.blocked(config, request), + ) + + +@router.post("/api/account/password") +def change_password(request: Request, body: PasswordBody) -> dict[str, int]: + """Change it, having proved you know the current one — or set the first. + + Asking for the current password matters even though you are already signed + in: it is what makes a borrowed unlocked laptop not equal to a permanent + takeover of the account. It is asked for whenever there is one, which on a + locked app is always; see the body of the function for the one case where + there is not. + + Every *other* session is closed on success. If the reason you are changing + the password is that someone may have it, leaving their session open makes + the change ceremonial — and the cost when that is not the reason is signing + in again on your phone, which is the right way round. + + Which is also why this is behind the same lockout as the front door, and not + only behind a session. `current` is the password, guessed here rather than + at `/login`, and a borrowed unlocked laptop is exactly the situation the + field exists for — so leaving it ungated would leave one door counted and + one door open, at scrypt speed, ending in a password change that signs the + owner out everywhere. + """ + config = _config(request) + if _pinned(request): + raise HTTPException( + 409, f"The password is pinned by {config.password_env}. Change it there.") + store = _store(request) + + # The first one is set without proving anything, because there is nothing + # to prove: an app with no password is one nothing was asked for, and the + # only way to reach this route is from a session it never demanded. That + # is the unlocked laptop, and setting a password *there* — before the + # thing is ever deployed — is the order this should encourage. A locked + # app never lands here: `auth.bootstrap` gives it a password at startup, + # so `current` is always required for the app that has a door. + if store.has_password(): + _refuse_while_locked(request) + if not store.check_password(body.current): + _warn(request, "wrong current password") + raise HTTPException(403, "That is not the current password.") + if len(body.new) < MIN_LENGTH: + raise HTTPException(400, f"Use at least {MIN_LENGTH} characters.") + + store.set_password(body.new) + closed = store.close_all(keep=token_from(request, config)) + return {"signed_out": closed} + + +@router.delete("/api/account/sessions/{session_id}") +def revoke(session_id: int, request: Request) -> dict[str, bool]: + """Drop one session. Revoking your own is allowed — it is a logout with an + extra click, and refusing it would be a rule to explain for no gain.""" + if not _store(request).close_by_id(session_id): + raise HTTPException(404, "That session is already gone.") + return {"ok": True} + + +@router.post("/api/account/sessions/close") +def close_others(request: Request) -> dict[str, int]: + """Sign out everywhere but here. The reason sessions are rows.""" + store = _store(request) + return {"signed_out": store.close_all(keep=token_from(request, _config(request)))} + + +# ------------------------------------------------------------------- passkeys +# +# Four routes behind the session (register in two steps, rename, revoke) and two +# in front of it (sign in). The ceremony itself is `auth/passkeys.py`; these are +# the round trips it needs, and the translation of its two exceptions into +# status codes. + + +@router.post("/api/account/passkeys/options") +def passkey_options(request: Request) -> dict[str, object]: + """Step one of registering: a challenge for `navigator.credentials.create`.""" + _require_passkeys(request) + return pk.register_options(_config(request), request, _store(request)) + + +@router.post("/api/account/passkeys") +def add_passkey(request: Request, body: PasskeyBody) -> dict[str, int]: + """Step two: check what the authenticator signed, and keep the public key. + + Requires a session, like everything else under `/api/account`. Registering a + passkey is granting a new way in, so it is gated on already being in — the + password is what proves that, and the ceremony does not replace it. + """ + _require_passkeys(request) + try: + return {"id": pk.register_verify(_config(request), request, _store(request), + body.credential, body.label)} + except pk.PasskeyError as exc: + # `PasskeyMalformed` is one of these and answers the same way: from + # behind a session, a body that is not a ceremony and a ceremony that + # did not check out are both "that did not work, press it again". + raise HTTPException(400, str(exc)) from None + + +@router.post("/api/account/passkeys/{passkey_id}/label") +def rename_passkey(passkey_id: int, body: LabelBody, request: Request) -> dict[str, bool]: + """Name it. A key you cannot name is one you will never dare revoke.""" + if not _store(request).rename_passkey(passkey_id, body.label): + raise HTTPException(404, "That passkey is already gone.") + return {"ok": True} + + +@router.delete("/api/account/passkeys/{passkey_id}") +def drop_passkey(passkey_id: int, request: Request) -> dict[str, bool]: + """Revoke one. The password is still there, so this cannot lock you out — + which is exactly why the password is still there.""" + if not _store(request).delete_passkey(passkey_id): + raise HTTPException(404, "That passkey is already gone.") + return {"ok": True} + + +@router.post("/login/passkey/options", include_in_schema=False) +def passkey_login_options(request: Request) -> dict[str, object]: + """Step one of signing in. Open, necessarily — there is no session yet. + + Handing out a challenge to anyone who asks costs nothing: it is a random + number, single-use, and worth nothing without a private key that never left + the authenticator. + """ + _require_passkeys(request) + store = _store(request) + if not store.passkey_ids(): + raise HTTPException(404, "No passkeys are registered on this app.") + return pk.login_options(_config(request), request, store) + + +@router.post("/login/passkey", include_in_schema=False) +def passkey_login(request: Request, response: Response, + body: PasskeyLoginBody) -> dict[str, str]: + """Step two: a good signature from a registered key opens a session. + + The same session a password opens — one kind of session, whichever door it + came through, so revoking and expiry have one implementation. + + Deliberately not behind the password lockout. An attacker hammering the + front door locks it for the owner too (see `AuthStore.check_password` on + why there is one counter), and a passkey that skips that queue is what + makes it survivable. It is safe to skip: there is nothing to guess here, + only a signature to produce. + """ + if not _locked(request): + raise HTTPException(400, "This app is not password-protected.") + _require_passkeys(request) + try: + pk.login_verify(_config(request), request, _store(request), body.credential) + except pk.PasskeyMalformed as exc: + # Not a failed sign-in — a request that was never a ceremony. 400 rather + # than 401, and no warning, because there is no credential here to have + # been wrong about. + raise HTTPException(400, str(exc)) from None + except pk.PasskeyError as exc: + _warn(request, "passkey refused") + raise HTTPException(401, str(exc)) from None + + _open_session(request, response) + return {"next": "/"} + + +def _require_passkeys(request: Request) -> None: + """501 rather than 500 when the extra is not installed, and a sentence + saying what to install. + + "Not implemented" is the honest reading: the route exists and this build + cannot perform it. The account element watches for exactly this status and + takes the passkey section off the screen, so a missing optional dependency + costs a paragraph rather than a broken page. + + Only the package is checked here — an insecure origin blocks *registration* + in the browser, not these endpoints, and the account screen has already + said so. + """ + if not pk.available(): + raise HTTPException(501, pk.blocked(_config(request), request)) + + +# --------------------------------------------------------------------- helpers + +def _open_session(request: Request, response: Response) -> None: + """One session, one cookie, whichever door was used.""" + config = _config(request) + token = _store(request).open_session( + request.headers.get("user-agent", ""), config.session_days) + https = request.url.scheme == "https" + response.set_cookie( + cookie_name(config, https), + token, + max_age=config.session_days * 86400, + httponly=True, # JavaScript cannot read it, so an XSS cannot post it on + secure=is_secure(config, https), + samesite="lax", # see the module docstring on why there is no CSRF token + path="/", + ) + + +def _refuse_while_locked(request: Request) -> None: + """The lockout, in front of every place a password can be guessed. + + Read *before* the guess is looked at, so hammering while locked out does not + extend the lockout — see `AuthStore.locked_for`. Called from both doors + rather than living inside `check_password`, because the store counts and + this decides: the same wrong answer is a refusal here and a bootstrap + comparison in `auth.bootstrap`, which must not be able to lock anyone out of + their own restart. + """ + wait = _store(request).locked_for() + if wait: + _warn(request, "attempt while locked out") + raise HTTPException(429, f"Too many attempts. Try again in {_spell(wait)}.", + headers={"Retry-After": str(wait)}) + + +def _warn(request: Request, what: str) -> None: + """Write down that someone got it wrong, and who. + + The whole record of who is knocking. There is no access log worth the name — + uvicorn is run quietly — and no table of attempts, because the log is the + record and a second copy of it would only ever disagree. WARNING so it + survives that level, one line per refusal, and the address first after + `from` so a fail2ban filter is one regex; see docs/password-protection.md. + + The message does not name the package, because the logger already does: + with the usual `%(name)s: %(message)s` this module is `auth.router`, and a + line reading `auth.router: auth: wrong password` says it twice and reads + like a bug in the logging. + + Nothing is *decided* from the address — `AuthStore.check_password` says why + it cannot be — so this is where it earns its keep: at the layer that can + drop the connection instead of answering it. + """ + log.warning("%s from %s", what, client_address(request)) + + +def _safe_next(target: str) -> str: + r"""Where to go after signing in — only ever inside this app. + + The classic open redirect: a real login page, a real successful sign-in, and + someone else's site afterwards. So this accepts one shape only — a path + beginning with a single `/`, with no host in it — and turns everything else + into `/`. + + A backslash is refused because a browser treats it as a slash in exactly + this position: `/\evil.com` is parsed as `//evil.com` and is a host, which + is how a version of this that rejected only a literal `//` sent people to + another site. And the value is checked decoded as well as raw, because it + reaches here through a query string that `URLSearchParams` has already + unescaped once — `%5C` is a backslash to everything downstream of the + address bar, and no `next` this app produces needs an escaped slash to + survive. + """ + if not _local(target) or not _local(unquote(target)): + return "/" + return target + + +def _local(target: str) -> bool: + r"""Whether this is a path inside this app and nothing else. + + `urlsplit` rather than more string tests, because it is the parser the + browser's own rules are written against — it strips the tabs and newlines a + browser strips, so `/\t/evil.com` is seen the way the browser will see it, + as a host, and not as the local path a naive `startswith` reads. + """ + if not target.startswith("/") or target.startswith("//") or "\\" in target: + return False + try: + parts = urlsplit(target) + except ValueError: # an unparseable authority, e.g. `//[` + return False + return not parts.scheme and not parts.netloc + + +def _spell(seconds: int) -> str: + if seconds < 60: + return f"{seconds} seconds" + return f"{round(seconds / 60)} minutes" diff --git a/auth/schemas.py b/auth/schemas.py new file mode 100644 index 0000000..f679152 --- /dev/null +++ b/auth/schemas.py @@ -0,0 +1,56 @@ +"""The five bodies this package accepts. Small on purpose. + +A ceremony's `credential` is passed through as an opaque dict — it is the +browser's `PublicKeyCredential` serialised by `toJSON()`, and its shape is the +WebAuthn spec's business rather than ours. Restating it as a model here would +be a second, staler copy of a structure the library already parses and +validates properly. +""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, Field + + +class LoginBody(BaseModel): + """The whole login form. No username field, because there is no username — + asking for one would be theatre, and a wrong guess at it would be an error + message about something that does not exist.""" + + #: Bounded, because it is the one field an unauthenticated stranger can + #: send: scrypt is memory-hard by design and a megabyte of "password" is a + #: megabyte this process hashes before saying no. + password: str = Field(max_length=1000) + #: Where the middleware was sending them before it asked. Sanitised on the + #: way back out — see `_safe_next` in `auth/router.py`. + next: str = "/" + + +class PasswordBody(BaseModel): + """Changing it. `current` is required even though the caller is signed in; + `auth/router.py` says why.""" + + current: str = Field(max_length=1000) + new: str = Field(max_length=1000) + + +class PasskeyBody(BaseModel): + """A finished registration ceremony, on its way to be checked.""" + + credential: dict[str, Any] + #: What to call it in the list. Suggested by the browser, editable after. + label: str = "" + + +class PasskeyLoginBody(BaseModel): + """A finished authentication ceremony. Same reasoning as above.""" + + credential: dict[str, Any] + + +class LabelBody(BaseModel): + """Renaming a passkey.""" + + label: str diff --git a/auth/static/account.js b/auth/static/account.js new file mode 100644 index 0000000..8bee28e --- /dev/null +++ b/auth/static/account.js @@ -0,0 +1,505 @@ +/* The account screen, for an app that has not written a UI for it. + * + * + * + * + * That is the whole of it. The element asks `/api/account` what there is to + * know, and draws the four things there are to do about it: change the + * password, look at what is signed in and revoke any of it, register and + * revoke passkeys, and sign out. + * + * One file, and the styles are in it rather than beside it, because the shadow + * root scopes them anyway and a second file is a second route, a second cache + * header and a second thing to forget to copy. Themed from outside through the + * same `--desk-jobs-*` custom properties `` reads -- set them once + * on `desk-jobs, desk-schedules, auth-account` and the queue's elements and + * this one look like one application rather than three. + * + * Failures are said in place, next to the button that caused them, *and* + * raised as an `auth-account:error` event carrying the server's own `detail`, + * so an app with toasts can show one without this element knowing what a toast + * is. Same split as `desk-jobs:notice`. + * + * Attributes, both optional: + * base="/api" prefix for the HTTP calls, if the router is mounted deeper + * login="/login" where signing out sends the browser + */ + +const CSS = ` +:host { + --desk-jobs-bg: #16181d; + --desk-jobs-fg: #e7e9ee; + --desk-jobs-dim: #9aa1ad; + --desk-jobs-accent: #4c8dff; + --desk-jobs-error: #ff6b6b; + --desk-jobs-radius: 10px; + --desk-jobs-font: system-ui, -apple-system, "Segoe UI", sans-serif; + --desk-jobs-gap: 8px; + + display: block; + font-family: var(--desk-jobs-font); + font-size: 14px; + color: var(--desk-jobs-fg); +} +:host([hidden]) { display: none; } + +.panel { display: flex; flex-direction: column; gap: calc(var(--desk-jobs-gap) * 2); } + +section { display: flex; flex-direction: column; gap: var(--desk-jobs-gap); } +h3 { font-size: 13px; text-transform: uppercase; letter-spacing: .06em; + color: var(--desk-jobs-dim); margin: 0; font-weight: 600; } + +.said { color: var(--desk-jobs-dim); } +.said.bad { color: var(--desk-jobs-error); } +.what { font-weight: 600; } + +.row { + display: flex; align-items: baseline; justify-content: space-between; + gap: var(--desk-jobs-gap); + padding: 6px 0; + border-top: 1px solid color-mix(in srgb, var(--desk-jobs-fg) 12%, transparent); +} +.row:first-of-type { border-top: 0; } +.row .lines { display: flex; flex-direction: column; min-width: 0; } +.row .acts { display: flex; gap: var(--desk-jobs-gap); flex: none; } + +form { display: flex; flex-wrap: wrap; gap: var(--desk-jobs-gap); align-items: center; } +input { + font: inherit; color: inherit; background: transparent; + border: 1px solid color-mix(in srgb, var(--desk-jobs-fg) 25%, transparent); + border-radius: 6px; padding: 3px 7px; min-width: 11rem; +} + +button { + font: inherit; + color: var(--desk-jobs-dim); + background: none; + border: 1px solid currentColor; + border-radius: 6px; + padding: 2px 9px; + cursor: pointer; +} +button:hover { color: var(--desk-jobs-accent); } +button.danger:hover { color: var(--desk-jobs-error); } +button[disabled] { opacity: .4; cursor: default; } + +.badge { + font-size: 12px; padding: 0 6px; border-radius: 999px; + border: 1px solid currentColor; color: var(--desk-jobs-dim); +} +`; + +/** What the server says when this build has no `webauthn` installed. The + * element takes the whole passkey section off the screen rather than leaving + * buttons that can only ever fail -- and 501 is the status that says "this + * route exists and this build cannot do it", which is exactly the case. */ +const NOT_BUILT = 501; + +class AuthAccount extends HTMLElement { + #root; + #panel; + #account = null; + #busy = false; + /** The one line under the status block: what the last button did, or why it + * did not. Kept on the element rather than in the DOM because every action + * reloads the account and redraws everything. */ + #message = ""; + #bad = false; + /** Set once the server has answered 501 to a passkey call. `/api/account` + * already says `passkeys_available`, and this is the same answer arriving + * the other way -- from an app that was restarted without the extra while + * this page was open. */ + #passkeysGone = false; + + constructor() { + super(); + this.#root = this.attachShadow({ mode: "open" }); + const style = document.createElement("style"); + style.textContent = CSS; + this.#panel = document.createElement("div"); + this.#panel.className = "panel"; + this.#root.append(style, this.#panel); + } + + connectedCallback() { this.#load(); } + + get #base() { return this.getAttribute("base") ?? ""; } + get #login() { return this.getAttribute("login") ?? "/login"; } + + // ------------------------------------------------------------------ server + + /** One fetch helper, so every failure surfaces the same way: the server's + * own sentence, in place and as an event. A 401 means the session ended + * under us, which is a redirect rather than a message. */ + async #api(path, options = {}) { + const response = await fetch(this.#base + path, { + credentials: "same-origin", + headers: Object.assign({ Accept: "application/json" }, + options.body ? { "Content-Type": "application/json" } : {}), + ...options, + }); + let data = null; + const text = await response.text(); + if (text) { try { data = JSON.parse(text); } catch { data = null; } } + if (!response.ok) { + const error = new Error((data && data.detail) + || `The server answered ${response.status} and said nothing about why.`); + error.status = response.status; + throw error; + } + return data; + } + + async #load() { + try { + this.#account = await this.#api("/api/account"); + this.#render(); + } catch (err) { + if (err.status === 401) { location.assign(this.#login); return; } + this.#fail(err); + this.#render(); + } + } + + /** Run one button's worth of work: disable the panel, say what went wrong, + * and reload the account either way -- the server is the only thing that + * knows what the sessions and passkeys are now. */ + async #act(work) { + if (this.#busy) return; + this.#busy = true; + this.#message = ""; + this.#bad = false; + this.#render(); + try { + await work(); + } catch (err) { + if (err.status === 401) { location.assign(this.#login); return; } + if (err.status === NOT_BUILT) this.#passkeysGone = true; + this.#fail(err); + } finally { + this.#busy = false; + } + await this.#load(); + } + + #fail(err) { + this.#message = String(err.message || err); + this.#bad = true; + this.dispatchEvent(new CustomEvent("auth-account:error", { + detail: { detail: this.#message, status: err.status || 0 }, + bubbles: true, composed: true, + })); + } + + #say(text) { + this.#message = text; + this.#bad = false; + } + + // ------------------------------------------------------------------ drawing + + #render() { + const account = this.#account; + this.#panel.replaceChildren(); + if (!account) { + this.#panel.append(this.#note(this.#message || "Loading…", this.#bad)); + return; + } + + this.#panel.append(this.#status(account)); + if (this.#message) this.#panel.append(this.#note(this.#message, this.#bad)); + this.#panel.append(this.#password(account)); + if (account.sessions.length) this.#panel.append(this.#sessions(account)); + if (this.#showPasskeys(account)) this.#panel.append(this.#passkeys(account)); + if (account.locked) this.#panel.append(this.#signOut()); + } + + #showPasskeys(account) { + return account.passkeys_available && !this.#passkeysGone; + } + + #status(account) { + const section = this.#section("This app"); + section.append(this.#line( + account.locked + ? "Locked. Every request but the login page, the health probe and the static files needs a session." + : "Not locked. The bind address is loopback and `auth.enabled` is `auto`, so anyone who can reach this app is already sitting at the machine.")); + if (account.password_set_at) { + section.append(this.#line(`Password last set ${when(account.password_set_at)}.`)); + } else { + section.append(this.#line("No password has been set on this database yet.")); + } + if (account.pinned) { + section.append(this.#line( + `The password is pinned by ${account.pinned_by || "the environment"}, so it cannot be changed here. Change it there and restart.`)); + } + if (account.passkeys_blocked && account.passkeys_available) { + section.append(this.#line(account.passkeys_blocked)); + } + return section; + } + + #password(account) { + const section = this.#section("Password"); + if (account.pinned) { + section.append(this.#line( + `Set by ${account.pinned_by || "the environment"}. Empty that variable, restart, and this form comes back.`)); + return section; + } + + // Nothing to prove when there is nothing set yet, which is what an + // unlocked app on a laptop looks like before it is ever deployed. Asking + // for a current password there is a field with no right answer, and the + // form could only ever be refused -- see `change_password` in router.py. + const first = !account.password_set_at; + const form = document.createElement("form"); + const current = field("password", "Current password"); + const next = field("password", first ? "Choose a password" : "New password"); + const go = document.createElement("button"); + go.textContent = first ? "Set password" : "Change password"; + go.disabled = this.#busy; + form.append(...(first ? [next, go] : [current, next, go])); + form.addEventListener("submit", (event) => { + event.preventDefault(); + this.#act(async () => { + const result = await this.#api("/api/account/password", { + method: "POST", + body: JSON.stringify({ current: current.value, new: next.value }), + }); + current.value = next.value = ""; + if (first) return this.#say("Password set."); + this.#say(result.signed_out + ? `Password changed. ${count(result.signed_out, "other session")} signed out.` + : "Password changed."); + }); + }); + section.append(form); + // Every *other* session goes when the password changes: if the reason you + // are changing it is that someone may have it, leaving their session open + // makes the change ceremonial. Said here so it is not a surprise. + section.append(this.#line(first + ? "Nothing asks for it until the app binds an address that is not loopback -- or `auth.enabled: always`. Set it before you deploy, not after." + : "Changing it signs out every other session, and keeps this one.")); + return section; + } + + #sessions(account) { + const section = this.#section("Signed in"); + for (const session of account.sessions) { + const row = this.#row( + session.label || "Unnamed browser", + `${session.current ? "This browser · " : ""}last seen ${when(session.last_seen_at)} · started ${when(session.created_at)}`); + if (session.current) row.querySelector(".acts").append(badge("this one")); + row.querySelector(".acts").append(this.#button("Revoke", "danger", () => + this.#act(async () => { + await this.#api(`/api/account/sessions/${session.id}`, { method: "DELETE" }); + // Revoking your own is allowed -- it is a logout with an extra click + // -- and the next request is the one that discovers it. + this.#say("Session revoked."); + }))); + section.append(row); + } + if (account.sessions.length > 1) { + section.append(this.#button("Sign out everywhere else", "", () => + this.#act(async () => { + const result = await this.#api("/api/account/sessions/close", { method: "POST" }); + this.#say(`${count(result.signed_out, "session")} signed out.`); + }))); + } + return section; + } + + #passkeys(account) { + const section = this.#section("Passkeys"); + for (const key of account.passkeys) { + const row = this.#row( + key.label || "Passkey", + [key.transports.join(", "), + `added ${when(key.created_at)}`, + key.last_used_at ? `last used ${when(key.last_used_at)}` : "never used", + ].filter(Boolean).join(" · ")); + const acts = row.querySelector(".acts"); + acts.append(this.#button("Rename", "", () => { + const label = prompt("Call this passkey what?", key.label || ""); + if (label === null) return; + this.#act(async () => { + await this.#api(`/api/account/passkeys/${key.id}/label`, { + method: "POST", body: JSON.stringify({ label }), + }); + this.#say("Renamed."); + }); + })); + acts.append(this.#button("Revoke", "danger", () => this.#act(async () => { + await this.#api(`/api/account/passkeys/${key.id}`, { method: "DELETE" }); + // Safe by construction: the password is still there, which is why + // revoking a key can never lock anyone out. + this.#say("Passkey revoked."); + }))); + section.append(row); + } + + if (account.passkeys_blocked) { + // Registering needs a secure origin; verifying does not. So the list + // above is still worth drawing, and only the button goes. + section.append(this.#line(account.passkeys_blocked)); + return section; + } + const add = this.#button("Add a passkey", "", () => this.#register()); + add.disabled = this.#busy; + section.append(add); + return section; + } + + async #register() { + const passkey = await loadPasskeyHelper(); + if (!passkey || !passkey.supported()) { + this.#say("This browser will not create a passkey here. It needs HTTPS, or localhost."); + this.#render(); + return; + } + // Two round trips: a challenge, then what the authenticator signed. The + // browser's own prompt happens between them, and a cancelled prompt is an + // exception rather than an error status -- hence `explain`. + await this.#act(async () => { + const options = await this.#api("/api/account/passkeys/options", { method: "POST" }); + let made; + try { + made = await passkey.create(options); + } catch (err) { + throw new Error(passkey.explain(err)); + } + const label = suggest(); + await this.#api("/api/account/passkeys", { + method: "POST", body: JSON.stringify({ credential: made, label }), + }); + this.#say(`Registered "${label}". Rename it if that is not what it is.`); + }); + } + + #signOut() { + const section = this.#section("This session"); + section.append(this.#button("Sign out", "danger", async () => { + try { + await this.#api("/logout", { method: "POST" }); + } catch { /* the cookie is going either way */ } + location.assign(this.#login); + })); + return section; + } + + // --------------------------------------------------------------- furniture + + #section(title) { + const section = document.createElement("section"); + const head = document.createElement("h3"); + head.textContent = title; + section.append(head); + return section; + } + + #line(text) { + const line = document.createElement("div"); + line.className = "said"; + line.textContent = text; + return line; + } + + #note(text, bad) { + const line = this.#line(text); + if (bad) line.classList.add("bad"); + return line; + } + + #row(what, said) { + const row = document.createElement("div"); + row.className = "row"; + const lines = document.createElement("div"); + lines.className = "lines"; + const name = document.createElement("span"); + name.className = "what"; + name.textContent = what; + const under = document.createElement("span"); + under.className = "said"; + under.textContent = said; + lines.append(name, under); + const acts = document.createElement("div"); + acts.className = "acts"; + row.append(lines, acts); + return row; + } + + #button(text, className, onClick) { + const button = document.createElement("button"); + button.textContent = text; + if (className) button.className = className; + button.disabled = this.#busy; + button.addEventListener("click", onClick); + return button; + } +} + +/** The WebAuthn helper is a classic script on `window`, shared with the login + * page, which is plain HTML served before any of the app's scripts. Loaded on + * demand rather than up front, so a screen nobody registers a passkey from + * never fetches it. */ +let helper = null; +function loadPasskeyHelper() { + if (window.passkey) return Promise.resolve(window.passkey); + helper = helper || new Promise((resolve) => { + const script = document.createElement("script"); + script.src = "/_auth/passkey.js"; + script.onload = () => resolve(window.passkey || null); + script.onerror = () => resolve(null); + document.head.append(script); + }); + return helper; +} + +/** A name the authenticator can be told apart by, before anyone renames it. + * Better than "Passkey" and worse than what a person would call it, which is + * why Rename is beside it. */ +function suggest() { + const agent = navigator.userAgent || ""; + for (const [needle, name] of [["Windows", "Windows"], ["Mac", "Mac"], + ["iPhone", "iPhone"], ["iPad", "iPad"], + ["Android", "Android"], ["Linux", "Linux"]]) { + if (agent.includes(needle)) return `${name} passkey`; + } + return "Passkey"; +} + +function badge(text) { + const span = document.createElement("span"); + span.className = "badge"; + span.textContent = text; + return span; +} + +function field(type, placeholder) { + const input = document.createElement("input"); + input.type = type; + input.placeholder = placeholder; + input.setAttribute("aria-label", placeholder); + input.required = true; + input.autocomplete = type === "password" ? "new-password" : "off"; + return input; +} + +function count(n, thing) { + return `${n} ${thing}${n === 1 ? "" : "s"}`; +} + +/** An ISO stamp as something to read. Local time, because the person reading + * it is at a machine and the server's UTC is nobody's afternoon. */ +function when(stamp) { + if (!stamp) return "never"; + const at = new Date(stamp); + if (Number.isNaN(at.getTime())) return stamp; + return at.toLocaleString(); +} + +if (!customElements.get("auth-account")) { + customElements.define("auth-account", AuthAccount); +} + +export { AuthAccount }; diff --git a/auth/static/passkey.js b/auth/static/passkey.js new file mode 100644 index 0000000..5a58e31 --- /dev/null +++ b/auth/static/passkey.js @@ -0,0 +1,100 @@ +// The browser half of WebAuthn: two ceremonies and the byte-shuffling they need. +// +// Shared by the login page and the account element, which is why it is a file +// rather than inline. Both need exactly these conversions, and getting them +// subtly wrong in one of two copies is the kind of bug that only shows up on +// the device you do not have. +// +// Not a module, and on `window` rather than exported: the login page is plain +// HTML served before any of the app's own scripts, and a classic script is the +// one shape both it and the element can load. +// +// ## Why the conversions exist at all +// +// WebAuthn's JavaScript API speaks ArrayBuffer; its JSON encoding speaks +// base64url. So every challenge and credential id has to be decoded on the way +// into `navigator.credentials` and encoded on the way back out. Newer browsers +// offer `PublicKeyCredential.parseRequestOptionsFromJSON()` and a `toJSON()` on +// the result, which do this for you -- deliberately not used here, because +// feature-detecting them means two code paths through the one part of this that +// is fiddly, and the ten lines below work everywhere without a branch. +// +// The server never sees an ArrayBuffer and never sends one: `options_to_json` +// on one side and these helpers on the other are the whole contract. + +const bufFrom = (s) => + Uint8Array.from(atob(s.replace(/-/g, '+').replace(/_/g, '/')), + c => c.charCodeAt(0)).buffer; + +const b64From = (b) => + btoa(String.fromCharCode(...new Uint8Array(b))) + .replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); + +// `supported` is asked before any button is drawn. `isSecureContext` is the +// browser's own answer to "will WebAuthn run here at all" -- false on plain +// HTTP anywhere but localhost -- and asking it here means the page can say so +// rather than letting `credentials.get()` reject with a DOMException that reads +// like a bug. +const supported = () => + Boolean(window.PublicKeyCredential) && window.isSecureContext; + +async function create(options) { + options.challenge = bufFrom(options.challenge); + options.user.id = bufFrom(options.user.id); + (options.excludeCredentials || []).forEach(c => { c.id = bufFrom(c.id); }); + + const credential = await navigator.credentials.create({publicKey: options}); + return { + id: credential.id, + rawId: b64From(credential.rawId), + type: credential.type, + response: { + clientDataJSON: b64From(credential.response.clientDataJSON), + attestationObject: b64From(credential.response.attestationObject), + // Advisory, and absent on older authenticators -- it is what lets the + // account screen say "usb, nfc" beside a key instead of just a name. + transports: credential.response.getTransports + ? credential.response.getTransports() : [], + }, + clientExtensionResults: {}, + }; +} + +async function get(options) { + options.challenge = bufFrom(options.challenge); + (options.allowCredentials || []).forEach(c => { c.id = bufFrom(c.id); }); + + const credential = await navigator.credentials.get({publicKey: options}); + return { + id: credential.id, + rawId: b64From(credential.rawId), + type: credential.type, + response: { + clientDataJSON: b64From(credential.response.clientDataJSON), + authenticatorData: b64From(credential.response.authenticatorData), + signature: b64From(credential.response.signature), + userHandle: credential.response.userHandle + ? b64From(credential.response.userHandle) : null, + }, + clientExtensionResults: {}, + }; +} + +// What a cancelled ceremony should say. Pressing Escape, unplugging the key, or +// letting the prompt time out all arrive as exceptions, and the raw +// DOMException text ("The operation either timed out or was not allowed...") +// reads as a failure of the application rather than as "you cancelled it". +function explain(err) { + if (err && (err.name === 'NotAllowedError' || err.name === 'AbortError')) { + return 'Cancelled - no passkey was used.'; + } + if (err && err.name === 'InvalidStateError') { + return 'That authenticator already has a passkey for this app.'; + } + if (err && err.name === 'SecurityError') { + return 'This page is not a secure origin, so the browser will not use a passkey here.'; + } + return String(err && err.message ? err.message : err).replace('Error: ', ''); +} + +window.passkey = {supported, create, get, explain}; diff --git a/auth/store.py b/auth/store.py new file mode 100644 index 0000000..e65ea57 --- /dev/null +++ b/auth/store.py @@ -0,0 +1,505 @@ +"""One password, and the sessions it opens. Nothing else knows a hash. + +Four tables in whatever SQLite file it is given — the app's own, normally, so +there is one file to back up and SQLite's writer lock keeps auth honest with +everything else in it. Every name starts `auth_`, because the file is shared: +a `sessions` table is a name an app is entitled to want for itself. + +Four decisions worth arguing for, because each of them is the kind of thing +that gets "simplified" later by someone who does not know why it is here. + +**The password is never stored, and neither is anything reversible.** What goes +in the file is `scrypt(password, salt)` and the salt. scrypt is in the standard +library, so this costs no dependency, and it is *memory*-hard rather than +merely slow — the parameters below make one guess cost 16 MB, which is what +takes a stolen database file out of range of a GPU. A plain SHA-256 of the +password would fit in the same column and be worth nothing. + +**A session token is random, not signed.** The alternative — a signed cookie +carrying the claim, JWT-shaped — needs a signing key that has to be generated, +stored, kept out of logs and backups, and rotated, and it buys statelessness +that a single-user app on one SQLite file has no use for. Here the cookie is 32 +random bytes and the database holds only its SHA-256, so: + +- the file cannot be read to forge a session, the same way it cannot be read to + recover the password; +- signing out everywhere is one `DELETE`, which no signed token can offer + without keeping exactly the server-side state this already has; +- there is no key to leak. + +Hashing the token before storing it is the same move as hashing the password +and for the same reason. It is only SHA-256 rather than scrypt because the +input is already 32 bytes of `secrets`, so there is no guessing to slow down. + +**Failed attempts are counted in the file, not in memory.** A counter on the +process resets when the process does, which on a server that reloads or +crash-loops is a free reset for whoever is guessing. See `check_password`. + +**Passkeys sit alongside all of that rather than replacing it.** What is stored +for one is a *public* key, which is the whole point — there is no secret here +to protect, because the private half never leaves the authenticator. The +ceremony that uses these rows is `auth/passkeys.py`; this module only keeps +them. +""" + +from __future__ import annotations + +import hashlib +import hmac +import math +import os +import secrets +import sqlite3 +from contextlib import closing +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from auth.models import Passkey, SessionInfo + +#: scrypt cost. `n` is the work factor, and 2**14 with `r=8` is the widely +#: recommended interactive setting: about 16 MB and ~50 ms per guess, which is +#: imperceptible on a login and ruinous on a dictionary. Raising `n` later is +#: safe — the parameters are not stored, so a change simply invalidates the +#: existing hash and `set_password` writes one at the new cost. Which is worth +#: knowing before raising it: it means the current password stops working and +#: has to be set again. +_N, _R, _P = 2**14, 8, 1 +_DKLEN = 32 + +#: scrypt's own guard against a caller asking for more memory than it meant to. +#: The default ceiling is below what `_N` above needs, so it has to be raised +#: here or every derivation is a `ValueError` — and the failure would be at the +#: door, on the first login, in production. +_MAXMEM = 64 * 1024 * 1024 + +#: How many wrong answers before the door slows down, and how slow it gets. +#: Five is past a typo and a stale password manager entry, and short of anything +#: systematic. The delay then doubles per failure to a quarter of an hour — +#: enough that guessing is pointless, little enough that locking yourself out on +#: a Sunday is not a crisis. +_GRACE = 5 +_FIRST_LOCK = timedelta(seconds=30) +_MAX_LOCK = timedelta(minutes=15) + +#: How many doublings can still change the answer. Derived from the two above +#: rather than written down, so re-tuning either of them cannot silently move +#: the ceiling. See `_lockout` for what goes wrong without it. +_MAX_STEPS = math.ceil(math.log2(_MAX_LOCK / _FIRST_LOCK)) + +#: How stale `last_seen_at` may get before a request bothers to update it. +#: Without this, every single request — every socket frame's sibling fetch, +#: every poll of a page left open — takes SQLite's write lock to move a +#: timestamp nobody reads more precisely than "today", and contends with a +#: worker trying to write actual data. Five minutes is far finer than the +#: account screen's "last seen" line can usefully show. +_SEEN_EVERY = timedelta(minutes=5) + +#: The tables. Created on construction rather than by a migration step, because +#: the only schema change this has ever had is "it did not exist yet", and a +#: migration runner for four `CREATE TABLE IF NOT EXISTS` is a thing to +#: maintain in exchange for nothing. +SCHEMA = """ +CREATE TABLE IF NOT EXISTS auth_credential ( + id INTEGER PRIMARY KEY CHECK (id = 1), + salt BLOB NOT NULL, + hash BLOB NOT NULL, + updated_at TEXT NOT NULL, + failures INTEGER NOT NULL DEFAULT 0, + locked_until TEXT NOT NULL DEFAULT '' +); +CREATE TABLE IF NOT EXISTS auth_sessions ( + id INTEGER PRIMARY KEY, + token_hash TEXT NOT NULL UNIQUE, + label TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + expires_at TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS auth_passkeys ( + id INTEGER PRIMARY KEY, + credential_id BLOB NOT NULL UNIQUE, + public_key BLOB NOT NULL, + sign_count INTEGER NOT NULL DEFAULT 0, + transports TEXT NOT NULL DEFAULT '', + label TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + last_used_at TEXT NOT NULL DEFAULT '' +); +CREATE TABLE IF NOT EXISTS auth_challenges ( + challenge BLOB PRIMARY KEY, + purpose TEXT NOT NULL, + expires_at TEXT NOT NULL +); +""" + + +def _now() -> datetime: + return datetime.now(UTC) + + +def _stamp(when: datetime) -> str: + return when.isoformat() + + +def _token_hash(token: str) -> str: + return hashlib.sha256(token.encode()).hexdigest() + + +def _derive(password: str, salt: bytes) -> bytes: + return hashlib.scrypt(password.encode(), salt=salt, n=_N, r=_R, p=_P, + dklen=_DKLEN, maxmem=_MAXMEM) + + +class AuthStore: + """The four `auth_*` tables, opened on the path it is handed. + + store = AuthStore("data/app.db") + + Its own connections, opened per call and closed after, rather than one held + open: SQLite in WAL mode is happy with that, a connection per call cannot be + shared across threads by accident, and the middleware runs this on a worker + thread from an async request. + """ + + def __init__(self, path: str | os.PathLike[str]) -> None: + self.path = Path(path) + self.path.parent.mkdir(parents=True, exist_ok=True) + with closing(self._connect()) as conn, conn: + conn.executescript(SCHEMA) + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.path, timeout=10) + conn.row_factory = sqlite3.Row + # WAL so a login is not blocked by whatever else is writing to this + # file — the job queue, most likely, which writes constantly. + conn.execute("PRAGMA journal_mode=WAL") + return conn + + # -------------------------------------------------------------- password + + def has_password(self) -> bool: + """Whether this database has ever had one set. False on a fresh file, + which is what makes the app print a generated one at startup.""" + with closing(self._connect()) as conn: + return conn.execute( + "SELECT 1 FROM auth_credential WHERE id=1").fetchone() is not None + + def set_password(self, password: str) -> None: + """Set or replace it. A fresh salt every time, so setting the same + password twice does not produce the same row — which is what stops the + file from answering "did it change?" to anyone who kept a copy. + + Deliberately does not close existing sessions. Whether a password + change should sign your other devices out is a policy question with a + real answer either way, and it belongs to the caller that knows why the + password is changing; `auth/router.py` decides. + """ + salt = secrets.token_bytes(16) + with closing(self._connect()) as conn, conn: + conn.execute( + """INSERT INTO auth_credential + (id, salt, hash, updated_at, failures, locked_until) + VALUES (1, ?, ?, ?, 0, '') + ON CONFLICT(id) DO UPDATE SET + salt=excluded.salt, hash=excluded.hash, + updated_at=excluded.updated_at, failures=0, locked_until=''""", + (salt, _derive(password, salt), _stamp(_now())), + ) + + def locked_for(self) -> int: + """Seconds until the next attempt will be looked at, or 0. + + Read before checking rather than after, so a locked-out caller is told + to wait instead of having a guess counted — otherwise a fast enough + attacker extends their own lockout forever and never notices, and, more + to the point, an honest person hammering the button digs a deeper hole + with every press. + """ + with closing(self._connect()) as conn: + row = conn.execute( + "SELECT locked_until FROM auth_credential WHERE id=1").fetchone() + if row is None or not row["locked_until"]: + return 0 + remaining = datetime.fromisoformat(row["locked_until"]) - _now() + return max(0, int(remaining.total_seconds() + 0.999)) + + def check_password(self, password: str) -> bool: + """Right or wrong, and record which. + + `hmac.compare_digest` rather than `==`, so the comparison takes the same + time whatever the first wrong byte is. That is close to superstition at + this scale — the network jitter in front of it is larger than the signal + — but it is one call, and the habit is worth more than the microseconds. + + Note that a wrong answer still costs a full scrypt derivation, including + on a database with no password set at all. That is the point: the delay + an attacker meets is the same one the honest user meets, and it is the + reason `_GRACE` can be as generous as it is. + + **There is one counter, not one per caller, and that is on purpose.** + The obvious improvement is to lock out the address that is guessing + instead of the password, so that a stranger cannot keep the owner + waiting at their own door. It cannot be built honestly here. The only + address this app can see is the one its proxy puts in a header — the + peer is a container on the same Docker network — and a header is set by + whoever is talking. Keyed on that, an attacker gets a fresh, empty + counter per request by changing one line of their own request, while the + owner still gets locked out by their own typos: strictly worse than + this, and it would look like protection. So the address is written to + the log instead, at WARNING, where fail2ban can act on it at the layer + that actually knows who connected — see `_warn` in `auth/router.py`. + What is left for this counter to do is make guessing pointless, which it + does whoever is guessing. + """ + with closing(self._connect()) as conn: + row = conn.execute( + "SELECT salt, hash, failures FROM auth_credential WHERE id=1" + ).fetchone() + if row is None: + _derive(password, b"no password here") + return False + ok = hmac.compare_digest(_derive(password, row["salt"]), row["hash"]) + with conn: + if ok: + conn.execute( + "UPDATE auth_credential SET failures=0, locked_until='' WHERE id=1") + else: + failures = int(row["failures"]) + 1 + conn.execute( + "UPDATE auth_credential SET failures=?, locked_until=? WHERE id=1", + (failures, _stamp(_now() + _lockout(failures)) + if failures >= _GRACE else ""), + ) + return ok + + def changed_at(self) -> str: + """When the password was last set, for the account screen to show.""" + with closing(self._connect()) as conn: + row = conn.execute( + "SELECT updated_at FROM auth_credential WHERE id=1").fetchone() + return row["updated_at"] if row else "" + + # -------------------------------------------------------------- sessions + + def open_session(self, label: str, days: int) -> str: + """A new session. Returns the raw token — the only time it exists. + + `token_urlsafe(32)` is 256 bits from the OS CSPRNG, which is past + anybody's ability to guess and short enough to sit in a cookie. + """ + token = secrets.token_urlsafe(32) + started = _now() + with closing(self._connect()) as conn, conn: + conn.execute( + """INSERT INTO auth_sessions + (token_hash, label, created_at, last_seen_at, expires_at) + VALUES (?,?,?,?,?)""", + (_token_hash(token), label[:200], _stamp(started), _stamp(started), + _stamp(started + timedelta(days=days))), + ) + return token + + def touch(self, token: str) -> SessionInfo | None: + """The session this token names, if it is live, marking it seen. + + Expiry is enforced in the `WHERE`, so a session that has run out is + never returned even though its row is still there — the row is cleared + by `purge`, and until then it is simply not a session. + + `last_seen_at` is what makes the sessions list worth reading: without it + every row says the same thing, and you cannot tell the laptop you use + daily from the phone you signed in on once. It is written at most once + every `_SEEN_EVERY` — this runs on every request there is, and a + timestamp is not worth the write lock. + """ + digest = _token_hash(token) + now = _now() + stamp = _stamp(now) + with closing(self._connect()) as conn: + row = conn.execute( + "SELECT * FROM auth_sessions WHERE token_hash=? AND expires_at > ?", + (digest, stamp), + ).fetchone() + if row is None: + return None + seen = row["last_seen_at"] + if now - datetime.fromisoformat(seen) >= _SEEN_EVERY: + with conn: + conn.execute( + "UPDATE auth_sessions SET last_seen_at=? WHERE token_hash=?", + (stamp, digest)) + seen = stamp + return _session(row, seen, current=True) + + def sessions(self, current: str = "") -> list[SessionInfo]: + """Every live session, most recently seen first, with this one flagged.""" + digest = _token_hash(current) if current else "" + stamp = _stamp(_now()) + with closing(self._connect()) as conn: + rows = conn.execute( + """SELECT * FROM auth_sessions WHERE expires_at > ? + ORDER BY last_seen_at DESC, id DESC""", (stamp,)).fetchall() + return [_session(r, r["last_seen_at"], current=r["token_hash"] == digest) + for r in rows] + + def close_session(self, token: str) -> None: + with closing(self._connect()) as conn, conn: + conn.execute("DELETE FROM auth_sessions WHERE token_hash=?", + (_token_hash(token),)) + + def close_by_id(self, session_id: int) -> bool: + """Revoke one session from the account screen. By row id rather than by + token, because the screen has never seen a token and must not.""" + with closing(self._connect()) as conn, conn: + cur = conn.execute("DELETE FROM auth_sessions WHERE id=?", (session_id,)) + return cur.rowcount > 0 + + def close_all(self, keep: str = "") -> int: + """Sign out everywhere, optionally sparing the caller's own session. + + The whole reason sessions are rows rather than signed tokens. Returns + how many went, so the UI can say. + """ + with closing(self._connect()) as conn, conn: + if keep: + cur = conn.execute("DELETE FROM auth_sessions WHERE token_hash != ?", + (_token_hash(keep),)) + else: + cur = conn.execute("DELETE FROM auth_sessions") + return cur.rowcount + + # -------------------------------------------------------------- passkeys + + def add_passkey(self, credential_id: bytes, public_key: bytes, + sign_count: int, transports: str, label: str) -> int: + """Register one. Returns its row id. + + `credential_id` is unique, so registering the same authenticator twice + is an error rather than a second row that would make "revoke this one" + ambiguous. The registration ceremony avoids reaching this by handing the + browser an `excludeCredentials` list — see `passkeys.register_options`. + """ + with closing(self._connect()) as conn, conn: + cur = conn.execute( + """INSERT INTO auth_passkeys + (credential_id, public_key, sign_count, transports, label, created_at) + VALUES (?,?,?,?,?,?)""", + (credential_id, public_key, sign_count, transports, + label[:100], _stamp(_now())), + ) + return int(cur.lastrowid or 0) + + def passkeys(self) -> list[Passkey]: + """Every registered authenticator, newest first. No key material: this + is what the account screen lists, and it has no use for a public key.""" + with closing(self._connect()) as conn: + rows = conn.execute( + "SELECT * FROM auth_passkeys ORDER BY created_at DESC, id DESC").fetchall() + return [Passkey(id=r["id"], label=r["label"], created_at=r["created_at"], + last_used_at=r["last_used_at"], + transports=[t for t in r["transports"].split(",") if t]) + for r in rows] + + def passkey_ids(self) -> list[bytes]: + """Raw credential ids, for the `allowCredentials` and + `excludeCredentials` lists the two ceremonies hand the browser.""" + with closing(self._connect()) as conn: + return [r["credential_id"] for r in + conn.execute("SELECT credential_id FROM auth_passkeys")] + + def passkey_for(self, credential_id: bytes) -> tuple[int, bytes, int] | None: + """`(id, public_key, sign_count)` for the credential that just signed, + or None if this app has never seen it.""" + with closing(self._connect()) as conn: + row = conn.execute( + "SELECT id, public_key, sign_count FROM auth_passkeys WHERE credential_id=?", + (credential_id,), + ).fetchone() + if row is None: + return None + return row["id"], row["public_key"], int(row["sign_count"]) + + def record_passkey_use(self, passkey_id: int, sign_count: int) -> None: + with closing(self._connect()) as conn, conn: + conn.execute( + "UPDATE auth_passkeys SET sign_count=?, last_used_at=? WHERE id=?", + (sign_count, _stamp(_now()), passkey_id)) + + def rename_passkey(self, passkey_id: int, label: str) -> bool: + with closing(self._connect()) as conn, conn: + cur = conn.execute("UPDATE auth_passkeys SET label=? WHERE id=?", + (label[:100], passkey_id)) + return cur.rowcount > 0 + + def delete_passkey(self, passkey_id: int) -> bool: + with closing(self._connect()) as conn, conn: + cur = conn.execute("DELETE FROM auth_passkeys WHERE id=?", (passkey_id,)) + return cur.rowcount > 0 + + # ------------------------------------------------------------ challenges + + def remember_challenge(self, challenge: bytes, purpose: str, + seconds: int = 300) -> None: + """Keep a handed-out challenge until it is answered or runs out. + + Five minutes is the browser's own ceremony timeout with room to spare; + a challenge outliving that is one nobody is going to answer. + """ + with closing(self._connect()) as conn, conn: + # Expired, unanswered ceremonies must not accumulate until restart: + # the endpoint that hands them out needs no session. + conn.execute("DELETE FROM auth_challenges WHERE expires_at <= ?", + (_stamp(_now()),)) + conn.execute( + "INSERT OR REPLACE INTO auth_challenges (challenge, purpose, expires_at) " + "VALUES (?,?,?)", + (challenge, purpose, _stamp(_now() + timedelta(seconds=seconds)))) + + def take_challenge(self, challenge: bytes, purpose: str) -> bool: + """Spend it. True if it was outstanding for this purpose, and it is now + gone whether it was or not. + + Single-use is the property that matters: without the delete, a captured + response could be replayed for as long as the challenge lived. The + `purpose` is checked too, so a challenge issued for registering cannot + be answered with a login. + """ + with closing(self._connect()) as conn, conn: + cur = conn.execute( + "DELETE FROM auth_challenges WHERE challenge=? AND purpose=? " + "AND expires_at > ?", + (challenge, purpose, _stamp(_now()))) + return cur.rowcount > 0 + + def purge(self) -> int: + """Drop sessions and challenges that have expired. Nothing depends on + this — `touch`, `sessions` and `take_challenge` all filter on expiry — + so it is housekeeping, run at startup, not a security boundary.""" + stamp = _stamp(_now()) + with closing(self._connect()) as conn, conn: + cur = conn.execute("DELETE FROM auth_sessions WHERE expires_at <= ?", (stamp,)) + gone = cur.rowcount + conn.execute("DELETE FROM auth_challenges WHERE expires_at <= ?", (stamp,)) + return gone + + +def _lockout(failures: int) -> timedelta: + """30s on the fifth wrong answer, doubling, capped at a quarter hour. + + The exponent is capped, not only the delay it produces. `2 ** steps` is an + ordinary Python int and will keep doubling happily past anything a + `timedelta` can hold, so a few dozen wrong answers in a row used to end in + an OverflowError raised out of `check_password` — which is a 500 on every + login attempt until a right password lands, from an attacker who only ever + sent wrong ones. Once the cap has won, further doublings are arithmetic + nobody reads. + """ + steps = min(failures - _GRACE, _MAX_STEPS) + return min(_FIRST_LOCK * (2 ** steps), _MAX_LOCK) + + +def _session(row: sqlite3.Row, last_seen: str, current: bool) -> SessionInfo: + return SessionInfo( + id=row["id"], label=row["label"], created_at=row["created_at"], + last_seen_at=last_seen, expires_at=row["expires_at"], current=current, + ) diff --git a/auth/templates/login.html b/auth/templates/login.html new file mode 100644 index 0000000..fe28318 --- /dev/null +++ b/auth/templates/login.html @@ -0,0 +1,174 @@ + + + + + +{name} — sign in + +{stylesheet} + + + + + +
+
{name}
+ +
+ + + + + + + +
+ + + + + +
+ + + + + diff --git a/auth/tests/__init__.py b/auth/tests/__init__.py new file mode 100644 index 0000000..5e7d368 --- /dev/null +++ b/auth/tests/__init__.py @@ -0,0 +1,20 @@ +"""A package, so `from auth.tests import PASSWORD` works wherever pytest is run +from, and the two lines every module here needs live in one place rather than +in an importable `conftest`. + +Excluded from the wheel by `pyproject.toml`: a module folder's tests travel +with the folder when it is copied into a project, not into an installed app. +""" + +from __future__ import annotations + +from typing import Any + +#: Long enough to be a real password and fixed so a failure names it. +PASSWORD = "correct-horse-battery" + + +def sign_in(client: Any, password: str = PASSWORD, **body: object) -> Any: + """Through the front door, which is a POST and not a form submit: the login + page is the only page here and it talks to this endpoint with `fetch`.""" + return client.post("/login", json={"password": password, **body}) diff --git a/auth/tests/conftest.py b/auth/tests/conftest.py new file mode 100644 index 0000000..7969fe3 --- /dev/null +++ b/auth/tests/conftest.py @@ -0,0 +1,79 @@ +"""The fixtures: an app that wants a password, and the same one that does not. + +The interesting question is asked twice. `locked` is an app that was told it +binds a real address and therefore wants a password; `open_app` is the same +code on loopback, which is what an app on a laptop is. The app itself is +`harness.py`, built here rather than imported, because this folder is not +allowed to reach for `app/`. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from auth import AuthStore +from auth.tests import PASSWORD, sign_in +from auth.tests.harness import build + + +@pytest.fixture(autouse=True) +def no_pinned_password(monkeypatch: pytest.MonkeyPatch) -> None: + """The variable pins the password while it is set, which would quietly + change what half of these tests are testing. A test that wants it sets it.""" + monkeypatch.delenv("APP_PASSWORD", raising=False) + + +@pytest.fixture +def make_client(tmp_path: Path) -> Iterator[Callable[..., TestClient]]: + """`make_client(enabled="always")`: an app, its store seeded, a client on it. + + A factory rather than a fixture per shape, because half the tests here are + about one setting being different, and a fixture per setting is a file of + fixtures nobody can hold in their head. + """ + clients: list[TestClient] = [] + + def make(*, host: str = "0.0.0.0", password: str | None = PASSWORD, + base_url: str = "http://testserver", **settings: object) -> TestClient: + db = tmp_path / f"auth{len(clients)}.db" + if password is not None: + AuthStore(db).set_password(password) + client = TestClient(build(db, host=host, **settings), + follow_redirects=False, base_url=base_url) + clients.append(client) + # Entered, so the lifespan runs and the bootstrap with it. Closed at + # the end of the test, which is what releases the SQLite file on + # Windows before pytest tries to delete the temp directory. + return client.__enter__() + + yield make + for client in reversed(clients): + client.__exit__(None, None, None) + + +@pytest.fixture +def store(tmp_path: Path) -> AuthStore: + return AuthStore(tmp_path / "store.db") + + +@pytest.fixture +def locked(make_client: Callable[..., TestClient]) -> TestClient: + """An app that wants a password, with one already set.""" + return make_client(enabled="always") + + +@pytest.fixture +def open_app(make_client: Callable[..., TestClient]) -> TestClient: + """The same app on loopback under `auto` — no password asked for.""" + return make_client(enabled="auto", host="127.0.0.1", password=None) + + +@pytest.fixture +def signed_in(locked: TestClient) -> TestClient: + """The locked app, already through the door.""" + assert sign_in(locked).status_code == 200 + return locked diff --git a/auth/tests/harness.py b/auth/tests/harness.py new file mode 100644 index 0000000..ba461d0 --- /dev/null +++ b/auth/tests/harness.py @@ -0,0 +1,68 @@ +"""An app to lock, built here rather than imported. + +`auth/` has to survive being copied into a project that has never heard of this +one, so its tests are not allowed to reach for `app/` or `pipeline/` either -- +`tests/test_coupling.py` reads the imports of every file under this folder and +says so. What they need is a FastAPI app with a page, an API route, a socket, +and a couple of neighbours to prove the open list is matched the way it claims. +That is this file, and it is nine routes. + +A module rather than a conftest fixture because two tests build the same +database twice, to see what a restart does, and a fixture cannot be asked for +twice in one test. +""" + +from __future__ import annotations + +import os + +from fastapi import FastAPI, WebSocket +from fastapi.responses import PlainTextResponse + +from auth import AuthConfig, AuthStore, connect + + +def build(db: str | os.PathLike[str], *, host: str = "0.0.0.0", + **settings: object) -> FastAPI: + """One app, locked or not depending on what it is told the server binds.""" + app = FastAPI() + connect(app, AuthStore(db), AuthConfig(**settings), host=host, + open=("/static/",), stylesheet="/static/app.css") + + @app.get("/", response_class=PlainTextResponse) + def front() -> str: + return "the front page" + + @app.get("/deep/page", response_class=PlainTextResponse) + def deep() -> str: + return "a page someone bookmarked" + + @app.get("/api/thing") + def thing() -> dict[str, bool]: + return {"ok": True} + + @app.get("/health") + def health() -> dict[str, bool]: + return {"ok": True} + + # The neighbours, so "matched whole, not as a prefix" is proved against + # routes that exist rather than against two 404s. + @app.get("/healthz") + def healthz() -> dict[str, bool]: + return {"ok": True} + + @app.get("/loginx", response_class=PlainTextResponse) + def loginx() -> str: + return "not the login page" + + @app.get("/static/app.css", response_class=PlainTextResponse) + def stylesheet() -> str: + return "body { }" + + @app.websocket("/ws/jobs") + async def socket(ws: WebSocket) -> None: + await ws.accept() + await ws.send_json({"type": "jobs"}) + await ws.close() + + return app diff --git a/auth/tests/softkey.py b/auth/tests/softkey.py new file mode 100644 index 0000000..45870a7 --- /dev/null +++ b/auth/tests/softkey.py @@ -0,0 +1,169 @@ +"""A WebAuthn authenticator in software, so the ceremony can be tested. + +Passkeys cannot be exercised the way the rest of this package can. There is no +HTTP call to fake: the interesting half runs inside a browser, talking to a +secure enclave or a stick in a USB port, and neither is available to pytest. +The usual answer is to test the two endpoints for their status codes, assert +nothing about what they actually verify, and find out whether any of it works +after deploying. That is not a test, and this is the alternative. + +`SoftKey` is a real authenticator that happens to live in this file. It holds a +P-256 key, and it produces exactly the bytes a hardware key produces -- the same +CBOR attestation object, the same authenticator data with the same flags and +counter, the same ECDSA signature over the same concatenation. `py_webauthn` +verifies it because there is nothing to tell apart: the library's checks are +cryptographic, and this satisfies them by doing the cryptography. + +What that buys is that every server-side branch is covered for real. A wrong +origin fails because the signature covers the origin. A replayed response fails +because the challenge is gone. A cloned key is caught because the counter went +backwards. None of those are simulated. + +What it cannot cover is the browser's own half: that it refuses insecure +origins, that it will not hand a credential to the wrong site, that the user was +actually asked. Those are the browser's promises, they are the reason passkeys +are worth having, and they are not ours to test. + +`cbor2` and `cryptography` are `webauthn`'s own dependencies, so this file +costs the project nothing it did not already have with the extra installed -- +and nothing at all without it, since the module that imports this one skips +first. + +Reference: WebAuthn Level 3, section 6.1 (authenticator data) and 6.5.4 +(packed/none attestation). The layouts below are that spec and nothing else. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import struct +from base64 import urlsafe_b64encode +from typing import Any + +import cbor2 +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ec + +#: Flags byte of authenticator data. UP is "a user was present" (they touched +#: it), UV is "a user was verified" (PIN or fingerprint), AT is "attested +#: credential data follows", which is set on registration and never after. +UP, UV, AT = 0x01, 0x04, 0x40 + + +def b64(raw: bytes) -> str: + """base64url, unpadded -- what WebAuthn puts every byte string in.""" + return urlsafe_b64encode(raw).decode().rstrip("=") + + +class SoftKey: + """One authenticator holding one credential. + + Construct it, `register()` against a challenge, then `sign()` challenges + with it. `sign_count` increments the way a security key's does; pass + `counts=False` to imitate a platform passkey, which is the case that makes + the clone check inapplicable. + """ + + def __init__(self, *, sign_count: int = 0, counts: bool = True) -> None: + self.key = ec.generate_private_key(ec.SECP256R1()) + self.credential_id = os.urandom(32) + self.sign_count = sign_count + #: Whether the counter moves. A real security key's does; most platform + #: passkeys report a constant 0 and opt out of clone detection. + self.counts = counts + + # ------------------------------------------------------------------ keys + + def cose_key(self) -> bytes: + """The public key as a COSE_Key map -- the format `authData` carries. + + `1: 2` is key type EC2, `3: -7` is ES256, and `-1: 1` is the P-256 + curve; `-2`/`-3` are the affine coordinates, each padded to exactly 32 + bytes because a leading zero byte is significant here and `int.to_bytes` + is the only thing that reliably keeps it. + """ + numbers = self.key.public_key().public_numbers() + return cbor2.dumps({ + 1: 2, 3: -7, -1: 1, + -2: numbers.x.to_bytes(32, "big"), + -3: numbers.y.to_bytes(32, "big"), + }) + + # --------------------------------------------------------- the ceremonies + + def register(self, challenge: bytes, rp_id: str, origin: str, + transports: list[str] | None = None) -> dict[str, Any]: + """Answer a creation challenge. Returns what `create().toJSON()` would.""" + auth_data = ( + hashlib.sha256(rp_id.encode()).digest() # rpIdHash + + bytes([UP | UV | AT]) # flags + + struct.pack(">I", self.sign_count) # signCount, big-endian + + b"\x00" * 16 # AAGUID: none, per `none` + + struct.pack(">H", len(self.credential_id)) + + self.credential_id + + self.cose_key() + ) + attestation = cbor2.dumps( + {"fmt": "none", "attStmt": {}, "authData": auth_data}) + return { + "id": b64(self.credential_id), + "rawId": b64(self.credential_id), + "type": "public-key", + "response": { + "clientDataJSON": b64(self._client_data("webauthn.create", + challenge, origin)), + "attestationObject": b64(attestation), + "transports": transports or ["internal"], + }, + "clientExtensionResults": {}, + } + + def sign(self, challenge: bytes, rp_id: str, origin: str) -> dict[str, Any]: + """Answer an authentication challenge. `get().toJSON()`'s shape. + + The signature covers `authenticatorData || SHA-256(clientDataJSON)`, + which is what binds one signature to one challenge *and* one origin -- + change either and the same key produces a signature that will not + verify. That is the property the tests lean on. + """ + if self.counts: + self.sign_count += 1 + auth_data = ( + hashlib.sha256(rp_id.encode()).digest() + + bytes([UP | UV]) # no AT: not a registration + + struct.pack(">I", self.sign_count) + ) + client_data = self._client_data("webauthn.get", challenge, origin) + signature = self.key.sign( + auth_data + hashlib.sha256(client_data).digest(), + ec.ECDSA(hashes.SHA256()), + ) + return { + "id": b64(self.credential_id), + "rawId": b64(self.credential_id), + "type": "public-key", + "response": { + "clientDataJSON": b64(client_data), + "authenticatorData": b64(auth_data), + "signature": b64(signature), + "userHandle": b64(b"app"), + }, + "clientExtensionResults": {}, + } + + @staticmethod + def _client_data(ceremony: str, challenge: bytes, origin: str) -> bytes: + """What the *browser* would assemble and the authenticator signs over. + + `crossOrigin` is included because real browsers include it, and the + point of this class is to produce bytes that are not distinguishable + from a real one's. + """ + return json.dumps({ + "type": ceremony, + "challenge": b64(challenge), + "origin": origin, + "crossOrigin": False, + }).encode() diff --git a/auth/tests/test_door.py b/auth/tests/test_door.py new file mode 100644 index 0000000..69fa110 --- /dev/null +++ b/auth/tests/test_door.py @@ -0,0 +1,489 @@ +"""The door: who gets past it, what a refusal looks like, and the cookie. + + uv run pytest auth/tests/test_door.py -q +""" + +from __future__ import annotations + +import logging +import sqlite3 +from collections.abc import Callable +from contextlib import closing +from urllib.parse import parse_qs, urlsplit + +import pytest +from fastapi.testclient import TestClient +from starlette.websockets import WebSocketDisconnect + +from auth import AuthStore +from auth.middleware import OPEN +from auth.tests import PASSWORD, sign_in +from auth.tests.harness import build + + +# ---------------------------------------------------------- the open door + +def test_loopback_under_auto_needs_no_password(open_app: TestClient): + """The default on a laptop: the machine's own login is already the door.""" + assert open_app.get("/").status_code == 200 + assert open_app.get("/api/thing").status_code == 200 + with open_app.websocket_connect("/ws/jobs") as socket: + assert socket.receive_json()["type"] == "jobs" + + +def test_the_account_says_it_is_open(open_app: TestClient): + assert open_app.get("/api/account").json()["locked"] is False + + +def test_the_login_page_bounces_when_there_is_no_door(open_app: TestClient): + """A form that would replace a session you do not need.""" + page = open_app.get("/login") + assert page.status_code == 303 + assert sign_in(open_app).status_code == 400 + + +def test_never_is_open_off_loopback_too(make_client: Callable[..., TestClient]): + """`never` means exactly what it says: something else is doing the + authentication, and this is not second-guessing it.""" + client = make_client(enabled="never", host="0.0.0.0") + assert client.get("/api/thing").status_code == 200 + + +# -------------------------------------------------------- the locked door + +def test_everything_is_shut(locked: TestClient): + """Deny by default: the API, the page, and the one worth naming -- + `/openapi.json` is every path, every payload shape and every field name of + the application, handed to whoever asks. It is a GET of something that is + not an API path, so the refusal is the redirect a browser wants.""" + assert locked.get("/api/thing").status_code == 401 + assert locked.get("/deep/page").status_code == 303 + assert locked.get("/openapi.json").status_code == 303 + + +def test_a_page_is_redirected_and_an_api_call_is_not(locked: TestClient): + """A browser gets sent to the form; a fetch gets a 401 it can act on -- + a login page arriving where JSON was asked for is a 200 full of HTML.""" + page = locked.get("/") + assert page.status_code == 303 + assert page.headers["location"] == "/login" + + refused = locked.get("/api/thing") + assert refused.status_code == 401 + assert refused.json() == {"detail": "Not signed in"} + assert "application/json" in refused.headers["content-type"] + + +def test_a_deep_link_survives_the_login_intact(locked: TestClient): + """`next` is carried in a query string, so a path holding a `&` or a `%` + has to be escaped on the way to the login page -- or it arrives truncated, + or as a different path than the one that was asked for.""" + sent = locked.get("/items/a&b%25c") + assert sent.status_code == 303 + where = sent.headers["location"] + assert where.startswith("/login?next=") + + # What `URLSearchParams` on the login page will read out of that query. + target = parse_qs(urlsplit(where).query)["next"][0] + assert target == "/items/a&b%c" + assert sign_in(locked, next=target).json()["next"] == "/items/a&b%c" + + +def test_the_open_list_is_the_four_doors_it_says_it_is(locked: TestClient): + """Deny-by-default is only as good as the list of exceptions is short, so + the list is asserted rather than trusted, and an entry has to be argued for + here before it ships. Four: `/login` and the passkey ceremony under + `/login/`, because there is no other way in; `/_auth/`, which is the two + scripts those pages are; and `/health`, because a proxy's probe has no + session to offer and a healthcheck that needs one is a healthcheck that + reports the login page.""" + assert OPEN == ("/login", "/login/", "/_auth/", "/health") + + +def test_an_open_name_is_matched_whole_and_a_slash_is_a_prefix(locked: TestClient): + """`/health` does not open `/healthz`, and `/login` does not open + `/loginx` -- both of which are real routes here, so this is a refusal and + not a 404 wearing one.""" + assert locked.get("/health").status_code == 200 + assert locked.get("/static/app.css").status_code == 200 + assert locked.get("/_auth/account.js").status_code == 200 + assert locked.post("/login/passkey/options").status_code != 401 + + assert locked.get("/healthz").status_code == 303 + assert locked.get("/loginx").status_code == 303 + + +def test_the_login_page_draws_no_passkey_button_without_a_passkey(locked: TestClient): + """Two halves, and the second one is why this test exists at all. + + The flag is substituted false, so the script never unhides the button -- + and the button is `hidden`, which is a browser default of `display: none` + that *any* class rule carrying a `display` overrides. `.orpass` sets + `display: grid`, so the first version of this page offered "Use a passkey" + on an app with no passkeys, and pressing it 404s. + """ + page = locked.get("/login").text + assert "const passkeysOffered = false;" in page + assert "[hidden] { display: none !important; }" in page + + +def test_a_query_string_cannot_widen_the_open_list(locked: TestClient): + assert locked.get("/api/thing?next=/health").status_code == 401 + + +def test_the_websocket_is_shut_too(locked: TestClient): + """The reason this is a middleware and not a dependency. A route + dependency would not cover the socket, and the socket is where an app + streams the state of everything it is doing.""" + with pytest.raises(WebSocketDisconnect), locked.websocket_connect("/ws/jobs"): + pass + + +def test_the_websocket_opens_with_a_session(locked: TestClient): + sign_in(locked) + with locked.websocket_connect("/ws/jobs") as socket: + assert socket.receive_json()["type"] == "jobs" + + +def test_a_wrong_password_is_refused(locked: TestClient): + assert sign_in(locked, "nope").status_code == 401 + assert locked.get("/api/thing").status_code == 401 + + +def test_signing_in_opens_everything(locked: TestClient): + assert sign_in(locked).status_code == 200 + assert locked.get("/api/thing").status_code == 200 + assert locked.get("/").status_code == 200 + + +def test_logout_ends_the_session(signed_in: TestClient): + assert signed_in.post("/logout").status_code == 200 + assert signed_in.get("/api/thing").status_code == 401 + + +def test_next_stays_inside_the_app(locked: TestClient): + r"""An open redirect is a real login page, a real sign-in, and someone + else's site afterwards. + + The backslash is the one that gets through: a browser parses `/\evil.com` + as `//evil.com`, and the address bar hands `%5C` to the page already + decoded -- so neither is the local path a leading-slash test reads them as. + """ + for target in ("//evil.example.com/", "/\\evil.example.com", + "/%5Cevil.example.com", "https://evil.example.com", + "\\\\evil.example.com", "/\t/evil.example.com"): + assert sign_in(locked, next=target).json()["next"] == "/", target + assert sign_in(locked, next="/deep/page?x=1").json()["next"] == "/deep/page?x=1" + + +# ------------------------------------------------------------- the cookie + +def cookies(response) -> list[str]: + """The `Set-Cookie` headers, read off the wire. + + Off the wire and not out of the jar, because the jar is a browser and does + the very thing under test: httpx refuses to store a `__Host-` cookie that + arrived over plain HTTP, so asking it what it kept cannot tell a correct + header apart from one it threw away. + """ + return [value.lower() for value in response.headers.get_list("set-cookie")] + + +def test_the_cookie_is_httponly_lax_and_not_secure_over_http(locked: TestClient): + """An XSS must not be able to read it, another origin must not be able to + make the browser send it -- and a Secure cookie set over plain HTTP is one + the browser never sends back, which is an app nobody can sign in to.""" + (sent,) = cookies(sign_in(locked)) + assert sent.startswith("app_session=") + assert "httponly" in sent + assert "samesite=lax" in sent + assert "path=/" in sent + assert "secure" not in sent + assert "__host-" not in sent + + +def test_secure_cookie_always_sends_the_host_prefixed_name( + make_client: Callable[..., TestClient]): + """`secure_cookie: always` is the setting for a proxy that terminates TLS, + where the app sees plain HTTP and `auto` would read as "no". + + Both halves have to move together. `__Host-` is a promise to the browser + that the cookie is Secure, path `/` and carries no Domain, and a browser + silently drops one that is not -- which presents as a login that appears to + work and lands you back on the login page, with nothing in any log. + """ + client = make_client(enabled="always", secure_cookie="always") + (sent,) = cookies(sign_in(client)) + assert sent.startswith("__host-app_session=") + assert "secure" in sent + assert "path=/" in sent + assert "domain=" not in sent + + +def test_a_renamed_cookie_is_used_under_both_names( + make_client: Callable[..., TestClient]): + """The name is a setting because two apps from this boilerplate on one + hostname would otherwise share a cookie and sign each other out.""" + client = make_client(enabled="always", cookie="other_session") + (sent,) = cookies(sign_in(client)) + assert sent.startswith("other_session=") + assert client.get("/api/thing").status_code == 200 + assert {v.split("=")[0] for v in cookies(client.post("/logout"))} == { + "other_session", "__host-other_session"} + + +def test_logout_clears_both_names(signed_in: TestClient): + """Which name was set depends on `secure_cookie` and on the scheme the + login happened over, and neither is known here. Clearing only the one this + request would have set leaves the other in the browser, still offered on + every request -- a signed-out session the door goes on opening for.""" + cleared = signed_in.post("/logout") + sent = cookies(cleared) + assert {value.split("=")[0] for value in sent} == { + "app_session", "__host-app_session"} + for value in sent: + assert "max-age=0" in value or "expires=thu, 01 jan 1970" in value + + +# ------------------------------------------------------------ the account + +def test_the_account_lists_this_session(signed_in: TestClient): + account = signed_in.get("/api/account").json() + assert account["locked"] is True + assert account["pinned"] is False + assert sum(s["current"] for s in account["sessions"]) == 1 + assert account["password_set_at"] + + +def test_revoked_elsewhere_means_locked_out_here(signed_in: TestClient): + """The reason sessions are rows rather than signed tokens: a token that + carries its own claim cannot be taken back.""" + account = signed_in.get("/api/account").json() + mine = next(s["id"] for s in account["sessions"] if s["current"]) + assert signed_in.delete(f"/api/account/sessions/{mine}").status_code == 200 + assert signed_in.get("/api/thing").status_code == 401 + assert signed_in.delete(f"/api/account/sessions/{mine}").status_code == 401 + + +def test_closing_the_others_keeps_this_one(signed_in: TestClient): + other = TestClient(signed_in.app, follow_redirects=False) + sign_in(other) + assert other.get("/api/thing").status_code == 200 + + closed = signed_in.post("/api/account/sessions/close") + assert closed.json() == {"signed_out": 1} + assert signed_in.get("/api/thing").status_code == 200 + assert other.get("/api/thing").status_code == 401 + + +def test_changing_the_password_needs_the_old_one(signed_in: TestClient): + bad = signed_in.post("/api/account/password", + json={"current": "wrong", "new": "a-much-longer-one"}) + assert bad.status_code == 403 + + +def test_the_first_password_is_set_without_a_current_one(open_app: TestClient): + """The laptop before it is ever deployed: no password has ever been set, + and the account screen is where you set one. Asking for the current + password there is a field with no right answer, and the form could only + ever be refused -- which is what it did until this test existed. The app + that has a door never reaches this branch: `auth.bootstrap` gives a locked + app a password at startup, so there is always one to prove.""" + assert open_app.get("/api/account").json()["password_set_at"] == "" + set_it = open_app.post("/api/account/password", + json={"current": "", "new": "a-much-longer-one"}) + assert set_it.status_code == 200 + assert open_app.get("/api/account").json()["password_set_at"] + # And the second change asks for the first, the way it always did. + again = open_app.post("/api/account/password", + json={"current": "", "new": "another-long-one"}) + assert again.status_code == 403 + + +def test_a_short_new_password_is_refused(signed_in: TestClient): + short = signed_in.post("/api/account/password", + json={"current": PASSWORD, "new": "short"}) + assert short.status_code == 400 + assert "8 characters" in short.json()["detail"] + + +def test_changing_it_keeps_you_in_and_signs_others_out(signed_in: TestClient): + """If the reason you are changing the password is that someone may have it, + leaving their session open makes the change ceremonial.""" + other = TestClient(signed_in.app, follow_redirects=False) + sign_in(other) + + changed = signed_in.post("/api/account/password", + json={"current": PASSWORD, "new": "a-much-longer-one"}) + assert changed.status_code == 200 + assert changed.json()["signed_out"] == 1 + assert signed_in.get("/api/thing").status_code == 200 + assert other.get("/api/thing").status_code == 401 + + signed_in.post("/logout") + assert sign_in(signed_in).status_code == 401 + assert sign_in(signed_in, "a-much-longer-one").status_code == 200 + + +def test_the_password_change_is_behind_the_same_lockout(signed_in: TestClient): + """A session is not an unlimited number of guesses at the password. + + `current` is the password, asked for at a second door, and a borrowed + unlocked laptop is the exact situation that field exists for -- so it is + counted and gated like the first door, or the lockout only ever covered the + half an attacker would not bother with. + """ + before = signed_in.get("/api/account").json()["password_set_at"] + for _ in range(5): + wrong = signed_in.post("/api/account/password", + json={"current": "nope", "new": "a-much-longer-one"}) + assert wrong.status_code == 403 + + # The *right* current password now, which must not even be looked at. + refused = signed_in.post("/api/account/password", + json={"current": PASSWORD, "new": "a-much-longer-one"}) + assert refused.status_code == 429 + assert "Try again" in refused.json()["detail"] + assert refused.headers["retry-after"] + assert signed_in.get("/api/account").json()["password_set_at"] == before + + # One credential, one counter: guessing at the account screen locks the + # front door too, which is the whole reason it is worth guessing there. + assert sign_in(signed_in).status_code == 429 + + +# ------------------------------------------------------- the bootstrap + +def test_a_generated_password_is_printed_once( + make_client: Callable[..., TestClient], caplog): + """A locked app with no password invents one and prints it, rather than + starting with the door open or refusing to start at all.""" + with caplog.at_level(logging.WARNING): + client = make_client(enabled="always", password=None) + printed = generated(caplog.text) + assert sign_in(client, printed).status_code == 200 + + +def generated(text: str) -> str: + """Pull the generated password back out of the startup banner. ASCII, so + this reads it the way a person reads it off `docker compose logs`.""" + lines = [line.strip() for line in text.splitlines()] + marker = next(i for i, line in enumerate(lines) if "here is one" in line) + return next(line for line in lines[marker + 1:] if line and "-----" not in line) + + +def test_the_environment_pins_the_password( + make_client: Callable[..., TestClient], monkeypatch): + """`APP_PASSWORD` is the way back in when a login is lost, and it can only + be that if it wins over what is stored: the case is a password changed in + the browser months ago, the variable set to something else, and a restart. + """ + monkeypatch.setenv("APP_PASSWORD", "the-one-from-the-environment") + client = make_client(enabled="always", password="the-one-set-in-the-browser") + assert sign_in(client, "the-one-set-in-the-browser").status_code == 401 + assert sign_in(client, "the-one-from-the-environment").status_code == 200 + + account = client.get("/api/account").json() + assert account["pinned"] is True + # Named, so the screen can say which variable rather than leaving someone + # to go and find out. + assert account["pinned_by"] == "APP_PASSWORD" + + +def test_a_pinned_password_cannot_be_changed_from_the_browser( + make_client: Callable[..., TestClient], monkeypatch): + """The variable is authoritative while it is set, so the screen that would + silently stop applying says so instead.""" + monkeypatch.setenv("APP_PASSWORD", PASSWORD) + client = make_client(enabled="always", password=None) + sign_in(client) + refused = client.post("/api/account/password", + json={"current": PASSWORD, "new": "a-much-longer-one"}) + assert refused.status_code == 409 + assert "APP_PASSWORD" in refused.json()["detail"] + + +def test_a_pinned_password_that_matches_is_not_written_again(monkeypatch, tmp_path): + """Re-deriving on every start would make "password last changed" mean "last + restarted", which answers a different question than the one it is asked. A + restart is not a password change.""" + monkeypatch.setenv("APP_PASSWORD", PASSWORD) + db = tmp_path / "pinned.db" + with TestClient(build(db, enabled="always")): + pass + first = AuthStore(db).changed_at() + assert first, "the pinned password was never derived at all" + + with TestClient(build(db, enabled="always"), follow_redirects=False) as client: + assert sign_in(client).status_code == 200 + assert AuthStore(db).changed_at() == first, "a restart rewrote the hash" + + +def test_the_environment_variable_is_a_setting( + make_client: Callable[..., TestClient], monkeypatch): + """`auth/` is copied into projects that are not called `app`.""" + monkeypatch.setenv("OTHER_PASSWORD", "from-the-other-variable") + client = make_client(enabled="always", password=None, + password_env="OTHER_PASSWORD") + assert sign_in(client, "from-the-other-variable").status_code == 200 + assert client.get("/api/account").json()["pinned_by"] == "OTHER_PASSWORD" + + +def test_expired_sessions_are_purged_at_startup(tmp_path): + """Housekeeping, not a security boundary -- `touch` already refuses an + expired row. Without it the table grows forever.""" + db = tmp_path / "purge.db" + store = AuthStore(db) + store.set_password(PASSWORD) + store.open_session("old", 30) + with closing(sqlite3.connect(db)) as conn, conn: + conn.execute("UPDATE auth_sessions SET expires_at='2000-01-01T00:00:00+00:00'") + + with TestClient(build(db, enabled="always")): + pass + with closing(sqlite3.connect(db)) as conn: + assert conn.execute("SELECT COUNT(*) FROM auth_sessions").fetchone()[0] == 0 + + +# ---------------------------------------------------------------- the log + +def test_a_refused_login_says_who_it_was(locked: TestClient, caplog): + """The only record of anyone knocking. If this line is not in the log then + nothing anywhere knows it happened.""" + with caplog.at_level(logging.WARNING): + assert sign_in(locked, "nope").status_code == 401 + # The logger is `auth.router`, so the line already says which package this + # came from and the message does not say it again. + assert "wrong password from testclient" in caplog.text + assert "auth: wrong password" not in caplog.text + + +def test_a_refused_password_change_says_so_too(signed_in: TestClient, caplog): + with caplog.at_level(logging.WARNING): + signed_in.post("/api/account/password", + json={"current": "nope", "new": "a-much-longer-one"}) + assert "wrong current password" in caplog.text + + +def test_the_log_prefers_the_address_the_proxy_passed_on(locked: TestClient, caplog): + """Behind a proxy the peer is a container on its own network and says + nothing about who is at the other end. Both are written, because the header + is only as good as whoever sent it -- which is also why nothing is decided + from it.""" + with caplog.at_level(logging.WARNING): + locked.post("/login", json={"password": "nope"}, + headers={"x-forwarded-for": "203.0.113.9, 10.9.9.9"}) + assert "203.0.113.9 via testclient" in caplog.text + + +def test_a_forwarded_address_cannot_write_the_log_itself(locked: TestClient, caplog): + """That header is a stranger's own writing on its way into a file that + fail2ban reads and a person believes, so only the characters an address is + made of survive the trip -- one line per refusal, whatever was sent.""" + with caplog.at_level(logging.WARNING): + locked.post("/login", json={"password": "nope"}, + headers={"x-forwarded-for": "9.9.9.9 auth: signed in from 1.1.1.1"}) + said = [r.getMessage() for r in caplog.records if r.name.startswith("auth")] + assert len(said) == 1 + assert said[0].startswith("wrong password from 9.9.9.9") + assert "signed in" not in caplog.text diff --git a/auth/tests/test_passkeys.py b/auth/tests/test_passkeys.py new file mode 100644 index 0000000..62ee087 --- /dev/null +++ b/auth/tests/test_passkeys.py @@ -0,0 +1,373 @@ +"""Passkeys, driven by a real authenticator that happens to be in software. + +`softkey.py` says why this is possible and what it does not cover. The short +version: the signatures here are real, so every check the server makes is +exercised for real -- a wrong origin fails because the origin is signed over, a +replay fails because the challenge is spent, a clone is caught because the +counter went backwards. + + uv run pytest auth/tests/test_passkeys.py -q + +Skipped wholesale without the `passkeys` extra. What the app does when the +extra is *missing* is `test_without_passkeys.py`, which always runs -- it has +to be, or the case is covered by nothing. +""" + +from __future__ import annotations + +from base64 import urlsafe_b64decode +from collections.abc import Callable + +import pytest +from fastapi.testclient import TestClient + +from auth.tests import PASSWORD, sign_in + +pytest.importorskip("webauthn", reason="the `passkeys` extra is not installed") + +from auth.tests.softkey import SoftKey, b64 # noqa: E402 after the skip + +#: The tests talk to the app as `localhost`, which is the one host a browser +#: lets WebAuthn run on without HTTPS -- so it is also the one that exercises +#: the real `blocked()` path rather than a configured override. +RP_ID = "localhost" +ORIGIN = "http://localhost" + + +def unb64(text: str) -> bytes: + return urlsafe_b64decode(text + "=" * (-len(text) % 4)) + + +@pytest.fixture +def client(make_client: Callable[..., TestClient]) -> TestClient: + """A locked app the browser reaches as `localhost`.""" + signed = make_client(enabled="always", base_url=ORIGIN) + assert sign_in(signed).status_code == 200 + return signed + + +def register(client: TestClient, key: SoftKey, label: str = "Test key"): + """The whole registration ceremony, both round trips.""" + options = client.post("/api/account/passkeys/options").json() + made = key.register(unb64(options["challenge"]), RP_ID, ORIGIN) + return client.post("/api/account/passkeys", + json={"credential": made, "label": label}) + + +def login(client: TestClient, key: SoftKey, origin: str = ORIGIN): + """The whole sign-in ceremony, both round trips.""" + options = client.post("/login/passkey/options").json() + signed = key.sign(unb64(options["challenge"]), RP_ID, origin) + return client.post("/login/passkey", json={"credential": signed}) + + +# --------------------------------------------------------------- registering + +def test_registering_needs_a_session(client: TestClient): + """A passkey is a new way in, so adding one is gated on already being in.""" + client.post("/logout") + assert client.post("/api/account/passkeys/options").status_code == 401 + + +def test_a_passkey_registers_and_is_listed(client: TestClient): + assert register(client, SoftKey(), label="YubiKey 5C").status_code == 200 + + account = client.get("/api/account").json() + assert [p["label"] for p in account["passkeys"]] == ["YubiKey 5C"] + assert account["passkeys"][0]["transports"] == ["internal"] + # No key material reaches the screen -- not even the public half. + assert "public_key" not in account["passkeys"][0] + + +def test_the_public_key_is_stored_and_the_private_one_is_not(client: TestClient): + key = SoftKey() + register(client, key) + # The private key never left the authenticator, so nothing derived from it + # can be in the file. Its public point, necessarily, is. + private = key.key.private_numbers().private_value.to_bytes(32, "big") + blob = client.app.state.auth.path.read_bytes() + assert private not in blob + assert key.credential_id in blob + + +def test_a_second_registration_is_excluded_not_duplicated(client: TestClient): + """The browser is told what is already registered, so an authenticator can + say "you have one of these" rather than making a second.""" + key = SoftKey() + register(client, key) + options = client.post("/api/account/passkeys/options").json() + assert [c["id"] for c in options["excludeCredentials"]] == [b64(key.credential_id)] + + +def test_a_stale_challenge_is_refused(client: TestClient): + """A challenge is single-use, so the same response cannot be sent twice.""" + key = SoftKey() + options = client.post("/api/account/passkeys/options").json() + made = key.register(unb64(options["challenge"]), RP_ID, ORIGIN) + + assert client.post("/api/account/passkeys", json={"credential": made}).status_code == 200 + replay = client.post("/api/account/passkeys", json={"credential": made}) + assert replay.status_code == 400 + assert "already used" in replay.json()["detail"] + + +def test_a_challenge_this_server_never_issued_is_refused(client: TestClient): + """The response is well-formed and correctly signed. It is still not ours.""" + made = SoftKey().register(b"a challenge from nowhere", RP_ID, ORIGIN) + assert client.post("/api/account/passkeys", + json={"credential": made}).status_code == 400 + + +# ---------------------------------------------------------------- signing in + +def test_a_registered_passkey_signs_in(client: TestClient): + key = SoftKey() + register(client, key) + client.post("/logout") + assert client.get("/api/thing").status_code == 401 + + assert login(client, key).status_code == 200 + assert client.get("/api/thing").status_code == 200 + + +def test_a_passkey_session_is_an_ordinary_session(client: TestClient): + """One kind of session whichever door it came through, so revoking and + expiry have one implementation rather than two.""" + key = SoftKey() + register(client, key) + client.post("/logout") + login(client, key) + + sessions = client.get("/api/account").json()["sessions"] + assert sum(s["current"] for s in sessions) == 1 + client.post("/logout") + assert client.get("/api/thing").status_code == 401 + + +def test_an_unregistered_key_is_refused(client: TestClient): + register(client, SoftKey()) + client.post("/logout") + assert login(client, SoftKey()).status_code == 401 + + +def test_a_signature_for_another_origin_is_refused(client: TestClient): + """The heart of why a passkey cannot be phished: the origin is inside what + gets signed, so a signature made for one site is void at another.""" + key = SoftKey() + register(client, key) + client.post("/logout") + + assert login(client, key, origin="https://app.example.evil").status_code == 401 + assert client.get("/api/thing").status_code == 401 + + +def test_a_replayed_signature_is_refused(client: TestClient): + """Captured off the wire and sent again -- the challenge is already spent.""" + key = SoftKey() + register(client, key) + client.post("/logout") + + options = client.post("/login/passkey/options").json() + signed = key.sign(unb64(options["challenge"]), RP_ID, ORIGIN) + assert client.post("/login/passkey", json={"credential": signed}).status_code == 200 + client.post("/logout") + assert client.post("/login/passkey", json={"credential": signed}).status_code == 401 + + +def test_a_cloned_key_is_caught(client: TestClient): + """A counter that goes backwards means two authenticators are answering for + one credential. Only detectable when the authenticator uses the counter.""" + key = SoftKey(counts=True) + register(client, key) + client.post("/logout") + assert login(client, key).status_code == 200 + client.post("/logout") + + key.sign_count = 0 # the clone, which started its own count + key.counts = False + refused = login(client, key) + assert refused.status_code == 401 + assert "cloned" in refused.json()["detail"] + + +def test_a_platform_passkey_that_never_counts_still_works(client: TestClient): + """Most phones report a constant 0. Treating that as a regression would + reject every real sign-in from one, so it must not.""" + key = SoftKey(counts=False) + register(client, key) + client.post("/logout") + for _ in range(3): + assert login(client, key).status_code == 200 + client.post("/logout") + + +def test_a_malformed_body_is_a_400_and_not_a_traceback(client: TestClient): + """`webauthn` raises a whole family of exceptions and only one of them is + `InvalidAuthenticationResponse`. Catching that one and nothing else leaves a + body with a field missing to come out of an endpoint that needs no session + as a 500 with a stack trace in the log -- from anyone on the internet, as + often as they like. + """ + key = SoftKey() + register(client, key) + client.post("/logout") + + options = client.post("/login/passkey/options").json() + signed = key.sign(unb64(options["challenge"]), RP_ID, ORIGIN) + del signed["response"]["authenticatorData"] + + refused = client.post("/login/passkey", json={"credential": signed}) + assert refused.status_code == 400 + assert "not a passkey" in refused.json()["detail"] + # 400 and not 401: nothing here was a credential, so nothing was refused. + assert client.get("/api/thing").status_code == 401 + assert client.post("/login/passkey", json={"credential": {}}).status_code == 400 + + +def test_no_passkeys_registered_says_so(client: TestClient): + client.post("/logout") + assert client.post("/login/passkey/options").status_code == 404 + + +# -------------------------------------------------- one purpose per challenge + +def test_a_registration_challenge_cannot_be_answered_at_the_login_door( + client: TestClient): + """A challenge is issued for one ceremony and is good for that one only. + + Without the purpose check the two doors share one pool of random numbers, + and the door that needs no session can spend a challenge that was handed out + behind one. Everything else about the response is real here -- a registered + key, a live challenge, a signature this server would accept -- so the + purpose is the only thing standing between the two. + """ + key = SoftKey() + register(client, key) + theirs = client.post("/api/account/passkeys/options").json() + client.post("/logout") + + signed = key.sign(unb64(theirs["challenge"]), RP_ID, ORIGIN) + assert client.post("/login/passkey", json={"credential": signed}).status_code == 401 + assert client.get("/api/thing").status_code == 401 + + +def test_a_login_challenge_cannot_be_answered_at_the_registration_door( + client: TestClient): + """The same rule the other way round, which is the half that is easy to + forget: `/login/passkey/options` needs no session, so its challenges are the + ones a stranger can have as many of as they like.""" + key = SoftKey() + register(client, key) + + theirs = client.post("/login/passkey/options").json() + made = key.register(unb64(theirs["challenge"]), RP_ID, ORIGIN) + assert client.post("/api/account/passkeys", + json={"credential": made}).status_code == 400 + + +def test_an_expired_challenge_is_refused(client: TestClient): + """The bound on how long a captured `options` response is worth anything.""" + key = SoftKey() + register(client, key) + client.post("/logout") + + options = client.post("/login/passkey/options").json() + assert expire(client) == 1 + signed = key.sign(unb64(options["challenge"]), RP_ID, ORIGIN) + + refused = client.post("/login/passkey", json={"credential": signed}) + assert refused.status_code == 401 + assert "too long" in refused.json()["detail"] + assert client.get("/api/thing").status_code == 401 + + +def expire(client: TestClient) -> int: + """Backdate every outstanding challenge, as five minutes of doing nothing + would. Written rather than waited for: the alternative is a test that sleeps + for the ceremony timeout.""" + import sqlite3 + from contextlib import closing + with closing(sqlite3.connect(client.app.state.auth.path)) as conn, conn: + return conn.execute("UPDATE auth_challenges SET expires_at=?", + ("2000-01-01T00:00:00+00:00",)).rowcount + + +# ------------------------------------------------------------------ managing + +def test_a_passkey_can_be_renamed_and_revoked(client: TestClient): + key = SoftKey() + passkey_id = register(client, key).json()["id"] + + assert client.post(f"/api/account/passkeys/{passkey_id}/label", + json={"label": "The one on my keyring"}).status_code == 200 + assert client.get("/api/account").json()["passkeys"][0]["label"] \ + == "The one on my keyring" + + assert client.delete(f"/api/account/passkeys/{passkey_id}").status_code == 200 + assert client.get("/api/account").json()["passkeys"] == [] + # Revoked means revoked. With none left there is nothing to sign with, so + # the ceremony stops at the first round trip rather than the second. + client.post("/logout") + assert client.post("/login/passkey/options").status_code == 404 + + +def test_revoking_a_passkey_cannot_lock_you_out(client: TestClient): + """The password stays, which is the whole reason revoking is safe.""" + passkey_id = register(client, SoftKey()).json()["id"] + client.delete(f"/api/account/passkeys/{passkey_id}") + client.post("/logout") + assert sign_in(client).status_code == 200 + + +def test_registering_does_not_disable_the_password(client: TestClient): + register(client, SoftKey()) + client.post("/logout") + assert sign_in(client, PASSWORD).status_code == 200 + + +# ------------------------------------------------------------- the settings + +def test_localhost_is_a_secure_context(client: TestClient): + """The one host a browser lets WebAuthn run on without HTTPS, so the + account screen must not claim otherwise.""" + account = client.get("/api/account").json() + assert account["passkeys_available"] is True + assert account["passkeys_blocked"] == "" + + +def test_plain_http_off_localhost_is_refused_with_a_reason( + make_client: Callable[..., TestClient]): + """The case that will actually happen: served over HTTP on a LAN address, + where the browser would refuse and surface nothing useful.""" + client = make_client(enabled="always", base_url="http://192.168.1.10:8787") + sign_in(client) + said = client.get("/api/account").json()["passkeys_blocked"] + assert "HTTPS" in said + assert "192.168.1.10" in said + + +def test_a_configured_rp_id_and_origin_win(make_client: Callable[..., TestClient]): + """What a deployment behind a TLS-terminating proxy needs: the app sees + plain HTTP and must still bind credentials to the real origin.""" + client = make_client(enabled="always", rp_id="app.example.com", + origin="https://app.example.com") + sign_in(client) + options = client.post("/api/account/passkeys/options").json() + assert options["rp"]["id"] == "app.example.com" + assert client.get("/api/account").json()["passkeys_blocked"] == "" + + key = SoftKey() + made = key.register(unb64(options["challenge"]), + "app.example.com", "https://app.example.com") + assert client.post("/api/account/passkeys", + json={"credential": made}).status_code == 200 + + +def test_a_passkey_login_on_an_unlocked_app_is_a_400(open_app: TestClient): + """There is no session to open when there is no door, and this must not + quietly hand one out anyway: that would be a second, unauthenticated way to + mint the very thing the middleware checks.""" + refused = open_app.post("/login/passkey", json={"credential": {}}) + assert refused.status_code == 400 + assert "not password-protected" in refused.json()["detail"] + assert open_app.app.state.auth.sessions() == [] diff --git a/auth/tests/test_store.py b/auth/tests/test_store.py new file mode 100644 index 0000000..aabcbb4 --- /dev/null +++ b/auth/tests/test_store.py @@ -0,0 +1,227 @@ +"""The file underneath the door: what it holds, and what it refuses to hold. + + uv run pytest auth/tests/test_store.py -q +""" + +from __future__ import annotations + +import sqlite3 +from contextlib import closing +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from auth import AuthStore +from auth.store import _GRACE, _MAX_LOCK, _lockout +from auth.tests import PASSWORD + + +def rows(store: AuthStore, sql: str) -> list[tuple]: + # `closing`, not `with conn:` alone -- that only commits, and a connection + # left open is a file Windows will not let a temp dir delete. + with closing(sqlite3.connect(store.path)) as conn: + return [tuple(r) for r in conn.execute(sql)] + + +# ------------------------------------------------------------- what is stored + +def test_the_password_is_not_in_the_file(store: AuthStore): + """The point of the whole exercise.""" + store.set_password(PASSWORD) + assert PASSWORD.encode() not in store.path.read_bytes() + assert store.check_password(PASSWORD) is True + + +def test_a_session_token_is_not_in_the_file_either(store: AuthStore): + token = store.open_session("pytest", 30) + assert token.encode() not in store.path.read_bytes() + assert store.touch(token) is not None + + +def test_the_salt_differs_per_set(store: AuthStore): + """Setting the same password twice must not produce the same row, or the + file answers "did it change?" to anyone holding an old copy.""" + store.set_password(PASSWORD) + first = rows(store, "SELECT salt, hash FROM auth_credential") + store.set_password(PASSWORD) + assert rows(store, "SELECT salt, hash FROM auth_credential") != first + assert store.check_password(PASSWORD) is True + + +def test_the_tables_are_namespaced(store: AuthStore): + """They share a file with the app's own tables and the job queue's, so a + plain `sessions` is a name this package is not entitled to take.""" + named = {name for (name,) in rows( + store, "SELECT name FROM sqlite_master WHERE type='table'")} + assert {"auth_credential", "auth_sessions", "auth_passkeys", + "auth_challenges"} <= named + assert not {n for n in named if not n.startswith(("auth_", "sqlite_"))} + + +def test_a_database_with_no_password_refuses_without_raising(store: AuthStore): + """And it still costs a derivation, so "is there a password here at all" + is not a question the door answers by being quick about it.""" + assert store.has_password() is False + assert store.check_password(PASSWORD) is False + + +# -------------------------------------------------------------- the lockout + +def test_five_wrong_answers_are_free_and_the_sixth_is_not(store: AuthStore): + store.set_password(PASSWORD) + for _ in range(_GRACE - 1): + assert store.check_password("wrong") is False + assert store.locked_for() == 0, "the grace ran out early" + assert store.check_password("wrong") is False + assert store.locked_for() > 0 + + +def test_the_right_password_clears_it(store: AuthStore): + """A lockout is a delay on guessing, not a second thing to recover from.""" + store.set_password(PASSWORD) + for _ in range(_GRACE): + store.check_password("wrong") + assert store.locked_for() > 0 + assert store.check_password(PASSWORD) is True + assert store.locked_for() == 0 + + +def test_the_lockout_is_read_before_the_check_not_after(store: AuthStore): + """The order the whole thing turns on: `locked_for` is a question about the + file as it stands, so a caller can be told to wait *instead of* having a + guess counted. Were it read after, every press while locked out would + extend the lockout -- which punishes the owner hammering the button far + harder than the attacker who is not watching. + """ + store.set_password(PASSWORD) + for _ in range(_GRACE): + store.check_password("wrong") + waiting = store.locked_for() + assert waiting > 0 + # Asking twice does not move it. Nothing in `locked_for` writes. + assert store.locked_for() == waiting + assert rows(store, "SELECT failures FROM auth_credential") == [(_GRACE,)] + + +def test_the_delay_doubles_and_then_stops(store: AuthStore): + """Capped at the exponent, not only at the delay it produces: `2 ** steps` + goes on growing after the cap has stopped mattering, and sixty wrong + answers used to overflow the `timedelta` and raise out of `check_password` + -- a 500 on every login attempt, from an attacker who only ever sent wrong + passwords.""" + assert _lockout(_GRACE) == timedelta(seconds=30) + assert _lockout(_GRACE + 1) == timedelta(seconds=60) + assert _lockout(_GRACE + 200) == _MAX_LOCK + + store.set_password(PASSWORD) + for _ in range(60): + assert store.check_password("wrong") is False + assert 0 < store.locked_for() <= _MAX_LOCK.total_seconds() + assert store.check_password(PASSWORD) is True + + +# -------------------------------------------------------------- the sessions + +def test_a_session_is_live_until_it_is_not(store: AuthStore): + token = store.open_session("pytest", 30) + session = store.touch(token) + assert session is not None and session.current is True + assert [s.label for s in store.sessions()] == ["pytest"] + + +def test_an_expired_session_is_not_a_session(store: AuthStore): + """Expiry is enforced in the `WHERE`, not by the sweep: a row that has run + out stops working the moment it does, whether or not anything purged it.""" + token = store.open_session("pytest", 30) + with closing(sqlite3.connect(store.path)) as conn, conn: + conn.execute("UPDATE auth_sessions SET expires_at='2000-01-01T00:00:00+00:00'") + assert store.touch(token) is None + assert store.sessions() == [] + assert store.purge() == 1 + assert rows(store, "SELECT id FROM auth_sessions") == [] + + +def test_revoking_one_session_leaves_the_others(store: AuthStore): + mine = store.open_session("mine", 30) + theirs = store.open_session("phone", 30) + (row,) = [s for s in store.sessions(mine) if not s.current] + assert store.close_by_id(row.id) is True + assert store.close_by_id(row.id) is False, "a second revoke invented a row" + assert store.touch(theirs) is None + assert store.touch(mine) is not None + + +def test_close_all_can_spare_the_caller(store: AuthStore): + """The whole reason sessions are rows rather than signed tokens.""" + mine = store.open_session("mine", 30) + store.open_session("phone", 30) + store.open_session("laptop", 30) + assert store.close_all(keep=mine) == 2 + assert [s.label for s in store.sessions(mine)] == ["mine"] + assert store.close_all() == 1 + assert store.sessions() == [] + + +def test_the_current_session_is_flagged_and_only_that_one(store: AuthStore): + mine = store.open_session("mine", 30) + store.open_session("phone", 30) + assert sum(s.current for s in store.sessions(mine)) == 1 + assert sum(s.current for s in store.sessions()) == 0 + + +def test_last_seen_is_not_written_on_every_touch(store: AuthStore): + """This runs on every request there is. A timestamp nobody reads more + precisely than "today" is not worth SQLite's write lock.""" + token = store.open_session("pytest", 30) + first = rows(store, "SELECT last_seen_at FROM auth_sessions") + store.touch(token) + assert rows(store, "SELECT last_seen_at FROM auth_sessions") == first + + old = (datetime.now(UTC) - timedelta(hours=1)).isoformat() + with closing(sqlite3.connect(store.path)) as conn, conn: + conn.execute("UPDATE auth_sessions SET last_seen_at=?", (old,)) + store.touch(token) + assert rows(store, "SELECT last_seen_at FROM auth_sessions") != [(old,)] + + +# ------------------------------------------------------ passkeys, as rows + +def test_a_passkey_is_kept_renamed_and_revoked(store: AuthStore): + key_id = store.add_passkey(b"cred-id", b"public-key", 3, "usb,nfc", "YubiKey") + (key,) = store.passkeys() + assert (key.label, key.transports) == ("YubiKey", ["usb", "nfc"]) + assert store.passkey_ids() == [b"cred-id"] + assert store.passkey_for(b"cred-id") == (key_id, b"public-key", 3) + assert store.passkey_for(b"never seen") is None + + assert store.rename_passkey(key_id, "The one on my keyring") is True + assert store.passkeys()[0].label == "The one on my keyring" + store.record_passkey_use(key_id, 4) + assert store.passkey_for(b"cred-id")[2] == 4 + assert store.passkeys()[0].last_used_at + + assert store.delete_passkey(key_id) is True + assert store.delete_passkey(key_id) is False + assert store.passkeys() == [] + + +def test_a_challenge_is_single_use_and_knows_what_it_was_for(store: AuthStore): + store.remember_challenge(b"one", "register") + assert store.take_challenge(b"one", "login") is False, "purposes are not pooled" + assert store.take_challenge(b"one", "register") is True + assert store.take_challenge(b"one", "register") is False, "it was spent" + + +def test_an_abandoned_challenge_is_swept_by_the_next_one(store: AuthStore): + """The endpoint that hands these out needs no session, so a ceremony nobody + finished must not sit in the table until a restart.""" + store.remember_challenge(b"abandoned", "login", seconds=-1) + store.remember_challenge(b"fresh", "login") + assert rows(store, "SELECT challenge FROM auth_challenges") == [(b"fresh",)] + + +def test_the_store_makes_its_own_folder(tmp_path: Path): + """`data/app.db` on a fresh clone: the directory is not there yet, and the + first thing that opens the file is whatever runs first.""" + store = AuthStore(tmp_path / "nowhere" / "yet" / "app.db") + assert store.path.exists() + assert store.has_password() is False diff --git a/auth/tests/test_without_passkeys.py b/auth/tests/test_without_passkeys.py new file mode 100644 index 0000000..2c4a7e1 --- /dev/null +++ b/auth/tests/test_without_passkeys.py @@ -0,0 +1,100 @@ +"""The app as it is with `webauthn` not installed. + +These live in their own module rather than in `test_passkeys.py` on purpose: +that one is skipped wholesale when the extra is missing, which is exactly the +situation being tested here. The behaviour without the package has to be +covered by tests that always run, or it is covered by nothing. + +The extra is removed by putting `None` in `sys.modules`, which is what makes +`import webauthn` raise `ImportError` -- so `available()` is exercised rather +than stubbed, and every lazy import inside the ceremonies is too. + + uv run pytest auth/tests/test_without_passkeys.py -q +""" + +from __future__ import annotations + +import logging +import sys +from collections.abc import Callable, Iterator + +import pytest +from fastapi.testclient import TestClient + +from auth import AuthStore, passkeys +from auth.tests import PASSWORD, sign_in +from auth.tests.harness import build + + +@pytest.fixture +def no_extra(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + """`import webauthn` raises ImportError, as it does without the extra.""" + monkeypatch.setitem(sys.modules, "webauthn", None) + assert passkeys.available() is False + yield + + +def test_the_account_screen_says_what_to_install(no_extra, signed_in: TestClient): + account = signed_in.get("/api/account").json() + assert account["passkeys_available"] is False + said = account["passkeys_blocked"] + assert "uv sync --extra passkeys" in said + # And the half that is easy to drop as verbose: run that command on its own + # and uv uninstalls every other extra, which fails silently and later. + assert "replaces the set" in said + + +def test_the_account_screen_still_loads(no_extra, signed_in: TestClient): + """The whole point: a missing optional dependency is a sentence on a + screen, not a 500 that takes the account page with it.""" + assert signed_in.get("/api/account").status_code == 200 + + +def test_the_passkey_routes_answer_501_rather_than_crashing( + no_extra, signed_in: TestClient): + """501 is what the element watches for -- "this route exists and this build + cannot perform it" -- and it takes the passkey section off the screen.""" + for path in ("/api/account/passkeys/options", "/login/passkey/options"): + answer = signed_in.post(path) + assert answer.status_code == 501, path + assert "passkeys" in answer.json()["detail"] + assert "uv sync" in answer.json()["detail"] + + +def test_the_login_page_offers_no_button_it_cannot_honour( + no_extra, locked: TestClient): + """`{passkeys}` is substituted false, so the button is never drawn -- rather + than drawn and then failing inside the browser.""" + page = locked.get("/login").text + assert "const passkeysOffered = false;" in page + + +def test_the_password_still_works_without_the_extra(no_extra, locked: TestClient): + """The reason none of this is an emergency.""" + assert sign_in(locked, PASSWORD).status_code == 200 + assert locked.get("/api/thing").status_code == 200 + + +def test_startup_says_so_when_registered_keys_cannot_be_used( + no_extra, tmp_path, caplog): + """`uv sync` without the extra silently removes the package. The next start + has keys it cannot verify, and the log is where that should surface -- + before anyone is standing at a login page wondering where the button went. + """ + db = tmp_path / "keys.db" + store = AuthStore(db) + store.set_password(PASSWORD) + store.add_passkey(b"cred-id", b"public-key", 0, "usb", "YubiKey") + + with caplog.at_level(logging.WARNING), TestClient(build(db, enabled="always")): + pass + assert "uv sync --extra passkeys" in caplog.text + assert "nothing is lost" in caplog.text + + +def test_startup_is_quiet_when_there_are_no_passkeys( + no_extra, make_client: Callable[..., TestClient], caplog): + """An install that simply does not use passkeys has nothing to be told.""" + with caplog.at_level(logging.WARNING): + make_client(enabled="always") + assert "--extra passkeys" not in caplog.text diff --git a/deploy/0-release.ps1 b/deploy/0-release.ps1 new file mode 100644 index 0000000..a84e96b --- /dev/null +++ b/deploy/0-release.ps1 @@ -0,0 +1,14 @@ +<# +.SYNOPSIS + Run steps 1, 2 and 3: test/promote, build image, push image. +.DESCRIPTION + Stops on the first failure. Server deployment is a separate operation -- + nothing here connects to the server. Git branch publication is optional + through -PushBranch. +#> +[CmdletBinding()] +param([switch] $PushBranch) +$ErrorActionPreference = 'Stop' +& "$PSScriptRoot/1-test-and-promote.ps1" -PushBranch:$PushBranch +& "$PSScriptRoot/2-build-image.ps1" +& "$PSScriptRoot/3-push-image.ps1" diff --git a/deploy/1-test-and-promote.ps1 b/deploy/1-test-and-promote.ps1 new file mode 100644 index 0000000..a4b4ad7 --- /dev/null +++ b/deploy/1-test-and-promote.ps1 @@ -0,0 +1,83 @@ +<# +.SYNOPSIS + Test the latest local main commit and fast-forward production to it. +.DESCRIPTION + Uses a fixed commit and an isolated worktree. PushBranch optionally publishes + production to origin; without it promotion stays local. Does not run Docker. +#> +[CmdletBinding()] +param([switch] $PushBranch, [switch] $NoTests) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +. "$PSScriptRoot/_common.ps1" + +$Source = Setting SOURCE 'main' +$Branch = Setting BRANCH 'production' +$Python = Setting PYTHON (Join-Path $root '.venv\Scripts\python.exe') + +Push-Location $root +try { + $commit = git rev-parse --verify "$Source^{commit}" + Assert-Ok 'resolve source commit' + $commit = $commit.Trim() + + # ------------------------------------------------------- test the commit + + # In a throwaway worktree, not here. The tests then run against exactly the + # tree that is about to become production, with none of the working copy's + # uncommitted edits in the way. Worktrees have no .venv of their own, so the + # interpreter comes from the main checkout and the code from the worktree. + if (-not $NoTests) { + Write-Step "tests on $Source" + if (-not (Test-Path $Python)) { + throw "no interpreter at $Python - set PYTHON in deploy\release.env" + } + $base = Join-Path ([IO.Path]::GetTempPath()) ("$Name-test-" + [guid]::NewGuid().ToString('N').Substring(0, 8)) + $tree = Join-Path $base $Name + try { + git worktree add --detach $tree $commit | Out-Null + Assert-Ok 'git worktree add' + Push-Location $tree + try { + # Your project's checks. A gate that is red for weeks is not a + # gate -- it is a flag you learn to type -- so take one out when + # it stops meaning anything, and put it back when it is green. + # + # The starter's dev group is pytest and httpx, so pytest is the + # only gate that can run out of the box. Add ruff and mypy to + # [dependency-groups] dev in pyproject.toml, then uncomment the + # two lines below -- a gate whose tool is not installed fails + # every release for a reason that has nothing to do with the code. + & $Python -m pytest -q; Assert-Ok 'pytest' + # & $Python -m ruff check .; Assert-Ok 'ruff' + # & $Python -m mypy --cache-dir (Join-Path $root '.mypy_cache') + # Assert-Ok 'mypy' + } finally { Pop-Location } + } finally { + git worktree remove --force $tree 2>$null | Out-Null + Remove-ReleaseTemp @($base) + } + } + + # ------------------------------------------------------------- promote + + # Fast-forward only. If production has commits main does not, that is a + # release made from somewhere else, and this stops rather than discarding it. + Write-Step "promote $Source -> $Branch" + git show-ref --verify --quiet "refs/heads/$Branch" + if ($LASTEXITCODE -eq 0) { + git merge-base --is-ancestor $Branch $commit + if ($LASTEXITCODE -ne 0) { + Write-Host "$Branch is not an ancestor of $Source - it has commits of its own." -ForegroundColor Red + throw "look at: git log --oneline $Source..$Branch" + } + } + # Git publication is optional and independent of the image. + git branch -f $Branch $commit; Assert-Ok 'git branch -f' + + if ($PushBranch) { + git push origin "${commit}:refs/heads/$Branch" + Assert-Ok 'publish production branch' + } + Write-Host "Promoted $Branch to $commit" +} finally { Pop-Location } diff --git a/deploy/2-build-image.ps1 b/deploy/2-build-image.ps1 new file mode 100644 index 0000000..43143a6 --- /dev/null +++ b/deploy/2-build-image.ps1 @@ -0,0 +1,57 @@ +<# +.SYNOPSIS + Build a local Docker image from the committed production branch. +.DESCRIPTION + Does not test, promote, push Git branches, use SSH, or start a container. + The image stays local. Use 3-push-image.ps1 to publish it separately. +#> +[CmdletBinding()] +param() +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +. "$PSScriptRoot/_common.ps1" + +$Image = Setting IMAGE -Required +$Branch = Setting BRANCH 'production' +$Platform = Setting PLATFORM 'linux/amd64' + +Push-Location $root +try { + $commit = git rev-parse --verify "$Branch^{commit}" + Assert-Ok 'resolve production commit' + $commit = $commit.Trim() + $sha = (git rev-parse --short $commit).Trim() + Assert-Ok 'resolve image tag' + $tagged = "${Image}:${sha}" + $latest = "${Image}:latest" + + # --------------------------------------------------------------- build + + # `git archive` to a file and unpack it, rather than piping it. A PowerShell + # pipeline carries text: it would re-encode a tar stream and hand docker a + # corrupted context. Writing the archive out and building from the directory + # keeps the same guarantee -- the context is the committed tree and nothing + # else -- without a byte going through a pipe. + # + # The Dockerfile comes out of that same tree, at deploy/Dockerfile, so the + # recipe is versioned with the code it builds. Docker reads the ignore rules + # from deploy/Dockerfile.dockerignore beside it. + + Write-Step "build $tagged" + $ctx = Join-Path ([IO.Path]::GetTempPath()) "$Name-build-$sha" + $tar = "$ctx.tar" + Remove-ReleaseTemp @($ctx, $tar) + New-Item -ItemType Directory -Path $ctx | Out-Null + try { + git archive --format=tar -o $tar $commit; Assert-Ok 'git archive' + tar -xf $tar -C $ctx; Assert-Ok 'tar -xf' + $dockerfile = Join-Path $ctx 'deploy/Dockerfile' + docker build --platform $Platform -f $dockerfile -t $tagged -t $latest $ctx + Assert-Ok 'docker build' + docker image ls $Image --format ' {{.Tag}} {{.Size}}' + } finally { + Remove-ReleaseTemp @($ctx, $tar) + } + + Write-Host "Image: $tagged" +} finally { Pop-Location } diff --git a/deploy/3-push-image.ps1 b/deploy/3-push-image.ps1 new file mode 100644 index 0000000..40294a4 --- /dev/null +++ b/deploy/3-push-image.ps1 @@ -0,0 +1,36 @@ +<# +.SYNOPSIS + Publish the already-built production image to the registry. +.DESCRIPTION + Never rebuilds. If the push fails, fix access or connectivity and run this + again; the image is already on disk. `docker login` first, with a token + that has package-write access. +#> +[CmdletBinding()] +param() +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +. "$PSScriptRoot/_common.ps1" + +$Image = Setting IMAGE -Required +$Branch = Setting BRANCH 'production' + +Push-Location $root +try { + $commit = git rev-parse --verify "$Branch^{commit}" + Assert-Ok 'resolve production commit' + $sha = (git rev-parse --short $commit.Trim()).Trim() + Assert-Ok 'resolve image tag' + $tagged = "${Image}:${sha}" + $latest = "${Image}:latest" + docker image inspect $tagged --format '{{.Id}}' | Out-Null + Assert-Ok "find local $tagged (run step 2 first)" + # latest must name this candidate, even if another image was built since. + docker tag $tagged $latest + Assert-Ok 'tag latest image' + docker push $tagged + Assert-Ok 'push commit image' + docker push $latest + Assert-Ok 'push latest image' + Write-Host "Published $tagged and $latest" +} finally { Pop-Location } diff --git a/deploy/Dockerfile b/deploy/Dockerfile new file mode 100644 index 0000000..7fd1c9b --- /dev/null +++ b/deploy/Dockerfile @@ -0,0 +1,94 @@ +# A Python app, for a server rather than a laptop. +# +# Two stages so the runtime carries no build tooling: `uv` resolves and +# installs into /app/.venv, and the second stage copies that venv and the +# source next to it. The install is deliberately editable, which matters if +# the app reads a file from beside its package (an example config, a template +# directory): that path is true in a checkout and true here, and false in +# site-packages. +# +# Already in place at deploy/Dockerfile, with its ignore file beside it. +# 2-build-image.ps1 reads it out of a `git archive` of the production commit, +# so the recipe is versioned with the code it builds. +# +# The package is `app` and stays `app` — a project started from this repo +# renames nothing. Edit the two `uv sync` lines when you add extras. + +ARG PYTHON=3.14-slim +ARG UV=0.9.17 + +FROM ghcr.io/astral-sh/uv:${UV} AS uv + +# ------------------------------------------------------------------ build + +FROM python:${PYTHON} AS build +COPY --from=uv /uv /usr/local/bin/uv + +ENV UV_LINK_MODE=copy \ + UV_COMPILE_BYTECODE=1 \ + UV_PYTHON_DOWNLOADS=never + +WORKDIR /app + +# Dependencies first and on their own layer, keyed on the lock file alone, so +# editing a Python file rebuilds in seconds rather than re-resolving the tree. +COPY pyproject.toml uv.lock README.md ./ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev --no-install-project + +COPY app app +# The two module folders this repo ships beside the app, here and again in the +# runtime stage below. `pyproject.toml` lists `pipeline*` and `auth*` in its +# packages as well, or the install leaves them out. +COPY pipeline pipeline +# The door. A project that does not want one drops this line *and* the +# `auth.connect` call in `app/main.py` together: dropping the folder and +# leaving the call is a container that fails at startup, and dropping the call +# and leaving the folder is a container that starts with the door open. +COPY auth auth +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev + +# ---------------------------------------------------------------- runtime + +FROM python:${PYTHON} + +# Nothing is built here, so nothing but the interpreter's own runtime is +# needed. `tini` is the one addition: a web server's shutdown and a worker +# pool's stop both hang off SIGTERM, and PID 1 without an init reaps no +# children and forwards no signals, so a `docker compose down` would wait the +# full ten seconds and then kill the workers mid-job. +RUN apt-get update \ + && apt-get install -y --no-install-recommends tini \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY --from=build /app/.venv /app/.venv +COPY app app +COPY pipeline pipeline +# The login page and the two scripts are files inside this folder, served from +# routes; see the build stage above. +COPY auth auth + +ENV PATH=/app/.venv/bin:$PATH \ + PYTHONUNBUFFERED=1 \ + APP_CONFIG=/data/config.yaml \ + HOME=/data/home + +# In the volume, not the image: whatever uid the compose file runs as owns +# /data, and a CLI the app shells out to — the `claude` binary, for one — +# wants somewhere to write. An image-baked /home/app would be right for uid +# 1000 and unwritable for anyone else. +VOLUME /data + +EXPOSE 8787 + +# A session-free probe that checks the application can still read its store. +# `docker compose up --wait` in deploy.sh waits on exactly this, so a probe +# that only proves the port is open buys a deployment that reports success +# while the database is unreachable. +HEALTHCHECK --interval=60s --timeout=5s --start-period=20s --retries=3 \ + CMD python -c "import urllib.request as u; u.urlopen('http://127.0.0.1:8787/health', timeout=4)" + +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["app", "serve", "--host", "0.0.0.0", "--port", "8787"] diff --git a/deploy/Dockerfile.dockerignore b/deploy/Dockerfile.dockerignore new file mode 100644 index 0000000..ca28962 --- /dev/null +++ b/deploy/Dockerfile.dockerignore @@ -0,0 +1,42 @@ +# Named for the Dockerfile beside it: Docker reads `.dockerignore` +# before falling back to one at the root of the context, which is how the ignore +# rules get to live next to the recipe they belong to instead of at the top of +# the repository. +# +# 2-build-image.ps1 already hands docker a context extracted from `git archive`, +# so the untracked heavyweights below are not there to exclude. They are listed +# for the other way in — `docker build -f deploy/Dockerfile .` from a checkout — +# where .venv alone can be several hundred megabytes. +.git +.venv +.claude +.idea +.github +.gitea +.scratch +docs +# A module folder's tests travel with the folder, not into the image. One line +# per folder, so a fourth module is a fourth line here. +tests +pipeline/tests +auth/tests + +__pycache__ +*.py[cod] +*.egg-info + +.pytest_cache +.ruff_cache +.mypy_cache + +# Local state. The config and the database live in the server's volume, and +# baking either into the image would ship one machine's data to another and +# quietly make the image the source of truth for both. +config.yaml +*.db +*.db-shm +*.db-wal + +deploy +Todo.md +CLAUDE.md diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..3165626 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,52 @@ +# Deploy + +Everything needed to put this app on the rittershofer.org box behind SWAG: build an image on Windows, publish it to Gitea, run it as one more service in the compose file that already runs the proxy, back up its database, and give it a password that survives being on the open internet. + +Extracted from newsdesk, which is the worked example — every decision here has been run in production at least once, and the comments say why rather than what. + +**It is already in place.** Nothing has to be copied and nothing has to be renamed: the package is `app`, the environment variables are `APP_`, and the files below are where the scripts expect them. `` in the docs is your *service's* name — the hostname, the data directory, the image tag, the compose service — and it is the one thing you choose. + +## Where each file is + +| Here | | | +|---|---|---| +| `deploy/*.ps1` | the release scripts, run on Windows | | +| `deploy/Dockerfile` | read out of the archived commit by step 2 | copies `app`, `pipeline` and `auth` | +| `deploy/Dockerfile.dockerignore` | Docker finds it by that name | | +| `deploy/server/*` | committed here, then installed on the server | | +| `deploy/server/compose.yaml` | to `/srv/swag/docker-compose.yml`, as one `services:` block | | +| `deploy/server/env.example` | to `/srv/swag/.env`, as appended variables, `chmod 600` | | +| `deploy/server/deploy.sh`, `deploy/server/backup.sh` | to `/srv/swag/`, next to the compose file | | +| `deploy/server/swag/app.subdomain.conf` | to SWAG's `proxy-confs/`, renamed to `.subdomain.conf` | | + +`db.ps1` expects `server/` beside it, which is why `deploy/server/` is committed here as well as installed on the server. + +**Keep the `.ps1` files to ASCII.** Windows PowerShell 5.1 reads a script without a byte-order mark as Windows-1252, so an em dash becomes three characters ending in a closing quote: the string ends in the middle of itself and the script fails to parse before it runs a line. Write `--`. newsdesk hit this on a comment nobody thought was code, and `tests/test_packaging.py` here holds every script in this folder to it. + +`.gitignore` already covers `deploy/release.env` and `deploy/backups/`. + +## The docs + +They are one directory up, in [../docs/](../docs), because they are about the hosting rather than about these scripts. + +- **[../docs/hosting.md](../docs/hosting.md)** — what the box is, the seven steps to add a service, what the templates already decide for you, and the things that have already gone wrong on it. Start here for a service that is not on the box yet. +- **[../docs/release.md](../docs/release.md)** — the Windows side: `release.env`, the four numbered scripts, what each one refuses to do, and why the tests run in a throwaway worktree. +- **[../docs/server.md](../docs/server.md)** — installing, deploying, backups, moving a SQLite database in either direction, rolling back, and what to verify before relying on any of it. +- **[../docs/llm-backends.md](../docs/llm-backends.md)** — asking Claude from a deployed app: API key or subscription token, selected by which one is in the environment, with a self-contained reference implementation. +- **[../docs/password-protection.md](../docs/password-protection.md)** — one password on a public hostname: the lockout, sessions, passkeys, and how much throttling is actually worth building. `auth/` is the implementation of it. + +## A new service, in order + +1. DNS, then certificate, then registry login, then data directory — [../docs/hosting.md](../docs/hosting.md). +2. Copy `release.env.example` to `release.env` and fill it in. That is the only file this step touches: everything else is already committed. +3. `.\deploy\0-release.ps1` to test, promote, build and publish. +4. Install the server files, edit `.env`, `bash deploy.sh` — [../docs/server.md](../docs/server.md). +5. Put the model credential in `.env` if the app needs one — [../docs/llm-backends.md](../docs/llm-backends.md). +6. Get in, rotate the bootstrap password, register a passkey — [../docs/password-protection.md](../docs/password-protection.md). +7. Schedule `backup.sh`, then restore one of its backups somewhere else to prove it works. + +Step 1 of `0-release.ps1` runs `pytest` in a throwaway worktree, with `ruff` and `mypy` commented out beside it: this repository's dev group is pytest and httpx, and a gate whose tool is not installed fails every release for a reason that has nothing to do with the code. Add the tools, then uncomment the gates. + +## What this is not + +Not a framework and not a generator: nothing here is imported or extended, it is copied and edited. That is the point — three services with three edited copies beats three services sharing one abstraction that has to be right for all of them. What this repository changes is only *where the copy comes from*: `boilerplate/deploy/` is the source of truth, and news and health hold copies of it. diff --git a/deploy/_common.ps1 b/deploy/_common.ps1 new file mode 100644 index 0000000..0e5587a --- /dev/null +++ b/deploy/_common.ps1 @@ -0,0 +1,66 @@ +<# +.SYNOPSIS + Shared plumbing for the release scripts. Dot-source it; do not run it. + +.DESCRIPTION + Dot-sourcing runs in the caller's scope, so everything defined here -- + $root, $settings, Setting, Assert-Ok, Write-Step, Remove-ReleaseTemp -- is + the caller's afterwards. One copy of the release.env reader, because three + copies is three places for it to drift. +#> + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +#: The repository root, worked out from this file rather than from the working +#: directory, so the scripts can be run from anywhere. +$root = Split-Path -Parent $PSScriptRoot + +function Assert-Ok([string] $What) { + if ($LASTEXITCODE -ne 0) { throw "$What failed (exit $LASTEXITCODE)" } +} + +function Write-Step([string] $Text) { + Write-Host '' + Write-Host "== $Text" -ForegroundColor Cyan +} + +$envFile = Join-Path $PSScriptRoot 'release.env' +if (-not (Test-Path $envFile)) { + throw "no deploy\release.env - copy deploy\release.env.example and fill it in" +} +$settings = @{} +foreach ($line in Get-Content $envFile) { + if ($line -match '^\s*(#|$)') { continue } + # A line with no `=` is a typo, not a setting. Say so here rather than + # letting it surface later as a null reference three functions away. + if ($line -notmatch '=') { throw "release.env: not a KEY=VALUE line: $line" } + $key, $value = $line -split '=', 2 + $settings[$key.Trim()] = $value.Trim().Trim('"') +} + +function Setting([string] $Name, [string] $Default, [switch] $Required) { + if ($settings.ContainsKey($Name) -and $settings[$Name]) { return $settings[$Name] } + if ($Required) { throw "set $Name in deploy\release.env" } + return $Default +} + +#: What temporary directories this release is allowed to delete. Everything +#: below is created by these scripts and named for it. +$Name = Setting NAME -Required + +# Only ever deletes a -* entry directly under the temp directory. A +# guard, not a convenience: these paths are built by string concatenation and +# handed to `Remove-Item -Recurse -Force`, and one empty variable in that +# sentence is a very bad afternoon. +function Remove-ReleaseTemp([string[]] $Paths) { + $tempRoot = [IO.Path]::GetFullPath([IO.Path]::GetTempPath()).TrimEnd('\') + '\' + foreach ($path in $Paths) { + $resolved = [IO.Path]::GetFullPath($path) + if (-not $resolved.StartsWith($tempRoot, [StringComparison]::OrdinalIgnoreCase) -or + -not ([IO.Path]::GetFileName($resolved)).StartsWith("$Name-")) { + throw "Unsafe release temporary path: $resolved" + } + if (Test-Path -LiteralPath $resolved) { Remove-Item -LiteralPath $resolved -Recurse -Force } + } +} diff --git a/deploy/db.ps1 b/deploy/db.ps1 new file mode 100644 index 0000000..e6f3734 --- /dev/null +++ b/deploy/db.ps1 @@ -0,0 +1,142 @@ +<# +.SYNOPSIS + The database, in both directions. + +.DESCRIPTION + push this machine's database -> the server + pull a fresh snapshot from the server -> .\backups + + `push` is the one-off at the start: production begins knowing what you + already have, rather than starting empty. `pull` is the one you keep + running -- on a schedule from the server itself (server\backup.sh under + cron) and by hand from here whenever you want the copy somewhere the + server is not. + + `push` moves the database and nothing else. It does not stop the app before + or start it after: a deployment knows what else is moving and in what order, + and this does not. Stop the app on the server first -- the import is a file + swap under SQLite's feet and is refused while the app is up. Start it again + afterwards. + + Both go through SQLite's own `VACUUM INTO`. The database is in WAL mode, so + the .db file on disk is always missing whatever is still in the -wal beside + it, and copying the three files with the app running is a race you lose + quietly -- a database that opens fine and is short a day's writes. + + After a push the two copies diverge from the first thing written on either. + There is no merge for that and this does not pretend there is: pick the one + that is authoritative, and use the other read-only or not at all. + +.EXAMPLE + .\deploy\db.ps1 push + +.EXAMPLE + .\deploy\db.ps1 pull +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [ValidateSet('push', 'pull')] + [string] $Action, + + # Skip the confirmation before replacing the server's database. + [switch] $Yes +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +. "$PSScriptRoot/_common.ps1" +Set-Location $root + +$SshHost = Setting SSH_HOST -Required +$RemoteDir = Setting REMOTE_DIR -Required +# Where the data actually is. Beside the compose file unless APP_DATA put it +# under a shared root; then REMOTE_DATA has to say so. +$RemoteData = Setting REMOTE_DATA "$RemoteDir/data" +$Python = Setting PYTHON (Join-Path $root '.venv\Scripts\python.exe') +$LocalDb = Setting LOCAL_DB (Join-Path $root "$Name.db") + +switch ($Action) { + +# ---------------------------------------------------------------- push + +'push' { + if (-not (Test-Path $LocalDb)) { throw "no $LocalDb" } + + $stamp = Get-Date -Format 'yyyyMMdd-HHmmss' + $snap = Join-Path $root "$Name-$stamp.db.gz" + Write-Host "snapshotting $LocalDb" + + # One Python call for both halves. `VACUUM INTO` is the only way to get a + # consistent single file out of a WAL database, and Python is already the + # thing that has to be here -- PowerShell has no gzip that writes the format + # the server's `gunzip` expects. + $snapshot = @' +import gzip, pathlib, shutil, sqlite3, sys, tempfile +source, target = sys.argv[1], pathlib.Path(sys.argv[2]) +with tempfile.TemporaryDirectory() as work: + plain = pathlib.Path(work) / "snapshot.db" + db = sqlite3.connect(source) + try: + db.execute("VACUUM INTO ?", (str(plain),)) + finally: + db.close() + with plain.open("rb") as raw, gzip.open(target, "wb") as out: + shutil.copyfileobj(raw, out) +'@ + & $Python -c $snapshot $LocalDb $snap; Assert-Ok 'snapshot' + ' {0:N1} MB' -f ((Get-Item $snap).Length / 1MB) | Write-Host + + try { + Write-Host '' + Write-Host "This replaces the database on $SshHost with this one." -ForegroundColor Yellow + Write-Host 'Anything written there since is gone.' -ForegroundColor Yellow + Write-Host 'The app must already be stopped there, and stays stopped after.' -ForegroundColor Yellow + if (-not $Yes) { + $reply = Read-Host 'go ahead? [y/N]' + if ($reply -notmatch '^[yY]') { Write-Host 'stopped.'; return } + } + + scp -C $snap "${SshHost}:$RemoteData/incoming.db.gz"; Assert-Ok 'scp database' + scp (Join-Path $PSScriptRoot 'server\import_database.py') "${SshHost}:$RemoteDir/import_database.py" + Assert-Ok 'scp importer' + } finally { + Remove-Item -Force $snap -ErrorAction SilentlyContinue + } + + # Base64 rather than stdin: PowerShell has no `<` for a native command, and + # piping the text would rewrite the script's line endings on the way. + $b64 = [Convert]::ToBase64String([IO.File]::ReadAllBytes((Join-Path $PSScriptRoot 'server\import.sh'))) + ssh $SshHost "echo $b64 | base64 -d | bash -s -- '$RemoteDir'"; Assert-Ok 'import' + Write-Host "pushed. The app is still stopped; start it when the rest of" + Write-Host 'the deployment is in place:' + Write-Host " ssh $SshHost `"cd '$RemoteDir' && docker compose up -d --wait $Name`"" +} + +# ---------------------------------------------------------------- pull + +'pull' { + $into = Join-Path $root 'backups' + New-Item -ItemType Directory -Force -Path $into | Out-Null + + # The server's own copy of backup.sh, not a streamed one. It is the same + # script cron runs, it is already there, and it locates its own directory + # through BASH_SOURCE -- which is exactly the thing a script read from stdin + # does not have. import.sh takes its directory as an argument and so can be + # streamed; this one cannot. + Write-Host "snapshotting on $SshHost" + ssh $SshHost "cd '$RemoteDir' && bash backup.sh"; Assert-Ok 'backup' + + $newest = (ssh $SshHost "ls -1t '$RemoteData'/backups/$Name-*.db.gz | head -1").Trim() + Assert-Ok 'listing backups' + if (-not $newest) { throw 'no backup appeared on the server' } + + scp -C "${SshHost}:$newest" $into; Assert-Ok 'scp' + $file = Split-Path -Leaf $newest + Write-Host "backups\$file" + Write-Host '' + Write-Host 'To read it here, gunzip it, or in Python:' + Write-Host " import gzip, shutil; shutil.copyfileobj(gzip.open(r'backups\$file'), open('read-only.db','wb'))" +} + +} diff --git a/deploy/release.env.example b/deploy/release.env.example new file mode 100644 index 0000000..f45a704 --- /dev/null +++ b/deploy/release.env.example @@ -0,0 +1,28 @@ +# Copy to release.env beside these scripts and fill it in. Never committed: +# it names your registry and, for db.ps1, your server. +# +# Local main is the source. Fetching it first is your choice, not the script's. + +# The service's name. Used for the image tag, for the temporary directories +# these scripts create, and for the database file db.ps1 moves. One word, no +# spaces — it ends up in paths. +NAME=app + +IMAGE=git.rittershofer.org/wolf/app +PLATFORM=linux/amd64 +BRANCH=production +SOURCE=main + +# Defaults to the checkout's .venv/Scripts/python.exe. Only needed for the +# tests in step 1 and for db.ps1's snapshot. +# PYTHON=C:/Users/Coolkau/code/app/.venv/Scripts/python.exe + +# db.ps1 uses SSH. The numbered release scripts never do. REMOTE_DIR is the +# directory holding the compose file; REMOTE_DATA is where APP_DATA points, +# and defaults to REMOTE_DIR/data when left blank. +SSH_HOST= +REMOTE_DIR=/srv/swag +REMOTE_DATA=/srv/data/app + +# The database this machine pushes, if it is not /.db. +# LOCAL_DB=C:/Users/Coolkau/code/app/app.db diff --git a/deploy/server/backup.sh b/deploy/server/backup.sh new file mode 100644 index 0000000..1dcc78e --- /dev/null +++ b/deploy/server/backup.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# On the server, next to the compose file, or with its directory as $1. One +# snapshot of the database, taken through the running container, kept +# alongside the others. +# +# bash backup.sh # into $APP_DATA/backups +# 0 4 * * * bash /srv/swag/backup.sh >> /srv/swag/backup.log 2>&1 +# +# `VACUUM INTO` rather than copying the file: SQLite is in WAL mode, so the +# .db on disk is missing whatever is still in the -wal, and `cp` of a live +# database is a coin toss. VACUUM INTO takes a read lock, writes one +# self-contained and compacted file, and never blocks a writer for long enough +# to matter on a database of this size. +# +# Rename `app` and `APP_DATA` to your service's name. + +set -euo pipefail +cd "${1:-$(dirname "$(readlink -f "$0")")}" + +# Same source of truth as the compose file and deploy.sh. +DATA=$(sed -n 's/^[[:space:]]*APP_DATA=\([^#]*\).*/\1/p' .env | tail -1 | xargs || true) +DATA=${DATA:-./data} + +KEEP=${KEEP:-14} +stamp=$(date +%Y%m%d-%H%M%S) + +mkdir -p "$DATA/backups" + +docker compose exec -T app python -c " +import sqlite3 +db = sqlite3.connect('/data/app.db') +db.execute('VACUUM INTO ?', ('/data/backups/app-$stamp.db',)) +db.close() +" +gzip -f "$DATA/backups/app-$stamp.db" +echo "$DATA/backups/app-$stamp.db.gz $(du -h "$DATA/backups/app-$stamp.db.gz" | cut -f1)" + +# Oldest first, everything past KEEP goes. Named by timestamp, so sorting the +# names sorts them by age. +ls -1 "$DATA"/backups/app-*.db.gz 2>/dev/null \ + | sort \ + | head -n -"$KEEP" \ + | while read -r old; do rm -f "$old"; echo "removed $old"; done diff --git a/deploy/server/compose.yaml b/deploy/server/compose.yaml new file mode 100644 index 0000000..d9238d3 --- /dev/null +++ b/deploy/server/compose.yaml @@ -0,0 +1,60 @@ +# The service block, for the compose file that already runs your proxy. +# +# It needs to be on the same Docker network as SWAG, because that is how nginx +# reaches it — by container name, on the network Compose gives every project. +# Being in the same file is what puts it there; nothing else about it is +# related to the other services, and nothing else needs to be. +# +# Paste this `app:` block into that file's `services:`, add the variables from +# `env.example` to the `.env` next to it, and it is `docker compose up -d app` +# from then on — which recreates this container and no other. +# +# Rename `app` and `APP_` throughout to your service's name. + +services: + app: + image: ${APP_IMAGE:?set APP_IMAGE in .env}:${APP_TAG:-latest} + container_name: app + restart: unless-stopped + + # A plain directory on the host rather than a managed volume, so a backup + # is `tar` and it is owned by you rather than by root. Relative to this + # file by default; set APP_DATA to an absolute path to keep every + # container's data under one root, which is the thing a backup wants. + volumes: + - ${APP_DATA:-./data}:/data + + # The container follows the host directory's uid instead of the other way + # round — the same PUID/PGID the rest of the stack already runs with. + user: "${PUID:-1000}:${PGID:-1000}" + + environment: + TZ: ${TZ:-Europe/Berlin} + # Which model answers, decided here rather than in the image. With + # `backend: auto` the API key wins when there is one and the + # subscription CLI is the fallback, so filling in ANTHROPIC_API_KEY + # later switches it over with no rebuild. See docs/llm-backends.md. + CLAUDE_CODE_OAUTH_TOKEN: ${CLAUDE_CODE_OAUTH_TOKEN:-} + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} + # Normally empty. Set it to pin the password — the way back in when a + # login is lost — then unset it and recreate to hand control of the + # password back to the app. See docs/password-protection.md. + APP_PASSWORD: ${APP_PASSWORD:-} + + # No `ports:`. Nothing on the host reaches it; SWAG does, by container + # name, on the project's network. That is what keeps the app's port off + # the internet even if the firewall is wrong. + + # `deploy: resources: limits:` is Swarm-only and silently ignored by + # `docker compose up`. This is the one that works. 0 is unlimited, which + # is the honest default until the box has been watched under real load. + mem_limit: ${APP_MEM:-0} + + # The log is the record — nothing here writes a second copy of what went + # wrong — so it is worth keeping a few rotations of, and worth capping on + # a small disk. + logging: + driver: json-file + options: + max-size: "20m" + max-file: "5" diff --git a/deploy/server/deploy.sh b/deploy/server/deploy.sh new file mode 100644 index 0000000..c50fb73 --- /dev/null +++ b/deploy/server/deploy.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Run on the server, next to the compose file, or with its directory as $1. +# No remote shell required: the release scripts on Windows publish an image +# and stop there, and this is the other half. +# +# Rename `app` and `APP_DATA` to your service's name. +set -euo pipefail +cd "${1:-$(dirname "$(readlink -f "$0")")}" + +[[ -s .env ]] || { echo 'Install .env before deploying.' >&2; exit 1; } + +# Where the data directory is depends on APP_DATA, which the compose file also +# reads. Pulled out of .env rather than sourced, because Compose's .env is not +# a shell script and running it as one is how a stray quote becomes a surprise. +DATA=$(sed -n 's/^[[:space:]]*APP_DATA=\([^#]*\).*/\1/p' .env | tail -1 | xargs || true) +DATA=${DATA:-./data} + +[[ -s "$DATA/config.yaml" ]] || { + echo "Install $DATA/config.yaml before deploying." >&2 + exit 1 +} + +# Pull explicitly, here and nowhere else: the compose file has no pull policy, +# so starting or importing never fetches an image on its own. A registry +# failure must leave the running service alone, not deploy a cache. +echo 'Pulling the configured image…' +if ! docker compose pull app; then + echo 'Image pull failed; app was not restarted.' >&2 + exit 1 +fi +# Use the image just pulled. Do not start or recreate other services in the +# file. `--wait` holds until the container's own HEALTHCHECK passes. +if ! docker compose up -d --no-deps --pull never --wait --wait-timeout 240 app; then + docker compose ps app >&2 || true + echo 'Deployment failed. Check docker compose logs app.' >&2 + exit 1 +fi +docker compose ps app diff --git a/deploy/server/env.example b/deploy/server/env.example new file mode 100644 index 0000000..a927461 --- /dev/null +++ b/deploy/server/env.example @@ -0,0 +1,42 @@ +# The variables this service adds to the `.env` next to your compose file. +# Written by hand once, on the server, and never synced from a repository — it +# holds the secrets and it is the record of which image is running. +# +# chmod 600. Rename APP_ to your service's name throughout. + +APP_IMAGE=git.rittershofer.org/wolf/app +APP_TAG=latest # follows published images; pin a commit tag for rollback + +# Where the app keeps its database, its config and its backups. Relative to +# the compose file if you leave it out. Set it to an absolute path to put it +# under the one root every container on the box writes to — the thing that +# makes a backup one target instead of a hunt. +APP_DATA=/srv/data/app + +# The same PUID/PGID the rest of the stack runs as. The data directory must be +# owned by it. +PUID=1000 +PGID=1000 +TZ=Europe/Berlin + +# Which model answers. Set one or the other, not both — see +# docs/llm-backends.md for what each one means and what it costs. +# +# CLAUDE_CODE_OAUTH_TOKEN a year-long token for the Claude subscription, from +# `claude setup-token` on a machine you are signed in +# on. Nothing billed per token. +# ANTHROPIC_API_KEY per-token billing, no local CLI, the unattended +# path. Preferred by `auto` when both are set. +CLAUDE_CODE_OAUTH_TOKEN= +ANTHROPIC_API_KEY= + +# Leave empty. The first locked start generates a password and prints it to +# `docker compose logs app`; after that it is the app's Account screen's to +# change. Fill this in only to force one — it is the way back in — then empty +# it again and recreate, or the Account screen refuses to change what the +# environment pins. See docs/password-protection.md. +APP_PASSWORD= + +# Unlimited until the box has been watched through the heaviest thing the app +# does. Then something like 1500m. +APP_MEM=0 diff --git a/deploy/server/import.sh b/deploy/server/import.sh new file mode 100644 index 0000000..340552e --- /dev/null +++ b/deploy/server/import.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Called on the server after uploading incoming.db.gz and import_database.py. +# `db.ps1 push` does both and then runs this; by hand it is the same two files +# in the same two places. +# +# This replaces the database and does nothing else. It does not stop the app +# and it does not start it: whoever is running a deployment knows what else is +# moving and in what order, and a tool that helpfully restarts things underneath +# that is a tool fighting its caller. By hand it is +# `docker compose stop app` and `docker compose up -d --wait app` around this. +# +# What it will not do is import into a running app. The replacement is a file +# swap under SQLite's feet, and the live database is checkpointed first, which +# needs every writer gone — `import_database.py` refuses a busy database, and +# this refuses an app that is up, because the clearer error is the earlier one. +set -euo pipefail +cd "${1:?server directory required}" + +# Same source of truth as the compose file, deploy.sh and backup.sh. +DATA=$(sed -n 's/^[[:space:]]*APP_DATA=\([^#]*\).*/\1/p' .env | tail -1 | xargs || true) +DATA=${DATA:-./data} + +[[ -s "$DATA/incoming.db.gz" && -s import_database.py ]] + +if [[ -n "$(docker compose ps -q --status running app 2>/dev/null)" ]]; then + echo 'app is running. Stop it before importing:' >&2 + echo ' docker compose stop app' >&2 + exit 1 +fi + +# The one-off container uses compose's uid and its mounted /data. `--pull +# never`: an import runs on the image already here, whatever the installed +# compose file says about pulling. Getting a new image is deploy.sh's job. +docker compose run --rm --no-deps --pull never -T --entrypoint python app - < import_database.py +rm -f "$DATA/incoming.db.gz" diff --git a/deploy/server/import_database.py b/deploy/server/import_database.py new file mode 100644 index 0000000..f7e606a --- /dev/null +++ b/deploy/server/import_database.py @@ -0,0 +1,77 @@ +"""Offline database replacement, run as the compose service's uid. + +The app container must be stopped. Only stdlib is used so this can be sent +over stdin to either the old or the new image during a deployment. + +Two things to edit for your service: `DATABASE` and `REQUIRED_TABLES`. The +table check is what stops a right-shaped-but-wrong database — last project's, +or a half-downloaded one — from landing on top of a good one. +""" + +from __future__ import annotations + +import gzip +import os +import shutil +import sqlite3 +import tempfile +from contextlib import closing +from datetime import UTC, datetime +from pathlib import Path + +DATABASE = Path("/data/app.db") +INCOMING = Path("/data/incoming.db.gz") +BACKUPS = Path("/data/backups") + +#: Any one of these missing means this is not our database. Name tables that +#: have existed since the first schema, not ones a recent migration added. +REQUIRED_TABLES = {"items"} + + +def replace_database(database: Path, archive: Path, backups: Path) -> Path | None: + """Validate first, snapshot the stopped database, then atomically replace it.""" + fd, name = tempfile.mkstemp(prefix=".incoming-", suffix=".db", dir=database.parent) + staged = Path(name) + backup = None + try: + with os.fdopen(fd, "wb") as output, gzip.open(archive, "rb") as source: + shutil.copyfileobj(source, output) + output.flush() + os.fsync(output.fileno()) + with closing(sqlite3.connect(staged)) as candidate: + if candidate.execute("PRAGMA integrity_check").fetchall() != [("ok",)]: + raise ValueError("Incoming database failed integrity_check") + tables = {row[0] for row in candidate.execute( + "SELECT name FROM sqlite_master WHERE type='table'")} + if not REQUIRED_TABLES <= tables: + raise ValueError("Incoming file is not this application's database") + + if database.exists(): + backups.mkdir(parents=True, exist_ok=True) + stamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S-%f") + snapshot = backups / f"{database.stem}-{stamp}.db" + backup = snapshot.with_suffix(".db.gz") + with closing(sqlite3.connect(database)) as current: + current.execute("VACUUM INTO ?", (str(snapshot),)) + # Refuse a competing connection instead of removing a live WAL. + busy, _, _ = current.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() + if busy: + raise RuntimeError("Database is busy; stop every writer before importing") + # Keep the uncompressed recovery copy if compression fails. + with snapshot.open("rb") as source, gzip.open(backup, "xb") as output: + shutil.copyfileobj(source, output) + with backup.open("r+b") as saved: + os.fsync(saved.fileno()) + snapshot.unlink() + + for suffix in ("-wal", "-shm"): + Path(str(database) + suffix).unlink(missing_ok=True) + os.replace(staged, database) + return backup + finally: + staged.unlink(missing_ok=True) + + +if __name__ == "__main__": + saved = replace_database(DATABASE, INCOMING, BACKUPS) + print(f"Database replaced; previous snapshot: {saved or 'none (first import)'}") diff --git a/deploy/server/swag/app.subdomain.conf b/deploy/server/swag/app.subdomain.conf new file mode 100644 index 0000000..9bc8d0e --- /dev/null +++ b/deploy/server/swag/app.subdomain.conf @@ -0,0 +1,64 @@ +## Rename to .subdomain.conf, copy to SWAG's /config/nginx/proxy-confs/, +## validate, reload. +## +## The subdomain has to exist before this does anything: add to SWAG's +## SUBDOMAINS (or run it with a wildcard cert) and point the DNS record at the +## same address as the rest of the stack. See docs/hosting.md. + +## Per-IP throttle for the login endpoints, if the app has a password. Declared +## here at `http` scope — SWAG includes proxy-confs outside any server block, +## which is what makes this legal in this file. 1 MB of zone holds about 16,000 +## addresses. Delete both this and the `location` below if there is no login. +limit_req_zone $binary_remote_addr zone=app_login:1m rate=6r/m; + +server { + listen 443 ssl; + listen [::]:443 ssl; + + server_name app.rittershofer.org; + + include /config/nginx/ssl.conf; + # Put proxy defaults at server scope so location timeouts override them + # through inheritance, rather than duplicate directives from proxy.conf. + include /config/nginx/proxy.conf; + + # 0 is no limit. Set a real one if strangers can post to this app; leave it + # off when the only uploads are yours and can be large. + client_max_body_size 0; + + # Guessing the password is already pointless if the app locks out after a + # handful of wrong answers (see docs/password-protection.md). This is the + # cheap half of the rest: attempts are slowed per address, at the proxy, + # before they reach the app at all. `burst=5 nodelay` lets a person who + # fat-fingers it a few times through immediately and then holds the rate. + location ~ ^/login(/passkey)?$ { + limit_req zone=app_login burst=5 nodelay; + limit_req_status 429; + + include /config/nginx/resolver.conf; + set $upstream_app app; + set $upstream_port 8787; + set $upstream_proto http; + proxy_pass $upstream_proto://$upstream_app:$upstream_port; + } + + location / { + include /config/nginx/resolver.conf; + + set $upstream_app app; + set $upstream_port 8787; + set $upstream_proto http; + proxy_pass $upstream_proto://$upstream_app:$upstream_port; + + # Only if the app holds a WebSocket open. SWAG's proxy.conf already + # sets the Upgrade headers; what it also sets is a read timeout of a + # couple of minutes, which would close every open tab's socket on the + # minute and leave the browser reconnecting for no reason. + proxy_read_timeout 86400s; + proxy_send_timeout 86400s; + } +} + +## `listen 443 ssl` without `http2` is correct — `http2 on;` is set once at +## `http` scope in nginx.conf, and the `listen ... http2` parameter is +## deprecated in nginx 1.25 and later. diff --git a/docs/box-docker-compose.yml b/docs/box-docker-compose.yml new file mode 100644 index 0000000..8e82067 --- /dev/null +++ b/docs/box-docker-compose.yml @@ -0,0 +1,175 @@ +services: + + mealie: + image: ghcr.io/mealie-recipes/mealie:latest + container_name: mealie + restart: unless-stopped + + # `deploy: resources: limits:` is Swarm-only and was silently ignored, + # so this container has never actually had a memory limit. This one works. + mem_limit: 1000m + + # Was 9925:9000 on every interface — the internet, if the firewall is ever + # wrong. SWAG reaches it by container name on the shared network; this is + # only the fallback for when you are on the box itself. + ports: + - "127.0.0.1:9925:9000" + + # Was a Docker-managed volume under /var/lib/docker, root-owned and + # invisible. A plain directory next to this file, so a backup is `tar`. + volumes: + - /srv/data/mealie:/app/data + + environment: + ALLOW_SIGNUP: "false" + PUID: 1000 + PGID: 1000 + TZ: Europe/Brussels + MAX_WORKERS: 1 + WEB_CONCURRENCY: 1 + BASE_URL: https://mealie.rittershofer.org + # From .env, chmod 600. Never in this file. + OPENAI_API_KEY: ${OPENAI_API_KEY:?set OPENAI_API_KEY in .env} + # A base, not an endpoint. The old /v1/models made it request + # /v1/models/chat/completions. + OPENAI_BASE_URL: https://api.openai.com/v1 + + logging: + driver: json-file + options: + max-size: "20m" + max-file: "5" + + gitea: + image: gitea/gitea:latest + container_name: gitea + restart: unless-stopped + environment: + - USER_UID=1000 + - USER_GID=1000 + volumes: + - /srv/data/gitea:/data + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro + logging: + driver: json-file + options: + max-size: "20m" + max-file: "5" + + newsdesk: + image: ${NEWSDESK_IMAGE:?set NEWSDESK_IMAGE in .env}:${NEWSDESK_TAG:-latest} + container_name: newsdesk + restart: unless-stopped + user: "${PUID:-1000}:${PGID:-1000}" + volumes: + - /srv/data/newsdesk:/data + environment: + TZ: ${TZ:-Europe/Berlin} + CLAUDE_CODE_OAUTH_TOKEN: ${CLAUDE_CODE_OAUTH_TOKEN:-} + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} + NEWSDESK_PASSWORD: ${NEWSDESK_PASSWORD:-} + mem_limit: ${NEWSDESK_MEM:-0} + logging: + driver: json-file + options: + max-size: "20m" + max-file: "5" + + bookly: + image: git.rittershofer.org/wolf/bookly:${BOOKLY_TAG:-latest} + container_name: bookly + restart: unless-stopped + user: "1000:1000" + + volumes: + - /srv/data/bookly:/data + + environment: + TZ: Europe/Brussels + # Required: without it the demo would be open to anyone who finds the hostname, so Compose refuses to start. + BOOKLY_PASSWORD: ${BOOKLY_PASSWORD:?set BOOKLY_PASSWORD in .env} + # subscription: the Claude CLI with CLAUDE_CODE_OAUTH_TOKEN (same variable newsdesk reads). + # api: ANTHROPIC_API_KEY, billed per token, lower latency. + BOOKLY_BACKEND: ${BOOKLY_BACKEND:-subscription} + CLAUDE_CODE_OAUTH_TOKEN: ${CLAUDE_CODE_OAUTH_TOKEN:-} + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} + + # No `ports:`. SWAG reaches :8000 by container name; nothing else can. + + mem_limit: 0 + + logging: + driver: json-file + options: + max-size: "20m" + max-file: "5" + + health: + image: ${HEALTH_IMAGE:?set HEALTH_IMAGE in .env}:${HEALTH_TAG:-latest} + container_name: health + restart: unless-stopped + + # Warehouse, journal, job table, speech models, tiles, backups: all of it + # under one host directory, so a backup is one target. + volumes: + - ${HEALTH_DATA:-./data/health}:/data + + user: "${PUID:-1000}:${PGID:-1000}" + + environment: + TZ: ${TZ:-Europe/Berlin} + HEALTH_TIMEZONE: ${TZ:-Europe/Berlin} + # Which model writes the evening entry. The subscription token now; add + # ANTHROPIC_API_KEY and `docker compose up -d health` (not `restart`) + # to move to the API. Shared with the other services in this file. + CLAUDE_CODE_OAUTH_TOKEN: ${CLAUDE_CODE_OAUTH_TOKEN:-} + ANTHROPIC_API_KEY: ${HEALTH_ANTHROPIC_API_KEY:-} + # Normally empty. Set it to pin the password -- the way back in when a + # login is lost -- then empty it and recreate to hand the password back + # to the Settings page. + HEALTH_PASSWORD: ${HEALTH_PASSWORD:-} + # The speech model. `small` (0.7 GB) if turbo (1.9 GB while it runs) + # does not fit beside everything else on the box. + HEALTH_WHISPER_MODEL: ${HEALTH_WHISPER_MODEL:-large-v3-turbo} + + # No `ports:` and no `networks:`: SWAG reaches it by name on this file's + # network, and nothing else reaches it at all. + + # `deploy: resources: limits:` is Swarm-only and ignored here. 0 is + # unlimited until a transcription has been watched on the box. + mem_limit: ${HEALTH_MEM:-0} + + logging: + driver: json-file + options: + max-size: "20m" + max-file: "5" + + swag: + image: lscr.io/linuxserver/swag:latest + container_name: swag + cap_add: + - NET_ADMIN + environment: + - PUID=1000 + - PGID=1000 + - TZ=Europe/Brussels + - URL=rittershofer.org + - VALIDATION=http + # Commas and nothing else — SWAG splits on the comma alone, so a space + # asks for a hostname with a leading space. `news` added for newsdesk; + # `collabora` dropped, since nothing serves it any more. + - SUBDOMAINS=www,mealie,git,news,play,body + - ONLY_SUBDOMAINS=false + volumes: + - /etc/config/swag:/config + ports: + - 443:443 + - 80:80 + logging: + driver: json-file + options: + max-size: "20m" + max-file: "5" + restart: unless-stopped diff --git a/docs/hosting.md b/docs/hosting.md new file mode 100644 index 0000000..e37f240 --- /dev/null +++ b/docs/hosting.md @@ -0,0 +1,112 @@ +# Deploying a service to rittershofer.org + +The ground every service on this box shares: what the box is, the seven steps to add a service to it, the templates those steps need, and the things that have already gone wrong. [release.md](release.md) is how an image gets built and published; [server.md](server.md) is what happens once it has been. + +`` below is your *service's* name: the hostname, the data directory, the image tag, the compose service. It is the one thing you choose. The Python package stays `app` and the environment variables stay `APP_`, here and in every project started from this repository — nothing inside the checkout is renamed, and the files these steps need are already in place under `deploy/`. + +## What the box is + +A single Hetzner server running Ubuntu 24.04 on x86_64, with a 37 GB root filesystem and no separate data disk. One Docker Compose project holds everything. + + /srv/swag/docker-compose.yml every service, one file, project name `swag` + /srv/swag/.env every secret, chmod 600, never in the compose file + /srv/data// every service's data, owned 1000:1000 + /etc/config/swag/ SWAG's /config: nginx confs, certificates, logs + +A snapshot of that whole compose file, every neighbour included, is [box-docker-compose.yml](box-docker-compose.yml): what a new service block lands beside, and where the SWAG subdomain list lives. + +`swag` is nginx, certbot and fail2ban in one container. It terminates TLS for every hostname and proxies to the others **by container name** over the project's default network. That shared network is the only reason all these services live in one compose file — nothing else about them is related. + +Images are published to the Gitea instance on the same box, at `git.rittershofer.org/wolf/`. Everything runs as uid/gid 1000 and `TZ=Europe/Brussels`. + +Which compose file that is comes off the running container rather than out of memory: + +```bash +docker inspect swag --format '{{index .Config.Labels "com.docker.compose.project.config_files"}}' +``` + +Declaring the network `external` from a second Compose project would work too, and would hardcode the first project's name into the second — a rename away from a confusing failure. + +## The seven steps + +**1. DNS.** `.rittershofer.org` must resolve to the server *before* anything else. Confirm with `getent hosts .rittershofer.org`. + +**2. Certificate.** Add `` to `SUBDOMAINS` in the swag service's environment. That list separates on commas and nothing else — a space after one asks for a hostname with a leading space. SWAG runs certbot on every container start and requests **one certificate covering every name in the list**, so a name that does not answer on port 80 fails the whole request and leaves the existing names unrenewed. This is why step 1 comes first. Restart swag, then check `docker logs swag` for the issuance. If SWAG already holds a wildcard certificate covering the name, reuse it and skip this step entirely. + +**3. Registry login.** On the build machine a Gitea token with package **write**; on the server, package **read**. Same command both ends, token at the password prompt: + +```bash +docker login git.rittershofer.org -u wolf +``` + +Git credentials and Docker registry credentials are separate. A successful Git push does not log Docker in. + +**4. Data directory.** Root's `mkdir` produces a root-owned directory, and the container runs as 1000, so the chown is not optional: + +```bash +sudo mkdir -p /srv/data/ && sudo chown -R 1000:1000 /srv/data/ +``` + +**5. Service block** from [../deploy/server/compose.yaml](../deploy/server/compose.yaml) into `/srv/swag/docker-compose.yml`, and the variables from [../deploy/server/env.example](../deploy/server/env.example) appended to `/srv/swag/.env`. + +**6. Proxy conf** — [../deploy/server/swag/app.subdomain.conf](../deploy/server/swag/app.subdomain.conf) renamed to `.subdomain.conf` in `/etc/config/swag/nginx/proxy-confs/`. SWAG's `/config` is a host directory bind-mounted into the container; read the host side off the running container rather than assuming where it is: + +```bash +docker inspect swag --format '{{range .Mounts}}{{if eq .Destination "/config"}}{{.Source}}{{end}}{{end}}' +``` + + /nginx/proxy-confs/ the per-site confs, where yours goes + /nginx/proxy.conf the shared defaults every site includes + /nginx/ssl.conf certificate paths and TLS settings + /nginx/resolver.conf Docker's internal DNS + /etc/letsencrypt/live/ the certificates themselves + +Then validate, and reload only if validation passes: + +```bash +docker exec swag nginx -t && docker exec swag nginx -s reload +``` + +Confirm the other sites still respond afterwards. + +**7. Deploy.** Name the service, so nothing else in the file is recreated: + +```bash +cd /srv/swag && docker compose up -d +``` + +## What the templates already decide for you + +Both templates are in [../deploy/server/](../deploy/server), commented in place. The decisions worth knowing before you read them: + +**No `ports:`.** SWAG reaches the container by name on the project network. That is what keeps the app's port off the internet even if the firewall is wrong. + +**No `networks:` either.** Being in that compose file is what puts it on the right one. + +**`user: "1000:1000"`.** Container uid is host uid — there is no translation layer. Without this the process runs as whatever the image declares, usually root, and everything it writes into `/srv/data` lands root-owned. + +**`mem_limit:`, not `deploy: resources: limits:`.** The latter is Swarm-only and silently ignored by `docker compose up`. 0 is unlimited, which is the honest default until the box has been watched under real load. + +**`logging:` with a size cap.** Without it, json-file logs grow until the disk does not. + +**`listen 443 ssl` without `http2`.** `http2 on;` is set once at `http` scope in `nginx.conf`, and the `listen ... http2` parameter is deprecated in nginx 1.25 and later. + +**`proxy_read_timeout 86400s`** only if the app holds a WebSocket open. SWAG's `proxy.conf` already sets the Upgrade headers; what it also sets is a read timeout of a couple of minutes, which closes an idle socket on the minute. Delete those two lines if there is no socket. + +## Things that have already bitten + +**Disk.** 37 GB, and it has been full. Three causes, all recurring: `:latest` orphans the previous image on every pull and nothing cleans them up (a weekly `docker image prune -f` cron handles it — no `-a`, so pinned images survive); journald defaults to 10% of the filesystem and is capped here at 200 MB; and VS Code Remote installs a fresh server per version under `~/.vscode-server` forever. + +**`cp -a "$SRC/."` with an unset `SRC`** copies the root filesystem. Use `"${SRC:?}"` and echo the value before running anything destructive with it. + +**SWAG never touches your live confs**, only the `.sample` files beside them. That is deliberate and it means they drift for years. Diff them occasionally: + +```bash +cd /etc/config/swag/nginx && for f in *.conf site-confs/*.conf proxy-confs/*.subdomain.conf; do echo "=== $f"; diff "$f" "$f.sample"; done +``` + +Take the samples wholesale and re-apply your own edits. `nginx.conf` and the site/proxy confs must be updated **together** — older ones include `ssl.conf` at `http` scope, newer ones per server block, and mixing the two produces `no "ssl_certificate" is defined`. + +**SQLite is never safe to copy live.** Stop the container, or snapshot with `VACUUM INTO` through the running one. A `cp` of a WAL-mode database is a coin toss. See [server.md](server.md#the-database). + +**Secrets in `.env`, never in the compose file.** It is easy to paste a compose file into a chat window; it is harder to do that by accident with a file you had to `cat` deliberately. diff --git a/docs/llm-backends.md b/docs/llm-backends.md new file mode 100644 index 0000000..621ef3d --- /dev/null +++ b/docs/llm-backends.md @@ -0,0 +1,241 @@ +# Asking Claude, from a deployed app + +One question — *ask Claude this, get JSON back* — and two ways to answer it, chosen by which credential is present in the environment. The caller asks without knowing which one ran. + + api One call to the Claude API. Wants ANTHROPIC_API_KEY. No local + install, no subprocess, per-token billing. The unattended path. + claude_code The Claude Agent SDK drives the `claude` CLI, on the + subscription it is signed in with, authenticated by + CLAUDE_CODE_OAUTH_TOKEN. Nothing billed per token. + +This is the reusable part of the deployment: the compose file passes both variables through, the app prefers the key when there is one, and switching from the subscription to the API is a line in `.env` and `docker compose up -d` — no rebuild, no code change. + +## The two credentials + +**`CLAUDE_CODE_OAUTH_TOKEN`** comes from `claude setup-token`, run interactively on a machine you are already signed in on. It lasts a year. Generating it needs browser authorization, so no deployment script can automate it — you produce it on your laptop and paste it into the server's `.env` with an editor. Then `docker compose up -d app`, because a `restart` will not pick it up. + +**`ANTHROPIC_API_KEY`** is per-token billing from the Console. Nothing to renew, nothing to sign in to, and it is what production should run on once the thing is real. + +Set one. If both are set, `auto` takes the key. Neither, and every call fails with a notice naming the reason, which is the correct outcome — better a batch that fails saying "no credentials" than one that waits for a human who is not there. + +The API key and the OAuth token are not interchangeable and do not substitute for each other. `ANTHROPIC_API_KEY` in the environment does not make the CLI work, and an OAuth token does not authenticate an API call. + +**Neither belongs in the image or in the compose file.** They live in the `.env` beside it, `chmod 600`, injected as environment variables at container creation. A compose file gets pasted into chat windows; a file you had to `cat` deliberately does not. + +## Why there is a `backend` setting and not just detection + +Detection answers "what can run here". The setting answers "what should". They differ in two cases worth naming: forcing `api` on a machine that also has the CLI, so a benchmark measures the thing you meant; and `off`, which makes everything needing a model fail on purpose, with a notice that says which switch did it. An app that silently degrades to no-model is worse than one that stops. + +## What the `claude_code` path deliberately does not get + +No tools (`tools=[]`, `allowed_tools=[]`), no filesystem settings (`setting_sources=None`, so no CLAUDE.md, no skills, no MCP servers leak into a classification call), no session to resume. One text-in, JSON-out turn — the same shape as the API call it stands in for. The CLI is built for agent work, and left to its defaults it will bring all of that with it. + +## Thinking has to be stated, not defaulted + +The CLI thinks unprompted; the Messages API thinks only when asked. So "say nothing about thinking" means opposite things on the two sides, and the two backends stop being interchangeable. State it explicitly to both. It is not a small difference: on a batch of twenty-five short classifications, the CLI's default cost 128 s against 17 s for the same answers. + +Some models support manual extended thinking only and reject `adaptive`. Check before spending a call, not after. + +## Both take the same JSON schema + +The Agent SDK accepts the Messages API's `json_schema` output format and hands the parsed object back on `ResultMessage.structured_output`. That is what makes this two backends rather than two implementations: neither path is the degraded one, and moving between them is a config line. + +## Reference implementation + +Self-contained — no imports from your project, both SDKs optional. It is the shape, not the whole thing: the real one in newsdesk also streams partial answers out for a progress bar, retries, and classifies API errors by status code so a 401 and a 429 tell you different things. + +```python +"""Ask Claude a question, constrained to a JSON schema, get a dict back. + +Two backends, one seam. `ANTHROPIC_API_KEY` selects the API; otherwise the +Claude Agent SDK drives the `claude` CLI on a subscription, authenticated by +`CLAUDE_CODE_OAUTH_TOKEN`. Both are optional dependencies: + + uv pip install anthropic # the api backend + uv pip install claude-agent-sdk # the claude_code backend +""" + +from __future__ import annotations + +import asyncio +import importlib.util +import json +import os +from dataclasses import dataclass +from typing import Any + + +class AgentError(Exception): + """The backend was there and the call still did not produce an answer.""" + + +@dataclass(slots=True) +class AgentConfig: + #: "auto", "api", "claude_code", or "off" to make every call fail. + backend: str = "auto" + model: str = "claude-haiku-4-5-20251001" + timeout: float = 300.0 + max_tokens: int = 8192 + #: "off" or "adaptive". Stated to both backends, never defaulted. + thinking: str = "off" + + +@dataclass(slots=True) +class Reply: + data: dict[str, Any] + backend: str + model: str + usage: dict[str, Any] + + +def backend(settings: AgentConfig) -> str: + """Which backend will answer, or "" for nobody.""" + if settings.backend == "off": + return "" + if settings.backend in ("api", "claude_code"): + return settings.backend + if any(os.environ.get(name) for name in + ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_PROFILE")): + return "api" + # The SDK bundles its own `claude`, so having the package is enough. Not + # proof the call will work — the CLI may be signed out — but that surfaces + # as an AgentError on the call, which the caller handles anyway. + if importlib.util.find_spec("claude_agent_sdk") is not None: + return "claude_code" + return "" + + +def _thinking(setting: str) -> dict[str, str]: + return {"type": "adaptive"} if setting == "adaptive" else {"type": "disabled"} + + +def ask_json(settings: AgentConfig, *, system: str, question: dict[str, Any], + schema: dict[str, Any]) -> Reply: + """Ask `question`, constrained to `schema`, and return the parsed answer. + + `system` should be byte-identical across the calls in a run — it is what + the API path caches. `question` is the part that changes. + """ + name = backend(settings) + if not name: + raise AgentError("no model backend is available: set ANTHROPIC_API_KEY " + "or install claude-agent-sdk with CLAUDE_CODE_OAUTH_TOKEN") + if name == "api": + return _ask_api(settings, system, question, schema) + return _ask_claude_code(settings, system, question, schema) + + +def _ask_api(settings: AgentConfig, system: str, question: dict[str, Any], + schema: dict[str, Any]) -> Reply: + try: + import anthropic + except ImportError as exc: + raise AgentError("the anthropic SDK isn't installed") from exc + + try: + client = anthropic.Anthropic(timeout=settings.timeout) + response = client.messages.create( + model=settings.model, + max_tokens=settings.max_tokens, + # The instructions are byte-identical across every call; only the + # payload below changes. Caching them is most of the saving. + system=[{"type": "text", "text": system, + "cache_control": {"type": "ephemeral"}}], + messages=[{"role": "user", "content": json.dumps(question)}], + thinking=_thinking(settings.thinking), + output_config={"format": {"type": "json_schema", "schema": schema}}, + ) + except anthropic.APIError as exc: + # Carry the status through in the text: 401 ("your key is wrong") and + # 429 ("slow down") want different reactions, and the message body + # alone does not always separate them. + status = getattr(exc, "status_code", "") + raise AgentError(f"the API call failed: {status} {exc}".strip()) from exc + + text = next((b.text for b in response.content if b.type == "text"), "") + return Reply(data=json.loads(text), backend="api", + model=response.model or settings.model, + usage=response.usage.model_dump() if response.usage else {}) + + +def _ask_claude_code(settings: AgentConfig, system: str, question: dict[str, Any], + schema: dict[str, Any]) -> Reply: + try: + from claude_agent_sdk import ( + ClaudeAgentOptions, ClaudeSDKError, ResultMessage, query, + ) + except ImportError as exc: + raise AgentError("the Claude Agent SDK isn't installed") from exc + + options = ClaudeAgentOptions( + system_prompt=system, + model=settings.model or None, + # A classification call, not an agent: nothing to run, nothing to read, + # and no project settings — so no CLAUDE.md, no skills and no MCP + # servers get in front of the model on the way to an answer. + tools=[], + allowed_tools=[], + permission_mode="dontAsk", + setting_sources=None, + thinking=_thinking(settings.thinking), + output_format={"type": "json_schema", "schema": schema}, + # The API takes max_tokens as a parameter; the CLI takes it only as an + # environment variable. Without this the setting is quietly API-only. + # The SDK merges this into the inherited environment rather than + # replacing it, so naming one variable does not cost the CLI its PATH. + env={"CLAUDE_CODE_MAX_OUTPUT_TOKENS": str(settings.max_tokens)}, + ) + + async def ask() -> Reply: + answer: Reply | None = None + async for message in query(prompt=json.dumps(question), options=options): + if not isinstance(message, ResultMessage): + continue + if message.is_error: + raise AgentError("; ".join(message.errors or []) or message.result + or f"the agent ended as {message.subtype}") + data = message.structured_output + if not isinstance(data, dict): + # Ended without structured output — a limit, an interruption. + # `result` is the raw text if there is any, worth one attempt. + data = json.loads(message.result or "") + answer = Reply(data=data, backend="claude_code", + model=settings.model, usage=message.usage or {}) + if answer is None: + raise AgentError("the agent returned no result") + return answer + + try: + # asyncio.run, so this is a synchronous call like the other backend. + # From inside a running loop, await `ask()` directly instead. + return asyncio.run(asyncio.wait_for(ask(), settings.timeout)) + except TimeoutError as exc: + raise AgentError(f"no answer within {settings.timeout:.0f}s") from exc + except ClaudeSDKError as exc: + # The SDK has its own exception hierarchy — a CLI that exited, a line + # it could not parse, an answer that overran max_tokens — and none of + # it is an AgentError. Left alone, those escape a caller that catches + # AgentError per item and take down a whole run. Every failure of this + # backend is an AgentError or none of them are. + raise AgentError(str(exc)) from exc +``` + +Using it: + +```python +settings = AgentConfig() +schema = { + "type": "object", + "properties": {"tags": {"type": "array", "items": {"type": "string"}}}, + "required": ["tags"], +} +reply = ask_json(settings, system="Tag this article. Answer with the schema.", + question={"title": title, "body": body}, schema=schema) +print(reply.data["tags"], reply.backend) +``` + +## The two failures that will actually happen + +**A changed token in `.env` appears to do nothing.** `docker compose restart` does not update environment variables. `docker compose up -d app` recreates the container, which does. + +**The token expired and nothing said so.** The setup token lasts a year, and the failure looks like a model call failing rather than a credential problem. Read the error text — this is why the SDK's own message is passed through intact rather than summarised into "model call failed". diff --git a/docs/password-protection.md b/docs/password-protection.md new file mode 100644 index 0000000..92a07d3 --- /dev/null +++ b/docs/password-protection.md @@ -0,0 +1,106 @@ +# One password, on the open internet + +The shape used by a single-user app that has to sit on a public hostname because that is where the browser is. No user management, no usernames, no registration: one password and the sessions it opens. Everything below is about making that enough. + +This is the reasoning; `auth/` in this repository is the code, already written and already wired into `app/main.py`. `auth/README.md` is the ten-minute version of how to take the folder somewhere else. + +## Locked by what it is bound to, not by a flag + +The switch is the bind address. On `127.0.0.1` the door is already the machine's own login, and asking for a password again is theatre. On `0.0.0.0` it is reachable from elsewhere and locks itself. + +Deriving it rather than configuring it removes the failure that looks fine: "deployed it, forgot to switch auth on". The production config then says `enabled: always` anyway — redundant with the bind address and worth saying out loud. + +```yaml +auth: + enabled: always # auto | always | never + session_days: 30 + # `auto` marks the cookie Secure when the request arrived over HTTPS. Behind + # a TLS-terminating proxy the app only ever sees plain HTTP and cannot work + # that out, so behind SWAG this must be `always`. + secure_cookie: always +``` + +## The password itself + +Stored as an scrypt hash, never in the config file. The first locked start either takes `APP_PASSWORD` from the environment or generates one and prints it to the log once. After that it is the app's own Account screen that changes it. + +`APP_PASSWORD` is the way back in when a login is lost, and it is authoritative while it is set — the Account screen refuses to change what the environment pins. Set it, get in, empty it, recreate the container, then change the password in the app. + +Two consequences worth writing down: a generated password is printed to the container log, so the retained log holds it until it rotates — rotate the password once you are in. And a password the *user* chooses is the weak spot, not the lockout: a minimum length of 8 is a very different thing from the 16 random characters the app generates. + +## The lockout + +Five wrong answers are free — past a typo and a stale password manager entry, short of anything systematic. After that each attempt waits, starting at 30 s and doubling to a ceiling of 15 minutes. That is about a hundred guesses a day against a secret with far more than a hundred possibilities, which is what "guessing is pointless" means in practice. + +Three details that matter more than the numbers: + +**Read the lockout before checking the password, not after.** Otherwise a fast attacker extends their own lockout forever and never notices, and — more to the point — an honest person hammering the button digs a deeper hole with every press. + +**A wrong answer still costs a full scrypt derivation.** The delay an attacker meets is the one the honest user meets, and it is what lets the free-attempt count be as generous as it is. + +**Cap the doubling.** `30s * 2**failures` with an uncapped counter eventually overflows whatever holds it, and a bot that has sent ten thousand guesses turns every future login attempt into a 500 — a denial of service delivered by arithmetic. + +## One counter, not one per address + +The obvious improvement is to lock out the address that is guessing, so a stranger cannot keep the owner waiting at their own door. Behind a proxy it cannot be built honestly: the only address the app sees is the one the proxy puts in a header, the peer is a container on the same Docker network, and a header is set by whoever is talking. Keyed on that, an attacker gets a fresh, empty counter per request by editing one line of their own request, while the owner still gets locked out by their own typos. Strictly worse, and it would look like protection. + +So the address is **logged, not decided from** — at the layer that knows who connected, the decision can be made properly. See the throttling section below. + +The honest cost of one global counter: an attacker hammering the door keeps password login locked for the owner too. A passkey login that skips the lockout is what makes that survivable. + +## Sessions + +One kind of session whichever door opened it, so revoking and expiry have one implementation. A row per session with its user agent and last-seen time, a screen listing them, a button to revoke one and a button to close all the others. `purge()` on startup drops the expired ones — otherwise that table grows forever. + +The cookie: `HttpOnly` so an XSS cannot read it, `Secure` per the config above, `SameSite=Lax`, `path=/`. + +**No CSRF token, and that is a decision rather than an omission.** A CSRF token defends a cookie-authenticated app against another origin submitting a form on your behalf. `SameSite=Lax` already refuses to send the cookie on any cross-site request that is not a top-level navigation, and if no `GET` in the app changes anything, the attack the token would stop cannot be mounted. Adding one anyway means a second secret, a second thing to rotate, and a second way for a request to fail that the UI has to explain. Where that reasoning stops holding is a browser old enough to ignore `SameSite` entirely. + +## Passkeys + +Here already, behind the `passkeys` extra: `uv sync --all-extras`, and without it the routes answer 501 and the Account tab simply does not draw the section. + +A passkey and a security key are one feature: a YubiKey registers through the same flow as a phone, because to WebAuthn they are the same thing. + +A passkey is bound to an origin, and that binding is what makes it unphishable — the browser only hands the credential back to the origin that created it. So it needs HTTPS and a real hostname, with `localhost` the one exception, which is what lets it be tried before deploying. Behind a proxy, state both explicitly: + +```yaml +auth: + rp_id: app.rittershofer.org + rp_name: app + origin: https://app.rittershofer.org +``` + +Left blank they are derived from the request, which behind a TLS-terminating proxy derives `http://` and rejects every ceremony as an origin mismatch. Changing `rp_id` later invalidates every key already registered against the old one. + +**The password does not go away**, deliberately: it is the way back in if a key is lost, which is why revoking a passkey can never lock you out. + +## Throttling a stranger who is guessing + +The lockout already makes the secret safe. What is left is keeping the door usable for you, and keeping the traffic off the app. Two layers, in the order worth adding them: + +**1. Rate limit in nginx.** Five lines in the proxy conf, included in [../deploy/server/swag/app.subdomain.conf](../deploy/server/swag/app.subdomain.conf). Per-address, no state to manage, no capabilities, and it cannot lock you out — it just makes requests wait. This is the part that earns its place. + +**2. fail2ban, only if a real attack turns up.** SWAG ships it. It reads nginx's access log, where the address is the one that actually connected rather than a header, and bans in the firewall across every service on the box. A filter matching the login routes with a 401/403/429 status, and a jail pointing at `/config/log/nginx/access.log`, is about ten lines. + +What fail2ban adds over the throttle is narrow and worth being honest about: your own password login stays usable during an attack, the ban covers the whole box rather than one site, and you get a list of who has been banned. It adds no protection of the password — that is already done. Against that it costs `NET_ADMIN` on the SWAG container, a regex that breaks silently if the log format changes, and an unban procedure for when it catches you. + +Add it with a real log sample to test the regex against: + +```bash +docker exec swag fail2ban-regex /config/log/nginx/access.log /config/fail2ban/filter.d/app.conf +``` + +If the domain is proxied through Cloudflare, nginx logs Cloudflare's addresses and a ban hits Cloudflare rather than the attacker. Check that before building anything on top of the log. + +## What to log + +One WARNING line per refusal, with the address first after `from`, so a filter is one regex: + +``` +WARNING auth.router: wrong password from 203.0.113.9 +``` + +The log is the record. A second table of attempts in the database would only ever disagree with it. If the app is run at `log_level="warning"` there is no access log of its own, which makes this the whole record of who is knocking. + +The forwarded address is the one piece of a stranger's own writing that ends up in that log, so strip it to the characters an address is made of before writing it. A line in this log is something a filter reads and a person believes, and neither should be forgeable by whoever is knocking. diff --git a/docs/release.md b/docs/release.md new file mode 100644 index 0000000..d269e2e --- /dev/null +++ b/docs/release.md @@ -0,0 +1,51 @@ +# Building and publishing a release + +Build on Windows, run on the server. Nothing in this half connects to the server: it tests a commit, promotes it, builds an image and pushes it to the registry. [server.md](server.md) is the other half. + +The scripts are already in place in `deploy/`, with `deploy/server/` beside them — `db.ps1` expects that — and the Dockerfile at `deploy/Dockerfile`, which is where `2-build-image.ps1` looks for it inside the archived commit. The package is `app`; there is nothing to rename. + +## Once + +Copy `deploy/release.env.example` to `deploy/release.env` and fill it in. `.gitignore` already covers `deploy/release.env` and `deploy/backups/`. + +Log Docker in with a token that has package-write access, before the first push: + +```bash +docker login git.rittershofer.org -u wolf +``` + +Enter the token at Docker's password prompt, not in source files or chat. Git credentials and Docker registry credentials are separate: a successful Git push does not log Docker in. + +## Every release + +```powershell +.\deploy\1-test-and-promote.ps1 # test local main, fast-forward local production +.\deploy\2-build-image.ps1 # build production's image locally +.\deploy\3-push-image.ps1 # push the built image to the registry +``` + +The number is the order. Each script stops on failure, can be run on its own, and works out the repository root for itself, so the working directory does not matter. To run all three, use `.\deploy\0-release.ps1` — numbered zero so it sorts above the steps it runs. + +**Step 1 never builds; step 2 never pushes; step 3 never rebuilds or deploys.** That separation is the point of the numbering: a failed push is `3` again, not the whole thing again. + +## What each step is actually doing + +**1 — test and promote.** Resolves `main` to a commit, checks that commit out into a throwaway worktree, and runs the project's checks there. The worktree is what makes this honest: the tests run against exactly the tree that is about to become production, with none of the working copy's uncommitted edits in the way. Worktrees have no `.venv` of their own, so the interpreter comes from the main checkout and the code from the worktree. + +Promotion is a fast-forward only. If `production` has commits `main` does not, that is a release made from somewhere else, and the script stops rather than discarding it. + +Git publication is optional: run `git push origin production` yourself, or pass `-PushBranch`. `production` identifies the tested candidate; the server's running image identifies the deployed release. + +**2 — build.** `git archive` of the production commit, unpacked to a temporary directory, built from there. Writing the archive out rather than piping it is deliberate — a PowerShell pipeline carries text, so it would re-encode the tar stream and hand Docker a corrupted context. The build publishes two tags locally: the short commit sha and `latest`. + +Docker builds an **image**. No container starts or restarts on this machine. + +**3 — push.** Re-tags `latest` onto this candidate, then pushes both. It never rebuilds, so a push that failed on connectivity is just this step again. + +## Uncommitted work is never included + +Every step reads from git, not from the working copy. An edit you have not committed is not in the tests, not in the image, and not in the release. That is worth knowing in both directions: it cannot bite you by accident, and it will not pick up the fix you forgot to commit. + +## Version pinning and rollback + +`latest` moves on every publication, so it cannot identify a previous release. Commit tags can: keep them in the registry, and set `APP_TAG=` in the server's `.env` to pin or roll back. See [server.md](server.md#roll-back) for what else a rollback has to check. diff --git a/docs/server.md b/docs/server.md new file mode 100644 index 0000000..105033a --- /dev/null +++ b/docs/server.md @@ -0,0 +1,75 @@ +# Server setup and operation + +What happens once an image has been published. The release half is [release.md](release.md); the box itself and the seven steps for putting a new service on it are [hosting.md](hosting.md). + +Deployment is server-local. No Git checkout, build tooling or SSH from the development machine is needed for it — `db.ps1` is the one exception and it moves a database, not a release. + +## Install the files + +`APP_DATA` decides where the data directory goes. Left out, it is `./data` beside the compose file. Set to an absolute path, it can sit under the one root every container on the box writes to — which is what makes a backup a single target rather than a hunt. + +```bash +sudo mkdir -p /srv/data/app/home /srv/data/app/backups +sudo chown -R 1000:1000 /srv/data/app +``` + +`home/` is there because `HOME=/data/home` in the Dockerfile: a CLI the app shells out to needs somewhere writable, and the image is not it. + +Then: + +- the `app:` block from [../deploy/server/compose.yaml](../deploy/server/compose.yaml) into the compose file that runs the proxy +- the variables from [../deploy/server/env.example](../deploy/server/env.example) appended to the `.env` beside it, `chmod 600` +- [../deploy/server/deploy.sh](../deploy/server/deploy.sh) and [../deploy/server/backup.sh](../deploy/server/backup.sh) next to that compose file +- the app's own `config.yaml` into `$APP_DATA/config.yaml` +- [../deploy/server/swag/app.subdomain.conf](../deploy/server/swag/app.subdomain.conf) into SWAG's `proxy-confs/`, renamed + +Edit `.env` for the actual data path and UID/GID. Keep `APP_TAG=latest` for ordinary deployments or choose a published commit tag to pin. + +## Deploy + +```bash +bash /srv/swag/deploy.sh +``` + +It runs `docker compose pull app` first, and only on success runs `docker compose up -d --no-deps --pull never --wait --wait-timeout 240 app`, using that image and waiting for its `HEALTHCHECK`. Other services in the file are left alone. A failed pull stops before startup; an unhealthy startup exits nonzero. Inspect `docker compose ps` and `docker compose logs app` on failure. + +**After changing credentials in `.env`, use `up -d`, not `restart`.** A restart does not update environment variables — the same container starts again with the environment it was created with. This is the single most common way an updated token appears not to have worked. + +## The database + +Never overwrite a live SQLite database with a plain file copy. It is in WAL mode, so the `.db` on disk is always missing whatever is still in the `-wal` beside it, and copying the three files while the app is running is a race you lose quietly: a database that opens fine and is short a day's writes. + +**Backups.** `bash /srv/swag/backup.sh` takes a `VACUUM INTO` snapshot through the running container, gzips it into `$APP_DATA/backups`, and deletes everything past `KEEP` (14 by default). Schedule it with cron: + +``` +0 4 * * * bash /srv/swag/backup.sh >> /srv/swag/backup.log 2>&1 +``` + +Keep an off-server copy, and test restoring it. A backup nobody has restored is a hypothesis. + +**Moving a database onto the server.** `.\deploy\db.ps1 push` takes a snapshot of this machine's database, uploads it with the importer, and runs the import. The server validates the incoming file — `integrity_check` plus the tables it must have — and snapshots the live database into `/data/backups` before replacing it, atomically. + +It does not stop the app and does not start it: **stop it yourself first** and start it afterwards. The import is refused while the app is up. Get the image the database needs into the registry and onto the server first, because older code refuses to open a newer schema. + +If the app has accounts, importing replaces them too, so do the account setup against the final database rather than before the import. + +**Taking one off.** `.\deploy\db.ps1 pull` runs the server's own `backup.sh` and downloads the newest snapshot into `backups\`. Work on a copy of it, never on a file you intend to push back: after a push the two copies diverge from the first write on either, and there is no merge for that. + +## Roll back + +Record the current image tag or digest before an update. Then set that commit tag in `.env` and run `deploy.sh` again. `latest` moves on publication, so it cannot name a previous release — this only works if commit-tagged images stay in the registry. + +Check schema compatibility before rolling code back. If a release migrated the database incompatibly, restore a matching backup while the app is stopped. No automated rollback overwrites the database, and that is deliberate. + +## Before relying on it + +Verify, in this order, and write down what you saw rather than closing it from local reasoning: + +1. HTTPS from outside, on the real hostname. +2. Login, and a passkey if the app has them — passkeys need `rp_id`/`origin` to match the hostname exactly. +3. The app's heaviest real operation, once, end to end. +4. Persistence: `docker compose up -d --force-recreate app`, then check the data is still there. +5. A restore of an off-server backup copy. +6. Memory under that heaviest operation, before choosing `APP_MEM`. + +The health endpoint checks the web app and its store. It does not check model credentials, and a green container says nothing about whether the API key works. diff --git a/pipeline/README.md b/pipeline/README.md new file mode 100644 index 0000000..ce3d5c3 --- /dev/null +++ b/pipeline/README.md @@ -0,0 +1,172 @@ +# pipeline + +A job queue that lives in a SQLite table. Workers per resource, batching with a report job, dependency chains, dedupe, retries, cancellation that can kill, a WebSocket job list, and a standalone worker. No broker. + +Needs Python 3.12+ and `pydantic`. `pipeline/web/` also needs `fastapi`; an app without a web face never imports it. + +The source of truth is `boilerplate/pipeline/`. Copies elsewhere — news, health — are copies: take the whole folder, and run its tests where it lands. + +```bash +cp -r boilerplate/pipeline yourproject/pipeline +python -m pytest yourproject/pipeline/tests +``` + +## A job + +A job is a plain function. `@job` registers it and returns it unchanged, so it stays callable from a terminal or a test with no queue. + +```python +from pipeline import NULL, JobQueue, Jobs, job + +@job("greet", "Saying hello", resource="cpu") +def greet(config, store, *, context=NULL, name: str = "") -> dict: + context.progress(0, 1, f"greeting {name}") + context.check() # stops here if Kill was pressed + return {"said": f"hello {name}"} + +greet(config, store, name="you") # direct call, no queue + +queue = JobQueue(Jobs("app.db"), config, store) +queue.enqueue("greet", {"name": "you"}) +queue.drain() # run everything queued, then stop +``` + +`config` and `store` are whatever your app uses. The queue passes them through untouched. The payload is the keyword arguments, stored as JSON on the row. + +| `@job(...)` | | +|---|---| +| `resource="cpu"` | which worker pool runs it. Pool sizes: `JobQueue(workers={"cpu": 1, "http": 8, "llm": 4})` | +| `solo=True` | at most one queued or running at a time; a second enqueue returns the first | +| `max_attempts=3, retry_delay=60` | retried by the queue, not by a sleeping worker | +| `isolate=True, timeout=600` | runs in its own process, so Kill and the deadline actually end it | + +## A web app + +```python +from fastapi import FastAPI +from pipeline import Jobs +from pipeline.web import connect + +app = FastAPI() +queue = connect(app, Jobs("app.db"), config, store) +``` + +That mounts `GET /api/jobs`, `POST /api/jobs`, `POST /api/jobs/{id}/cancel`, the `/ws/jobs` socket, and starts the worker with the app. For a job list with progress and Kill buttons, put this in a page: + +```html + + +``` + +It themes from CSS custom properties (`--desk-jobs-bg`, `--desk-jobs-accent`, …) and raises a `desk-jobs:notice` DOM event for failures instead of wording them. The wording is the app's. + +## Many jobs and one report + +```python +@job("fetch_all") +def fetch_all(config, store, *, context=NULL): + urls = store.pending_urls() + context.fan_out("fetch", [{"urls": urls[i:i + 10]} for i in range(0, len(urls), 10)], + then="myapp.jobs:after_fetch") + +def after_fetch(config, store, tally, carries, *, context): + # tally.done / .failed / .cancelled, and tally.results: what each child returned + context.fan_out("parse", ...) # the next step; this is a cascade +``` + +`then` is `module:function` because it is stored on a row. The report job runs whether the children succeeded or not. `Task(kind, finder=...)` is the same thing with the finding done for you; register it in `TASKS` to make it a button. + +## Chains + +```python +a = queue.enqueue("fetch", {"url": url}) +b = queue.enqueue("extract", {}, depends_on=[a.id], after="ok") # "done" = however A ended + +@job("extract") +def extract(config, store, *, context=NULL, html: str = ""): + html = html or context.upstream().get("html", "") # A's result +``` + +B reads A's result off A's row. Do not queue a row you mean to fill in later. The fallback keyword keeps `extract` callable by hand, since `upstream()` is `{}` outside the queue. + +Results are reports: counts, ids, errors. A result over 64 KB fails the job with `ResultTooLarge` rather than being cut down. Put data in your store and return its ids, or raise `pipeline.table.MAX_RESULT` if a project really needs more. + +## Ending work that does not look up + +A thread cannot be interrupted, so `check()` is cooperative. Two cases can be killed: + +- **A command** (ffmpeg, stockfish): `context.spawn([...], timeout=600)` kills the process tree on Kill or deadline. +- **Python that blocks** (`sklearn.fit`): `@job(..., isolate=True, timeout=600)`. The job runs as `python -m pipeline.child`, which rebuilds `(config, store)` from a bootstrap you provide: + +```python +def bootstrap(config_path: str = ""): # myapp/jobs.py; importing it registers the jobs + config = load_config(config_path) + return config, Store(config.database) + +JobQueue(jobs, config, store, bootstrap="myapp.jobs:bootstrap", bootstrap_arg=config_path) +``` + +`timeout` without `isolate` is refused at registration: a thread's deadline could only mark the row failed while the work carried on. + +## Schedules + +A schedule is a row that says "enqueue this kind with this payload on this cadence". Same SQLite file as the jobs, so there is no second process to keep alive and a restart between two runs loses nothing. + +```python +from pipeline.schedule import declare + +declare("nightly tick", "tick", "daily at 06:00", label="A tick, nightly") +declare("hourly count", "count", "every 1h", {"n": 5}) +``` + +`declare` goes next to `@job` in the app's jobs module and fills `SCHEDULES` the way `@job` fills `JOBS`. On `JobQueue.start()` every declared name with no row gets one, and a row that already exists is left exactly as it was edited — cadence, payload and enabled alike. So a declaration is a starting point, not a setting: switching one off on screen survives the next deploy. + +Five cadences and nothing else: + +| | | +|---|---| +| `every 15m` `every 6h` `every 2d` | elapsed time. Unaffected by a daylight-saving change, because an interval is an interval | +| `daily at 06:00` | 06:00 on the wall clock of `JobQueue(tz=...)`, on both sides of a change | +| `weekly mon 07:00` | the same, on that weekday. Long names (`monday`) are accepted and stored short | + +Anything else raises `ValueError` listing all five, which is the sentence the API hands a person who typed something else. There is no seconds unit: the loop looks every `SCHEDULE_TICK` (30 s), and a cadence it cannot keep is one to refuse rather than round. + +The tick runs inside the drain loop — no second thread, no second clock — and once on `start()`, so an app that was down over 06:00 runs that job when it comes back. Once: a firing sets `next_run` from *now* rather than stepping through the slots it missed, so there is nowhere for a backlog to accumulate. A schedule whose last job is still queued or running skips its turn and advances instead of piling copies of it up, and so does one whose kind is `solo` while another is under way. A row whose kind is no longer registered never fires, keeps its place, and is listed as `unknown kind` for somebody to delete. + +```python +queue = JobQueue(jobs, config, store, tz="Europe/Copenhagen") # or schedules=False +``` + +Over HTTP, from `pipeline.web.connect`: `GET/POST /api/schedules`, `PATCH`/`DELETE /api/schedules/{id}`, `POST /api/schedules/{id}/run` (fires now without moving the next run). Every refusal is `{"detail": ""}` — an unregistered kind 404s naming what there is, a payload missing keys 422s naming them, a bad cadence 422s with all five wordings. Any change pushes a `schedules` message on `/ws/jobs`. + +```html + + +``` + +`` is the table: name, kind, cadence edited in place, an on/off toggle, next and last run with the last job's state, Run now, Delete, and a row for adding one. Same `--desk-jobs-*` custom properties as ``, and just as silent about failures — it raises `desk-schedules:error` carrying the server's `detail` and leaves the wording to the app. + +## A worker without the web app + +```bash +python -m pipeline.worker --db app.db --bootstrap myapp.jobs:bootstrap --bootstrap-arg config.yaml +python -m pipeline.worker ... --until-idle # drain and exit, for cron +python -m pipeline.worker ... --timezone Europe/Copenhagen +python -m pipeline.worker ... --no-schedules # every worker after the first +``` + +Any number of workers may share one file. A claim is one atomic statement and a claimed job is held on a lease; a worker that dies has its jobs re-queued by whichever is still alive. + +This worker fires schedules too, which is what makes it a complete replacement for the web app's own rather than half of one. Exactly one process per file should: `--no-schedules` is what the second and third pass, because the pile-up guard reads `last_job_id` and neither of two simultaneous firings has written it when the other looks. + +## Housekeeping + +Finished jobs older than `JobQueue(retain=30 * 86400)` are purged hourly while idle. Unfinished jobs and the newest ending of each kind are kept regardless. `retain=0` keeps everything. + +The table is WAL-mode SQLite. Back it up with `VACUUM INTO`, not a file copy, and keep it off network filesystems. + +## More + +- `pipeline.md` — the reference card: every entity, the timings, the limits. +- `__init__.py` — what each module is for. +- `tests/` — every claim above, including two worker processes racing on one file and a schedule either side of a daylight-saving change. diff --git a/pipeline/__init__.py b/pipeline/__init__.py new file mode 100644 index 0000000..151a170 --- /dev/null +++ b/pipeline/__init__.py @@ -0,0 +1,127 @@ +"""The pipeline: how work is queued, run, watched and stopped. + + from pipeline import JobQueue, Jobs, job, RunContext, Task, NULL + +**This folder is meant to be copied.** It imports nothing from the app around +it and nothing outside the standard library but pydantic — and FastAPI, in +`pipeline/web/`, which an app that has no web face never imports. Drop the +directory into another project, point `Jobs` at a SQLite file, and it works. +`pipeline.md` beside this file is the write-up, and it travels with the folder. + +**The source of truth is the boilerplate repo's copy.** news and health hold +copies; a fix lands here first and is copied out with `git archive`. + +Nothing here knows what a job *does*. The jobs themselves are the app's — +`app/jobs.py` for the starter — and import `@job` from here. That is the one +rule: the pipeline is imported by the work, never the other way round, so +there is no cycle to break and no lazy import to remember. + + models.py `Job`, `Run`, `JobsSnapshot`, `Notice` and the `Model` base + they share — the shapes the queue stores and serves + notices.py a failure turned into one line worth putting on screen. + Only the part the queue needs; classifying a failure is the + app's, because a useful sentence names things only it knows + registry.py `@job`, `Spec`, and `JOBS` — how a function becomes a job, + how many attempts it is worth, and whether it runs in a + thread or in a child process of its own + schedule.py `Cadence`, `declare` and `SCHEDULES` — the five wordings a + standing instruction may be written in, when each next comes + round, and how an app ships a default schedule next to `@job` + context.py `RunContext` — what a running job is handed: progress, + cancellation, notices, its parents' results, the fan-out, + the subprocess helper, and the two things it can raise + table.py `Jobs` — the jobs table, on whatever SQLite file it is given + child.py the other side of `isolate=True`: one job, one process, + reconstructed from its row + queue.py `JobQueue` — the worker loop that drains that table + task.py `Task` — plain code that finds work, cuts it into batches + and queues them, plus the one final job every task ends + with and the tally it hands to the callback. `Bound` is + what a batch carries when how often to try it is the + batch's own business rather than the job kind's + worker.py `python -m pipeline.worker`: the same queue with no web app + around it, for a second machine, a cron entry or a script + web/ the queue over HTTP, the socket that reports on it, and the + `` element that renders it. Optional: an app + with no web face never imports it, and so never needs + FastAPI + +What the pipeline asks of an app is small and is typed where it is asked: +handlers take `(config, store)` and the queue passes those through untouched, +`Jobs` wants a file path, and `isolate=True` wants a `bootstrap` that can +rebuild that pair in a child process. That is the whole of what a second app +has to provide. `README.md` beside this file is the ten-minute version. + +Within the package the order is `models, notices > registry > schedule > +context > table > child > task > queue > worker > web`, and `tests/test_architecture.py` holds +every file to it, and to one door: what is listed below is the whole of what +anything outside may use. +""" + +from __future__ import annotations + +from pipeline import notices +from pipeline.context import NULL, Cancelled, Retry, RunContext, spawn +from pipeline.models import ( + TERMINAL, CascadeProgress, Job, JobState, JobsSnapshot, Notice, + NoticeEntry, NoticeKind, NoticeLevel, Run, Schedule, TaskProgress, +) +from pipeline.queue import ( + ChildFailed, JobQueue, Listener, NoticeListener, ScheduleListener, + local_zone, +) +from pipeline.registry import JOBS, Handler, Spec, job +from pipeline.schedule import ( + DAYS, FORMS, SCHEDULES, BadCadence, Cadence, Declared, declare, +) +from pipeline.table import MAX_RESULT, Jobs, ResultTooLarge, RunCancelled +from pipeline.task import FINAL, TASKS, Batch, Bound, Tally, Task, fan_out + +__all__ = [ + "DAYS", + "FINAL", + "FORMS", + "JOBS", + "MAX_RESULT", + "NULL", + "SCHEDULES", + "TASKS", + "TERMINAL", + "BadCadence", + "Batch", + "Bound", + "Cadence", + "Cancelled", + "CascadeProgress", + "ChildFailed", + "Declared", + "Handler", + "Job", + "JobQueue", + "JobState", + "Jobs", + "JobsSnapshot", + "Listener", + "Notice", + "NoticeEntry", + "NoticeKind", + "NoticeLevel", + "NoticeListener", + "ResultTooLarge", + "Retry", + "Run", + "RunCancelled", + "RunContext", + "Schedule", + "ScheduleListener", + "Spec", + "Tally", + "Task", + "TaskProgress", + "declare", + "fan_out", + "job", + "local_zone", + "notices", + "spawn", +] diff --git a/pipeline/child.py b/pipeline/child.py new file mode 100644 index 0000000..512d189 --- /dev/null +++ b/pipeline/child.py @@ -0,0 +1,140 @@ +"""The other side of `isolate=True`: one job, one process, and then gone. + + python -m pipeline.child --db news.db --job 412 \ + --bootstrap newsdesk.jobs:bootstrap --out C:/Temp/tmp123.json + +Run by the worker, never by a person, and the whole reason it can exist is +that **a job row is already a complete description of the work.** A kind, a +JSON payload, a database file. Nothing is pickled and nothing is sent: the +child is told where to look and reads the same row the worker read. + +Which is also the division of labour that keeps the state machine intact. +The child writes *progress* — a non-terminal update, and the snapshot the UI +reads is a `SELECT`, so a child's progress reaches the browser for free. The +child does **not** write the ending. It hands its result back through a file +and exits, and the worker writes `done` or `failed` exactly as it does for a +job that ran in a thread. One writer of the terminal state, as before. + +A file rather than stdout because a handler is arbitrary code and arbitrary +code prints. Parsing a result out of a stream something else is also writing +to is a bug waiting for the first `print` in a fetch loop. + +The `bootstrap` is the one thing an app has to provide, and it has two jobs: +import the modules that fill `JOBS`, and return the `(config, store)` pair +the handlers take. Neither survives a process boundary, so neither is sent — +it is rebuilt here from the config file the app already knows how to read. +""" + +from __future__ import annotations + +import argparse +import importlib +import json +import logging +import sys +import traceback +from pathlib import Path +from typing import Any + +from pipeline.context import Cancelled, RunContext +from pipeline.registry import JOBS +from pipeline.table import Jobs + +log = logging.getLogger(__name__) + +#: What the worker reads off the exit code. Anything else is the process +#: dying of something that was never ours to catch — an OOM kill, a segfault +#: in a native library — and the worker reports it as the failure it is. +OK = 0 +FAILED = 1 +CANCELLED = 2 + + +def resolve(name: str) -> Any: + """`module:attribute` to the attribute. The same spelling a callback uses.""" + module, _, attr = name.partition(":") + if not module or not attr: + raise ValueError(f"a bootstrap is 'module:function', not {name!r}") + return getattr(importlib.import_module(module), attr) + + +def context_for(jobs: Jobs, job_id: int, task_id: int) -> RunContext: + """A context that reports to the table and asks it about cancellation. + + Every channel the in-process context has, except the two that cannot + cross a process boundary and are not wanted here anyway: `on_task` and + `on_fan_out` both queue work, and an isolated job is isolated precisely + because it is one long piece of work rather than an organiser of other + work. Both fall back to their no-ops, which report that nothing took it. + + Cancellation is a read of the row, which is what makes Kill work at all + from here: the button writes `cancel_requested` in the web process, and + this is the only way this process would ever hear about it. The worker + also kills us outright — this is the polite half, so a handler that calls + `check()` stops at a line of its own choosing and gets to clean up. + """ + return RunContext( + on_progress=lambda done, total, message: jobs.progress( + job_id, done, total, message), + is_cancelled=lambda: jobs.cancel_requested(job_id), + is_killed=lambda: bool(task_id) and jobs.cancelled(task_id), + on_notice=lambda notice: print( + json.dumps({"notice": notice.model_dump()}), file=sys.stderr), + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="pipeline.child", description=__doc__) + parser.add_argument("--db", required=True, help="the SQLite file the jobs table is on") + parser.add_argument("--job", required=True, type=int, help="which row to run") + parser.add_argument("--bootstrap", required=True, + help="module:function returning (config, store)") + parser.add_argument("--bootstrap-arg", default=None, + help="one string handed to the bootstrap, e.g. a config path") + parser.add_argument("--out", required=True, help="where to write the result JSON") + args = parser.parse_args(argv) + + out = Path(args.out) + jobs = Jobs(args.db) + job = jobs.get(args.job) + if job is None: + out.write_text(json.dumps({"error": f"no job {args.job}"}), encoding="utf-8") + return FAILED + + try: + build = resolve(args.bootstrap) + config, store = (build() if args.bootstrap_arg is None + else build(args.bootstrap_arg)) + spec = JOBS.get(job.kind) + if spec is None: + raise KeyError( + f"{job.kind!r} is not a registered job in this process — " + f"does {args.bootstrap} import the module that declares it?" + ) + context = context_for(jobs, job.id, job.task_id) + kwargs = dict(job.payload) + if spec.wants_context: + kwargs["context"] = context + result = spec.fn(config, store, **kwargs) + except Cancelled: + out.write_text(json.dumps({}), encoding="utf-8") + return CANCELLED + except BaseException as exc: + # The traceback goes in the file as well as to stderr. stderr is the + # worker's fallback and is captured either way; the file is what + # survives a child that also wrote something confusing to the stream. + out.write_text(json.dumps({ + "error": f"{type(exc).__name__}: {exc}", + "traceback": traceback.format_exc(), + }), encoding="utf-8") + traceback.print_exc() + return FAILED + + out.write_text(json.dumps({"result": result or {}}, default=str), encoding="utf-8") + return OK + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, + format="%(levelname)s %(name)s: %(message)s") + raise SystemExit(main()) diff --git a/pipeline/context.py b/pipeline/context.py new file mode 100644 index 0000000..e2ae973 --- /dev/null +++ b/pipeline/context.py @@ -0,0 +1,446 @@ +"""What a running job is handed, and the two things it can raise. + + def handler(config, store, *, context=NULL, limit=None) -> dict + +This is the job-side half of the contract; `registry.py` is the other — how +a function becomes queueable. A pipeline stage imports this to report +progress from inside its own loops, and nothing else: no queue, no worker. +Nothing here imports anything else from desk at module level, so it is safe to +import from anywhere. + +Everything in here is optional to the job that receives it. Ignore `context` +and the job still runs — it just cannot be stopped part-way and shows no +progress. Use it and you get both, for two lines. +""" + +from __future__ import annotations + +import logging +import os +import signal +import subprocess +import tempfile +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import Any + +log = logging.getLogger(__name__) + +#: How often a spawned process is looked in on: whether it has finished, +#: whether Kill has been pressed, whether its deadline has passed. Frequent +#: enough that a Kill lands in well under a second, rare enough that waiting +#: an hour for ffmpeg costs a few thousand cheap syscalls. +POLL = 0.25 + + +class Cancelled(Exception): + """Raised by `RunContext.check` when the job has been asked to stop. + + Caught by the worker and recorded as `cancelled` rather than `failed`. A + handler should let it travel — catching it is how you build the thing the + Kill button can't kill. + """ + + +class Retry(Exception): + """Raised by a handler that wants this same work attempted again later. + + An ordinary failure is already retried, within whatever bound the job was + queued with — so this is not how a job asks to be retried at all. It is how + it says *when*: `after` is the wait the other end named, a `Retry-After` + header or a rate limit's own answer, and the queue uses it in place of the + backoff it would otherwise have guessed. Guessing is the thing worth + avoiding here — a minute is either a minute of doing nothing when the + service was ready, or another refusal charged against the bound. + + It does not buy an attempt. A job queued with one attempt raises this and + fails, exactly as any other exception would: the bound is ours to set and + the wait is the world's to name, and a handler that could extend its own + bound would be a policy nobody declared. + """ + + def __init__(self, message: str = "", *, after: float | None = None) -> None: + super().__init__(message) + #: Seconds to wait, or None to leave it to the job's own backoff. + self.after = after + + +def _no_progress(done: int, total: int, message: str) -> None: + return None + + +def _no_task(task: Any) -> bool: + return False + + +def _no_stop(why: str) -> int: + """Where "stop the rest of this run" goes when there is no run to stop. + + The direct path is one batch after another in the caller's own thread, and + the caller is standing there: it sees each failure as it happens and can + stop by not asking again. Nothing to cancel, so nothing is cancelled. + """ + return 0 + + +def _no_fan_out(*_args: Any, **_kwargs: Any) -> tuple[int, ...]: + """Where a fan-out goes when there is no queue to put it in. + + Empty rather than an error, and the same answer `task` gives for the same + reason: a handler called directly — from a terminal, a notebook, a test — + is being run by somebody who is standing there, and the useful behaviour + is to do its own work and report that nothing was queued behind it. A + handler that must know either way checks the return. + """ + return () + + +def _log_only(notice: Any) -> None: + """Where a notice goes when nothing is watching for one. + + A direct caller — `newsdesk job pull` in a terminal, a notebook, a test — has + no socket, so the log is the destination. It is not a *lesser* one: it goes + through the same `notices.render`, so the terminal gets the sentence, the + command to run and the is-this-a-bug verdict, exactly as the browser does. + """ + from pipeline.notices import render + + log.warning("%s\n%s", notice.code, render(notice)) + + +def _never() -> bool: + return False + + +def kill_tree(proc: subprocess.Popen[Any]) -> None: + """Kill a spawned process *and* whatever it started. + + The children are the point. `proc.kill()` reaches the process we launched + and nothing it launched itself, so killing a shell leaves the ffmpeg it + ran holding the core — which is the exact failure a deadline exists to + prevent. Both branches need the process to have been started detached, + which `spawn` does. + + Best-effort by construction: between deciding to kill and killing, the + process may have ended on its own and its id may have been reused. So + every call is guarded, and a failure to kill something that is already + gone is not news. + """ + if proc.poll() is not None: + return + try: + if os.name == "nt": + # No process groups to signal, so ask Windows to walk the tree. + # `/T` is children, `/F` is without asking. + subprocess.run(["taskkill", "/T", "/F", "/PID", str(proc.pid)], + capture_output=True, check=False, timeout=10) + else: + # Guarded by `os.name` above rather than by the type checker, + # which is reading this file on Windows and knows these three do + # not exist there. + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) # type: ignore[attr-defined,unused-ignore] + except Exception: + log.debug("could not kill %s", proc.pid, exc_info=True) + try: + proc.wait(timeout=5) + except Exception: + log.warning("process %s did not go away after being killed", proc.pid) + + +def spawn(argv: Sequence[str], *, check: Callable[[], None] | None = None, + timeout: float | None = None, poll: float = POLL, + **popen: Any) -> subprocess.CompletedProcess[bytes]: + """Run a command, watching for Kill and for a deadline while it does. + + The building block under `RunContext.spawn` and under `isolate=True`, and + the answer to the one thing a thread cannot do. A handler that blocks in + `subprocess.run` cannot be stopped, because stopping it would mean tearing + a thread out of a syscall; a handler that blocks *here* can, because what + is expensive is the child and a child can be killed. The thread is only + sleeping, which costs nothing. + + Output goes to temporary files rather than pipes. `PIPE` plus a poll loop + is a deadlock: a child that fills the pipe buffer blocks on the write + while we block on the poll, and neither side is wrong. A file cannot fill + up in the way that matters, so the loop is free to look away. + + Started detached — its own process group, or Windows' equivalent — purely + so `kill_tree` has something to aim at. + """ + detach: dict[str, Any] = ({"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP} + if os.name == "nt" else {"start_new_session": True}) + with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err: + popen.setdefault("stdout", out) + popen.setdefault("stderr", err) + proc = subprocess.Popen(list(argv), **detach, **popen) + deadline = time.monotonic() + timeout if timeout else None + try: + while proc.poll() is None: + if check is not None: + check() # raises Cancelled on Kill + if deadline is not None and time.monotonic() > deadline: + kill_tree(proc) + raise TimeoutError( + f"{argv[0]} was still running after {timeout:g}s") + time.sleep(poll) + except BaseException: + kill_tree(proc) + raise + out.seek(0) + err.seek(0) + return subprocess.CompletedProcess( + list(argv), proc.returncode, + out.read() if popen["stdout"] is out else b"", + err.read() if popen["stderr"] is err else b"") + + +@dataclass(slots=True) +class RunContext: + """The handle a running job has on the run it is part of. + + Deliberately a handful of plain callables rather than a reference to the + queue: a handler cannot reach back into the worker, reschedule itself or + read other jobs, and a test can hand it a lambda without standing anything + up. + """ + + on_progress: Callable[[int, int, str], None] = _no_progress + is_cancelled: Callable[[], bool] = _never + #: Whether the *task* this job belongs to was killed — a different + #: question from `is_cancelled`, which is about this job alone. See + #: `killed`. + is_killed: Callable[[], bool] = _never + on_notice: Callable[[Any], None] = _log_only + on_task: Callable[[Any], bool] = _no_task + on_stop: Callable[[str], int] = _no_stop + #: Where `fan_out` puts the jobs it asks for. Supplied by the queue, which + #: is the only thing that can write rows; `_no_fan_out` under `NULL`. + on_fan_out: Callable[..., tuple[int, ...]] = _no_fan_out + #: How the jobs this one waited for ended, oldest first. Empty for a job + #: that waited for nothing. Typed loosely for the same reason `notice` is: + #: nothing in this module may import the rest of desk, and the shape it + #: carries is `pipeline.models.Job`. + waited_for: tuple[Any, ...] = () + + def upstream(self) -> dict[str, Any]: + """What the jobs this one waited for returned, merged into one dict. + + The read half of a chain. `A -> B -> C` needs A's output to reach B, + and this is how it arrives: B does not have its payload filled in by + A, it reads A's result off A's row. Which is the whole difference — + **a row is complete when it is written.** A payload with a hole in it + meant for an earlier job to fill would be claimable the instant that + job landed, so a worker could take it between the write and the + commit; it could not carry a `dedup_key`, which has to be known at + queue time; and it could not be re-run from its own row, which is + what a retry does and what `max_attempts` is. + + Merged, later parents winning, because the overwhelmingly common case + is one parent and `upstream()["text"]` is what that should read like. + A job with several parents that care about the same key wants + `waited_for` and to decide for itself. + + Empty under `NULL`, which is what makes the fallback work: + + def handler(config, store, *, context=NULL, html=""): + html = html or context.upstream().get("html", "") + + From the queue nobody passes `html` and it comes from the parent; from + a terminal you pass it yourself. Without that line a chained handler + is the one kind in the pipeline you cannot simply call. + """ + merged: dict[str, Any] = {} + for parent in self.waited_for: + result = getattr(parent, "result", None) + if isinstance(result, dict): + merged |= result + return merged + + def fan_out(self, kind: str, payloads: Sequence[dict[str, Any]], *, + then: str = "", settle: str = "", label: str = "", + carries: dict[str, Any] | None = None, + max_attempts: int | None = None, + retry_delay: float | None = None, + retry_backoff: float | None = None) -> tuple[int, ...]: + """Queue one job per payload, and one job behind them that reports. + + Returns the ids of the children. The reporting job is not among them: + it is the pipeline's own kind, it is nothing an app writes, and it is + the reason this method exists rather than a handler assembling a + `depends_on` list by hand every time. + + def handler(config, store, *, context=NULL): + chunks = chunk(needs_a_body(store)) + context.fan_out("body_batch", [{"uids": c} for c in chunks], + then="newsdesk.jobs.cascade:step") + + `then` is `module:function`, resolved where it is run, and is handed + `(config, store, tally, carries)` once every child is over however it + ended — see `Tally` in `task.py`. A name rather than a function + because what carries it is a table row, and no row can hold a + function. + + One transaction, so a fan-out is never half-queued. The children go + into a run of their own within this job's cascade, which is what makes + the callback able to fan out again: that is a cascade, and there is no + cascade object anywhere — step N's callback queues step N+1. + + Nothing is queued under `NULL`, and `()` says so. + """ + return self.on_fan_out( + kind, list(payloads), then=then, settle=settle, label=label, + carries=dict(carries or {}), max_attempts=max_attempts, + retry_delay=retry_delay, retry_backoff=retry_backoff) + + def spawn(self, argv: Sequence[str], *, timeout: float | None = None, + **popen: Any) -> subprocess.CompletedProcess[bytes]: + """Run an external command, killable, with this job's Kill wired in. + + What to reach for when the expensive thing is a binary — ffmpeg, + stockfish, whisper.cpp. Because the work is in the child and not in + this thread, Kill is no longer cooperative in the way `check` is: the + child is killed outright, and its children with it. Blocking a worker + thread on a process costs nothing, since the thread is only sleeping. + + `timeout` is a real deadline for the same reason, and the only kind of + deadline the pipeline offers that can actually end the work. A job + kind that wants one over Python code rather than over a command wants + `isolate=True` — see `registry.py`. + + Raises `Cancelled` on Kill and `TimeoutError` on the deadline, having + killed the process tree either way. Non-zero exit is *not* an error + here: the returncode is handed back, because plenty of commands say + something useful with it and only the handler knows which. + """ + return spawn(argv, check=self.check, timeout=timeout, **popen) + + def progress(self, done: int = 0, total: int = 0, message: str = "") -> None: + """Say how far along this is. Units are the handler's own business — + sources, files, minutes of audio — the UI renders `done/total`. + + The message has one shape, and every job in this codebase uses it: + + a lowercase participle, what it is being done to, and — when the + job is walking a list — `, n of m`. + + "reading Golem, 3 of 12". "fetching the body from theverge.com, 40 of + 212". "training on your votes" for a job that is one step and has no + list to be three of. Not an em dash, not a middot, and never a bare + identifier: this used to say `wordpress`, which named the module that + reads the feed rather than the outlet, and read as a fragment of a log + line that had escaped onto the status strip. + + `n` is the item being worked on now and so runs one ahead of `done`, + which counts the ones that are finished. They are answering different + questions — the fraction in the message belongs to the name beside it, + and the bar belongs to the job — and a line saying "reading Golem, 0 of + 12" while Golem is being read would be wrong about the only thing it + was there to say. A phase that has nothing left to name ends on the + collective — "reading every source, 12 of 12" — so the same participle + carries the line from first item to last, and a full fraction is what + says it is over. Past tense is not used: it dates a line that is about + to be replaced by the next phase's. + + A job's *label* is the same grammar capitalised, because a label names + the work rather than narrating it: "Tagging 25 items, 2 of 9". + """ + self.on_progress(done, total, message) + + def check(self) -> None: + """Stop now if Kill has been pressed. + + Cancellation is cooperative because the work is synchronous: there is + no safe way to tear a thread out of the middle of an HTTP fetch or a + model load, so the job says where stopping is fine. Call it between + units of work, not inside tight inner loops. + """ + if self.is_cancelled(): + raise Cancelled() + + def killed(self) -> bool: + """Whether the task this job belongs to was killed. + + Not the question `check` asks. That one is about this job, and killing + one job is dropping one batch. This one is about the run, and killing + a run is skipping the whole step — which the job that reports on the + step has to be able to say, since it is spared the kill precisely so + that somebody hears about it. The counts cannot tell the two apart: a + batch someone killed by hand and a batch killed by the task going + down both arrive as `cancelled`. + + Asked when asked, not read at the start, so a kill landing while the + reporting job runs is still in what it reports. + """ + return bool(self.is_killed()) + + def step(self, done: int = 0, total: int = 0, message: str = "") -> None: + """`progress` then `check` — what the top of a loop body always wants.""" + self.progress(done, total, message) + self.check() + + def task(self, task: Any) -> bool: + """Queue a `Task` once this job has finished well. + + Recorded rather than queued: the worker starts the task after this job + lands in `done`, so a task never outlives the failure of the job that + described it, and a handler still cannot reach the queue. It names what + it wants to happen and that is all. + + The task runs in the worker, synchronously, the moment the job is over + — the same plain code a request would run. It is handed the queue as + its sink and this job's cascade, so its batches land in the run this + one belongs to. + + Returns whether anything took it. `False` under `NULL`, where there is + no queue to take it: a callback invoked from a notebook or a Claude + Code session has to do the work itself, and this is how it finds out + that nobody else will. + """ + return bool(self.on_task(task)) + + def stop(self, why: str = "") -> int: + """Cancel this task's remaining queued batches. Returns how many. + + For the failure that is not this batch's: a spend limit, a signed-out + CLI, no model on the machine at all. The backend has answered, the + answer is the same one every sibling is about to be given, and the + only thing left to decide is whether to spend the other twenty-two + calls finding that out. Twenty-three identical notices saying "looks + like a bug" is what not deciding looked like. + + Only what is still *queued* stops. A batch already in flight is a + request in the air and a bill either way, so it runs to its own end — + and this one, the one that asked, keeps its own failure. The task's + final job is spared for the reason `Jobs.cancel_run` spares it: + reporting is not work, and a run that stopped early still has to say + so. `why` is written onto each cancelled row, so a batch that never + ran says on screen why it didn't. + + Not a kill. Nobody pressed anything, so the task's row is not flagged + and `killed` stays false: the cascade goes on to the next stage, as it + does after any batch that did not land. Whether the *next* task should + stop too is that cascade's question, not this one's. + """ + return int(self.on_stop(why)) + + def notice(self, notice: Any) -> None: + """Tell the user something went wrong. See `pipeline/notices.py`. + + The counterpart to `log.exception`, not a replacement for it: log the + traceback for afterwards, raise a notice for now. Use it for what a + person has to act on — a login to renew, a feed that has stopped + resolving — and not for what the program can simply get on with. + + Under a queue it is written to the notice log and pushed to whoever is + watching; called directly it is logged and nothing else. Safe to call + repeatedly: the same `code` and `key` again is one row with a count, + and only the first interrupts anyone. So give `key` whatever the + failure is about — the source, the article — or two different + failures of one kind are told as one. + """ + self.on_notice(notice) + + +#: For calling a handler outside the queue. Reports nowhere, never cancels. +NULL = RunContext() diff --git a/pipeline/models.py b/pipeline/models.py new file mode 100644 index 0000000..f2ba0f4 --- /dev/null +++ b/pipeline/models.py @@ -0,0 +1,432 @@ +"""The shapes the queue stores and serves: a job, a run, a snapshot, a notice. + +One set of models, not two — there is no ORM here, so the row the table +returns and the object the API serves are the same declaration. `Model` is +the base an app's own shapes are welcome to extend as well, because the one +thing it does (tolerate a `sqlite3.Row`, ignore columns nobody declared) is +wanted by every table anyone will ever `SELECT *` from. + +Pydantic is the package's only dependency outside the standard library, and +this is the file that wants it. Everything else here would run without it. +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from typing import Annotated, Any, Literal, Self + +from pydantic import (BaseModel, BeforeValidator, ConfigDict, computed_field) + +#: A column that holds JSON. SQLite hands back the text; everything above the +#: storage layer wants the dict. Decoded here so neither side has to remember, +#: and tolerant of a malformed value because a job whose payload failed to +#: parse should still be listable and cancellable rather than un-readable. +JsonDict = Annotated[ + dict[str, Any], + BeforeValidator(lambda v: v if isinstance(v, dict) else _loads(v)), +] + + +def _loads(value: Any) -> dict[str, Any]: + try: + parsed = json.loads(value or "{}") + except (TypeError, ValueError): + return {} + return parsed if isinstance(parsed, dict) else {} + + +#: The same for a column holding a JSON array of job ids. Tolerant for the same +#: reason: a job whose dependency list will not parse has already been let +#: through by the queue, which reads the column in SQL — so refusing to build +#: the model here would only hide it from the list it is holding up. +JsonIds = Annotated[ + list[int], + BeforeValidator(lambda v: v if isinstance(v, list) else _ids(v)), +] + + +def _ids(value: Any) -> list[int]: + try: + parsed = json.loads(value or "[]") + except (TypeError, ValueError): + return [] + return [int(i) for i in parsed] if isinstance(parsed, list) else [] + + +class Model(BaseModel): + """Base: tolerate sqlite3.Row and reject fields nobody declared.""" + + model_config = ConfigDict(from_attributes=True, extra="forbid") + + @classmethod + def from_row(cls, row: Any) -> Self: + """Build one from a database row, ignoring columns nobody declared. + + `SELECT *` into a model that forbids extras means the next + `ALTER TABLE` is a 500 in whatever route reads that table — raised a + long way from the migration that caused it, over a column the API was + never going to serve anyway. A storage column the API has not heard of + is not something to hand out; it is something to ignore until someone + declares a field for it. + + The half of `extra="forbid"` worth keeping still works. A column a + model *requires* going missing — a rename, a typo — is a validation + error exactly as before. Only additions go quiet. + """ + return cls.model_validate( + {k: v for k, v in dict(row).items() if k in cls.model_fields} + ) + + +# ------------------------------------------------------------------- jobs + +#: The five states a job can be in. `queued` and `running` are live; the other +#: three are terminal and a job never leaves them. +JobState = Literal["queued", "running", "done", "failed", "cancelled"] + +#: Everything that is over, however it ended. Used to decide what the dedup +#: index constrains, what the UI greys out, and when to stop pushing updates. +TERMINAL: frozenset[str] = frozenset({"done", "failed", "cancelled"}) + + +class Job(Model): + """One unit of queued work, as stored. + + `payload` is the handler's keyword arguments and `result` is what it + returned — both JSON in SQLite, both dicts everywhere else, translated + once here rather than at each call site. + + `done`/`total` are deliberately untyped as units. A pull counts sources, a + transcription counts minutes of audio; the UI renders "3/12" and a bar and + does not need to know which. + """ + + id: int + kind: str + label: str = "" + payload: JsonDict = {} + state: JobState = "queued" + done: int = 0 + total: int = 0 + message: str = "" + result: JsonDict = {} + error: str = "" + dedup_key: str = "" + #: Which jobs this one waits for, and whether it needs them to have + #: succeeded (`ok`) or merely to be over (`done`). + depends_on: JsonIds = [] + after: str = "ok" + #: Which kind of worker runs it: `llm`, `http`, `cpu`. + resource: str = "cpu" + #: Which cascade this belongs to, and which of its tasks. A job queued on + #: its own is its own cascade and its own task. Both name a row in `runs` + #: — a series of their own, not this one's. See `Run`. + cascade_id: int = 0 + task_id: int = 0 + cancel_requested: bool = False + #: How many times this has been started, counted by `claim`. One after the + #: first run, whether that run ended well, badly, or with the worker dying + #: under it. + attempt: int = 0 + #: The bound, counting the first attempt — so `1` is the default and means + #: no second try. Carried on the row rather than looked up from the job + #: kind, because the attempt that decides is the one after a restart, in a + #: worker that may be running a different version of the code. + max_attempts: int = 1 + #: The wait before the next attempt, and what multiplies it each time + #: round. `1.0` is a flat wait. + retry_delay: float = 0.0 + retry_backoff: float = 1.0 + #: Not before this. Empty means now, which is why it is `''` rather than + #: NULL: the column is compared as text and `''` sorts before every stamp, + #: so "is it due?" is one comparison with no special case in it. + run_after: str = "" + created_at: str + started_at: str | None = None + finished_at: str | None = None + + @property + def live(self) -> bool: + return self.state not in TERMINAL + + @computed_field # type: ignore[prop-decorator] + @property + def waiting(self) -> bool: + """Queued, but not until its next-attempt time comes round. + + Serialised, unlike its two neighbours, because it is the one of the + three a client cannot work out for itself. `live` is a state, and + `exhausted` is `attempt`, `max_attempts` and `state`, all of which go + out on the row; this one is `run_after` against *now*, and the only + clock worth asking is the one the stamp was written by. A browser four + minutes fast would otherwise draw a job as due while the queue goes on + passing over it. + + `run_after` travels too and still has to: this says *that* it is + waiting and the stamp says until when, which is the difference between + greying a row out and counting down on it. + + The resting state of a failure that is going to be tried again, and the + thing that makes it different from a failure: it is still `queued`, so + nothing is running it, nothing holds a slot for it, and whatever waits + on it goes on waiting. Compared as text, which is safe because + `run_after` is written in the same UTC ISO form as every other stamp + here — see `table.py`. + """ + return self.state == "queued" and self.run_after > datetime.now(UTC).isoformat() + + @property + def exhausted(self) -> bool: + """Failed on the last attempt it was allowed, rather than the only one. + + Worth telling apart, because the two want opposite things done about + them. A job that failed with attempts still to come is not this — it is + back in the queue and `waiting`. A job that failed at `max_attempts=1` + is not this either: nobody ever said it should be retried, so nothing + was used up. This is the third case, the one where the policy ran its + course and the work still did not land, and it is the only one where + trying again is a decision a person has to make. + """ + return self.state == "failed" and self.attempt >= self.max_attempts > 1 + + +class Run(Model): + """A cascade or a task: an identity, and whether it was killed. + + No state and no progress: the jobs carrying this id are the record of what + happened, and a second copy of it here could only go out of step. What it + holds is what cannot be worked out from them — that this id is a cascade + rather than a task, what it was called, when it began, and the one fact its + jobs cannot tell you, which is that somebody stopped it. + """ + + id: int + kind: str + name: str = "" + #: What to call it on screen. A task is not a job, so this is the only + #: place it can be written down. + label: str = "" + created_at: str + #: When it was killed, or "" for a run nobody stopped. A cascade exists + #: nowhere but this row, so this is where its Kill has to live. + cancelled_at: str = "" + + +class Schedule(Model): + """A standing instruction: enqueue this kind, with this payload, this often. + + A row rather than a timer, for the reason every other waiting thing here is + a row: nothing is alive between runs, so a process restarted at 05:59 still + fires the 06:00 one, and a second worker on the same file reads the same + schedule rather than keeping a second copy of it. + + `name` is unique and is the handle declared defaults are matched on — see + `pipeline/schedule.py`. Everything else about a row is the user's to edit, + the cadence and whether it is on at all included, and redeclaring a default + never overwrites those. + + `next_run` and `last_run` are UTC ISO stamps, like every other time in this + table, and are compared as text — so a cadence worded in a local timezone + still sorts correctly against `now` and reads the same from any process. + Empty means never, which for `next_run` also means not due, and is why it + is `''` rather than NULL. + """ + + id: int + name: str + kind: str + #: What to call it on screen, from `declare(label=...)`. Empty means the + #: job kind's own label, which is what the queued job then carries. + label: str = "" + payload: JsonDict = {} + cadence: str + enabled: bool = True + next_run: str = "" + last_run: str = "" + #: The job the last firing queued, and so the answer to "is one still + #: going?" — which is what stops a slow job from piling up behind itself. + last_job_id: int = 0 + #: How that job ended, or `queued`/`running` while it has not. Joined on in + #: `Jobs.schedules` rather than fetched per row: it is one column on a query + #: that is already running, and the time is no use on screen without it. + last_state: str = "" + created_at: str + + +class TaskProgress(Model): + """One task of a live cascade, as a tally of the jobs carrying its id. + + Counts and not the jobs themselves. A task is a body fetch shared over + twenty-three jobs and there is nothing to tell them apart — the ones still + going are in `JobsSnapshot.running` with their own `done`/`total`, and the + ones that landed are a number, because a landed job is a full box and full + boxes are all alike. + + Derived on read, never stored, for the reason `runs` holds no state: the + job rows are the record of what happened and a second copy of the tally + could only go out of step with them. + """ + + #: The `runs` row this task is, and the id `/api/tasks/{id}/cancel` takes. + id: int + #: What the task is — `body`, `tag`. Its own name, minted with the `runs` + #: row before any of its jobs existed, and how a cascade's list of tasks is + #: matched against it. + name: str = "" + #: How to say that on screen: "Fetching what the feeds left out". The + #: task's own, off the `runs` row, because a task is not a job and has no + #: handler to carry one. + label: str = "" + done: int = 0 + failed: int = 0 + cancelled: int = 0 + running: int = 0 + queued: int = 0 + + +class CascadeProgress(Model): + """A cascade that is still going, and how far each of its tasks has got. + + Only the tasks that exist. A cascade splits at every boundary — how many + jobs summarising becomes is a query that is only meaningful once the + bodies have landed — so the ones ahead have no jobs, no ids and nothing to + count. Which of them are coming is the app's own business, in its cascade + module and served beside this; here there is nothing to say about them. + """ + + #: The `runs` row, and the id `/api/cascades/{id}/cancel` takes. + id: int + #: Which cascade it is a run of — `pull`. For a job or a task queued on + #: its own, which is a cascade of one, that name and a word saying it + #: stands alone: `pull on its own`. The two are different runs and the + #: name is where they are told apart, so it is also what the strip shows. + name: str = "" + #: In the order they began, which for a cascade is the order they run. + tasks: list[TaskProgress] = [] + + +class JobsSnapshot(Model): + """What the queue looks like right now — the whole WebSocket payload. + + Sent on connect and after every state change, rather than diffed. The + whole thing is a few hundred bytes and one shape that is always complete + is worth more here than the bytes a patch protocol would save. + """ + + #: Everything under way. A list because the queue has several workers, so + #: "the running job" stopped being a single thing. + running: list[Job] = [] + queued: list[Job] = [] + recent: list[Job] = [] + #: The cascades any of that belongs to, tallied by task. Here rather than + #: behind a URL of its own because a tally of job states changes exactly + #: when this message does: asked for separately it would arrive a beat + #: later and draw a bar that disagreed with the list above it. And it + #: cannot be counted from that list — `recent` is the last thirty jobs, so + #: by the time tagging starts the body batches that landed have dropped + #: off the end and a bar drawn from it would count backwards. + cascades: list[CascadeProgress] = [] + + +# ---------------------------------------------------------------- notices + +#: How loud a `Notice` is, and the four colours a reader renders them in. +#: `error` is red — "you have to do something about this". `warning` is amber, +#: the same amber a work panel uses for a row that wants you — "this went +#: wrong and carried on". `info` is grey, worth saying once. `good` is green +#: and is the odd one out: nothing in `notices.py` produces one, because that +#: module is about failure — it exists for a UI to confirm something worked +#: without inventing a second mechanism to say so. +NoticeLevel = Literal["info", "good", "warning", "error"] + +#: Whether a failure is the world's or ours. See `Notice.kind`. +NoticeKind = Literal["expected", "bug"] + + +class Notice(Model): + """Something went wrong that you, not the log, need to know about *now*. + + Broadcast to whatever browsers are open, and written to the `notices` + table so that one opened afterwards can read it — see `NoticeEntry`, and + `pipeline/notices.py` for why it is kept after all. + + So the division of labour is: + + this "the tagger stopped, your claude login expired" — on + screen, the moment it happens, with enough to tell you + whether to shrug, fix something, or go and read the log. + the table the same sentence, findable once it has faded, with how + often it has been said. Cleared by you, or by a fortnight + of not being said again. + the log everything, with the traceback, still there tomorrow. + + Which means a notice does not have to be complete. It has to be *enough to + orient you*, and `where` points at the rest. + """ + + #: Stable dotted identifier — `agent.auth`, `source.failed`. What the UI + #: dedupes a burst on, and what a test asserts against, so the prose in + #: `text` can be reworded without breaking either. + code: str + level: NoticeLevel = "warning" + #: The distinction worth drawing when something breaks: is this the world + #: misbehaving in a way it is entitled to, or is this ours? + #: + #: expected an expired login, a feed returning 403, a rate limit. Not a + #: defect. Either you fix it or you wait, and either way there + #: is nothing to debug. + #: bug an exception we did not anticipate. Something to go and read + #: the log about, and probably to fix in the code. + #: + #: Getting this wrong in either direction is expensive — hunting a bug that + #: was a dead feed, or shrugging at a real one — so it is a field rather + #: than something to infer from the wording. + kind: NoticeKind = "expected" + text: str + #: What to do about it, when there is something: `claude auth login`. Empty + #: when there isn't, which is worth saying honestly rather than padding. + fix: str = "" + #: The raw error. Shown folded — it is the bit you would paste into a bug + #: report, and the bit that tells you which log line to look for. + detail: str = "" + #: Where to look for the whole story. Always the log; this names the part. + where: str = "" + #: What this is about within its code — a source name, a job kind. Lets the + #: UI collapse a burst of the same failure into one line with a count. + key: str = "" + + +class NoticeEntry(Notice): + """A notice as the log holds it: one row per `(code, key)`. + + What `Jobs.record_notice` hands back and what `/api/notices` lists. The + same failure said again is this row with a higher `count` and a later + `last_at`, never a second row — which is also how a client knows whether + to interrupt anyone: `count == 1` is news, anything above it is a repeat + of something already said and still sitting in the log. + """ + + id: int + first_at: str + last_at: str + count: int = 1 + + +__all__ = [ + "TERMINAL", + "CascadeProgress", + "Job", + "JobState", + "JobsSnapshot", + "JsonDict", + "JsonIds", + "Model", + "Notice", + "NoticeEntry", + "NoticeKind", + "NoticeLevel", + "Run", + "Schedule", + "TaskProgress", +] diff --git a/pipeline/notices.py b/pipeline/notices.py new file mode 100644 index 0000000..1590781 --- /dev/null +++ b/pipeline/notices.py @@ -0,0 +1,136 @@ +"""Turning a failure into one line worth putting on screen. + +The whole of this module's job is a handoff. Something broke; the log has the +traceback and always will; what the log cannot do is *tell you it happened* +while you are looking at the app wondering why nothing appeared. So a notice +is pushed to whoever is watching — and, since nobody usually is, written down. + +**It is stored, and it was not always.** The first design kept nothing: a +notices table would be a second record of what the log already records, with +rows that go stale and a dismiss button to maintain. That argument assumed +somebody was watching when the notice was raised, and somebody who could read +a server's log when they were not. Neither holds for a pull that runs on a +schedule and a reader holding a phone. What it cost was three things, each +reported as a defect of its own: + + a line that faded was gone twelve seconds, then nowhere to find it + nothing open, nothing heard the usual case, and the notice was lost + every run said it again "already said" was known only to a tab + +So `Jobs.record_notice` keeps one row per `(code, key)`, and the three are one +fix. The same failure said again is that row with a higher count, which is how +a client tells news from a repeat: it interrupts for the first and only counts +the rest. A line somebody dismissed is still in the log, so dismissing means +later rather than never. A row goes when somebody clears it, or a fortnight +after it was last said — by which time it has stopped being true, which is the +only staleness the first design was right to worry about. + +The table is not the log and does not try to be. No traceback, no history of +each occurrence: a sentence, what to do, the raw error, how often, and when. + +This module is only the part the queue itself needs: the notice every failed +job gets, the check that stops it being said twice, and the rendering a +terminal wants. **Classifying a failure is the app's job**, because a useful +sentence names things only the app knows — its model backend, its feeds, its +config keys. An app writes its own vocabulary next to its handlers and passes +notices in through `RunContext.notice`; newsdesk's is `desk/notices.py`. + +What a notice has to carry is not the full story but enough to orient you: + + text what happened, in a sentence, in terms of what you lost by it. + kind whether this is the world misbehaving — an expired login, a feed + gone 403 — or a bug of ours. That is the fork in what you do next, + and guessing it from an error string is exactly the work you should + not have to do at a glance. + fix the command, when there is one. +""" + +from __future__ import annotations + +import logging +from collections.abc import Iterable + +from pipeline.models import Notice, NoticeKind + +log = logging.getLogger(__name__) + +#: Words that mean the failure happened at the other end of a wire. An app's +#: own classifier will want more than this; `whose_fault` is the cheap default +#: for the failures the queue meets on its own. +THEIR_FAULT = ( + "http", "url", "timeout", "connection", "ssl", "certificate", "socket", + "dns", "resolve", "403", "404", "429", "500", "502", "503", "refused", + "unreachable", "forbidden", "not found", "unavailable", "moved", +) + + +def job_failed(kind: str, label: str, error: BaseException | str) -> Notice: + """A job that ended in `failed`. + + A handler raising something it did not expect is the definition of ours to + fix, so this is the one place that says `bug` outright rather than working + it out. + """ + return Notice( + code="job.failed", level="error", kind="bug", key=kind, + text=f"{label or kind} failed.", + detail=str(error).strip(), + where=f"the `job … ({kind}) failed` traceback in the log", + ) + + +def already_said(said: Iterable[Notice], error: str) -> bool: + """Whether one of these notices is about this failure. + + The caller is the worker, which announces `job_failed` for every job that + ends in `failed` — and must not, when the handler already announced a + classified notice about the very exception that ended it. "agent.backend + is `off`, so nothing that needs a model runs" followed by "Tagging a batch + failed — looks like a bug" is two contradictory lines about one failure, + and the wrong one is the louder: it sends you to a traceback about a + machine that is doing exactly what it was configured to do. + + Matched on `detail` rather than on identity, because what the handler + classified is an exception and what reaches the worker is that same + exception rendered `Type: message` — the same string with a prefix. Which + is also the line in the right place: a notice about something else the job + met on the way, a source that 403'd during a pull that then crashed over + something unrelated, does not match, and that crash still gets its own + line as the unexplained failure it is. + """ + return any(notice.detail.strip() and notice.detail.strip() in error + for notice in said) + + +def render(notice: Notice) -> str: + """The same notice, for a terminal. + + The browser is the first-class destination, but it is not the only one: a + handler run from a terminal hits every one of these failures and deserves + the same sentence, the same command to run, and the same "is this mine to + fix". Without this the CLI got the code and the text and silently lost the + two fields that make a notice worth having. + + Rendered rather than logged in pieces so that the wording lives in one + place. An app changes a rule and both destinations change together. + """ + lines = [notice.text] + if notice.fix: + lines.append(f" -> {notice.fix}") + if notice.kind == "bug": + lines.append(f" this looks like a bug rather than a bad day; see {notice.where}" + if notice.where else " this looks like a bug rather than a bad day") + return "\n".join(lines) + + +def whose_fault(error: BaseException | str) -> NoticeKind: + """Whether a failure is the other end's or ours. + + A 403 is the outlet's business and there is nothing here to fix. A + `KeyError: 'entries'` is us assuming a shape the other end didn't have, and + it will keep happening until someone changes the code — so it belongs in + front of you as a defect, not as weather. + """ + text = f"{type(error).__name__} {error}".lower() if isinstance( + error, BaseException) else str(error).lower() + return "expected" if any(hint in text for hint in THEIR_FAULT) else "bug" diff --git a/pipeline/pipeline.md b/pipeline/pipeline.md new file mode 100644 index 0000000..63865ea --- /dev/null +++ b/pipeline/pipeline.md @@ -0,0 +1,395 @@ +# `pipeline` — the job queue + +## What it is + +A jobs **table in SQLite** with an async worker draining it. Handlers are plain synchronous functions run in threads. + +| | | +|---|---| +| Packages | stdlib here — `sqlite3`, `asyncio`, `dataclasses`, `inspect` — plus pydantic, which `pipeline/models.py` wants | +| Not used | no Celery / RQ / Redis / Huey, no `multiprocessing`, no in-memory queue | +| Workers | threads, several at once, one per running job, from a pool of the queue's own | +| Storage | one SQLite file (WAL). Point it at the app's DB or one of its own | +| Concurrency | threads within a worker; any number of workers on one file — `claim` is one atomic `UPDATE`, and a claimed job is held on a lease its worker renews. `python -m pipeline.worker` starts one with no web app; `pipeline/tests/test_housekeeping.py` races two | +| Web | `pipeline.web.connect(app, jobs, config, store)` — routes, socket, the `` and `` elements, and the worker in the app's lifespan | +| States | `queued → running → done \| failed \| cancelled`, and `running → queued` again for a job with an attempt left | +| Scheduling | five cadences on a `schedules` row in the same file — `every 15m`, `daily at 06:00`, `weekly mon 07:00` — fired from the drain loop, defaults declared next to `@job` | +| Door | `from pipeline import DAYS, FINAL, FORMS, JOBS, MAX_RESULT, NULL, SCHEDULES, TASKS, TERMINAL, BadCadence, Batch, Bound, Cadence, Cancelled, CascadeProgress, ChildFailed, Declared, Handler, Job, JobQueue, JobState, Jobs, JobsSnapshot, Listener, Notice, NoticeEntry, NoticeKind, NoticeLevel, NoticeListener, ResultTooLarge, Retry, Run, RunCancelled, RunContext, Schedule, ScheduleListener, Spec, Tally, Task, TaskProgress, declare, fan_out, job, local_zone, notices, spawn` | +| Copyable | yes, and that is the point — `cp -r pipeline yourproject/`. Nothing here imports `desk` or `newsdesk`, and `tests/test_architecture.py` fails the build if it ever does. See `docs/pipeline-package.md` | + +The table *is* the queue, which is why a job survives a restart, why the UI's job list is a `SELECT`, and why a worker can live in another process. + +## The Entities + +**Job** — one physical unit of work, admitted to the queue and run on one worker. It has a resource class, so the queue knows which worker to run it on. A job is the only thing the queue runs, and the only thing that has a row in `jobs`. + +**Handler** — the function that defines the job, aka the code. `@job` registers the handler in the queue. The handler is the piece of code which is called when the job is about to run. + +**Task** — **plain code that creates jobs, and is never a job.** A business step comes in — fetch every feed, give every article a body — and is translated into jobs the queue can run, typically by cutting the work into batch payloads. A task mints a run id for itself, queues one job per payload tagged with that id, and queues one **final job** behind them with `after="done"` on all of them, which is how it comes to be reported on. Then it is over: nothing of the task is alive afterwards, only its jobs and the callback the final job carries. + +Everything a task queues, it queues in one go. That is what makes it plain synchronous code and not a job: finding the work is a query, and a query is cheap enough to run inside the request that pressed the button, or in the worker thread that has just finished the job before. If the planning itself needs real work done first, that work is a job, and what follows it is the next task of a cascade. + +**Cascade** — a series of tasks, for when each can only be planned once the one before it is over, or where there is a natural breaking point between them. Structurally it is the same shape as a task, and the pipeline does not distinguish the two. Both are rows in `runs`, both group jobs by an id, both are killed by the same call, and neither has an object of its own. The difference is a **convention**, and it is worth stating because nothing enforces it: + +> A **task** queues everything it wants at once. +> A **cascade** is where the breakpoints are: it queues the next task only once the last one is over. + +The final job of a task is what calls back to the cascade to start the next task. + +Jobs are tied to a task and a cascade by `task_id` and `cascade_id`. + +**Schedule** — **a row that queues a job, and is never a job itself.** "Enqueue this kind with this payload on this cadence": a name, a kind, a payload, a cadence, whether it is on, when it is next due and what it queued last time. It is a row rather than a timer for the same reason a queued job is: nothing is alive between runs, so a process restarted at 05:59 still fires the 06:00 one, and two workers on the file read one schedule instead of each holding a copy of it in memory. It queues *jobs*, not tasks — a schedule that wants a whole cascade queues the job that starts one. + +**Cadence** — one of five wordings, and the only five: `every 15m`, `every 6h`, `every 2d`, `daily at 06:00`, `weekly mon 07:00`. Not cron, which is five fields of positional punctuation that nobody reads back correctly. The interval forms are elapsed time and ignore the timezone; the clock forms are wall-clock in `JobQueue(tz=...)` and are 06:00 on both sides of a daylight-saving change. The two mean different things and a person picking one has already chosen which. + +**Declaration** — a default schedule an app ships, written next to `@job` with `declare(name, kind, cadence, payload, label)`. `SCHEDULES` is to it what `JOBS` is to `@job`. On start, a declared name with no row gets one and a name that has one is not touched, so the declarations are where a schedule *comes from* and the table is what it *is*. + +## How a job has to look + +```python +def handler(config, store, *, context=NULL, limit=None) -> dict: ... +``` + +`config` and `store` are the app's, passed through untouched. Payload keys arrive as keyword arguments — **declare the ones you take**; an undeclared key is a `TypeError` recorded against the job, and `**kwargs` would swallow a typo. `context` is optional: a handler that takes neither progress nor cancellation says so by not declaring the parameter, and `registry.py` works that out once at registration rather than inspecting a signature per run. One payload key is reserved — a job whose payload names `context` fails, because the worker's own is what goes there. + +`limit` is a **convention, not a contract**. Nothing in the pipeline reads it. It is an ordinary payload key that most of newsdesk's sweeps happen to accept, so that a person can ask for a short run of one, and a handler is free not to have it. + +The return value should be a JSON-serialisable dict, and is stored as the job's result. A handler that returns something else — `None`, a list, a count — is not an error: the worker keeps the ending and stores an empty result, so the job is `done` with `{}`. Which means a result nobody can read is silent, and worth knowing when a handler's tally goes missing rather than its job failing. + +**An exception ends the job in `failed`.** The traceback goes to the log, the exception is stored on the row as `Type: message`, and a `job_failed` notice goes to the screen saying it looks like a bug — *unless* the handler already announced a notice about that same failure, in which case the handler's sentence stands alone rather than being contradicted underneath. The `except` is deliberately broad: a handler is a source plugin, a model call, a subprocess, and anything it raises is that one job's failure and nothing else's. Letting it escape would take the worker down and strand every job behind it. + +## Minimum example + +```python +from pipeline import Jobs, JobQueue, job, RunContext, NULL + +@job("greet", "Saying hello", solo=True, resource="cpu") +def run_greet(config, store, *, context: RunContext = NULL, name="world") -> dict: + context.step(0, 1, f"greeting {name}") # progress + cancellation check + return {"greeting": f"hello {name}"} + +run_greet(cfg, store, name="wolf") # 1. straight call — no queue at all + +queue = JobQueue(Jobs(cfg.database), cfg, store) # 2. queued +await queue.start() +queue.enqueue("greet", {"name": "wolf"}) +``` + +`@job` returns the function unchanged, so every job stays importable and callable on its own. Registration is by import — importing `newsdesk.jobs` fills `JOBS`. + +## Workers + +`JobQueue.WORKERS = {"llm": 4, "http": 8, "cpu": 1}` — how many workers of each kind there are, and so how many jobs of that resource run at once. An app passes its own; newsdesk has a `workers` section. A resource named in neither gets one. Counted from the table, not from this process, so a second worker on the same file shares the four model workers rather than helping itself to four more. + +The count belongs to the worker, not the job kind: tagging and summarising both ask a model, so they share the same four. + +Our queue is fanning out threads not processes. Therefore, a worker is a thread, not a program. + +## `depends_on` + +A job may name jobs it waits for, and say what it wants of them. + +| | | +|---|---| +| `after="ok"` | admit only if all of them succeeded; fail with them otherwise. The default, and what one job needing another's output means. | +| `after="done"` | admit once all of them are over, however they ended. | + +Waiting costs nothing: `claim` passes the job over and it stays queued. No timer, no listener, nothing holding a place, nothing lost to a restart. + +`done` exists for the job that reports on the others — success-only semantics would mean the report never exists when there is something to report. An id naming nothing is satisfied. + +A **cancel** and a **failure** are the same thing here: both travel to the `after="ok"` jobs waiting on them, and neither holds back an `after="done"` one. Killing a job is dropping that batch, not stopping the run it belongs to, so the job that reports still runs and counts it among the ones that did not land. + +**What a waiting job is handed.** `context.waited_for` is a tuple of `pipeline.models.Job` — the whole row for each job this one named, oldest id first, and empty for a job that waited for nothing. Each carries its `state` and its `result`, which is everything a report needs. Read at claim time off the jobs themselves, never counted up as they landed, so there is no tally to keep in step and nothing to lose to a restart half way through. An id that named a job which no longer exists is simply absent from the tuple, so `len(waited_for)` is what was found, not what was asked for. + +## `solo` + +**One job of this kind queued or running at a time.** Two tabs pressing Fetch is one pull, and the second caller is handed the job already going rather than an error, because what they asked for is that the work be under way and it is. Enforced by a partial unique index over the unfinished states, so it holds across every tab, every client and every process at once — not by a flag in whichever browser happened to ask. + +**Off unless a job says otherwise.** A sweep that forgets to say `solo=True` runs twice and wastes a minute. A batch kind that forgets to say `solo=False` has seven of its eight batches quietly handed the first one, and the work simply never happens. The default belongs on the side that fails loudly. + +A job that names `depends_on` never dedups, whatever it declared, since the whole point of a batch is that there are several. + +**The job you are handed back may not be yours.** Dedup matches on the kind alone, so a duplicate is answered with whatever live job holds that key — which can belong to another cascade entirely. Press Fetch while an overnight pull is part way through its own `pull` job and you are handed *that* job: the right answer to "is a pull under way", and the wrong one to "is *my* pull under way". Nothing rewrites its `cascade_id`, so a caller that means to follow the run it started has to notice the ids it got back are not the ones it minted. + +## Retrying + +```python +@job("extract", "Reading the page", resource="http", + max_attempts=3, retry_delay=60, retry_backoff=2.0) +def run_extract(config, store, *, context=NULL, url=""): ... +``` + +**`max_attempts` counts the first attempt**, so `1` is the default and means what every job here meant before the column existed: run it once, and an exception is the end of it. An attempt that fails with attempts left goes back to `queued` with a `run_after` on it, `retry_delay × retry_backoff ⁿ` seconds hence. + +**Waiting costs exactly what `depends_on` costs, which is nothing.** The wait is a column, not a sleeping worker: `claim` passes the row over until its time comes round, so no worker, no thread and no lease is spent on an hour's backoff, and the poll that exists to notice another process's enqueue is what notices the hour is up. Sleeping in a handler would hold a worker for the duration, and is the thing this exists instead of. + +**And because it is a column it survives the process that wrote it.** A retry scheduled for 09:40 is taken at 09:40 by whichever worker is alive then, including one started at 09:38 in a different process. There is nothing to hand over, because nothing was alive to hand. + +**Three endings, and `Jobs.retry` picks between them.** `cancelled`, if a Kill landed while the attempt ran — a retry that put the job straight back would be a button that stopped nothing and said it had. Back to `queued`, if there are attempts left. `failed`, if there are not, and `Job.exhausted` says so from then on: the policy ran its course and the work still did not land, which is the only one of the three where trying again is a person's decision. A job that failed at `max_attempts=1` is not exhausted — nothing was ever used up. + +| | | +|---|---| +| `job.waiting` | queued, with its next attempt still to come | +| `job.exhausted` | failed on the last attempt it was allowed, rather than the only one | +| `job.attempt` | how many times it has been *started*, stamped by `claim` in the statement that takes the row | +| `jobs.waiting()` | every row in the first of those states, oldest first | +| `jobs.blocked()` | everything `claim` would pass over — waiting on a time, or on another job | + +**A handler can name the wait but not the bound.** `raise Retry("rate limited", after=120)` puts the other end's own answer in place of the backoff, which is worth having because the alternative is a guess: a minute is either a minute of doing nothing when the service was ready, or another refusal charged against the bound. It buys no extra attempt — a job queued with one attempt that raises `Retry` fails exactly as it would on any other exception. The bound is ours to declare and the wait is the world's to name. + +**A retry says nothing to the screen.** A notice is for what a person has to act on, and while attempts remain there is nothing to act on: the queue is already doing what they would have asked for. The log has every attempt either way; only the one that runs out announces. + +**Retrying is off by default**, for the reason `solo` is off by default. A fetch that should have been retried and was not costs one item until the next sweep finds it again. Work retried three times that never should have been is three times the writes, or three times the bill. + +`queue.enqueue(kind, delay=600)` schedules one job for later with no retry involved at all. `queue.enqueue(kind, max_attempts=n, retry_delay=s, retry_backoff=b)` overrides what the kind declared for one piece of work, and `dedup_key=` overrides the kind's own key — which is how one *item's* acquisition is made unrepeatable while the kind stays freely repeatable, so two presses of a reader's retry button collide in the unique index rather than in one browser tab. + +**A batch can carry its own bound.** A task's finder may return `Batch(payload, Bound(max_attempts=3, delay=300, backoff=2.0))` instead of a bare payload, and those numbers go onto that job's row. That is where a plugin's declared retry policy lands: newsdesk cuts its body batches per plugin profile precisely because the row holds one bound, so a batch mixing two outlets could only carry one of their declarations. A finder returning plain dicts — every task that existed before this — gets `Bound()`, which is one attempt and no wait. The direct path (`task.run`) ignores bounds entirely: a bound is a number the queue enforces by putting a row back with a time on it, and there is no row. + +## Typical way of dealing with batches + +A task queues its batches as independent jobs, and one **final job** behind them. It depends on all of them with `after="done"`, so it runs however they ended, and it is handed each one's state and result in `context.waited_for`. It is the only job of the task that calls back. + +A task that finds nothing to do is the ordinary case, not the empty one: it queues no batches and its final job runs immediately, with an empty `waited_for`, and moves the cascade on. Which is why a switch turned off, a backlog already cleared and a stage this pull does not want are all the same thing here — a finder that returns no payloads — and none of them stops the chain. + +**The callback is a name in the final job's payload, not a function.** Nothing in a table can hold a function, and a name is what makes the callback survive a restart and be callable from another process; the worker running the final job resolves `module:function` to imported code. `newsdesk.jobs.cascade:step` reads as documentation in the payload, and carries its own import, where a registry would have to be filled before the worker looked in it. + +It is called with the task's tally and whatever the task fixed at the time it was made: + +```python +def step(config, store, tally: Tally, payload: dict, *, context: RunContext) -> dict | None: ... +``` + +`payload` is `Task.carries`, stored in the final job's row, so it survives a restart and a worker in another process. Whatever the callback returns is merged into the final job's own result; `None` is fine. A task with no callback stops there, which is what being asked for on its own means. + +**`settle` is a second name in the same payload, resolved the same way, and it belongs to the task rather than to what follows.** The final job runs it before the callback, over the same tally. It is for the decision a batch is in no position to make: a batch is one of forty and cannot see what the other thirty-nine are doing, so anything that must hold across the whole run — a cap on what the run may add, a merge of what several batches each asked for — is worked out once, here. Newsdesk's tagger is the one that has one: a batch proposes new tags and adds none, and the settle merges the proposals, takes at most `max_new_tags_per_run` of them and goes back to tag the items that asked. A task started on its own has no callback and still settles, because settling is its own work and not a message to anybody. + +**`Tally`** is how the batches ended, read off their rows at the moment the final job runs: + +| | | +|---|---| +| `done` `failed` `cancelled` | how many batches ended each way, and `jobs` for the sum | +| `results` | what each batch returned, in the order they were queued | +| `killed` | whether the **task itself** was killed | + +`killed` is the one that cannot be inferred from the counts, and is the reason it is there. A task killed while its batches ran leaves every one of them `cancelled`, and so does a person killing every batch by hand — identical numbers meaning opposite things: *skip this step*, or *drop this work*. Only the `runs` row knows, so the final job asks it. What to do about it is the cascade's decision and nothing here presumes it; newsdesk's pull skips the stage and carries on to the next. + +## Passing data + +Three ways to pass data. Can be mixed and merged. + +1. **Store.** A job reads its input or persists its output directly into a storage. +2. **Parent.** A job receives data as a parameter, put in its payload by whatever queued it — a batch's uids, a cascade's options. Fixed at queue time, and the only kind of data a job that runs first can hand to a job that runs later. +3. **Cross job via `context.waited_for`.** The `after="done"` job gets the jobs it waited on: their `state` and their `result`. Read, never pushed — nothing is counted while the work runs, so there is no tally to lose. + +The usual arrangement is the third feeding the callback: the final job reads how its batches went and hands the tally to the cascade, which turns it into the next `Task`. + +`context.upstream()` is the third one merged into a dict, later parents winning, which is what a chain of one parent wants to read like. It is `{}` under `NULL`, which is what keeps a chained handler callable by hand: + +```python +@job("extract") +def extract(config, store, *, context=NULL, html: str = ""): + html = html or context.upstream().get("html", "") +``` + +From the queue nobody passes `html`; from a terminal you pass it yourself. A handler reading `waited_for[0].result` directly would be the one kind in the pipeline you cannot simply call. + +**The rule is that whoever has the data writes the row.** Never insert a row meaning an earlier job to fill its payload in later: a worker claims it the moment the parent is `done`, so it can be claimed between that write and its commit; `dedup_key` has to be known at queue time; and `max_attempts` re-runs it from its own row. Reading a parent has none of that, because the row was complete when written. + +Bulk data belongs in the store. `Tally.results` is a query across the children's `result` TEXT columns and those columns never shrink — counts, ids and errors are free, a few hundred large dicts are not. + +## `fan_out` — N jobs and the one that reports on them + +```python +@job("body") +def body(config, store, *, context=NULL): + context.fan_out("body_batch", [{"uids": c} for c in chunks], + then="newsdesk.jobs.cascade:step") +``` + +One call, one transaction. The reporting job is `FINAL`, the pipeline's own kind, and an app never writes one. The children go into a run of their own inside the caller's cascade, which is what lets the callback fan out again — step N queues step N+1, and that is the whole of what a cascade is. + +Because the thing fanning out is itself a job it has a context, so this works from inside a handler as well as from a request. `Task` is the same three statements with a finder in front, which is why a task and a cascade are one shape and one primitive covers both. + +`()` under `NULL`: a handler called directly is being run by somebody standing there, and the useful answer is to do its own work and report that nothing was queued behind it. + +## Running it + +```python +queue = JobQueue(Jobs("app.db"), config, store) # then, in an app: +connect(app, Jobs("app.db"), config, store) # pipeline.web: routes, socket, element, lifespan +queue.drain(timeout=60) # in a script: run what is queued, stop +``` + +```bash +python -m pipeline.worker --db app.db --bootstrap myapp.jobs:bootstrap --bootstrap-arg config.yaml +``` + +The worker is the same `JobQueue` with no web app, on the same table, and may run beside the web app's own. `--until-idle` drains and exits. The bootstrap is the one `isolate=True` uses. + +## Ending work that will not end itself + +Cancellation is cooperative because the work is synchronous — but two kinds of work never look up, and for those there is a real kill. + +**A command.** `context.spawn(argv, timeout=...)` runs it, polls for Kill and for the deadline, and kills the **process tree** on either — `taskkill /T` on Windows, `killpg` elsewhere, which is why the child is started detached. `proc.kill()` alone would leave the ffmpeg a shell started still holding the core. Blocking a worker thread on this costs nothing: the thread is only sleeping. Output goes to temporary files, not `PIPE`, because a child that fills the pipe buffer while the loop polls is a deadlock. A non-zero exit is handed back rather than raised. + +**Python that cannot be interrupted.** `@job(..., isolate=True, timeout=600)` runs the job in a child process, which is the only thing that can be killed mid-`sklearn.fit`. Nothing is pickled: a job row is already a complete description of the work, so the child is handed `--db` and `--job` and reads the same row. It writes *progress* to that table — which is how its progress reaches the browser with nothing forwarding it, the snapshot being a `SELECT` — and hands its result back through a file. The worker still writes the ending, so there is exactly one writer of the terminal state, as before. + +`isolate=True` needs `bootstrap="module:function"` on the queue, returning `(config, store)`: neither survives a process boundary, so both are rebuilt in the child, and the same import is what fills `JOBS` there. `bootstrap_arg` is one string passed to it — an app's config path, which the child cannot learn any other way. + +**`timeout` without `isolate` raises at registration.** A deadline over a thread could only mark the row failed and walk away — the work would carry on holding the core, and two things would intend to write one row. The refusal names both ways out. + +## `context.task` — a job asking for what comes next + +Newsdesk's body task fetches each eligible article and passes its HTML to `desk.tools.article.extract_article`. Its storage writer saves structured content, format, provenance and a derived plain-text projection atomically; existing enrichment jobs continue to read the generated `body` view. Empty or late fetch results preserve previously acquired content. The body job still uses the existing wall detector and candidate rechecks, and does not yet execute plugin-declared acquisition *graphs* — one declared step, the one whose tool is `extract_article`, is what it runs. Its declared **retry policy** is executed: a failure is classified with `desk.tools.failures`, the plugin decides whether that category is worth another attempt, and the batch raises so the queue puts it back under the bound its row carries. Batches are therefore cut per plugin profile. A second attempt asks the table again and so re-fetches only what is still missing, and never overwrites text that has since arrived. Structured bodies are retained without the former 60,000-character clipping; downstream model tasks remain responsible for their own input limits. + +Hacker News discovery now runs as the `hacker_news` outlet through shared RSS fetching. Its item destination stays the publisher URL, so the subsequent body task extracts that article and records that URL as provenance. Its supplied discussion link is a separate typed input, and feed points/comment statistics stay in source evidence rather than the summary or preview. Its declared tagging policy runs: the destination article is what the tagger reads, and a link whose page would not extract is tagged from its title. Its summary policy stays off — the summariser selects only profiles that ask for an abstract, and nobody has decided that a front-page link earns one. + +A handler that discovers work only by doing its own can name the task that should follow it: + +```python +context.task(cascade.task("body", cascade="pull")) +``` + +**Recorded, not queued.** The worker starts it after this job lands in `done`, on the same thread that ran the handler, and hands it this job's `cascade_id` so its batches land in the run this one belongs to. Three consequences worth knowing: a task never outlives the failure of the job that described it, a handler still cannot reach the queue, and a job's row lands a moment *before* the jobs that follow from it exist. + +It returns whether anything took it — `False` under `NULL`, where there is no queue. A callback invoked from a notebook or a Claude Code session gets that answer and has to do the work itself. + +## `Cancelled` and `RunCancelled` + +Two exceptions with similar names, and only one of them is a handler's business. + +**`Cancelled`** is raised by `context.check()` inside a handler when this job has been asked to stop. The worker catches it and records `cancelled` rather than `failed`. A handler should let it travel; catching it is how you build the thing the Kill button cannot kill. A handler that never checks is not lost either — the worker asks once more when the handler returns, and a job asked to stop while it ran ends `cancelled`, keeping whatever it made and queueing nothing. + +**`RunCancelled`** is raised by `Jobs.enqueue` for a job whose `cascade_id` or `task_id` names a killed run, except for an `after='done'` job under a killed *task* — the same carve-out `cancel_run` and `stop_task` make, so a kill landing while a task is still fanning out its batches cannot take the final job with it. A killed cascade refuses it too, which is what "nothing reports and nothing follows" means. It is not a handler's business and a handler will not see it: `JobQueue.enqueue` catches it, logs it, and returns **`None`** instead of a `Job`. So a caller that queues a job has to expect no job back — `Task.start` skips the id it did not get, and a `depends_on` list is that much shorter. Nothing is told that a run is over; a callback tries to queue the next task and the table says no, which is the same answer across processes, after a restart, and when the kill landed mid-run. + +## Ids + +Every job has an `id` of its own, and carries a `task_id` and a `cascade_id`. Both are grouping ids: nothing depends on a task, but everything belonging to one can be found by it and killed together. Two of them rather than one because they are two groupings and both have a button. + +**Killing a run** is one operation for a task and a cascade alike — `cancel_run`. Three things: flag the row, end its queued jobs, ask its running ones. And a fourth that is not an operation at all: **`enqueue` refuses a job whose `cascade_id` or `task_id` names a killed run**, which is what stops a callback from reviving a killed task or moving a killed cascade on. + +Three buttons, three sizes of stop: killing a **job** drops one batch, killing a **task** skips one step, killing a **cascade** ends the run. + +**Stopping a run is not killing it.** `Jobs.stop_task` ends a task's queued batches and nothing else: the row is not flagged, so `Tally.killed` stays false and nobody is told a person pressed anything; running batches keep their requests and their own endings; the final job still reports. It is what a handler reaches through `RunContext.stop(why)`, for the failure that is the backend's answer to the whole run rather than to one batch — a spend limit, an expired login, no model on the machine. The reason is written to `error` on each cancelled row, so a batch that never ran says on screen why it didn't. `desk/modeltask.py` is the one caller, and `notices.stops_the_run` is what decides. + +The one asymmetry between killing a `cascade` and killing a `task` is what the last two of those mean. Kill a **task** and its final job is spared: it still runs, and what it reports says the *task* was killed rather than only how many of its batches ended cancelled — see `Tally.killed`. Kill the **cascade** and the final job dies with it, so nothing reports and nothing follows — the one stop no cascade gets a say in, because there is nothing left of it to ask. + + cascade_id = jobs.new_run("cascade", "pull") # before a single job exists + jobs.of_cascade(n) · jobs.of_task(n) · jobs.run(n) · jobs.cancel_run(n) + jobs.cascades() + POST /api/cascades/{id}/cancel · /api/tasks/{id}/cancel · /api/jobs/{id}/cancel + +A job queued with neither begins both: something asked for it on its own, so it is a cascade of one task of one job. A task started with no cascade does the same, for the same reason. + +The cascade that comes of it says so in its name — `pull on its own` rather than `pull` — because a chain named `pull` is a different run, and `live_run("cascade", "pull")` is how a caller asks whether that chain is already going. Answering it with a lone job would be a Fetch button that starts nothing and greys itself over a run that is one job long. The task keeps its plain name (`pull`), since the task really is that task however it was started, and the strip draws its box from that. + +## Schedules + +The tick is in the drain loop, at the top of `_turn`, before the claim — not in a thread of its own and not on the idle path with the reclaim and the purge. Not a second thread, because then two things would race to enqueue one row. Not on the idle path, because reclaim and purge are repairs that can wait for a quiet moment and a schedule is *work*: on a queue that is never idle, 06:00 would never come round. The check itself is a clock comparison, so paying it every pass costs nothing. It also runs once in `start()`, after reconciliation, so a default declared today fires on the start that introduced it. + +**Overdue fires once, not once per slot missed.** A firing sets `next_run` from *now*, never by stepping through the times it was down for. There is one `next_run` column and nowhere for a backlog to accumulate even in principle, so an app off for three days runs its hourly job once when it comes back. + +**No pile-up.** Before firing, two questions: is the job from the last firing still queued or running (`last_job_id`), and is this kind `solo` with one already under way? Either way `next_run` advances and nothing is queued. A job slower than its own cadence must not grow a queue of itself, and a `solo` kind somebody has already pressed must not be recorded as this schedule's firing. + +**A row the code no longer supports keeps its place.** An unregistered kind, or a cadence nothing can parse (only reachable by editing the file by hand), is logged once per process and left exactly as it is — `next_run` included — so registering the handler again runs it rather than starting another cadence. `` marks the row `unknown kind`; deleting it is deliberate. + +**Reconciliation never overwrites and never deletes.** `Jobs.add_schedule(..., only_if_new=True)` is the whole of it: one statement, so two workers starting at once insert a declared default once between them. A declaration removed from the code leaves its row behind, because a deploy that silently forgot a schedule somebody relied on is worse than an orphan that says so. + +**One scheduler per file.** `JobQueue(schedules=False)` and `pipeline.worker --no-schedules` are what every worker after the first passes. The workers themselves race correctly — that is `claim` — but the pile-up guard reads `last_job_id`, which neither of two simultaneous firings has written when the other looks. + + declare(name, kind, cadence, payload=None, label="") # next to @job + Cadence.parse(text) · str(cadence) · cadence.next_after(moment, zone) + jobs.schedules() · jobs.schedule(id) · jobs.add_schedule(...) + jobs.update_schedule(id, **fields) · jobs.delete_schedule(id) + jobs.due_schedules(now) · jobs.mark_fired(id, job_id, next_run) + queue.run_schedule(id) · queue.notify_schedules() + GET/POST /api/schedules · PATCH/DELETE /api/schedules/{id} + POST /api/schedules/{id}/run + +## The three tables + +**`jobs`** is the queue, and holds rather more than the states: the `payload` it was queued with and the `result` it ended with, both JSON; `done`, `total` and `message`, which are the progress bar and are written by `context.progress`; `error`, the exception string of a failure; `dedup_key`, `depends_on` and `after`; `resource`; `cascade_id` and `task_id`; the retry policy and where it has got to, as `attempt`, `max_attempts`, `retry_delay` and `retry_backoff`; and five times — `created_at`, `started_at`, `seen_at` (the lease), `run_after` (the next attempt) and `finished_at`. `cancel_requested` is a flag, because a kill is a request in the table rather than a message to whichever process happens to be running the job. + +The policy is on the row rather than read off the job kind when a failure happens, for the same reason the payload is: the attempt that has to decide may be in another process, after a restart, running a version of the code where the kind's numbers are different or the kind is gone. + +**`schedules`** is the standing instructions: `id`, `name` (unique, and the handle a declaration is matched on), `kind`, `label`, `payload`, `cadence`, `enabled`, `next_run`, `last_run`, `last_job_id`, `created_at`. `next_run` and `last_run` are UTC ISO stamps like every other time here, and `next_run` is compared as text in `due_schedules` for the reason `run_after` is in `claim`: `''` sorts before every stamp, so "never" and "not due" are one comparison with no coalesce in it. A cadence worded in a local timezone is turned into one of these before it is stored, so `pipeline/schedule.py` is the only file that reads a zone at all. + +**`runs`** is where a cascade's id and a task's come from, and it is deliberately almost empty: `id`, `kind` (`cascade` or `task`), `name`, `label`, `created_at`, `cancelled_at`. + +What it does **not** have is the point of it. No state, no progress, no counts, no `finished_at`: the jobs carrying the id are the record of what happened, and a second copy of that could only go out of step with them. `cascades()` tallies a run by reading its jobs. So the table exists for exactly two facts its jobs cannot supply — that a number was handed out *before* there was a job to take it from, and that somebody killed this run. `name` carries a third job: it is how `live_run` answers "is one already going", which is why a cascade minted only because a lone job needed an id is named apart from a chain somebody asked for (see `on_its_own`). + +## `ModelTask` — next door, not here + +`ModelTask` left this package: it imports `desk.config` and `desk.tools.agent`, so it was never the queue's. It is `desk/modeltask.py` now, and this section stays because it is still the thing most jobs in newsdesk are. + +`desk/modeltask.py` is one layer above `desk/tools/agent.py`. `agent` answers *how do I reach a model*; this answers *what do I send it and what do I do with what comes back*, once, for every task that needs one. Tagging and summarising are the two, and what differs between them is all content: which rows are waiting, what goes in the batch, what the model is told, what shape it must answer in, what to do with the answer, what the run has to settle once every batch is in, and how a row that never gets one is given up on. That set of callables **is** a `ModelTask`; the module is everything that is not one. + +`modeltask.of(TASK)` turns it into an ordinary `Task`, so it is queued, watched and killed like any other stage. `modeltask.one` is what a single batch job does: check for a kill, build the payload from the uids it was given, ask, apply. Two things about it are worth reading before changing: + +- **The prompt is built when the batch runs, not when it was handed out.** Tagging's carries the vocabulary as it stands at that moment, so a tag somebody adds by hand mid-run reaches the batches still to come. It does not carry what a sibling batch proposed, because a batch proposes and adds nothing: the run's own additions are worked out once, in `settle`. +- **`pending`, `waiting` and `build_batch` are given the config as well as the store.** Which rows are waiting is no longer a question a table can answer on its own: in newsdesk both backlogs are whatever the installed plugins declare about their own items, and the plugins are found through the config. Both tasks now use it; the argument is still there for one that does not. +- **A kill during the call does not throw the answer away.** `check()` is called before the model is asked, so a Kill pressed while the job sat queued makes no paid call at all. Once the request is in the air the money is spent, so the answer is applied and the worker records the cancellation at return. The asymmetry is deliberate and looks like an oversight otherwise. + +**A row is given up on eventually.** "The next run finds the work again" is unbounded, and for a row no model will ever answer usefully — a segment whose only honest abstract is one sentence, a uid dropped from every reply — unbounded means paying for the same call on every pull for as long as the row exists. So a row handed over and left unanswered is charged a turn through `ModelTask.missed`, and one charged three of them stops being offered until a person asks for it deliberately. A batch that *failed* charges nothing: it never reached a model, so it proved nothing about its rows. Every `ModelTask` must supply `missed`, and there is no default, because the default is the bug. + +**A refusal is the one failure that is about a row.** The platform runs a classifier over the request and rejects the whole thing at once, naming nothing — and a news archive meets it regularly, because reporting on a worm or a supply-chain compromise is written in the vocabulary it watches for. `one` answers it by asking again in halves (`_narrow`) until the batch is the row, which is then charged and leaves the backlog after three. Halves rather than one at a time: two flagged rows in a batch is ordinary, since they arrive from the same day's news, and stepping through twenty singly would pay eighteen calls for what four find. The notice is `agent.refused` and is deliberately not `standing` — the account is fine and the next batch is about different rows. + +**What was given up on is listed, and said once.** A refused row is charged with `why="refused"`, kept in `raw.why.`, so the list can tell a refusal from a row the model left out. The strip interrupts only for given-up rows without `raw.seen.`; its × or opening Jobs & history sets that mark on the server, so a reload does not say them again and a row given up on later still does. Try again there reopens a row with one attempt (`reopen_given_up(..., attempts=1)`) and queues its `_batch` job at once, so a row refused again is back on the list after one call rather than after three pulls. + +## `notices` + +**Two halves since the extraction.** `pipeline/notices.py` is what the queue itself needs — the notice a failed job gets, the check that stops it being said twice, the rendering a terminal wants. `desk/notices.py` is newsdesk's vocabulary: `Rule`, `RULES`, `classify`, `stops_the_run`, `source_failed`. An app writes its own, because a useful sentence names things only the app knows. + +`pipeline/notices.py` turns a failure into one line worth putting on screen. A handler calls `context.notice(...)`, and the queue writes it to the `notices` table and pushes it to whoever is watching — the counterpart to `log.exception`, not a replacement for it: log the traceback for afterwards, raise a notice for now. + +**A notice is stored, one row per `(code, key)`.** The first design stored nothing, on the argument that a table would duplicate the log. That assumed somebody was watching when the notice was raised; a scheduled pull and a reader on a phone mean nobody usually is. `Jobs.record_notice` upserts the row and returns it as a `NoticeEntry` with `count`, `first_at` and `last_at`, and that row is what goes over the socket. `count == 1` is news and anything above is a repeat, so a client interrupts for the first and only counts the rest — a refused article is announced on the pull that met it, not on the two retries after. Every notice still reaches `on_notice`; what interrupts a person is the interface's decision. + +A row goes when it is cleared (`DELETE /api/notices/{id}`, or `/api/notices` for all), after which the same failure is news again, or when `JobQueue.NOTICE_QUIET` (14 days) has passed since it was *last* said. `GET /api/notices` lists the log, newest first. A write that fails is logged and the notice is pushed anyway: it never costs the job that raised it. The table holds no traceback and no per-occurrence history; those stay in the log. + +A notice carries three things beyond its text: `kind`, which says whether this is the world misbehaving or a bug of ours — the fork in what you do next, and exactly the guess you should not have to make from an error string; `fix`, the command when there is one; and `key`, which with `code` names the row in the log — so a feed that has been failing all week is one line with a count rather than one per pull, and two refused articles are two. `classify` is the interface, and `RULES` is the catalogue it reads, which doubles as the list of what this will ever interrupt you about. Everything in the catalogue is marked `expected`; the fallback is marked `bug`, which is the whole point of enumerating them — a failure nobody wrote a rule for is one to go and read about. + +For a job author, two rules. Use a notice for what a person has to act on, not for what the program can get on with. And classify your own failure if you can: the worker announces "looks like a bug" for every job that ends `failed`, and `already_said` is what stops it contradicting a handler that had already explained itself. + +A rule also says whether the failure it names is `standing` — caused by the account or the machine rather than by this batch, so every sibling of the run would be told the same thing. `stops_the_run(notice)` is that answer, and a task for the model cancels what is still queued rather than paying for twenty-two more refusals. A limit, an expired login and an unreachable model are standing; a timeout, an oversized answer and a reply that was not JSON are not. + +## Timing + +Seven numbers, all on `JobQueue`, plus one on `Jobs`. None of them is tunable and none is meant to be; they are here so that "why did that take a minute" has an answer. + +| | | | +|---|---|---| +| `IDLE_POLL` | 1 s | How long to sleep before looking again when the table was empty. Only reached when nothing woke us — an enqueue in this process is instant, so this is what notices one made by another, and what notices a scheduled attempt whose time has come. | +| `PUSH_EVERY` | 0.25 s | Fastest a running job's progress may reach the browser. The write is not throttled; the push is. | +| `CANCEL_CACHE` | 0.25 s | How long `check()` caches its answer. It reads the table so Kill works across processes, and a handler calling it in a loop would otherwise turn one button press into a query storm. | +| `HEARTBEAT_EVERY` | 5 s | How often a worker restamps `seen_at` on the jobs it holds. | +| `STALE_AFTER` | 30 s | How long a `running` row may go unsaid-about before another worker reads it as abandoned. | +| `STOP_GRACE` | 5 s | How long a shutdown waits for handlers already in a thread. | +| `SCHEDULE_TICK` | 30 s | How often the loop looks for a schedule whose time has come, and the resolution the cadences promise — which is why `pipeline/schedule.py` has no seconds unit. | +| `Jobs.LINGER` | 15 s | How long a finished cascade stays in `cascades()`, so a run you watched is still on screen the moment after it lands. | + +`HEARTBEAT_EVERY` and `STALE_AFTER` are far apart on purpose: the gap is how many beats may be missed to a stalled loop or a slow disk before live work is re-queued underneath a worker that is still there, which is the one mistake this must not make. And because reclamation runs at most once per `STALE_AFTER` on the idle path, a crashed worker's job comes back after up to about a minute rather than after thirty seconds. + +## Limits + +- **Cancellation in a thread is cooperative.** No `context.check()` in the loop, no Kill. A running handler in a thread is never interrupted, not even on `stop()`, because Python cannot kill a thread from outside. Work that might need killing runs in a process: `context.spawn` for a command, `isolate=True` for Python. See "Ending work that will not end itself". +- **A shutdown waits `STOP_GRACE` for its handlers, then walks away.** A thread cannot be cut off, only waited for or abandoned, and both are wrong as an absolute: waiting is a reload held up by a model call, abandoning is a handler still writing to a file the process considers finished with. Most handlers use the moment and end properly; the rest are left running, and land their own ending in the table whenever they reach it. +- **A shutdown hands its leases back, a crash cannot.** `stop()` expires the lease on whatever was still going when the grace ran out, because a process on its way out knows outright what silence can only imply. So a restart re-queues that work at once instead of a window later, and only a real crash pays the wait. Expired rather than re-queued, so a handler thread outliving the shutdown still writes its own ending rather than being overwritten by a second run of itself. +- **A job's ending is written by the thread that ran it.** The handler call and the terminal `done` / `failed` / `cancelled` are one function on one thread; the loop's part is claiming the job and pushing the new state out afterwards. Written back on the loop, the ending would be the first thing a `stop()` costs: cancelling an abandoned job's coroutine cancels the write, while the handler itself carries on to the end regardless — so a reload landing mid-run would leave finished work looking unstarted, and the next start would run it a second time. For a batch that asks a model that is a second bill for something already paid for. The task a job named with `context.task` is queued by that same thread, for the same reason and still only once the job has landed `done`. Which is the one thing to know when reading the table from outside: a job's row lands a moment before the jobs that follow from it exist, so an empty `running` and `queued` is not by itself the end of a run. +- **A kill outlives the crash that interrupted it.** A job asked to stop, whose worker then died, is reclaimed to `cancelled` rather than back to `queued`. The request is in the table exactly so that it survives the process that was meant to act on it. +- **A dead worker is noticed late, on purpose.** A `running` row carries `seen_at`, stamped by `claim` and renewed every `HEARTBEAT_EVERY`. SQLite reads and writes run off the event loop so they do not delay its heartbeat. A lease expires after `STALE_AFTER`; reclamation runs at startup and at most once per `STALE_AFTER` on the idle path, so recovery can take about a minute. A worker excludes jobs it still holds from reclamation, renews their expired leases and warns that its loop stalled. Another process can still reclaim an expired lease. +- **Every handler must be safe to run twice.** Not advice — the rule the rest of this depends on. A reclaimed lease, a shutdown that abandoned a thread, a second process on the same file, an attempt that failed half way through: each of them can put a handler on the same rows again, and the queue does not promise otherwise. So write for it. Read what is still waiting rather than trusting the uids you were handed, upsert rather than insert, and write each unit as it lands so a re-run has less to redo. `max_attempts` bounds how often that happens on a failure; it is not something to lean on, because the paths that re-run a handler without asking it are the ones that were always there. +- **Handlers must be sync.** An async handler is not awaited. +- **A crashed worker holds its slots until it is reclaimed.** `claim` counts `running` rows, so the survivor cannot use a crashed worker's slots until the lease expires and its next reclamation pass runs, about a minute in the worst case with a responsive loop and database. A stalled worker never re-queues its own held jobs. Socket sends run concurrently with a one-second timeout per subscriber, so an unresponsive reader cannot delay drain wake-ups indefinitely. +- **Handlers run in a pool of the queue's own**, sized to the worker counts, because the interpreter's shared pool is sized from the cpu count instead. On a machine smaller than `workers` the shared one would run fewer jobs than the table admits and, worse, queue the loop's own claim and heartbeat behind the handlers, which is a worker starving itself of the thread it needs to hold its leases. +- **Progress is a write per call.** `context.step` is one `UPDATE` in its own transaction, on its own connection. The push to the browser is throttled; the write is not, because the row is the record and a second process reads it. Report per batch rather than per row in anything with thousands of items. +- **One bad turn does not stop the loop.** A locked database is logged and retried on the next pass rather than ending the drain task, because a worker that died silently looks exactly like a healthy one with nothing to do. +- **`depends_on` names jobs, not tasks.** A job waits on ids it was given; a task id is for finding and cancelling, not for waiting. +- **No priorities.** Jobs are claimed oldest first within a resource; there is no way to jump the queue but to be queued earlier. +- **Five cadences, not cron.** `every 15m`, `every 6h`, `every 2d`, `daily at 06:00`, `weekly mon 07:00`, and nothing else — no seconds, no day-of-month, no "every weekday", no ranges. Anything the five do not cover is a job that decides for itself whether today is its day, or cron running `pipeline.worker --until-idle` as before. +- **A schedule is not an alarm.** It is checked every `SCHEDULE_TICK`, so "06:00" means within half a minute of it, and a machine asleep at 06:00 runs the job when it wakes rather than at 06:00. What it will not do is run it once per slot missed. +- **History is purged.** Finished jobs older than `JobQueue(retain=...)`, 30 days by default, are deleted hourly while idle. Kept regardless: unfinished jobs, the newest ending of each kind in each state (what `last(kind)` reads), and runs still pointed at. `retain=0` keeps everything. +- **A result is at most 64 KB of JSON, and a bigger one fails the job.** Past `MAX_RESULT` the job ends `failed` with `ResultTooLarge`, the size and the keys it returned, a notice goes out, and nothing of the result is written. It is not retried, since the same handler returns the same result. Every snapshot and tally reads results back, so data belongs in the app's store; raise `MAX_RESULT` if a project really needs bigger results. +- **A task for the model gives up on a row eventually.** See `ModelTask` above: three unanswered turns and a row stops being offered until a person asks for it deliberately (`reset_tagging` with no slug, `reset_summaries`). `ModelTask.missed` is the seam and `status.py` says how many. +- **Migrations are `_reconcile`.** Columns are added, and dropped, by reading the catalogue on open, because `CREATE TABLE IF NOT EXISTS` leaves an existing table alone and `PRAGMA user_version` on that file belongs to the app. +- **SQLite means one machine.** Not a network queue. diff --git a/pipeline/queue.py b/pipeline/queue.py new file mode 100644 index 0000000..a76e429 --- /dev/null +++ b/pipeline/queue.py @@ -0,0 +1,1135 @@ +"""The worker: one loop, draining the jobs table. + +Deliberately not an in-memory queue. The table *is* the queue, which buys three +things that matter more here than the microseconds it costs: + + it survives a restart a pull interrupted by a reload comes back queued + rather than vanishing half-done. + it is the history the list the UI shows is a `SELECT`, not a + separate ring buffer kept in step by hand. + the worker can move out nothing here is reachable only from the web + process. `Jobs.claim` is a single atomic statement, + so a second worker — one started from a terminal, + or by Claude Code — races correctly against this + one instead of double-running a job. + +Handlers are synchronous (feedparser, trafilatura, sklearn, and soon whisper +all block), so each runs in a thread and the loop stays free to push WebSocket +updates while it works. That is also why cancellation is cooperative: see +`RunContext.check`. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import logging +import sys +import tempfile +import time +from collections.abc import Awaitable, Callable, Coroutine, Sequence +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime, tzinfo +from pathlib import Path +from typing import Any, ClassVar +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +from pipeline.child import CANCELLED +from pipeline.registry import JOBS, Spec +from pipeline.context import Cancelled, Retry, RunContext, spawn +from pipeline.models import Job, JobsSnapshot, Notice, Schedule +from pipeline.notices import already_said, job_failed, render +from pipeline.schedule import SCHEDULES, Cadence +from pipeline.table import Jobs, ResultTooLarge, RunCancelled +from pipeline.task import fan_out as fan_out_jobs + +log = logging.getLogger(__name__) + + +class ChildFailed(Exception): + """What an `isolate=True` job raises when its process came back badly. + + An ordinary exception on purpose, so the worker's own `except Exception` + ends the job exactly as it ends a handler that raised in a thread: the + retry bound applies, the notice is the same, and nothing above has to know + which side of a process boundary the failure happened on. + """ + + +Listener = Callable[[JobsSnapshot], Awaitable[None]] +#: Where a notice goes. Separate from `Listener` because the two have +#: opposite lifetimes: a snapshot is state, replaced by the next one, and a +#: notice is an event that is gone once delivered. Folding an event into a +#: state payload is how you get a banner nobody can dismiss. +#: +#: A coroutine rather than any awaitable, because it is handed to +#: `run_coroutine_threadsafe` from the worker thread and that is what it takes. +NoticeListener = Callable[[Notice], Coroutine[Any, Any, None]] + +#: Where the schedule list goes when it changes. State, like `Listener`, and +#: separate from it because the two change at different moments: the queue +#: pushes jobs on every claim and every ending, and schedules only when one +#: fires, is skipped, or is edited. Folding them together would mean sending +#: the schedules on every progress frame of a long job. +#: +#: A coroutine, for the same reason `NoticeListener` is one: it is handed to +#: `run_coroutine_threadsafe` from a worker thread or a request thread. +ScheduleListener = Callable[[list[Schedule]], Coroutine[Any, Any, None]] + + +def local_zone(name: str = "") -> tzinfo: + """The zone a queue's clock cadences are read in. + + A name when there is one, and otherwise the machine's own — which is what + a script, a test or a notebook gets for free, and is right often enough + that asking for it would be ceremony. A deployed app names it, because a + server runs on UTC and nobody reads their day in it. + + A name the system has no zone for is a warning and the machine's zone, not + a refusal: a mistyped timezone is not a reason for the queue not to start, + and `daily at 06:00` an hour out is a great deal better than no worker. + """ + if not name: + return datetime.now().astimezone().tzinfo or UTC + try: + return ZoneInfo(name) + except (ZoneInfoNotFoundError, ValueError): + log.warning("no timezone called %r; using this machine's own for " + "daily and weekly schedules", name) + return datetime.now().astimezone().tzinfo or UTC + + + +class JobQueue: + """Workers per resource. Start it once, enqueue from anywhere, cancel by id. + + `jobs` is the table — see `table.py` — and `config` and `store` are + whatever the app's handlers take as their two positional arguments. The + queue never reads them; it passes them to a handler it runs, and to a + `Task` it starts, and has no opinion about either. + """ + + #: How long to sleep before looking for work again when the table was + #: empty. Only reached when nothing woke us — an enqueue in this process + #: is instant; this is what notices one made by another process. + IDLE_POLL = 1.0 + + #: Fastest a running job's progress is allowed to reach the browser. A + #: handler that reports per item would otherwise push hundreds of frames a + #: second, which no eye and no socket wants. + PUSH_EVERY = 0.25 + + #: How many jobs of each resource may run at once, when a caller does not + #: say. The app's own `workers` section is where a person changes it; this is + #: what a test or a notebook gets for free. A resource named in neither + #: gets one, which is the safe way to be wrong. + #: + #: The count belongs to the worker, not the job kind: two kinds that both + #: ask a model share the same four instead of each helping itself to four. + WORKERS: ClassVar[dict[str, int]] = {"llm": 4, "http": 8, "cpu": 1} + + #: How often a worker says the jobs it holds are still being run, and how + #: long a job may go unsaid-about before another worker decides nobody is + #: running it and puts it back in the queue. Beaten from the event loop + #: rather than from the handler, so a job that blocks in a thread for an + #: hour still holds its lease — being slow is not being dead. + #: + #: Far apart on purpose: the gap is how many beats may be missed to a + #: stalled loop or a slow disk before live work is re-queued underneath a + #: worker that is still there, which is the one mistake this must not make. + HEARTBEAT_EVERY = 5.0 + STALE_AFTER = 30.0 + + #: How long a stop waits for handlers already in a thread to come back + #: before walking away from them. A thread cannot be cut off, only waited + #: for or abandoned, and both are wrong as an absolute: waiting means a + #: reload held up by a model call, abandoning means a handler still + #: writing to a file the process thinks it has finished with. So: long + #: enough for a fetch to land, short enough not to be noticed, and + #: whatever is still going when it expires is left to its own thread, + #: which still writes the ending. The lease is what covers the case + #: where the process goes before that thread does. + STOP_GRACE = 5.0 + + #: How long a cancellation check is cached. `check()` reads the table so + #: that Kill works across processes; without this a handler calling it in a + #: loop would turn one button press into a query storm. + CANCEL_CACHE = 0.25 + + #: How often the loop asks the table to forget old jobs. Hourly: history + #: does not need to be exact to the minute, and the delete is one + #: statement over an indexed column. + PURGE_EVERY = 3600.0 + + #: How often the loop looks for a schedule whose time has come, when a + #: caller does not say. Half a minute, which is the resolution the + #: cadences promise: the finest of them is `every 1m`, and a schedule is a + #: standing instruction rather than an alarm. It is also why there is no + #: seconds unit in `pipeline/schedule.py` — a cadence this cannot keep is + #: one it should refuse rather than round. + SCHEDULE_TICK = 30.0 + + #: How long a notice nobody cleared stays in the log once it has stopped + #: being said. A fortnight: long enough that a failure you meant to look + #: at on the weekend is still there, short enough that the log is about + #: now. Counted from the last time it was said — see `Jobs.purge_notices`. + NOTICE_QUIET = 14 * 86400.0 + + def __init__(self, jobs: Jobs, config: Any, store: Any, *, + listener: Listener | None = None, + on_notice: NoticeListener | None = None, + on_schedules: ScheduleListener | None = None, + workers: dict[str, int] | None = None, + bootstrap: str = "", bootstrap_arg: str = "", + retain: float = 30 * 86400, + tz: str = "", schedules: bool = True, + schedule_tick: float | None = None) -> None: + self.jobs = jobs + self.config = config + self.store = store + #: `module:function` returning this app's `(config, store)`, for the + #: child process an `isolate=True` job runs in — see `_isolated`. Only + #: those kinds need it, so an app with none may leave it empty and a + #: queue that meets one without it says so rather than guessing. + self.bootstrap = bootstrap + #: Handed to the bootstrap in the child — for an app, the path of the + #: config file it was started with, which the child has no other way + #: to learn. Empty calls the bootstrap with no argument. + self.bootstrap_arg = bootstrap_arg + #: Seconds of finished history to keep, or 0 to keep it all. See + #: `Jobs.purge` for what survives regardless. + self.retain = retain + #: What `daily at 06:00` is 06:00 in. An IANA name, or empty for the + #: machine's own — see `local_zone`. Only the clock cadences read it; + #: `every 6h` is six elapsed hours wherever the machine thinks it is. + self.zone = local_zone(tz) + #: Whether this worker fires schedules at all. Off for a second worker + #: on the same file: they would both fire every due row, and two of + #: them would fire it twice — the pile-up guard reads `last_job_id`, + #: which neither has written yet at the moment the other looks. One + #: scheduler per file, as many workers as you like. + self.schedules = schedules + self.schedule_tick = (self.SCHEDULE_TICK if schedule_tick is None + else schedule_tick) + self._listener = listener + self._on_notice = on_notice + self._on_schedules = on_schedules + self.workers = dict(workers if workers is not None else self.WORKERS) + #: The jobs this process has in hand, by the task running each. A map + #: rather than a set because the ids are what the heartbeat writes. + self._running: dict[asyncio.Task[None], int] = {} + self._loop: asyncio.AbstractEventLoop | None = None + self._task: asyncio.Task[None] | None = None + self._pulse: asyncio.Task[None] | None = None + #: Threads for handlers, and for nothing else. See `start`. + self._pool: ThreadPoolExecutor | None = None + self._wake = asyncio.Event() + self._stopping = False + self._reclaimed_at = 0.0 + self._purged_at = 0.0 + self._scheduled_at = 0.0 + #: Schedules this process has already complained about, so a row with + #: a cadence nothing can parse or a kind nothing registers is said + #: once rather than twice a minute for as long as the app is up. The + #: same restraint `notices.already_said` shows, at the size it needs + #: to be here. + self._said_about: set[tuple[int, str]] = set() + + # ----------------------------------------------------------- lifecycle + + async def run_until_idle(self, timeout: float | None = None) -> bool: + """Start, work until nothing is queued or running, and stop. + + Returns whether it got there, as against running out of `timeout`. The + batch shape — a script, a cron entry, `python -m pipeline.worker + --until-idle`, a test — where the web app's shape is `start` and never + stopping. + + "Nothing queued" includes jobs waiting for a later attempt, which is + what a caller draining a queue means: a retry due in a minute is still + work this run was asked to finish. Pass `timeout` when that could be + a long wait. + """ + await self.start() + deadline = None if timeout is None else time.monotonic() + timeout + try: + while True: + snap = await asyncio.to_thread(self.jobs.snapshot) + if not snap.running and not snap.queued and not self._running: + return True + if deadline is not None and time.monotonic() > deadline: + return False + await asyncio.sleep(0.05) + finally: + await self.stop() + + def drain(self, timeout: float | None = None) -> bool: + """`run_until_idle` from code that has no event loop running.""" + return asyncio.run(self.run_until_idle(timeout)) + + async def start(self) -> None: + self._loop = asyncio.get_running_loop() + self._stopping = False # a stopped queue can be started again + # Made here rather than in `__init__` because an `asyncio.Event` binds + # to the first loop that uses it and refuses every later one. A queue + # started, stopped and started again — a test, a reload — would + # otherwise be waiting on a loop that has been closed since. + self._wake = asyncio.Event() + # Handlers get threads of their own rather than the interpreter's + # shared pool. That pool is sized from the cpu count, so on a small + # machine it is smaller than `workers` and two things go wrong at + # once: fewer jobs run than the table admits, and the loop's own + # claim and heartbeat queue up behind the handlers. A worker starved + # of a thread stops renewing its leases and has its live jobs + # re-queued underneath it, which is the one mistake the lease exists + # to prevent. So: as many threads as there are workers, plus a few + # for resources the app never sized, which the table runs one at a + # time each. + self._pool = ThreadPoolExecutor( + max_workers=sum(self.workers.values()) + 4, + thread_name_prefix="desk-handler") + await asyncio.to_thread(self._reclaim) + if self.schedules: + # Both here rather than left to the first turn of the loop, and in + # this order. Reconciling first means a default declared today has + # a row before anything looks for due ones, so a new schedule fires + # on the start that introduced it rather than the one after. And + # firing here means an app that was down over 06:00 runs the 06:00 + # job when it comes back, which is the reading of "overdue" that a + # person expects — once, not once per slot missed. + await asyncio.to_thread(self._reconcile_schedules) + await asyncio.to_thread(self._fire_due) + + self._task = asyncio.create_task(self._drain(), name="desk-worker") + self._pulse = asyncio.create_task(self._heartbeat(), name="desk-heartbeat") + + async def stop(self) -> None: + """Stop taking new work and let what is running finish. + + An in-flight handler is a thread and is not interrupted — killing it + mid-write is exactly what the jobs table exists to avoid. So they are + given `STOP_GRACE` to come back on their own, which most of them use, + and whatever is still going after that is walked away from. + + Walked away from, and not taken from: the ending of a job is written + by the thread running it, so a handler that misses the window still + lands its own `done` or `failed` in the row whenever it gets there. + What is handed back here is the lease, because a shutdown is the one + moment a process can say outright that it is no longer watching these. + That saves the next start from waiting out the silence for the ones + whose thread this process did not outlive — and only those, since a + row its thread did finish is no longer `running` for a reclaim to + take. + """ + self._stopping = True + self._wake.set() + for task in (self._task, self._pulse): + if task is not None: + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + # A moment to land in, and no more. Cancelling the task does not + # reach into the thread, so the ones that miss this window keep going + # regardless; what the wait buys is that the common case — a fetch, a + # write — finishes properly instead of being torn off mid-sentence. + if self._running: + await asyncio.wait(list(self._running), timeout=self.STOP_GRACE) + held = list(self._running.values()) + for task in list(self._running): + task.cancel() + self._running.clear() + # Said out loud rather than left to the lease. Only what this process + # holds, so a shutdown here never speaks for a job another worker is + # part way through. + await asyncio.to_thread(self.jobs.release, held) + # Forget the loop as well as the task. It is about to be closed, and a + # later `wake()` or `notify()` holding a reference to a closed loop + # raises rather than being the no-op it should be. + if self._pool is not None: + # Not waited for, same as the tasks: a handler in a thread is not + # coming back on request, and the interpreter joins it at exit. + self._pool.shutdown(wait=False) + self._pool = None + self._loop = None + self._task = None + self._pulse = None + + # -------------------------------------------------------------- public + + def enqueue(self, kind: str, payload: dict[str, Any] | None = None, *, + label: str = "", depends_on: Sequence[int] = (), after: str = "ok", + cascade_id: int = 0, task_id: int = 0, delay: float = 0.0, + max_attempts: int | None = None, retry_delay: float | None = None, + retry_backoff: float | None = None, + dedup_key: str | None = None) -> Job | None: + """Queue work by name. Returns the job — or the live one it deduped to. + + `depends_on` holds it back until those jobs are over; `after` says + whether it needs them to have succeeded. Dedup is skipped for a job + that waits on something, because the point of one is that there will + be several — eight of a kind, each waiting on its own batch. + + `delay` queues it for later without anything staying alive in the + meantime: the row carries the time and `claim` passes over it until + then, so "in ten minutes" survives a restart four minutes in. + + `max_attempts` and the two waits override what the kind declared, for + a caller that knows more about this one piece of work than the handler + does — which is what a plugin's own retry policy is: the outlet, not + the job kind, is what knows how long to leave a publisher alone. They + go onto the row, so the attempt that reads them may be in another + process, after a restart, under a plugin file that has since changed. + + `label` overrides the kind's, for one piece of work that can say more + about itself than its kind can. A queued job has no `message` — that is + written by a running handler — so the label is the whole of what the + queue list has to show, and the kind's is by construction the same + string on every job of that kind. A caller that knows how the work was + cut says so here. Empty means the kind's, which is every caller but + one. + + `dedup_key` overrides the kind's own, which is how one *item's* + acquisition is made unrepeatable while the kind stays freely + repeatable: two presses of a reader's retry button collide on the key + and the second is handed the first one's job, across tabs, clients and + processes, because the collision happens in a unique index rather than + in whichever page asked. + + `None` when the run it would belong to has been killed. Logged and + dropped rather than raised: whatever asked for this — a cascade's step, + a task fanning out its batches — has no way of knowing and nothing to + do about it. The kill has already happened; this is it arriving. + """ + spec = JOBS.get(kind) + if spec is None: + raise KeyError( + f"no job registered as {kind!r} (have: {', '.join(sorted(JOBS)) or 'none'})" + ) + solo = spec.solo and not depends_on + try: + job = self.jobs.enqueue( + kind, label or spec.label, payload, + dedup_key=(kind if solo else "") if dedup_key is None + else dedup_key, + depends_on=depends_on, after=after, resource=spec.resource, + cascade_id=cascade_id, task_id=task_id, delay=delay, + max_attempts=spec.max_attempts if max_attempts is None + else max_attempts, + retry_delay=spec.retry_delay if retry_delay is None + else retry_delay, + retry_backoff=spec.retry_backoff if retry_backoff is None + else retry_backoff) + except RunCancelled as refused: + log.info("%s", refused) + return None + self.wake() + self.notify() + return job + + def new_run(self, kind: str, name: str = "", label: str = "") -> int: + """Mint an id for a cascade or a task. Half of what a `Task` needs of + a sink; `enqueue` and `cancelled` are the rest.""" + return self.jobs.new_run(kind, name, label) + + def cancelled(self, run_id: int) -> bool: + return self.jobs.cancelled(run_id) + + def start_task(self, task: Any, *, cascade_id: int = 0) -> int: + """Run a `Task` here and now, and return its run id. + + Here and now because a task is plain code: it asks the store what there + is to do and queues it. That is a query and a handful of inserts, which + is cheap enough to do in the request that pressed the button or in the + worker thread that has just finished the job before. + """ + return task.start(self.config, self.store, self, cascade_id=cascade_id) + + def cancel(self, job_id: int) -> Job | None: + """Ask a job to stop. Queued ones end here; running ones on their next + `context.check()`. Returns the job as it now stands, or None if there + isn't one.""" + job = self.jobs.request_cancel(job_id) + self.wake() + self.notify() + return job + + def cancel_run(self, run_id: int) -> int: + """Kill a task or a cascade — one operation for both. How many ended. + + Killing a task is "skip this one": its batches end, its final job still + runs, and what it reports says the task was killed rather than only how + many batches ended cancelled. What happens next is then the cascade's + to decide, and newsdesk's pull cascade decides to carry on to the next + stage. Killing the cascade takes the final job with it, so nothing + reports and nothing follows — the one decision that is not the + cascade's, because there is nothing left of it to ask. + `Jobs.cancel_run` is where all of that is one statement. + """ + ended = self.jobs.cancel_run(run_id) + self.wake() + self.notify() + return ended + + def stop_task(self, task_id: int, why: str = "") -> int: + """End a task's queued batches, sparing what is running and its final + job. How many ended. + + Not a kill and not written as one: the task's row stays as it is, so + nothing reports that a person stopped this. What a handler reaches + through `RunContext.stop`, for the failure that is the backend's + answer to the whole run — see `Jobs.stop_task`. + """ + ended = self.jobs.stop_task(task_id, error=why) + self.wake() + self.notify() + return ended + + def snapshot(self) -> JobsSnapshot: + return self.jobs.snapshot() + + def get(self, job_id: int) -> Job | None: + return self.jobs.get(job_id) + + def wake(self) -> None: + """Tell the loop to look for work now instead of at the next poll. + + A no-op when no worker is running in this process — enqueueing without + one is legitimate (a command queueing work for the app to pick up), and + the row is in the table either way. + """ + if self._loop is None or self._loop.is_closed(): + return + with contextlib.suppress(RuntimeError): + self._loop.call_soon_threadsafe(self._wake.set) + + def run_schedule(self, schedule_id: int) -> Job | None: + """Fire one schedule now, leaving when it is next due alone. + + What Run now presses. `next_run` is untouched on purpose: a person + asking for this run is not asking to move the nightly one, and a button + that quietly rescheduled what it touched would be a button nobody could + use twice. + + `last_run` and `last_job_id` are written, because they are the truth + about this schedule — it did just run, and that is the job it queued. + Which also means the next tick will not start a second one while this + is still going. + """ + row = self.jobs.schedule(schedule_id) + if row is None: + return None + job = self.enqueue(row.kind, row.payload, label=row.label) + self.jobs.update_schedule(row.id, last_run=datetime.now(UTC).isoformat(), + last_job_id=job.id if job else 0) + self.notify_schedules() + return job + + def notify_schedules(self) -> None: + """Push the schedule list to listeners. Safe from any thread. + + Called by whatever changed one — a firing here, a route over there — + rather than polled, so a row edited in one tab redraws in the others + without either of them asking every few seconds. + """ + if self._loop is None or self._on_schedules is None or self._loop.is_closed(): + return + with contextlib.suppress(RuntimeError): + asyncio.run_coroutine_threadsafe(self._push_schedules(), self._loop) + + def announce(self, notice: Notice) -> None: + """Write one notice down, then push it to whoever is watching. Safe + from any thread. + + Written first, because the push is fire-and-forget and the usual + listener is nobody: a pull runs on a schedule and the phone that would + have shown the line is in a pocket. It used to stop there — "the log + already has it" — which was true of a file on a server and no use to + the person the notice was for. The table is what a tab opened + afterwards reads, and where a line that faded can be found again. + + What is pushed is the row, count included, so a listener can tell news + from a repeat of something already in the log. That judgement is left + to it on purpose: every notice still goes out, and what interrupts a + person is the interface's business. See `NoticeEntry`. + + Called from the worker thread, so the push hops the loop the same way + `notify` does, and delivery is still never waited for. + """ + # Logged as well, always. The socket is the fast path, the table is + # what is on screen tomorrow, and the log is the only one of the three + # with the traceback beside it. + log.warning("%s\n%s", notice.code, render(notice)) + try: + notice = self.jobs.record_notice(notice) + except Exception: + # A notice is already the report of a failure. One that cannot be + # written is still worth pushing, and is never worth failing the + # job that raised it. + log.exception("could not record the %s notice; pushing it anyway", + notice.code) + if self._loop is None or self._on_notice is None or self._loop.is_closed(): + return + with contextlib.suppress(RuntimeError): + asyncio.run_coroutine_threadsafe(self._on_notice(notice), self._loop) + + def notify(self) -> None: + """Push the queue's state to listeners. Safe from any thread.""" + if self._loop is None or self._listener is None or self._loop.is_closed(): + return + with contextlib.suppress(RuntimeError): + asyncio.run_coroutine_threadsafe(self._announce(), self._loop) + + # --------------------------------------------------------------- guts + + async def _drain(self) -> None: + """Claim whatever there is a free worker for, then wait to be woken. + + Claiming until it comes back empty is what lets the table decide how + many run rather than this loop: it only asks, and stops when told. It never counts anything itself, which is + the only way a second process sharing the file gets the same answer. + + One bad turn does not end it. A locked database or a file that went + away for a moment would otherwise take this task down with an + exception nobody is waiting to receive, and the process would sit + there looking healthy while draining nothing at all. + """ + while not self._stopping: + try: + await self._turn() + except asyncio.CancelledError: + raise + except Exception: + log.exception("the drain loop hit an error; carrying on") + await asyncio.sleep(self.IDLE_POLL) + + async def _turn(self) -> None: + """One pass: take what there is room for, or wait to be woken. + + `IDLE_POLL` is also what notices a job whose next attempt has come + round. Nothing wakes the loop for one — the row said it was due at a + moment nobody was listening for — so the poll that exists to notice + another process's enqueue notices this too, and a retry starts within a + second of its time. Which is why a scheduled job needs no timer here: + there was already something looking. + """ + # Before the claim, not after it with the other housekeeping. Reclaim + # and purge are reached only when there was nothing to take, which is + # right for both — they are repairs, and a busy queue has more pressing + # work. A schedule is the opposite: it is work, and a queue that is + # never idle would be one where 06:00 never came round. The check + # itself is a clock comparison, so paying it on every pass costs + # nothing worth measuring. + if self.schedules and (time.monotonic() - self._scheduled_at + >= self.schedule_tick): + await asyncio.to_thread(self._fire_due) + job = await asyncio.to_thread(self.jobs.claim, self.workers) + if job is not None: + self._start(job) + return + # Nothing to take, which is the moment to ask whether some other + # worker's `running` row has gone quiet. Here rather than only at + # start, so a process that dies is cleaned up by whichever one is + # still alive instead of waiting for somebody to be restarted. + if time.monotonic() - self._reclaimed_at >= self.STALE_AFTER: + await asyncio.to_thread(self._reclaim) + if self.retain and time.monotonic() - self._purged_at >= self.PURGE_EVERY: + await asyncio.to_thread(self._purge) + self._wake.clear() + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(self._wake.wait(), self.IDLE_POLL) + + def _reclaim(self) -> None: + """Recover silent workers' jobs while renewing our own stalled leases.""" + self._reclaimed_at = time.monotonic() + stale = self.jobs.reclaim_stale( + self.STALE_AFTER, except_ids=list(self._running.values())) + if stale: + log.info("re-queued %d job(s) whose worker stopped reporting in", stale) + + def _purge(self) -> None: + """Forget finished jobs older than `retain`. Only when idle, like the + reclaim above it, so it never competes with work for the file.""" + self._purged_at = time.monotonic() + try: + gone = self.jobs.purge(self.retain) + # The notices with them, on their own clock: a job is history once + # it has finished, a notice once it has stopped being said. + self.jobs.purge_notices(self.NOTICE_QUIET) + except Exception: + log.exception("could not purge old jobs; carrying on") + return + if gone: + log.info("forgot %d finished job(s) older than %g days", + gone, self.retain / 86400) + + # ----------------------------------------------------------- schedules + + def _reconcile_schedules(self) -> None: + """Give every declared default a row, and leave every existing one be. + + The whole of what a declaration buys: a name nothing has a row for gets + one, and a name that does have one is not touched — not its cadence, + not its payload, not whether it is switched on. So the declarations in + an app's jobs module are where a schedule *comes from*, and the table + is what it *is*; turning one off in the UI survives the next deploy, + and changing one in the code reaches only the installs that never had + it. + + Which is also why nothing here deletes. A declaration that goes away + leaves its row behind, `unknown kind` against it if the handler went + too, for a person to delete deliberately — the alternative is a deploy + that silently forgets a schedule somebody was relying on. + """ + now = datetime.now(UTC) + for declared in SCHEDULES.values(): + row = self.jobs.add_schedule( + declared.name, declared.kind, declared.cadence, + payload=declared.payload, label=declared.label, + next_run=Cadence.parse(declared.cadence) + .next_after(now, self.zone).isoformat(), + only_if_new=True) + if row is not None: + log.info("schedule %r declared: %s %s, first at %s", + row.name, row.kind, row.cadence, row.next_run) + + def _fire_due(self) -> None: + """Queue a job for every schedule whose time has come. One pass. + + From the drain loop's own thread pool, not a timer and not a second + thread: there is one place that decides what runs, and adding a second + clock would mean two things racing to enqueue the same row. + + Three ways a due schedule does not fire: + + unknown kind the handler is gone — a plugin uninstalled, a + deploy that dropped a job. Said once and then left + exactly as it is, `next_run` included, so that + registering the handler again runs it rather than + waiting out another cadence. + `` marks the row `unknown kind` + and a person deletes it if it is not coming back. + bad cadence only reachable by editing the table by hand, since + every route parses before it writes. Same + treatment, and for the same reason: fixing the + wording is meant to be enough. + still going the last job this schedule queued is queued or + running, or the kind is `solo` and one is under + way. Here `next_run` *does* advance and nothing is + queued, which is the difference between a schedule + and a backlog: a job slower than its own cadence + must not grow a queue of itself. + """ + self._scheduled_at = time.monotonic() + now = datetime.now(UTC) + moved = False + for row in self.jobs.due_schedules(now.isoformat()): + spec = JOBS.get(row.kind) + try: + when = Cadence.parse(row.cadence).next_after(now, self.zone) + except ValueError as bad: + self._say_once(row, "cadence", "schedule %r will not run: %s", + row.name, bad) + continue + if spec is None: + self._say_once(row, "kind", + "schedule %r wants %r, which no handler is " + "registered for; it will not run until one is", + row.name, row.kind) + continue + moved = True + if self._still_going(row, spec): + log.info("schedule %r was due, but its last job is still " + "going; next at %s", row.name, when.isoformat()) + self.jobs.update_schedule(row.id, next_run=when.isoformat()) + continue + job = self.enqueue(row.kind, row.payload, label=row.label) + self.jobs.mark_fired(row.id, job.id if job else 0, when.isoformat()) + log.info("schedule %r fired %s as job %s; next at %s", row.name, + row.kind, job.id if job else "nothing", when.isoformat()) + if moved: + self.notify_schedules() + + def _still_going(self, row: Schedule, spec: Spec) -> bool: + """Whether firing this schedule would pile one job on another. + + Two questions, because a schedule can collide with itself two ways. + Its own last job may still be going, which is the slow-job case. Or the + kind may be `solo` and something else may have started one — a button, + another schedule, a cascade — in which case `enqueue` would hand back + that job and the schedule would record somebody else's work as its own + firing. + """ + if row.last_job_id: + last = self.jobs.get(row.last_job_id) + if last is not None and last.live: + return True + return bool(spec.solo and self.jobs.last(row.kind, states=("queued", "running"))) + + def _say_once(self, row: Schedule, about: str, message: str, *args: Any) -> None: + """Log a schedule's own defect the first time this process meets it. + + A tick every thirty seconds would otherwise make one broken row into + two thousand log lines a day, which is how a log stops being read. + Keyed on the row and what is wrong with it, so fixing the cadence and + then breaking the kind still says both. + """ + if (row.id, about) in self._said_about: + return + self._said_about.add((row.id, about)) + log.warning(message, *args) + + async def _push_schedules(self) -> None: + """Read the list off the loop, for the reason `_announce` does.""" + if self._on_schedules is None: + return + try: + await self._on_schedules(await asyncio.to_thread(self.jobs.schedules)) + except Exception: + log.exception("schedule listener raised; carrying on") + + async def _heartbeat(self) -> None: + """Say the jobs this process holds are still being run. + + From the loop, not from the handler: a handler is arbitrary blocking + code in a thread and may not come back for an hour, and a lease it can + only renew between units of work would make "slow" indistinguishable + from "dead". The loop is the thing that is still there either way, and + if it is not, then this process really has stopped and its jobs really + should go back. + """ + while not self._stopping: + await asyncio.sleep(self.HEARTBEAT_EVERY) + ids = list(self._running.values()) + if not ids: + continue + try: + await asyncio.to_thread(self.jobs.heartbeat, ids) + except Exception: + # A missed beat is survivable — the next one lands well inside + # the lease — and a pulse that dies on one is not, because + # every job this process holds would be re-queued under it. + log.exception("heartbeat failed; carrying on") + + def _start(self, job: Job) -> None: + """Run one job alongside whatever else is going. + + The finished callback wakes the loop rather than the loop watching the + set: a worker has just come free and something queued may now fit in it, + and that is the same signal an enqueue sends. + """ + task = asyncio.create_task(self._execute(job), name=f"job-{job.id}") + self._running[task] = job.id + + def done(finished: asyncio.Task[None]) -> None: + self._running.pop(finished, None) + with contextlib.suppress(asyncio.CancelledError): + finished.result() + self._wake.set() + + task.add_done_callback(done) + + async def _execute(self, job: Job) -> None: + spec = JOBS.get(job.kind) + await self._announce() + + if spec is None: + # Reachable: a job queued by a newer version, or by a plugin that + # is no longer installed. Failing it says so and clears the row, + # rather than leaving it claimed forever. + await asyncio.to_thread(self.jobs.finish, + job.id, "failed", error=f"no handler registered for {job.kind!r}") + await self._announce() + return + + tasks: list[Any] = [] + said: list[Notice] = [] + + fn, label = spec.fn, spec.label + args = (self.config, self.store) + if spec.isolate: + # The row is the whole input, so nothing is sent — the child is + # told where to look. Its signature matches a handler's on + # purpose: everything below this line, the ending, the cancel + # check on return, the retry, is the same code either way. + fn = self._isolated(job, spec) + + def run() -> None: + """The handler and this job's ending, in the same thread. + + Together because the thread is the half of this that survives a + shutdown. `stop()` cancels the coroutine of a job whose handler + missed the grace period, and a cancelled coroutine never reaches + the line after its `await` — so an ending written back there is + precisely the ending an abandoned job never gets. The handler runs + on to the end regardless, writes whatever it writes, and leaves a + `running` row for the next start to re-queue and run a second + time: work already done, done again, and for anything that asks a + model a second bill for it. + + Nothing in here needs the loop. Everything that does — the push + saying the row moved — stays outside. + """ + try: + if "context" in job.payload: + raise TypeError("context is reserved for the worker") + context = self._context(job, tasks, said) + extra = dict(job.payload) + if spec.wants_context: + extra["context"] = context + result = fn(*args, **extra) + result = result if isinstance(result, dict) else {} + if self.jobs.cancel_requested(job.id): + # Asked to stop while it ran, and it never looked — a handler + # with no `check()`, or one whose last unit was already under + # way. The return is its last chance to look, so it is taken + # here: the job ends cancelled, keeping what it made, and + # queues nothing. Ending `done` would start the next task + # under a cascade that has just been killed — a Kill that did + # nothing. (The guard in `Jobs.enqueue` would refuse the jobs + # anyway; this is the same answer one step earlier, and without + # a run minted for nothing.) + self.jobs.finish(job.id, "cancelled", result=result) + return + self.jobs.finish(job.id, "done", result=result) + for task in tasks: + # After the job landed, never before: a task that outlived the + # failure of the job that described it would be work nobody + # asked for. Here rather than back on the loop for the same + # two reasons the ending is — a finder query and a handful of + # inserts are no more the loop's business than the handler + # was, and a job abandoned by a shutdown still asks for what + # it said should follow. Its own failure is logged and no + # more: the job it came from is already over and did what it + # was asked. + try: + self.start_task(task, cascade_id=job.cascade_id) + except Exception: + log.exception("job %d asked for a task that would not start", + job.id) + except Cancelled: + self.jobs.finish(job.id, "cancelled") + except Retry as again: + # The handler naming its own wait — a rate limit's answer, a + # `Retry-After`. Logged rather than tracebacked: this is the + # world saying "not yet", which is a fact about the service and + # not a defect here. Whether there is another attempt to give + # it is still the row's to decide. + log.info("job %d (%s) asked to come back: %s", job.id, job.kind, again) + self._ended_badly(job, f"{type(again).__name__}: {again}", + label, said, delay=again.after) + except ResultTooLarge as exc: + # Loud and final. Not `_ended_badly`, because a retry would run + # the same handler to the same oversized result; and not + # quiet, because the alternative to failing is dropping data. + log.error("job %d (%s) %s", job.id, job.kind, exc) + error = f"{type(exc).__name__}: {exc}" + self.jobs.finish(job.id, "failed", error=error) + self.announce(job_failed(job.kind, label, error)) + except Exception as exc: + # Deliberately broad. A handler is arbitrary third-party-ish code + # (a source plugin, a model, a subprocess); anything it raises is + # this job's failure and nothing else's, and letting it escape + # would take the worker down and strand every job behind it. + log.exception("job %d (%s) failed", job.id, job.kind) + self._ended_badly(job, f"{type(exc).__name__}: {exc}", label, said) + + try: + await asyncio.get_running_loop().run_in_executor(self._pool, run) + finally: + await self._announce() + + def _ended_badly(self, job: Job, error: str, label: str, + said: list[Notice], *, delay: float | None = None) -> None: + """An attempt that did not land: another one later, or the end of it. + + Called from the handler's thread, like the ending it replaces, and for + the same reason — a shutdown that abandoned the thread must not lose + the row's move to whatever comes next. `Jobs.retry` decides which move + that is; here there is only the question of what to say about it. + + A failure with another attempt coming says nothing to the screen. A + notice is for what a person has to act on, and there is nothing to act + on yet: the queue is already doing the thing they would have asked for. + The log has it, as it has everything. Only the attempt that runs out + announces, and it announces the failure that ended it. + """ + ended = self.jobs.retry(job.id, error=error, delay=delay) + if ended is not None and ended.live: + log.info("job %d (%s) goes back in the queue: attempt %d of %d", + job.id, job.kind, job.attempt, job.max_attempts) + return + # And a notice, so it reaches the screen and not only the log — unless + # this failure has already put one there. A handler that classified its + # own failure said the useful thing about it, which login expired or + # which switch is off; adding "failed, looks like a bug" underneath + # contradicts it on the same screen, and sends you to a traceback about + # a machine that is working correctly. + # + # `job_failed` keeps the case it was written for: a handler that raised + # something nobody caught and said nothing about it. That really is + # ours to fix, and it is still the only thing here that says `bug` + # without working it out. + if ended is not None and ended.state == "cancelled": + return # a Kill that landed mid-attempt; nothing broke + if not already_said(said, error): + self.announce(job_failed(job.kind, label, error)) + + def _isolated(self, job: Job, spec: Spec) -> Callable[..., dict[str, Any]]: + """A stand-in handler that runs this job in a child process. + + Returned rather than called so the worker's thread calls it, in the + place the real handler would have been called: the ending, the retry + and the cancel-on-return are then one piece of code for both paths. + + `context` is accepted and dropped. The child builds its own — one that + writes to the same table, which is how its progress reaches the + browser without anything being forwarded. What cannot cross is the + pair the handlers take, hence `bootstrap`. + + Failure is read off the exit code and the result file, not off stderr. + A child that died of something Python never saw — an OOM kill, a + segfault in a native library — leaves no file, and gets the only + honest report available: its exit code and whatever it managed to say. + """ + def run_in_child(*_args: Any, context: RunContext | None = None, + **_payload: Any) -> dict[str, Any]: + # Checked in here rather than above, where it reads more naturally + # and would be raised while the worker was still assembling the + # run — outside the `try` that turns a handler's exception into + # this job's ending, so the row would stay `running` for the lease + # to clean up and the reason would be in the log only. + if not self.bootstrap: + raise RuntimeError( + f"{job.kind!r} is isolate=True, but this queue has no " + "bootstrap. A child process cannot be handed the app's " + "config and store, so it has to rebuild them: pass " + "bootstrap='module:function' to JobQueue, returning " + "(config, store)." + ) + with tempfile.TemporaryDirectory(prefix="pipeline-job-") as tmp: + out = Path(tmp) / "result.json" + argv = [sys.executable, "-m", "pipeline.child", + "--db", str(self.jobs.path), "--job", str(job.id), + "--bootstrap", self.bootstrap, "--out", str(out)] + if self.bootstrap_arg: + argv += ["--bootstrap-arg", self.bootstrap_arg] + done = spawn(argv, check=context.check if context else None, + timeout=spec.timeout or None) + said = json.loads(out.read_text(encoding="utf-8")) if out.exists() else {} + if done.returncode == CANCELLED: + raise Cancelled() + if done.returncode != 0 or "error" in said: + detail = said.get("error") or ( + done.stderr.decode("utf-8", "replace").strip().splitlines()[-1:] + or [f"exited {done.returncode}"])[0] + if said.get("traceback"): + log.error("job %d (%s) failed in its own process:\n%s", + job.id, job.kind, said["traceback"]) + raise ChildFailed(detail) + result = said.get("result") + return result if isinstance(result, dict) else {} + + return run_in_child + + def _context(self, job: Job, tasks: list[Any], + said: list[Notice]) -> RunContext: + """The `RunContext` for one run. The callbacks are called from the worker + thread, so they are written to be safe there. + + `said` collects what the handler announced while it ran, which is how + the failure above knows whether this ending has already been explained. + """ + pushed_at = 0.0 + checked_at = 0.0 + cancelled = False + + def on_progress(done: int, total: int, message: str) -> None: + nonlocal pushed_at + self.jobs.progress(job.id, done, total, message) + moment = time.monotonic() + if moment - pushed_at >= self.PUSH_EVERY: + pushed_at = moment + self.notify() + + def is_cancelled() -> bool: + # Reads the table rather than a flag on this object, which is what + # lets a job be killed from the UI while the worker runs somewhere + # else entirely. + nonlocal checked_at, cancelled + moment = time.monotonic() + if moment - checked_at >= self.CANCEL_CACHE: + checked_at = moment + cancelled = self.jobs.cancel_requested(job.id) + return cancelled + + def is_killed() -> bool: + # The task's row, not this job's request. A batch asked to stop is + # one batch dropped; a killed task is the whole step skipped, and + # nothing in the counts tells those apart. Uncached, unlike the + # check above: the job that asks is the one reporting at the end + # of a task, and it asks once. + return bool(job.task_id) and self.jobs.cancelled(job.task_id) + + def on_task(task: Any) -> bool: + # Collected rather than queued, and always taken: the worker starts + # what is in here once the job lands in `done`. See + # `RunContext.task` for why that is the promise being made. + tasks.append(task) + return True + + def on_stop(why: str) -> int: + # A job with no task behind it is the whole run, and there is + # nothing queued under it to spare. Its own failure is the answer. + if not job.task_id: + return 0 + ended = self.stop_task(job.task_id, why) + log.warning("job %d stopped the rest of task %d: %s (%d batch(es))", + job.id, job.task_id, why or "no reason given", ended) + return ended + + def on_notice(notice: Notice) -> None: + # Kept as well as pushed. One job is one thread, so appending here + # needs no lock, and the list dies with the run. + said.append(notice) + self.announce(notice) + + def on_fan_out(kind: str, payloads: list[dict[str, Any]], *, + then: str, settle: str, label: str, + carries: dict[str, Any], + max_attempts: int | None, + retry_delay: float | None, + retry_backoff: float | None) -> tuple[int, ...]: + """`RunContext.fan_out`, from the worker thread. + + Inside this job's cascade rather than its task: the children are a + step of their own, so killing them is not killing the step that + asked for them, and the callback fanning out again makes a chain + rather than a nest. A job that fans out with no cascade of its own + begins one, which is `fan_out`'s business and not this closure's. + """ + return fan_out_jobs( + self, kind, payloads, then=then, settle=settle, label=label, + carries=carries, cascade_id=job.cascade_id, + max_attempts=max_attempts, retry_delay=retry_delay, + retry_backoff=retry_backoff) + + return RunContext( + on_progress=on_progress, + is_cancelled=is_cancelled, + is_killed=is_killed, + on_notice=on_notice, + on_task=on_task, + on_stop=on_stop, + on_fan_out=on_fan_out, + waited_for=tuple(self.jobs.outcomes(job.depends_on)), + ) + + async def _announce(self) -> None: + """Read snapshots off the loop so slow SQLite cannot delay heartbeats.""" + if self._listener is None: + return + try: + await self._listener(await asyncio.to_thread(self.jobs.snapshot)) + except Exception: + log.exception("job listener raised; carrying on") diff --git a/pipeline/registry.py b/pipeline/registry.py new file mode 100644 index 0000000..a17c8af --- /dev/null +++ b/pipeline/registry.py @@ -0,0 +1,195 @@ +"""How a function becomes a job. + +The whole contract is one line: + + def handler(config, store, *, context=NULL, limit=None) -> dict + +`@job` records that function against a name and **returns it unchanged**. That +is the point of this module rather than an accident of it: a handler stays an +ordinary function you can import and call. + + run_pull(config, store) # sync, no queue, no event loop + run_pull(config, store, limit=5) # payload keys are just keyword args + +Declared keyword arguments, not `**payload`: a key the handler does not name +is reported against the job, and a handler that swallowed everything would +turn a misspelt payload key into work that quietly did the wrong thing. + +Which means a job is testable, reusable from a command, and reusable from +another job, without any of them going near the queue. The queue's entire +involvement is calling the same function with a real `context` and the stored +payload spread over it. + +What that `context` *is* lives next door in `context.py`. Neither module +imports the handlers: they live in the app (`newsdesk/jobs/` for newsdesk), +import `@job` from here, and fill `JOBS` when they are imported. +""" + +from __future__ import annotations + +import inspect +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + + +#: A handler takes (config, store), keyword-only `context`, and whatever payload it +#: declares. It returns a JSON-serialisable dict, which is stored as the job's +#: result and shown when the job is done. +Handler = Callable[..., dict[str, Any]] + + +@dataclass(frozen=True, slots=True) +class Spec: + kind: str + label: str + fn: Handler + #: Whether `fn` actually accepts `context`. Worked out once, here, so the + #: worker doesn't inspect a signature per run — and so that "any function + #: with the right shape is queueable" holds for a function that wants + #: neither progress nor cancellation and says so by not taking the + #: argument. Passing `context` to one of those would be a TypeError, which + #: would surface as the job mysteriously failing. + wants_context: bool = True + #: When true, only one job of this kind may be queued or running at a time + #: — enforced in the database, so it holds across every client at once. + #: For the sweeps: one `pull`, one `rank`, however many tabs are open. + #: + #: Off by default, because the two ways of being wrong are not equal. A + #: sweep that forgets to say `solo=True` runs twice and wastes a minute; + #: a batch kind that forgets to say `solo=False` has seven of its eight + #: batches silently handed back the first one, and the work simply never + #: happens. The default belongs on the side that fails loudly. + solo: bool = False + #: What running one spends, and so which kind of worker runs it: + #: `llm` for anything that asks a model, `http` for a fetch, `cpu` for + #: work that wants the machine to itself. Declared here rather than worked + #: out, because the handler is arbitrary code and only its author knows. + resource: str = "cpu" + #: How many attempts this kind of work is worth, counting the first, and + #: how long to wait between them. Copied onto every job queued of this kind + #: — see `pipeline/table.py`, where the row rather than the kind is + #: what a later attempt reads. + #: + #: One by default, which is no retry and is what every job did before this + #: existed. Retrying is off by default for the same reason `solo` is: the + #: two ways of being wrong are not equal. A fetch that should have been + #: retried and was not costs one item until the next sweep finds it again; + #: work retried three times that never should have been is three times the + #: writes, or three times the bill. + max_attempts: int = 1 + retry_delay: float = 0.0 + retry_backoff: float = 1.0 + #: Run this kind in a child process of its own rather than in a worker + #: thread. What it buys is the one thing a thread cannot give: the job can + #: be *ended*. A handler inside `sklearn.fit` or a tight Python loop + #: cannot be stopped, because no signal and no exception reaches a thread + #: in the middle of a C call — only a separate process can be killed. + #: + #: Nothing has to be pickled to make it work, which is why it is cheap + #: here and expensive elsewhere: a job row is already a complete + #: description of the work — a kind, a JSON payload, a database. The child + #: is handed those three and rebuilds the rest. See `pipeline/child.py`. + #: + #: The cost is that the queue needs a `bootstrap` able to build the app's + #: `(config, store)` from nothing, since neither survives a process + #: boundary. One function per app. + #: + #: Not for work that is expensive because of a *command* it runs — + #: ffmpeg, stockfish. That wants `RunContext.spawn`, which kills the same + #: way for none of the setup. + isolate: bool = False + #: Seconds this kind may run before it is ended, or 0 for no deadline. + #: Requires `isolate`, and `job` refuses the pair without it: a deadline + #: over a thread could only mark the row failed and walk away, leaving the + #: work still running and still holding the core it was killed for + #: holding — and leaving two things intending to write the same row. A + #: promise the pipeline cannot keep is worse than no promise. + timeout: float = 0.0 + #: Payload keys the handler cannot run without: its keyword arguments + #: that have no default. A kind with any is not a button, because a press + #: sends an empty payload and the job would fail on the TypeError. + needs: tuple[str, ...] = () + + +#: name -> spec. Explicit registration, like `routers/ROUTERS`: a new job is +#: a change to this project, and the cost of importing its module once in +#: the app's handlers module is one line. Empty until something imports +#: `newsdesk.jobs`, which every entry point does before it looks here. +JOBS: dict[str, Spec] = {} + + +def job(kind: str, label: str = "", *, solo: bool = False, + resource: str = "cpu", max_attempts: int = 1, + retry_delay: float = 0.0, + retry_backoff: float = 1.0, + isolate: bool = False, + timeout: float = 0.0) -> Callable[[Handler], Handler]: + """Register a plain function as queueable work. + + Returns the function untouched, so importing it and calling it directly + goes on working exactly as before it was decorated. + + `solo=True` for a job there should only ever be one of at a time — see + `Spec.solo`, which is where the default and its reasoning live. + + `max_attempts=3, retry_delay=60` for work that is worth another go when + the other end is having a bad minute. The retry is the queue's, not the + handler's: a failed attempt puts the row back in the queue with a time on + it, so nothing sleeps and nothing holds a worker while it waits. A + retrying job must be safe to run twice, which every handler here has to be + anyway — see the pipeline's usage card. + + `isolate=True, timeout=600` for work that has to be endable: it runs in a + child process, and ten minutes in it is killed rather than asked. See + `Spec.isolate` for what that costs and `RunContext.spawn` for the cheaper + answer when the expensive part is a command rather than Python. + """ + if timeout and not isolate: + raise ValueError( + f"{kind!r} asks for a {timeout:g}s timeout without isolate=True. " + "A job in a thread cannot be ended — the deadline could only mark " + "the row failed while the work carried on. Either isolate it, or " + "give the handler a RunContext.spawn(timeout=...) to hold." + ) + if timeout < 0: + raise ValueError(f"{kind!r} asks for a negative timeout: {timeout!r}") + + def register(fn: Handler) -> Handler: + if kind in JOBS and JOBS[kind].fn is not fn: + raise ValueError( + f"two different handlers are both registered as {kind!r}: " + f"{JOBS[kind].fn.__qualname__} and {fn.__qualname__}" + ) + JOBS[kind] = Spec( + kind=kind, label=label or kind, fn=fn, solo=solo, + wants_context=_accepts_context(fn), needs=_required(fn), + resource=resource, max_attempts=max_attempts, + retry_delay=retry_delay, retry_backoff=retry_backoff, + isolate=isolate, timeout=timeout, + ) + return fn + + return register + + +def _required(fn: Handler) -> tuple[str, ...]: + """`fn`'s keyword-only parameters without a default, in declared order.""" + try: + params = inspect.signature(fn).parameters + except (TypeError, ValueError): + return () + return tuple(name for name, p in params.items() + if p.kind is inspect.Parameter.KEYWORD_ONLY + and p.default is inspect.Parameter.empty) + + +def _accepts_context(fn: Handler) -> bool: + """Whether `fn` has a `context` parameter, or a `**kwargs` that would swallow one.""" + try: + params = inspect.signature(fn).parameters + except (TypeError, ValueError): + return True # unintrospectable; assume the contract + return "context" in params or any( + p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values() + ) diff --git a/pipeline/schedule.py b/pipeline/schedule.py new file mode 100644 index 0000000..7207142 --- /dev/null +++ b/pipeline/schedule.py @@ -0,0 +1,273 @@ +"""Cadences: the five wordings a schedule may be written in, and nothing else. + +A schedule is a row that says "enqueue this kind with this payload on this +cadence". This module is only the *cadence* half: the grammar, the canonical +spelling, and when the next run falls. The row is in `table.py`, the firing is +in `queue.py`, and the buttons are in `web/schedules.py`. + +Five wordings, deliberately, rather than cron: + + every 15m · every 6h · every 2d · daily at 06:00 · weekly mon 07:00 + +Cron is five fields of positional punctuation that nobody reads back +correctly, and the schedules a small app actually wants are these. A wording +outside them raises `ValueError` listing what is accepted, which is the +sentence the API hands a person who typed something else. Long weekday names +are taken as well as short ones, because `weekly monday 07:00` is what people +write; `__str__` gives back the short form, so a row stores one spelling +however it was typed. + +**The interval forms are intervals and the clock forms are clock times.** +`every 6h` is six hours later, measured in real elapsed time, so it neither +skips nor doubles across a daylight-saving change. `daily at 06:00` is 06:00 on +the wall clock of the queue's timezone, on both sides of one — which is the +whole reason `next_after` takes a zone rather than doing UTC arithmetic. The +two mean different things and a person picking one has already chosen which. + +Defaults are declared next to `@job`, in the app's own jobs module: + + from pipeline.schedule import declare + declare("nightly tick", "tick", "daily at 06:00", label="A tick, nightly") + +`declare` fills `SCHEDULES` the way `@job` fills `JOBS`, and `JobQueue.start()` +inserts a row for any declared name that has none. It parses the cadence there +and then, so a typo is an ImportError at startup rather than a schedule that +quietly never fires. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from datetime import UTC, datetime, time, timedelta, tzinfo +from typing import Any, Literal + +#: Short weekday names, Monday first, which is also `datetime.weekday()`'s +#: numbering. The short form is canonical: it is what `__str__` writes and so +#: what the table stores, however the row was typed. +DAYS: tuple[str, ...] = ("mon", "tue", "wed", "thu", "fri", "sat", "sun") + +#: The long names, accepted on input for the same index. Spelled out rather +#: than derived from `calendar.day_name`, which is locale-dependent — a cadence +#: that parses on one machine and not on another is the bug this avoids. +LONG_DAYS: tuple[str, ...] = ("monday", "tuesday", "wednesday", "thursday", + "friday", "saturday", "sunday") + +#: What each interval unit is worth. Seconds are deliberately absent: a queue +#: that looks for due schedules every thirty seconds cannot honour "every 10s", +#: and a cadence the scheduler cannot keep is worse than one it refuses. +UNITS: dict[str, int] = {"m": 60, "h": 3600, "d": 86400} + +#: What a person is told when the wording is not one of the five. One sentence, +#: because it goes out as `{"detail": ...}` and ends up in a toast. +FORMS = ("every 15m", "every 6h", "every 2d", "daily at 06:00", "weekly mon 07:00") + +_EVERY = re.compile(r"^every (\d+) ?([mhd])$") +_DAILY = re.compile(r"^daily at (\d{1,2}):(\d{2})$") +_WEEKLY = re.compile(r"^weekly ([a-z]+) (\d{1,2}):(\d{2})$") + + +class BadCadence(ValueError): + """A wording that is not one of the five. A `ValueError`, so a caller that + only knows it is parsing something still catches it.""" + + +def _refuse(text: str, because: str = "is not a cadence") -> BadCadence: + """The one refusal, with the forms always in it. + + Always, even when the wording was nearly right and this could have said + only what was wrong with it: the person reading this is typing into a box + and what they need is what to type, not a diagnosis. The specific + complaint goes in front of the list rather than instead of it. + """ + return BadCadence( + f"{text!r} {because}. Write one of: " + + ", ".join(repr(f) for f in FORMS) + + " — long weekday names are fine too." + ) + + +@dataclass(frozen=True, slots=True) +class Cadence: + """One of the five wordings, parsed. Immutable, and its own spelling. + + Built through `parse`; the fields are what the wording said and are left + public so a test can assert on them. `str(cadence)` is the canonical form + and is what goes in the table, so two spellings of the same thing — + `weekly monday 07:00` and `weekly mon 07:00` — are one row and compare + equal. + """ + + kind: Literal["every", "daily", "weekly"] + #: `every`: how many of `unit`. Unused by the clock forms. + n: int = 0 + unit: str = "" + #: `daily` and `weekly`: the wall-clock time, in the queue's zone. + hour: int = 0 + minute: int = 0 + #: `weekly`: 0 is Monday, matching `datetime.weekday()`. -1 otherwise. + day: int = -1 + + def __str__(self) -> str: + if self.kind == "every": + return f"every {self.n}{self.unit}" + if self.kind == "daily": + return f"daily at {self.hour:02d}:{self.minute:02d}" + return f"weekly {DAYS[self.day]} {self.hour:02d}:{self.minute:02d}" + + @property + def seconds(self) -> int: + """The interval, for the `every` forms. Zero for the clock ones, which + have no fixed length — a day is 23 or 25 hours twice a year.""" + return self.n * UNITS[self.unit] if self.kind == "every" else 0 + + @classmethod + def parse(cls, text: str) -> Cadence: + """One of the five wordings, or `BadCadence` naming all five. + + Case and inner spacing are forgiven — `Every 15 M` is `every 15m` — + because they are typing, not meaning. Everything else is refused: a + cadence that is nearly right is the one that has to fail loudly, since + the alternative is a row that looks scheduled and never fires. + """ + said = " ".join(str(text or "").lower().split()) + + found = _EVERY.match(said) + if found: + n = int(found.group(1)) + if n < 1: + raise _refuse(text) + return cls(kind="every", n=n, unit=found.group(2)) + + found = _DAILY.match(said) + if found: + hour, minute = int(found.group(1)), int(found.group(2)) + _check_clock(text, hour, minute) + return cls(kind="daily", hour=hour, minute=minute) + + found = _WEEKLY.match(said) + if found: + name = found.group(1) + day = (DAYS.index(name) if name in DAYS + else LONG_DAYS.index(name) if name in LONG_DAYS else -1) + if day < 0: + raise _refuse(text) + hour, minute = int(found.group(2)), int(found.group(3)) + _check_clock(text, hour, minute) + return cls(kind="weekly", hour=hour, minute=minute, day=day) + + raise _refuse(text) + + def next_after(self, moment: datetime, zone: tzinfo = UTC) -> datetime: + """When this next falls, strictly after `moment`. Always UTC-aware. + + Strictly after, so a schedule fired at exactly its own time does not + find itself due again on the same instant. + + `every` is elapsed time and ignores the zone: six hours is six hours, + so a run neither vanishes nor doubles when the clocks move. The two + clock forms are the opposite and want the zone for exactly that reason + — 06:00 is 06:00 on both sides of a change, which means the interval + across one is 23 or 25 hours, and that is the thing the wording + promised. + + Walking forward a day at a time rather than doing arithmetic on the + date: it is at most eight steps, it handles "later today" and "this + weekday next week" with one piece of code, and each candidate is built + from a wall-clock time and then converted, which is what keeps the + promise above. + """ + moment = moment if moment.tzinfo else moment.replace(tzinfo=UTC) + if self.kind == "every": + # Converted to UTC *before* the addition, not after. Adding a + # timedelta to an aware datetime is wall-clock arithmetic within + # its own zone — `06:30 + 24h` on the day the clocks go forward is + # 06:30 the next day, which is 23 real hours. That is exactly the + # promise the interval forms do not make, and the whole reason + # somebody writes `daily` when they want the other one. + return moment.astimezone(UTC) + timedelta(seconds=self.seconds) + + local = moment.astimezone(zone) + for step in range(9): # today, then a full week of weekdays + day = local.date() + timedelta(days=step) + if self.kind == "weekly" and day.weekday() != self.day: + continue + when = datetime.combine(day, time(self.hour, self.minute), tzinfo=zone) + if when > local: + return when.astimezone(UTC) + # Unreachable: a daily cadence lands within two steps and a weekly one + # within eight. Raised rather than returned-as-None so that if the + # calendar ever surprises us it says so here. + raise BadCadence(f"{self} never comes round after {moment.isoformat()}") + + +def _check_clock(text: str, hour: int, minute: int) -> None: + if not (0 <= hour <= 23 and 0 <= minute <= 59): + raise _refuse(text, "is not a time of day — hours run 00-23 and " + "minutes 00-59") + + +# ------------------------------------------------------------ declared defaults + +@dataclass(frozen=True, slots=True) +class Declared: + """A schedule an app ships with, before any row exists for it. + + `name` is the handle and is the whole of the reconciliation: a declared + name with no row gets one, a declared name that already has one is left + exactly as the user edited it. Which means the defaults here are a + *starting point*, not a setting — turning one off in the UI survives the + next restart, and that is the point of the row being the truth. + """ + + name: str + kind: str + cadence: str + payload: dict[str, Any] = field(default_factory=dict) + label: str = "" + + +#: name -> declaration. Filled by `declare` when the app's jobs module is +#: imported, exactly as `JOBS` is filled by `@job`, and read once per +#: `JobQueue.start()`. Empty in a project that declares none, which is fine: +#: every schedule can be added from the UI instead. +SCHEDULES: dict[str, Declared] = {} + + +def declare(name: str, kind: str, cadence: str, + payload: dict[str, Any] | None = None, + label: str = "") -> Declared: + """Ship a default schedule. Goes next to `@job`, in the app's jobs module. + + declare("nightly tick", "tick", "daily at 06:00") + declare("hourly count", "count", "every 1h", {"n": 5}) + + The cadence is parsed here, so a typo is an error the moment the module is + imported — at startup, with a traceback pointing at this line — rather than + a schedule that sits in the table looking fine and never fires. + + The kind is *not* checked, because it usually cannot be: `declare` and + `@job` are read top to bottom in one file and a declaration above its + handler would be refused for a job that exists a dozen lines later. An + unregistered kind is caught where it matters instead — `JobQueue` never + fires one and `` says `unknown kind` against the row. + + Declaring the same name twice with different terms is refused, like two + handlers on one `@job` name: the second would silently win, and which + module imported first is not a thing anybody should have to know. + """ + Cadence.parse(cadence) + declared = Declared(name=name, kind=kind, cadence=str(Cadence.parse(cadence)), + payload=dict(payload or {}), label=label) + if name in SCHEDULES and SCHEDULES[name] != declared: + raise ValueError( + f"two different schedules are both declared as {name!r}: " + f"{SCHEDULES[name].kind} {SCHEDULES[name].cadence!r} and " + f"{declared.kind} {declared.cadence!r}" + ) + SCHEDULES[name] = declared + return declared + + +__all__ = ["DAYS", "FORMS", "LONG_DAYS", "SCHEDULES", "UNITS", "BadCadence", + "Cadence", "Declared", "declare"] diff --git a/pipeline/table.py b/pipeline/table.py new file mode 100644 index 0000000..3ac5760 --- /dev/null +++ b/pipeline/table.py @@ -0,0 +1,1319 @@ +"""The jobs table: the queue's own storage, on whatever SQLite file it is given. + +The pipeline used to keep its rows in the application's store, which meant +the application's schema knew what a job was and the queue could not exist +without it. Now the table is the pipeline's: it creates it on open, owns its +shape, and asks nothing of the file beyond being SQLite. Point it at the +app's database and the jobs sit next to the app's rows, one file to back up; +point it at a file of its own and a worker runs with no app at all. + +Why a table rather than a list in memory is in `queue.py`. Why every method +here is one statement is the same reason: the file is shared by whoever +opens it, so a claim has to be atomic and a state change has to be a single +`UPDATE` that a second process cannot half-see. + +The shape is `CREATE TABLE IF NOT EXISTS`, which leaves a file that already +has the table exactly as it is — so a column added here would never reach one. +`_reconcile` is how a column arrives instead: read the catalogue, add what is +missing. Idempotent, so it runs on every open and there is no version to keep +in step. There is deliberately no counter of our own — `PRAGMA user_version` +on this file belongs to newsdesk (`newsdesk/storage/sql/schema.py`), and a +second migration series numbering itself on the same pragma would fight it. +""" + +from __future__ import annotations + +import json +import logging +import sqlite3 +from collections.abc import Mapping, Sequence +from contextlib import closing, contextmanager +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +from pipeline.models import ( + CascadeProgress, Job, JobsSnapshot, Notice, NoticeEntry, Run, Schedule, + TaskProgress, +) + +log = logging.getLogger(__name__) + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS jobs ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL, + label TEXT NOT NULL DEFAULT '', + payload TEXT NOT NULL DEFAULT '{}', + state TEXT NOT NULL DEFAULT 'queued', + done INTEGER NOT NULL DEFAULT 0, + total INTEGER NOT NULL DEFAULT 0, + message TEXT NOT NULL DEFAULT '', + result TEXT NOT NULL DEFAULT '{}', + error TEXT NOT NULL DEFAULT '', + dedup_key TEXT NOT NULL DEFAULT '', + -- Which jobs this one waits for, and what it wants of them. `after='ok'` + -- is one job needing another's work to have happened, and fails with it; + -- `after='done'` waits only for them to be over, however they ended, and + -- is what a job whose purpose is to report on them needs — success-only + -- would mean the report never exists when there is something to report. + -- An id that names nothing is satisfied: it finished before this was + -- asked, or was never queued. + depends_on TEXT NOT NULL DEFAULT '[]', + after TEXT NOT NULL DEFAULT 'ok', + -- What this job spends while it runs: `llm` for anything that asks a + -- model, `http` for a fetch, `cpu` for work that wants the machine to + -- itself. Named on the job rather than counted in the worker, because a + -- second process opening this file has to be counted too — four model + -- calls is a fact about the subscription, not about one program. + resource TEXT NOT NULL DEFAULT 'cpu', + -- Which cascade this job belongs to, and which of its tasks. Both name a + -- row in `runs`, and both are set before the job is written — a task + -- exists before any of its jobs do, so its id is minted rather than + -- scavenged from whichever job it happened to queue first. + -- + -- Two of them rather than one, because they are two different groupings + -- and the buttons want both: stopping a task is not stopping the cascade + -- it is part of. + cascade_id INTEGER NOT NULL DEFAULT 0, + task_id INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, + started_at TEXT, + -- The lease. A worker with a job in hand stamps this every few seconds + -- from its event loop, so a `running` row says not only that somebody + -- claimed it but that somebody is still there. Without it the only way to + -- read a `running` row on a shared file is to assume its worker died, and + -- a second process starting up would re-queue the first one's live work + -- and run it twice. Written by `claim` as well, so a job is leased from + -- the moment it is taken rather than from the first beat after it. + seen_at TEXT, + finished_at TEXT, + -- Cancellation is a request in the table, not a flag in the process that + -- happens to be running the job. A worker polls it, so Kill works whether + -- the worker is the web app's own thread or a second process that + -- opened the same file. + cancel_requested INTEGER NOT NULL DEFAULT 0, + -- The retry policy, and where it has got to. All five are on the row + -- rather than read off the job kind when a failure happens, because the + -- attempt that has to decide may be in another process, after a restart, + -- running a version of the code where the kind's numbers are different or + -- the kind is gone. A job carries its own bound for the same reason it + -- carries its own payload. + -- + -- `attempt` is how many times this has been *started*, stamped by `claim` + -- in the same statement that takes the row — so a worker dying under a job + -- costs it an attempt, which is the reading that stops a job that kills + -- workers from being fed them forever. + attempt INTEGER NOT NULL DEFAULT 0, + -- Counting the first attempt, so 1 is "run it once" and is the default: + -- every job that existed before this column behaves exactly as it did. + max_attempts INTEGER NOT NULL DEFAULT 1, + retry_delay REAL NOT NULL DEFAULT 0, + retry_backoff REAL NOT NULL DEFAULT 1, + -- Not before this. The whole of how a job waits without being run: it + -- stays `queued`, `claim` passes over it, and no worker, no thread and no + -- lease is spent on the wait. Durable because it is a column — a next + -- attempt an hour away outlives the process that scheduled it, which a + -- timer in a worker could not. + -- + -- `''` rather than NULL, and compared as text: `''` sorts before every + -- stamp, so "due" is one comparison with no coalesce and no special case. + run_after TEXT NOT NULL DEFAULT '' +); + +-- Where a cascade's id and a task's come from. A row is identity, and one +-- thing more: whether it was killed. No state and no progress — the jobs +-- carrying the id are the record of what happened and a second copy could +-- only go out of step. The kill is the one fact about a run its jobs cannot +-- tell you, and a cascade exists nowhere else, so it is stored here beside +-- the job's own `cancel_requested`. +-- +-- It exists so that a number can be handed out *before* there is a job to +-- take it from, and so that "the pull you started at 09:04" has somewhere to +-- be written down. +-- +-- `name` is what was asked for, and it is also how "is one already going?" +-- is answered — so the two ways a cascade row comes about have to be told +-- apart by it. One is a chain somebody named; the other is minted only +-- because a lone job or a lone task needed an id to carry. The second is +-- named by `on_its_own` and so never answers for the first. +CREATE TABLE IF NOT EXISTS runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + kind TEXT NOT NULL, + name TEXT NOT NULL DEFAULT '', + -- What to call it on screen: "Fetching what the feeds left out". A task is + -- not a job and has no handler to carry a label, so it is written here + -- with the name, by whoever minted the run. + label TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + cancelled_at TEXT NOT NULL DEFAULT '' +); + +-- What the queue told somebody, kept so it can be read again. One row per +-- `(code, key)`: the same failure said again is this row with a higher count +-- and a later `last_at`, which is how "already said" is a fact about the table +-- rather than about whichever browser tab happened to be open. See +-- `pipeline/notices.py` for why this exists at all. +-- +-- `place` is `Notice.where`, which is not a name SQL will take unquoted. +CREATE TABLE IF NOT EXISTS notices ( + id INTEGER PRIMARY KEY, + code TEXT NOT NULL, + key TEXT NOT NULL DEFAULT '', + level TEXT NOT NULL DEFAULT 'warning', + kind TEXT NOT NULL DEFAULT 'expected', + text TEXT NOT NULL, + fix TEXT NOT NULL DEFAULT '', + detail TEXT NOT NULL DEFAULT '', + place TEXT NOT NULL DEFAULT '', + first_at TEXT NOT NULL, + last_at TEXT NOT NULL, + count INTEGER NOT NULL DEFAULT 1, + UNIQUE (code, key) +); + +-- Standing instructions: enqueue this kind, with this payload, this often. +-- Here rather than in a scheduler process of its own, which is the whole +-- design: there is no second process to keep alive, a restart between two +-- runs loses nothing, and two workers on this file read one schedule instead +-- of each holding a timer for it. +-- +-- `name` is unique because it is the handle: a default declared in the app's +-- jobs module is matched to its row by name, and a name that already has a row +-- keeps whatever the user edited. That is what makes the declarations a +-- starting point rather than a setting — turning one off in the UI survives +-- the next restart. +-- +-- `next_run` and `last_run` are UTC ISO stamps like every other time here, and +-- `next_run` is compared as text in `due_schedules` for the same reason +-- `run_after` is in `claim`: `''` sorts before every stamp, so "never" and +-- "not due" need no coalesce and no special case. A cadence worded in a local +-- timezone is turned into one of these before it is stored — see +-- `pipeline/schedule.py`, which is the only place a zone is read. +-- +-- `last_job_id` is what stops a schedule piling up behind itself: the firing +-- checks whether that job is still queued or running before starting another. +CREATE TABLE IF NOT EXISTS schedules ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + kind TEXT NOT NULL, + label TEXT NOT NULL DEFAULT '', + payload TEXT NOT NULL DEFAULT '{}', + cadence TEXT NOT NULL, + enabled INTEGER NOT NULL DEFAULT 1, + next_run TEXT NOT NULL DEFAULT '', + last_run TEXT NOT NULL DEFAULT '', + last_job_id INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, + UNIQUE (name) +); +""" + +#: Made after `_reconcile`, not with the table. An index names columns, and on +#: a file that predates one of them `CREATE INDEX` fails before the `ALTER` +#: that would have added it — which is a database that will not open, over an +#: index. Splitting them puts the two in the only order that always works. +INDEXES = """ +CREATE INDEX IF NOT EXISTS idx_jobs_state ON jobs(state, id); +CREATE INDEX IF NOT EXISTS idx_jobs_cascade ON jobs(cascade_id, id); +CREATE INDEX IF NOT EXISTS idx_jobs_task ON jobs(task_id, id); + +-- For the one question that asks about a moment rather than about an id: +-- which cascades ended just now, which is how a finished run stays on screen +-- for a few seconds. Without it that clause is a scan of every job kept, taken +-- on every push. `purge` is what bounds "every job kept". +CREATE INDEX IF NOT EXISTS idx_jobs_finished ON jobs(finished_at); + +-- Partial, so it constrains only jobs that have not finished yet. This is what +-- makes "one pull at a time" a property of the system rather than of whichever +-- client asked: a second tab and a command collide here, where a +-- flag in one browser could not see the others. Enqueueing a duplicate hands +-- back the live job instead of failing. +CREATE UNIQUE INDEX IF NOT EXISTS idx_jobs_dedup ON jobs(dedup_key) + WHERE dedup_key <> '' AND state IN ('queued', 'running'); +""" + +LIVE = ("queued", "running") + +#: What holds a job back. Both halves read the dependency's own state rather +#: than a count kept alongside it, so nothing has to be incremented as jobs +#: land and nothing can be lost to a restart mid-run. +#: And whether a worker of its resource is free. `workers` arrives as JSON so +#: one parameter carries every count; a resource it does not mention gets one. +_HAS_ROOM = """ + (SELECT count(*) FROM jobs r WHERE r.state = 'running' AND r.resource = q.resource) + < coalesce(json_extract(?2, '$."' || q.resource || '"'), 1) +""" + +_READY = """ + NOT EXISTS (SELECT 1 FROM json_each(q.depends_on) d + JOIN jobs dep ON dep.id = d.value + WHERE dep.state IN ('queued','running')) + AND (q.after = 'done' OR NOT EXISTS ( + SELECT 1 FROM json_each(q.depends_on) d + JOIN jobs dep ON dep.id = d.value + WHERE dep.state IN ('failed','cancelled'))) +""" + +#: And whether its time has come. The second thing that holds a job back, and +#: it is the same kind of thing as the first: a clause in the claim, not an +#: object waiting anywhere. A job scheduled for an hour hence costs a row and +#: nothing else — no worker, no thread, no lease, no timer to be lost to a +#: restart — and the loop that looks for work every second is what notices +#: when the hour is up. +#: +#: `?1` is the same `now` the claim stamps with, so a job due at exactly this +#: instant is taken by the pass that finds it rather than the one after. +_DUE = "q.run_after <= ?1" + + + +#: The lease of a job nobody is running any more. Older than every real +#: stamp, so `reclaim_stale` takes the row on its next look rather than after +#: a silence long enough to infer what the worker already knew. +_LONG_AGO = datetime.fromtimestamp(0, UTC).isoformat() + + +def _now() -> str: + return datetime.now(UTC).isoformat() + + +def _in(seconds: float) -> str: + """A stamp that many seconds from now, or `''` for no wait at all. + + Written in the same UTC ISO form as every other time in this table, + because `run_after` is compared as text in `claim` and a second format in + the same column would sort by whichever digits it happened to start with. + Empty for nothing to wait for, which is what makes the comparison free of + a special case: `''` is before every stamp there is. + """ + if seconds <= 0: + return "" + return (datetime.now(UTC) + timedelta(seconds=seconds)).isoformat() + + +def on_its_own(name: str) -> str: + """What to call the cascade minted for a lone job or a lone task. + + Two different things would otherwise both be a cascade named `pull`: the + chain that starts with a pull and goes on to five more tasks, and the one + job somebody pressed by itself. `live_run` answers by name, so it would + hand the first back for the second — a Fetch that starts nothing, greys + its own button, and returns a run that is over a second later. + + A task is not renamed, only the cascade wrapped round it: the `pull` task + of a lone pull really is the pull task, and the strip draws its box from + that name. It is the chain that was never asked for. + + The space is the whole of the guarantee. A job kind and a task name are + identifiers — `pull`, `body_batch` — so nothing that mints a cascade + deliberately can ever collide with this. + """ + return f"{name} on its own" + + +class RunCancelled(Exception): + """Raised by `enqueue` for a job under a run that has been killed. + + The whole of how a kill stops what has not been queued yet. Nobody tells a + cascade it is over: its step tries to queue the next task, and the table + says no — across processes, after a restart, and when the kill landed in + the middle of the run. The queue logs it and drops the job. + """ + + +#: The largest `result` a job may leave on its row, as JSON. A result is a +#: report — counts, ids, an error — and the snapshot and every tally read it +#: back, so data returned instead of written to the app's store would make +#: every one of those queries slower for as long as the row is kept. +#: +#: Past this the job *fails*. It used to keep the keys and log a warning, which +#: is losing data where nobody is looking: the next job in a chain would get a +#: result with its payload missing and carry on. Raise the limit if a project +#: really wants bigger results; do not make it quiet. +MAX_RESULT = 64 * 1024 + + +class ResultTooLarge(ValueError): + """A handler returned more than `MAX_RESULT` bytes of JSON. + + Raised by `Jobs.finish` before anything is written, so the row is not + touched. The worker ends the job `failed` with this message and does not + retry it: the same handler would return the same result. + """ + + +def _result(job_id: int, result: dict[str, Any] | None) -> str: + text = json.dumps(result or {}) + if len(text) > MAX_RESULT: + raise ResultTooLarge( + f"returned {len(text) // 1024} KB, and a job's result is capped at " + f"{MAX_RESULT // 1024} KB (pipeline.table.MAX_RESULT). Write the " + f"data to the store and return its ids; keys returned: " + f"{', '.join(sorted(str(k) for k in (result or {}))[:20])}") + return text + + +def _entry(row: sqlite3.Row) -> NoticeEntry: + """A `notices` row as the model, with `place` given back its real name.""" + fields = dict(row) + fields["where"] = fields.pop("place") + return NoticeEntry.model_validate(fields) + + +class Jobs: + """The jobs table on one file. Every statement against it is here.""" + + def __init__(self, path: str | Path) -> None: + self.path = Path(path).expanduser() + self.path.parent.mkdir(parents=True, exist_ok=True) + with closing(self._connect()) as conn, conn: + conn.execute("PRAGMA journal_mode=WAL") + conn.executescript(SCHEMA) + self._reconcile(conn) + conn.executescript(INDEXES) + + @staticmethod + def _reconcile(conn: sqlite3.Connection) -> None: + """Add any column `SCHEMA` has grown since this file was created.""" + runs = {r["name"] for r in conn.execute("PRAGMA table_info(runs)")} + for column in ("cancelled_at", "label"): + if runs and column not in runs: + conn.execute( + f"ALTER TABLE runs ADD COLUMN {column} TEXT NOT NULL DEFAULT ''") + have = {r["name"] for r in conn.execute("PRAGMA table_info(jobs)")} + # The agent handoff, on a file made while it existed. Dropped rather + # than left: a column nobody writes is a question every reader of this + # table has to answer for itself, and the answer is that a job the + # model cannot do now fails and says why. + for gone in ("handoff", "answer", "instructions", "skill"): + if gone in have: + conn.execute(f"ALTER TABLE jobs DROP COLUMN {gone}") + for column, ddl in (("seen_at", "TEXT"), + ("depends_on", "TEXT NOT NULL DEFAULT '[]'"), + ("after", "TEXT NOT NULL DEFAULT 'ok'"), + ("resource", "TEXT NOT NULL DEFAULT 'cpu'"), + ("cascade_id", "INTEGER NOT NULL DEFAULT 0"), + ("task_id", "INTEGER NOT NULL DEFAULT 0"), + # The retry policy, arriving on a file that + # predates it. Every default is what the file + # already behaved as: one attempt, no wait, due + # now — so a database migrated by this reads + # exactly as it did before the column existed. + ("attempt", "INTEGER NOT NULL DEFAULT 0"), + ("max_attempts", "INTEGER NOT NULL DEFAULT 1"), + ("retry_delay", "REAL NOT NULL DEFAULT 0"), + ("retry_backoff", "REAL NOT NULL DEFAULT 1"), + ("run_after", "TEXT NOT NULL DEFAULT ''")): + if column not in have: + conn.execute(f"ALTER TABLE jobs ADD COLUMN {column} {ddl}") + + def _connect(self) -> sqlite3.Connection: + # Thirty seconds, and whoever else opens this file uses the same number + # — see `newsdesk/storage/sql/store.py`, which says why the longer of + # two timeouts on one file is the one to keep. + conn = sqlite3.connect(self.path, timeout=30) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA busy_timeout=30000") + return conn + + def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]: + with closing(self._connect()) as conn: + return conn.execute(sql, params).fetchall() + + def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None: + with closing(self._connect()) as conn: + return conn.execute(sql, params).fetchone() + + def _write(self, sql: str, params: Any = ()) -> int: + """One statement in its own transaction. Returns the rowcount.""" + with closing(self._connect()) as conn, conn: + return conn.execute(sql, params).rowcount + + # --------------------------------------------------------------- queue + + def enqueue(self, kind: str, label: str = "", payload: dict[str, Any] | None = None, + dedup_key: str = "", depends_on: Sequence[int] = (), + after: str = "ok", resource: str = "cpu", + cascade_id: int = 0, task_id: int = 0, delay: float = 0.0, + max_attempts: int = 1, retry_delay: float = 0.0, + retry_backoff: float = 1.0) -> Job: + """Queue one job. A duplicate hands back the live one instead of adding. + + `dedup_key` is enforced by a partial unique index over the unfinished + states, so "one pull at a time" holds across every client at once. An + empty key opts out, for jobs where several at once is the point. + + Returning the existing job rather than raising is deliberate: the + caller asked for that work to be underway, and it is. + + `depends_on` holds this job back until those jobs are over — see the + column comments in `SCHEMA` for what `after` chooses between. + + A job under a killed run is refused with `RunCancelled`. That is where + a kill reaches everything that had not been queued when the button was + pressed: the next task of a killed cascade, the batches a callback was + about to fan out. Checked in the job's own transaction, so a kill + landing between the check and the insert loses the race by a statement + rather than by a process. + + The refusal spares an `after='done'` job under a killed *task*, which + is the same carve-out `cancel_run` and `stop_task` make and for the + same reason: killing a task is "skip this one", and its final job is + what says so. Without it a kill landing while a task was still fanning + out its batches would refuse the final job too, and the task would sit + with nothing queued, nothing running and nobody to report that it had + been killed. A killed *cascade* still refuses it — there is nothing + left to report to. + + `delay` queues it for later: the row goes in `queued` with a + `run_after` that many seconds hence, and `claim` passes over it until + then. It is the durable form of "in five minutes" — the row is written + now, so the wait survives the process that asked for it, and nothing is + alive in the meantime. + + `max_attempts` and the two retry numbers are the bound `retry` will + enforce, stored on the row. The defaults are one attempt and no wait, + which is every job that has ever been queued here. + """ + if after not in ("ok", "done"): + raise ValueError(f"after must be 'ok' or 'done', not {after!r}") + if max_attempts < 1: + raise ValueError(f"max_attempts counts the first attempt, so it is " + f"at least 1, not {max_attempts!r}") + try: + with closing(self._connect()) as conn, conn: + killed = conn.execute( + """SELECT id FROM runs WHERE cancelled_at <> '' + AND (id = ?1 OR (id = ?2 AND ?3 <> 'done'))""", + (int(cascade_id), int(task_id), after), + ).fetchone() + if killed is not None: + raise RunCancelled( + f"run {killed['id']} was killed; {kind!r} is not queued") + # A job queued with neither begins both: something asked for + # this on its own, so it is a cascade of one task of one job. + # Minted in the job's own transaction, so a duplicate that + # rolls back takes its ids with it — a `runs` row with no job + # under it would be an identity for nothing. + # + # The cascade says it stands alone and the task does not: the + # task is the `pull` task however it was started, but a chain + # named `pull` is a different thing from this and has to stay + # findable. See `on_its_own`. + cascade_id = int(cascade_id) or self._mint( + conn, "cascade", on_its_own(kind), label) + task_id = int(task_id) or self._mint(conn, "task", kind, label) + job_id = conn.execute( + """INSERT INTO jobs (kind, label, payload, state, dedup_key, + depends_on, after, resource, cascade_id, task_id, + max_attempts, retry_delay, retry_backoff, + run_after, created_at) + VALUES (?,?,?,'queued',?,?,?,?,?,?,?,?,?,?,?)""", + (kind, label, json.dumps(payload or {}), dedup_key, + json.dumps([int(i) for i in depends_on]), after, + resource or "cpu", int(cascade_id), int(task_id), + int(max_attempts), float(retry_delay), float(retry_backoff), + _in(delay), _now()), + ).lastrowid + except sqlite3.IntegrityError: + existing = self._row( + """SELECT * FROM jobs + WHERE dedup_key=? AND state IN ('queued','running') + ORDER BY id DESC LIMIT 1""", + (dedup_key,), + ) + if existing is None: + raise + return Job.from_row(existing) + job = self.get(job_id) if job_id is not None else None + if job is None: + raise RuntimeError(f"job {job_id} disappeared between write and read") + return job + + def claim(self, workers: Mapping[str, int] | None = None) -> Job | None: + """Take the oldest ready job and mark it running, atomically. + + One statement, so two workers racing cannot both take the same job — + which matters the moment a worker runs outside the web process. Ready + rather than merely queued: a job with unmet dependencies is passed over + and stays where it is, costing nothing while it waits, because nothing + is running it. That is the whole of "waiting" here — no timer, no + listener, no object holding a place. + + `workers` is how many of each kind there are, and so how many jobs of + that resource may run at once. Counted + from the table rather than from anything this process is holding, so a + worker in a second process shares the same four model slots instead of + helping itself to four more. A resource nobody sized gets one. + + `seen_at` is stamped with `started_at`, so the job is leased from the + moment it is taken: a worker that dies in its first second still leaves + a row that goes stale rather than one that never had a heartbeat at + all and has to be guessed about. + + A job whose `run_after` has not come round is passed over the same way + a job with unmet dependencies is, and costs exactly as little. `attempt` + goes up here, in the statement that takes the row, because that is the + only place a count of runs cannot be lost or doubled — a worker that + dies a second later has still started this job once, and the row says + so without anyone having to write it down afterwards. + """ + with closing(self._connect()) as conn, conn: + row = conn.execute( + f"""UPDATE jobs SET state='running', started_at=?1, seen_at=?1, + attempt = attempt + 1 + WHERE id = (SELECT q.id FROM jobs q WHERE q.state='queued' + AND {_DUE} AND {_READY} AND {_HAS_ROOM} + ORDER BY q.id LIMIT 1) + RETURNING *""", + (_now(), json.dumps(dict(workers or {}))), + ).fetchone() + return Job.from_row(row) if row else None + + def running(self) -> list[Job]: + """Everything under way, oldest first.""" + return [Job.from_row(r) for r in self._rows( + "SELECT * FROM jobs WHERE state='running' ORDER BY id")] + + @contextmanager + def serialize_starts(self): + """Serialize cascade discovery and creation across processes. + + A separate SQLite lock file leaves the jobs database writable while + the task creates its batches. SQLite releases the lock on a crash. + """ + with closing(sqlite3.connect(str(self.path) + ".start-lock", timeout=60)) as conn: + conn.execute("BEGIN IMMEDIATE") + try: + yield + finally: + conn.rollback() + + def new_run(self, kind: str, name: str = "", label: str = "") -> int: + """Mint an id for a cascade or a task, before it has queued anything. + + Its own series, not a job's. A task is a thing before any of its jobs + exist — it is what decides how many there will be — so taking the id + of whichever job it happened to queue first would be describing the + task by one of its own consequences. The same goes for what it is + called: `name` is what it is, `label` is how to say it. + """ + with closing(self._connect()) as conn, conn: + return self._mint(conn, kind, name, label) + + @staticmethod + def _mint(conn: sqlite3.Connection, kind: str, name: str, + label: str = "") -> int: + """One `runs` row, on the caller's connection, so it can share a + transaction with the job that will carry the id.""" + return int(conn.execute( + "INSERT INTO runs (kind, name, label, created_at) VALUES (?,?,?,?)", + (kind, name, label or name, _now()), + ).lastrowid or 0) + + def run(self, run_id: int) -> Run | None: + """What a cascade or task id names: its kind, its name, when it began.""" + row = self._row("SELECT * FROM runs WHERE id=?", (int(run_id),)) + return Run.from_row(row) if row else None + + def of_cascade(self, cascade_id: int) -> list[Job]: + """Every job of one cascade, oldest first.""" + return [Job.from_row(r) for r in self._rows( + "SELECT * FROM jobs WHERE cascade_id=? ORDER BY id", (int(cascade_id),))] + + def of_task(self, task_id: int) -> list[Job]: + """Every job of one task: its batches, and the final job behind them.""" + return [Job.from_row(r) for r in self._rows( + "SELECT * FROM jobs WHERE task_id=? ORDER BY id", (int(task_id),))] + + def live_run(self, kind: str, name: str) -> Run | None: + """The newest run of that kind and name with a job still going. + + What "is a pull already under way?" means, and what no job kind can + answer: `solo` stops a second `pull` while the first runs and says + nothing once the run has moved on to bodies. A run is live for as long + as any of its jobs is, which is the span the Fetch button means. + + The name is the whole of the question, which is why the cascade minted + for something standing alone is named apart — see `on_its_own`. A lone + `pull` job is not the pull chain, and answering with it would be this + saying yes to a question nobody asked. + """ + row = self._row( + """SELECT * FROM runs r WHERE r.kind=? AND r.name=? AND r.cancelled_at='' + AND EXISTS (SELECT 1 FROM jobs j WHERE j.cascade_id=r.id + AND j.state IN ('queued','running')) + ORDER BY r.id DESC LIMIT 1""", + (kind, name), + ) + return Run.from_row(row) if row else None + + def outcomes(self, ids: Sequence[int]) -> list[Job]: + """How each of those jobs ended, oldest first. + + What a final job is handed, and the only thing it needs: seven done + and one failed is read off the jobs themselves, at the moment it is + asked, rather than counted up as they landed. There is nothing to + increment and so nothing to lose to a restart half way through. + """ + if not ids: + return [] + holes = ",".join("?" for _ in ids) + return [Job.from_row(r) for r in self._rows( + f"SELECT * FROM jobs WHERE id IN ({holes}) ORDER BY id", + tuple(int(i) for i in ids))] + + def blocked(self) -> list[Job]: + """Queued jobs that `claim` would pass over. For the UI and for tests. + + Both reasons, because from here they are the same fact: this row is in + the queue and is not going to be taken yet. Which of the two it is — + something it waits for, or a time it waits until — is on the row, in + `depends_on` and `run_after`. + """ + return [Job.from_row(r) for r in self._rows( + f"""SELECT * FROM jobs q WHERE q.state='queued' + AND NOT ({_DUE} AND {_READY}) ORDER BY q.id""", + (_now(),))] + + def progress(self, job_id: int, done: int = 0, total: int = 0, + message: str = "") -> None: + self._write("UPDATE jobs SET done=?, total=?, message=? WHERE id=?", + (done, total, message, job_id)) + + def finish(self, job_id: int, state: str, result: dict[str, Any] | None = None, + error: str = "") -> Job | None: + """Move a job to one of the terminal states, once, and settle what waited. + + A job that ended badly is the moment anything depending on it can be + answered, so it is answered here rather than swept for later: nothing + should sit queued forever waiting on something that is never coming. + """ + self._write( + """UPDATE jobs SET state=?, result=?, error=?, finished_at=? + WHERE id=? AND state NOT IN ('done','failed','cancelled')""", + (state, _result(job_id, result), error, _now(), job_id), + ) + if state in ("failed", "cancelled"): + self._settle_dependents() + return self.get(job_id) + + def retry(self, job_id: int, *, error: str = "", + delay: float | None = None) -> Job | None: + """End an attempt that went badly: another one later, or give up. + + The one place the bound is read, and so the one place a job stops + being tried. Three endings, and which one it is depends on nothing the + caller knows: + + asked to stop `cancelled`. A Kill that landed while the attempt + ran must not be undone by the attempt failing — + that would be a button that quietly did nothing. + attempts left back to `queued`, with `run_after` set and the + lease handed back. Nothing runs it, nothing holds a + place for it, and whatever depends on it goes on + waiting, because it is still live. + none left `failed`, and `Job.exhausted` from then on. + + The wait is `retry_delay` multiplied by `retry_backoff` once for each + attempt already made, unless `delay` overrides it — which is how a + server's own answer to "come back in 120 seconds" is honoured instead + of guessed at. A hint does not buy an extra attempt: the bound is + ours, the wait is the world's. + + The error is kept on the row either way, so a job sitting in the queue + with an hour to wait still says what went wrong last time rather than + looking like ordinary work nobody has got to yet. + + Read then written, rather than one statement like everything else + here: the arithmetic wants the row's own numbers, and this is the one + method whose row is `running` and held by the worker calling it, so + there is no second writer to lose a race with. + """ + job = self.get(job_id) + if job is None: + return None + if job.cancel_requested: + return self.finish(job_id, "cancelled", error=error) + if job.attempt >= job.max_attempts: + return self.finish(job_id, "failed", error=error) + wait = delay if delay is not None else ( + job.retry_delay * job.retry_backoff ** max(job.attempt - 1, 0)) + self._write( + """UPDATE jobs SET state='queued', error=?, run_after=?, message='', + started_at=NULL, seen_at=NULL + WHERE id=? AND state='running'""", + (error, _in(wait), job_id), + ) + return self.get(job_id) + + def waiting(self) -> list[Job]: + """Queued jobs whose next attempt is still to come, oldest first.""" + return [Job.from_row(r) for r in self._rows( + "SELECT * FROM jobs WHERE state='queued' AND run_after > ? ORDER BY id", + (_now(),))] + + def _settle_dependents(self) -> None: + """End queued jobs that can no longer run, and their dependents in turn. + + Only jobs that wanted success. Killing one job is not stopping the run + it belongs to — it is dropping that batch, exactly as a batch that + failed is dropped, and the work comes round next time because every + task rediscovers its own backlog. So the job waiting to report on the + batches still runs, counts this one among those that did not land, and + the chain carries on: to a cascade a killed batch and a failed one are + the same thing. Stopping a whole task or a whole cascade is + `cancel_run`, which is a different button and says so. + + Looped because a dependent is itself depended on, and each pass ends + one more generation. It terminates: every pass moves at least one job + out of `queued`, and nothing here puts one back. + """ + while self._write( + """UPDATE jobs SET state='failed', finished_at=?, message='', + error='a job it waited for did not finish' + WHERE jobs.state = 'queued' AND jobs.after = 'ok' + AND EXISTS (SELECT 1 FROM json_each(jobs.depends_on) d + JOIN jobs dep ON dep.id = d.value + WHERE dep.state IN ('failed','cancelled'))""", + (_now(),), + ): + pass + + def get(self, job_id: int) -> Job | None: + row = self._row("SELECT * FROM jobs WHERE id=?", (job_id,)) + return Job.from_row(row) if row else None + + def last(self, kind: str, *, states: tuple[str, ...] = ("done",)) -> Job | None: + """The most recent job of a kind that reached one of `states`. + + Ordered by `id` rather than `finished_at`: the id is the order things + were asked for, which is what "the last pull" means. + """ + placeholders = ",".join("?" for _ in states) + row = self._row( + f"""SELECT * FROM jobs WHERE kind=? AND state IN ({placeholders}) + ORDER BY id DESC LIMIT 1""", + (kind, *states), + ) + return Job.from_row(row) if row else None + + # -------------------------------------------------------- cancelling + + def request_cancel(self, job_id: int) -> Job | None: + """Ask for a job to stop. + + A queued job is finished here and now — nothing is part-way through. + A running one only gets the request recorded; the worker notices on + its next `context.check()` and writes the terminal state itself, + because only it knows when it has reached a point where stopping is + safe. + + A job waiting for its next attempt is a queued job and ends here with + the rest of them. That is the point of the wait being a column rather + than a sleeping worker: there is a row to cancel, from any process, at + any moment of the hour it is waiting. + """ + job = self.get(job_id) + if job is None or not job.live: + return job + if job.state == "queued": + return self.finish(job_id, "cancelled") + self._write("UPDATE jobs SET cancel_requested=1 WHERE id=?", (job_id,)) + return self.get(job_id) + + def cancel_run(self, run_id: int) -> int: + """Kill a run — a task or a cascade, the same operation for both. + + Three things: flag the row, end the queued jobs carrying that id, and + ask the running ones. Plus the fourth that is not here: `enqueue` + refuses anything later queued under it. That is the whole of how a kill + stops the pipeline, and the flag is what makes it hold across + processes, after a restart, and when the kill lands mid-run. + + The one asymmetry is deliberate and is what the two buttons mean. A + cascade's id takes everything in it. A task's id spares the job waiting + on `after='done'` — its final job — because reporting is not work: + killing a task is "skip this one", not "make it never end". So the + final job still runs and still calls the step, and what it hands over + says the *task* was killed rather than only how many batches ended + cancelled — the two are indistinguishable in the counts and mean + opposite things, and `Jobs.cancelled` is where that flag is read back. + What to do about it is then the cascade's decision, not this call's: + newsdesk's skips the stage and carries on. The step's attempt to queue + the next task goes ahead if only the task was killed, and is refused if + the cascade was killed too. + """ + self._write( + "UPDATE runs SET cancelled_at=? WHERE id=? AND cancelled_at=''", + (_now(), int(run_id))) + # Queued jobs end here and now — nothing is part-way through. A running + # one is asked and stops at its next `context.check()`, because only it + # knows when it has reached a point where stopping is safe. + where = "(cascade_id=?1 OR (task_id=?1 AND after<>'done'))" + ended = self._write( + f"""UPDATE jobs SET state='cancelled', finished_at=?2, message='' + WHERE {where} AND state='queued'""", + (int(run_id), _now()), + ) + asked = self._write( + f"UPDATE jobs SET cancel_requested=1 WHERE {where} AND state='running'", + (int(run_id),)) + self._settle_dependents() + return ended + asked + + def stop_task(self, task_id: int, error: str = "") -> int: + """End a task's queued batches without killing the task. How many. + + What a batch calls when the failure it just met is the backend's + answer to the whole run rather than to its own rows — a spend limit, a + login that has expired, no model on the machine. The siblings behind it + would each make the same call and get the same refusal, and each put + the same notice on screen; twenty-three of those is the bug this + exists to stop. + + Three differences from `cancel_run`, and each of them is the point: + + the run is not flagged nothing was killed. `Tally.killed` means a + person pressed Kill and the step is to be + skipped, and borrowing the flag to mean + "the model said no" would make the two + indistinguishable in the one place that + can still tell them apart. + running jobs are left their requests are in the air and are a + bill either way. They end however they end + — including the one that called this, which + keeps its own `failed` rather than being + cancelled by its own instruction. + the reason is kept written to `error` on each row, so a batch + that never ran says why on screen instead + of looking like one somebody dropped. + + The final job is spared, as it is by a kill and for the same reason: + reporting is not work, and a run that stopped early is exactly the one + that has something to report. + """ + ended = self._write( + """UPDATE jobs SET state='cancelled', finished_at=?, message='', + error=? WHERE task_id=? AND after<>'done' AND state='queued'""", + (_now(), error, int(task_id)), + ) + if ended: + self._settle_dependents() + return ended + + def cancelled(self, run_id: int) -> bool: + """Whether this task or cascade has been killed.""" + row = self._row("SELECT cancelled_at FROM runs WHERE id=?", (int(run_id),)) + return bool(row and row[0]) + + def cancel_requested(self, job_id: int) -> bool: + row = self._row("SELECT cancel_requested FROM jobs WHERE id=?", (job_id,)) + return bool(row[0]) if row else False + + # ---------------------------------------------------------- schedules + + #: Every schedule, with how its last job ended. One statement, because the + #: list and the socket push both want the join and neither wants a query + #: per row — and because a row read in two halves can disagree with itself. + _SCHEDULES = """ + SELECT s.*, coalesce(j.state, '') AS last_state + FROM schedules s LEFT JOIN jobs j ON j.id = s.last_job_id + """ + + def schedules(self) -> list[Schedule]: + """Every schedule, by name. The order the UI lists them in. + + By name rather than by id, because the name is what a person reads and + an id is the order they happened to be added in — including the order + the app's declared defaults were imported in, which is not a fact about + anything. + """ + return [Schedule.from_row(r) for r in + self._rows(self._SCHEDULES + " ORDER BY s.name")] + + def schedule(self, schedule_id: int) -> Schedule | None: + row = self._row(self._SCHEDULES + " WHERE s.id=?", (int(schedule_id),)) + return Schedule.from_row(row) if row else None + + def add_schedule(self, name: str, kind: str, cadence: str, *, + payload: dict[str, Any] | None = None, label: str = "", + enabled: bool = True, next_run: str = "", + only_if_new: bool = False) -> Schedule | None: + """Write one schedule. `None` when `only_if_new` and the name is taken. + + `only_if_new` is the whole of reconciliation: the queue offers every + declared default on every start, and a name that already has a row is + left exactly as the user edited it — cadence, payload, enabled and all. + Which is what makes a declaration a starting point rather than a + setting somebody has to fight. + + One statement, so two workers starting at once insert a declared + default once between them rather than racing a read against a write. + The conflict is the second one finding the first one's row. + """ + clash = "DO NOTHING" if only_if_new else """DO UPDATE SET + kind=excluded.kind, label=excluded.label, + payload=excluded.payload, cadence=excluded.cadence, + enabled=excluded.enabled, next_run=excluded.next_run""" + with closing(self._connect()) as conn, conn: + row = conn.execute( + f"""INSERT INTO schedules (name, kind, label, payload, cadence, + enabled, next_run, created_at) + VALUES (?,?,?,?,?,?,?,?) + ON CONFLICT (name) {clash} + RETURNING id""", + (name, kind, label, json.dumps(payload or {}), cadence, + int(bool(enabled)), next_run, _now()), + ).fetchone() + return self.schedule(row["id"]) if row else None + + #: What `update_schedule` will write, and the whole of it. A whitelist + #: rather than trust, because the column names go into the statement: the + #: values are parameters but the keys cannot be, so this is the boundary. + #: `name` is not here — it is the handle a declared default is matched on, + #: and renaming a row would hand its edits back to a freshly inserted one. + EDITABLE = ("kind", "label", "payload", "cadence", "enabled", + "next_run", "last_run", "last_job_id") + + def update_schedule(self, schedule_id: int, **fields: Any) -> Schedule | None: + """Change some of one schedule's columns. The row as it now stands. + + Also how a firing that decided not to fire moves on: setting + `next_run` alone is "skip this slot", which is what happens when the + last job of this schedule is still going. Nothing else is touched, so + `last_run` goes on saying when it last actually ran. + """ + wanted = {k: v for k, v in fields.items() if k in self.EDITABLE} + if not wanted: + return self.schedule(schedule_id) + if "payload" in wanted and not isinstance(wanted["payload"], str): + wanted["payload"] = json.dumps(wanted["payload"] or {}) + if "enabled" in wanted: + wanted["enabled"] = int(bool(wanted["enabled"])) + sets = ", ".join(f"{k}=?" for k in wanted) + self._write(f"UPDATE schedules SET {sets} WHERE id=?", + (*wanted.values(), int(schedule_id))) + return self.schedule(schedule_id) + + def delete_schedule(self, schedule_id: int) -> bool: + """Forget one. Not an error when it has already gone — two tabs.""" + return bool(self._write("DELETE FROM schedules WHERE id=?", + (int(schedule_id),))) + + def due_schedules(self, now: str) -> list[Schedule]: + """Enabled schedules whose time has come, oldest due first. + + Text comparison against a UTC ISO `now`, for the reason `claim` uses + one: `''` sorts before every stamp, so a schedule with no next run — + one that has never been given a time — is simply not due, with no + coalesce and no special case. + + Oldest first so that a queue coming back after a long silence fires + what was due longest ago before what was due a minute ago, which is + the order a person would have run them in. + """ + return [Schedule.from_row(r) for r in self._rows( + self._SCHEDULES + """ WHERE s.enabled=1 AND s.next_run <> '' + AND s.next_run <= ? ORDER BY s.next_run""", + (now,))] + + def mark_fired(self, schedule_id: int, job_id: int, next_run: str) -> Schedule | None: + """Record that this schedule just queued that job, and when it is next. + + One statement, because the three move together: a `last_job_id` written + without the `next_run` beside it would be a schedule that fires again + on the following pass, every pass, for as long as that job runs. + """ + self._write( + """UPDATE schedules SET last_run=?, last_job_id=?, next_run=? + WHERE id=?""", + (_now(), int(job_id), next_run, int(schedule_id))) + return self.schedule(schedule_id) + + # ------------------------------------------------------------ repairs + + def heartbeat(self, ids: Sequence[int]) -> int: + """Say those running jobs are still being run. Returns how many rows. + + The other half of `reclaim_stale`. Called on a timer by the worker + that holds them, and against the table rather than any object, because + the process that has to read it is a different one. + + Only `running` rows are touched: a job that finished between the beat + being scheduled and the beat landing must not have its lease refreshed + under its terminal state. + """ + if not ids: + return 0 + holes = ",".join("?" for _ in ids) + return self._write( + f"UPDATE jobs SET seen_at=? WHERE state='running' AND id IN ({holes})", + (_now(), *(int(i) for i in ids))) + + def release(self, ids: Sequence[int]) -> int: + """Hand back the lease on jobs this worker is walking away from. + + A process on its way out knows something `reclaim_stale` can only wait + to infer: that these jobs have nobody running them as of now. Saying so + is what makes a restart pick them up at once rather than a lease later. + Only a crash, which by definition says nothing, pays the wait. + + The lease is expired rather than the job re-queued, and that is the + point of doing it this way: the row stays `running`, so a handler + thread that outlives the shutdown still writes its own ending into it + rather than being overwritten by a second run of itself. That ending + is the thread's to write — see `JobQueue._execute`, where the handler + call and the terminal write are one function — which is what makes + expiring the lease the safe half-measure it reads as. Only the next + `reclaim_stale` moves the row, and only if that ending never came. + """ + if not ids: + return 0 + holes = ",".join("?" for _ in ids) + return self._write( + f"UPDATE jobs SET seen_at=? WHERE state='running' AND id IN ({holes})", + (_LONG_AGO, *(int(i) for i in ids))) + + def purge(self, older_than: float) -> int: + """Delete jobs that finished more than `older_than` seconds ago. + + Returns how many. Without this both tables grow for as long as the app + runs. Three things are kept whatever their age: + + anything not finished a queued or running row is work, not + history. + the newest ending of each `last(kind)` answers "when did this last + kind in each state succeed" from the table, and a status + line saying "never pulled" because the + last pull was five weeks ago is wrong. + a run with jobs left a run is deleted only once nothing + points at it and it is old itself — one + minted a moment ago may have no jobs yet + because its finder is still looking. + + One transaction, so a snapshot never sees jobs gone and their runs + still there. + """ + cutoff = (datetime.now(UTC) - timedelta(seconds=older_than)).isoformat() + with closing(self._connect()) as conn, conn: + gone = conn.execute( + """DELETE FROM jobs + WHERE state IN ('done','failed','cancelled') + AND finished_at IS NOT NULL AND finished_at < ? + AND id NOT IN (SELECT MAX(id) FROM jobs + WHERE state IN ('done','failed','cancelled') + GROUP BY kind, state)""", + (cutoff,)).rowcount + conn.execute( + """DELETE FROM runs WHERE created_at < ? + AND id NOT IN (SELECT cascade_id FROM jobs) + AND id NOT IN (SELECT task_id FROM jobs)""", + (cutoff,)) + return int(gone or 0) + + # ------------------------------------------------------------- notices + + def record_notice(self, notice: Notice) -> NoticeEntry: + """Write one notice into the log, and say how often it has been said. + + One statement, for the reason every write here is one: two batches of + one run meet the same expired login in the same second, and a read + followed by a write would file it twice. The conflict is the second + batch finding the first one's row. + + A repeat takes the newer wording as well as the count. The text is a + sentence about now — the refused article's title, the feed that died + this time — and a log showing last week's sentence beside today's + count is describing two different moments. + + `count == 1` on what comes back is how a caller knows this is news. See + `NoticeEntry`. + """ + now = _now() + with closing(self._connect()) as conn, conn: + row = conn.execute( + """INSERT INTO notices (code, key, level, kind, text, fix, + detail, place, first_at, last_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (code, key) DO UPDATE SET + count = count + 1, last_at = excluded.last_at, + level = excluded.level, kind = excluded.kind, + text = excluded.text, fix = excluded.fix, + detail = excluded.detail, place = excluded.place + RETURNING *""", + (notice.code, notice.key, notice.level, notice.kind, notice.text, + notice.fix, notice.detail, notice.where, now, now)).fetchone() + return _entry(row) + + def notices(self, limit: int = 200) -> list[NoticeEntry]: + """The log, most recently said first.""" + return [_entry(row) for row in self._rows( + "SELECT * FROM notices ORDER BY last_at DESC, id DESC LIMIT ?", + (int(limit),))] + + def clear_notice(self, notice_id: int) -> bool: + """Done with this one. The same failure afterwards is news again.""" + return bool(self._write("DELETE FROM notices WHERE id=?", (int(notice_id),))) + + def clear_notices(self) -> int: + return self._write("DELETE FROM notices") + + def purge_notices(self, quiet_for: float) -> int: + """Forget what has not been said again for `quiet_for` seconds. + + By `last_at`, not `first_at`: a feed that has failed every evening for + a month is a row a month old that was true an hour ago, and deleting + it would make tonight's failure look like the first. What goes is what + stopped happening, which is the only kind of notice that can be + dropped without anyone having read it — and dropping it is what lets + the same failure, months on, interrupt you again. + """ + cutoff = (datetime.now(UTC) - timedelta(seconds=quiet_for)).isoformat() + return self._write("DELETE FROM notices WHERE last_at < ?", (cutoff,)) + + def reclaim_stale(self, older_than: float = 30.0, *, + except_ids: Sequence[int] = ()) -> int: + """Put jobs whose worker stopped saying it was there back in the queue. + + A `running` row is a claim plus a lease: `claim` stamps `seen_at` and + the worker holding the job restamps it every few seconds. So a row + whose lease has run out is one whose worker went away mid-job, and + re-queueing is the right reading of that — the work was asked for and + never happened. Handlers have to tolerate being re-run. + + The lease is what makes this safe on a file two processes share. Left + as "every `running` row", a second worker starting up would re-queue + the first one's live jobs and both would run them; `older_than` is how + long a worker has to have been silent before that reading is the only + one left. Which is also why this runs on a timer and not only at + start: whichever process is still alive is the one that cleans up + after the one that is not. + + A row with no `seen_at` at all — claimed by a version of this file + that predates the column — falls back to `started_at`, so an old + orphan is still reclaimed once it is old enough. + + One of these does not come back: a job that had been asked to stop + before its worker died ends `cancelled` instead of returning to the + queue. Re-queueing it would be a Kill that a crash quietly undid, and + the request is in the table precisely so that it outlives the process + that was supposed to act on it. + + A worker excludes the ids it still holds. Their expired leases mean + its loop stalled, not that its handler threads stopped; renew them and + warn instead of running the same handler twice in this process. + """ + cutoff = (datetime.now(UTC) - timedelta(seconds=older_than)).isoformat() + ids = tuple(int(i) for i in except_ids) + holes = ",".join("?" for _ in ids) + if ids: + renewed = self._write( + f"UPDATE jobs SET seen_at=? WHERE state='running' " + f"AND coalesce(seen_at, started_at, created_at) < ? " + f"AND id IN ({holes})", (_now(), cutoff, *ids)) + if renewed: + log.warning("worker loop stalled; renewed %d held job lease(s)", renewed) + excluded = f" AND id NOT IN ({holes})" if ids else "" + taken = self._write( + """UPDATE jobs SET + state = CASE WHEN cancel_requested THEN 'cancelled' ELSE 'queued' END, + finished_at = CASE WHEN cancel_requested THEN ?1 ELSE NULL END, + started_at=NULL, seen_at=NULL, cancel_requested=0 + WHERE state='running' + AND coalesce(seen_at, started_at, created_at) < ?2""" + excluded, + (_now(), cutoff, *ids)) + if taken: + # Cheap and only on a reclaim, which is rare. A job that came back + # cancelled is a job something may have been waiting for. + self._settle_dependents() + return taken + + # ------------------------------------------------------------ reading + + #: How long a finished cascade stays in `cascades`. Nothing depends on it + #: — it is so that a run you watched is still on screen the moment after + #: it lands, rather than vanishing on the push that says it is over, which + #: is the one push you wanted to see. + LINGER = timedelta(seconds=15) + + def cascades(self) -> list[CascadeProgress]: + """Every cascade with something live in it, tallied task by task. + + One query and a grouping. The rows are every job of those cascades and + not only the live ones, because a task's landed jobs are most of what + a bar of it is and they are long gone from `snapshot`'s `recent`. + + `finished_at` is set only when a job reaches a terminal state, so the + second half of that `IN` needs no state list — and without one it can + use the index rather than reading every job that ever finished. + + Ordered by the id of the first job of each, which for a cascade whose + tasks queue one another is the order they ran. + """ + rows = self._rows( + """SELECT j.cascade_id, j.task_id, j.id, j.state, j.label, + c.name AS cascade_name, t.name AS task_name, + t.label AS task_label + FROM jobs j + LEFT JOIN runs c ON c.id = j.cascade_id + LEFT JOIN runs t ON t.id = j.task_id + WHERE j.cascade_id IN ( + SELECT cascade_id FROM jobs WHERE state IN ('queued','running') + UNION + SELECT cascade_id FROM jobs WHERE finished_at > ?) + ORDER BY j.id""", + ((datetime.now(UTC) - self.LINGER).isoformat(),)) + + runs: dict[int, CascadeProgress] = {} + tasks: dict[tuple[int, int], TaskProgress] = {} + for row in rows: + run = runs.get(row["cascade_id"]) + if run is None: + run = runs[row["cascade_id"]] = CascadeProgress( + id=row["cascade_id"], + # A job queued outside any cascade begins one named after + # itself and for standing alone, so there is always a name + # and no chain answers to it; the fallback is for a row + # from before `runs` existed. + name=row["cascade_name"] or "") + task = tasks.get((row["cascade_id"], row["task_id"])) + if task is None: + # Both off the `runs` row, minted before any of these jobs + # existed. The first job's label is the fallback, for a run + # from before the column and for a job queued on its own. + task = tasks[(row["cascade_id"], row["task_id"])] = TaskProgress( + id=row["task_id"], name=row["task_name"] or "", + label=row["task_label"] or row["label"] or "") + run.tasks.append(task) + # One counter per state, by name. A state the model has no field + # for is skipped rather than raised: this is a drawing, and it + # must not be what takes the queue's whole snapshot down. + if row["state"] in TaskProgress.model_fields: + setattr(task, row["state"], getattr(task, row["state"]) + 1) + return list(runs.values()) + + def snapshot(self, recent: int = 30) -> JobsSnapshot: + """The whole queue as a UI shows it, in one read. + + Live and finished jobs in one statement, so a job that finishes while + this runs is in one list or the other and never in both. + """ + rows = [Job.from_row(r) for r in self._rows( + """SELECT * FROM jobs WHERE state IN ('queued','running') + UNION ALL + SELECT * FROM (SELECT * FROM jobs + WHERE state IN ('done','failed','cancelled') + ORDER BY id DESC LIMIT ?)""", + (recent,), + )] + live = sorted((j for j in rows if j.state in ("queued", "running")), + key=lambda j: j.id) + return JobsSnapshot( + cascades=self.cascades(), + running=[j for j in live if j.state == "running"], + queued=[j for j in live if j.state == "queued"], + recent=sorted((j for j in rows if j.state not in ("queued", "running")), + key=lambda j: j.id, reverse=True), + ) diff --git a/pipeline/task.py b/pipeline/task.py new file mode 100644 index 0000000..e7e4345 --- /dev/null +++ b/pipeline/task.py @@ -0,0 +1,495 @@ +"""A task: plain code that makes jobs, and is never a job itself. + + task = Task(kind="body_batch", name="body", finder=needs_a_body) + task.start(config, store, queue) # queued: batches, then one final job + task.run(config, store) # here and now, no queue at all + +A task is a business step — fetch every feed, give every article a body, tag +what has never been tagged. Given the config and the store it finds its own +work, cuts it into batch payloads, mints a run id for itself, queues one job +per payload under that id, and queues one final job behind them with +`after="done"` on all of them. Then it is over. Nothing of the task is alive +afterwards: only its jobs, and the callback the final job carries. + +**It runs in whoever calls it.** The HTTP request behind a button, a final +job's handler, a Claude Code session, a test. Finding the work is a query, and +a query is cheap enough to do there — making the preparation asynchronous +would buy a few milliseconds and cost a whole job kind, a payload, a failure +mode and a row in the history that says nothing a person wants to read. + +**A cascade is the same shape one level up**, and the pipeline does not tell +them apart: both are runs, both group jobs by an id, both are killed the same +way. The difference is a convention, and it is worth stating because nothing +enforces it: + + a task queues everything it wants at once, and only its final job + calls back. + a cascade is where the breakpoints are: it queues the next task only + once the last one is over. + +**The callback is a name, not a function.** Nothing in a table can hold a +function, and the final job's payload is a table row — so the name travels +and the worker that runs the job resolves it. `module:function`, imported +where it is used. A registry would work too, and was the other candidate; the +dotted name wins because it carries its own import, where a registry has to be +filled before the worker looks in it, which is exactly the import-order trap +`JOBS` already has to be careful about. It also reads as documentation in the +job's payload: `newsdesk.jobs.cascade:step` says where to go and look. +""" + +from __future__ import annotations + +import importlib +import logging +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass, field, replace +from typing import Any, Protocol + +from pipeline.context import NULL, Cancelled, RunContext +from pipeline.registry import JOBS, job +from pipeline.table import on_its_own + +log = logging.getLogger(__name__) + +#: The job kind every task ends with. One kind for every task there will ever +#: be, because what it does is the same every time: read how the batches went +#: and call the name in its payload. +FINAL = "advance" + +#: A payload for one batch: the keyword arguments its job will be called with. +Payload = dict[str, Any] + + +@dataclass(frozen=True, slots=True) +class Bound: + """How many times one batch may be attempted, and how long it waits. + + The defaults are one attempt and no wait, which is every batch of every + task that existed before this: a job with no policy fails at its first + exception exactly as it always did. + + It exists because a bound is sometimes a property of the *batch* rather + than of the job kind — a plugin declares how long to wait before asking + its publisher again, so two batches of `body_batch` cut from two outlets + carry two different numbers. The values travel onto the row (see + `Jobs.enqueue`), because the attempt that has to decide may be running in + another process, after a restart, under a plugin file that has since + changed. + """ + + #: Counting the first attempt, so 1 is "run it once". + max_attempts: int = 1 + #: Seconds before the second attempt. + delay: float = 0.0 + #: Multiplied into the delay once per attempt already made. + backoff: float = 1.0 + + +@dataclass(frozen=True, slots=True) +class Batch: + """One batch of a task, when its bound is its own. + + A finder may return these instead of plain payloads. Two shapes rather + than one because the second is what nearly every task wants and reads as + what it is — `[{"uids": [...]}, ...]` is a list of batches, and wrapping + each of them to say "and the default bound" would be ceremony on every + task to serve the one that needs it. + """ + + payload: Payload + bound: Bound = Bound() + #: What to call this one batch on screen, or "" to be called whatever its + #: job kind is called. The kind's label is the same string on every batch + #: of the task — nine rows reading "Tagging a batch" — because a label is + #: declared once, at registration, by something that cannot see how the + #: work was cut. Only the finder knows that, so this is where a batch that + #: can say which batch it is says so. See `desk/modeltask.py`. + label: str = "" + + +#: Given the app's config and store, what there is to do — one payload per +#: batch, or one `Batch` where the bound is the batch's own. An empty list is +#: a legitimate answer and the common one: a task with nothing waiting costs +#: one query, queues no batches, and its final job runs immediately and moves +#: the cascade on. +Finder = Callable[[Any, Any], list[Any]] + + +def _batch(found: Payload | Batch) -> Batch: + """One thing a finder returned, as a batch. A bare payload gets the + default bound, which is the behaviour it had before bounds existed.""" + return found if isinstance(found, Batch) else Batch(found) + + +#: name -> how to build that task, standing on its own. Filled by the app the +#: way `JOBS` is, and for the same reason: a task is as much a thing to start +#: as a job is, so whatever offers the one — `/api/jobs`, the Jobs view, a +#: script — has to be able to find the other by name. +TASKS: dict[str, Callable[[], Task]] = {} + + +class Sink(Protocol): + """Where a task puts the jobs it makes. + + Two of them: `JobQueue`, when a task is started from a request or a + handler, and nothing else. It is a protocol rather than the class so that + `Task` does not import the queue — the dependency runs the other way, and a + test can hand it a list. + """ + + def new_run(self, kind: str, name: str = "", label: str = "") -> int: ... + + def cancelled(self, run_id: int) -> bool: ... + + def enqueue(self, kind: str, payload: Payload | None = None, *, + label: str = "", depends_on: Sequence[int] = (), after: str = "ok", + cascade_id: int = 0, task_id: int = 0, + max_attempts: int | None = None, + retry_delay: float | None = None, + retry_backoff: float | None = None) -> Any: ... + + +@dataclass(frozen=True, slots=True) +class Task: + """One business step: what a batch of it is, how to find them, who hears. + + Knows nothing about what the work *is* — that is the batch job's, and the + finder's. This is the shape they are held in. + """ + + #: The job kind one batch runs as. A task of one batch names an ordinary + #: job kind (`pull`); a task of many names the batch kind (`body_batch`). + kind: str + #: How the work is found. Called once, in the caller, before anything is + #: queued. + finder: Finder + #: What this task is — `body`, `tag`. Its identity: what a cascade's list + #: matches against, and what the `runs` row is minted under. The job kind + #: when nobody says. + name: str = "" + #: How to say that on screen: "Fetching what the feeds left out". Written + #: onto the `runs` row, because a task is not a job and has no handler to + #: carry a label. The name when nobody says. + label: str = "" + #: `module:function` for the final job to call, or "" for a task that + #: reports to nobody — one started on its own, from a button or a script. + callback: str = "" + #: `module:function` the final job runs first, or "" for a task whose + #: batches finish their own work. A name for the same reason the callback + #: is one: it is a row in a table. + #: + #: The seam exists for a decision no batch can make on its own. A batch is + #: one of forty and cannot see the other thirty-nine, so anything that has + #: to hold across the whole run — a cap on what the run may add, a merge of + #: what several batches each asked for — has exactly one place it can be + #: made correctly, and that is the job that runs when they are all over. + #: It belongs to the task rather than to the callback because it *is* the + #: task's work: a task started on its own has no callback and still has to + #: settle. See `newsdesk/jobs/tagging.py` for the one that does. + settle: str = "" + #: What the callback is handed alongside the tally. Fixed when the task is + #: made, and stored in the final job's payload, so it survives a restart + #: and a worker in another process. + carries: dict[str, Any] = field(default_factory=dict) + #: How many items a payload is worth, for the direct path's progress bar. + #: Only the direct path asks: as a job a batch is on its own and reports + #: its own rows, which is what the boxes on screen are drawn from. + size: Callable[[Payload], int] | None = None + #: How the direct path's batch results become one result. The queue never + #: calls it — there each batch is its own job with its own row. + fold: Callable[[Any, Any, list[Any]], dict[str, Any]] | None = None + + @property + def named(self) -> str: + return self.name or self.kind + + @property + def called(self) -> str: + return self.label or self.named + + # ------------------------------------------------------------- queued + + def start(self, config: Any, store: Any, sink: Sink, *, + cascade_id: int = 0) -> int: + """Find the work, queue it, and be over. Returns the task's run id. + + The id is minted before a single job exists, because a task is a thing + before its jobs are — it is what decides how many there will be — and + naming it after whichever job it happened to queue first would be + describing it by one of its own consequences. + + With no cascade given it begins one: something asked for this on its + own, so it is a cascade of one task. Named apart from any chain of the + same name — `pull on its own`, not `pull` — because pressing Tag is + not starting the chain that ends in tagging, and whoever asks whether + that chain is going has to be told no. See `on_its_own`. + + Returns 0 when the cascade it belongs to has been killed. Nothing is + told; it tries, and the table says no. `Jobs.enqueue` is where that is + actually enforced, across processes and after a restart; this only + saves minting a run for jobs that would all be refused. + """ + task_id, _ = queue_batches( + sink, self.kind, lambda: self.finder(config, store), + name=self.named, label=self.called, then=self.callback, + settle=self.settle, carries=self.carries, cascade_id=cascade_id) + return task_id + + # ------------------------------------------------------------- direct + + def run(self, config: Any, store: Any, *, + context: RunContext = NULL) -> dict[str, Any]: + """Do the batches here and now, in order, with no queue at all. + + What a Claude Code session, a notebook and a test get. The same batch + handler the worker would call, with the same payloads from the same + finder — there is no second implementation to drift. + + A batch that raises is kept and the rest carry on, which is what the + queue does with eight independent jobs of which one fails. Unless + *nothing* got through: then the first exception is raised, because a + run that returned "0 done" with the explanation only in the log would + be a failure reported as a success. A `Cancelled` travels either way — + it is an instruction from the person at the keyboard, not a batch that + did not land. + + A batch's `Bound` is ignored here, and that is not an omission: a + bound is a number the *queue* enforces by putting a row back with a + time on it, and there is no queue and no row on this path. Asked for + directly, a batch is attempted once and its failure is returned to the + caller, who is standing right there. + """ + spec = JOBS.get(self.kind) + if spec is None: + raise KeyError( + f"{self.named!r} makes jobs of kind {self.kind!r}, which " + f"nothing registers (have: {', '.join(sorted(JOBS)) or 'none'})") + payloads = [_batch(found).payload for found in self.finder(config, store)] + sizes = [self._size(p) for p in payloads] + total = sum(sizes) + + results: list[Any] = [] + failure: Exception | None = None + at = 0 + for payload, size in zip(payloads, sizes, strict=True): + context.check() + extra = {"context": self._placed(context, at, total, size)} \ + if spec.wants_context else {} + try: + results.append(dict(spec.fn(config, store, **extra, **payload) or {})) + except Cancelled: + raise + except Exception as exc: + log.exception("%s: a batch of %s did not land", self.named, self.kind) + results.append(exc) + failure = failure or exc + finally: + # Position through the backlog, not what was answered: a batch + # the model skipped half of still moved the run that far, and a + # bar that goes backwards is worse than one that is approximate. + at += size + if failure is not None and not any(isinstance(r, dict) for r in results): + raise failure + if self.fold is not None: + return self.fold(config, store, results) + return {"batches": sum(1 for r in results if isinstance(r, dict)), + "failed": sum(1 for r in results if isinstance(r, Exception)), + "results": [r for r in results if isinstance(r, dict)]} + + def _size(self, payload: Payload) -> int: + return int(self.size(payload)) if self.size is not None else 0 + + def _placed(self, context: RunContext, at: int, total: int, + size: int) -> RunContext: + """The batch's context, with its progress shifted into the whole run. + + A batch run as a job is on its own and counts its own rows. Run here it + is one of several in a row, and a bar that restarts per batch would be + a bar that goes backwards five times. + """ + if not total: + return context + return replace(context, on_progress=lambda done, _t, message: + context.progress(at + min(done, size), total, message)) + + +# ------------------------------------------------------------- the final job + +@dataclass(frozen=True, slots=True) +class Tally: + """How a task's batches ended, as the callback is handed it. + + Read off the jobs at the moment the final job runs, never counted up as + they landed — so there is nothing to increment and nothing to lose to a + restart half way through. + """ + + done: int = 0 + failed: int = 0 + cancelled: int = 0 + #: Whether the task itself was killed, as against however many of its + #: batches happen to have ended cancelled. The counts cannot say: one + #: batch killed by hand and every batch killed by the task going down look + #: identical there and mean opposite things — drop this batch, or skip + #: this whole step. Only the run's row knows, which is why it is asked. + killed: bool = False + #: What each batch returned, in the order they were queued. + results: tuple[dict[str, Any], ...] = () + + @property + def jobs(self) -> int: + return self.done + self.failed + self.cancelled + + @classmethod + def of(cls, outcomes: Sequence[Any], *, killed: bool = False) -> Tally: + states = [getattr(j, "state", "") for j in outcomes] + return cls( + done=states.count("done"), + failed=states.count("failed"), + cancelled=states.count("cancelled"), + killed=killed, + results=tuple(dict(getattr(j, "result", {}) or {}) for j in outcomes), + ) + + def as_dict(self) -> dict[str, Any]: + return {"jobs": self.jobs, "done": self.done, "failed": self.failed, + "cancelled": self.cancelled, "killed": self.killed} + + +def queue_batches(sink: Sink, kind: str, + found: Callable[[], Iterable[Payload | Batch]], *, + name: str = "", label: str = "", then: str = "", + settle: str = "", carries: dict[str, Any] | None = None, + cascade_id: int = 0, + max_attempts: int | None = None, + retry_delay: float | None = None, + retry_backoff: float | None = None) -> tuple[int, tuple[int, ...]]: + """Mint a task, queue one job per batch, and queue the job that reports. + + The one implementation behind both `Task.start` and `fan_out`. Returns the + task's run id and the children's job ids; `(0, ())` when the cascade it + would join has been killed. + + `found` is called only after the runs exist, which is the order a task has + always had: a task is a thing before its jobs are, since it is what decides + how many there will be. + + A plain payload takes the kind's own retry policy, or the uniform one + passed here. A `Batch` carries its own `Bound`, which wins — that is what a + batch cut from one plugin profile needs when the profile declares how long + to leave its publisher alone. + + With no cascade given it begins one: something asked for this on its own, + so it is a cascade of one task, named apart from any chain of the same name + (`pull on its own`, not `pull`). See `on_its_own`. + """ + if cascade_id and sink.cancelled(cascade_id): + return 0, () + name = name or kind + label = label or name + cascade_id = cascade_id or sink.new_run("cascade", on_its_own(name), label) + task_id = sink.new_run("task", name, label) + + ids: list[int] = [] + for item in found(): + if isinstance(item, Batch): + queued = sink.enqueue(kind, dict(item.payload), label=item.label, + cascade_id=cascade_id, task_id=task_id, + max_attempts=item.bound.max_attempts, + retry_delay=item.bound.delay, + retry_backoff=item.bound.backoff) + else: + queued = sink.enqueue(kind, dict(item), + cascade_id=cascade_id, task_id=task_id, + max_attempts=max_attempts, + retry_delay=retry_delay, + retry_backoff=retry_backoff) + if queued is not None: + ids.append(queued.id) + # Behind all of them, and part of the same task — killing the task spares + # it, because reporting is not work and "skip this task" is not "make it + # never end". That holds where the kill landed mid-loop too: `Jobs.enqueue` + # carves `after='done'` out of a task's kill, so this row is written and + # the task ends by saying it was killed. None here is the *cascade* going + # down, which takes the report with it by design: there is nothing left + # to report to. See `Jobs.cancel_run`. + sink.enqueue(FINAL, {"callback": then, "settle": settle, + "payload": dict(carries or {})}, + depends_on=ids, after="done", + cascade_id=cascade_id, task_id=task_id) + return task_id, tuple(ids) + + +def fan_out(sink: Sink, kind: str, payloads: Sequence[Payload | Batch], *, + then: str = "", settle: str = "", label: str = "", + carries: dict[str, Any] | None = None, + cascade_id: int = 0, + max_attempts: int | None = None, + retry_delay: float | None = None, + retry_backoff: float | None = None) -> tuple[int, ...]: + """Queue these payloads as jobs, and one job behind them that reports. + + What `RunContext.fan_out` reaches. `Task` is the same call with a finder in + front, which is why a handler whose whole body is one of these *is* a task. + The children go into a run of their own inside the caller's cascade, so the + callback can fan out again and the result is a chain rather than a nest. + + Returns the children's ids. The reporting job is not among them — it is + behind them by construction, and a caller that waited on it would be + holding a worker open to do so. + """ + _, ids = queue_batches( + sink, kind, lambda: payloads, label=label, then=then, settle=settle, + carries=carries, cascade_id=cascade_id, max_attempts=max_attempts, + retry_delay=retry_delay, retry_backoff=retry_backoff) + return ids + + +@job(FINAL, "Moving on") +def run_final(config: Any, store: Any, *, context: RunContext = NULL, + callback: str = "", settle: str = "", + payload: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """The one job every task ends with: tally the batches, call the name. + + It runs for the milliseconds it takes to read how the batches went and + hand that to whoever asked — nothing is alive between two tasks, and + nothing was waiting while the last one ran. + + `after="done"`, so it runs however the batches ended. That is the whole + reason the mode exists: a job whose purpose is to report that one of eight + failed cannot be held back by that failure. + + A task with no callback stops here, which is what being asked for on its + own means. + + The settle runs first and the callback second, because the settle is the + last of this task's own work and the callback is what comes after this + task: a cascade told to move on before the tags of the stage it is leaving + exist would be a stage reporting itself finished early. + + It asks whether its own task was killed, as well as counting how the + batches ended, because those are two different facts that arrive looking + the same. A task killed while its batches ran leaves them all `cancelled`, + and so does a person killing every batch by hand; the first is "skip this + step", the second is "drop this work". Only the run's row can tell them + apart, and the callback needs to be told which it is — the whole reason + this job is spared the kill is so that somebody hears. + """ + tally = Tally.of(context.waited_for, killed=context.killed()) + out = tally.as_dict() + if settle: + out |= resolve(settle)(config, store, tally, context=context) or {} + if not callback: + return out + return {**out, **(resolve(callback)(config, store, tally, + dict(payload or {}), context=context) or {})} + + +def resolve(name: str) -> Callable[..., dict[str, Any] | None]: + """`module:function` to the function. See this module's docstring for why + a callback, or a settle, is a name at all.""" + module, _, attr = name.partition(":") + if not module or not attr: + raise ValueError(f"a callback is 'module:function', not {name!r}") + return getattr(importlib.import_module(module), attr) diff --git a/pipeline/tests/__init__.py b/pipeline/tests/__init__.py new file mode 100644 index 0000000..a637904 --- /dev/null +++ b/pipeline/tests/__init__.py @@ -0,0 +1,2 @@ +"""The pipeline's own tests. They travel with the folder, so a copy can +prove itself where it lands: `python -m pytest pipeline/tests`.""" diff --git a/pipeline/tests/isolated_handlers.py b/pipeline/tests/isolated_handlers.py new file mode 100644 index 0000000..71e3f4a --- /dev/null +++ b/pipeline/tests/isolated_handlers.py @@ -0,0 +1,74 @@ +"""Handlers for the isolation tests, in a module a child process can import. + +Not a test file — pytest does not collect it — and deliberately separate from +the tests that use it, because the point of `isolate=True` is that the work +runs in a *different process*. That process gets nothing from this one: it +imports `pipeline.tests.isolated_handlers` itself, by the name in +`--bootstrap`, which is exactly what an app's own bootstrap has to do. + +It also documents the whole of the contract, which is two lines: import the +modules that fill `JOBS`, and return the `(config, store)` pair the handlers +take. Here both are dicts, since nothing in the pipeline reads them. +""" + +from __future__ import annotations + +import os +import time +from typing import Any + +from pipeline import NULL, RunContext, job + + +@job("iso_ok", "Working in its own process", isolate=True) +def iso_ok(config: Any, store: Any, *, context: RunContext = NULL, + n: int = 0) -> dict[str, Any]: + """Reports which process it ran in, so a test can prove it was not ours.""" + context.progress(n, n, f"counting to {n}") + return {"doubled": n * 2, "pid": os.getpid()} + + +@job("iso_raises", "Failing in its own process", isolate=True) +def iso_raises(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]: + raise RuntimeError("the model said no") + + +@job("iso_slow", "Taking too long on purpose", isolate=True, timeout=1.0) +def iso_slow(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]: + """Uninterruptible on purpose: no `check()`, so only a kill ends it. + + `time.sleep` stands in for `sklearn.fit` — the difference that matters is + not what it does but that nothing in it ever asks whether to stop, which + is the case a thread has no answer for. + """ + time.sleep(120) + return {"never": True} + + +def bootstrap() -> tuple[Any, Any]: + """What `--bootstrap` names. Importing this module registered the jobs.""" + return {"config": "rebuilt in the child"}, {"store": "rebuilt in the child"} + + +# ---------------------------------------------------- two workers, one file + +@job("mp_mark", "Leaving a mark") +def mp_mark(config: Any, store: Any, *, context: RunContext = NULL, + i: int = 0) -> dict[str, Any]: + """Leave a file named after this job and the process that ran it. + + A file per run rather than a line in a shared one, because creating a + uniquely named file is atomic on every platform and appending from two + processes is not. A job run twice leaves two files with the same number. + """ + import uuid + from pathlib import Path + + (Path(store) / f"{i}-{os.getpid()}-{uuid.uuid4().hex}").touch() + time.sleep(0.02) # long enough that both workers get a share + return {"i": i} + + +def marking(marks: str) -> tuple[Any, Any]: + """The bootstrap `pipeline.worker` is given: the marks directory is the store.""" + return {}, marks diff --git a/pipeline/tests/scheduled_handlers.py b/pipeline/tests/scheduled_handlers.py new file mode 100644 index 0000000..c34b7d1 --- /dev/null +++ b/pipeline/tests/scheduled_handlers.py @@ -0,0 +1,39 @@ +"""A job and a declared schedule, in a module a worker process can import. + +Not a test file — pytest does not collect it — and separate from the tests that +use it for the same reason `isolated_handlers.py` is: the process under test is +a *different* process, and it gets nothing from this one. It imports this by +the name in `--bootstrap`, which fills `JOBS` and `SCHEDULES` exactly as an +app's own jobs module does. + +Separate from `isolated_handlers.py` as well, because importing a module that +calls `declare` fills a global: a test suite that imported this one would find +its own schedule registry non-empty before it had declared anything. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from pipeline import NULL, RunContext, declare, job + + +@job("sw_mark", "Leaving a mark") +def sw_mark(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]: + """Touch a file in the store directory, so the parent process can see from + outside whether this ran at all.""" + (Path(store) / "fired").touch() + return {"fired": True} + + +#: The declaration itself is what one half of the worker test is about: a +#: worker with no web app around it reconciles declared defaults like any +#: other, so a machine running only `python -m pipeline.worker` still gets +#: them. +declare("marker", "sw_mark", "every 1h", label="Leaving a mark, hourly") + + +def marking(marks: str) -> tuple[Any, Any]: + """The bootstrap `pipeline.worker` is given: the marks directory is the store.""" + return {}, marks diff --git a/pipeline/tests/test_connect.py b/pipeline/tests/test_connect.py new file mode 100644 index 0000000..311abe2 --- /dev/null +++ b/pipeline/tests/test_connect.py @@ -0,0 +1,113 @@ +"""The web face of the queue, from an app that has written nothing but `connect`. + + python -m pytest pipeline/tests/test_web.py -q +""" + +from __future__ import annotations + +import sys +import tempfile +import time +from contextlib import asynccontextmanager +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +pytest.importorskip("fastapi") + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from pipeline import NULL, Jobs, RunContext, job +from pipeline.web import connect, jobs_router + + +@job("w_hello", "Saying hello") +def w_hello(config, store, *, context: RunContext = NULL, name: str = "") -> dict: + context.progress(1, 1, f"greeting {name}") + return {"said": f"hello {name}", "config": config["app"]} + + +@pytest.fixture +def jobs(): + with tempfile.TemporaryDirectory() as tmp: + yield Jobs(Path(tmp) / "web.db") + + +def settled(client: TestClient, job_id: int, timeout: float = 10.0) -> dict: + """The job as the HTTP API reports it, once it has finished.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + snap = client.get("/api/jobs").json() + for row in snap["recent"]: + if row["id"] == job_id: + return row + time.sleep(0.05) + raise AssertionError(f"job {job_id} did not finish") + + +def test_connect_is_the_whole_of_the_wiring(jobs): + """Routes, socket and a running worker, from one call. + + The app below has nothing else: no lifespan, no broadcaster, no router of + its own. A job enqueued over HTTP runs to `done` because the worker came up + with the app, and the socket hands a new tab the current state on connect. + """ + app = FastAPI() + queue = connect(app, jobs, {"app": "tiny"}, None) + assert app.state.queue is queue + + with TestClient(app) as client: + made = client.post("/api/jobs", json={"kind": "w_hello", + "payload": {"name": "you"}}) + assert made.status_code == 202, made.text + ended = settled(client, made.json()["id"]) + assert ended["state"] == "done" + assert ended["result"] == {"said": "hello you", "config": "tiny"} + + with client.websocket_connect("/ws/jobs") as socket: + hello = socket.receive_json() + assert hello["type"] == "jobs" + assert any(row["id"] == ended["id"] for row in hello["recent"]) + + assert queue._task is None or queue._task.done(), "and it stopped with the app" + + +def test_connect_keeps_the_apps_own_lifespan(jobs): + """An app with startup code of its own keeps it, and the worker is already + up when that code runs — so startup code may enqueue.""" + seen: list[str] = [] + + @asynccontextmanager + async def lifespan(app: FastAPI): + seen.append("app started") + app.state.queue.enqueue("w_hello", {"name": "from startup"}) + yield + seen.append("app stopped") + + app = FastAPI(lifespan=lifespan) + connect(app, jobs, {"app": "tiny"}, None) + + with TestClient(app) as client: + # By id: the snapshot reads live and recent jobs separately, so a job + # that finishes in between is listed in both. + snap = client.get("/api/jobs").json() + (job_id,) = {r["id"] for r in snap["queued"] + snap["running"] + + snap["recent"] if r["kind"] == "w_hello"} + assert settled(client, job_id)["state"] == "done" + assert seen == ["app started", "app stopped"] + + +def test_the_element_is_served_by_the_queues_own_router(): + """Two lines in a template and a new app has a job list. Served from a + route, because a router cannot mount a static directory and needing the + app to mount one is the setup the element exists to remove.""" + app = FastAPI() + app.include_router(jobs_router) + answer = TestClient(app).get("/_jobs/jobs.js") + + assert answer.status_code == 200 + assert answer.headers["content-type"].startswith("text/javascript") + assert 'customElements.define("desk-jobs"' in answer.text diff --git a/pipeline/tests/test_housekeeping.py b/pipeline/tests/test_housekeeping.py new file mode 100644 index 0000000..8b94c10 --- /dev/null +++ b/pipeline/tests/test_housekeeping.py @@ -0,0 +1,209 @@ +"""What keeps a queue healthy for months: retention, bounded results, whose +retry policy wins, draining, and two real processes on one file. + + python -m pytest pipeline/tests/test_housekeeping.py -q +""" + +from __future__ import annotations + +import os +import sqlite3 +import subprocess +import sys +import tempfile +from contextlib import closing +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from pipeline import NULL, Batch, Bound, JobQueue, Jobs, Task, job +from pipeline.models import Job + +ROOT = Path(__file__).resolve().parents[2] + + +@job("h_plain", "A job with a policy of its own", max_attempts=3, retry_delay=7) +def h_plain(config, store, *, context=NULL, i: int = 0) -> dict: + return {"i": i} + + +@job("h_big", "Returning far too much", max_attempts=3) +def h_big(config, store, *, context=NULL) -> dict: + return {"rows": ["x" * 100] * 2000, "count": 2000} + + +@pytest.fixture +def jobs(): + with tempfile.TemporaryDirectory() as tmp: + yield Jobs(Path(tmp) / "h.db") + + +def got(found: Job | None) -> Job: + assert found is not None + return found + + +def age(jobs: Jobs, job_id: int, days: float) -> None: + """Pretend a job finished `days` ago, and its runs were minted then too.""" + when = (datetime.now(UTC) - timedelta(days=days)).isoformat() + with closing(sqlite3.connect(jobs.path)) as conn, conn: + conn.execute("UPDATE jobs SET finished_at=? WHERE id=?", (when, job_id)) + conn.execute("UPDATE runs SET created_at=? WHERE id IN " + "(SELECT cascade_id FROM jobs WHERE id=? UNION " + " SELECT task_id FROM jobs WHERE id=?)", (when, job_id, job_id)) + + +def runs(jobs: Jobs) -> int: + with closing(sqlite3.connect(jobs.path)) as conn: + return conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0] + + +# ---------------------------------------------------------------- retention + +def test_purge_forgets_old_history_but_not_the_last_of_each_kind(jobs): + """Without this both tables grow for as long as the app runs. + + What survives an old cutoff: anything unfinished, and the newest ending of + each kind in each state — because `last(kind)` is how an app says "last + pulled five weeks ago", and deleting it would make that "never". + """ + old = [got(jobs.enqueue("h_plain")) for _ in range(3)] + for queued in old: + jobs.finish(queued.id, "done") + age(jobs, queued.id, days=40) + waiting = got(jobs.enqueue("h_plain")) # never finished + + gone = jobs.purge(30 * 86400) + + assert gone == 2, "two of the three old ones went" + assert got(jobs.last("h_plain")).id == old[-1].id, "the newest stayed" + assert got(jobs.get(waiting.id)).state == "queued", "unfinished work is not history" + + +def test_purge_leaves_recent_history_alone(jobs): + recent = got(jobs.enqueue("h_plain")) + jobs.finish(recent.id, "done") + before = runs(jobs) + + assert jobs.purge(30 * 86400) == 0 + assert jobs.get(recent.id) is not None + assert runs(jobs) == before + + +def test_purge_drops_runs_nothing_points_at_any_more(jobs): + first = got(jobs.enqueue("h_plain")) + second = got(jobs.enqueue("h_plain")) + for queued in (first, second): + jobs.finish(queued.id, "done") + age(jobs, queued.id, days=40) + before = runs(jobs) + + jobs.purge(30 * 86400) + + assert runs(jobs) < before, "the purged job's cascade and task went with it" + assert jobs.run(second.task_id) is not None, "the kept job's runs did not" + + +# ----------------------------------------------------------- bounded results + +def test_a_result_too_large_fails_the_job_loudly_and_once(jobs): + """Past `MAX_RESULT` the job fails rather than losing data quietly: the row + says why and how to fix it, a notice goes to whoever is watching, and it is + not retried — the same handler would return the same result.""" + said: list = [] + + async def on_notice(notice) -> None: + said.append(notice) + + queued = got(jobs.enqueue("h_big", max_attempts=3)) + JobQueue(jobs, None, None, on_notice=on_notice).drain(timeout=10) + + ended = got(jobs.get(queued.id)) + assert ended.state == "failed" + assert "ResultTooLarge" in ended.error and "store" in ended.error, ended.error + assert "count, rows" in ended.error, "it names what was returned" + assert ended.result == {}, "nothing of it was written" + assert ended.attempt == 1, "and it was not tried again" + assert [n.code for n in said] == ["job.failed"], "and somebody was told" + + +def test_a_result_at_the_limit_is_kept_whole(jobs): + """The limit is a line, not a tax: everything under it is stored as is.""" + queued = got(jobs.enqueue("h_plain", payload={"i": 1})) + JobQueue(jobs, None, None).drain(timeout=10) + assert got(jobs.get(queued.id)).result == {"i": 1} + + +# -------------------------------------------------------- whose policy wins + +def test_a_task_with_plain_payloads_keeps_the_kinds_own_retry_policy(jobs): + """`@job(max_attempts=3)` has to mean something when a task queues it. + + It used not to: a plain payload was given the default bound explicitly, + which silently overrode whatever the kind declared. + """ + queue = JobQueue(jobs, None, None) + Task(kind="h_plain", finder=lambda c, s: [{"i": 1}]).start(None, None, queue) + + (child,) = [j for j in jobs.snapshot().queued if j.kind == "h_plain"] + assert (child.max_attempts, child.retry_delay) == (3, 7.0) + + +def test_a_batch_with_its_own_bound_overrides_the_kind(jobs): + """The other half: a batch cut from one plugin profile carries that + profile's policy, and that wins, because the outlet knows its publisher.""" + queue = JobQueue(jobs, None, None) + Task(kind="h_plain", finder=lambda c, s: [ + Batch({"i": 1}, Bound(max_attempts=5, delay=60))]).start(None, None, queue) + + (child,) = [j for j in jobs.snapshot().queued if j.kind == "h_plain"] + assert (child.max_attempts, child.retry_delay) == (5, 60.0) + + +# ------------------------------------------------------------------ draining + +def test_drain_runs_everything_queued_and_stops(jobs): + """The batch shape: a script or a cron entry with no web app around it.""" + ids = [got(jobs.enqueue("h_plain", payload={"i": i})).id for i in range(5)] + + assert JobQueue(jobs, None, None).drain(timeout=10) is True + assert {got(jobs.get(i)).state for i in ids} == {"done"} + + +# ------------------------------------------------- two processes, one file + +def test_two_worker_processes_never_run_the_same_job_twice(tmp_path): + """The claim the table-as-queue design rests on, tested the only way that + means anything: two separate `python -m pipeline.worker` processes draining + one file at once. Every job leaves a marker file named after itself and the + process that ran it; each job must have exactly one. + """ + db = tmp_path / "shared.db" + marks = tmp_path / "marks" + marks.mkdir() + jobs = Jobs(db) + for i in range(40): + jobs.enqueue("mp_mark", payload={"i": i}) + + env = dict(os.environ, PYTHONPATH=str(ROOT)) + argv = [sys.executable, "-m", "pipeline.worker", "--db", str(db), + "--bootstrap", "pipeline.tests.isolated_handlers:marking", + "--bootstrap-arg", str(marks), "--workers", "cpu=2", "--until-idle"] + workers = [subprocess.Popen(argv, env=env, cwd=ROOT, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + for _ in range(2)] + for worker in workers: + out, _ = worker.communicate(timeout=120) + assert worker.returncode == 0, out.decode(errors="replace")[-2000:] + + ran: dict[int, list[str]] = {} + for mark in marks.iterdir(): + i, pid = mark.name.split("-")[:2] + ran.setdefault(int(i), []).append(pid) + assert sorted(ran) == list(range(40)), "every job ran" + doubled = {i: pids for i, pids in ran.items() if len(pids) > 1} + assert not doubled, f"and none ran twice: {doubled}" + assert {j.state for j in jobs.snapshot().recent} == {"done"} diff --git a/pipeline/tests/test_isolation.py b/pipeline/tests/test_isolation.py new file mode 100644 index 0000000..6f75a60 --- /dev/null +++ b/pipeline/tests/test_isolation.py @@ -0,0 +1,320 @@ +"""Ending work that will not end itself, and how data moves between jobs. + +Four claims, and they fail independently: + + a deadline is honest `timeout` without `isolate` is refused at + registration rather than promising a kill a + thread cannot deliver. + a process can be killed an isolated job that never calls `check()` is + still over when its deadline passes, and its + row says so. + a chain reads, never writes `upstream()` is how B gets A's output — off A's + row, so B's own row was complete when written. + a fan-out is one call `fan_out` queues the children and the job that + reports on them, and an app never writes the + second one. + + python3 -m pytest tests/test_isolation.py -q +""" + +from __future__ import annotations + +import asyncio +import os +import sys +import tempfile +import time +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from pipeline import NULL, Cancelled, JobQueue, Jobs, RunContext, job, spawn +from pipeline.models import Job + +# Imported for the `@job` registrations, which the *parent* needs as much as +# the child does: the worker reads `isolate` and `timeout` off the spec before +# it decides to start a process at all. An app imports its handlers in every +# entry point for this reason; here that import is this line. +from pipeline.tests import isolated_handlers # noqa: F401 + +BOOTSTRAP = "pipeline.tests.isolated_handlers:bootstrap" + + +@job("t_child", "A child of a fan-out") +def t_child(config, store, *, context=NULL, i: int = 0) -> dict: + """Queued by `fan_out` and never run in these tests — what is under test is + the shape of the rows it made, which is the part an app never writes.""" + return {"i": i} + + +@pytest.fixture +def jobs(): + with tempfile.TemporaryDirectory() as tmp: + yield Jobs(Path(tmp) / "t.db") + + +@pytest.fixture +def child_path(monkeypatch): + """Put the folder holding `pipeline` where a child process can import it. + + What an installed app gets for free and a checkout does not: the child is a + fresh interpreter, and the checkout is on *our* path only because pytest + put it there. + """ + root = Path(__file__).resolve().parents[2] + monkeypatch.setenv("PYTHONPATH", str(root)) + + +def drain(queue: JobQueue, *, timeout: float = 20.0) -> None: + """Run the worker until nothing is queued, running, or held by this one.""" + + async def go() -> None: + await queue.start() + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + snap = queue.jobs.snapshot() + if not snap.running and not snap.queued and not queue._running: + break + await asyncio.sleep(0.02) + await queue.stop() + + asyncio.run(go()) + + +def got(found: Job | None) -> Job: + assert found is not None + return found + + +# --------------------------------------------------- a deadline is honest + +def test_a_timeout_without_isolation_is_refused(): + """The pipeline will not offer a kill it cannot perform. + + A deadline over a job in a thread could only mark the row failed and walk + away: the work would carry on holding the core it was killed for holding, + and two things would intend to write one row. So the pair is refused where + it is declared, with the two ways out named. + """ + with pytest.raises(ValueError) as refused: + @job("t_threaded_deadline", timeout=5.0) + def handler(config, store, *, context=NULL): + return {} + + said = str(refused.value) + assert "isolate=True" in said, "it names the way to get a real deadline" + assert "RunContext.spawn" in said, "and the cheaper way, for a command" + + +def test_isolation_without_a_timeout_is_fine(): + """Only the one combination is refused. An isolated job with no deadline + runs until it is done or killed by hand, which is every job the pipeline + has ever had.""" + + @job("t_isolated_no_deadline", isolate=True) + def handler(config, store, *, context=NULL): + return {} + + from pipeline import JOBS + assert JOBS["t_isolated_no_deadline"].isolate + assert JOBS["t_isolated_no_deadline"].timeout == 0.0 + + +def test_a_queue_with_no_bootstrap_says_so(jobs): + """An isolated job needs its app rebuilt in the child, and a queue that + cannot do it fails the job with the reason rather than with a TypeError + from somewhere inside a subprocess.""" + + @job("t_needs_bootstrap", isolate=True) + def handler(config, store, *, context=NULL): + return {} + + queued = got(jobs.enqueue("t_needs_bootstrap")) + drain(JobQueue(jobs, None, None)) # no bootstrap= + ended = got(jobs.get(queued.id)) + assert ended.state == "failed" + assert "bootstrap" in ended.error, f"it names what is missing: {ended.error}" + + +# ------------------------------------------------ a process can be killed + +def test_an_isolated_job_runs_somewhere_else_and_its_result_comes_back(jobs, child_path): + """The happy path, and the proof that it was a real process. + + Nothing was pickled to get there: the child was handed a database and a + row id and rebuilt the rest. Which is also why its progress is on the row + — it wrote it there itself, to the same table the snapshot reads. + """ + queued = got(jobs.enqueue("iso_ok", payload={"n": 21})) + drain(JobQueue(jobs, None, None, bootstrap=BOOTSTRAP)) + + ended = got(jobs.get(queued.id)) + assert ended.state == "done", ended.error + assert ended.result["doubled"] == 42, "the handler's return travelled back" + assert ended.result["pid"] != os.getpid(), "and it ran in another process" + assert ended.message == "counting to 21", "its progress reached our table" + + +def test_an_isolated_job_that_raises_fails_with_what_it_said(jobs, child_path): + """A failure across a process boundary reads like any other failure: the + row is `failed`, the error is the exception, and the retry bound applies + as it would to a handler that raised in a thread.""" + queued = got(jobs.enqueue("iso_raises")) + drain(JobQueue(jobs, None, None, bootstrap=BOOTSTRAP)) + + ended = got(jobs.get(queued.id)) + assert ended.state == "failed" + assert "the model said no" in ended.error, ended.error + + +def test_a_deadline_ends_a_job_that_never_looks_up(jobs, child_path): + """The claim the whole feature exists for. + + `iso_slow` sleeps for two minutes and never calls `check()`. In a thread + that is unkillable and the worker could only lie about having stopped it. + In a process of its own it is over in about a second, and the row says + why — which is what a person reading the job list needs. + """ + queued = got(jobs.enqueue("iso_slow")) + started = time.monotonic() + drain(JobQueue(jobs, None, None, bootstrap=BOOTSTRAP)) + took = time.monotonic() - started + + ended = got(jobs.get(queued.id)) + assert ended.state == "failed", f"it ended: {ended.state}" + assert took < 15, f"and on its deadline, not in its own time: {took:.1f}s" + + +# ------------------------------------------------------ spawn, on its own + +def test_spawn_kills_the_process_when_the_job_is_cancelled(): + """Kill, for a handler whose work is a command. + + Cooperative in name only from the handler's side: it is blocked in + `spawn`, not in a loop with a `check()` in it, and the child is killed + outright. What makes that safe is that the thread was only sleeping. + """ + asked = [0] + + def is_cancelled() -> bool: + asked[0] += 1 + return asked[0] > 2 # pressed, a moment in + + context = RunContext(is_cancelled=is_cancelled) + started = time.monotonic() + with pytest.raises(Cancelled): + context.spawn([sys.executable, "-c", "import time; time.sleep(120)"]) + assert time.monotonic() - started < 10, "it did not wait the two minutes" + + +def test_spawn_raises_on_its_deadline_and_leaves_nothing_running(): + """The deadline, and that the process is gone after it — not merely + abandoned, which is the failure mode the whole thing is against.""" + started = time.monotonic() + with pytest.raises(TimeoutError): + spawn([sys.executable, "-c", "import time; time.sleep(120)"], timeout=0.5) + assert time.monotonic() - started < 10 + + +def test_spawn_hands_back_the_exit_code_rather_than_raising(): + """A non-zero exit is not an error here. Plenty of commands say something + useful with one, and only the handler knows which.""" + done = spawn([sys.executable, "-c", "import sys; sys.exit(3)"]) + assert done.returncode == 3 + + +def test_spawn_survives_a_command_that_writes_more_than_a_pipe_holds(): + """Output goes to a file, not a `PIPE`, and this is why. + + A child that fills the pipe buffer blocks on the write while the poll loop + blocks on the read, and neither side is wrong — the job simply never ends. + A megabyte is far past any platform's buffer. + """ + done = spawn([sys.executable, "-c", "print('x' * 1000000)"], timeout=60) + assert done.returncode == 0 + assert len(done.stdout) >= 1000000 + + +# ---------------------------------------------- a chain reads, never writes + +def test_upstream_merges_what_the_parents_returned(): + """How B gets A's output: off A's row, not out of B's payload. + + Which is the property worth having. B's row was complete when it was + written, so it can carry a dedup key, a retry can re-run it from itself, + and no worker can claim it between an earlier job's write and its commit. + """ + a = Job(id=1, kind="a", created_at="", result={"html": "

hi

", "code": 200}) + b = Job(id=2, kind="b", created_at="", result={"code": 204}) + context = RunContext(waited_for=(a, b)) + + assert context.upstream() == {"html": "

hi

", "code": 204}, "later parents win" + + +def test_upstream_is_empty_under_null_so_a_chained_handler_is_still_callable(): + """The one line that keeps a chained handler an ordinary function. + + Every other handler in the pipeline can be called from a terminal. One + that read `waited_for[0].result` directly could not — it would raise + IndexError under `NULL` — so `upstream()` answers `{}` and the handler + falls back to the keyword it declared. + """ + assert NULL.upstream() == {} + + @job("t_chained") + def handler(config, store, *, context=NULL, html: str = "") -> dict: + return {"html": html or context.upstream().get("html", "")} + + assert handler(None, None, html="passed by hand") == {"html": "passed by hand"} + assert handler(None, None) == {"html": ""}, "and it does not blow up with neither" + + +# ----------------------------------------------------- a fan-out is one call + +def test_fan_out_queues_the_children_and_the_job_that_reports_on_them(jobs): + """One call, and the reporting job is not the app's to write. + + The children get a run of their own inside the caller's cascade, and the + report waits on all of them with `after='done'` — the mode that exists + precisely so a job reporting that one of three failed is not held back by + that failure. + """ + queue = JobQueue(jobs, None, None) + parent = got(jobs.enqueue("t_parent")) + context = queue._context(parent, [], []) + + children = context.fan_out( + "t_child", [{"i": 1}, {"i": 2}, {"i": 3}], then="mod:fn") + + assert len(children) == 3, "it answers with the children's ids" + queued = jobs.snapshot().queued + report = [j for j in queued if j.kind == "advance"] + assert len(report) == 1, "and queued exactly one job to report on them" + assert sorted(report[0].depends_on) == sorted(children) + assert report[0].after == "done" + assert report[0].payload["callback"] == "mod:fn" + assert {j.task_id for j in queued if j.id in children} == {report[0].task_id}, \ + "children and report are one run" + + +def test_fan_out_queues_nothing_under_null_and_says_so(): + """A handler called directly is being run by somebody who is standing + there, and the useful answer is to do its own work and report that nothing + was queued behind it — the same answer `task` gives, for the same reason.""" + assert NULL.fan_out("t_child", [{"i": 1}]) == () + + +def test_a_fan_out_carries_what_the_caller_puts_on_it(jobs): + """`carries` goes *around* the children to the callback: static input from + whoever asked, as against `Tally.results`, which is what the children + themselves returned.""" + queue = JobQueue(jobs, None, None) + parent = got(jobs.enqueue("t_parent")) + context = queue._context(parent, [], []) + + context.fan_out("t_child", [{"i": 1}], then="mod:fn", carries={"since": "monday"}) + + report = [j for j in jobs.snapshot().queued if j.kind == "advance"] + assert report[0].payload["payload"] == {"since": "monday"} diff --git a/pipeline/tests/test_notice_log.py b/pipeline/tests/test_notice_log.py new file mode 100644 index 0000000..0dd461e --- /dev/null +++ b/pipeline/tests/test_notice_log.py @@ -0,0 +1,134 @@ +"""The notice log: said once, kept until cleared, forgotten when it stops. + +A notice used to be a line that faded in twelve seconds in whichever tab was +open, and was raised as often as not while none was. These hold the three +things the table is for: a repeat is the same row and not news, a cleared row +makes the next one news again, and what nobody clears goes when it has stopped +being said rather than when it was first said. + + python -m pytest pipeline/tests/test_notice_log.py -q +""" + +from __future__ import annotations + +import sqlite3 +import sys +import tempfile +from contextlib import closing +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from pipeline import NULL, JobQueue, Jobs, Notice, RunContext, job +from pipeline.web import connect + + +@job("n_refused", "Meeting the same refusal as last time") +def n_refused(config, store, *, context: RunContext = NULL) -> dict: + context.notice(Notice(code="agent.refused", key="Worm hits npm", + text="No abstract: Worm hits npm", fix="Nothing to do.")) + return {} + + +@pytest.fixture +def jobs(): + with tempfile.TemporaryDirectory() as tmp: + yield Jobs(Path(tmp) / "n.db") + + +def refusal(text: str = "No abstract: Worm hits npm") -> Notice: + return Notice(code="agent.refused", key="Worm hits npm", text=text, + where="the tagging lines in the server log") + + +def test_the_same_notice_again_is_the_same_row_with_a_count(jobs): + first = jobs.record_notice(refusal()) + again = jobs.record_notice(refusal("No abstract, again: Worm hits npm")) + other = jobs.record_notice(Notice(code="agent.refused", key="Other", text="x")) + + assert (first.count, again.count) == (1, 2), "one is news, two is a repeat" + assert again.id == first.id and other.id != first.id, "keyed by code and key" + assert again.first_at == first.first_at and again.last_at >= first.last_at + assert again.text.startswith("No abstract, again"), "the newer wording is kept" + assert again.where == first.where, "and `where` survives its SQL column name" + assert [n.key for n in jobs.notices()] == ["Other", "Worm hits npm"] + + +def test_clearing_one_makes_the_next_occurrence_news_again(jobs): + said = jobs.record_notice(refusal()) + + assert jobs.clear_notice(said.id) + assert not jobs.clear_notice(said.id), "gone already is an answer, not an error" + assert jobs.record_notice(refusal()).count == 1 + + +def test_a_notice_is_forgotten_when_it_stops_being_said_not_when_it_began(jobs): + """By `last_at`. A feed that has failed nightly for a month is an old row + that was true an hour ago, and dropping it would make tonight's failure + read as the first.""" + chronic = jobs.record_notice(Notice(code="source.failed", key="rss", text="x")) + over = jobs.record_notice(Notice(code="source.failed", key="old", text="y")) + month = (datetime.now(UTC) - timedelta(days=30)).isoformat() + with closing(sqlite3.connect(jobs.path)) as conn, conn: + conn.execute("UPDATE notices SET first_at=? WHERE id=?", (month, chronic.id)) + conn.execute("UPDATE notices SET first_at=?, last_at=? WHERE id=?", + (month, month, over.id)) + + assert jobs.purge_notices(14 * 86400) == 1 + assert [n.key for n in jobs.notices()] == ["rss"] + + +def test_a_job_that_repeats_itself_is_heard_as_a_repeat(jobs): + """The whole path. Every notice still reaches the listener — what + interrupts a person is the interface's business — and the second carries + the count that says it is nothing new.""" + heard: list = [] + + async def on_notice(notice) -> None: + heard.append(notice) + + for _ in range(2): + jobs.enqueue("n_refused") + JobQueue(jobs, None, None, on_notice=on_notice).drain(timeout=10) + + assert [n.count for n in heard] == [1, 2] + assert len(jobs.notices()) == 1 + + +def test_a_log_that_cannot_be_written_does_not_cost_the_notice(jobs, monkeypatch): + heard: list = [] + + async def on_notice(notice) -> None: + heard.append(notice) + + def broken(notice): + raise sqlite3.OperationalError("database is locked") + + monkeypatch.setattr(jobs, "record_notice", broken) + done = jobs.enqueue("n_refused") + JobQueue(jobs, None, None, on_notice=on_notice).drain(timeout=10) + + assert [n.code for n in heard] == ["agent.refused"] + assert jobs.get(done.id).state == "done", "and the job that said it is unharmed" + + +def test_the_log_is_read_and_cleared_over_http(jobs): + app = FastAPI() + connect(app, jobs, None, None) + jobs.record_notice(refusal()) + jobs.record_notice(Notice(code="source.failed", key="rss", text="rss failed")) + + with TestClient(app) as client: + listed = client.get("/api/notices").json() + assert [n["code"] for n in listed] == ["source.failed", "agent.refused"] + assert listed[1]["where"] == "the tagging lines in the server log" + + assert client.delete(f"/api/notices/{listed[0]['id']}").json() == {"cleared": True} + assert client.delete(f"/api/notices/{listed[0]['id']}").json() == {"cleared": False} + assert client.delete("/api/notices").json() == {"cleared": 1} + assert client.get("/api/notices").json() == [] diff --git a/pipeline/tests/test_retries.py b/pipeline/tests/test_retries.py new file mode 100644 index 0000000..bb77b10 --- /dev/null +++ b/pipeline/tests/test_retries.py @@ -0,0 +1,355 @@ +"""Waiting to be tried again, without anything being alive while it waits. + +The claim under test is that a job between attempts costs a row and nothing +else. No worker, no thread, no lease, no timer — so the tests that matter here +are the ones that ask what *else* the queue can do while a job waits, and what +survives the process that scheduled the wait. + +Each of the four properties step 12 asks for is pinned on its own, because +they fail independently: a bound that is off by one still waits correctly, a +`run_after` that never reaches the table still retries in one process, and a +Kill that cannot reach a waiting row is invisible until somebody presses it. + + python3 -m pytest tests/test_retries.py -q +""" + +from __future__ import annotations + +import asyncio +import sqlite3 +import sys +import tempfile +import time +from contextlib import closing +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from pipeline.models import Job +from pipeline import NULL, JobQueue, Jobs, Retry, job + + +@pytest.fixture +def jobs(): + """The pipeline's table on a file of its own. No app, no store — nothing + here needs one, which is the point of the table owning its own schema.""" + with tempfile.TemporaryDirectory() as tmp: + yield Jobs(Path(tmp) / "t.db") + + +def made(queued: Job | None) -> Job: + """The job an enqueue made. `enqueue` answers None for a killed run, which + in these tests nothing is.""" + assert queued is not None + return queued + + +def got(found: Job | None) -> Job: + """The job a read must have found — a None here is a test failure.""" + assert found is not None + return found + + +def lease(jobs: Jobs, job_id: int) -> tuple[str | None, str | None]: + """The `started_at` and `seen_at` of one job: whether anybody has it.""" + with closing(sqlite3.connect(jobs.path)) as conn: + row = conn.execute( + "SELECT started_at, seen_at FROM jobs WHERE id=?", (job_id,)).fetchone() + return row[0], row[1] + + +def bring_forward(jobs: Jobs, job_id: int) -> None: + """Move a waiting job's next attempt into the past, as if its wait were + over. What the clock does, done in one statement so a test of an hour's + backoff does not take an hour.""" + with closing(sqlite3.connect(jobs.path)) as conn, conn: + conn.execute("UPDATE jobs SET run_after=? WHERE id=?", + ((datetime.now(UTC) - timedelta(hours=1)).isoformat(), job_id)) + + +def drain(queue: JobQueue, *, timeout: float = 5.0) -> None: + """Run the worker until the table is empty and this worker holds nothing. + + Both halves, because a job's last acts belong to its handler thread and + between them the table reads as idle while it is not. A job waiting for its + next attempt keeps this going, since it is still `queued` — which is the + behaviour the retry tests are leaning on. + """ + + async def go() -> None: + await queue.start() + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + snap = queue.jobs.snapshot() + if not snap.running and not snap.queued and not queue._running: + break + await asyncio.sleep(0.01) + await queue.stop() + + asyncio.run(go()) + + +# ------------------------------------------------------- not holding a place + +def test_a_failed_attempt_goes_back_to_the_queue_with_a_time_on_it(jobs): + """The resting state of a retry, and everything that is true of it.""" + queued = made(jobs.enqueue("r_wait", max_attempts=3, retry_delay=600)) + assert got(jobs.claim({})).attempt == 1, "claim counts the attempt" + + again = got(jobs.retry(queued.id, error="RuntimeError: the feed was down")) + assert again.state == "queued", "it is in the queue, not failed" + assert again.waiting and not again.exhausted + assert again.error == "RuntimeError: the feed was down", "it says why it waits" + assert lease(jobs, queued.id) == (None, None), "nobody holds it" + assert jobs.running() == [], "and nothing is running it" + assert [j.id for j in jobs.waiting()] == [queued.id] + assert [j.id for j in jobs.blocked()] == [queued.id] + + +def test_a_waiting_job_does_not_occupy_a_worker(jobs): + """The whole of why the wait is a column. One cpu worker, one job waiting + ten minutes for its next attempt, and the worker is free the entire time — + a handler that slept would have held it and stopped the queue dead.""" + waiting = made(jobs.enqueue("r_slot", max_attempts=2, retry_delay=600)) + assert jobs.claim({"cpu": 1}) is not None + jobs.retry(waiting.id, error="not this time") + + other = made(jobs.enqueue("r_other")) + assert got(jobs.claim({"cpu": 1})).id == other.id, "the slot was free" + assert got(jobs.get(waiting.id)).state == "queued" + + +def test_a_job_queued_for_later_is_not_claimed_before_its_time(jobs): + """`delay` on the enqueue itself: the durable form of "in ten minutes".""" + later = made(jobs.enqueue("r_later", delay=600)) + assert later.waiting + assert jobs.claim({}) is None + bring_forward(jobs, later.id) + assert got(jobs.claim({})).id == later.id + + +def test_what_waits_on_a_retrying_job_goes_on_waiting(jobs): + """A retry is still live work, so a dependent is held rather than settled. + + The alternative would be the worse half of both: a batch that is coming + back in a minute counted among the ones that did not land, and a final job + reporting on a task that is not over. + """ + batch = made(jobs.enqueue("r_batch", max_attempts=2, retry_delay=600)) + final = made(jobs.enqueue("r_final", depends_on=[batch.id], after="done")) + jobs.claim({}) + jobs.retry(batch.id, error="again in a minute") + + assert [j.id for j in jobs.blocked()] == [batch.id, final.id] + bring_forward(jobs, batch.id) + assert got(jobs.claim({})).id == batch.id, "the batch goes first" + jobs.finish(batch.id, "done") + assert got(jobs.claim({})).id == final.id + + +# --------------------------------------------------------------- the bounds + +def test_the_bound_counts_the_first_attempt(jobs): + """`max_attempts=3` is three runs of the handler, not three retries.""" + runs = [] + + @job("r_bounded", "Bounded", max_attempts=3) + def handler(config, store, *, context=NULL): + runs.append(1) + raise RuntimeError("this was never going to work") + + queue = JobQueue(jobs, None, None) + queued = made(queue.enqueue("r_bounded")) + drain(queue) + + ended = got(jobs.get(queued.id)) + assert len(runs) == 3, "it ran exactly as many times as it was allowed" + assert ended.state == "failed" and ended.attempt == 3 + assert ended.exhausted, "and says the policy ran out rather than merely failing" + + +def test_a_job_with_no_policy_fails_at_the_first_exception(jobs): + """The default is one attempt, which is every job queued before any of + this existed. Exhausted is not what that is: nothing was used up.""" + runs = [] + + @job("r_unbounded", "Unbounded") + def handler(config, store, *, context=NULL): + runs.append(1) + raise RuntimeError("no") + + queue = JobQueue(jobs, None, None) + queued = made(queue.enqueue("r_unbounded")) + drain(queue) + + ended = got(jobs.get(queued.id)) + assert len(runs) == 1 + assert ended.state == "failed" and not ended.exhausted + + +def test_an_attempt_that_lands_stops_the_retrying(jobs): + """The case the bound exists for: the second go works, and the job is done + rather than carrying a memory of having failed once.""" + runs = [] + + @job("r_second_time", "Second time", max_attempts=4) + def handler(config, store, *, context=NULL): + runs.append(1) + if len(runs) < 2: + raise RuntimeError("the first one never works") + return {"runs": len(runs)} + + queue = JobQueue(jobs, None, None) + queued = made(queue.enqueue("r_second_time")) + drain(queue) + + ended = got(jobs.get(queued.id)) + assert ended.state == "done" and ended.result == {"runs": 2} + assert ended.attempt == 2 and not ended.exhausted + + +def test_the_wait_lengthens_with_the_backoff(jobs): + """Doubling, read off the two stamps rather than waited out.""" + queued = made(jobs.enqueue("r_backoff", max_attempts=4, + retry_delay=100, retry_backoff=2.0)) + + waits = [] + for _ in range(3): + assert jobs.claim({}) is not None + again = got(jobs.retry(queued.id, error="again")) + waits.append( + datetime.fromisoformat(again.run_after) - datetime.now(UTC)) + bring_forward(jobs, queued.id) + + seconds = [w.total_seconds() for w in waits] + assert [round(s / 100) for s in seconds] == [1, 2, 4] + + +def test_a_server_hint_replaces_the_backoff(jobs): + """Honouring what the other end said, rather than guessing at it. + + The bound is not part of the bargain: a hint moves the next attempt, it + does not add one. + """ + queued = made(jobs.enqueue("r_hinted", max_attempts=3, retry_delay=600)) + jobs.claim({}) + again = got(jobs.retry(queued.id, error="429", delay=5)) + waited = datetime.fromisoformat(again.run_after) - datetime.now(UTC) + assert 0 < waited.total_seconds() <= 5 + + +def test_a_handler_can_name_its_own_wait(jobs): + """`Retry(after=...)` from inside the handler, which is where a + `Retry-After` header is actually read.""" + + @job("r_asked", "Asked to come back", max_attempts=3, retry_delay=600) + def handler(config, store, *, context=NULL): + raise Retry("rate limited", after=7) + + queue = JobQueue(jobs, None, None) + queued = made(queue.enqueue("r_asked")) + + async def one_pass() -> None: + await queue.start() + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + if got(jobs.get(queued.id)).waiting: + break + await asyncio.sleep(0.01) + await queue.stop() + + asyncio.run(one_pass()) + + waiting = got(jobs.get(queued.id)) + assert waiting.waiting and waiting.attempt == 1 + assert "rate limited" in waiting.error + waited = datetime.fromisoformat(waiting.run_after) - datetime.now(UTC) + assert 0 < waited.total_seconds() <= 7, "the handler's number, not the policy's" + + +def test_asking_to_come_back_does_not_extend_the_bound(jobs): + """A handler cannot grant itself a policy nobody declared.""" + runs = [] + + @job("r_asked_once", "Asked once") + def handler(config, store, *, context=NULL): + runs.append(1) + raise Retry("come back in a second", after=0) + + queue = JobQueue(jobs, None, None) + queued = made(queue.enqueue("r_asked_once")) + drain(queue) + + assert len(runs) == 1 + assert got(jobs.get(queued.id)).state == "failed" + + +# ------------------------------------------------------------- across a stop + +def test_the_next_attempt_survives_the_process_that_scheduled_it(jobs): + """The wait is a column, so it outlives everything that is not the file. + + A second `Jobs` on the same path is what another process sees, and what + this one sees after a restart: a queued row it will not take yet, and takes + the moment its time comes round. Nothing was handed over, because there + was nothing alive to hand. + """ + queued = made(jobs.enqueue("r_restart", max_attempts=3, retry_delay=600)) + jobs.claim({}) + jobs.retry(queued.id, error="the feed was down") + + restarted = Jobs(jobs.path) + assert restarted.claim({}) is None, "still waiting, on a table that never saw it fail" + assert [j.id for j in restarted.waiting()] == [queued.id] + + bring_forward(restarted, queued.id) + taken = got(restarted.claim({})) + assert taken.id == queued.id + assert taken.attempt == 2, "and it is the second attempt, not the first again" + + +# ------------------------------------------------------------- cancellation + +def test_cancelling_a_job_that_is_waiting_to_be_retried_ends_it(jobs): + """Kill on a waiting row. It is queued, so it ends here and now — there is + no worker to ask and nothing part-way through.""" + queued = made(jobs.enqueue("r_killable", max_attempts=3, retry_delay=600)) + jobs.claim({}) + jobs.retry(queued.id, error="not yet") + + ended = got(jobs.request_cancel(queued.id)) + assert ended.state == "cancelled" + assert jobs.waiting() == [] + + +def test_killing_the_run_takes_a_job_waiting_to_be_retried_with_it(jobs): + """The same for the task and cascade buttons: an hour's wait is not an + hour during which a run cannot be stopped.""" + cascade = jobs.new_run("cascade", "r_run") + task = jobs.new_run("task", "r_task") + queued = made(jobs.enqueue("r_in_a_run", max_attempts=3, retry_delay=600, + cascade_id=cascade, task_id=task)) + jobs.claim({}) + jobs.retry(queued.id, error="not yet") + + assert jobs.cancel_run(task) == 1 + assert got(jobs.get(queued.id)).state == "cancelled" + + +def test_a_kill_during_the_attempt_is_not_undone_by_the_retry(jobs): + """The one ordering that would make Kill a lie. + + Press it while the attempt runs, the attempt then fails, and a retry that + only looked at the bound would put the job straight back in the queue — + a button that stopped nothing and said it had. + """ + queued = made(jobs.enqueue("r_killed_mid", max_attempts=3, retry_delay=0)) + jobs.claim({}) + jobs.request_cancel(queued.id) + + ended = got(jobs.retry(queued.id, error="it failed as well")) + assert ended.state == "cancelled" + assert not ended.waiting diff --git a/pipeline/tests/test_schedule.py b/pipeline/tests/test_schedule.py new file mode 100644 index 0000000..70c947f --- /dev/null +++ b/pipeline/tests/test_schedule.py @@ -0,0 +1,701 @@ +"""The scheduler: the five wordings, the table, the firing, and the routes. + + python -m pytest pipeline/tests/test_schedule.py -q + +Three things here are worth knowing before changing anything. + +The daylight-saving tests are the reason `next_after` takes a zone at all, and +they are written around a real transition rather than a synthetic one, because +the bug they guard is precisely that `+ timedelta(days=1)` looks correct. + +The firing tests call `JobQueue._fire_due` directly. It is a private, and that +is deliberate: driving it through `start()` would mean asserting on a loop that +ticks on a clock, and a test that sleeps to make a scheduler fire is a test +that fails on a busy machine. One test does start a real queue, to prove the +tick is wired into the loop at all. + +`SCHEDULES` is a module-level registry, like `JOBS`, so anything here that +declares into it does so through the `declared` fixture, which puts it back. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +from datetime import UTC, datetime, timedelta +from pathlib import Path +from zoneinfo import ZoneInfo + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from pipeline import NULL, JobQueue, Jobs, RunContext, job +from pipeline.schedule import SCHEDULES, BadCadence, Cadence, declare + +ROOT = Path(__file__).resolve().parents[2] +COPENHAGEN = ZoneInfo("Europe/Copenhagen") + + +@job("s_tick", "A scheduled tick") +def s_tick(config, store, *, context: RunContext = NULL, note: str = "") -> dict: + store.append(note) + return {"note": note} + + +@job("s_solo", "One at a time", solo=True) +def s_solo(config, store, *, context: RunContext = NULL) -> dict: + return {} + + +@job("s_needs", "Needing a payload") +def s_needs(config, store, *, context: RunContext = NULL, target: str) -> dict: + return {"target": target} + + +@pytest.fixture +def jobs(): + with tempfile.TemporaryDirectory() as tmp: + yield Jobs(Path(tmp) / "s.db") + + +@pytest.fixture +def queue(jobs): + """A queue that is never started: the tests below fire it by hand. + + `tz="UTC"` so a cadence's next run is the same number wherever this runs. + """ + return JobQueue(jobs, {}, [], tz="UTC") + + +@pytest.fixture +def declared(): + """Borrow the declaration registry and give it back. + + `SCHEDULES` is global, like `JOBS`, and a test that left something in it + would give every later test in the session a schedule it never asked for. + """ + saved = dict(SCHEDULES) + SCHEDULES.clear() + yield SCHEDULES + SCHEDULES.clear() + SCHEDULES.update(saved) + + +def ago(seconds: float) -> str: + return (datetime.now(UTC) - timedelta(seconds=seconds)).isoformat() + + +# ------------------------------------------------------------ the grammar + +WORDINGS = [ + ("every 15m", "every 15m"), + ("every 6h", "every 6h"), + ("every 2d", "every 2d"), + ("daily at 06:00", "daily at 06:00"), + ("weekly mon 07:00", "weekly mon 07:00"), + # Written how people write them, stored how the table stores them. + ("weekly monday 07:00", "weekly mon 07:00"), + ("Weekly Sunday 23:59", "weekly sun 23:59"), + ("EVERY 15 M", "every 15m"), + (" daily at 6:05 ", "daily at 06:05"), + ("every 1d", "every 1d"), +] + + +@pytest.mark.parametrize("typed,canonical", WORDINGS) +def test_every_wording_parses_and_comes_back_canonical(typed, canonical): + """Five forms, one spelling each. + + The round trip is the point: the table stores `str(cadence)`, so two + people writing the same schedule two ways end up with one row that reads + the same to both of them. + """ + cadence = Cadence.parse(typed) + assert str(cadence) == canonical + assert str(Cadence.parse(canonical)) == canonical + assert Cadence.parse(typed) == Cadence.parse(canonical) + + +def test_the_units_are_what_they_say(): + assert Cadence.parse("every 15m").seconds == 900 + assert Cadence.parse("every 6h").seconds == 6 * 3600 + assert Cadence.parse("every 2d").seconds == 2 * 86400 + # A clock cadence has no fixed length, and says so rather than guessing. + assert Cadence.parse("daily at 06:00").seconds == 0 + + +@pytest.mark.parametrize("wording", [ + "sometimes", "", "cron 0 6 * * *", "every 15", "every m", "every 0h", + "every 15s", "daily 06:00", "daily at 25:00", "daily at 06:60", + "weekly 07:00", "weekly funday 07:00", "every 15m please", +]) +def test_a_wording_outside_the_five_is_refused_with_all_five(wording): + """And the refusal is the sentence a person is shown. + + It goes out of the API as `{"detail": ...}` and into a toast, so it has to + say what to type instead rather than merely that this was wrong. + """ + with pytest.raises(ValueError) as refused: + Cadence.parse(wording) + said = str(refused.value) + assert isinstance(refused.value, BadCadence) + for form in ("every 15m", "every 6h", "every 2d", + "daily at 06:00", "weekly mon 07:00"): + assert form in said + + +# ------------------------------------------------------------- next_after + +def test_next_after_is_strictly_after(): + """A schedule fired at its own time must not find itself due again on the + same instant, which would be one firing per tick for ever.""" + at_six = datetime(2025, 6, 1, 6, 0, tzinfo=COPENHAGEN) + when = Cadence.parse("daily at 06:00").next_after(at_six, COPENHAGEN) + assert when.astimezone(COPENHAGEN) == at_six + timedelta(days=1) + + +def test_next_after_always_comes_back_in_utc(): + """Because it is written straight into the table, where every other time + is UTC and the column is compared as text.""" + when = Cadence.parse("daily at 06:00").next_after( + datetime(2025, 6, 1, 12, 0, tzinfo=COPENHAGEN), COPENHAGEN) + assert when.tzinfo is UTC + assert when.isoformat().endswith("+00:00") + + +def test_weekly_lands_on_the_day_it_names(): + monday = Cadence.parse("weekly mon 07:00") + # A Wednesday: the next Monday is five days away, not tomorrow. + from_wed = monday.next_after(datetime(2025, 6, 4, 9, 0, tzinfo=COPENHAGEN), + COPENHAGEN).astimezone(COPENHAGEN) + assert from_wed.weekday() == 0 and (from_wed.hour, from_wed.minute) == (7, 0) + assert from_wed.date() == datetime(2025, 6, 9).date() + # Monday morning before seven: today, not next week. + from_mon = monday.next_after(datetime(2025, 6, 9, 6, 0, tzinfo=COPENHAGEN), + COPENHAGEN).astimezone(COPENHAGEN) + assert from_mon.date() == datetime(2025, 6, 9).date() + + +def test_daily_keeps_its_wall_clock_across_both_dst_changes(): + """06:00 is 06:00 on both sides, which is the whole reason for the zone. + + Spring forward is a 23-hour day and autumn back is a 25-hour one, so the + real elapsed time between two runs is not 24 hours — and it must not be, + because what the wording promised is a time of day. + """ + daily = Cadence.parse("daily at 06:00") + + # Europe/Copenhagen, 2025: forward on 30 March, back on 26 October. + forward = daily.next_after( + datetime(2025, 3, 29, 6, 30, tzinfo=COPENHAGEN), COPENHAGEN) + local = forward.astimezone(COPENHAGEN) + assert (local.hour, local.minute) == (6, 0) + assert local.date() == datetime(2025, 3, 30).date() + # 05:30Z to 04:00Z: 22h30m of real time for a 23h30m step of the clock. + assert forward - datetime(2025, 3, 29, 6, 30, tzinfo=COPENHAGEN) \ + == timedelta(hours=22, minutes=30) + + back = daily.next_after( + datetime(2025, 10, 25, 6, 30, tzinfo=COPENHAGEN), COPENHAGEN) + local = back.astimezone(COPENHAGEN) + assert (local.hour, local.minute) == (6, 0) + assert local.date() == datetime(2025, 10, 26).date() + assert back - datetime(2025, 10, 25, 6, 30, tzinfo=COPENHAGEN) \ + == timedelta(hours=24, minutes=30) + + +def test_weekly_keeps_its_wall_clock_across_a_dst_change(): + when = Cadence.parse("weekly sun 07:00").next_after( + datetime(2025, 3, 26, 9, 0, tzinfo=COPENHAGEN), COPENHAGEN) + local = when.astimezone(COPENHAGEN) + assert local.date() == datetime(2025, 3, 30).date() # the change itself + assert (local.hour, local.minute) == (7, 0) + + +def test_an_interval_is_elapsed_time_and_ignores_the_change(): + """The other half of the promise, and the reason both wordings exist. + + `every 24h` across a spring-forward is 24 real hours and therefore lands + an hour later on the clock. A person who wanted 06:00 wrote `daily`. + """ + start = datetime(2025, 3, 29, 6, 30, tzinfo=COPENHAGEN) + when = Cadence.parse("every 24h").next_after(start, COPENHAGEN) + assert when - start == timedelta(hours=24) + assert when.astimezone(COPENHAGEN).hour == 7 + + +def test_a_naive_moment_is_read_as_utc(): + """Rather than raising. Every stamp in the table is UTC, so a caller that + lost the tzinfo on the way is very nearly certainly holding one.""" + naive = datetime(2025, 6, 1, 12, 0) + assert Cadence.parse("every 1h").next_after(naive) \ + == datetime(2025, 6, 1, 13, 0, tzinfo=UTC) + + +# ---------------------------------------------------------------- the table + +def test_the_table_holds_a_schedule_through_its_life(jobs): + row = jobs.add_schedule("nightly", "s_tick", "daily at 06:00", + payload={"note": "hello"}, label="A nightly tick", + next_run="2030-01-01T06:00:00+00:00") + assert row is not None + assert (row.name, row.kind, row.cadence) == ("nightly", "s_tick", "daily at 06:00") + assert row.payload == {"note": "hello"} and row.enabled is True + assert row.last_job_id == 0 and row.last_run == "" and row.last_state == "" + + assert [s.name for s in jobs.schedules()] == ["nightly"] + assert jobs.schedule(row.id) == row + + edited = jobs.update_schedule(row.id, cadence="every 15m", enabled=False, + payload={"note": "other"}) + assert edited.cadence == "every 15m" and edited.enabled is False + assert edited.payload == {"note": "other"} + # Untouched columns stay as they were. + assert edited.name == "nightly" and edited.label == "A nightly tick" + + assert jobs.delete_schedule(row.id) is True + assert jobs.delete_schedule(row.id) is False # two tabs; not an error + assert jobs.schedules() == [] + + +def test_a_name_is_taken_once(jobs): + assert jobs.add_schedule("one", "s_tick", "every 15m") is not None + assert jobs.add_schedule("one", "s_tick", "every 6h", only_if_new=True) is None + assert jobs.schedules()[0].cadence == "every 15m" + + +def test_schedules_carry_the_state_of_their_last_job(jobs): + row = jobs.add_schedule("one", "s_tick", "every 15m") + queued = jobs.enqueue("s_tick") + jobs.mark_fired(row.id, queued.id, "2030-01-01T00:00:00+00:00") + assert jobs.schedules()[0].last_state == "queued" + jobs.claim({"cpu": 1}) + jobs.finish(queued.id, "failed", error="no") + assert jobs.schedules()[0].last_state == "failed" + # And the time it fired, which is not the time it is next due. + listed = jobs.schedules()[0] + assert listed.last_run and listed.next_run == "2030-01-01T00:00:00+00:00" + + +def test_due_schedules_passes_over_what_is_off_or_not_yet(jobs): + jobs.add_schedule("due", "s_tick", "every 15m", next_run=ago(60)) + jobs.add_schedule("off", "s_tick", "every 15m", next_run=ago(60), enabled=False) + jobs.add_schedule("later", "s_tick", "every 15m", + next_run="2099-01-01T00:00:00+00:00") + jobs.add_schedule("never", "s_tick", "every 15m") # no time at all + due = jobs.due_schedules(datetime.now(UTC).isoformat()) + assert [s.name for s in due] == ["due"] + + +# --------------------------------------------------------------- the firing + +def test_the_tick_fires_a_due_row_and_moves_it_on(queue, jobs): + row = jobs.add_schedule("regular", "s_tick", "every 15m", + payload={"note": "scheduled"}, next_run=ago(30)) + queue._fire_due() + + queued = jobs.snapshot().queued + assert [j.kind for j in queued] == ["s_tick"] + assert queued[0].payload == {"note": "scheduled"} + + after = jobs.schedule(row.id) + assert after.last_job_id == queued[0].id + assert after.last_run + # Fifteen minutes from now, not from the moment it was due. + assert after.next_run > datetime.now(UTC).isoformat() + assert after.next_run < (datetime.now(UTC) + timedelta(minutes=16)).isoformat() + + +def test_a_schedule_does_not_pile_up_behind_its_own_job(queue, jobs): + """The job from the last firing is still queued, so this one is skipped. + + Skipped and *moved on*, not left due: a schedule whose job takes longer + than its cadence would otherwise fire on every tick for as long as the job + ran, and the queue would fill with copies of work already under way. + """ + row = jobs.add_schedule("regular", "s_tick", "every 15m", next_run=ago(30)) + queue._fire_due() + first = jobs.schedule(row.id).last_job_id + + jobs.update_schedule(row.id, next_run=ago(1)) # due again + queue._fire_due() + + assert len(jobs.snapshot().queued) == 1 + again = jobs.schedule(row.id) + assert again.last_job_id == first # nothing new was queued + assert again.next_run > datetime.now(UTC).isoformat() + + +def test_a_solo_kind_already_underway_holds_its_schedule_back(queue, jobs): + """The other way a schedule collides: not with itself, with somebody else. + + A button, a second schedule or a cascade started one, and `enqueue` would + hand that job back — so the schedule would record work it did not ask for + as its own firing. + """ + pressed = jobs.enqueue("s_solo", dedup_key="s_solo") + row = jobs.add_schedule("solo", "s_solo", "every 15m", next_run=ago(30)) + queue._fire_due() + after = jobs.schedule(row.id) + assert after.last_job_id == 0 # it never fired + assert after.next_run > datetime.now(UTC).isoformat() + assert [j.id for j in jobs.snapshot().queued] == [pressed.id] + + +def test_downtime_is_caught_up_once_and_not_once_per_slot(queue, jobs): + """Three days off an hourly schedule is one job, not seventy-two. + + The row carries one `next_run`, and a firing sets it from *now* rather + than stepping through what was missed — so there is nowhere for a backlog + to accumulate even in principle. + """ + row = jobs.add_schedule("hourly", "s_tick", "every 1h", next_run=ago(3 * 86400)) + queue._fire_due() + assert len(jobs.snapshot().queued) == 1 + after = jobs.schedule(row.id) + assert after.next_run > datetime.now(UTC).isoformat() + assert after.next_run < (datetime.now(UTC) + timedelta(minutes=61)).isoformat() + + +def test_an_unknown_kind_never_fires_and_keeps_its_place(queue, jobs): + """A handler that went away in a deploy. The row stays exactly as it is — + time included — so reinstalling the plugin runs it rather than starting + another cadence from scratch.""" + row = jobs.add_schedule("gone", "s_vanished", "every 15m", next_run=ago(30)) + queue._fire_due() + assert jobs.snapshot().queued == [] + assert jobs.schedule(row.id).next_run == row.next_run + queue._fire_due() # and it says so once + assert queue._said_about == {(row.id, "kind")} + + +def test_a_cadence_the_table_cannot_parse_never_fires(queue, jobs): + """Only reachable by editing the file by hand, since every route parses + first. It must not take the tick down with it.""" + jobs.add_schedule("bent", "s_tick", "every 15m", next_run=ago(30)) + jobs._write("UPDATE schedules SET cadence='sometimes'") + queue._fire_due() + assert jobs.snapshot().queued == [] + assert jobs.schedules()[0].cadence == "sometimes" # left to be fixed + + +def test_a_disabled_row_is_not_fired(queue, jobs): + jobs.add_schedule("off", "s_tick", "every 15m", next_run=ago(30), enabled=False) + queue._fire_due() + assert jobs.snapshot().queued == [] + + +def test_run_now_fires_without_moving_the_next_run(queue, jobs): + row = jobs.add_schedule("nightly", "s_tick", "daily at 06:00", + next_run="2099-01-01T06:00:00+00:00") + started = queue.run_schedule(row.id) + assert started is not None and started.kind == "s_tick" + after = jobs.schedule(row.id) + assert after.next_run == "2099-01-01T06:00:00+00:00" + # But it did just run, and that is the job it queued — so the next tick + # will not start a second one while this is still going. + assert after.last_job_id == started.id and after.last_run + + +# ------------------------------------------------------------ reconciliation + +def test_declared_defaults_get_a_row_on_start(declared, jobs): + declare("nightly tick", "s_tick", "daily at 06:00", {"note": "declared"}, + label="A tick, nightly") + JobQueue(jobs, {}, [], tz="UTC")._reconcile_schedules() + row = jobs.schedules()[0] + assert (row.name, row.kind, row.cadence) == ("nightly tick", "s_tick", + "daily at 06:00") + assert row.payload == {"note": "declared"} and row.label == "A tick, nightly" + assert row.next_run # and it knows when it is next + + +def test_reconciliation_keeps_what_the_user_edited(declared, jobs): + """The whole of what a declaration means: a starting point, not a setting. + + Switching a default off, or slowing it down, has to survive the next + deploy — otherwise the only way to be rid of one is to edit the code and + redeploy, which is not a thing the person running the app can do. + """ + declare("nightly tick", "s_tick", "daily at 06:00") + queue = JobQueue(jobs, {}, [], tz="UTC") + queue._reconcile_schedules() + row = jobs.schedules()[0] + jobs.update_schedule(row.id, cadence="every 2d", enabled=False, + payload={"note": "mine"}) + + queue._reconcile_schedules() # a restart + + again = jobs.schedules()[0] + assert again.id == row.id + assert again.cadence == "every 2d" + assert again.enabled is False + assert again.payload == {"note": "mine"} + + +def test_a_declaration_that_goes_away_leaves_its_row(declared, jobs): + """Nothing here deletes. A deploy that silently forgot a schedule somebody + was relying on is the failure this avoids; an orphan says `unknown kind` + and waits to be deleted deliberately.""" + declare("nightly tick", "s_tick", "daily at 06:00") + queue = JobQueue(jobs, {}, [], tz="UTC") + queue._reconcile_schedules() + declared.clear() + queue._reconcile_schedules() + assert [s.name for s in jobs.schedules()] == ["nightly tick"] + + +def test_declaring_one_name_twice_differently_is_refused(declared): + declare("nightly", "s_tick", "daily at 06:00") + declare("nightly", "s_tick", "daily at 06:00") # the same: fine + with pytest.raises(ValueError) as refused: + declare("nightly", "s_tick", "daily at 07:00") + assert "nightly" in str(refused.value) + + +def test_a_declared_cadence_is_parsed_where_it_is_written(declared): + """At import, with a traceback pointing at the line — rather than at 06:00 + in a log nobody is reading.""" + with pytest.raises(ValueError): + declare("bad", "s_tick", "every so often") + assert "bad" not in declared + + +# ------------------------------------------------------------- a real queue + +def test_a_started_queue_fires_what_is_overdue(jobs, declared): + """The one test that drives the loop rather than the method. + + Everything above proves the firing decides correctly; this proves it is + wired into `start()` at all, which is the wire nobody notices is missing + until a schedule silently stops running. + """ + ran: list[str] = [] + jobs.add_schedule("regular", "s_tick", "every 15m", + payload={"note": "from the loop"}, next_run=ago(30)) + queue = JobQueue(jobs, {}, ran, tz="UTC", workers={"cpu": 1}) + assert queue.drain(timeout=10) is True + assert ran == ["from the loop"] + assert jobs.schedules()[0].last_state == "done" + + +def test_a_queue_with_schedules_off_fires_nothing(jobs, declared): + ran: list[str] = [] + jobs.add_schedule("regular", "s_tick", "every 15m", next_run=ago(30)) + queue = JobQueue(jobs, {}, ran, tz="UTC", schedules=False) + assert queue.drain(timeout=10) is True + assert ran == [] + assert jobs.schedules()[0].last_job_id == 0 + + +def test_an_unknown_timezone_is_a_warning_and_not_a_refusal(jobs, caplog): + """A mistyped zone must not be the reason a worker will not start.""" + queue = JobQueue(jobs, {}, [], tz="Mars/Olympus_Mons") + assert queue.zone is not None + assert Cadence.parse("daily at 06:00").next_after(datetime.now(UTC), queue.zone) + + +# ------------------------------------------------------------------ the web + +@pytest.fixture +def client(jobs, declared): + pytest.importorskip("fastapi") + from fastapi import FastAPI + from fastapi.testclient import TestClient + from pipeline.web import connect + + app = FastAPI() + connect(app, jobs, {}, [], tz="UTC", schedules=False) + with TestClient(app) as client: + yield client + + +def test_the_element_is_served(client): + answer = client.get("/_jobs/schedules.js") + assert answer.status_code == 200 + assert "javascript" in answer.headers["content-type"] + assert "desk-schedules" in answer.text + + +def test_adding_listing_editing_and_deleting(client): + added = client.post("/api/schedules", json={ + "kind": "s_tick", "cadence": "weekly monday 07:00", + "payload": {"note": "over http"}}) + assert added.status_code == 201 + row = added.json() + # Named after its kind, since none was given, and stored canonically. + assert row["name"] == "s_tick" and row["cadence"] == "weekly mon 07:00" + assert row["next_run"] and row["unknown"] is False + + assert [s["id"] for s in client.get("/api/schedules").json()] == [row["id"]] + + off = client.patch(f"/api/schedules/{row['id']}", json={"enabled": False}) + assert off.status_code == 200 and off.json()["enabled"] is False + # And only that: a PATCH of one field does not resubmit the rest. + assert off.json()["cadence"] == "weekly mon 07:00" + assert off.json()["payload"] == {"note": "over http"} + + faster = client.patch(f"/api/schedules/{row['id']}", json={"cadence": "every 15m"}) + assert faster.json()["cadence"] == "every 15m" + # A new cadence moves the time with it, rather than leaving the old one. + assert faster.json()["next_run"] != row["next_run"] + + gone = client.delete(f"/api/schedules/{row['id']}") + assert gone.status_code == 200 and gone.json() == {"deleted": True} + assert client.delete(f"/api/schedules/{row['id']}").json() == {"deleted": False} + + +def test_a_second_schedule_of_one_kind_is_named_by_counting(client): + first = client.post("/api/schedules", + json={"kind": "s_tick", "cadence": "every 15m"}).json() + second = client.post("/api/schedules", + json={"kind": "s_tick", "cadence": "every 6h"}).json() + assert (first["name"], second["name"]) == ("s_tick", "s_tick 2") + + +def test_a_name_somebody_typed_is_refused_rather_than_altered(client): + client.post("/api/schedules", + json={"name": "nightly", "kind": "s_tick", "cadence": "every 15m"}) + clash = client.post("/api/schedules", + json={"name": "nightly", "kind": "s_tick", "cadence": "every 6h"}) + assert clash.status_code == 409 + assert "nightly" in clash.json()["detail"] + + +def test_an_unregistered_kind_is_a_404_naming_what_there_is(client): + refused = client.post("/api/schedules", + json={"kind": "s_nonesuch", "cadence": "every 15m"}) + assert refused.status_code == 404 + detail = refused.json()["detail"] + assert "s_nonesuch" in detail and "s_tick" in detail + + +def test_a_payload_missing_what_the_handler_needs_is_a_422_naming_it(client): + refused = client.post("/api/schedules", + json={"kind": "s_needs", "cadence": "every 15m"}) + assert refused.status_code == 422 + assert "target" in refused.json()["detail"] + # And with it, the same request is fine. + assert client.post("/api/schedules", json={ + "kind": "s_needs", "cadence": "every 15m", + "payload": {"target": "somewhere"}}).status_code == 201 + + +def test_a_bad_cadence_is_a_422_listing_the_accepted_wordings(client): + refused = client.post("/api/schedules", + json={"kind": "s_tick", "cadence": "sometimes"}) + assert refused.status_code == 422 + detail = refused.json()["detail"] + assert isinstance(detail, str) + for form in ("every 15m", "every 6h", "every 2d", + "daily at 06:00", "weekly mon 07:00"): + assert form in detail + + +def test_an_edit_is_validated_the_same_way(client): + row = client.post("/api/schedules", + json={"kind": "s_tick", "cadence": "every 15m"}).json() + assert client.patch(f"/api/schedules/{row['id']}", + json={"cadence": "now and then"}).status_code == 422 + assert client.patch(f"/api/schedules/{row['id']}", + json={"kind": "s_needs"}).status_code == 422 + assert client.patch("/api/schedules/9999", + json={"enabled": False}).status_code == 404 + + +def test_run_now_queues_the_job(client, jobs): + row = client.post("/api/schedules", + json={"kind": "s_tick", "cadence": "daily at 06:00"}).json() + started = client.post(f"/api/schedules/{row['id']}/run") + assert started.status_code == 202 + assert started.json()["kind"] == "s_tick" + after = client.get("/api/schedules").json()[0] + assert after["last_job_id"] == started.json()["id"] + assert after["next_run"] == row["next_run"] + assert client.post("/api/schedules/9999/run").status_code == 404 + + +def test_a_row_whose_handler_went_away_is_flagged_not_hidden(client, jobs): + jobs.add_schedule("orphan", "s_vanished", "every 15m") + row = next(s for s in client.get("/api/schedules").json() + if s["name"] == "orphan") + assert row["unknown"] is True + # Listed and deletable, which is the whole of what can be done about it. + assert client.delete(f"/api/schedules/{row['id']}").json() == {"deleted": True} + + +def test_the_socket_pushes_the_schedule_list_on_every_change(client): + """So a row edited in one tab redraws in the others without polling. + + The opening message is the *job* list — the socket is shared — which is + why `` reads `/api/schedules` once on connect and follows + this afterwards. + """ + with client.websocket_connect("/ws/jobs") as socket: + assert socket.receive_json()["type"] == "jobs" + added = client.post("/api/schedules", + json={"kind": "s_tick", "cadence": "every 15m"}).json() + pushed = _next_of(socket, "schedules") + assert [s["id"] for s in pushed["schedules"]] == [added["id"]] + assert pushed["schedules"][0]["cadence"] == "every 15m" + + client.patch(f"/api/schedules/{added['id']}", json={"cadence": "every 6h"}) + assert _next_of(socket, "schedules")["schedules"][0]["cadence"] == "every 6h" + + client.delete(f"/api/schedules/{added['id']}") + assert _next_of(socket, "schedules")["schedules"] == [] + + +def _next_of(socket, type_: str, tries: int = 12) -> dict: + """The next message of one type, stepping over the job pushes between.""" + for _ in range(tries): + message = socket.receive_json() + if message["type"] == type_: + return message + raise AssertionError(f"no {type_} message arrived") + + +# --------------------------------------------------------------- the worker + +def _worker(db: Path, marks: Path, *extra: str) -> subprocess.CompletedProcess: + argv = [sys.executable, "-m", "pipeline.worker", "--db", str(db), + "--bootstrap", "pipeline.tests.scheduled_handlers:marking", + "--bootstrap-arg", str(marks), "--until-idle", *extra] + return subprocess.run(argv, env=dict(os.environ, PYTHONPATH=str(ROOT)), + cwd=ROOT, capture_output=True, timeout=120) + + +def test_the_standalone_worker_fires_schedules(tmp_path): + """A machine running only `python -m pipeline.worker` still gets its 06:00 + job, which is what makes it a complete replacement for the web app's + worker rather than half of one.""" + db, marks = tmp_path / "w.db", tmp_path / "marks" + marks.mkdir() + jobs = Jobs(db) + jobs.add_schedule("marker", "sw_mark", "every 1h", next_run=ago(60)) + + done = _worker(db, marks) + assert done.returncode == 0, done.stdout.decode("utf-8", "replace") + assert (marks / "fired").exists() + assert jobs.schedules()[0].last_job_id + + +def test_no_schedules_turns_it_off_for_a_second_worker(tmp_path): + """One scheduler per file, as many workers as you like: the pile-up guard + reads `last_job_id`, which neither of two simultaneous firings has written + yet when the other looks.""" + db, marks = tmp_path / "w.db", tmp_path / "marks" + marks.mkdir() + jobs = Jobs(db) + jobs.add_schedule("marker", "sw_mark", "every 1h", next_run=ago(60)) + + done = _worker(db, marks, "--no-schedules") + assert done.returncode == 0, done.stdout.decode("utf-8", "replace") + assert not (marks / "fired").exists() + assert jobs.schedules()[0].last_job_id == 0 + # Not even the declared default was reconciled: nothing about schedules + # happens in a worker that was told not to do them. + assert [s.name for s in jobs.schedules()] == ["marker"] diff --git a/pipeline/web/__init__.py b/pipeline/web/__init__.py new file mode 100644 index 0000000..6dfee7c --- /dev/null +++ b/pipeline/web/__init__.py @@ -0,0 +1,59 @@ +"""The parts of a web app that are the pipeline's, not the application's. + + from pipeline.web import connect + queue = connect(app, Jobs("app.db"), config, store) + +An app of this shape has a queue, and therefore has the same endpoints and the +same socket over it: list, kinds, enqueue, cancel, and `/ws/jobs`. None of that +knows what the work is *for*, so it is here rather than copied into each app — +and a socket's reconnect and fan-out behaviour is exactly the sort of thing a +second copy gets subtly wrong. + + app.py `connect`: all of the below wired into a FastAPI app, and + the worker started and stopped with it, in one call + broadcaster.py one message out to every connected socket + jobs.py the queue over HTTP, the socket that reports on it, and + the one route that serves the element below + schedules.py schedules over HTTP: list, add, edit, delete, run now, + with the readable refusals a thing that runs while nobody + is watching has to have + deps.py how those routes reach the queue and the bus + schemas.py the request bodies they take + static/jobs.js ``: the queue on screen. Two lines in a + template and a new app has a job list with progress and a + Kill button, before it has written any UI of its own + static/schedules.js + ``: the same two lines for the schedule + table, themed from the same custom properties + + + + +The element is the one thing here with an opinion about how something looks, +and it is deliberate: this is the part nobody wants to write twice. It keeps +to a shadow root and themes from custom properties, so an app overrides it +without forking it, and it says nothing about *failures* — it raises an event +and leaves the wording to the app, which is the same split `notices.py` makes +on the server. newsdesk does not use it: it had a richer strip of its own +before this existed, which is what an app is expected to grow into. + +What an app still owns: its own factory, its own store and config +dependencies, its own routers, its own page. +""" + +from __future__ import annotations + +from pipeline.web.app import connect +from pipeline.web.broadcaster import Broadcaster +from pipeline.web.deps import BroadcasterDep, QueueDep, get_broadcaster, get_queue +from pipeline.web.jobs import STATIC, router as jobs_router +from pipeline.web.schedules import router as schedules_router, served +from pipeline.web.schemas import ( + EnqueueBody, JobKind, NewSchedule, ScheduleEdit, ScheduleRow, +) + +__all__ = ["STATIC", "Broadcaster", "BroadcasterDep", "EnqueueBody", + "JobKind", "NewSchedule", "QueueDep", "ScheduleEdit", "ScheduleRow", + "connect", "get_broadcaster", "get_queue", "jobs_router", + "schedules_router", "served", +] diff --git a/pipeline/web/app.py b/pipeline/web/app.py new file mode 100644 index 0000000..49d6bfe --- /dev/null +++ b/pipeline/web/app.py @@ -0,0 +1,86 @@ +"""The whole web face of the queue, in one call. + + app = FastAPI() + queue = connect(app, Jobs("app.db"), config, store) + +That is the HTTP routes, the `/ws/jobs` socket, the `` and +`` elements, the schedule endpoints, the broadcaster that feeds +the socket, and the worker starting and stopping with the app. Before this +every app wrote the same thirty lines of glue — three callbacks, a lifespan, +two `app.state` slots, two `include_router`s — which is the part of "a day of +pipeline per project" that was still left. + +Keyword arguments go to `JobQueue`: `workers=`, `bootstrap=`, `retain=`, `tz=` +and the rest. `tz` is the one a deployed app has to pass: a server runs on UTC +and `daily at 06:00` means 06:00 where the person reading it lives. The queue +is returned for the app's own routes and commands to enqueue on; it is also on +`app.state.queue`, which is where `QueueDep` finds it. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any + +from fastapi import FastAPI + +from pipeline import JobQueue, Jobs, JobsSnapshot, Notice, Schedule +from pipeline.web.broadcaster import Broadcaster +from pipeline.web.jobs import router +from pipeline.web.schedules import router as schedules_router, served + + +def connect(app: FastAPI, jobs: Jobs, config: Any, store: Any, + **queue_options: Any) -> JobQueue: + """Give `app` a queue: routes, socket, element, and a worker in its lifespan. + + The lifespan is wrapped rather than replaced, so an app's own lifespan — + passed to `FastAPI(lifespan=...)` before or after this — still runs, inside + the queue's: the worker is up before the app's startup code and still up + through its shutdown code, so either may enqueue. + """ + bus = Broadcaster() + + async def publish(snapshot: JobsSnapshot) -> None: + """State: the whole queue, replaced wholesale by the next one.""" + await bus.broadcast({"type": "jobs", **snapshot.model_dump()}) + + async def announce(notice: Notice) -> None: + """An event: one failure, out to every open tab. The row it was + written to, so a tab can tell news from a repeat — see `NoticeEntry`.""" + await bus.broadcast({"type": "notice", **notice.model_dump()}) + + async def scheduled(rows: list[Schedule]) -> None: + """State, on its own clock: the schedule list, whenever one changes. + + Separate from `jobs` rather than folded into it because the two change + at completely different rates — a running job pushes four times a + second and a schedule a few times a day — and because a client that + only draws the table should not have to ignore everything else. + """ + await bus.broadcast({"type": "schedules", + "schedules": [served(row).model_dump() for row in rows]}) + + queue = JobQueue(jobs, config, store, listener=publish, on_notice=announce, + on_schedules=scheduled, **queue_options) + app.state.queue = queue + app.state.bus = bus + app.include_router(router) + app.include_router(schedules_router) + + inner = app.router.lifespan_context + + @asynccontextmanager + async def lifespan(this: FastAPI) -> AsyncIterator[Any]: + # The worker needs a running event loop, which exists here and not + # where `connect` was called. + await queue.start() + try: + async with inner(this) as state: + yield state + finally: + await queue.stop() + + app.router.lifespan_context = lifespan + return queue diff --git a/pipeline/web/broadcaster.py b/pipeline/web/broadcaster.py new file mode 100644 index 0000000..ae2a568 --- /dev/null +++ b/pipeline/web/broadcaster.py @@ -0,0 +1,56 @@ +"""One message out to every connected socket. + +Deliberately knows nothing about jobs. It holds sockets, sends dicts, and drops +the ones that have gone away — which is the whole of what a fan-out is, and is +the reason the next thing that wants to push (a tag landing, a transcript +finishing, another reader in another tab) can use this rather than grow a +second copy of it. + +A dead socket is discovered by writing to it, not by asking: a browser tab that +was closed, slept or navigated away leaves a connection that looks open until +the send fails. So every failure is a removal, and `broadcast` never raises — +one stale subscriber must not cost the others their update. +""" + +from __future__ import annotations + +import logging +import asyncio +from typing import Any + +from fastapi import WebSocket + +log = logging.getLogger(__name__) + + +class Broadcaster: + def __init__(self) -> None: + self._subscribers: set[WebSocket] = set() + + @property + def count(self) -> int: + return len(self._subscribers) + + async def subscribe(self, ws: WebSocket, hello: dict[str, Any] | None = None) -> None: + """Register an accepted socket, and send it the current state. + + The immediate send matters: a client that connects mid-job would + otherwise see nothing at all until the next change, which for a long + job is a blank progress bar next to work that is visibly happening. + """ + self._subscribers.add(ws) + if hello is not None: + await self._send(ws, hello) + + def unsubscribe(self, ws: WebSocket) -> None: + self._subscribers.discard(ws) + + async def broadcast(self, message: dict[str, Any]) -> None: + await asyncio.gather(*(self._send(ws, message) + for ws in tuple(self._subscribers))) + + async def _send(self, ws: WebSocket, message: dict[str, Any]) -> None: + try: + await asyncio.wait_for(ws.send_json(message), timeout=1.0) + except Exception: + self._subscribers.discard(ws) diff --git a/pipeline/web/deps.py b/pipeline/web/deps.py new file mode 100644 index 0000000..7e329cf --- /dev/null +++ b/pipeline/web/deps.py @@ -0,0 +1,34 @@ +"""How a route reaches the queue and the socket bus. + +Both are built once by the app's factory and parked on `app.state`. Routes ask +for them as dependencies rather than closing over them, so a router module is +importable on its own and a test can replace either through +`app.dependency_overrides`. + +Only the two things that are the *pipeline's*. An app's own store and config +are its own business and stay in its own `deps` — the queue and the bus are +here because `pipeline.web.jobs` is here, and it needs them whichever app it is +mounted in. +""" + +from __future__ import annotations + +from typing import Annotated + +from fastapi import Depends +from starlette.requests import HTTPConnection + +from pipeline import JobQueue +from pipeline.web.broadcaster import Broadcaster + + +def get_queue(request: HTTPConnection) -> JobQueue: + return request.app.state.queue + + +def get_broadcaster(request: HTTPConnection) -> Broadcaster: + return request.app.state.bus + + +QueueDep = Annotated[JobQueue, Depends(get_queue)] +BroadcasterDep = Annotated[Broadcaster, Depends(get_broadcaster)] diff --git a/pipeline/web/jobs.py b/pipeline/web/jobs.py new file mode 100644 index 0000000..0decbe8 --- /dev/null +++ b/pipeline/web/jobs.py @@ -0,0 +1,219 @@ +"""The work queue over HTTP, and the socket that reports on it. + +Four operations and one socket. Anything the UI can start, it starts by +enqueueing a job here; anything it needs to know about running work, it learns +from `/ws/jobs`. There is no polling endpoint on purpose — a queue that pushes +is both less code and less latency than one that is asked every second. +""" + +from __future__ import annotations + +import logging +import asyncio +from pathlib import Path + +from fastapi import APIRouter, HTTPException, WebSocket, WebSocketDisconnect +from fastapi.responses import FileResponse + +from pipeline import FINAL, JOBS, TASKS +from pipeline.models import Job, JobsSnapshot, NoticeEntry, Run +from pipeline.web.deps import BroadcasterDep, QueueDep +from pipeline.web.schemas import EnqueueBody, JobKind + +log = logging.getLogger(__name__) + +router = APIRouter(tags=["jobs"]) + +#: The element that renders all this, shipped with the package. Served from a +#: route rather than mounted as a static directory because a router cannot +#: mount one — only an app can — and needing the app to mount something is +#: exactly the line of setup this is meant to remove. One file, so one route. +STATIC = Path(__file__).resolve().parent / "static" + + +@router.get("/_jobs/jobs.js", include_in_schema=False) +def element() -> FileResponse: + """``: the queue on screen, for an app with no UI for it yet. + + + + + Cached for an hour rather than forever. It changes when the package is + copied into a project and then hardly ever, and an app that wants it + fingerprinted can serve the file itself from `STATIC`. + """ + return FileResponse(STATIC / "jobs.js", media_type="text/javascript", + headers={"Cache-Control": "public, max-age=3600"}) + + +@router.get("/api/jobs") +def list_jobs(queue: QueueDep) -> JobsSnapshot: + """What is running and what recently finished. + + The same shape the socket pushes, so a client that cannot hold a socket + open — a script, a `curl`, another tab — sees exactly what the + UI sees. + """ + return queue.snapshot() + + +@router.get("/api/jobs/kinds") +def kinds() -> list[JobKind]: + """What can be started. Populated by `@job` and by the app's `TASKS`, so a + new handler or a new task appears here without being registered twice. + + Tasks and jobs in one list because they are one thing to press: a task is + what a person means by "tag everything", and that it becomes eight jobs is + the pipeline's business rather than the button's. + + Which is also why a job some task queues is not listed. `tag_batch` without + a list of uids is a button that does nothing, and the final job without the + batches it reports on is a button that reports nothing. If a task queues + it, the task is the thing to offer — worked out from `TASKS` rather than + declared, so a new batch kind cannot be forgotten here. + + Nor is a job that needs a payload, such as `mark_reclassify` without a + `source`: a press sends none, so its button could only ever fail. + """ + made_for_you = {build().kind for build in TASKS.values()} | {FINAL} + listed = [ + JobKind(kind=spec.kind, label=spec.label, solo=spec.solo) + for spec in JOBS.values() + if spec.kind not in made_for_you and not spec.needs + ] + listed += [JobKind(kind=name, label=build().called, solo=True) + for name, build in TASKS.items()] + return sorted(listed, key=lambda k: k.kind) + + +@router.post("/api/jobs", status_code=202) +def enqueue(body: EnqueueBody, queue: QueueDep) -> Job | Run: + """Start work: one job, or one task. + + A kind that names a task starts it rather than being refused — one task, + on its own, which is a cascade of one and comes back as that run. Refusing + with "use /api/cascades" was the other reading and is worse: every caller + would have to know which names had become tasks, when what they asked for + is exactly what this does. + + A name that is both a job and a task is the job. A task of a single batch + is that batch, so the two are the same work, and one row says it more + plainly than two. + + 202 rather than 201, because the useful thing has not happened yet — this + only promises that it will. A `solo` job that is already underway comes + back as itself rather than as an error: the caller asked for that work to + be in progress, and it is. + """ + if body.kind not in JOBS and body.kind in TASKS: + task_id = queue.start_task(TASKS[body.kind]()) + run = queue.jobs.run(task_id) + if run is None: + raise HTTPException(409, f"{body.kind!r} would not start") + return run + spec = JOBS.get(body.kind) + missing = [k for k in (spec.needs if spec else ()) if k not in body.payload] + if missing: + raise HTTPException(422, f"{body.kind!r} needs {', '.join(missing)} " + "in its payload") + try: + job = queue.enqueue(body.kind, body.payload) + except KeyError as exc: + raise HTTPException(404, str(exc)) from exc + if job is None: + raise HTTPException(409, "the run it belongs to was killed") + return job + + +@router.post("/api/jobs/{job_id}/cancel") +def cancel(job_id: int, queue: QueueDep) -> Job: + """Stop a job. + + A queued one stops immediately. A running one stops at its next checkpoint, + so this returns while it is still `running` with `cancel_requested` set — + the socket reports the actual end. A job that has already finished is + returned unchanged rather than treated as an error; the button was simply + pressed a moment too late. + """ + job = queue.cancel(job_id) + if job is None: + raise HTTPException(404, "no such job") + return job + + +@router.post("/api/tasks/{task_id}/cancel") +def cancel_task(task_id: int, queue: QueueDep) -> dict[str, int]: + """Stop one task: every batch of it. + + Not the cascade it belongs to, and not the task's final job: that still + runs, reports the cancels and calls back, so the cascade knows how this + ended — "skip this", rather than "stop". Stopping is the cascade's own + button, below. + """ + return {"ended": queue.cancel_run(task_id)} + + +@router.post("/api/cascades/{cascade_id}/cancel") +def cancel_cascade(cascade_id: int, queue: QueueDep) -> dict[str, int]: + """Stop a whole cascade: every task of it, and every job of those. + + The same operation as above — a task and a cascade are one thing to the + pipeline — and two endpoints because they are two buttons and mean + different things to whoever presses one. + + An id nobody has heard of, or one already over, ends nothing and says so + rather than being an error — the same reading as pressing Kill a moment + too late. + """ + return {"ended": queue.cancel_run(cascade_id)} + + +@router.get("/api/notices") +def list_notices(queue: QueueDep) -> list[NoticeEntry]: + """Everything the queue has said that nobody has cleared, newest first. + + What a line on screen was always missing: somewhere to be found again. A + notice fades in seconds and is raised, as often as not, while no tab is + open to show it. This is the same sentence with its fix and its raw error, + how many times it has been said, and when it last was. + """ + return queue.jobs.notices() + + +@router.delete("/api/notices/{notice_id}") +def clear_notice(notice_id: int, queue: QueueDep) -> dict[str, bool]: + """Done with this one. Not an error when it has already gone: two tabs, or + the fortnight running out between the list and the press.""" + return {"cleared": queue.jobs.clear_notice(notice_id)} + + +@router.delete("/api/notices") +def clear_notices(queue: QueueDep) -> dict[str, int]: + return {"cleared": queue.jobs.clear_notices()} + + +@router.websocket("/ws/jobs") +async def jobs_socket(ws: WebSocket, queue: QueueDep, bus: BroadcasterDep) -> None: + """Push queue state on connect and on every change. + + Two message types go out here: `jobs`, the whole queue state, replaced by + each new one; and `notice`, a single failure as the log now holds it, its + count included. See `pipeline/notices.py` for what a client does with that. + + The receive loop exists only to notice the client leaving — nothing the + browser sends is read. Starting and cancelling go over HTTP, where they get + status codes and error bodies; making the socket bidirectional would mean + inventing a second request protocol inside it for no gain. + """ + await ws.accept() + snapshot = await asyncio.to_thread(queue.snapshot) + await bus.subscribe(ws, {"type": "jobs", **snapshot.model_dump()}) + try: + while True: + await ws.receive_text() + except WebSocketDisconnect: + pass + except Exception: + log.debug("jobs socket closed unexpectedly", exc_info=True) + finally: + bus.unsubscribe(ws) diff --git a/pipeline/web/schedules.py b/pipeline/web/schedules.py new file mode 100644 index 0000000..ed82f7c --- /dev/null +++ b/pipeline/web/schedules.py @@ -0,0 +1,218 @@ +"""Schedules over HTTP: list, add, edit, delete, run now. + +Five routes and no socket of its own. Changes go out on `/ws/jobs` as a +`schedules` message — the same socket the job list already holds open, because +a second one would be a second reconnect loop to get subtly wrong, for a list +that changes a few times a day. + +Every refusal here is a sentence a person can act on, which is the whole +reason these are not four lines of `queue.jobs.*` behind a generic validator: + + an unregistered kind 404, and it names what is registered + a payload missing keys 422, and it names the keys + a cadence it cannot read 422, and it lists all five accepted wordings + +They arrive as `{"detail": ""}`, which is FastAPI's own shape, so an +app's `api()` helper turns any of them into one toast carrying the server's +words rather than inventing its own — see `app/static/app.js` in the starter. + +Validating the kind and the payload *here*, rather than letting the job fail at +06:00, is the point of the endpoint existing at all. A schedule is the one way +of starting work where nobody is watching when it runs. +""" + +from __future__ import annotations + +import logging +from datetime import UTC, datetime + +from fastapi import APIRouter, HTTPException +from fastapi.responses import FileResponse + +from pipeline import JOBS, JobQueue +from pipeline.models import Job, Schedule +from pipeline.schedule import Cadence +from pipeline.web.deps import QueueDep +from pipeline.web.jobs import STATIC +from pipeline.web.schemas import NewSchedule, ScheduleEdit, ScheduleRow + +log = logging.getLogger(__name__) + +router = APIRouter(tags=["schedules"]) + + +@router.get("/_jobs/schedules.js", include_in_schema=False) +def element() -> FileResponse: + """``: the schedule table on screen. + + + + + Served and cached exactly as `/_jobs/jobs.js` is, and it reads the same + `--desk-jobs-*` custom properties, so an app that themed one has themed + both. + """ + return FileResponse(STATIC / "schedules.js", media_type="text/javascript", + headers={"Cache-Control": "public, max-age=3600"}) + + +def listed(queue: JobQueue) -> list[ScheduleRow]: + """The list as it goes out, over HTTP and over the socket alike. + + One function for both, so the two cannot drift: a client that reads the + REST list once and then follows the socket must not be handed two shapes of + the same thing. + """ + return [served(row) for row in queue.jobs.schedules()] + + +def served(row: Schedule) -> ScheduleRow: + """One schedule, plus the one thing the table cannot know: whether anything + is still registered to run it. Worked out here rather than stored, because + it is a fact about the code that is loaded and not about the row.""" + return ScheduleRow(**row.model_dump(), unknown=row.kind not in JOBS) + + +def _check(kind: str, payload: dict) -> None: + """Refuse a kind nothing runs, or a payload its handler cannot take.""" + spec = JOBS.get(kind) + if spec is None: + raise HTTPException(404, f"no job is registered as {kind!r}. Registered: " + f"{', '.join(sorted(JOBS)) or 'none'}.") + missing = [k for k in spec.needs if k not in payload] + if missing: + raise HTTPException(422, f"{kind!r} cannot run without " + f"{', '.join(missing)} in its payload.") + + +def _cadence(text: str) -> Cadence: + """Parse, or 422 with the accepted wordings. + + `Cadence` already writes that sentence — see `pipeline/schedule.py` — so it + is passed through rather than reworded here. One description of the grammar + in the codebase, and it is next to the parser that enforces it. + """ + try: + return Cadence.parse(text) + except ValueError as bad: + raise HTTPException(422, str(bad)) from bad + + +def _free_name(queue: JobQueue, wanted: str) -> str: + """A name nothing else has: `tick`, then `tick 2`, then `tick 3`. + + Only for a request that gave none. A name somebody typed is refused rather + than quietly altered, because the name is the handle, and being handed a + different one is how you edit the wrong row a month later. + """ + taken = {row.name for row in queue.jobs.schedules()} + if wanted not in taken: + return wanted + n = 2 + while f"{wanted} {n}" in taken: + n += 1 + return f"{wanted} {n}" + + +@router.get("/api/schedules") +def list_schedules(queue: QueueDep) -> list[ScheduleRow]: + """Every schedule, by name, with how its last job ended. + + The same list the socket pushes, so a client that cannot hold one open — a + script, a `curl`, a second tab — sees exactly what the element sees. + """ + return listed(queue) + + +@router.post("/api/schedules", status_code=201) +def add_schedule(body: NewSchedule, queue: QueueDep) -> ScheduleRow: + """Add one. 201, because unlike a job this really does now exist. + + `name` is optional and defaults to the kind, made unique by counting. A + name that was given and is taken is a conflict rather than a second row: + two rows answering to one name is exactly the ambiguity the handle exists + to prevent. + + `next_run` is worked out here, from now, so a schedule added at 09:00 + saying `every 6h` is next at 15:00 rather than at some moment the table + invented. + """ + _check(body.kind, body.payload) + cadence = _cadence(body.cadence) + name = body.name.strip() or _free_name(queue, body.kind) + row = queue.jobs.add_schedule( + name, body.kind, str(cadence), payload=body.payload, label=body.label, + enabled=body.enabled, + next_run=cadence.next_after(datetime.now(UTC), queue.zone).isoformat(), + only_if_new=True) + if row is None: + raise HTTPException(409, f"there is already a schedule called {name!r}.") + queue.notify_schedules() + return served(row) + + +@router.patch("/api/schedules/{schedule_id}") +def edit_schedule(schedule_id: int, body: ScheduleEdit, queue: QueueDep) -> ScheduleRow: + """Change part of one. Only what was sent is touched. + + A new cadence moves `next_run` with it, because the alternative is a row + that says `every 15m` and does not run for six hours: the old time is an + answer to a question nobody is asking any more. Switching a row back on + does the same. It would come right on its own — a firing advances to the + next slot from now and never walks through the missed ones — but the time + on screen would be a lie until it did. + """ + row = queue.jobs.schedule(schedule_id) + if row is None: + raise HTTPException(404, f"there is no schedule {schedule_id}.") + # Unset and null are one thing here: every field is optional and every + # column is NOT NULL, so `{"cadence": null}` can only be a client leaving + # a key in rather than a request to blank the column. + fields = {k: v for k, v in body.model_dump(exclude_unset=True).items() + if v is not None} + if "kind" in fields or "payload" in fields: + _check(fields.get("kind", row.kind), fields.get("payload", row.payload)) + now = datetime.now(UTC) + if "cadence" in fields: + cadence = _cadence(fields["cadence"]) + fields["cadence"] = str(cadence) + fields["next_run"] = cadence.next_after(now, queue.zone).isoformat() + elif fields.get("enabled") and not row.enabled: + fields["next_run"] = _cadence(row.cadence).next_after(now, queue.zone).isoformat() + changed = queue.jobs.update_schedule(schedule_id, **fields) + queue.notify_schedules() + return served(changed or row) + + +@router.delete("/api/schedules/{schedule_id}") +def delete_schedule(schedule_id: int, queue: QueueDep) -> dict[str, bool]: + """Forget one. Not an error when it has already gone — two tabs. + + A declared default deleted here comes back on the next start, which is + what declaring one means. Switching it off is how to be rid of it for + good: that edit is the row's, and reconciliation never overwrites one. + """ + deleted = queue.jobs.delete_schedule(schedule_id) + if deleted: + queue.notify_schedules() + return {"deleted": deleted} + + +@router.post("/api/schedules/{schedule_id}/run", status_code=202) +def run_now(schedule_id: int, queue: QueueDep) -> Job: + """Queue this schedule's job now, without moving when it is next due. + + 202 like `/api/jobs`, and for the same reason: what comes back is the + promise, not the outcome. A `solo` kind already under way comes back as + that job rather than as an error — the caller asked for the work to be in + progress, and it is. + """ + row = queue.jobs.schedule(schedule_id) + if row is None: + raise HTTPException(404, f"there is no schedule {schedule_id}.") + _check(row.kind, row.payload) + job = queue.run_schedule(schedule_id) + if job is None: + raise HTTPException(409, f"{row.kind!r} was not queued: the run it " + "would have belonged to was killed.") + return job diff --git a/pipeline/web/schemas.py b/pipeline/web/schemas.py new file mode 100644 index 0000000..0cab3e4 --- /dev/null +++ b/pipeline/web/schemas.py @@ -0,0 +1,75 @@ +"""What the queue's endpoints accept and promise back. + +Job-shaped, so they live with the router that serves them rather than in an +app's schema module. The `Job`, `JobsSnapshot` and `Schedule` they sit +alongside are in `pipeline/models.py`; these are only ever request bodies, or a +stored shape with one derived field added for the client. +""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel + +from pipeline.models import Schedule + + +class EnqueueBody(BaseModel): + """Start a job. `payload` is spread over the handler's keyword arguments, + so what a kind accepts is the signature of its function and nothing else + has to restate it.""" + + kind: str + payload: dict[str, Any] = {} + + +class JobKind(BaseModel): + """One entry from the job registry, for a UI that wants to offer them.""" + + kind: str + label: str + solo: bool + + +class NewSchedule(BaseModel): + """Add a schedule. `name` is optional because the kind is usually answer + enough — one `tick` schedule wants no name of its own, and the second is + named by counting rather than by making somebody think of one.""" + + kind: str + cadence: str + name: str = "" + label: str = "" + payload: dict[str, Any] = {} + enabled: bool = True + + +class ScheduleEdit(BaseModel): + """Change part of a schedule. Every field optional, and read with + `exclude_unset`, so sending `{"enabled": false}` switches one off without + also resubmitting a cadence somebody edited in another tab a second ago. + + `name` is not here: it is the handle a declared default is matched on, and + renaming a row would hand its edits to a fresh one inserted under the old + name on the next start. + """ + + kind: str | None = None + label: str | None = None + cadence: str | None = None + payload: dict[str, Any] | None = None + enabled: bool | None = None + + +class ScheduleRow(Schedule): + """A schedule as it is served: the row, plus whether anything still runs it. + + `unknown` is not stored, because it is a fact about the code that is loaded + rather than about the row — a handler removed in a deploy makes it true + without the table changing, and reinstalling the plugin makes it false + again. The element greys such a row and says `unknown kind`; the queue + never fires one. + """ + + unknown: bool = False diff --git a/pipeline/web/static/jobs.js b/pipeline/web/static/jobs.js new file mode 100644 index 0000000..613eb64 --- /dev/null +++ b/pipeline/web/static/jobs.js @@ -0,0 +1,280 @@ +/* The queue, on screen, for an app that has not written a UI yet. + * + * + * + * + * That is the whole of it. The element finds `/ws/jobs` itself, shows what is + * running with a progress bar and a Kill button each, says how many are + * waiting, and reconnects when the server restarts under you. + * + * One file, and the styles are in it rather than beside it, because the + * shadow root scopes them anyway and a second file is a second route, a + * second cache header and a second thing to forget to copy. Themed from + * outside through the custom properties at the top of `CSS` — set them on + * `desk-jobs` or on `:root` and nothing here has to be forked. + * + * What it deliberately does not do is talk about failures. It raises a + * `desk-jobs:notice` event and leaves the wording to the app, because a + * useful sentence names things only the app knows — its backends, its feeds, + * its config keys. `pipeline/notices.py` is the same split on the server. + * + * Attributes, all optional: + * base="/api" prefix for the HTTP calls, if the router is mounted deeper + * socket="..." the socket's URL, if it is not `/ws/jobs` on this origin + * probe="/api/status" + * an authenticated endpoint to try when the socket closes. A + * refused handshake and a dead server look identical from + * here — the server rejects an unauthenticated upgrade before + * completing it, so the browser reports the same abnormal + * close for "your session expired" as for "the server is + * gone". One cheap request tells them apart. Without this the + * element assumes the server went away, which is the more + * common reason and the less confusing thing to be wrong about. + * idle="hide" with nothing running or queued, take up no space. The + * default, since a chip that is always there is furniture. + */ + +const CSS = ` +:host { + --desk-jobs-bg: #16181d; + --desk-jobs-fg: #e7e9ee; + --desk-jobs-dim: #9aa1ad; + --desk-jobs-accent: #4c8dff; + --desk-jobs-error: #ff6b6b; + --desk-jobs-radius: 10px; + --desk-jobs-font: system-ui, -apple-system, "Segoe UI", sans-serif; + --desk-jobs-gap: 8px; + + display: block; + font-family: var(--desk-jobs-font); + font-size: 13px; + color: var(--desk-jobs-fg); +} +:host([hidden]) { display: none; } + +.strip { + display: flex; + flex-direction: column; + gap: var(--desk-jobs-gap); + background: var(--desk-jobs-bg); + border-radius: var(--desk-jobs-radius); + padding: 10px 12px; +} +.job { display: grid; grid-template-columns: 1fr auto; gap: 2px 10px; align-items: center; } +.what { font-weight: 600; } +.said { color: var(--desk-jobs-dim); grid-column: 1; } +.count { color: var(--desk-jobs-dim); font-variant-numeric: tabular-nums; } + +/* A bar only when there is a fraction to draw. A handler that reports no + * total is working on one thing, and an empty trough reads as stalled. */ +.bar { + grid-column: 1 / -1; + height: 3px; + border-radius: 2px; + background: color-mix(in srgb, var(--desk-jobs-fg) 15%, transparent); + overflow: hidden; +} +.bar > i { display: block; height: 100%; background: var(--desk-jobs-accent); transition: width .2s; } + +button { + font: inherit; + color: var(--desk-jobs-dim); + background: none; + border: 1px solid currentColor; + border-radius: 6px; + padding: 1px 8px; + cursor: pointer; +} +button:hover { color: var(--desk-jobs-error); } +button[disabled] { opacity: .4; cursor: default; } + +.offline { color: var(--desk-jobs-error); } +`; + +/** Backing off rather than hammering. The usual reason a socket closed is the + * server restarting under --reload, which comes back inside a second; the + * ceiling is for when it is not coming back at all. */ +const FIRST_WAIT = 500; +const LONGEST_WAIT = 30000; + +/** How long down is long enough to say so. A reload blinks the socket every + * time a file is saved, and a warning that flashes on every save is one you + * stop reading — which would cost us the case that matters, where the server + * really is gone and the strip is quietly showing state from a minute ago. */ +const QUIET_ABOUT_IT_FOR = 4000; + +class DeskJobs extends HTMLElement { + #root; + #socket = null; + #wait = FIRST_WAIT; + #timer = null; + #state = { running: [], queued: [], recent: [] }; + #offline = false; + #killing = new Set(); + + constructor() { + super(); + this.#root = this.attachShadow({ mode: "open" }); + const style = document.createElement("style"); + style.textContent = CSS; + this.#root.append(style, document.createElement("div")); + } + + connectedCallback() { this.#connect(); this.#render(); } + + disconnectedCallback() { + clearTimeout(this.#timer); + this.#socket?.close(); + this.#socket = null; + } + + get #base() { return this.getAttribute("base") ?? ""; } + + get #socketUrl() { + const given = this.getAttribute("socket"); + if (given) return given; + const scheme = location.protocol === "https:" ? "wss" : "ws"; + return `${scheme}://${location.host}${this.#base}/ws/jobs`; + } + + #connect() { + let socket; + try { + socket = new WebSocket(this.#socketUrl); + } catch { + this.#reconnect(); // a malformed URL, or no network + return; + } + this.#socket = socket; + + socket.onopen = () => { + this.#wait = FIRST_WAIT; + this.#offline = false; + this.#render(); + }; + socket.onmessage = (event) => { + let message; + try { message = JSON.parse(event.data); } catch { return; } + if (message.type === "jobs") { + const { type, ...state } = message; + this.#state = state; + this.#render(); + } else if (message.type === "notice") { + // The app's to word. Composed, so a listener on `document` hears it + // through the shadow boundary. + this.dispatchEvent(new CustomEvent("desk-jobs:notice", { + detail: message, bubbles: true, composed: true, + })); + } + }; + socket.onerror = () => socket.close(); + socket.onclose = async () => { + if (this.#socket !== socket) return; // a later connection owns us now + this.#socket = null; + await this.#whyClosed(); + this.#reconnect(); + }; + } + + /** Ask before assuming. See the `probe` attribute. */ + async #whyClosed() { + const probe = this.getAttribute("probe"); + if (!probe) return; + try { + const answer = await fetch(this.#base + probe, { credentials: "same-origin" }); + if (answer.status === 401 || answer.status === 403) { + this.dispatchEvent(new CustomEvent("desk-jobs:signed-out", { + bubbles: true, composed: true, + })); + } + } catch { /* offline; the strip says so below */ } + } + + #reconnect() { + if (this.#wait >= QUIET_ABOUT_IT_FOR) { + this.#offline = true; + this.#render(); + } + clearTimeout(this.#timer); + this.#timer = setTimeout(() => this.#connect(), this.#wait); + this.#wait = Math.min(this.#wait * 2, LONGEST_WAIT); + } + + async #kill(id) { + this.#killing.add(id); + this.#render(); + try { + await fetch(`${this.#base}/api/jobs/${id}/cancel`, + { method: "POST", credentials: "same-origin" }); + } finally { + // The socket reports the actual ending — a running job stops at its next + // checkpoint, so the button going quiet is the whole of what happens here. + this.#killing.delete(id); + } + } + + #render() { + const { running = [], queued = [] } = this.#state; + const strip = this.#root.lastElementChild; + this.hidden = this.getAttribute("idle") !== "show" + && !running.length && !queued.length && !this.#offline; + + strip.className = "strip"; + strip.replaceChildren(); + + if (this.#offline) { + const said = document.createElement("div"); + said.className = "offline"; + said.textContent = "Not connected — what you see here may be out of date."; + strip.append(said); + } + + for (const job of running) { + strip.append(this.#row(job)); + } + if (queued.length) { + const waiting = document.createElement("div"); + waiting.className = "count"; + waiting.textContent = `${queued.length} waiting`; + strip.append(waiting); + } + } + + #row(job) { + const row = document.createElement("div"); + row.className = "job"; + + const what = document.createElement("span"); + what.className = "what"; + what.textContent = job.label || job.kind; + row.append(what); + + const kill = document.createElement("button"); + kill.textContent = "Kill"; + kill.disabled = this.#killing.has(job.id) || job.cancel_requested; + kill.addEventListener("click", () => this.#kill(job.id)); + row.append(kill); + + if (job.message) { + const said = document.createElement("span"); + said.className = "said"; + said.textContent = job.message; + row.append(said); + } + if (job.total > 0) { + const bar = document.createElement("div"); + bar.className = "bar"; + const fill = document.createElement("i"); + fill.style.width = `${Math.min(100, (job.done / job.total) * 100)}%`; + bar.append(fill); + row.append(bar); + } + return row; + } +} + +if (!customElements.get("desk-jobs")) { + customElements.define("desk-jobs", DeskJobs); +} + +export { DeskJobs }; diff --git a/pipeline/web/static/schedules.js b/pipeline/web/static/schedules.js new file mode 100644 index 0000000..75ace62 --- /dev/null +++ b/pipeline/web/static/schedules.js @@ -0,0 +1,543 @@ +/* The schedule table, on screen, for an app that has not written a UI yet. + * + * + * + * + * A row per schedule — name, kind, cadence, on or off, next run, last run and + * how that job ended — plus Run now, Delete, and a row at the bottom for + * adding one. The kinds come from `/api/jobs/kinds`, so a new `@job` is a new + * option here without this file being touched. + * + * It reads `/ws/jobs` for `schedules` messages, so a row edited in another tab + * redraws in this one, and a firing moves the times without anybody polling. + * The socket's opening message is the *job* list, though, so the table is + * fetched once on connect: the push says what changed, not what there is. The + * job list is read as it goes past as well, for the last column — a schedule + * is pushed when it fires, which is before its job has ended, so the state it + * ended in can only come from there. + * + * The same one-file, shadow-root, custom-property shape as ``, and + * deliberately the same `--desk-jobs-*` names: an app that themed one element + * has themed both, and two schemes for two halves of one page is the thing + * nobody wants to maintain. + * + * Failures are not worded here. A refused change raises `desk-schedules:error` + * carrying the server's own `detail` — which the endpoints promise is one + * readable sentence — and the app toasts it. Notices arriving on the socket + * are re-raised as `desk-jobs:notice`, the same event `` raises, so + * an app that listens for one is listening for both. + * + * Attributes, all optional: + * base="/api" prefix for the HTTP calls, if the router is mounted deeper + * socket="..." the socket's URL, if it is not `/ws/jobs` on this origin + */ + +const CSS = ` +:host { + --desk-jobs-bg: #16181d; + --desk-jobs-fg: #e7e9ee; + --desk-jobs-dim: #9aa1ad; + --desk-jobs-accent: #4c8dff; + --desk-jobs-error: #ff6b6b; + --desk-jobs-radius: 10px; + --desk-jobs-font: system-ui, -apple-system, "Segoe UI", sans-serif; + --desk-jobs-gap: 8px; + + display: block; + font-family: var(--desk-jobs-font); + font-size: 13px; + color: var(--desk-jobs-fg); +} +:host([hidden]) { display: none; } + +.panel { + background: var(--desk-jobs-bg); + border-radius: var(--desk-jobs-radius); + padding: 10px 12px; +} + +table { width: 100%; border-collapse: collapse; } +th, td { + text-align: left; + padding: 6px 8px 6px 0; + vertical-align: baseline; + border-bottom: 1px solid color-mix(in srgb, var(--desk-jobs-fg) 12%, transparent); +} +th { color: var(--desk-jobs-dim); font-weight: 600; white-space: nowrap; } +tr:last-child > td { border-bottom: 0; } +td.actions { text-align: right; white-space: nowrap; } + +.name { font-weight: 600; } +.kind, .when { color: var(--desk-jobs-dim); font-variant-numeric: tabular-nums; } +.off { opacity: .55; } +.bad { color: var(--desk-jobs-error); } + +/* The cadence is edited in place: a box that looks like text until you are in + * it, because a table of eight input fields reads as a form to fill in rather + * than as a list of what is running. */ +input.cadence, input.text { + font: inherit; + color: inherit; + background: none; + border: 1px solid transparent; + border-radius: 6px; + padding: 1px 4px; + min-width: 9em; +} +input.cadence:hover { border-color: color-mix(in srgb, var(--desk-jobs-fg) 20%, transparent); } +input:focus { outline: none; border-color: var(--desk-jobs-accent); } +input::placeholder { color: var(--desk-jobs-dim); } + +select { + font: inherit; + color: inherit; + background: var(--desk-jobs-bg); + border: 1px solid color-mix(in srgb, var(--desk-jobs-fg) 20%, transparent); + border-radius: 6px; + padding: 1px 4px; +} + +button { + font: inherit; + color: var(--desk-jobs-dim); + background: none; + border: 1px solid currentColor; + border-radius: 6px; + padding: 1px 8px; + margin-left: 6px; + cursor: pointer; +} +button:hover { color: var(--desk-jobs-accent); } +button.remove:hover { color: var(--desk-jobs-error); } +button[disabled] { opacity: .4; cursor: default; } + +.state { font-size: 11px; padding: 0 5px; border-radius: 999px; + border: 1px solid currentColor; margin-left: 6px; } +.state.failed, .state.cancelled { color: var(--desk-jobs-error); } +.state.running, .state.queued { color: var(--desk-jobs-accent); } + +.empty, .offline { color: var(--desk-jobs-dim); padding: 6px 0; } +.offline { color: var(--desk-jobs-error); } +`; + +/** The same backoff as ``: the usual reason a socket closed is the + * server restarting under --reload, which comes back inside a second. */ +const FIRST_WAIT = 500; +const LONGEST_WAIT = 30000; + +/** The grammar, in the one place a person is asked to use it. Two of the five + * as the placeholder, because a box wide enough for all five is a box wide + * enough to unbalance the row it is in — and all five on hover, which is also + * what the server's refusal says if they get it wrong anyway. */ +const FORMS = "every 15m · every 6h · every 2d · daily at 06:00 · weekly mon 07:00"; +const HINT = "every 15m · daily at 06:00"; + +/** A stamp as a person reads it. Today is a time, anything else is a date and + * a time, and "never" is said rather than left blank — an empty cell reads as + * a bug, and "never" is a fact. */ +function when(stamp) { + if (!stamp) return "never"; + const at = new Date(stamp); + if (isNaN(at)) return stamp; + const today = at.toDateString() === new Date().toDateString(); + const clock = at.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" }); + return today ? clock : `${at.toLocaleDateString()} ${clock}`; +} + +class DeskSchedules extends HTMLElement { + #root; + #socket = null; + #wait = FIRST_WAIT; + #timer = null; + #rows = []; + #kinds = []; + #offline = false; + #busy = new Set(); + #loaded = false; + + constructor() { + super(); + this.#root = this.attachShadow({ mode: "open" }); + const style = document.createElement("style"); + style.textContent = CSS; + this.#root.append(style, document.createElement("div")); + } + + connectedCallback() { + this.#render(); + this.#load(); + this.#connect(); + } + + disconnectedCallback() { + clearTimeout(this.#timer); + this.#socket?.close(); + this.#socket = null; + } + + get #base() { return this.getAttribute("base") ?? ""; } + + get #socketUrl() { + const given = this.getAttribute("socket"); + if (given) return given; + const scheme = location.protocol === "https:" ? "wss" : "ws"; + return `${scheme}://${location.host}${this.#base}/ws/jobs`; + } + + /* -- the server ------------------------------------------------------- */ + + /** Every call goes through here, so one place raises the event and one + * place knows that a refusal carries `{"detail": ""}`. */ + async #ask(path, options) { + const o = Object.assign({ credentials: "same-origin" }, options); + if (o.body !== undefined && typeof o.body !== "string") { + o.headers = Object.assign({ "Content-Type": "application/json" }, o.headers); + o.body = JSON.stringify(o.body); + } + let answer; + try { + answer = await fetch(this.#base + path, o); + } catch (e) { + this.#failed("Could not reach the server."); + throw e; + } + const text = await answer.text(); + let body = null; + if (text) { try { body = JSON.parse(text); } catch (e) { body = null; } } + if (!answer.ok) { + this.#failed((body && body.detail) + || `The server answered ${answer.status} and said nothing about why.`); + throw new Error(String((body && body.detail) || answer.status)); + } + return body; + } + + #failed(detail) { + // Composed, so a listener on `document` hears it through the shadow + // boundary. The wording is the server's; this element has no opinion. + this.dispatchEvent(new CustomEvent("desk-schedules:error", { + detail: { detail: String(detail) }, bubbles: true, composed: true, + })); + } + + async #load() { + try { + const [rows, kinds] = await Promise.all([ + this.#ask("/api/schedules"), + this.#ask("/api/jobs/kinds"), + ]); + this.#rows = rows || []; + this.#kinds = kinds || []; + } catch (e) { + /* already raised */ + } finally { + this.#loaded = true; + this.#render(); + } + } + + #connect() { + let socket; + try { + socket = new WebSocket(this.#socketUrl); + } catch { + this.#reconnect(); + return; + } + this.#socket = socket; + + socket.onopen = () => { + this.#wait = FIRST_WAIT; + if (this.#offline) { + // Down and back: what changed while we were away came as nothing, so + // the list is read again rather than waited for. + this.#offline = false; + this.#load(); + } + this.#render(); + }; + socket.onmessage = (event) => { + let message; + try { message = JSON.parse(event.data); } catch { return; } + if (message.type === "schedules") { + this.#rows = message.schedules || []; + this.#render(); + } else if (message.type === "jobs") { + this.#catchUp(message); + } else if (message.type === "notice") { + this.dispatchEvent(new CustomEvent("desk-jobs:notice", { + detail: message, bubbles: true, composed: true, + })); + } + }; + socket.onerror = () => socket.close(); + socket.onclose = () => { + if (this.#socket !== socket) return; // a later connection owns us + this.#socket = null; + this.#reconnect(); + }; + } + + /* The last column says how the last job ended, and the `schedules` message + * is pushed when a schedule *fires* — at which point that job has not ended + * and the cell reads `running`. Nothing pushes a schedule again when the + * job finishes, so without this the table goes on reporting a job as + * running long after it is done, which is the one thing this column is for. + * + * The job list arrives on this same socket four times a second while + * anything is going, and it carries the state of every job worth knowing + * about, so the answer is already here: take it off the snapshot rather + * than ask the server a second time. A job old enough to have left `recent` + * keeps whatever it was last seen as; its schedule has fired since, or it + * has been on screen for an hour. */ + #catchUp(snapshot) { + const states = new Map(); + for (const list of ["running", "queued", "recent"]) { + for (const job of snapshot[list] || []) states.set(job.id, job.state); + } + let moved = false; + for (const row of this.#rows) { + const state = states.get(row.last_job_id); + if (state && state !== row.last_state) { + row.last_state = state; + moved = true; + } + } + if (moved) this.#render(); + } + + #reconnect() { + if (this.#wait >= 4000) { + this.#offline = true; + this.#render(); + } + clearTimeout(this.#timer); + this.#timer = setTimeout(() => this.#connect(), this.#wait); + this.#wait = Math.min(this.#wait * 2, LONGEST_WAIT); + } + + /* One wrapper for every change: the row is greyed while the request is in + * the air, and the new list arrives on the socket rather than being patched + * in here — so what is on screen is always what the server holds. */ + async #change(id, run) { + this.#busy.add(id); + this.#render(); + try { + await run(); + } catch (e) { + /* already raised */ + } finally { + this.#busy.delete(id); + this.#render(); + } + } + + /* -- drawing ---------------------------------------------------------- */ + + #render() { + const panel = this.#root.lastElementChild; + panel.className = "panel"; + panel.replaceChildren(); + + if (this.#offline) { + const said = document.createElement("div"); + said.className = "offline"; + said.textContent = "Not connected — what you see here may be out of date."; + panel.append(said); + } + + const table = document.createElement("table"); + table.append(this.#head()); + const body = document.createElement("tbody"); + for (const row of this.#rows) body.append(this.#row(row)); + body.append(this.#adder()); + table.append(body); + panel.append(table); + + if (this.#loaded && !this.#rows.length) { + const empty = document.createElement("div"); + empty.className = "empty"; + empty.textContent = "Nothing is scheduled. Add one below."; + panel.insertBefore(empty, table); + } + } + + #head() { + const head = document.createElement("thead"); + const row = document.createElement("tr"); + for (const label of ["Name", "Kind", "Cadence", "On", "Next", "Last", ""]) { + const cell = document.createElement("th"); + cell.textContent = label; + row.append(cell); + } + head.append(row); + return head; + } + + #row(schedule) { + const row = document.createElement("tr"); + const busy = this.#busy.has(schedule.id); + if (!schedule.enabled) row.classList.add("off"); + + row.append(this.#cell("name", schedule.name)); + + const kind = this.#cell("kind", schedule.kind); + if (schedule.unknown) { + // Never fires, and says so rather than sitting there with a next run it + // is not going to keep. Deleting it is the only thing left to do. + const bad = document.createElement("span"); + bad.className = "state bad"; + bad.textContent = "unknown kind"; + kind.append(bad); + } + row.append(kind); + + const cadence = document.createElement("td"); + const box = document.createElement("input"); + box.className = "cadence"; + box.value = schedule.cadence; + box.title = FORMS; + box.disabled = busy; + const save = () => { + const wanted = box.value.trim(); + if (!wanted || wanted === schedule.cadence) { + box.value = schedule.cadence; + return; + } + this.#change(schedule.id, () => this.#ask(`/api/schedules/${schedule.id}`, + { method: "PATCH", body: { cadence: wanted } })); + }; + box.addEventListener("change", save); + box.addEventListener("keydown", (e) => { if (e.key === "Enter") box.blur(); }); + cadence.append(box); + row.append(cadence); + + const onOff = document.createElement("td"); + const toggle = document.createElement("input"); + toggle.type = "checkbox"; + toggle.checked = !!schedule.enabled; + toggle.disabled = busy; + toggle.title = schedule.enabled ? "switch this schedule off" : "switch it on"; + toggle.addEventListener("change", () => this.#change(schedule.id, + () => this.#ask(`/api/schedules/${schedule.id}`, + { method: "PATCH", body: { enabled: toggle.checked } }))); + onOff.append(toggle); + row.append(onOff); + + row.append(this.#cell("when", schedule.enabled ? when(schedule.next_run) : "—")); + + const last = this.#cell("when", when(schedule.last_run)); + if (schedule.last_state) { + const state = document.createElement("span"); + state.className = `state ${schedule.last_state}`; + state.textContent = schedule.last_state; + last.append(state); + } + row.append(last); + + const actions = document.createElement("td"); + actions.className = "actions"; + const now = document.createElement("button"); + now.textContent = "Run now"; + now.disabled = busy || schedule.unknown; + now.addEventListener("click", () => this.#change(schedule.id, + () => this.#ask(`/api/schedules/${schedule.id}/run`, { method: "POST" }))); + const remove = document.createElement("button"); + remove.className = "remove"; + remove.textContent = "Delete"; + remove.disabled = busy; + remove.addEventListener("click", () => this.#change(schedule.id, + () => this.#ask(`/api/schedules/${schedule.id}`, { method: "DELETE" }))); + actions.append(now, remove); + row.append(actions); + + return row; + } + + #cell(className, text) { + const cell = document.createElement("td"); + cell.className = className; + const span = document.createElement("span"); + span.textContent = text; + cell.append(span); + return cell; + } + + /** The add row. In the table rather than in a form above it, because what + * is being added is another one of these and the columns already say what + * each box is for. */ + #adder() { + const row = document.createElement("tr"); + const busy = this.#busy.has(0); + + const name = document.createElement("input"); + name.className = "text"; + name.placeholder = "optional"; + name.disabled = busy; + row.append(this.#wrap(name)); + + const kind = document.createElement("select"); + kind.disabled = busy || !this.#kinds.length; + for (const k of this.#kinds) { + const option = document.createElement("option"); + option.value = k.kind; + option.textContent = k.label || k.kind; + kind.append(option); + } + if (!this.#kinds.length) { + const none = document.createElement("option"); + none.textContent = this.#loaded ? "nothing registered" : "loading…"; + kind.append(none); + } + row.append(this.#wrap(kind)); + + const cadence = document.createElement("input"); + cadence.className = "text"; + cadence.placeholder = HINT; + cadence.title = FORMS; + cadence.disabled = busy; + row.append(this.#wrap(cadence)); + + row.append(document.createElement("td"), document.createElement("td"), + document.createElement("td")); + + const actions = document.createElement("td"); + actions.className = "actions"; + const add = document.createElement("button"); + add.textContent = "Add"; + add.disabled = busy; + const submit = () => { + if (!kind.value) return; + this.#change(0, async () => { + await this.#ask("/api/schedules", { + method: "POST", + body: { name: name.value.trim(), kind: kind.value, + cadence: cadence.value.trim() }, + }); + // Cleared only on success, so a refused cadence is still there to be + // corrected rather than retyped from memory. + name.value = ""; + cadence.value = ""; + }); + }; + add.addEventListener("click", submit); + for (const box of [name, cadence]) { + box.addEventListener("keydown", (e) => { if (e.key === "Enter") submit(); }); + } + actions.append(add); + row.append(actions); + return row; + } + + #wrap(control) { + const cell = document.createElement("td"); + cell.append(control); + return cell; + } +} + +if (!customElements.get("desk-schedules")) { + customElements.define("desk-schedules", DeskSchedules); +} + +export { DeskSchedules }; diff --git a/pipeline/worker.py b/pipeline/worker.py new file mode 100644 index 0000000..a5ce607 --- /dev/null +++ b/pipeline/worker.py @@ -0,0 +1,113 @@ +"""A worker with no web app around it. + + python -m pipeline.worker --db app.db --bootstrap myapp.jobs:bootstrap + python -m pipeline.worker --db app.db --bootstrap myapp.jobs:bootstrap \\ + --bootstrap-arg config.yaml --workers cpu=2,http=8 --until-idle + +What a second machine, a cron entry or a batch script runs. It is the same +`JobQueue` the web app starts, on the same table, so it races correctly against +any other worker on that file: `Jobs.claim` is one atomic statement and every +claimed job is held on a lease. Start as many as the file will bear. + +The bootstrap is the one `isolate=True` already asks for — `module:function` +returning `(config, store)` — and it is used for the same two reasons here: +importing it fills `JOBS`, and calling it builds the pair the handlers take. +An app that has written one for isolation has written this worker's too. + +`--until-idle` drains what is queued and exits, which is the batch shape. Without +it the worker runs until it is interrupted, and a SIGINT or SIGTERM stops it the +way the web app stops: handlers in flight get `JobQueue.STOP_GRACE` to land, +and what is still running after that goes back to the queue on its lease. + +**This worker fires schedules too**, which is what makes it a complete +replacement for the web app's own worker — a machine running only this still +gets its 06:00 job. `--no-schedules` turns that off, and is what every worker +after the first passes: they all share the jobs table safely, but the pile-up +guard reads `last_job_id`, which neither of two simultaneous firings has +written yet when the other looks. One scheduler per file, as many workers as +you like. `--timezone` is what the clock cadences are read in; without it they +are read in the machine's own zone, which on a server is UTC and is usually not +what the person who wrote `daily at 06:00` meant. +""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import logging +import signal + +from pipeline.child import resolve +from pipeline.queue import JobQueue +from pipeline.table import Jobs + +log = logging.getLogger(__name__) + + +def workers(text: str) -> dict[str, int]: + """`cpu=2,http=8` to `{"cpu": 2, "http": 8}`.""" + counts: dict[str, int] = {} + for part in filter(None, (p.strip() for p in text.split(","))): + name, _, n = part.partition("=") + if not name or not n.isdigit(): + raise argparse.ArgumentTypeError(f"not resource=count: {part!r}") + counts[name] = int(n) + return counts + + +async def serve(queue: JobQueue) -> None: + """Run until SIGINT or SIGTERM.""" + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for sig in (signal.SIGINT, signal.SIGTERM): + # Windows has no loop signal handlers; there Ctrl+C arrives as + # KeyboardInterrupt instead, and `main` turns it into the same stop. + with contextlib.suppress(NotImplementedError, RuntimeError): + loop.add_signal_handler(sig, stop.set) + await queue.start() + try: + await stop.wait() + finally: + await queue.stop() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="pipeline.worker", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--db", required=True, help="the SQLite file the jobs table is on") + parser.add_argument("--bootstrap", required=True, + help="module:function returning (config, store)") + parser.add_argument("--bootstrap-arg", default="", + help="one string handed to the bootstrap, e.g. a config path") + parser.add_argument("--workers", type=workers, default=None, + help="resource=count,... (default: JobQueue.WORKERS)") + parser.add_argument("--until-idle", action="store_true", + help="drain what is queued, then exit") + parser.add_argument("--no-schedules", action="store_true", + help="do not fire schedules here; for a second worker " + "on a file that already has a scheduler") + parser.add_argument("--timezone", default="", + help="what 'daily at 06:00' is 06:00 in " + "(default: this machine's own zone)") + args = parser.parse_args(argv) + + build = resolve(args.bootstrap) + config, store = build(args.bootstrap_arg) if args.bootstrap_arg else build() + queue = JobQueue(Jobs(args.db), config, store, workers=args.workers, + bootstrap=args.bootstrap, bootstrap_arg=args.bootstrap_arg, + tz=args.timezone, schedules=not args.no_schedules) + try: + if args.until_idle: + queue.drain() + else: + asyncio.run(serve(queue)) + except KeyboardInterrupt: + pass + return 0 + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s") + raise SystemExit(main()) diff --git a/tests/test_coupling.py b/tests/test_coupling.py new file mode 100644 index 0000000..d6cc527 --- /dev/null +++ b/tests/test_coupling.py @@ -0,0 +1,75 @@ +"""The coupling rule, held by reading the imports. + +`pipeline/` and `auth/` import nothing from `app/` and nothing from each +other. `app/` imports both. That is what makes each of them copyable on its +own into a project that has never heard of this one -- and it is exactly the +rule that breaks quietly, one convenient import at a time, until the folder is +a port rather than a copy. + +Read with `ast` rather than grepped: a string containing the word `app`, or a +line in a docstring showing what an app would write, is not an import, and a +test that cannot tell the difference is a test people learn to work around. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent + +#: The module folders and what each may not reach for. Listed rather than +#: derived so that adding a fourth folder is a decision, not an accident. +FORBIDDEN = { + "pipeline": ("app", "auth"), + "auth": ("app", "pipeline"), +} + + +def imported_modules(source: str) -> set[str]: + """Every top-level package name this file imports.""" + found: set[str] = set() + tree = ast.parse(source) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + found.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom): + # A relative import stays inside the folder by construction. + if node.level == 0 and node.module: + found.add(node.module.split(".")[0]) + return found + + +def python_files(folder: str) -> list[Path]: + root = ROOT / folder + if not root.is_dir(): + return [] + return sorted(p for p in root.rglob("*.py") if "__pycache__" not in p.parts) + + +CASES = [(folder, path) for folder in FORBIDDEN for path in python_files(folder)] + + +@pytest.mark.parametrize( + ("folder", "path"), CASES, + ids=[str(path.relative_to(ROOT)).replace("\\", "/") for _, path in CASES]) +def test_a_module_folder_imports_nothing_above_it(folder: str, path: Path): + imported = imported_modules(path.read_text(encoding="utf-8")) + for name in FORBIDDEN[folder]: + assert name not in imported, ( + f"{path.relative_to(ROOT)} imports {name!r}. " + f"{folder}/ has to survive being copied on its own.") + + +@pytest.mark.parametrize("folder", sorted(FORBIDDEN)) +def test_the_folder_is_actually_there(folder: str): + """A parametrised test over an empty list passes by saying nothing. + + So the rule is only proved for what exists, and this is what makes "it + exists" a thing the suite checks rather than a thing somebody assumes -- + the day a folder is moved or renamed, the coupling test would otherwise go + quiet instead of red. + """ + assert python_files(folder), f"no {folder}/ to check" diff --git a/tests/test_packaging.py b/tests/test_packaging.py new file mode 100644 index 0000000..7412e0d --- /dev/null +++ b/tests/test_packaging.py @@ -0,0 +1,75 @@ +"""What has to be true about the repository itself, not about the code in it. + +Three facts that are invisible until they cost a deployment: a PowerShell +script that will not parse, a folder that is not in the wheel, and a container +missing a package the app imports. +""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +SCRIPTS = sorted((ROOT / "deploy").rglob("*.ps1")) + + +@pytest.mark.parametrize("script", SCRIPTS, ids=lambda p: p.name) +def test_powershell_scripts_are_ascii(script: Path): + """Windows PowerShell 5.1 reads a BOM-less file as Windows-1252. + + So an em dash becomes three characters, the last of which is a closing + quote: the string ends in the middle of itself and the script fails to + parse before it runs a line. It happened on a comment nobody thought was + code. Write `--`, `->`, `...`. + """ + raw = script.read_bytes() + bad = [(i, b) for i, b in enumerate(raw) if b > 0x7F] + assert not bad, ( + f"{script.relative_to(ROOT)} has non-ASCII bytes at offsets " + f"{[i for i, _ in bad[:5]]} -- PowerShell 5.1 will mis-read them" + ) + + +def test_there_are_scripts_to_check(): + """A parametrised test over an empty list passes by saying nothing.""" + assert SCRIPTS, "no .ps1 found under deploy/ -- did the copy go wrong?" + + +def test_pyproject_packages_the_module_folders(): + data = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) + include = data["tool"]["setuptools"]["packages"]["find"]["include"] + for pattern in ("app*", "pipeline*", "auth*"): + assert pattern in include, f"{pattern} is missing: an install would leave it out" + + +def test_no_test_folder_travels_into_the_image(): + """`COPY auth auth` copies `auth/tests` with it unless the ignore file says + otherwise, and a module folder added later brings its own. + + Nothing breaks when they arrive, which is why nobody notices: the image is + just carrying a test suite, its fixtures and whatever they import, in a + container on a server. One line per folder, next to the recipe. + """ + ignored = {line.strip() for line + in (ROOT / "deploy" / "Dockerfile.dockerignore") + .read_text(encoding="utf-8").splitlines()} + for folder in sorted(p.parent for p in ROOT.glob("*/tests")): + name = f"{folder.name}/tests" + assert name in ignored, f"{name} is not in Dockerfile.dockerignore" + + +def test_the_dockerfile_copies_the_module_folders(): + """`COPY app app` alone builds an image whose app cannot import its queue. + + The failure is at startup, in a container, on a server -- which is the + worst of the three places to find out. + """ + dockerfile = (ROOT / "deploy" / "Dockerfile").read_text(encoding="utf-8") + lines = [line.strip() for line in dockerfile.splitlines()] + for folder in ("pipeline", "auth"): + copies = [line for line in lines if line == f"COPY {folder} {folder}"] + assert len(copies) == 2, ( + f"expected `COPY {folder} {folder}` in both stages, found {len(copies)}")