Skeleton: roledesk on desk and pipeline from ../news
Package layers models, config, storage, jobs, web, cli, held by tests/test_architecture.py with desk and pipeline as external trees and plugins/ limited to desk. roles.yaml is the default config, roledesk.db its own file, port 8790. create_app mounts pipeline.web.connect over Jobs(config.database); `roledesk job pull` drains a no-op pull and `roledesk db init` creates the file. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
36dbcbb29e
20 files changed
+2320
No files matched your search
@@ -0,0 +1,16 @@
|
||||
"""Where things are kept. One door: `from roledesk.storage import Store`.
|
||||
|
||||
schema.py the tables, and the numbered migrations that bring an older
|
||||
file to the current shape
|
||||
store.py `Store`, every SQL statement in the application
|
||||
|
||||
Written for roledesk rather than imported from newsdesk: `desk` has no storage
|
||||
package on purpose, and the connection plumbing here is a deliberate copy.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from roledesk.storage.schema import LATEST
|
||||
from roledesk.storage.store import Store
|
||||
|
||||
__all__ = ["LATEST", "Store"]
|
||||
@@ -0,0 +1,131 @@
|
||||
"""The tables, and how a database file gets from an old shape to this one.
|
||||
|
||||
`SCHEMA` is the current shape. A new file is created from it and stamped with
|
||||
`LATEST`, so it never runs a migration. `MIGRATIONS` is how an existing file
|
||||
catches up: ordered, numbered, applied once each, tracked in SQLite's own
|
||||
`PRAGMA user_version`.
|
||||
|
||||
Adding a column means editing both: the DDL in `SCHEMA` for new files, and an
|
||||
appended migration for old ones. Never renumber or edit a migration that has
|
||||
shipped.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SCHEMA = """
|
||||
-- The string map a desk plugin is handed as `self.state`. One namespace shared
|
||||
-- by every plugin, so a plugin prefixes its keys.
|
||||
CREATE TABLE IF NOT EXISTS state (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
"""
|
||||
|
||||
#: Applied after `SCHEMA` or the migrations, every open. An index may name a
|
||||
#: column only a migration adds, which is why these are kept apart.
|
||||
INDEXES = """
|
||||
"""
|
||||
|
||||
#: The table whose absence marks a file that has never been a roledesk
|
||||
#: database, and is therefore created from `SCHEMA` rather than migrated.
|
||||
MARKER = "state"
|
||||
|
||||
Step = str | Callable[[sqlite3.Connection], None]
|
||||
|
||||
MIGRATIONS: tuple[tuple[int, str, Step], ...] = (
|
||||
# Never run as a migration: a file without the marker table is created
|
||||
# from `SCHEMA` and stamped. It exists so the first real migration is 2.
|
||||
(1, "the first schema", ""),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
LATEST = MIGRATIONS[-1][0]
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
def _stamp(conn: sqlite3.Connection, version: int) -> None:
|
||||
# PRAGMA takes no bound parameters; the value is an int from MIGRATIONS.
|
||||
conn.execute(f"PRAGMA user_version = {int(version)}")
|
||||
|
||||
|
||||
def _statements(script: str) -> list[str]:
|
||||
"""A script split into statements, cut where `sqlite3.complete_statement`
|
||||
says one ends. Not `executescript`, which commits before it starts and
|
||||
autocommits each statement, so a failing migration would be half applied."""
|
||||
statements, buffer = [], ""
|
||||
for line in script.splitlines(keepends=True):
|
||||
buffer += line
|
||||
if sqlite3.complete_statement(buffer):
|
||||
statements.append(buffer.strip())
|
||||
buffer = ""
|
||||
if buffer.strip():
|
||||
statements.append(buffer.strip())
|
||||
return statements
|
||||
|
||||
|
||||
def migrate(conn: sqlite3.Connection) -> list[int]:
|
||||
"""Bring an existing database up to `LATEST`. Returns what it applied."""
|
||||
current = user_version(conn)
|
||||
if current > LATEST:
|
||||
raise RuntimeError(
|
||||
f"this database is at schema v{current}, but this roledesk only knows "
|
||||
f"up to v{LATEST}. Upgrade roledesk rather than let older code write to it.")
|
||||
applied = []
|
||||
for version, description, step in MIGRATIONS:
|
||||
if version <= current:
|
||||
continue
|
||||
log.info("migrating database to v%d: %s", version, description)
|
||||
_apply(conn, version, step)
|
||||
applied.append(version)
|
||||
return applied
|
||||
|
||||
|
||||
def _apply(conn: sqlite3.Connection, version: int, step: Step) -> None:
|
||||
"""One migration and its version stamp, in one explicit transaction.
|
||||
|
||||
Explicit because sqlite3 only opens a transaction implicitly before DML: an
|
||||
`ALTER` would otherwise be durable at once while the stamp stayed behind,
|
||||
and the next open would replay it into a "duplicate column" forever.
|
||||
"""
|
||||
started = not conn.in_transaction
|
||||
if started:
|
||||
conn.execute("BEGIN")
|
||||
try:
|
||||
if callable(step):
|
||||
step(conn)
|
||||
else:
|
||||
for statement in _statements(step):
|
||||
conn.execute(statement)
|
||||
_stamp(conn, version)
|
||||
except Exception:
|
||||
if started:
|
||||
conn.execute("ROLLBACK")
|
||||
raise
|
||||
if started:
|
||||
conn.execute("COMMIT")
|
||||
|
||||
|
||||
def initialise(conn: sqlite3.Connection) -> list[int]:
|
||||
"""Create or upgrade, whichever this file needs. Returns migrations applied."""
|
||||
fresh = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (MARKER,)
|
||||
).fetchone() is None
|
||||
applied: list[int] = []
|
||||
if fresh:
|
||||
# Atomic like a migration: a file half created would otherwise carry
|
||||
# the marker table and be taken for one to migrate.
|
||||
_apply(conn, LATEST, SCHEMA)
|
||||
else:
|
||||
applied = migrate(conn)
|
||||
for statement in _statements(INDEXES):
|
||||
conn.execute(statement)
|
||||
return applied
|
||||
@@ -0,0 +1,103 @@
|
||||
"""`Store`: every SQL statement in the application.
|
||||
|
||||
The connection plumbing is a copy of newsdesk's `Store`, on purpose: `desk`
|
||||
has no storage package. A connection is opened per call and closed again, so
|
||||
one caller's commit can never land inside another's transaction; in WAL mode
|
||||
that costs a file handle.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from collections.abc import Iterator, MutableMapping
|
||||
from contextlib import closing, contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from roledesk.storage import schema
|
||||
|
||||
|
||||
class _StateMap(MutableMapping[str, str]):
|
||||
"""`Store.state`: the key/value table, read and written one key at a time."""
|
||||
|
||||
def __init__(self, store: Store) -> None:
|
||||
self._store = store
|
||||
|
||||
def __getitem__(self, key: str) -> str:
|
||||
row = self._store._row("SELECT value FROM state WHERE key=?", (key,))
|
||||
if row is None:
|
||||
raise KeyError(key)
|
||||
return str(row["value"])
|
||||
|
||||
def __setitem__(self, key: str, value: str) -> None:
|
||||
self._store._write(
|
||||
"""INSERT INTO state (key, value) VALUES (?,?)
|
||||
ON CONFLICT(key) DO UPDATE SET value=excluded.value""",
|
||||
(key, value))
|
||||
|
||||
def __delitem__(self, key: str) -> None:
|
||||
if not self._store._write("DELETE FROM state WHERE key=?", (key,)):
|
||||
raise KeyError(key)
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
return iter([r["key"] for r in self._store._rows("SELECT key FROM state ORDER BY key")])
|
||||
|
||||
def __len__(self) -> int:
|
||||
row = self._store._row("SELECT COUNT(*) AS n FROM state")
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, path: str | Path) -> None:
|
||||
self.path = Path(path).expanduser()
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self.transaction() as conn:
|
||||
# Persistent: recorded in the file, inherited by every connection.
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
#: Migration versions this open applied; empty when already current.
|
||||
self.migrations_applied: list[int] = schema.initialise(conn)
|
||||
|
||||
# ------------------------------------------------------------ connections
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
"""A fresh connection. `foreign_keys` is off by default per connection;
|
||||
`busy_timeout` matches `pipeline/table.py`, which writes the same file;
|
||||
`synchronous=NORMAL` is the standard WAL companion."""
|
||||
conn = sqlite3.connect(self.path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
conn.execute("PRAGMA busy_timeout=30000")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
return conn
|
||||
|
||||
@contextmanager
|
||||
def transaction(self) -> Iterator[sqlite3.Connection]:
|
||||
"""One connection, one transaction, closed on the way out."""
|
||||
conn = self._connect()
|
||||
try:
|
||||
with conn:
|
||||
yield conn
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _rows(self, sql: str, params: Any = ()) -> list[sqlite3.Row]:
|
||||
with closing(self._connect()) as conn:
|
||||
return conn.execute(sql, params).fetchall()
|
||||
|
||||
def _row(self, sql: str, params: Any = ()) -> sqlite3.Row | None:
|
||||
with closing(self._connect()) as conn:
|
||||
row: sqlite3.Row | None = conn.execute(sql, params).fetchone()
|
||||
return row
|
||||
|
||||
def _write(self, sql: str, params: Any = ()) -> int:
|
||||
"""One statement in its own transaction. Returns the rowcount."""
|
||||
with self.transaction() as conn:
|
||||
return conn.execute(sql, params).rowcount
|
||||
|
||||
# ---------------------------------------------------------------- state
|
||||
|
||||
@property
|
||||
def state(self) -> MutableMapping[str, str]:
|
||||
"""The `state` table as a mapping. What the runner hands a plugin as
|
||||
`self.state`: it can keep a counter without being able to read roles."""
|
||||
return _StateMap(self)
|
||||
Reference in new issue
Block a user