Files
jobboard/app/config.py
T
2026-09-30 15:44:32 +02:00

275 lines
11 KiB
Python

"""Config loading: one YAML file, validated into a model.
The boilerplate's shape with roledesk's keys. A typo in a key is an error
naming the key, because every section forbids extras: a setting you believe
you have changed and have not is the expensive kind of typo. `sources` stays a
dict: one block per plugin, whose keys only the plugin knows.
The path comes from `app --config`, else `APP_CONFIG`, else `config.yaml`
beside the checkout. The Dockerfile sets `APP_CONFIG=/data/config.yaml`.
**A missing file is not an error.** Every field has a default that runs, so
`uv run app serve` works on a fresh clone. The committed `config.yaml` adds
the starter agents; `config.example.yaml` documents every key.
"""
from __future__ import annotations
import os
import sys
from pathlib import Path
from typing import Any
import yaml
from pydantic import Field, ValidationError, field_validator, model_validator
from app.models import Gate, Limits, Prompts, Weights
from auth import AuthConfig
from desk.config import AgentConfig, ModelTaskConfig, Section, WorkersConfig
#: Where `load()` looks when nobody says. Read at import so a test can set the
#: variable first, and `app --config` overrides it after.
DEFAULT_PATH = Path(os.environ.get("APP_CONFIG", "config.yaml"))
class ServerConfig(Section):
"""Where uvicorn binds. Anything but a loopback address makes
`auth.enabled: auto` lock the app."""
host: str = "127.0.0.1"
port: int = 8790
class ExtractConfig(ModelTaskConfig):
"""Turning a posting's title and text into the gate columns. See
app/jobs/extract.py. Off by default, so nobody is billed by surprise:
on the API backend every batch is a paid call. On, it reads only the
best roles of each enabled agent, as deep and as low as that agent's
own `limits` say, so the cost follows the ranking rather than the
boards."""
enabled: bool = False
model: str = "claude-haiku-4-5-20251001"
batch_size: int = Field(20, ge=1)
class SummariseConfig(ModelTaskConfig):
"""About 60 words on a role, for the list and the read page. See
app/jobs/summarise.py. Off by default like extract; on, a pull summarises
only the best of each enabled agent, as deep and as low as that agent's
own `limits` say, so the cost is bounded whatever the backlog. The read
page's Summarise button asks for one role whether this is on or not."""
enabled: bool = False
model: str = "claude-haiku-4-5-20251001"
#: A description is capped at 8000 characters per role, so ten is about
#: 20k tokens of input per call.
batch_size: int = Field(10, ge=1)
class EnrichConfig(Section):
"""Reading roles on their own pages after scoring, for the hosts whose
list carries no text. See app/jobs/enrich.py. A role is read when it has
no text and an enabled agent with a prompt ranks it ungated among its
best `top` and at `min_score` or above: the extract bar (`Limits`,
500 and 0.55) with the score a notch lower, since a role ranked on its
title alone is the one whose score its text moves most."""
enabled: bool = True
top: int = Field(500, ge=0)
min_score: float = Field(0.5, ge=0, le=1)
#: Reads per board and per run. Per board is the cap the pull had; the
#: boards are read in parallel on the `http` pool.
per_board: int = Field(200, ge=0)
per_run: int = Field(1000, ge=0)
#: Days before a page that gave no text is read again.
retry_days: int = Field(7, ge=1)
class Calibration(Section):
"""Where one embedding model's cosines sit, as `(lo, hi)` for fit and for
taste: `score.scale` maps `lo` to 0 and `hi` to 1 and clips outside, so a
term is a level rather than a rank within the run."""
fit: tuple[float, float]
taste: tuple[float, float]
@field_validator("fit", "taste")
@classmethod
def _lo_below_hi(cls, band: tuple[float, float]) -> tuple[float, float]:
lo, hi = band
if not -1 <= lo < hi <= 1:
raise ValueError(f"needs -1 <= lo < hi <= 1, got [{lo}, {hi}]")
return band
#: The built-in calibration, by embedding model. Measured on a copy of the
#: live database, 16,841 open roles (see docs/decisions.md). fit: pooled over
#: five intents the raw cosine has p1 0.753 and p99.9 0.871; hi sits above
#: every single intent's p99.9, so only about 5 of the starter agent's 4,156
#: ungated roles clip to 1. taste: from synthetic votes, 30 liked and 30
#: passed roles for each of three intents, since the copy had none (pooled p1
#: -0.140, p99.9 0.192); re-measure it once real votes exist.
CALIBRATION: dict[str, Calibration] = {
"intfloat/multilingual-e5-small": Calibration(fit=(0.75, 0.88), taste=(-0.14, 0.19)),
}
class EmbedConfig(Section):
"""Turning a role into a vector for `fit`. Needs sentence-transformers;
without it `embed` says so and does nothing, and `score` leaves fit out.
See app/jobs/embed.py.
`intfloat/multilingual-e5-small`, not the English-only
`sentence-transformers/all-MiniLM-L6-v2` roledesk started with: measured
on the live roles against the "Copenhagen engineering" agent's intent,
MiniLM separates engineering from non-engineering roles with an AUC of
0.68 (0.74 restricted to Danish-country roles), multilingual-e5-small
with an AUC of 0.70 (1.00 restricted to Danish-country roles), for about
3x the CPU time. See docs/decisions.md.
"""
#: Recorded on every vector. Changing it drops the old vectors, which are
#: a different geometry and never comparable. An e5 model needs its
#: "query: " / "passage: " prefixes, which app/jobs/embed.py adds by
#: matching this name, not a separate setting.
model: str = "intfloat/multilingual-e5-small"
#: Roles embedded per run at most, newest first.
per_run: int = Field(50_000, ge=1)
#: `{model: {fit: [lo, hi], taste: [lo, hi]}}`, over `CALIBRATION`: an
#: entry replaces the built-in one for that model. Keyed by model, so a
#: change of `model` cannot carry another geometry's constants along. A
#: model with neither is min-max scaled per run, with a warning in the log.
calibration: dict[str, Calibration] = Field(default_factory=dict)
def calibrated(self) -> Calibration | None:
"""The configured model's calibration, or None when it has none."""
return self.calibration.get(self.model) or CALIBRATION.get(self.model)
class AgentSeed(Section):
"""A search agent as `config.yaml` lists it. Imported into an empty
`agents` table once; after that the table is the answer. `prompts` is a
list of `{name, text}`; without it, `intent` is the one prompt, named
after the agent."""
name: str
prompts: Prompts | None = None
intent: str = ""
gates: list[Gate] = Field(default_factory=list)
weights: Weights = Field(default_factory=Weights)
limits: Limits = Field(default_factory=Limits)
enabled: bool = True
class SearchSeed(Section):
"""A saved search as `config.yaml` lists it: one `search` subscription,
imported once like the agents. `params` are the provider's own, see
plugins/search.py: `q`, `area` and `pages` for jobindex; `q`, `country`,
`where` and `pages` for adzuna; `q` and `region` for platsbanken. The provider is any reader of
`scope="search"`, so a new one needs no edit here."""
name: str
provider: str
params: dict[str, Any] = Field(default_factory=dict)
@field_validator("provider")
@classmethod
def _a_search_provider(cls, provider: str) -> str:
# `plugins` is a top-level folder beside `app` and is not on the path
# when this module is imported first, so it is reached the way
# `app.models` reaches it (`_declared_posted`), and only when a seed
# is validated.
root = str(Path(__file__).resolve().parents[1])
if root not in sys.path:
sys.path.insert(0, root)
from plugins.readers import PROVIDERS
if provider not in PROVIDERS:
raise ValueError(f"provider is one of {', '.join(PROVIDERS)}, not {provider!r}")
return provider
class Config(Section):
"""The whole file."""
server: ServerConfig = Field(default_factory=ServerConfig)
#: One SQLite file for the lot: roles and companies, the jobs table, the
#: notice log, the schedules, and auth's `auth_*` tables. One file is one
#: thing to back up.
database: Path = Path("roledesk.db")
#: What a clock on screen means, and what a `daily at 06:00` schedule is
#: 06:00 in. `every 6h` is six elapsed hours in any zone.
timezone: str = "Europe/Berlin"
log_level: str = "INFO"
#: The door, auth's own model. See auth/README.md.
auth: AuthConfig = Field(default_factory=AuthConfig)
#: Directories searched for `Source` plugins. A plugin imports `desk` only.
plugin_paths: list[str] = Field(default_factory=lambda: ["plugins"])
#: Workers per resource class. `http` is wide because a fetch is mostly waiting.
workers: WorkersConfig = Field(default_factory=WorkersConfig)
#: Who answers when a job needs a model. See desk/tools/agent.py.
agent: AgentConfig = Field(default_factory=AgentConfig)
extract: ExtractConfig = Field(default_factory=ExtractConfig)
summarise: SummariseConfig = Field(default_factory=SummariseConfig)
embed: EmbedConfig = Field(default_factory=EmbedConfig)
enrich: EnrichConfig = Field(default_factory=EnrichConfig)
#: Starter agents, imported once into an empty table.
agents: list[AgentSeed] = Field(default_factory=list)
#: Starter searches, imported once into an empty set of `search`
#: subscriptions.
searches: list[SearchSeed] = Field(default_factory=list)
#: One block per plugin: `enabled`, and whatever that plugin reads.
sources: dict[str, Any] = Field(default_factory=dict)
#: Where this came from, or None for the defaults. Never written back.
path: Path | None = Field(default=None, exclude=True)
@model_validator(mode="after")
def _credentials_live_in_the_database(self) -> Config:
"""`desk.tools.agent` reads a stored token from the app's database."""
self.agent.credentials_database = str(self.database)
return self
class ConfigError(ValueError):
"""A config file that exists but does not validate. The message names
every bad key, for the CLI to print as it is."""
def load(path: str | os.PathLike[str] | None = None) -> Config:
"""Read the YAML file into a `Config`, or return the defaults.
An empty file is the same as no file. A file that is not a mapping, or
that does not validate, is a `ConfigError`.
"""
where = Path(path).expanduser() if path else DEFAULT_PATH
if not where.exists():
return Config()
raw: Any = yaml.safe_load(where.read_text(encoding="utf-8"))
if raw is None:
return Config(path=where)
if not isinstance(raw, dict):
raise ConfigError(f"{where}: expected a mapping of settings, "
f"got {type(raw).__name__}")
try:
return Config(**raw, path=where)
except ValidationError as exc:
raise ConfigError(f"{where}:\n{_explain(exc)}") from None
def _explain(exc: ValidationError) -> str:
lines = []
for err in exc.errors():
where = ".".join(str(p) for p in err["loc"]) or "(top level)"
detail = err["msg"]
if err["type"] == "extra_forbidden":
detail = "unknown setting, check the spelling against config.example.yaml"
lines.append(f" {where}: {detail}")
return "\n".join(lines)