Files
jobboard/app/config.py
T

294 lines
12 KiB
Python

"""Config loading: one YAML file, validated into a model.
The boilerplate's shape with roledesk's keys. A typo in a key is an error
naming the key, because every section forbids extras: a setting you believe
you have changed and have not is the expensive kind of typo. `sources` stays a
dict: one block per plugin, whose keys only the plugin knows.
The path comes from `app --config`, else `APP_CONFIG`, else `config.yaml`
beside the checkout. The Dockerfile sets `APP_CONFIG=/data/config.yaml`.
**A missing file is not an error.** Every field has a default that runs, so
`uv run app serve` works on a fresh clone. The committed `config.yaml` adds
the starter agents; `config.example.yaml` documents every key.
"""
from __future__ import annotations
import os
import sys
from pathlib import Path
from typing import Any
import yaml
from pydantic import Field, ValidationError, field_validator, model_validator
from app.models import Gate, Limits, Prompts, Weights
from auth import AuthConfig
from desk.config import AgentConfig, ModelTaskConfig, Section, WorkersConfig
#: Where `load()` looks when nobody says. Read at import so a test can set the
#: variable first, and `app --config` overrides it after.
DEFAULT_PATH = Path(os.environ.get("APP_CONFIG", "config.yaml"))
class ServerConfig(Section):
"""Where uvicorn binds. Anything but a loopback address makes
`auth.enabled: auto` lock the app."""
host: str = "127.0.0.1"
port: int = 8790
class ExtractConfig(ModelTaskConfig):
"""Turning a posting's title and text into the gate columns. See
app/jobs/extract.py. Off by default, so nobody is billed by surprise:
on the API backend every batch is a paid call. On, it reads only the
best roles of each enabled agent, as deep and as low as that agent's
own `limits` say, so the cost follows the ranking rather than the
boards."""
enabled: bool = False
model: str = "claude-haiku-4-5-20251001"
batch_size: int = Field(20, ge=1)
class SummariseConfig(ModelTaskConfig):
"""About 60 words on a role, for the list and the read page. See
app/jobs/summarise.py. Off by default like extract; on, a pull summarises
the best of each enabled agent, as deep and as low as that agent's
own `limits` say, so the cost is bounded whatever the backlog, and every
role that is saved or liked, whatever its rank. The read page's Summarise button asks for one
role whether this is on or not."""
enabled: bool = False
model: str = "claude-haiku-4-5-20251001"
#: A description is capped at 8000 characters per role, so ten is about
#: 20k tokens of input per call.
batch_size: int = Field(10, ge=1)
class JudgeConfig(ModelTaskConfig):
"""Each search agent's best roles read against its CV, as three numbers
and a total. See app/jobs/judge.py. Off by default like extract; on, a
pull judges the best of each enabled agent with a CV, as deep and as low
as that agent's own `limits` say (`judge_top`, `judge_min`), and only
the roles whose judgement is missing or was read from another CV or
text."""
enabled: bool = False
model: str = "claude-haiku-4-5-20251001"
#: A posting is capped at 8000 characters and the CV rides in the system
#: prompt of every call, so batches stay small.
batch_size: int = Field(5, ge=1)
class EnrichConfig(Section):
"""Reading roles on their own pages after scoring, for the hosts whose
list carries no text. See app/jobs/enrich.py. A role is read when it has
no text and an enabled agent with a prompt ranks it ungated among its
best `top` and at `min_score` or above: the extract bar (`Limits`,
500 and 0.55) with the score a notch lower, since a role ranked on its
title alone is the one whose score its text moves most."""
enabled: bool = True
top: int = Field(500, ge=0)
min_score: float = Field(0.5, ge=0, le=1)
#: Reads per board and per run. Per board is the cap the pull had; the
#: boards are read in parallel on the `http` pool.
per_board: int = Field(200, ge=0)
per_run: int = Field(1000, ge=0)
#: Days before a page that gave no text is read again.
retry_days: int = Field(7, ge=1)
class Calibration(Section):
"""Where one embedding model's cosines sit, as `(lo, hi)` for fit and for
taste: `score.scale` maps `lo` to 0 and `hi` to 1 and clips outside, so a
term is a level rather than a rank within the run."""
fit: tuple[float, float]
taste: tuple[float, float]
@field_validator("fit", "taste")
@classmethod
def _lo_below_hi(cls, band: tuple[float, float]) -> tuple[float, float]:
lo, hi = band
if not -1 <= lo < hi <= 1:
raise ValueError(f"needs -1 <= lo < hi <= 1, got [{lo}, {hi}]")
return band
#: The built-in calibration, by embedding model. Measured on a copy of the
#: live database, 16,841 open roles (see docs/decisions.md). fit: pooled over
#: five intents the raw cosine has p1 0.753 and p99.9 0.871; hi sits above
#: every single intent's p99.9, so only about 5 of the starter agent's 4,156
#: ungated roles clip to 1. taste: from synthetic votes, 30 liked and 30
#: passed roles for each of three intents, since the copy had none (pooled p1
#: -0.140, p99.9 0.192); re-measure it once real votes exist.
CALIBRATION: dict[str, Calibration] = {
"intfloat/multilingual-e5-small": Calibration(fit=(0.75, 0.88), taste=(-0.14, 0.19)),
}
class EmbedConfig(Section):
"""Turning a role into a vector for `fit`. Needs sentence-transformers;
without it `embed` says so and does nothing, and `score` leaves fit out.
See app/jobs/embed.py.
`intfloat/multilingual-e5-small`, not the English-only
`sentence-transformers/all-MiniLM-L6-v2` roledesk started with: measured
on the live roles against the "Copenhagen engineering" agent's intent,
MiniLM separates engineering from non-engineering roles with an AUC of
0.68 (0.74 restricted to Danish-country roles), multilingual-e5-small
with an AUC of 0.70 (1.00 restricted to Danish-country roles), for about
3x the CPU time. See docs/decisions.md.
"""
#: Recorded on every vector. Changing it drops the old vectors, which are
#: a different geometry and never comparable. An e5 model needs its
#: "query: " / "passage: " prefixes, which app/jobs/embed.py adds by
#: matching this name, not a separate setting.
model: str = "intfloat/multilingual-e5-small"
#: Roles embedded per run at most, newest first.
per_run: int = Field(50_000, ge=1)
#: `{model: {fit: [lo, hi], taste: [lo, hi]}}`, over `CALIBRATION`: an
#: entry replaces the built-in one for that model. Keyed by model, so a
#: change of `model` cannot carry another geometry's constants along. A
#: model with neither is min-max scaled per run, with a warning in the log.
calibration: dict[str, Calibration] = Field(default_factory=dict)
def calibrated(self) -> Calibration | None:
"""The configured model's calibration, or None when it has none."""
return self.calibration.get(self.model) or CALIBRATION.get(self.model)
class AgentSeed(Section):
"""A search agent as `config.yaml` lists it. Imported into an empty
`agents` table once; after that the table is the answer. `prompts` is a
list of `{name, text}`; without it, `intent` is the one prompt, named
after the agent."""
name: str
prompts: Prompts | None = None
intent: str = ""
gates: list[Gate] = Field(default_factory=list)
weights: Weights = Field(default_factory=Weights)
limits: Limits = Field(default_factory=Limits)
enabled: bool = True
class SearchSeed(Section):
"""A saved search as `config.yaml` lists it: one `search` subscription,
imported once like the agents. `params` are the provider's own, see
plugins/search.py: `q`, `area` and `pages` for jobindex; `q`, `country`,
`where` and `pages` for adzuna; `q` and `region` for platsbanken; `q`,
`country` and `remote` for freehire; `q` and `region` for jobnet; `q`,
`region` and `remote` for jobbank. The provider is any reader of
`scope="search"`, so a new one needs no edit here."""
name: str
provider: str
params: dict[str, Any] = Field(default_factory=dict)
@field_validator("provider")
@classmethod
def _a_search_provider(cls, provider: str) -> str:
# `plugins` is a top-level folder beside `app` and is not on the path
# when this module is imported first, so it is reached the way
# `app.models` reaches it (`_declared_posted`), and only when a seed
# is validated.
root = str(Path(__file__).resolve().parents[1])
if root not in sys.path:
sys.path.insert(0, root)
from plugins.readers import PROVIDERS
if provider not in PROVIDERS:
raise ValueError(f"provider is one of {', '.join(PROVIDERS)}, not {provider!r}")
return provider
class Config(Section):
"""The whole file."""
server: ServerConfig = Field(default_factory=ServerConfig)
#: One SQLite file for the lot: roles and companies, the jobs table, the
#: notice log, the schedules, and auth's `auth_*` tables. One file is one
#: thing to back up.
database: Path = Path("roledesk.db")
#: What a clock on screen means, and what a `daily at 06:00` schedule is
#: 06:00 in. `every 6h` is six elapsed hours in any zone.
timezone: str = "Europe/Berlin"
log_level: str = "INFO"
#: The door, auth's own model. See auth/README.md.
auth: AuthConfig = Field(default_factory=AuthConfig)
#: Directories searched for `Source` plugins. A plugin imports `desk` only.
plugin_paths: list[str] = Field(default_factory=lambda: ["plugins"])
#: Workers per resource class. `http` is wide because a fetch is mostly waiting.
workers: WorkersConfig = Field(default_factory=WorkersConfig)
#: Who answers when a job needs a model. See desk/tools/agent.py.
agent: AgentConfig = Field(default_factory=AgentConfig)
extract: ExtractConfig = Field(default_factory=ExtractConfig)
summarise: SummariseConfig = Field(default_factory=SummariseConfig)
judge: JudgeConfig = Field(default_factory=JudgeConfig)
embed: EmbedConfig = Field(default_factory=EmbedConfig)
enrich: EnrichConfig = Field(default_factory=EnrichConfig)
#: Starter agents, imported once into an empty table.
agents: list[AgentSeed] = Field(default_factory=list)
#: Starter searches, imported once into an empty set of `search`
#: subscriptions.
searches: list[SearchSeed] = Field(default_factory=list)
#: One block per plugin: `enabled`, and whatever that plugin reads.
sources: dict[str, Any] = Field(default_factory=dict)
#: Where this came from, or None for the defaults. Never written back.
path: Path | None = Field(default=None, exclude=True)
@model_validator(mode="after")
def _credentials_live_in_the_database(self) -> Config:
"""`desk.tools.agent` reads a stored token from the app's database."""
self.agent.credentials_database = str(self.database)
return self
class ConfigError(ValueError):
"""A config file that exists but does not validate. The message names
every bad key, for the CLI to print as it is."""
def load(path: str | os.PathLike[str] | None = None) -> Config:
"""Read the YAML file into a `Config`, or return the defaults.
An empty file is the same as no file. A file that is not a mapping, or
that does not validate, is a `ConfigError`.
"""
where = Path(path).expanduser() if path else DEFAULT_PATH
if not where.exists():
return Config()
raw: Any = yaml.safe_load(where.read_text(encoding="utf-8"))
if raw is None:
return Config(path=where)
if not isinstance(raw, dict):
raise ConfigError(f"{where}: expected a mapping of settings, "
f"got {type(raw).__name__}")
try:
return Config(**raw, path=where)
except ValidationError as exc:
raise ConfigError(f"{where}:\n{_explain(exc)}") from None
def _explain(exc: ValidationError) -> str:
lines = []
for err in exc.errors():
where = ".".join(str(p) for p in err["loc"]) or "(top level)"
detail = err["msg"]
if err["type"] == "extra_forbidden":
detail = "unknown setting, check the spelling against config.example.yaml"
lines.append(f" {where}: {detail}")
return "\n".join(lines)