Provider names, the CLI's choices and the dupes rule's search hosts now come from readers.PROVIDERS, and the regions, work modes and remote words from the modules' REGIONS, MODES and REMOTE, read with getattr so main works before and after the readers merge. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
277 lines
11 KiB
Python
277 lines
11 KiB
Python
"""Config loading: one YAML file, validated into a model.
|
|
|
|
The boilerplate's shape with roledesk's keys. A typo in a key is an error
|
|
naming the key, because every section forbids extras: a setting you believe
|
|
you have changed and have not is the expensive kind of typo. `sources` stays a
|
|
dict: one block per plugin, whose keys only the plugin knows.
|
|
|
|
The path comes from `app --config`, else `APP_CONFIG`, else `config.yaml`
|
|
beside the checkout. The Dockerfile sets `APP_CONFIG=/data/config.yaml`.
|
|
|
|
**A missing file is not an error.** Every field has a default that runs, so
|
|
`uv run app serve` works on a fresh clone. The committed `config.yaml` adds
|
|
the starter agents; `config.example.yaml` documents every key.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import yaml
|
|
from pydantic import Field, ValidationError, field_validator, model_validator
|
|
|
|
from app.models import Gate, Limits, Prompts, Weights
|
|
from auth import AuthConfig
|
|
from desk.config import AgentConfig, ModelTaskConfig, Section, WorkersConfig
|
|
|
|
#: Where `load()` looks when nobody says. Read at import so a test can set the
|
|
#: variable first, and `app --config` overrides it after.
|
|
DEFAULT_PATH = Path(os.environ.get("APP_CONFIG", "config.yaml"))
|
|
|
|
|
|
class ServerConfig(Section):
|
|
"""Where uvicorn binds. Anything but a loopback address makes
|
|
`auth.enabled: auto` lock the app."""
|
|
|
|
host: str = "127.0.0.1"
|
|
port: int = 8790
|
|
|
|
|
|
class ExtractConfig(ModelTaskConfig):
|
|
"""Turning a posting's title and text into the gate columns. See
|
|
app/jobs/extract.py. Off by default, so nobody is billed by surprise:
|
|
on the API backend every batch is a paid call. On, it reads only the
|
|
best roles of each enabled agent, as deep and as low as that agent's
|
|
own `limits` say, so the cost follows the ranking rather than the
|
|
boards."""
|
|
|
|
enabled: bool = False
|
|
model: str = "claude-haiku-4-5-20251001"
|
|
batch_size: int = Field(20, ge=1)
|
|
|
|
|
|
class SummariseConfig(ModelTaskConfig):
|
|
"""About 60 words on a role, for the list and the read page. See
|
|
app/jobs/summarise.py. Off by default like extract; on, a pull summarises
|
|
only the best of each enabled agent, as deep and as low as that agent's
|
|
own `limits` say, so the cost is bounded whatever the backlog. The read
|
|
page's Summarise button asks for one role whether this is on or not."""
|
|
|
|
enabled: bool = False
|
|
model: str = "claude-haiku-4-5-20251001"
|
|
#: A description is capped at 8000 characters per role, so ten is about
|
|
#: 20k tokens of input per call.
|
|
batch_size: int = Field(10, ge=1)
|
|
|
|
|
|
class EnrichConfig(Section):
|
|
"""Reading roles on their own pages after scoring, for the hosts whose
|
|
list carries no text. See app/jobs/enrich.py. A role is read when it has
|
|
no text and an enabled agent with a prompt ranks it ungated among its
|
|
best `top` and at `min_score` or above: the extract bar (`Limits`,
|
|
500 and 0.55) with the score a notch lower, since a role ranked on its
|
|
title alone is the one whose score its text moves most."""
|
|
|
|
enabled: bool = True
|
|
top: int = Field(500, ge=0)
|
|
min_score: float = Field(0.5, ge=0, le=1)
|
|
#: Reads per board and per run. Per board is the cap the pull had; the
|
|
#: boards are read in parallel on the `http` pool.
|
|
per_board: int = Field(200, ge=0)
|
|
per_run: int = Field(1000, ge=0)
|
|
#: Days before a page that gave no text is read again.
|
|
retry_days: int = Field(7, ge=1)
|
|
|
|
|
|
class Calibration(Section):
|
|
"""Where one embedding model's cosines sit, as `(lo, hi)` for fit and for
|
|
taste: `score.scale` maps `lo` to 0 and `hi` to 1 and clips outside, so a
|
|
term is a level rather than a rank within the run."""
|
|
|
|
fit: tuple[float, float]
|
|
taste: tuple[float, float]
|
|
|
|
@field_validator("fit", "taste")
|
|
@classmethod
|
|
def _lo_below_hi(cls, band: tuple[float, float]) -> tuple[float, float]:
|
|
lo, hi = band
|
|
if not -1 <= lo < hi <= 1:
|
|
raise ValueError(f"needs -1 <= lo < hi <= 1, got [{lo}, {hi}]")
|
|
return band
|
|
|
|
|
|
#: The built-in calibration, by embedding model. Measured on a copy of the
|
|
#: live database, 16,841 open roles (see docs/decisions.md). fit: pooled over
|
|
#: five intents the raw cosine has p1 0.753 and p99.9 0.871; hi sits above
|
|
#: every single intent's p99.9, so only about 5 of the starter agent's 4,156
|
|
#: ungated roles clip to 1. taste: from synthetic votes, 30 liked and 30
|
|
#: passed roles for each of three intents, since the copy had none (pooled p1
|
|
#: -0.140, p99.9 0.192); re-measure it once real votes exist.
|
|
CALIBRATION: dict[str, Calibration] = {
|
|
"intfloat/multilingual-e5-small": Calibration(fit=(0.75, 0.88), taste=(-0.14, 0.19)),
|
|
}
|
|
|
|
|
|
class EmbedConfig(Section):
|
|
"""Turning a role into a vector for `fit`. Needs sentence-transformers;
|
|
without it `embed` says so and does nothing, and `score` leaves fit out.
|
|
See app/jobs/embed.py.
|
|
|
|
`intfloat/multilingual-e5-small`, not the English-only
|
|
`sentence-transformers/all-MiniLM-L6-v2` roledesk started with: measured
|
|
on the live roles against the "Copenhagen engineering" agent's intent,
|
|
MiniLM separates engineering from non-engineering roles with an AUC of
|
|
0.68 (0.74 restricted to Danish-country roles), multilingual-e5-small
|
|
with an AUC of 0.70 (1.00 restricted to Danish-country roles), for about
|
|
3x the CPU time. See docs/decisions.md.
|
|
"""
|
|
|
|
#: Recorded on every vector. Changing it drops the old vectors, which are
|
|
#: a different geometry and never comparable. An e5 model needs its
|
|
#: "query: " / "passage: " prefixes, which app/jobs/embed.py adds by
|
|
#: matching this name, not a separate setting.
|
|
model: str = "intfloat/multilingual-e5-small"
|
|
#: Roles embedded per run at most, newest first.
|
|
per_run: int = Field(50_000, ge=1)
|
|
#: `{model: {fit: [lo, hi], taste: [lo, hi]}}`, over `CALIBRATION`: an
|
|
#: entry replaces the built-in one for that model. Keyed by model, so a
|
|
#: change of `model` cannot carry another geometry's constants along. A
|
|
#: model with neither is min-max scaled per run, with a warning in the log.
|
|
calibration: dict[str, Calibration] = Field(default_factory=dict)
|
|
|
|
def calibrated(self) -> Calibration | None:
|
|
"""The configured model's calibration, or None when it has none."""
|
|
return self.calibration.get(self.model) or CALIBRATION.get(self.model)
|
|
|
|
|
|
class AgentSeed(Section):
|
|
"""A search agent as `config.yaml` lists it. Imported into an empty
|
|
`agents` table once; after that the table is the answer. `prompts` is a
|
|
list of `{name, text}`; without it, `intent` is the one prompt, named
|
|
after the agent."""
|
|
|
|
name: str
|
|
prompts: Prompts | None = None
|
|
intent: str = ""
|
|
gates: list[Gate] = Field(default_factory=list)
|
|
weights: Weights = Field(default_factory=Weights)
|
|
limits: Limits = Field(default_factory=Limits)
|
|
enabled: bool = True
|
|
|
|
|
|
class SearchSeed(Section):
|
|
"""A saved search as `config.yaml` lists it: one `search` subscription,
|
|
imported once like the agents. `params` are the provider's own, see
|
|
plugins/search.py: `q`, `area` and `pages` for jobindex; `q`, `country`,
|
|
`where` and `pages` for adzuna; `q` and `region` for platsbanken; `q`,
|
|
`country` and `remote` for freehire; `q` and `region` for jobnet; `q`,
|
|
`region` and `remote` for jobbank. The provider is any reader of
|
|
`scope="search"`, so a new one needs no edit here."""
|
|
|
|
name: str
|
|
provider: str
|
|
params: dict[str, Any] = Field(default_factory=dict)
|
|
|
|
@field_validator("provider")
|
|
@classmethod
|
|
def _a_search_provider(cls, provider: str) -> str:
|
|
# `plugins` is a top-level folder beside `app` and is not on the path
|
|
# when this module is imported first, so it is reached the way
|
|
# `app.models` reaches it (`_declared_posted`), and only when a seed
|
|
# is validated.
|
|
root = str(Path(__file__).resolve().parents[1])
|
|
if root not in sys.path:
|
|
sys.path.insert(0, root)
|
|
from plugins.readers import PROVIDERS
|
|
|
|
if provider not in PROVIDERS:
|
|
raise ValueError(f"provider is one of {', '.join(PROVIDERS)}, not {provider!r}")
|
|
return provider
|
|
|
|
|
|
class Config(Section):
|
|
"""The whole file."""
|
|
|
|
server: ServerConfig = Field(default_factory=ServerConfig)
|
|
|
|
#: One SQLite file for the lot: roles and companies, the jobs table, the
|
|
#: notice log, the schedules, and auth's `auth_*` tables. One file is one
|
|
#: thing to back up.
|
|
database: Path = Path("roledesk.db")
|
|
|
|
#: What a clock on screen means, and what a `daily at 06:00` schedule is
|
|
#: 06:00 in. `every 6h` is six elapsed hours in any zone.
|
|
timezone: str = "Europe/Berlin"
|
|
|
|
log_level: str = "INFO"
|
|
|
|
#: The door, auth's own model. See auth/README.md.
|
|
auth: AuthConfig = Field(default_factory=AuthConfig)
|
|
|
|
#: Directories searched for `Source` plugins. A plugin imports `desk` only.
|
|
plugin_paths: list[str] = Field(default_factory=lambda: ["plugins"])
|
|
#: Workers per resource class. `http` is wide because a fetch is mostly waiting.
|
|
workers: WorkersConfig = Field(default_factory=WorkersConfig)
|
|
#: Who answers when a job needs a model. See desk/tools/agent.py.
|
|
agent: AgentConfig = Field(default_factory=AgentConfig)
|
|
extract: ExtractConfig = Field(default_factory=ExtractConfig)
|
|
summarise: SummariseConfig = Field(default_factory=SummariseConfig)
|
|
embed: EmbedConfig = Field(default_factory=EmbedConfig)
|
|
enrich: EnrichConfig = Field(default_factory=EnrichConfig)
|
|
#: Starter agents, imported once into an empty table.
|
|
agents: list[AgentSeed] = Field(default_factory=list)
|
|
#: Starter searches, imported once into an empty set of `search`
|
|
#: subscriptions.
|
|
searches: list[SearchSeed] = Field(default_factory=list)
|
|
#: One block per plugin: `enabled`, and whatever that plugin reads.
|
|
sources: dict[str, Any] = Field(default_factory=dict)
|
|
|
|
#: Where this came from, or None for the defaults. Never written back.
|
|
path: Path | None = Field(default=None, exclude=True)
|
|
|
|
@model_validator(mode="after")
|
|
def _credentials_live_in_the_database(self) -> Config:
|
|
"""`desk.tools.agent` reads a stored token from the app's database."""
|
|
self.agent.credentials_database = str(self.database)
|
|
return self
|
|
|
|
|
|
class ConfigError(ValueError):
|
|
"""A config file that exists but does not validate. The message names
|
|
every bad key, for the CLI to print as it is."""
|
|
|
|
|
|
def load(path: str | os.PathLike[str] | None = None) -> Config:
|
|
"""Read the YAML file into a `Config`, or return the defaults.
|
|
|
|
An empty file is the same as no file. A file that is not a mapping, or
|
|
that does not validate, is a `ConfigError`.
|
|
"""
|
|
where = Path(path).expanduser() if path else DEFAULT_PATH
|
|
if not where.exists():
|
|
return Config()
|
|
raw: Any = yaml.safe_load(where.read_text(encoding="utf-8"))
|
|
if raw is None:
|
|
return Config(path=where)
|
|
if not isinstance(raw, dict):
|
|
raise ConfigError(f"{where}: expected a mapping of settings, "
|
|
f"got {type(raw).__name__}")
|
|
try:
|
|
return Config(**raw, path=where)
|
|
except ValidationError as exc:
|
|
raise ConfigError(f"{where}:\n{_explain(exc)}") from None
|
|
|
|
|
|
def _explain(exc: ValidationError) -> str:
|
|
lines = []
|
|
for err in exc.errors():
|
|
where = ".".join(str(p) for p in err["loc"]) or "(top level)"
|
|
detail = err["msg"]
|
|
if err["type"] == "extra_forbidden":
|
|
detail = "unknown setting, check the spelling against config.example.yaml"
|
|
lines.append(f" {where}: {detail}")
|
|
return "\n".join(lines)
|