Calibrate fit and taste per embedding model instead of min-max per run

scale() maps a cosine to clip((cos - lo) / (hi - lo), 0, 1) with constants
from CALIBRATION in app/config.py, keyed by model and replaceable under
embed.calibration. multilingual-e5-small: fit 0.75..0.88, taste -0.14..0.19
(taste from synthetic votes). An uncalibrated model falls back to min-max
and logs a warning once per run.

The extract and summarise bars default to 0.55 and 0.65, about 300 and 35
roles for the starter agent on the measured data; stored limits are kept.
An agent without an intent is left out of the model-stage scope, and each
score run names such agents in one score.no_intent notice.

Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus committed 2026-09-29 14:59:34 +02:00
1 parent 706b127bc0
commit b021838959
12 files changed
+336 -76

No files matched your search

+39 -1
View File
@@ -20,7 +20,7 @@ from pathlib import Path
from typing import Any, Literal
import yaml
from pydantic import Field, ValidationError, model_validator
from pydantic import Field, ValidationError, field_validator, model_validator
from app.models import Gate, Limits, Weights
from auth import AuthConfig
@@ -66,6 +66,35 @@ class SummariseConfig(ModelTaskConfig):
batch_size: int = Field(10, ge=1)
class Calibration(Section):
"""Where one embedding model's cosines sit, as `(lo, hi)` for fit and for
taste: `score.scale` maps `lo` to 0 and `hi` to 1 and clips outside, so a
term is a level rather than a rank within the run."""
fit: tuple[float, float]
taste: tuple[float, float]
@field_validator("fit", "taste")
@classmethod
def _lo_below_hi(cls, band: tuple[float, float]) -> tuple[float, float]:
lo, hi = band
if not -1 <= lo < hi <= 1:
raise ValueError(f"needs -1 <= lo < hi <= 1, got [{lo}, {hi}]")
return band
#: The built-in calibration, by embedding model. Measured on a copy of the
#: live database, 16,841 open roles (see docs/decisions.md). fit: pooled over
#: five intents the raw cosine has p1 0.753 and p99.9 0.871; hi sits above
#: every single intent's p99.9, so only about 5 of the starter agent's 4,156
#: ungated roles clip to 1. taste: from synthetic votes, 30 liked and 30
#: passed roles for each of three intents, since the copy had none (pooled p1
#: -0.140, p99.9 0.192); re-measure it once real votes exist.
CALIBRATION: dict[str, Calibration] = {
"intfloat/multilingual-e5-small": Calibration(fit=(0.75, 0.88), taste=(-0.14, 0.19)),
}
class EmbedConfig(Section):
"""Turning a role into a vector for `fit`. Needs sentence-transformers;
without it `embed` says so and does nothing, and `score` leaves fit out.
@@ -87,6 +116,15 @@ class EmbedConfig(Section):
model: str = "intfloat/multilingual-e5-small"
#: Roles embedded per run at most, newest first.
per_run: int = Field(50_000, ge=1)
#: `{model: {fit: [lo, hi], taste: [lo, hi]}}`, over `CALIBRATION`: an
#: entry replaces the built-in one for that model. Keyed by model, so a
#: change of `model` cannot carry another geometry's constants along. A
#: model with neither is min-max scaled per run, with a warning in the log.
calibration: dict[str, Calibration] = Field(default_factory=dict)
def calibrated(self) -> Calibration | None:
"""The configured model's calibration, or None when it has none."""
return self.calibration.get(self.model) or CALIBRATION.get(self.model)
class AgentSeed(Section):