Calibrate fit and taste per embedding model instead of min-max per run
scale() maps a cosine to clip((cos - lo) / (hi - lo), 0, 1) with constants from CALIBRATION in app/config.py, keyed by model and replaceable under embed.calibration. multilingual-e5-small: fit 0.75..0.88, taste -0.14..0.19 (taste from synthetic votes). An uncalibrated model falls back to min-max and logs a warning once per run. The extract and summarise bars default to 0.55 and 0.65, about 300 and 35 roles for the starter agent on the measured data; stored limits are kept. An agent without an intent is left out of the model-stage scope, and each score run names such agents in one score.no_intent notice. Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
1 parent
706b127bc0
commit
b021838959
12 files changed
+336
-76
No files matched your search
+39
-1
@@ -20,7 +20,7 @@ from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
import yaml
|
||||
from pydantic import Field, ValidationError, model_validator
|
||||
from pydantic import Field, ValidationError, field_validator, model_validator
|
||||
|
||||
from app.models import Gate, Limits, Weights
|
||||
from auth import AuthConfig
|
||||
@@ -66,6 +66,35 @@ class SummariseConfig(ModelTaskConfig):
|
||||
batch_size: int = Field(10, ge=1)
|
||||
|
||||
|
||||
class Calibration(Section):
|
||||
"""Where one embedding model's cosines sit, as `(lo, hi)` for fit and for
|
||||
taste: `score.scale` maps `lo` to 0 and `hi` to 1 and clips outside, so a
|
||||
term is a level rather than a rank within the run."""
|
||||
|
||||
fit: tuple[float, float]
|
||||
taste: tuple[float, float]
|
||||
|
||||
@field_validator("fit", "taste")
|
||||
@classmethod
|
||||
def _lo_below_hi(cls, band: tuple[float, float]) -> tuple[float, float]:
|
||||
lo, hi = band
|
||||
if not -1 <= lo < hi <= 1:
|
||||
raise ValueError(f"needs -1 <= lo < hi <= 1, got [{lo}, {hi}]")
|
||||
return band
|
||||
|
||||
|
||||
#: The built-in calibration, by embedding model. Measured on a copy of the
|
||||
#: live database, 16,841 open roles (see docs/decisions.md). fit: pooled over
|
||||
#: five intents the raw cosine has p1 0.753 and p99.9 0.871; hi sits above
|
||||
#: every single intent's p99.9, so only about 5 of the starter agent's 4,156
|
||||
#: ungated roles clip to 1. taste: from synthetic votes, 30 liked and 30
|
||||
#: passed roles for each of three intents, since the copy had none (pooled p1
|
||||
#: -0.140, p99.9 0.192); re-measure it once real votes exist.
|
||||
CALIBRATION: dict[str, Calibration] = {
|
||||
"intfloat/multilingual-e5-small": Calibration(fit=(0.75, 0.88), taste=(-0.14, 0.19)),
|
||||
}
|
||||
|
||||
|
||||
class EmbedConfig(Section):
|
||||
"""Turning a role into a vector for `fit`. Needs sentence-transformers;
|
||||
without it `embed` says so and does nothing, and `score` leaves fit out.
|
||||
@@ -87,6 +116,15 @@ class EmbedConfig(Section):
|
||||
model: str = "intfloat/multilingual-e5-small"
|
||||
#: Roles embedded per run at most, newest first.
|
||||
per_run: int = Field(50_000, ge=1)
|
||||
#: `{model: {fit: [lo, hi], taste: [lo, hi]}}`, over `CALIBRATION`: an
|
||||
#: entry replaces the built-in one for that model. Keyed by model, so a
|
||||
#: change of `model` cannot carry another geometry's constants along. A
|
||||
#: model with neither is min-max scaled per run, with a warning in the log.
|
||||
calibration: dict[str, Calibration] = Field(default_factory=dict)
|
||||
|
||||
def calibrated(self) -> Calibration | None:
|
||||
"""The configured model's calibration, or None when it has none."""
|
||||
return self.calibration.get(self.model) or CALIBRATION.get(self.model)
|
||||
|
||||
|
||||
class AgentSeed(Section):
|
||||
|
||||
Reference in new issue
Block a user