From b021838959418bbca77fcbd241c1943505f72ed1 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 14:59:34 +0200 Subject: [PATCH] Calibrate fit and taste per embedding model instead of min-max per run scale() maps a cosine to clip((cos - lo) / (hi - lo), 0, 1) with constants from CALIBRATION in app/config.py, keyed by model and replaceable under embed.calibration. multilingual-e5-small: fit 0.75..0.88, taste -0.14..0.19 (taste from synthetic votes). An uncalibrated model falls back to min-max and logs a warning once per run. The extract and summarise bars default to 0.55 and 0.65, about 300 and 35 roles for the starter agent on the measured data; stored limits are kept. An agent without an intent is left out of the model-stage scope, and each score run names such agents in one score.no_intent notice. Co-Authored-By: Claude Opus --- app/config.py | 40 +++++- app/jobs/score.py | 80 ++++++++---- app/models.py | 17 ++- app/static/agents.js | 2 +- app/storage/store.py | 5 +- app/templates/settings/agents.html | 6 +- config.example.yaml | 22 +++- docs/decisions.md | 14 ++- tests/test_extract.py | 16 ++- tests/test_score.py | 187 ++++++++++++++++++++++++++--- tests/test_storage.py | 17 ++- tests/test_web.py | 6 +- 12 files changed, 336 insertions(+), 76 deletions(-) diff --git a/app/config.py b/app/config.py index 95c4af5..8083766 100644 --- a/app/config.py +++ b/app/config.py @@ -20,7 +20,7 @@ from pathlib import Path from typing import Any, Literal import yaml -from pydantic import Field, ValidationError, model_validator +from pydantic import Field, ValidationError, field_validator, model_validator from app.models import Gate, Limits, Weights from auth import AuthConfig @@ -66,6 +66,35 @@ class SummariseConfig(ModelTaskConfig): batch_size: int = Field(10, ge=1) +class Calibration(Section): + """Where one embedding model's cosines sit, as `(lo, hi)` for fit and for + taste: `score.scale` maps `lo` to 0 and `hi` to 1 and clips outside, so a + term is a level rather than a rank within the run.""" + + fit: tuple[float, float] + taste: tuple[float, float] + + @field_validator("fit", "taste") + @classmethod + def _lo_below_hi(cls, band: tuple[float, float]) -> tuple[float, float]: + lo, hi = band + if not -1 <= lo < hi <= 1: + raise ValueError(f"needs -1 <= lo < hi <= 1, got [{lo}, {hi}]") + return band + + +#: The built-in calibration, by embedding model. Measured on a copy of the +#: live database, 16,841 open roles (see docs/decisions.md). fit: pooled over +#: five intents the raw cosine has p1 0.753 and p99.9 0.871; hi sits above +#: every single intent's p99.9, so only about 5 of the starter agent's 4,156 +#: ungated roles clip to 1. taste: from synthetic votes, 30 liked and 30 +#: passed roles for each of three intents, since the copy had none (pooled p1 +#: -0.140, p99.9 0.192); re-measure it once real votes exist. +CALIBRATION: dict[str, Calibration] = { + "intfloat/multilingual-e5-small": Calibration(fit=(0.75, 0.88), taste=(-0.14, 0.19)), +} + + class EmbedConfig(Section): """Turning a role into a vector for `fit`. Needs sentence-transformers; without it `embed` says so and does nothing, and `score` leaves fit out. @@ -87,6 +116,15 @@ class EmbedConfig(Section): model: str = "intfloat/multilingual-e5-small" #: Roles embedded per run at most, newest first. per_run: int = Field(50_000, ge=1) + #: `{model: {fit: [lo, hi], taste: [lo, hi]}}`, over `CALIBRATION`: an + #: entry replaces the built-in one for that model. Keyed by model, so a + #: change of `model` cannot carry another geometry's constants along. A + #: model with neither is min-max scaled per run, with a warning in the log. + calibration: dict[str, Calibration] = Field(default_factory=dict) + + def calibrated(self) -> Calibration | None: + """The configured model's calibration, or None when it has none.""" + return self.calibration.get(self.model) or CALIBRATION.get(self.model) class AgentSeed(Section): diff --git a/app/jobs/score.py b/app/jobs/score.py index 3b13093..f031092 100644 --- a/app/jobs/score.py +++ b/app/jobs/score.py @@ -14,13 +14,15 @@ A gate rejects on evidence, never on absence: a role whose column is empty is not gated by it. `extract` is what fills an empty column, and it reads only the roles that rank, so gating on an empty column would keep those roles out of extract's reach for good. - fit cos(role vector, intent vector), min-max scaled across - the roles this agent scores in this run: the closest is 1, - the farthest 0, a lone role 1 + fit cos(role vector, intent vector) as a level: the embedding + model's calibrated `lo` is 0 and `hi` 1, clipped outside + (see `scale`), so a role's fit does not depend on what + else is open recency 0.5 ** (days since posted_at / 14) affinity (the agent's affinity for the company + 1) / 3 - taste cos(role vector, liked direction), min-max scaled like fit, - where the direction is mean(liked vectors) - mean(passed vectors) + taste cos(role vector, liked direction), scaled like fit with the + model's taste calibration, where the direction is + mean(liked vectors) - mean(passed vectors) over this agent's votes; with only likes it is the liked mean, with only passes the negated passed mean; no votes, no term @@ -30,7 +32,10 @@ company (`affinity = 2`) skips the gates and is floored at 0.9; a hidden one (`-1`) is not scored at all. A role with no vector is scored without `fit` and `taste`, unless `fit` is all there is, when it is gated as "no vector". An agent with no votes scores every role without `taste`; the direction is -newsdesk's `centroid(liked) - centroid(disliked)`. +newsdesk's `centroid(liked) - centroid(disliked)`. An agent with no intent +has no `fit` either: its scores are not a match, so it is left out of what +extract and summarise read (`Store._top_scope`), and a run names such agents +in one notice. Recomputed in full on every run, and each agent's rows are replaced in one transaction: nothing is incremental, so nothing can drift. @@ -47,8 +52,8 @@ from typing import Any import numpy as np -from pipeline import NULL, RunContext, job -from app.config import Config +from pipeline import NULL, Notice, RunContext, job +from app.config import Calibration, Config from app.jobs import embed from app.models import Agent, Candidate, Gate, Weights from app.storage import Store @@ -110,19 +115,28 @@ def taste_direction(vectors: dict[str, np.ndarray], return direction -def scale(cosines: dict[str, float]) -> dict[str, float]: - """Min-max to 0..1 across one agent's run, so `fit` ranks roles rather - than stating a level. +def scale(cosines: dict[str, float], + band: tuple[float, float] | None) -> dict[str, float]: + """Cosines to 0..1 levels: `clip((cos - lo) / (hi - lo), 0, 1)` with the + embedding model's calibrated `band` (`app.config.CALIBRATION`). - `(cos + 1) / 2` was the level, and on multilingual-e5-small it spans about - 0.87 to 0.95 for real postings against a real intent (0.55 to 0.79 on the - all-MiniLM-L6-v2 this replaced), so a prefer gate weighted 0.2 moved a - score more than the whole spread of fit did either way. Scaled, the - closest role is 1 and the farthest 0. A single role, or roles that are all - equally close, get 1: there is nothing to tell them apart by. + Raw cosines on multilingual-e5-small sit between about 0.75 and 0.87 for + real postings against a real intent, so unscaled a prefer gate weighted + 0.2 moved a score more than the whole spread of fit. The band stretches + that spread to 0..1 with fixed constants, so a role's level is the same + in every run and a `limits` bar is an absolute threshold. + + Without a band, for a model nobody calibrated, it falls back to min-max + across the run: the closest role is 1 and the farthest 0, which ranks + rather than states a level. A single role, or roles that are all equally + close, then get 1: there is nothing to tell them apart by. """ if not cosines: return {} + if band is not None: + lo, hi = band + return {uid: min(max((value - lo) / (hi - lo), 0.0), 1.0) + for uid, value in cosines.items()} low, high = min(cosines.values()), max(cosines.values()) if high - low < 1e-9: return dict.fromkeys(cosines, 1.0) @@ -231,11 +245,24 @@ def _intent(store: Store, agent: Agent, model: str, # ------------------------------------------------------------------- the job +def no_intent(names: Sequence[str]) -> Notice: + """The notice for enabled agents without an intent: their scores have no + fit, so `Store._top_scope` keeps them from feeding extract and summarise.""" + return Notice( + code="score.no_intent", level="warning", kind="expected", key="score", + text=f"No intent, so ranked without fit and not read by extract or " + f"summarise: {', '.join(names)}.", + fix="Give the agent an intent under Settings, Search agents.", + where="app.jobs.score") + + def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray], - intent: np.ndarray | None, now: datetime) -> dict[str, int]: + intent: np.ndarray | None, now: datetime, + calibration: Calibration | None = None) -> dict[str, int]: """Two passes: the cosines of the roles this agent scores, to its intent - and to its taste direction, then every score with both scaled across - them. A gated role is left out of the scaling, so a role that fails a + and to its taste direction, then every score with both scaled by the + model's `calibration`, or across the run without one. A gated role is + left out of the cosines, so under the fallback a role that fails a require gate cannot stretch the range the ranked roles share. The voted roles are scored like any other; a liked one sits near the direction it helped make, which is what a like should do.""" @@ -251,7 +278,8 @@ def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray], cosines[candidate.uid] = cosine(role_vec, intent) if direction is not None: tastes[candidate.uid] = cosine(role_vec, direction) - fits, tastes = scale(cosines), scale(tastes) + fits = scale(cosines, calibration.fit if calibration else None) + tastes = scale(tastes, calibration.taste if calibration else None) rows: list[tuple[str, float, str]] = [] gated = 0 for candidate in candidates: @@ -274,11 +302,19 @@ def run_score(config: Config, store: Store, *, vectors = store.vectors(model) if agents else {} encoder = _Encoder(model, context) now = store.clock() + calibration = config.embed.calibrated() + if agents and calibration is None: + log.warning("no calibration for %s: fit and taste are min-max scaled across " + "each agent's roles in this run, so they rank rather than state a " + "level; add it under embed.calibration", model) + if blind := [agent.name for agent in agents if not agent.intent.strip()]: + context.notice(no_intent(blind)) scored = gated = 0 for n, agent in enumerate(agents): context.check() context.progress(n, len(agents), f"scoring for {agent.name}, {n + 1} of {len(agents)}") - counts = score_agent(store, agent, vectors, _intent(store, agent, model, encoder), now) + counts = score_agent(store, agent, vectors, _intent(store, agent, model, encoder), now, + calibration) scored += counts["scored"] gated += counts["gated"] log.info("%-20s %6d scored, %6d gated", agent.name, counts["scored"], counts["gated"]) diff --git a/app/models.py b/app/models.py index 46606d0..213e274 100644 --- a/app/models.py +++ b/app/models.py @@ -499,15 +499,20 @@ class Limits(Model): """How far down an agent's ranking the two model stages read. Each pair is a top N, the backstop, and a minimum score, the real stop: a role is read when it is among the agent's best `extract_top` ungated scores - and not below `extract_min`, and summarised likewise. A fresh role with - the worst fit scores about 0.27 on the default weights, a good one 0.6 - or more, so extract's floor is loose and summarise's, the long answer, - stricter.""" + and not below `extract_min`, and summarised likewise. Fit is a + calibrated level (`score.scale`), so a bar means the same in every run. + A fresh role with the worst fit scores about 0.27 on the default + weights, a perfect one about 0.87. Measured on 16,841 open roles for the + "Copenhagen engineering" starter agent (4,156 ungated, no votes), 0.55 + admits about 300 roles to extract and 0.65 about 35 to summarise, the + long answer; the top N catches an agent whose bar admits far more. An + agent keeps the limits stored with it; only a key its JSON lacks + follows these defaults.""" extract_top: int = Field(500, ge=0) - extract_min: float = Field(0.3, ge=0, le=1) + extract_min: float = Field(0.55, ge=0, le=1) summarise_top: int = Field(30, ge=0) - summarise_min: float = Field(0.5, ge=0, le=1) + summarise_min: float = Field(0.65, ge=0, le=1) class Agent(Model): diff --git a/app/static/agents.js b/app/static/agents.js index 9274b49..bace965 100644 --- a/app/static/agents.js +++ b/app/static/agents.js @@ -23,7 +23,7 @@ // What the model reads after a pull, per agent: the best `extract_top` // roles scoring at least `extract_min` get their gate columns filled, the // best `summarise_top` at `summarise_min` or more a summary. - const DEFAULT_LIMITS = { extract_top: 500, extract_min: 0.3, summarise_top: 30, summarise_min: 0.5 }; + const DEFAULT_LIMITS = { extract_top: 500, extract_min: 0.55, summarise_top: 30, summarise_min: 0.65 }; const NEW = "new"; /** @type {object[]} every agent, as the server lists them */ diff --git a/app/storage/store.py b/app/storage/store.py index a37b3b1..8366c1f 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -675,7 +675,9 @@ class Store: limits ride out of the window subquery beside it, so each row is held to the limits of the agent that ranked it; a key the JSON lacks is the `Limits` default. The role is `r`. A company an agent hid - needs no clause: `score` writes no rows for it under that agent.""" + needs no clause: `score` writes no rows for it under that agent. An + agent without an intent counts for nothing here: its scores have no + fit, so they say how fresh a role is, not whether it is wanted.""" defaults = Limits() return (f"""r.uid IN (SELECT uid FROM ( SELECT s.uid, s.score, a.limits, ROW_NUMBER() OVER ( @@ -683,6 +685,7 @@ class Store: ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n FROM scores s JOIN agents a ON a.id = s.agent_id AND a.enabled = 1 + AND TRIM(a.intent, ' ' || char(9, 10, 13)) <> '' JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL WHERE s.gated_reason = '') WHERE n <= COALESCE(json_extract(limits, '$.{stage}_top'), ?) diff --git a/app/templates/settings/agents.html b/app/templates/settings/agents.html index d95264f..9d8ac08 100644 --- a/app/templates/settings/agents.html +++ b/app/templates/settings/agents.html @@ -33,11 +33,11 @@
What the model reads - + - +
- Every pull scores every open role for free; these bound what the model then reads, per agent. The count is the backstop, the score the real stop. A new role with the worst fit scores about 0.27, a good one 0.6 or more. + Every pull scores every open role for free; these bound what the model then reads, per agent. The count is the backstop, the score the real stop. Fit is a level, the same in every run: a new role with the worst fit scores about 0.27, a perfect one about 0.87. An agent without an intent feeds neither stage.
Advanced