diff --git a/app/config.py b/app/config.py
index 95c4af5..8083766 100644
--- a/app/config.py
+++ b/app/config.py
@@ -20,7 +20,7 @@ from pathlib import Path
from typing import Any, Literal
import yaml
-from pydantic import Field, ValidationError, model_validator
+from pydantic import Field, ValidationError, field_validator, model_validator
from app.models import Gate, Limits, Weights
from auth import AuthConfig
@@ -66,6 +66,35 @@ class SummariseConfig(ModelTaskConfig):
batch_size: int = Field(10, ge=1)
+class Calibration(Section):
+ """Where one embedding model's cosines sit, as `(lo, hi)` for fit and for
+ taste: `score.scale` maps `lo` to 0 and `hi` to 1 and clips outside, so a
+ term is a level rather than a rank within the run."""
+
+ fit: tuple[float, float]
+ taste: tuple[float, float]
+
+ @field_validator("fit", "taste")
+ @classmethod
+ def _lo_below_hi(cls, band: tuple[float, float]) -> tuple[float, float]:
+ lo, hi = band
+ if not -1 <= lo < hi <= 1:
+ raise ValueError(f"needs -1 <= lo < hi <= 1, got [{lo}, {hi}]")
+ return band
+
+
+#: The built-in calibration, by embedding model. Measured on a copy of the
+#: live database, 16,841 open roles (see docs/decisions.md). fit: pooled over
+#: five intents the raw cosine has p1 0.753 and p99.9 0.871; hi sits above
+#: every single intent's p99.9, so only about 5 of the starter agent's 4,156
+#: ungated roles clip to 1. taste: from synthetic votes, 30 liked and 30
+#: passed roles for each of three intents, since the copy had none (pooled p1
+#: -0.140, p99.9 0.192); re-measure it once real votes exist.
+CALIBRATION: dict[str, Calibration] = {
+ "intfloat/multilingual-e5-small": Calibration(fit=(0.75, 0.88), taste=(-0.14, 0.19)),
+}
+
+
class EmbedConfig(Section):
"""Turning a role into a vector for `fit`. Needs sentence-transformers;
without it `embed` says so and does nothing, and `score` leaves fit out.
@@ -87,6 +116,15 @@ class EmbedConfig(Section):
model: str = "intfloat/multilingual-e5-small"
#: Roles embedded per run at most, newest first.
per_run: int = Field(50_000, ge=1)
+ #: `{model: {fit: [lo, hi], taste: [lo, hi]}}`, over `CALIBRATION`: an
+ #: entry replaces the built-in one for that model. Keyed by model, so a
+ #: change of `model` cannot carry another geometry's constants along. A
+ #: model with neither is min-max scaled per run, with a warning in the log.
+ calibration: dict[str, Calibration] = Field(default_factory=dict)
+
+ def calibrated(self) -> Calibration | None:
+ """The configured model's calibration, or None when it has none."""
+ return self.calibration.get(self.model) or CALIBRATION.get(self.model)
class AgentSeed(Section):
diff --git a/app/jobs/score.py b/app/jobs/score.py
index 3b13093..f031092 100644
--- a/app/jobs/score.py
+++ b/app/jobs/score.py
@@ -14,13 +14,15 @@ A gate rejects on evidence, never on absence: a role whose column is empty
is not gated by it. `extract` is what fills an empty column, and it reads
only the roles that rank, so gating on an empty column would keep those
roles out of extract's reach for good.
- fit cos(role vector, intent vector), min-max scaled across
- the roles this agent scores in this run: the closest is 1,
- the farthest 0, a lone role 1
+ fit cos(role vector, intent vector) as a level: the embedding
+ model's calibrated `lo` is 0 and `hi` 1, clipped outside
+ (see `scale`), so a role's fit does not depend on what
+ else is open
recency 0.5 ** (days since posted_at / 14)
affinity (the agent's affinity for the company + 1) / 3
- taste cos(role vector, liked direction), min-max scaled like fit,
- where the direction is mean(liked vectors) - mean(passed vectors)
+ taste cos(role vector, liked direction), scaled like fit with the
+ model's taste calibration, where the direction is
+ mean(liked vectors) - mean(passed vectors)
over this agent's votes; with only likes it is the liked mean,
with only passes the negated passed mean; no votes, no term
@@ -30,7 +32,10 @@ company (`affinity = 2`) skips the gates and is floored at 0.9; a hidden one
(`-1`) is not scored at all. A role with no vector is scored without `fit`
and `taste`, unless `fit` is all there is, when it is gated as "no vector".
An agent with no votes scores every role without `taste`; the direction is
-newsdesk's `centroid(liked) - centroid(disliked)`.
+newsdesk's `centroid(liked) - centroid(disliked)`. An agent with no intent
+has no `fit` either: its scores are not a match, so it is left out of what
+extract and summarise read (`Store._top_scope`), and a run names such agents
+in one notice.
Recomputed in full on every run, and each agent's rows are replaced in one
transaction: nothing is incremental, so nothing can drift.
@@ -47,8 +52,8 @@ from typing import Any
import numpy as np
-from pipeline import NULL, RunContext, job
-from app.config import Config
+from pipeline import NULL, Notice, RunContext, job
+from app.config import Calibration, Config
from app.jobs import embed
from app.models import Agent, Candidate, Gate, Weights
from app.storage import Store
@@ -110,19 +115,28 @@ def taste_direction(vectors: dict[str, np.ndarray],
return direction
-def scale(cosines: dict[str, float]) -> dict[str, float]:
- """Min-max to 0..1 across one agent's run, so `fit` ranks roles rather
- than stating a level.
+def scale(cosines: dict[str, float],
+ band: tuple[float, float] | None) -> dict[str, float]:
+ """Cosines to 0..1 levels: `clip((cos - lo) / (hi - lo), 0, 1)` with the
+ embedding model's calibrated `band` (`app.config.CALIBRATION`).
- `(cos + 1) / 2` was the level, and on multilingual-e5-small it spans about
- 0.87 to 0.95 for real postings against a real intent (0.55 to 0.79 on the
- all-MiniLM-L6-v2 this replaced), so a prefer gate weighted 0.2 moved a
- score more than the whole spread of fit did either way. Scaled, the
- closest role is 1 and the farthest 0. A single role, or roles that are all
- equally close, get 1: there is nothing to tell them apart by.
+ Raw cosines on multilingual-e5-small sit between about 0.75 and 0.87 for
+ real postings against a real intent, so unscaled a prefer gate weighted
+ 0.2 moved a score more than the whole spread of fit. The band stretches
+ that spread to 0..1 with fixed constants, so a role's level is the same
+ in every run and a `limits` bar is an absolute threshold.
+
+ Without a band, for a model nobody calibrated, it falls back to min-max
+ across the run: the closest role is 1 and the farthest 0, which ranks
+ rather than states a level. A single role, or roles that are all equally
+ close, then get 1: there is nothing to tell them apart by.
"""
if not cosines:
return {}
+ if band is not None:
+ lo, hi = band
+ return {uid: min(max((value - lo) / (hi - lo), 0.0), 1.0)
+ for uid, value in cosines.items()}
low, high = min(cosines.values()), max(cosines.values())
if high - low < 1e-9:
return dict.fromkeys(cosines, 1.0)
@@ -231,11 +245,24 @@ def _intent(store: Store, agent: Agent, model: str,
# ------------------------------------------------------------------- the job
+def no_intent(names: Sequence[str]) -> Notice:
+ """The notice for enabled agents without an intent: their scores have no
+ fit, so `Store._top_scope` keeps them from feeding extract and summarise."""
+ return Notice(
+ code="score.no_intent", level="warning", kind="expected", key="score",
+ text=f"No intent, so ranked without fit and not read by extract or "
+ f"summarise: {', '.join(names)}.",
+ fix="Give the agent an intent under Settings, Search agents.",
+ where="app.jobs.score")
+
+
def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
- intent: np.ndarray | None, now: datetime) -> dict[str, int]:
+ intent: np.ndarray | None, now: datetime,
+ calibration: Calibration | None = None) -> dict[str, int]:
"""Two passes: the cosines of the roles this agent scores, to its intent
- and to its taste direction, then every score with both scaled across
- them. A gated role is left out of the scaling, so a role that fails a
+ and to its taste direction, then every score with both scaled by the
+ model's `calibration`, or across the run without one. A gated role is
+ left out of the cosines, so under the fallback a role that fails a
require gate cannot stretch the range the ranked roles share. The voted
roles are scored like any other; a liked one sits near the direction it
helped make, which is what a like should do."""
@@ -251,7 +278,8 @@ def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
cosines[candidate.uid] = cosine(role_vec, intent)
if direction is not None:
tastes[candidate.uid] = cosine(role_vec, direction)
- fits, tastes = scale(cosines), scale(tastes)
+ fits = scale(cosines, calibration.fit if calibration else None)
+ tastes = scale(tastes, calibration.taste if calibration else None)
rows: list[tuple[str, float, str]] = []
gated = 0
for candidate in candidates:
@@ -274,11 +302,19 @@ def run_score(config: Config, store: Store, *,
vectors = store.vectors(model) if agents else {}
encoder = _Encoder(model, context)
now = store.clock()
+ calibration = config.embed.calibrated()
+ if agents and calibration is None:
+ log.warning("no calibration for %s: fit and taste are min-max scaled across "
+ "each agent's roles in this run, so they rank rather than state a "
+ "level; add it under embed.calibration", model)
+ if blind := [agent.name for agent in agents if not agent.intent.strip()]:
+ context.notice(no_intent(blind))
scored = gated = 0
for n, agent in enumerate(agents):
context.check()
context.progress(n, len(agents), f"scoring for {agent.name}, {n + 1} of {len(agents)}")
- counts = score_agent(store, agent, vectors, _intent(store, agent, model, encoder), now)
+ counts = score_agent(store, agent, vectors, _intent(store, agent, model, encoder), now,
+ calibration)
scored += counts["scored"]
gated += counts["gated"]
log.info("%-20s %6d scored, %6d gated", agent.name, counts["scored"], counts["gated"])
diff --git a/app/models.py b/app/models.py
index 46606d0..213e274 100644
--- a/app/models.py
+++ b/app/models.py
@@ -499,15 +499,20 @@ class Limits(Model):
"""How far down an agent's ranking the two model stages read. Each pair
is a top N, the backstop, and a minimum score, the real stop: a role
is read when it is among the agent's best `extract_top` ungated scores
- and not below `extract_min`, and summarised likewise. A fresh role with
- the worst fit scores about 0.27 on the default weights, a good one 0.6
- or more, so extract's floor is loose and summarise's, the long answer,
- stricter."""
+ and not below `extract_min`, and summarised likewise. Fit is a
+ calibrated level (`score.scale`), so a bar means the same in every run.
+ A fresh role with the worst fit scores about 0.27 on the default
+ weights, a perfect one about 0.87. Measured on 16,841 open roles for the
+ "Copenhagen engineering" starter agent (4,156 ungated, no votes), 0.55
+ admits about 300 roles to extract and 0.65 about 35 to summarise, the
+ long answer; the top N catches an agent whose bar admits far more. An
+ agent keeps the limits stored with it; only a key its JSON lacks
+ follows these defaults."""
extract_top: int = Field(500, ge=0)
- extract_min: float = Field(0.3, ge=0, le=1)
+ extract_min: float = Field(0.55, ge=0, le=1)
summarise_top: int = Field(30, ge=0)
- summarise_min: float = Field(0.5, ge=0, le=1)
+ summarise_min: float = Field(0.65, ge=0, le=1)
class Agent(Model):
diff --git a/app/static/agents.js b/app/static/agents.js
index 9274b49..bace965 100644
--- a/app/static/agents.js
+++ b/app/static/agents.js
@@ -23,7 +23,7 @@
// What the model reads after a pull, per agent: the best `extract_top`
// roles scoring at least `extract_min` get their gate columns filled, the
// best `summarise_top` at `summarise_min` or more a summary.
- const DEFAULT_LIMITS = { extract_top: 500, extract_min: 0.3, summarise_top: 30, summarise_min: 0.5 };
+ const DEFAULT_LIMITS = { extract_top: 500, extract_min: 0.55, summarise_top: 30, summarise_min: 0.65 };
const NEW = "new";
/** @type {object[]} every agent, as the server lists them */
diff --git a/app/storage/store.py b/app/storage/store.py
index a37b3b1..8366c1f 100644
--- a/app/storage/store.py
+++ b/app/storage/store.py
@@ -675,7 +675,9 @@ class Store:
limits ride out of the window subquery beside it, so each row is
held to the limits of the agent that ranked it; a key the JSON lacks
is the `Limits` default. The role is `r`. A company an agent hid
- needs no clause: `score` writes no rows for it under that agent."""
+ needs no clause: `score` writes no rows for it under that agent. An
+ agent without an intent counts for nothing here: its scores have no
+ fit, so they say how fresh a role is, not whether it is wanted."""
defaults = Limits()
return (f"""r.uid IN (SELECT uid FROM (
SELECT s.uid, s.score, a.limits, ROW_NUMBER() OVER (
@@ -683,6 +685,7 @@ class Store:
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
FROM scores s
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
+ AND TRIM(a.intent, ' ' || char(9, 10, 13)) <> ''
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
WHERE s.gated_reason = '')
WHERE n <= COALESCE(json_extract(limits, '$.{stage}_top'), ?)
diff --git a/app/templates/settings/agents.html b/app/templates/settings/agents.html
index d95264f..9d8ac08 100644
--- a/app/templates/settings/agents.html
+++ b/app/templates/settings/agents.html
@@ -33,11 +33,11 @@
- Every pull scores every open role for free; these bound what the model then reads, per agent. The count is the backstop, the score the real stop. A new role with the worst fit scores about 0.27, a good one 0.6 or more.
+ Every pull scores every open role for free; these bound what the model then reads, per agent. The count is the backstop, the score the real stop. Fit is a level, the same in every run: a new role with the worst fit scores about 0.27, a perfect one about 0.87. An agent without an intent feeds neither stage.Advanced