Calibrate fit and taste per embedding model instead of min-max per run

scale() maps a cosine to clip((cos - lo) / (hi - lo), 0, 1) with constants
from CALIBRATION in app/config.py, keyed by model and replaceable under
embed.calibration. multilingual-e5-small: fit 0.75..0.88, taste -0.14..0.19
(taste from synthetic votes). An uncalibrated model falls back to min-max
and logs a warning once per run.

The extract and summarise bars default to 0.55 and 0.65, about 300 and 35
roles for the starter agent on the measured data; stored limits are kept.
An agent without an intent is left out of the model-stage scope, and each
score run names such agents in one score.no_intent notice.

Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus committed 2026-09-29 14:59:34 +02:00
1 parent 706b127bc0
commit b021838959
12 files changed
+336 -76

No files matched your search

+39 -1
View File
@@ -20,7 +20,7 @@ from pathlib import Path
from typing import Any, Literal from typing import Any, Literal
import yaml import yaml
from pydantic import Field, ValidationError, model_validator from pydantic import Field, ValidationError, field_validator, model_validator
from app.models import Gate, Limits, Weights from app.models import Gate, Limits, Weights
from auth import AuthConfig from auth import AuthConfig
@@ -66,6 +66,35 @@ class SummariseConfig(ModelTaskConfig):
batch_size: int = Field(10, ge=1) batch_size: int = Field(10, ge=1)
class Calibration(Section):
"""Where one embedding model's cosines sit, as `(lo, hi)` for fit and for
taste: `score.scale` maps `lo` to 0 and `hi` to 1 and clips outside, so a
term is a level rather than a rank within the run."""
fit: tuple[float, float]
taste: tuple[float, float]
@field_validator("fit", "taste")
@classmethod
def _lo_below_hi(cls, band: tuple[float, float]) -> tuple[float, float]:
lo, hi = band
if not -1 <= lo < hi <= 1:
raise ValueError(f"needs -1 <= lo < hi <= 1, got [{lo}, {hi}]")
return band
#: The built-in calibration, by embedding model. Measured on a copy of the
#: live database, 16,841 open roles (see docs/decisions.md). fit: pooled over
#: five intents the raw cosine has p1 0.753 and p99.9 0.871; hi sits above
#: every single intent's p99.9, so only about 5 of the starter agent's 4,156
#: ungated roles clip to 1. taste: from synthetic votes, 30 liked and 30
#: passed roles for each of three intents, since the copy had none (pooled p1
#: -0.140, p99.9 0.192); re-measure it once real votes exist.
CALIBRATION: dict[str, Calibration] = {
"intfloat/multilingual-e5-small": Calibration(fit=(0.75, 0.88), taste=(-0.14, 0.19)),
}
class EmbedConfig(Section): class EmbedConfig(Section):
"""Turning a role into a vector for `fit`. Needs sentence-transformers; """Turning a role into a vector for `fit`. Needs sentence-transformers;
without it `embed` says so and does nothing, and `score` leaves fit out. without it `embed` says so and does nothing, and `score` leaves fit out.
@@ -87,6 +116,15 @@ class EmbedConfig(Section):
model: str = "intfloat/multilingual-e5-small" model: str = "intfloat/multilingual-e5-small"
#: Roles embedded per run at most, newest first. #: Roles embedded per run at most, newest first.
per_run: int = Field(50_000, ge=1) per_run: int = Field(50_000, ge=1)
#: `{model: {fit: [lo, hi], taste: [lo, hi]}}`, over `CALIBRATION`: an
#: entry replaces the built-in one for that model. Keyed by model, so a
#: change of `model` cannot carry another geometry's constants along. A
#: model with neither is min-max scaled per run, with a warning in the log.
calibration: dict[str, Calibration] = Field(default_factory=dict)
def calibrated(self) -> Calibration | None:
"""The configured model's calibration, or None when it has none."""
return self.calibration.get(self.model) or CALIBRATION.get(self.model)
class AgentSeed(Section): class AgentSeed(Section):
+58 -22
View File
@@ -14,13 +14,15 @@ A gate rejects on evidence, never on absence: a role whose column is empty
is not gated by it. `extract` is what fills an empty column, and it reads is not gated by it. `extract` is what fills an empty column, and it reads
only the roles that rank, so gating on an empty column would keep those only the roles that rank, so gating on an empty column would keep those
roles out of extract's reach for good. roles out of extract's reach for good.
fit cos(role vector, intent vector), min-max scaled across fit cos(role vector, intent vector) as a level: the embedding
the roles this agent scores in this run: the closest is 1, model's calibrated `lo` is 0 and `hi` 1, clipped outside
the farthest 0, a lone role 1 (see `scale`), so a role's fit does not depend on what
else is open
recency 0.5 ** (days since posted_at / 14) recency 0.5 ** (days since posted_at / 14)
affinity (the agent's affinity for the company + 1) / 3 affinity (the agent's affinity for the company + 1) / 3
taste cos(role vector, liked direction), min-max scaled like fit, taste cos(role vector, liked direction), scaled like fit with the
where the direction is mean(liked vectors) - mean(passed vectors) model's taste calibration, where the direction is
mean(liked vectors) - mean(passed vectors)
over this agent's votes; with only likes it is the liked mean, over this agent's votes; with only likes it is the liked mean,
with only passes the negated passed mean; no votes, no term with only passes the negated passed mean; no votes, no term
@@ -30,7 +32,10 @@ company (`affinity = 2`) skips the gates and is floored at 0.9; a hidden one
(`-1`) is not scored at all. A role with no vector is scored without `fit` (`-1`) is not scored at all. A role with no vector is scored without `fit`
and `taste`, unless `fit` is all there is, when it is gated as "no vector". and `taste`, unless `fit` is all there is, when it is gated as "no vector".
An agent with no votes scores every role without `taste`; the direction is An agent with no votes scores every role without `taste`; the direction is
newsdesk's `centroid(liked) - centroid(disliked)`. newsdesk's `centroid(liked) - centroid(disliked)`. An agent with no intent
has no `fit` either: its scores are not a match, so it is left out of what
extract and summarise read (`Store._top_scope`), and a run names such agents
in one notice.
Recomputed in full on every run, and each agent's rows are replaced in one Recomputed in full on every run, and each agent's rows are replaced in one
transaction: nothing is incremental, so nothing can drift. transaction: nothing is incremental, so nothing can drift.
@@ -47,8 +52,8 @@ from typing import Any
import numpy as np import numpy as np
from pipeline import NULL, RunContext, job from pipeline import NULL, Notice, RunContext, job
from app.config import Config from app.config import Calibration, Config
from app.jobs import embed from app.jobs import embed
from app.models import Agent, Candidate, Gate, Weights from app.models import Agent, Candidate, Gate, Weights
from app.storage import Store from app.storage import Store
@@ -110,19 +115,28 @@ def taste_direction(vectors: dict[str, np.ndarray],
return direction return direction
def scale(cosines: dict[str, float]) -> dict[str, float]: def scale(cosines: dict[str, float],
"""Min-max to 0..1 across one agent's run, so `fit` ranks roles rather band: tuple[float, float] | None) -> dict[str, float]:
than stating a level. """Cosines to 0..1 levels: `clip((cos - lo) / (hi - lo), 0, 1)` with the
embedding model's calibrated `band` (`app.config.CALIBRATION`).
`(cos + 1) / 2` was the level, and on multilingual-e5-small it spans about Raw cosines on multilingual-e5-small sit between about 0.75 and 0.87 for
0.87 to 0.95 for real postings against a real intent (0.55 to 0.79 on the real postings against a real intent, so unscaled a prefer gate weighted
all-MiniLM-L6-v2 this replaced), so a prefer gate weighted 0.2 moved a 0.2 moved a score more than the whole spread of fit. The band stretches
score more than the whole spread of fit did either way. Scaled, the that spread to 0..1 with fixed constants, so a role's level is the same
closest role is 1 and the farthest 0. A single role, or roles that are all in every run and a `limits` bar is an absolute threshold.
equally close, get 1: there is nothing to tell them apart by.
Without a band, for a model nobody calibrated, it falls back to min-max
across the run: the closest role is 1 and the farthest 0, which ranks
rather than states a level. A single role, or roles that are all equally
close, then get 1: there is nothing to tell them apart by.
""" """
if not cosines: if not cosines:
return {} return {}
if band is not None:
lo, hi = band
return {uid: min(max((value - lo) / (hi - lo), 0.0), 1.0)
for uid, value in cosines.items()}
low, high = min(cosines.values()), max(cosines.values()) low, high = min(cosines.values()), max(cosines.values())
if high - low < 1e-9: if high - low < 1e-9:
return dict.fromkeys(cosines, 1.0) return dict.fromkeys(cosines, 1.0)
@@ -231,11 +245,24 @@ def _intent(store: Store, agent: Agent, model: str,
# ------------------------------------------------------------------- the job # ------------------------------------------------------------------- the job
def no_intent(names: Sequence[str]) -> Notice:
"""The notice for enabled agents without an intent: their scores have no
fit, so `Store._top_scope` keeps them from feeding extract and summarise."""
return Notice(
code="score.no_intent", level="warning", kind="expected", key="score",
text=f"No intent, so ranked without fit and not read by extract or "
f"summarise: {', '.join(names)}.",
fix="Give the agent an intent under Settings, Search agents.",
where="app.jobs.score")
def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray], def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
intent: np.ndarray | None, now: datetime) -> dict[str, int]: intent: np.ndarray | None, now: datetime,
calibration: Calibration | None = None) -> dict[str, int]:
"""Two passes: the cosines of the roles this agent scores, to its intent """Two passes: the cosines of the roles this agent scores, to its intent
and to its taste direction, then every score with both scaled across and to its taste direction, then every score with both scaled by the
them. A gated role is left out of the scaling, so a role that fails a model's `calibration`, or across the run without one. A gated role is
left out of the cosines, so under the fallback a role that fails a
require gate cannot stretch the range the ranked roles share. The voted require gate cannot stretch the range the ranked roles share. The voted
roles are scored like any other; a liked one sits near the direction it roles are scored like any other; a liked one sits near the direction it
helped make, which is what a like should do.""" helped make, which is what a like should do."""
@@ -251,7 +278,8 @@ def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
cosines[candidate.uid] = cosine(role_vec, intent) cosines[candidate.uid] = cosine(role_vec, intent)
if direction is not None: if direction is not None:
tastes[candidate.uid] = cosine(role_vec, direction) tastes[candidate.uid] = cosine(role_vec, direction)
fits, tastes = scale(cosines), scale(tastes) fits = scale(cosines, calibration.fit if calibration else None)
tastes = scale(tastes, calibration.taste if calibration else None)
rows: list[tuple[str, float, str]] = [] rows: list[tuple[str, float, str]] = []
gated = 0 gated = 0
for candidate in candidates: for candidate in candidates:
@@ -274,11 +302,19 @@ def run_score(config: Config, store: Store, *,
vectors = store.vectors(model) if agents else {} vectors = store.vectors(model) if agents else {}
encoder = _Encoder(model, context) encoder = _Encoder(model, context)
now = store.clock() now = store.clock()
calibration = config.embed.calibrated()
if agents and calibration is None:
log.warning("no calibration for %s: fit and taste are min-max scaled across "
"each agent's roles in this run, so they rank rather than state a "
"level; add it under embed.calibration", model)
if blind := [agent.name for agent in agents if not agent.intent.strip()]:
context.notice(no_intent(blind))
scored = gated = 0 scored = gated = 0
for n, agent in enumerate(agents): for n, agent in enumerate(agents):
context.check() context.check()
context.progress(n, len(agents), f"scoring for {agent.name}, {n + 1} of {len(agents)}") context.progress(n, len(agents), f"scoring for {agent.name}, {n + 1} of {len(agents)}")
counts = score_agent(store, agent, vectors, _intent(store, agent, model, encoder), now) counts = score_agent(store, agent, vectors, _intent(store, agent, model, encoder), now,
calibration)
scored += counts["scored"] scored += counts["scored"]
gated += counts["gated"] gated += counts["gated"]
log.info("%-20s %6d scored, %6d gated", agent.name, counts["scored"], counts["gated"]) log.info("%-20s %6d scored, %6d gated", agent.name, counts["scored"], counts["gated"])
+11 -6
View File
@@ -499,15 +499,20 @@ class Limits(Model):
"""How far down an agent's ranking the two model stages read. Each pair """How far down an agent's ranking the two model stages read. Each pair
is a top N, the backstop, and a minimum score, the real stop: a role is a top N, the backstop, and a minimum score, the real stop: a role
is read when it is among the agent's best `extract_top` ungated scores is read when it is among the agent's best `extract_top` ungated scores
and not below `extract_min`, and summarised likewise. A fresh role with and not below `extract_min`, and summarised likewise. Fit is a
the worst fit scores about 0.27 on the default weights, a good one 0.6 calibrated level (`score.scale`), so a bar means the same in every run.
or more, so extract's floor is loose and summarise's, the long answer, A fresh role with the worst fit scores about 0.27 on the default
stricter.""" weights, a perfect one about 0.87. Measured on 16,841 open roles for the
"Copenhagen engineering" starter agent (4,156 ungated, no votes), 0.55
admits about 300 roles to extract and 0.65 about 35 to summarise, the
long answer; the top N catches an agent whose bar admits far more. An
agent keeps the limits stored with it; only a key its JSON lacks
follows these defaults."""
extract_top: int = Field(500, ge=0) extract_top: int = Field(500, ge=0)
extract_min: float = Field(0.3, ge=0, le=1) extract_min: float = Field(0.55, ge=0, le=1)
summarise_top: int = Field(30, ge=0) summarise_top: int = Field(30, ge=0)
summarise_min: float = Field(0.5, ge=0, le=1) summarise_min: float = Field(0.65, ge=0, le=1)
class Agent(Model): class Agent(Model):
+1 -1
View File
@@ -23,7 +23,7 @@
// What the model reads after a pull, per agent: the best `extract_top` // What the model reads after a pull, per agent: the best `extract_top`
// roles scoring at least `extract_min` get their gate columns filled, the // roles scoring at least `extract_min` get their gate columns filled, the
// best `summarise_top` at `summarise_min` or more a summary. // best `summarise_top` at `summarise_min` or more a summary.
const DEFAULT_LIMITS = { extract_top: 500, extract_min: 0.3, summarise_top: 30, summarise_min: 0.5 }; const DEFAULT_LIMITS = { extract_top: 500, extract_min: 0.55, summarise_top: 30, summarise_min: 0.65 };
const NEW = "new"; const NEW = "new";
/** @type {object[]} every agent, as the server lists them */ /** @type {object[]} every agent, as the server lists them */
+4 -1
View File
@@ -675,7 +675,9 @@ class Store:
limits ride out of the window subquery beside it, so each row is limits ride out of the window subquery beside it, so each row is
held to the limits of the agent that ranked it; a key the JSON lacks held to the limits of the agent that ranked it; a key the JSON lacks
is the `Limits` default. The role is `r`. A company an agent hid is the `Limits` default. The role is `r`. A company an agent hid
needs no clause: `score` writes no rows for it under that agent.""" needs no clause: `score` writes no rows for it under that agent. An
agent without an intent counts for nothing here: its scores have no
fit, so they say how fresh a role is, not whether it is wanted."""
defaults = Limits() defaults = Limits()
return (f"""r.uid IN (SELECT uid FROM ( return (f"""r.uid IN (SELECT uid FROM (
SELECT s.uid, s.score, a.limits, ROW_NUMBER() OVER ( SELECT s.uid, s.score, a.limits, ROW_NUMBER() OVER (
@@ -683,6 +685,7 @@ class Store:
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
FROM scores s FROM scores s
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1 JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
AND TRIM(a.intent, ' ' || char(9, 10, 13)) <> ''
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
WHERE s.gated_reason = '') WHERE s.gated_reason = '')
WHERE n <= COALESCE(json_extract(limits, '$.{stage}_top'), ?) WHERE n <= COALESCE(json_extract(limits, '$.{stage}_top'), ?)
+3 -3
View File
@@ -33,11 +33,11 @@
<fieldset class="weights limits"> <fieldset class="weights limits">
<legend>What the model reads</legend> <legend>What the model reads</legend>
<label>Extract the best <input type="number" id="a-extract-top" min="0" step="10" value="500"></label> <label>Extract the best <input type="number" id="a-extract-top" min="0" step="10" value="500"></label>
<label>down to a score of <input type="number" id="a-extract-min" min="0" max="1" step="0.05" value="0.3"></label> <label>down to a score of <input type="number" id="a-extract-min" min="0" max="1" step="0.05" value="0.55"></label>
<label>Summarise the best <input type="number" id="a-summarise-top" min="0" step="5" value="30"></label> <label>Summarise the best <input type="number" id="a-summarise-top" min="0" step="5" value="30"></label>
<label>down to a score of <input type="number" id="a-summarise-min" min="0" max="1" step="0.05" value="0.5"></label> <label>down to a score of <input type="number" id="a-summarise-min" min="0" max="1" step="0.05" value="0.65"></label>
</fieldset> </fieldset>
<span class="hint">Every pull scores every open role for free; these bound what the model then reads, per agent. The count is the backstop, the score the real stop. A new role with the worst fit scores about 0.27, a good one 0.6 or more.</span> <span class="hint">Every pull scores every open role for free; these bound what the model then reads, per agent. The count is the backstop, the score the real stop. Fit is a level, the same in every run: a new role with the worst fit scores about 0.27, a perfect one about 0.87. An agent without an intent feeds neither stage.</span>
<details class="advanced"> <details class="advanced">
<summary>Advanced</summary> <summary>Advanced</summary>
<label>Gates <label>Gates
+16 -6
View File
@@ -74,9 +74,16 @@ summarise:
# Danish-country roles (see docs/decisions.md), for about 3x the CPU time. An # Danish-country roles (see docs/decisions.md), for about 3x the CPU time. An
# e5 model's "query: " / "passage: " prefixes are added automatically, keyed # e5 model's "query: " / "passage: " prefixes are added automatically, keyed
# off this name. # off this name.
# calibration maps a model's cosines to 0..1 levels, as [lo, hi] for fit and
# for taste: lo is 0, hi is 1, clipped outside. multilingual-e5-small has
# built-in constants (app/config.py, CALIBRATION); an entry here replaces
# them for that model. A model with none is min-max scaled per run instead,
# so its fit is a rank, not a level, and the log says so.
embed: embed:
model: intfloat/multilingual-e5-small model: intfloat/multilingual-e5-small
per_run: 50000 per_run: 50000
calibration: {}
# intfloat/multilingual-e5-small: {fit: [0.75, 0.88], taste: [-0.14, 0.19]}
# One block per plugin, keyed by the plugin's name. The ats and search plugins # One block per plugin, keyed by the plugin's name. The ats and search plugins
# read their boards and searches from the subscriptions table, so they need # read their boards and searches from the subscriptions table, so they need
@@ -99,11 +106,14 @@ sources: {}
# limits is how deep into this agent's ranking the two model stages read, # limits is how deep into this agent's ranking the two model stages read,
# each a top N (the backstop) and a minimum score (the real stop): extract # each a top N (the backstop) and a minimum score (the real stop): extract
# reads the best extract_top roles at extract_min or more, summarise the best # reads the best extract_top roles at extract_min or more, summarise the best
# summarise_top at summarise_min or more. A fresh role with the worst fit # summarise_top at summarise_min or more. Fit is a calibrated level, so a bar
# scores about 0.27 on the default weights and a good one 0.6 or more. The # means the same in every run: a fresh role with the worst fit scores about
# defaults are {extract_top: 500, extract_min: 0.3, summarise_top: 30, # 0.27 on the default weights, a perfect one about 0.87. The defaults are
# summarise_min: 0.5}; a wide monitoring search raises them, a narrow hunt # {extract_top: 500, extract_min: 0.55, summarise_top: 30, summarise_min:
# lowers them. # 0.65}; measured for "Copenhagen engineering" on 16,841 open roles, 0.55
# admits about 300 roles to extract and 0.65 about 35 to summarise. A wide
# monitoring search lowers the bars and raises the counts, a narrow hunt the
# reverse. An agent without an intent has no fit and feeds neither stage.
# The Search agents sub-tab lists and edits them; the order menu on Roles # The Search agents sub-tab lists and edits them; the order menu on Roles
# picks one by name. Two more, beside the starter in config.yaml: # picks one by name. Two more, beside the starter in config.yaml:
agents: [] agents: []
@@ -117,7 +127,7 @@ agents: []
# - {field: country, values: [AT, BE, BG, HR, CY, CZ, DK, EE, FI, FR, DE, GR, HU, IE, IT, # - {field: country, values: [AT, BE, BG, HR, CY, CZ, DK, EE, FI, FR, DE, GR, HU, IE, IT,
# LV, LT, LU, MT, NL, PL, PT, RO, SK, SI, ES, SE, IS, LI, NO, # LV, LT, LU, MT, NL, PL, PT, RO, SK, SI, ES, SE, IS, LI, NO,
# GB, CH, EU], mode: require} # GB, CH, EU], mode: require}
# limits: {extract_top: 1000, extract_min: 0.3, summarise_top: 50, summarise_min: 0.5} # limits: {extract_top: 1000, extract_min: 0.45, summarise_top: 50, summarise_min: 0.6}
# - name: Europe hybrid # - name: Europe hybrid
# intent: >- # intent: >-
# Senior backend or platform engineer in a European tech hub, hybrid or # Senior backend or platform engineer in a European tech hub, hybrid or
+10 -4
View File
@@ -90,7 +90,13 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**Scoring is per search agent.** A role is scored as `(uid, agent) -> score`, because a role is only good for a given search. Each gate has a mode: `require` becomes a SQL `WHERE`, `prefer` becomes a weighted term in the score. Gated roles keep their row with a reason. `companies.affinity = 2` skips the gates and floors the score, `-1` mutes. **Scoring is per search agent.** A role is scored as `(uid, agent) -> score`, because a role is only good for a given search. Each gate has a mode: `require` becomes a SQL `WHERE`, `prefer` becomes a weighted term in the score. Gated roles keep their row with a reason. `companies.affinity = 2` skips the gates and floors the score, `-1` mutes.
**`fit` is the cosine between a role and the agent's intent, min-max scaled across the roles that agent scores in the run.** `(cos + 1) / 2` spanned only about 0.55 to 0.79 in the real run, so a prefer gate at 0.2 outweighed all of fit. A single role gets 1, gated roles stay out of the range, and the starter agents' prefer weights are 0.05. **`fit` is the cosine between a role and the agent's intent, mapped to a level by constants calibrated per embedding model: `clip((cos - lo) / (hi - lo), 0, 1)`.** `(cos + 1) / 2` spanned only about 0.87 to 0.95 on multilingual-e5-small, so a prefer gate at 0.2 outweighed all of fit. Min-max across the roles an agent scored in the run fixed the spread but made fit a rank: the best role of a poor set got 1, a role's fit and the 0.9 favourite floor's meaning drifted from run to run, and the `limits` bars that bound the paid stages compared an absolute number with a relative one. The constants are `CALIBRATION` in `app/config.py`, keyed by model name, and an entry under `embed.calibration` replaces a model's built-in one; keyed by model, a change of `embed.model` cannot carry another geometry's constants along. A model with none falls back to min-max per run, with gated roles left out of the range, and the log warns once per run. `taste` is mapped the same way with its own pair. The starter agents' prefer weights are 0.05.
**multilingual-e5-small's constants are fit 0.75 to 0.88 and taste -0.14 to 0.19, measured on a copy of the live database.** Over 16,841 open roles and five different intents the raw fit cosine pooled to p1 0.7528, p50 0.7989, p99 0.8528, p99.5 0.8595, p99.9 0.8711 and max 0.9031; per intent p1 ranged 0.7456 to 0.7700 and p99.9 0.8413 to 0.8773. `hi` is above every intent's p99.9, so about 5 of the "Copenhagen engineering" agent's 4,156 ungated roles clip to 1; at 0.86 (p99.5) 75 would, and lose their order where it matters most. The span 0.13 is close to the 0.145 min-max span it replaces, so a prefer weight keeps about its old size against fit. The copy had no votes, so taste comes from synthetic ones, 30 liked and 30 passed roles for each of three intents (pooled p1 -0.1404, p50 0.0429, p99.9 0.1924), and should be re-measured once real votes exist.
**The default bars are 0.55 for extract and 0.65 for summarise.** On the same data, the Copenhagen agent's ungated roles clearing each bar, min-max then calibrated, were: 0.3: 3,467 and 3,520; 0.4: 1,837 and 2,090; 0.5: 519 and 728; 0.55: 200 and 307; 0.6: 56 and 111; 0.65: 15 and 35. So 0.55 sends about 300 roles to extract and 0.65 about 35 to summarise, which its top 30 trims; the top-N backstops (500 and 30) stay for an agent whose bar admits more. The old defaults, 0.3 and 0.5, admit 3,520 and 728 on the calibrated scale, where only the top N would stop them. An agent keeps the limits stored with it, so one saved under the old defaults keeps 0.3 and 0.5 until it is edited: a migration cannot tell a stored default from a choice.
**An agent without an intent feeds neither model stage.** Its scores have no fit term, so they order roles by recency, affinity and prefer gates, and a fresh role of any kind clears a bar. `Store._top_scope` leaves such an agent out, and each score run raises one `score.no_intent` notice naming the enabled agents without an intent. The agent still ranks its own list.
**The embedding backend is effectively required.** The intent is a CV or a paragraph per agent, and TF-IDF cannot relate "Kubernetes" to "container orchestration". The model is `intfloat/multilingual-e5-small`, not the English-only `all-MiniLM-L6-v2` roledesk started with: the corpus turned European, and on a sample of the live roles (all open Danish-country roles plus a random sample of the rest, language labelled by a stop-word count) against the "Copenhagen engineering" agent's intent, MiniLM separates engineering from non-engineering roles with an AUC of 0.68 overall but only 0.74 restricted to Danish-country roles, where a Danish-country engineering role beats the rest's non-engineering median only 68% of the time; multilingual-e5-small reaches an AUC of 0.70 overall and 1.00 on Danish-country roles, for about 3x the CPU time. A second multilingual candidate, `paraphrase-multilingual-MiniLM-L12-v2`, measured *worse* than plain MiniLM (AUC 0.60) and was dropped. e5 needs its own "query: " / "passage: " prefixes, which `app/jobs/embed.py` adds by matching the model name, not a separate setting. **The embedding backend is effectively required.** The intent is a CV or a paragraph per agent, and TF-IDF cannot relate "Kubernetes" to "container orchestration". The model is `intfloat/multilingual-e5-small`, not the English-only `all-MiniLM-L6-v2` roledesk started with: the corpus turned European, and on a sample of the live roles (all open Danish-country roles plus a random sample of the rest, language labelled by a stop-word count) against the "Copenhagen engineering" agent's intent, MiniLM separates engineering from non-engineering roles with an AUC of 0.68 overall but only 0.74 restricted to Danish-country roles, where a Danish-country engineering role beats the rest's non-engineering median only 68% of the time; multilingual-e5-small reaches an AUC of 0.70 overall and 1.00 on Danish-country roles, for about 3x the CPU time. A second multilingual candidate, `paraphrase-multilingual-MiniLM-L12-v2`, measured *worse* than plain MiniLM (AUC 0.60) and was dropped. e5 needs its own "query: " / "passage: " prefixes, which `app/jobs/embed.py` adds by matching the model name, not a separate setting.
@@ -112,7 +118,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**The pull cascade is `pull → embed → score → extract → score → summarise`.** Every stage is always in the chain and a stage that is off finds nothing, as in newsdesk. The free stages run first: embed and score cost no tokens, and their score is what bounds the two model stages. Extract reads only what ranks, then score runs again so the gates read what it filled. A stage is addressed by its position in the tuple, since `score` is in it twice. Extract reads `on_ingest` only as a stage of a pull, so with `auto` and no model reachable a pull passes it quietly. **The pull cascade is `pull → embed → score → extract → score → summarise`.** Every stage is always in the chain and a stage that is off finds nothing, as in newsdesk. The free stages run first: embed and score cost no tokens, and their score is what bounds the two model stages. Extract reads only what ranks, then score runs again so the gates read what it filled. A stage is addressed by its position in the tuple, since `score` is in it twice. Extract reads `on_ingest` only as a stage of a pull, so with `auto` and no model reachable a pull passes it quietly.
**Extract is bounded like summarise: the top roles of each agent, by that agent's own `limits`.** Extract has short answers but would otherwise touch every open role, and the seed boards alone produced 29,000; at Haiku's rates that is tens of dollars for a first pass over roles nobody will look at. Bounded to the ranking, a pull reads a few hundred. `extract_top` is the backstop (default 500) and `extract_min` the real stop (default 0.3, loose): a fixed count stays flat whatever the boards produce, where a percentage would not. The bounds are per agent rather than config, because one agent may be a wide monitoring search that should read a thousand and another a narrow hunt that wants fifty; one global number fits neither. A batch dealt named uids finishes them whatever their rank, so a running batch is not cut short by a rescore. **Extract is bounded like summarise: the top roles of each agent, by that agent's own `limits`.** Extract has short answers but would otherwise touch every open role, and the seed boards alone produced 29,000; at Haiku's rates that is tens of dollars for a first pass over roles nobody will look at. Bounded to the ranking, a pull reads a few hundred. `extract_top` is the backstop (default 500) and `extract_min` the real stop (default 0.55, about 300 roles for the starter agent): a fixed count stays flat whatever the boards produce, where a percentage would not. The bounds are per agent rather than config, because one agent may be a wide monitoring search that should read a thousand and another a narrow hunt that wants fifty; one global number fits neither. A batch dealt named uids finishes them whatever their rank, so a running batch is not cut short by a rescore.
**`extract_batch` and `summarise_batch` are `isolate=True`, with a timeout.** `context.check()` only runs between the calls `modeltask.one` makes, not inside one, so a call that hangs on either backend was otherwise unkillable short of restarting the server; a process boundary is what lets a Kill, or the timeout itself, actually end it. One call is already bounded at `agent.timeout`, so the deadline is a few calls' worth (30 minutes for extract, 15 for summarise): a batch still running after that is stuck, not slow. The cost: an isolated job's context has no working `stop`, `task` or `fan_out` (see `pipeline/child.py`'s `context_for`), so a refusal or an account-level failure inside a batch can no longer stop the rest of that task's queued siblings — they run on to the same failure instead of being spared it. **`extract_batch` and `summarise_batch` are `isolate=True`, with a timeout.** `context.check()` only runs between the calls `modeltask.one` makes, not inside one, so a call that hangs on either backend was otherwise unkillable short of restarting the server; a process boundary is what lets a Kill, or the timeout itself, actually end it. One call is already bounded at `agent.timeout`, so the deadline is a few calls' worth (30 minutes for extract, 15 for summarise): a batch still running after that is stuck, not slow. The cost: an isolated job's context has no working `stop`, `task` or `fan_out` (see `pipeline/child.py`'s `context_for`), so a refusal or an account-level failure inside a batch can no longer stop the rest of that task's queued siblings — they run on to the same failure instead of being spared it.
@@ -142,7 +148,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**Affinity and votes are per agent.** A company hidden for one hunt may be a favourite for another, and a vote says what one agent should learn from, so both carry the agent id, and the Roles tab and the company list ask for one agent's. Without an agent id the API uses the default agent, the enabled one with the lowest id, so the CLI and older callers keep working. **Affinity and votes are per agent.** A company hidden for one hunt may be a favourite for another, and a vote says what one agent should learn from, so both carry the agent id, and the Roles tab and the company list ask for one agent's. Without an agent id the API uses the default agent, the enabled one with the lowest id, so the CLI and older callers keep working.
**`taste` is the liked-minus-passed centroid, as in newsdesk.** An agent's votes become one vector, the mean of the liked roles' embeddings minus the mean of the passed ones', and a role's taste term is its cosine to that vector, weighted beside fit, recency and affinity. It is the one term an agent learns rather than is told, and it costs no model call. **`taste` is the liked-minus-passed centroid, as in newsdesk.** An agent's votes become one vector, the mean of the liked roles' embeddings minus the mean of the passed ones', and a role's taste term is its cosine to that vector, calibrated like fit, weighted beside fit, recency and affinity. It is the one term an agent learns rather than is told, and it costs no model call.
**Roles are a feed with a reader, not links out.** `GET /roles/{uid}` renders the feed with its reader open on that role instead of sending the reader to the board, because 29,194 of 29,196 open board roles already have a `description` averaging 6,300 characters, and a search role's short teaser is worth reading in place before deciding whether to fetch the rest. **Roles are a feed with a reader, not links out.** `GET /roles/{uid}` renders the feed with its reader open on that role instead of sending the reader to the board, because 29,194 of 29,196 open board roles already have a `description` averaging 6,300 characters, and a search role's short teaser is worth reading in place before deciding whether to fetch the rest.
@@ -160,7 +166,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**A search role's posting text is fetched on demand, not in the pull.** `POST /api/roles/{uid}/text` scrapes `original_url` only when the read page asks for it, because following Jobindex's redirect for every role during a pull would count as a click in Jobindex's own statistics; one click for a posting somebody actually opened is fair. **A search role's posting text is fetched on demand, not in the pull.** `POST /api/roles/{uid}/text` scrapes `original_url` only when the read page asks for it, because following Jobindex's redirect for every role during a pull would count as a click in Jobindex's own statistics; one click for a posting somebody actually opened is fair.
**Summarising is bounded to the top roles of each agent, by that agent's own `limits`, and off by default.** The pull's `summarise` stage only considers roles with no summary yet that are in the top `summarise_top` (default 30) of an enabled agent's scores and at `summarise_min` (default 0.5, stricter than extract's since a summary is the long answer), so the cost stays flat whatever the backlog of open roles is. The bounds live on the agent, beside its gates and weights, for the same reason as extract's: a wide monitoring search and a narrow hunt want different depths, and the agent is where the difference is already expressed. `summarise.enabled: false` matches `extract`'s own default; the read page's Summarise button asks for one role's summary either way. **Summarising is bounded to the top roles of each agent, by that agent's own `limits`, and off by default.** The pull's `summarise` stage only considers roles with no summary yet that are in the top `summarise_top` (default 30) of an enabled agent's scores and at `summarise_min` (default 0.65, about 35 roles for the starter agent, stricter than extract's since a summary is the long answer), so the cost stays flat whatever the backlog of open roles is. The bounds live on the agent, beside its gates and weights, for the same reason as extract's: a wide monitoring search and a narrow hunt want different depths, and the agent is where the difference is already expressed. `summarise.enabled: false` matches `extract`'s own default; the read page's Summarise button asks for one role's summary either way.
**The schedules element's narrow layout is fixed upstream, in `../boilerplate`.** `<desk-schedules>` is a shadow root the app cannot lay out from outside, and `pipeline/` is a downstream copy refreshed with `git archive`, so a fix landing only in roledesk would be reverted by the next refresh. **The schedules element's narrow layout is fixed upstream, in `../boilerplate`.** `<desk-schedules>` is a shadow root the app cannot lay out from outside, and `pipeline/` is a downstream copy refreshed with `git archive`, so a fix landing only in roledesk would be reverted by the next refresh.
+10 -6
View File
@@ -62,7 +62,7 @@ def ranked(store: Store, *items: Item) -> None:
"""Score the roles for one enabled agent, best first, so they are in the """Score the roles for one enabled agent, best first, so they are in the
top extract reads. Every role given must already be stored.""" top extract reads. Every role given must already be stored."""
agents = store.agents() agents = store.agents()
agent_id = agents[0].id if agents else store.save_agent("Backend").id agent_id = agents[0].id if agents else store.save_agent("Backend", "python backend").id
store.replace_scores(agent_id, [(item.uid, 1.0 - n / 10, "") store.replace_scores(agent_id, [(item.uid, 1.0 - n / 10, "")
for n, item in enumerate(items)]) for n, item in enumerate(items)])
@@ -171,8 +171,9 @@ def test_only_the_best_of_each_agent_above_the_floor_is_read(
uid finishes it.""" uid finishes it."""
for sid in ("best", "second", "low", "unscored", "gated"): for sid in ("best", "second", "low", "unscored", "gated"):
store.upsert_role(posting(sid)) store.upsert_role(posting(sid))
one = store.save_agent("One", limits=Limits(extract_top=1, extract_min=0.5)) one = store.save_agent("One", "python backend",
two = store.save_agent("Two", enabled=False) limits=Limits(extract_top=1, extract_min=0.5))
two = store.save_agent("Two", "python backend", enabled=False)
store.replace_scores(one.id, [(posting("best").uid, 0.9, ""), store.replace_scores(one.id, [(posting("best").uid, 0.9, ""),
(posting("second").uid, 0.8, ""), (posting("second").uid, 0.8, ""),
(posting("low").uid, 0.2, ""), (posting("low").uid, 0.2, ""),
@@ -182,11 +183,14 @@ def test_only_the_best_of_each_agent_above_the_floor_is_read(
model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles]) model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles])
assert store.extract_pending() == 1 assert store.extract_pending() == 1
store.save_agent("One", agent_id=one.id, limits=Limits(extract_top=5, extract_min=0.5)) store.save_agent("One", "python backend", agent_id=one.id,
limits=Limits(extract_top=5, extract_min=0.5))
assert store.extract_pending() == 2, "the floor holds the low one back" assert store.extract_pending() == 2, "the floor holds the low one back"
store.save_agent("One", agent_id=one.id, limits=Limits(extract_top=5, extract_min=0)) store.save_agent("One", "python backend", agent_id=one.id,
limits=Limits(extract_top=5, extract_min=0))
assert store.extract_pending() == 3, "the disabled agent's pick never counts" assert store.extract_pending() == 3, "the disabled agent's pick never counts"
store.save_agent("One", agent_id=one.id, limits=Limits(extract_top=1, extract_min=0.5)) store.save_agent("One", "python backend", agent_id=one.id,
limits=Limits(extract_top=1, extract_min=0.5))
modeltask.run(extract.TASK, config, store) modeltask.run(extract.TASK, config, store)
assert [[r["uid"] for r in batch] for batch in model.asked] == [[posting("best").uid]] assert [[r["uid"] for r in batch] for batch in model.asked] == [[posting("best").uid]]
+170 -17
View File
@@ -3,6 +3,7 @@ by a deterministic fake. Nothing here downloads a model or reaches a board."""
from __future__ import annotations from __future__ import annotations
import re
import sys import sys
from collections.abc import Iterator, Sequence from collections.abc import Iterator, Sequence
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
@@ -19,13 +20,17 @@ if str(ROOT) not in sys.path:
from desk.models import Item # noqa: E402 from desk.models import Item # noqa: E402
from pipeline import FINAL, JOBS, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, Tally # noqa: E402 from pipeline import FINAL, JOBS, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, Tally # noqa: E402
from plugins.ats import ATSSource # noqa: E402 from plugins.ats import ATSSource # noqa: E402
from app.config import AgentSeed, Config # noqa: E402 from app.config import CALIBRATION, AgentSeed, Config, ConfigError # noqa: E402
from app.jobs import cascade, embed, score # noqa: E402 from app.jobs import cascade, embed, score # noqa: E402
from app.models import Candidate, Gate, Weights # noqa: E402 from app.models import Candidate, Gate, Limits, Weights # noqa: E402
from app.storage import Store # noqa: E402 from app.storage import Store # noqa: E402
NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC) NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
#: The shipped constants, which the fake is calibrated with so a test reads
#: the rule production runs; `uncalibrated` is the min-max fallback.
E5 = CALIBRATION["intfloat/multilingual-e5-small"]
#: The fake's whole vocabulary: one dimension per word, plus a constant one so #: The fake's whole vocabulary: one dimension per word, plus a constant one so
#: no text is the zero vector. #: no text is the zero vector.
WORDS = ("backend", "python", "frontend", "react", "data") WORDS = ("backend", "python", "frontend", "react", "data")
@@ -76,6 +81,12 @@ def store(tmp_path: Path, clock: Clock) -> Iterator[Store]:
@pytest.fixture @pytest.fixture
def config(tmp_path: Path) -> Config: def config(tmp_path: Path) -> Config:
return Config(database=str(tmp_path / "roledesk.db"),
embed={"model": "fake", "calibration": {"fake": E5.model_dump()}})
@pytest.fixture
def uncalibrated(tmp_path: Path) -> Config:
return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"}) return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
@@ -94,9 +105,17 @@ def run(config: Config, store: Store, job: Any) -> dict[str, Any]:
notices: list[Any] = [] notices: list[Any] = []
result = dict(job(config, store, context=RunContext(on_notice=notices.append))) result = dict(job(config, store, context=RunContext(on_notice=notices.append)))
result["notices"] = [n.code for n in notices] result["notices"] = [n.code for n in notices]
result["notice_texts"] = [n.text for n in notices]
return result return result
def level(text: str, intent: str, band: tuple[float, float] = E5.fit) -> float:
"""The calibrated fit the fake gives a role whose `document` is `text`."""
role_vec, intent_vec = FakeEmbedder().encode([text, intent])
lo, hi = band
return min(max((score.cosine(role_vec, intent_vec) - lo) / (hi - lo), 0.0), 1.0)
# ------------------------------------------------------------------ embed # ------------------------------------------------------------------ embed
def test_embed_writes_a_vector_per_open_role_once( def test_embed_writes_a_vector_per_open_role_once(
@@ -431,7 +450,8 @@ def test_two_agents_rank_the_same_roles_the_other_way_round(
result = run(config, store, score.run_score) result = run(config, store, score.run_score)
assert result["agents"] == 2 assert result["agents"] == 2
# Fit alone, scaled per agent: each agent's closer role is 1, the other 0. # Fit alone, calibrated: each agent's own role is at a cosine past `hi`,
# so 1, the other's near 0, below `lo`, so 0.
assert store.scores(backend.id)[back][0] == 1.0 assert store.scores(backend.id)[back][0] == 1.0
assert store.scores(backend.id)[front][0] == 0.0 assert store.scores(backend.id)[front][0] == 0.0
assert store.scores(frontend.id)[front][0] == 1.0 assert store.scores(frontend.id)[front][0] == 1.0
@@ -481,6 +501,7 @@ def test_taste_ranks_a_role_near_the_liked_one_above_one_near_the_passed(
scores = store.scores(agent.id) scores = store.scores(agent.id)
assert scores[near_liked][0] > scores[near_passed][0] assert scores[near_liked][0] > scores[near_passed][0]
# Cosines of about 0.7 and -0.7 to the direction, past both ends of the band.
assert scores[liked][0] == 1.0 and scores[passed][0] == 0.0 assert scores[liked][0] == 1.0 and scores[passed][0] == 0.0
assert all(reason == "" for _, reason in scores.values()) assert all(reason == "" for _, reason in scores.values())
@@ -513,35 +534,167 @@ def test_a_closed_liked_role_still_shapes_the_taste(
scores = store.scores(agent.id) scores = store.scores(agent.id)
assert liked not in scores assert liked not in scores
assert scores[near][0] == 1.0 and scores[far][0] == 0.0 # Near sits on the liked direction, past `hi`. Far is orthogonal to it,
# a cosine of about 0 and so a middling level, not the 0 min-max gave it.
assert scores[near][0] == 1.0
assert scores[far][0] == pytest.approx(
level("Web Engineer\nReact frontend", "Backend Engineer\nPython backend", E5.taste),
abs=1e-5)
assert 0 < scores[far][0] < 1
# ------------------------------------------------------------ fit scaling # ------------------------------------------------------------ fit scaling
def test_scale_maps_the_closest_to_one_and_the_farthest_to_zero() -> None: def test_scale_maps_a_cosine_to_its_calibrated_level_and_clips_both_ends() -> None:
# The spread the last real run showed under `(cos + 1) / 2`, on # multilingual-e5-small's fit band: lo 0.75 is 0, hi 0.88 is 1.
# multilingual-e5-small. assert E5.fit == (0.75, 0.88) and E5.taste == (-0.14, 0.19)
assert score.scale({"a": 0.87, "b": 0.95, "c": 0.91}) == pytest.approx( assert score.scale({"lo": 0.75, "mid": 0.815, "hi": 0.88, "p50": 0.7989},
E5.fit) == pytest.approx(
{"lo": 0.0, "mid": 0.5, "hi": 1.0, "p50": 0.0489 / 0.13})
assert score.scale({"under": 0.6, "negative": -0.2, "over": 0.9031, "one": 1.0},
E5.fit) == {"under": 0.0, "negative": 0.0, "over": 1.0, "one": 1.0}
assert score.scale({"none": 0.0, "liked": 0.25}, E5.taste) == pytest.approx(
{"none": 0.14 / 0.33, "liked": 1.0})
# A lone role, or roles that are all equally close, get their level.
assert score.scale({"only": 0.8}, E5.fit) == pytest.approx({"only": 0.05 / 0.13})
assert score.scale({}, E5.fit) == {}
def test_scale_without_a_band_falls_back_to_min_max() -> None:
assert score.scale({"a": 0.87, "b": 0.95, "c": 0.91}, None) == pytest.approx(
{"a": 0.0, "b": 1.0, "c": 0.5}) {"a": 0.0, "b": 1.0, "c": 0.5})
assert score.scale({"only": 0.3}) == {"only": 1.0} assert score.scale({"only": 0.3}, None) == {"only": 1.0}
assert score.scale({"a": 0.4, "b": 0.4}) == {"a": 1.0, "b": 1.0} assert score.scale({"a": 0.4, "b": 0.4}, None) == {"a": 1.0, "b": 1.0}
assert score.scale({}) == {} assert score.scale({}, None) == {}
def test_a_single_candidate_gets_full_fit( def test_a_single_candidate_gets_its_level_not_full_fit(
store: Store, config: Config, fake: FakeEmbedder) -> None: store: Store, config: Config, uncalibrated: Config, fake: FakeEmbedder) -> None:
"""Min-max gave the best of a poor set 1; a level gives it what it is."""
uid = role(store, "1", "Frontend Engineer", body="React apps") uid = role(store, "1", "Frontend Engineer", body="React apps")
agent = store.save_agent("Backend", "python backend developer", agent = store.save_agent("Backend", "python backend developer",
weights=Weights(fit=1, recency=0, affinity=0)) weights=Weights(fit=1, recency=0, affinity=0))
run(config, store, embed.run_embed) run(config, store, embed.run_embed)
run(config, store, score.run_score) run(config, store, score.run_score)
assert store.scores(agent.id)[uid] == (0.0, "")
run(uncalibrated, store, score.run_score)
assert store.scores(agent.id)[uid] == (1.0, "") assert store.scores(agent.id)[uid] == (1.0, "")
def test_a_gated_role_does_not_stretch_the_fit_range( def test_the_same_role_gets_the_same_fit_whatever_else_is_open(
store: Store, config: Config, fake: FakeEmbedder) -> None: store: Store, config: Config, uncalibrated: Config, fake: FakeEmbedder) -> None:
"""The point of a level: two runs over different candidate sets give the
role the same fit, where min-max gave it 1 beside a worse role and 0
beside a better one."""
mid = role(store, "1", "Backend Engineer", body="Python backend data")
worse = role(store, "2", "Frontend Engineer", body="React apps")
agent = store.save_agent("Backend", "python backend",
weights=Weights(fit=1, recency=0, affinity=0))
expected = level("Backend Engineer\nPython backend data", "python backend")
assert 0.5 < expected < 1
run(config, store, embed.run_embed)
run(config, store, score.run_score)
first = store.scores(agent.id)[mid][0]
run(uncalibrated, store, score.run_score)
assert store.scores(agent.id)[mid][0] == 1.0
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (NOW.isoformat(), worse))
role(store, "3", "Backend Engineer", body="Python")
run(config, store, embed.run_embed)
run(config, store, score.run_score)
second = store.scores(agent.id)[mid][0]
run(uncalibrated, store, score.run_score)
assert store.scores(agent.id)[mid][0] == 0.0
assert first == second == pytest.approx(expected, abs=1e-5)
def test_an_uncalibrated_model_falls_back_to_min_max_and_warns_once_a_run(
store: Store, uncalibrated: Config, fake: FakeEmbedder,
caplog: pytest.LogCaptureFixture) -> None:
close = role(store, "1", "Backend Engineer", body="Python backend data")
far = role(store, "2", "Data Engineer", body="Data")
only_fit = Weights(fit=1, recency=0, affinity=0)
one = store.save_agent("One", "python backend", weights=only_fit)
two = store.save_agent("Two", "python backend", weights=only_fit)
run(uncalibrated, store, embed.run_embed)
with caplog.at_level("WARNING", logger="app.jobs.score"):
run(uncalibrated, store, score.run_score)
warnings = [r.getMessage() for r in caplog.records if r.name == "app.jobs.score"]
assert len(warnings) == 1 and warnings[0].startswith("no calibration for fake")
for agent in (one, two):
assert store.scores(agent.id)[close][0] == 1.0
assert store.scores(agent.id)[far][0] == 0.0
def test_the_calibration_is_the_built_in_one_unless_the_config_replaces_it() -> None:
assert Config().embed.calibrated() == E5
assert Config(embed={"model": "fake"}).embed.calibrated() is None
mine = {"fit": [0.7, 0.9], "taste": [-0.2, 0.2]}
replaced = Config(embed={"calibration": {"intfloat/multilingual-e5-small": mine}})
assert replaced.embed.calibrated() is not None
assert replaced.embed.calibrated().fit == (0.7, 0.9) # type: ignore[union-attr]
# Keyed by model: another model's entry does not apply.
other = Config(embed={"model": "fake", "calibration": {"other": mine}})
assert other.embed.calibrated() is None
@pytest.mark.parametrize("entry, says", [
({"fit": [0.88, 0.75], "taste": [-0.14, 0.19]}, "embed.calibration.fake.fit"),
({"fit": [0.75, 0.75], "taste": [-0.14, 0.19]}, "lo < hi"),
({"fit": [0.75, 0.88], "taste": [-2, 0.19]}, "embed.calibration.fake.taste"),
({"fit": [0.75, 0.88]}, "embed.calibration.fake.taste"),
({"fit": [0.75, 0.88], "taste": [-0.14, 0.19], "spread": 1}, "unknown setting"),
])
def test_a_bad_calibration_is_a_config_error(
tmp_path: Path, entry: dict[str, Any], says: str) -> None:
import yaml
from app import config as config_mod
path = tmp_path / "config.yaml"
path.write_text(yaml.safe_dump({"embed": {"model": "fake",
"calibration": {"fake": entry}}}))
with pytest.raises(ConfigError, match=re.escape(says)):
config_mod.load(path)
def test_an_agent_without_an_intent_feeds_neither_model_stage_and_is_named(
store: Store, config: Config, clock: Clock, fake: FakeEmbedder) -> None:
"""Its scores have no fit, so they rank fresh roles, not wanted ones: the
newer role tops it, and extract and summarise must not read it on that."""
clock.at = NOW - timedelta(days=1)
wanted = role(store, "1", "Backend Engineer", body="Python backend")
clock.at = NOW
fresh = role(store, "2", "Frontend Engineer", body="React apps")
top_one = Limits(extract_top=1, extract_min=0, summarise_top=1, summarise_min=0)
store.save_agent("Backend", "python backend", limits=top_one)
blind = store.save_agent("Fresh", "", limits=top_one)
store.save_agent("Also fresh", " \n", limits=top_one)
store.save_agent("Off", "", enabled=False)
run(config, store, embed.run_embed)
result = run(config, store, score.run_score)
assert result["notices"] == ["score.no_intent"]
assert result["notice_texts"][0].endswith(": Also fresh, Fresh.")
ranked = sorted(store.scores(blind.id).items(), key=lambda kv: -kv[1][0])
assert ranked[0][0] == fresh, "the intent-less agent would pick the newer role"
assert [r.uid for r in store.extract_waiting(10)] == [wanted]
assert [r.uid for r in store.summarise_waiting(10)] == [wanted]
assert store.extract_pending() == store.summarise_pending() == 1
def test_a_gated_role_does_not_stretch_the_fit_range_under_the_fallback(
store: Store, uncalibrated: Config, fake: FakeEmbedder) -> None:
"""Calibrated, no other role moves a role's fit. Min-max, the fallback,
must still leave a gated role out of the range."""
config = uncalibrated
close = role(store, "1", "Backend Engineer", body="Python backend", place="Berlin") close = role(store, "1", "Backend Engineer", body="Python backend", place="Berlin")
far = role(store, "2", "Data Engineer", body="Data", place="Berlin") far = role(store, "2", "Data Engineer", body="Data", place="Berlin")
# Farther from the intent than either, but gated out by place. # Farther from the intent than either, but gated out by place.
@@ -562,8 +715,8 @@ def test_a_small_prefer_gate_no_longer_outweighs_fit(
"""The flaw the last real run showed: real cosines sit close together, so """The flaw the last real run showed: real cosines sit close together, so
under `(cos + 1) / 2` a prefer gate decided the order over fit. Here the under `(cos + 1) / 2` a prefer gate decided the order over fit. Here the
cosines are about 0.95 and 0.87: the old fit terms differ by 0.04, which cosines are about 0.95 and 0.87: the old fit terms differ by 0.04, which
at weight 0.6 is less than the gate's 0.05, so the remote role won. Scaled, at weight 0.6 is less than the gate's 0.05, so the remote role won.
the closer role wins.""" Calibrated, they are 1 and about 0.89, and the closer role wins."""
close = role(store, "1", "Backend Engineer", body="Python backend", remote="onsite") close = role(store, "1", "Backend Engineer", body="Python backend", remote="onsite")
far = role(store, "2", "Backend Engineer", body="Python backend data", remote="remote") far = role(store, "2", "Backend Engineer", body="Python backend data", remote="remote")
agent = store.save_agent("Backend", "python backend", agent = store.save_agent("Backend", "python backend",
+11 -6
View File
@@ -1033,24 +1033,29 @@ def test_summarise_waits_on_the_top_of_each_enabled_agent_by_its_own_limits(
a, b, c, d = (posting(s).uid for s in "abcd") a, b, c, d = (posting(s).uid for s in "abcd")
with store.transaction() as conn: with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (store.now(), d)) conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (store.now(), d))
one = store.save_agent("One", limits=Limits(summarise_top=1, summarise_min=0)) one = store.save_agent("One", "python backend",
two = store.save_agent("Two", limits=Limits(summarise_top=1, summarise_min=0)) limits=Limits(summarise_top=1, summarise_min=0))
two = store.save_agent("Two", "python backend",
limits=Limits(summarise_top=1, summarise_min=0))
store.replace_scores(one.id, [(d, 0.99, ""), (a, 0.9, ""), (b, 0.5, "")]) store.replace_scores(one.id, [(d, 0.99, ""), (a, 0.9, ""), (b, 0.5, "")])
store.replace_scores(two.id, [(a, 0.95, "remote"), (c, 0.7, ""), (b, 0.8, "")]) store.replace_scores(two.id, [(a, 0.95, "remote"), (c, 0.7, ""), (b, 0.8, "")])
assert {r.uid for r in store.summarise_waiting(10)} == {a, b} assert {r.uid for r in store.summarise_waiting(10)} == {a, b}
# Two goes deeper: its second, c, joins; One's second, b, was in already. # Two goes deeper: its second, c, joins; One's second, b, was in already.
store.save_agent("Two", agent_id=two.id, limits=Limits(summarise_top=2, summarise_min=0)) store.save_agent("Two", "python backend", agent_id=two.id,
limits=Limits(summarise_top=2, summarise_min=0))
assert {r.uid for r in store.summarise_waiting(10)} == {a, b, c} assert {r.uid for r in store.summarise_waiting(10)} == {a, b, c}
assert store.summarise_pending() == 3 assert store.summarise_pending() == 3
assert [r.uid for r in store.summarise_waiting(10, uids=[d])] == [d] assert [r.uid for r in store.summarise_waiting(10, uids=[d])] == [d]
# The floor is Two's own: c at 0.7 drops, One's b at 0.5 stays. # The floor is Two's own: c at 0.7 drops, One's b at 0.5 stays.
store.save_agent("Two", agent_id=two.id, store.save_agent("Two", "python backend", agent_id=two.id,
limits=Limits(summarise_top=2, summarise_min=0.75)) limits=Limits(summarise_top=2, summarise_min=0.75))
store.save_agent("One", agent_id=one.id, limits=Limits(summarise_top=2, summarise_min=0)) store.save_agent("One", "python backend", agent_id=one.id,
limits=Limits(summarise_top=2, summarise_min=0))
assert {r.uid for r in store.summarise_waiting(10)} == {a, b} assert {r.uid for r in store.summarise_waiting(10)} == {a, b}
# '{}' is the defaults, 30 at 0.5: One's b at 0.5 stays, Two's c at 0.7 is back. # '{}' is the defaults, 30 at 0.65: One's b at 0.5 drops, but Two's b at
# 0.8 and c at 0.7 clear it.
with store.transaction() as conn: with store.transaction() as conn:
conn.execute("UPDATE agents SET limits = '{}'") conn.execute("UPDATE agents SET limits = '{}'")
assert {r.uid for r in store.summarise_waiting(10)} == {a, b, c} assert {r.uid for r in store.summarise_waiting(10)} == {a, b, c}
+3 -3
View File
@@ -659,7 +659,7 @@ def test_the_agents_page_lists_and_edits_search_agents(client: TestClient) -> No
for limit in ("a-extract-top", "a-extract-min", "a-summarise-top", "a-summarise-min"): for limit in ("a-extract-top", "a-extract-min", "a-summarise-top", "a-summarise-min"):
assert f'id="{limit}"' in template assert f'id="{limit}"' in template
assert 'id="a-extract-top" min="0" step="10" value="500"' in page assert 'id="a-extract-top" min="0" step="10" value="500"' in page
assert 'id="a-summarise-min" min="0" max="1" step="0.05" value="0.5"' in page assert 'id="a-summarise-min" min="0" max="1" step="0.05" value="0.65"' in page
assert 'id="a-new"' in page and 'id="a-delete"' in page assert 'id="a-new"' in page and 'id="a-delete"' in page
assert "Save and score" in page assert "Save and score" in page
@@ -823,8 +823,8 @@ def test_an_agents_limits_are_saved_listed_and_checked(client: TestClient) -> No
"""How deep the model stages read is the agent's own; a score outside """How deep the model stages read is the agent's own; a score outside
0..1 is refused.""" 0..1 is refused."""
default = client.post("/api/agents", json={"name": "Wide"}).json() default = client.post("/api/agents", json={"name": "Wide"}).json()
assert default["limits"] == {"extract_top": 500, "extract_min": 0.3, assert default["limits"] == {"extract_top": 500, "extract_min": 0.55,
"summarise_top": 30, "summarise_min": 0.5} "summarise_top": 30, "summarise_min": 0.65}
narrow = {"extract_top": 50, "extract_min": 0.6, "summarise_top": 5, "summarise_min": 0.7} narrow = {"extract_top": 50, "extract_min": 0.6, "summarise_top": 5, "summarise_min": 0.7}
saved = client.post("/api/agents", json={"name": "Narrow", "limits": narrow}) saved = client.post("/api/agents", json={"name": "Narrow", "limits": narrow})
assert saved.status_code == 200 and saved.json()["limits"] == narrow assert saved.status_code == 200 and saved.json()["limits"] == narrow