Calibrate fit and taste per embedding model instead of min-max per run
scale() maps a cosine to clip((cos - lo) / (hi - lo), 0, 1) with constants from CALIBRATION in app/config.py, keyed by model and replaceable under embed.calibration. multilingual-e5-small: fit 0.75..0.88, taste -0.14..0.19 (taste from synthetic votes). An uncalibrated model falls back to min-max and logs a warning once per run. The extract and summarise bars default to 0.55 and 0.65, about 300 and 35 roles for the starter agent on the measured data; stored limits are kept. An agent without an intent is left out of the model-stage scope, and each score run names such agents in one score.no_intent notice. Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
1 parent
706b127bc0
commit
b021838959
12 files changed
+336
-76
No files matched your search
+39
-1
@@ -20,7 +20,7 @@ from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
import yaml
|
||||
from pydantic import Field, ValidationError, model_validator
|
||||
from pydantic import Field, ValidationError, field_validator, model_validator
|
||||
|
||||
from app.models import Gate, Limits, Weights
|
||||
from auth import AuthConfig
|
||||
@@ -66,6 +66,35 @@ class SummariseConfig(ModelTaskConfig):
|
||||
batch_size: int = Field(10, ge=1)
|
||||
|
||||
|
||||
class Calibration(Section):
|
||||
"""Where one embedding model's cosines sit, as `(lo, hi)` for fit and for
|
||||
taste: `score.scale` maps `lo` to 0 and `hi` to 1 and clips outside, so a
|
||||
term is a level rather than a rank within the run."""
|
||||
|
||||
fit: tuple[float, float]
|
||||
taste: tuple[float, float]
|
||||
|
||||
@field_validator("fit", "taste")
|
||||
@classmethod
|
||||
def _lo_below_hi(cls, band: tuple[float, float]) -> tuple[float, float]:
|
||||
lo, hi = band
|
||||
if not -1 <= lo < hi <= 1:
|
||||
raise ValueError(f"needs -1 <= lo < hi <= 1, got [{lo}, {hi}]")
|
||||
return band
|
||||
|
||||
|
||||
#: The built-in calibration, by embedding model. Measured on a copy of the
|
||||
#: live database, 16,841 open roles (see docs/decisions.md). fit: pooled over
|
||||
#: five intents the raw cosine has p1 0.753 and p99.9 0.871; hi sits above
|
||||
#: every single intent's p99.9, so only about 5 of the starter agent's 4,156
|
||||
#: ungated roles clip to 1. taste: from synthetic votes, 30 liked and 30
|
||||
#: passed roles for each of three intents, since the copy had none (pooled p1
|
||||
#: -0.140, p99.9 0.192); re-measure it once real votes exist.
|
||||
CALIBRATION: dict[str, Calibration] = {
|
||||
"intfloat/multilingual-e5-small": Calibration(fit=(0.75, 0.88), taste=(-0.14, 0.19)),
|
||||
}
|
||||
|
||||
|
||||
class EmbedConfig(Section):
|
||||
"""Turning a role into a vector for `fit`. Needs sentence-transformers;
|
||||
without it `embed` says so and does nothing, and `score` leaves fit out.
|
||||
@@ -87,6 +116,15 @@ class EmbedConfig(Section):
|
||||
model: str = "intfloat/multilingual-e5-small"
|
||||
#: Roles embedded per run at most, newest first.
|
||||
per_run: int = Field(50_000, ge=1)
|
||||
#: `{model: {fit: [lo, hi], taste: [lo, hi]}}`, over `CALIBRATION`: an
|
||||
#: entry replaces the built-in one for that model. Keyed by model, so a
|
||||
#: change of `model` cannot carry another geometry's constants along. A
|
||||
#: model with neither is min-max scaled per run, with a warning in the log.
|
||||
calibration: dict[str, Calibration] = Field(default_factory=dict)
|
||||
|
||||
def calibrated(self) -> Calibration | None:
|
||||
"""The configured model's calibration, or None when it has none."""
|
||||
return self.calibration.get(self.model) or CALIBRATION.get(self.model)
|
||||
|
||||
|
||||
class AgentSeed(Section):
|
||||
|
||||
+58
-22
@@ -14,13 +14,15 @@ A gate rejects on evidence, never on absence: a role whose column is empty
|
||||
is not gated by it. `extract` is what fills an empty column, and it reads
|
||||
only the roles that rank, so gating on an empty column would keep those
|
||||
roles out of extract's reach for good.
|
||||
fit cos(role vector, intent vector), min-max scaled across
|
||||
the roles this agent scores in this run: the closest is 1,
|
||||
the farthest 0, a lone role 1
|
||||
fit cos(role vector, intent vector) as a level: the embedding
|
||||
model's calibrated `lo` is 0 and `hi` 1, clipped outside
|
||||
(see `scale`), so a role's fit does not depend on what
|
||||
else is open
|
||||
recency 0.5 ** (days since posted_at / 14)
|
||||
affinity (the agent's affinity for the company + 1) / 3
|
||||
taste cos(role vector, liked direction), min-max scaled like fit,
|
||||
where the direction is mean(liked vectors) - mean(passed vectors)
|
||||
taste cos(role vector, liked direction), scaled like fit with the
|
||||
model's taste calibration, where the direction is
|
||||
mean(liked vectors) - mean(passed vectors)
|
||||
over this agent's votes; with only likes it is the liked mean,
|
||||
with only passes the negated passed mean; no votes, no term
|
||||
|
||||
@@ -30,7 +32,10 @@ company (`affinity = 2`) skips the gates and is floored at 0.9; a hidden one
|
||||
(`-1`) is not scored at all. A role with no vector is scored without `fit`
|
||||
and `taste`, unless `fit` is all there is, when it is gated as "no vector".
|
||||
An agent with no votes scores every role without `taste`; the direction is
|
||||
newsdesk's `centroid(liked) - centroid(disliked)`.
|
||||
newsdesk's `centroid(liked) - centroid(disliked)`. An agent with no intent
|
||||
has no `fit` either: its scores are not a match, so it is left out of what
|
||||
extract and summarise read (`Store._top_scope`), and a run names such agents
|
||||
in one notice.
|
||||
|
||||
Recomputed in full on every run, and each agent's rows are replaced in one
|
||||
transaction: nothing is incremental, so nothing can drift.
|
||||
@@ -47,8 +52,8 @@ from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from pipeline import NULL, RunContext, job
|
||||
from app.config import Config
|
||||
from pipeline import NULL, Notice, RunContext, job
|
||||
from app.config import Calibration, Config
|
||||
from app.jobs import embed
|
||||
from app.models import Agent, Candidate, Gate, Weights
|
||||
from app.storage import Store
|
||||
@@ -110,19 +115,28 @@ def taste_direction(vectors: dict[str, np.ndarray],
|
||||
return direction
|
||||
|
||||
|
||||
def scale(cosines: dict[str, float]) -> dict[str, float]:
|
||||
"""Min-max to 0..1 across one agent's run, so `fit` ranks roles rather
|
||||
than stating a level.
|
||||
def scale(cosines: dict[str, float],
|
||||
band: tuple[float, float] | None) -> dict[str, float]:
|
||||
"""Cosines to 0..1 levels: `clip((cos - lo) / (hi - lo), 0, 1)` with the
|
||||
embedding model's calibrated `band` (`app.config.CALIBRATION`).
|
||||
|
||||
`(cos + 1) / 2` was the level, and on multilingual-e5-small it spans about
|
||||
0.87 to 0.95 for real postings against a real intent (0.55 to 0.79 on the
|
||||
all-MiniLM-L6-v2 this replaced), so a prefer gate weighted 0.2 moved a
|
||||
score more than the whole spread of fit did either way. Scaled, the
|
||||
closest role is 1 and the farthest 0. A single role, or roles that are all
|
||||
equally close, get 1: there is nothing to tell them apart by.
|
||||
Raw cosines on multilingual-e5-small sit between about 0.75 and 0.87 for
|
||||
real postings against a real intent, so unscaled a prefer gate weighted
|
||||
0.2 moved a score more than the whole spread of fit. The band stretches
|
||||
that spread to 0..1 with fixed constants, so a role's level is the same
|
||||
in every run and a `limits` bar is an absolute threshold.
|
||||
|
||||
Without a band, for a model nobody calibrated, it falls back to min-max
|
||||
across the run: the closest role is 1 and the farthest 0, which ranks
|
||||
rather than states a level. A single role, or roles that are all equally
|
||||
close, then get 1: there is nothing to tell them apart by.
|
||||
"""
|
||||
if not cosines:
|
||||
return {}
|
||||
if band is not None:
|
||||
lo, hi = band
|
||||
return {uid: min(max((value - lo) / (hi - lo), 0.0), 1.0)
|
||||
for uid, value in cosines.items()}
|
||||
low, high = min(cosines.values()), max(cosines.values())
|
||||
if high - low < 1e-9:
|
||||
return dict.fromkeys(cosines, 1.0)
|
||||
@@ -231,11 +245,24 @@ def _intent(store: Store, agent: Agent, model: str,
|
||||
|
||||
# ------------------------------------------------------------------- the job
|
||||
|
||||
def no_intent(names: Sequence[str]) -> Notice:
|
||||
"""The notice for enabled agents without an intent: their scores have no
|
||||
fit, so `Store._top_scope` keeps them from feeding extract and summarise."""
|
||||
return Notice(
|
||||
code="score.no_intent", level="warning", kind="expected", key="score",
|
||||
text=f"No intent, so ranked without fit and not read by extract or "
|
||||
f"summarise: {', '.join(names)}.",
|
||||
fix="Give the agent an intent under Settings, Search agents.",
|
||||
where="app.jobs.score")
|
||||
|
||||
|
||||
def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
|
||||
intent: np.ndarray | None, now: datetime) -> dict[str, int]:
|
||||
intent: np.ndarray | None, now: datetime,
|
||||
calibration: Calibration | None = None) -> dict[str, int]:
|
||||
"""Two passes: the cosines of the roles this agent scores, to its intent
|
||||
and to its taste direction, then every score with both scaled across
|
||||
them. A gated role is left out of the scaling, so a role that fails a
|
||||
and to its taste direction, then every score with both scaled by the
|
||||
model's `calibration`, or across the run without one. A gated role is
|
||||
left out of the cosines, so under the fallback a role that fails a
|
||||
require gate cannot stretch the range the ranked roles share. The voted
|
||||
roles are scored like any other; a liked one sits near the direction it
|
||||
helped make, which is what a like should do."""
|
||||
@@ -251,7 +278,8 @@ def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
|
||||
cosines[candidate.uid] = cosine(role_vec, intent)
|
||||
if direction is not None:
|
||||
tastes[candidate.uid] = cosine(role_vec, direction)
|
||||
fits, tastes = scale(cosines), scale(tastes)
|
||||
fits = scale(cosines, calibration.fit if calibration else None)
|
||||
tastes = scale(tastes, calibration.taste if calibration else None)
|
||||
rows: list[tuple[str, float, str]] = []
|
||||
gated = 0
|
||||
for candidate in candidates:
|
||||
@@ -274,11 +302,19 @@ def run_score(config: Config, store: Store, *,
|
||||
vectors = store.vectors(model) if agents else {}
|
||||
encoder = _Encoder(model, context)
|
||||
now = store.clock()
|
||||
calibration = config.embed.calibrated()
|
||||
if agents and calibration is None:
|
||||
log.warning("no calibration for %s: fit and taste are min-max scaled across "
|
||||
"each agent's roles in this run, so they rank rather than state a "
|
||||
"level; add it under embed.calibration", model)
|
||||
if blind := [agent.name for agent in agents if not agent.intent.strip()]:
|
||||
context.notice(no_intent(blind))
|
||||
scored = gated = 0
|
||||
for n, agent in enumerate(agents):
|
||||
context.check()
|
||||
context.progress(n, len(agents), f"scoring for {agent.name}, {n + 1} of {len(agents)}")
|
||||
counts = score_agent(store, agent, vectors, _intent(store, agent, model, encoder), now)
|
||||
counts = score_agent(store, agent, vectors, _intent(store, agent, model, encoder), now,
|
||||
calibration)
|
||||
scored += counts["scored"]
|
||||
gated += counts["gated"]
|
||||
log.info("%-20s %6d scored, %6d gated", agent.name, counts["scored"], counts["gated"])
|
||||
|
||||
+11
-6
@@ -499,15 +499,20 @@ class Limits(Model):
|
||||
"""How far down an agent's ranking the two model stages read. Each pair
|
||||
is a top N, the backstop, and a minimum score, the real stop: a role
|
||||
is read when it is among the agent's best `extract_top` ungated scores
|
||||
and not below `extract_min`, and summarised likewise. A fresh role with
|
||||
the worst fit scores about 0.27 on the default weights, a good one 0.6
|
||||
or more, so extract's floor is loose and summarise's, the long answer,
|
||||
stricter."""
|
||||
and not below `extract_min`, and summarised likewise. Fit is a
|
||||
calibrated level (`score.scale`), so a bar means the same in every run.
|
||||
A fresh role with the worst fit scores about 0.27 on the default
|
||||
weights, a perfect one about 0.87. Measured on 16,841 open roles for the
|
||||
"Copenhagen engineering" starter agent (4,156 ungated, no votes), 0.55
|
||||
admits about 300 roles to extract and 0.65 about 35 to summarise, the
|
||||
long answer; the top N catches an agent whose bar admits far more. An
|
||||
agent keeps the limits stored with it; only a key its JSON lacks
|
||||
follows these defaults."""
|
||||
|
||||
extract_top: int = Field(500, ge=0)
|
||||
extract_min: float = Field(0.3, ge=0, le=1)
|
||||
extract_min: float = Field(0.55, ge=0, le=1)
|
||||
summarise_top: int = Field(30, ge=0)
|
||||
summarise_min: float = Field(0.5, ge=0, le=1)
|
||||
summarise_min: float = Field(0.65, ge=0, le=1)
|
||||
|
||||
|
||||
class Agent(Model):
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
// What the model reads after a pull, per agent: the best `extract_top`
|
||||
// roles scoring at least `extract_min` get their gate columns filled, the
|
||||
// best `summarise_top` at `summarise_min` or more a summary.
|
||||
const DEFAULT_LIMITS = { extract_top: 500, extract_min: 0.3, summarise_top: 30, summarise_min: 0.5 };
|
||||
const DEFAULT_LIMITS = { extract_top: 500, extract_min: 0.55, summarise_top: 30, summarise_min: 0.65 };
|
||||
const NEW = "new";
|
||||
|
||||
/** @type {object[]} every agent, as the server lists them */
|
||||
|
||||
@@ -675,7 +675,9 @@ class Store:
|
||||
limits ride out of the window subquery beside it, so each row is
|
||||
held to the limits of the agent that ranked it; a key the JSON lacks
|
||||
is the `Limits` default. The role is `r`. A company an agent hid
|
||||
needs no clause: `score` writes no rows for it under that agent."""
|
||||
needs no clause: `score` writes no rows for it under that agent. An
|
||||
agent without an intent counts for nothing here: its scores have no
|
||||
fit, so they say how fresh a role is, not whether it is wanted."""
|
||||
defaults = Limits()
|
||||
return (f"""r.uid IN (SELECT uid FROM (
|
||||
SELECT s.uid, s.score, a.limits, ROW_NUMBER() OVER (
|
||||
@@ -683,6 +685,7 @@ class Store:
|
||||
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
|
||||
FROM scores s
|
||||
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
|
||||
AND TRIM(a.intent, ' ' || char(9, 10, 13)) <> ''
|
||||
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
||||
WHERE s.gated_reason = '')
|
||||
WHERE n <= COALESCE(json_extract(limits, '$.{stage}_top'), ?)
|
||||
|
||||
@@ -33,11 +33,11 @@
|
||||
<fieldset class="weights limits">
|
||||
<legend>What the model reads</legend>
|
||||
<label>Extract the best <input type="number" id="a-extract-top" min="0" step="10" value="500"></label>
|
||||
<label>down to a score of <input type="number" id="a-extract-min" min="0" max="1" step="0.05" value="0.3"></label>
|
||||
<label>down to a score of <input type="number" id="a-extract-min" min="0" max="1" step="0.05" value="0.55"></label>
|
||||
<label>Summarise the best <input type="number" id="a-summarise-top" min="0" step="5" value="30"></label>
|
||||
<label>down to a score of <input type="number" id="a-summarise-min" min="0" max="1" step="0.05" value="0.5"></label>
|
||||
<label>down to a score of <input type="number" id="a-summarise-min" min="0" max="1" step="0.05" value="0.65"></label>
|
||||
</fieldset>
|
||||
<span class="hint">Every pull scores every open role for free; these bound what the model then reads, per agent. The count is the backstop, the score the real stop. A new role with the worst fit scores about 0.27, a good one 0.6 or more.</span>
|
||||
<span class="hint">Every pull scores every open role for free; these bound what the model then reads, per agent. The count is the backstop, the score the real stop. Fit is a level, the same in every run: a new role with the worst fit scores about 0.27, a perfect one about 0.87. An agent without an intent feeds neither stage.</span>
|
||||
<details class="advanced">
|
||||
<summary>Advanced</summary>
|
||||
<label>Gates
|
||||
|
||||
+16
-6
@@ -74,9 +74,16 @@ summarise:
|
||||
# Danish-country roles (see docs/decisions.md), for about 3x the CPU time. An
|
||||
# e5 model's "query: " / "passage: " prefixes are added automatically, keyed
|
||||
# off this name.
|
||||
# calibration maps a model's cosines to 0..1 levels, as [lo, hi] for fit and
|
||||
# for taste: lo is 0, hi is 1, clipped outside. multilingual-e5-small has
|
||||
# built-in constants (app/config.py, CALIBRATION); an entry here replaces
|
||||
# them for that model. A model with none is min-max scaled per run instead,
|
||||
# so its fit is a rank, not a level, and the log says so.
|
||||
embed:
|
||||
model: intfloat/multilingual-e5-small
|
||||
per_run: 50000
|
||||
calibration: {}
|
||||
# intfloat/multilingual-e5-small: {fit: [0.75, 0.88], taste: [-0.14, 0.19]}
|
||||
|
||||
# One block per plugin, keyed by the plugin's name. The ats and search plugins
|
||||
# read their boards and searches from the subscriptions table, so they need
|
||||
@@ -99,11 +106,14 @@ sources: {}
|
||||
# limits is how deep into this agent's ranking the two model stages read,
|
||||
# each a top N (the backstop) and a minimum score (the real stop): extract
|
||||
# reads the best extract_top roles at extract_min or more, summarise the best
|
||||
# summarise_top at summarise_min or more. A fresh role with the worst fit
|
||||
# scores about 0.27 on the default weights and a good one 0.6 or more. The
|
||||
# defaults are {extract_top: 500, extract_min: 0.3, summarise_top: 30,
|
||||
# summarise_min: 0.5}; a wide monitoring search raises them, a narrow hunt
|
||||
# lowers them.
|
||||
# summarise_top at summarise_min or more. Fit is a calibrated level, so a bar
|
||||
# means the same in every run: a fresh role with the worst fit scores about
|
||||
# 0.27 on the default weights, a perfect one about 0.87. The defaults are
|
||||
# {extract_top: 500, extract_min: 0.55, summarise_top: 30, summarise_min:
|
||||
# 0.65}; measured for "Copenhagen engineering" on 16,841 open roles, 0.55
|
||||
# admits about 300 roles to extract and 0.65 about 35 to summarise. A wide
|
||||
# monitoring search lowers the bars and raises the counts, a narrow hunt the
|
||||
# reverse. An agent without an intent has no fit and feeds neither stage.
|
||||
# The Search agents sub-tab lists and edits them; the order menu on Roles
|
||||
# picks one by name. Two more, beside the starter in config.yaml:
|
||||
agents: []
|
||||
@@ -117,7 +127,7 @@ agents: []
|
||||
# - {field: country, values: [AT, BE, BG, HR, CY, CZ, DK, EE, FI, FR, DE, GR, HU, IE, IT,
|
||||
# LV, LT, LU, MT, NL, PL, PT, RO, SK, SI, ES, SE, IS, LI, NO,
|
||||
# GB, CH, EU], mode: require}
|
||||
# limits: {extract_top: 1000, extract_min: 0.3, summarise_top: 50, summarise_min: 0.5}
|
||||
# limits: {extract_top: 1000, extract_min: 0.45, summarise_top: 50, summarise_min: 0.6}
|
||||
# - name: Europe hybrid
|
||||
# intent: >-
|
||||
# Senior backend or platform engineer in a European tech hub, hybrid or
|
||||
|
||||
+10
-4
@@ -90,7 +90,13 @@ What was decided for roledesk and the one reason for each. The longer reasoning
|
||||
|
||||
**Scoring is per search agent.** A role is scored as `(uid, agent) -> score`, because a role is only good for a given search. Each gate has a mode: `require` becomes a SQL `WHERE`, `prefer` becomes a weighted term in the score. Gated roles keep their row with a reason. `companies.affinity = 2` skips the gates and floors the score, `-1` mutes.
|
||||
|
||||
**`fit` is the cosine between a role and the agent's intent, min-max scaled across the roles that agent scores in the run.** `(cos + 1) / 2` spanned only about 0.55 to 0.79 in the real run, so a prefer gate at 0.2 outweighed all of fit. A single role gets 1, gated roles stay out of the range, and the starter agents' prefer weights are 0.05.
|
||||
**`fit` is the cosine between a role and the agent's intent, mapped to a level by constants calibrated per embedding model: `clip((cos - lo) / (hi - lo), 0, 1)`.** `(cos + 1) / 2` spanned only about 0.87 to 0.95 on multilingual-e5-small, so a prefer gate at 0.2 outweighed all of fit. Min-max across the roles an agent scored in the run fixed the spread but made fit a rank: the best role of a poor set got 1, a role's fit and the 0.9 favourite floor's meaning drifted from run to run, and the `limits` bars that bound the paid stages compared an absolute number with a relative one. The constants are `CALIBRATION` in `app/config.py`, keyed by model name, and an entry under `embed.calibration` replaces a model's built-in one; keyed by model, a change of `embed.model` cannot carry another geometry's constants along. A model with none falls back to min-max per run, with gated roles left out of the range, and the log warns once per run. `taste` is mapped the same way with its own pair. The starter agents' prefer weights are 0.05.
|
||||
|
||||
**multilingual-e5-small's constants are fit 0.75 to 0.88 and taste -0.14 to 0.19, measured on a copy of the live database.** Over 16,841 open roles and five different intents the raw fit cosine pooled to p1 0.7528, p50 0.7989, p99 0.8528, p99.5 0.8595, p99.9 0.8711 and max 0.9031; per intent p1 ranged 0.7456 to 0.7700 and p99.9 0.8413 to 0.8773. `hi` is above every intent's p99.9, so about 5 of the "Copenhagen engineering" agent's 4,156 ungated roles clip to 1; at 0.86 (p99.5) 75 would, and lose their order where it matters most. The span 0.13 is close to the 0.145 min-max span it replaces, so a prefer weight keeps about its old size against fit. The copy had no votes, so taste comes from synthetic ones, 30 liked and 30 passed roles for each of three intents (pooled p1 -0.1404, p50 0.0429, p99.9 0.1924), and should be re-measured once real votes exist.
|
||||
|
||||
**The default bars are 0.55 for extract and 0.65 for summarise.** On the same data, the Copenhagen agent's ungated roles clearing each bar, min-max then calibrated, were: 0.3: 3,467 and 3,520; 0.4: 1,837 and 2,090; 0.5: 519 and 728; 0.55: 200 and 307; 0.6: 56 and 111; 0.65: 15 and 35. So 0.55 sends about 300 roles to extract and 0.65 about 35 to summarise, which its top 30 trims; the top-N backstops (500 and 30) stay for an agent whose bar admits more. The old defaults, 0.3 and 0.5, admit 3,520 and 728 on the calibrated scale, where only the top N would stop them. An agent keeps the limits stored with it, so one saved under the old defaults keeps 0.3 and 0.5 until it is edited: a migration cannot tell a stored default from a choice.
|
||||
|
||||
**An agent without an intent feeds neither model stage.** Its scores have no fit term, so they order roles by recency, affinity and prefer gates, and a fresh role of any kind clears a bar. `Store._top_scope` leaves such an agent out, and each score run raises one `score.no_intent` notice naming the enabled agents without an intent. The agent still ranks its own list.
|
||||
|
||||
**The embedding backend is effectively required.** The intent is a CV or a paragraph per agent, and TF-IDF cannot relate "Kubernetes" to "container orchestration". The model is `intfloat/multilingual-e5-small`, not the English-only `all-MiniLM-L6-v2` roledesk started with: the corpus turned European, and on a sample of the live roles (all open Danish-country roles plus a random sample of the rest, language labelled by a stop-word count) against the "Copenhagen engineering" agent's intent, MiniLM separates engineering from non-engineering roles with an AUC of 0.68 overall but only 0.74 restricted to Danish-country roles, where a Danish-country engineering role beats the rest's non-engineering median only 68% of the time; multilingual-e5-small reaches an AUC of 0.70 overall and 1.00 on Danish-country roles, for about 3x the CPU time. A second multilingual candidate, `paraphrase-multilingual-MiniLM-L12-v2`, measured *worse* than plain MiniLM (AUC 0.60) and was dropped. e5 needs its own "query: " / "passage: " prefixes, which `app/jobs/embed.py` adds by matching the model name, not a separate setting.
|
||||
|
||||
@@ -112,7 +118,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning
|
||||
|
||||
**The pull cascade is `pull → embed → score → extract → score → summarise`.** Every stage is always in the chain and a stage that is off finds nothing, as in newsdesk. The free stages run first: embed and score cost no tokens, and their score is what bounds the two model stages. Extract reads only what ranks, then score runs again so the gates read what it filled. A stage is addressed by its position in the tuple, since `score` is in it twice. Extract reads `on_ingest` only as a stage of a pull, so with `auto` and no model reachable a pull passes it quietly.
|
||||
|
||||
**Extract is bounded like summarise: the top roles of each agent, by that agent's own `limits`.** Extract has short answers but would otherwise touch every open role, and the seed boards alone produced 29,000; at Haiku's rates that is tens of dollars for a first pass over roles nobody will look at. Bounded to the ranking, a pull reads a few hundred. `extract_top` is the backstop (default 500) and `extract_min` the real stop (default 0.3, loose): a fixed count stays flat whatever the boards produce, where a percentage would not. The bounds are per agent rather than config, because one agent may be a wide monitoring search that should read a thousand and another a narrow hunt that wants fifty; one global number fits neither. A batch dealt named uids finishes them whatever their rank, so a running batch is not cut short by a rescore.
|
||||
**Extract is bounded like summarise: the top roles of each agent, by that agent's own `limits`.** Extract has short answers but would otherwise touch every open role, and the seed boards alone produced 29,000; at Haiku's rates that is tens of dollars for a first pass over roles nobody will look at. Bounded to the ranking, a pull reads a few hundred. `extract_top` is the backstop (default 500) and `extract_min` the real stop (default 0.55, about 300 roles for the starter agent): a fixed count stays flat whatever the boards produce, where a percentage would not. The bounds are per agent rather than config, because one agent may be a wide monitoring search that should read a thousand and another a narrow hunt that wants fifty; one global number fits neither. A batch dealt named uids finishes them whatever their rank, so a running batch is not cut short by a rescore.
|
||||
|
||||
**`extract_batch` and `summarise_batch` are `isolate=True`, with a timeout.** `context.check()` only runs between the calls `modeltask.one` makes, not inside one, so a call that hangs on either backend was otherwise unkillable short of restarting the server; a process boundary is what lets a Kill, or the timeout itself, actually end it. One call is already bounded at `agent.timeout`, so the deadline is a few calls' worth (30 minutes for extract, 15 for summarise): a batch still running after that is stuck, not slow. The cost: an isolated job's context has no working `stop`, `task` or `fan_out` (see `pipeline/child.py`'s `context_for`), so a refusal or an account-level failure inside a batch can no longer stop the rest of that task's queued siblings — they run on to the same failure instead of being spared it.
|
||||
|
||||
@@ -142,7 +148,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning
|
||||
|
||||
**Affinity and votes are per agent.** A company hidden for one hunt may be a favourite for another, and a vote says what one agent should learn from, so both carry the agent id, and the Roles tab and the company list ask for one agent's. Without an agent id the API uses the default agent, the enabled one with the lowest id, so the CLI and older callers keep working.
|
||||
|
||||
**`taste` is the liked-minus-passed centroid, as in newsdesk.** An agent's votes become one vector, the mean of the liked roles' embeddings minus the mean of the passed ones', and a role's taste term is its cosine to that vector, weighted beside fit, recency and affinity. It is the one term an agent learns rather than is told, and it costs no model call.
|
||||
**`taste` is the liked-minus-passed centroid, as in newsdesk.** An agent's votes become one vector, the mean of the liked roles' embeddings minus the mean of the passed ones', and a role's taste term is its cosine to that vector, calibrated like fit, weighted beside fit, recency and affinity. It is the one term an agent learns rather than is told, and it costs no model call.
|
||||
|
||||
**Roles are a feed with a reader, not links out.** `GET /roles/{uid}` renders the feed with its reader open on that role instead of sending the reader to the board, because 29,194 of 29,196 open board roles already have a `description` averaging 6,300 characters, and a search role's short teaser is worth reading in place before deciding whether to fetch the rest.
|
||||
|
||||
@@ -160,7 +166,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning
|
||||
|
||||
**A search role's posting text is fetched on demand, not in the pull.** `POST /api/roles/{uid}/text` scrapes `original_url` only when the read page asks for it, because following Jobindex's redirect for every role during a pull would count as a click in Jobindex's own statistics; one click for a posting somebody actually opened is fair.
|
||||
|
||||
**Summarising is bounded to the top roles of each agent, by that agent's own `limits`, and off by default.** The pull's `summarise` stage only considers roles with no summary yet that are in the top `summarise_top` (default 30) of an enabled agent's scores and at `summarise_min` (default 0.5, stricter than extract's since a summary is the long answer), so the cost stays flat whatever the backlog of open roles is. The bounds live on the agent, beside its gates and weights, for the same reason as extract's: a wide monitoring search and a narrow hunt want different depths, and the agent is where the difference is already expressed. `summarise.enabled: false` matches `extract`'s own default; the read page's Summarise button asks for one role's summary either way.
|
||||
**Summarising is bounded to the top roles of each agent, by that agent's own `limits`, and off by default.** The pull's `summarise` stage only considers roles with no summary yet that are in the top `summarise_top` (default 30) of an enabled agent's scores and at `summarise_min` (default 0.65, about 35 roles for the starter agent, stricter than extract's since a summary is the long answer), so the cost stays flat whatever the backlog of open roles is. The bounds live on the agent, beside its gates and weights, for the same reason as extract's: a wide monitoring search and a narrow hunt want different depths, and the agent is where the difference is already expressed. `summarise.enabled: false` matches `extract`'s own default; the read page's Summarise button asks for one role's summary either way.
|
||||
|
||||
**The schedules element's narrow layout is fixed upstream, in `../boilerplate`.** `<desk-schedules>` is a shadow root the app cannot lay out from outside, and `pipeline/` is a downstream copy refreshed with `git archive`, so a fix landing only in roledesk would be reverted by the next refresh.
|
||||
|
||||
|
||||
+10
-6
@@ -62,7 +62,7 @@ def ranked(store: Store, *items: Item) -> None:
|
||||
"""Score the roles for one enabled agent, best first, so they are in the
|
||||
top extract reads. Every role given must already be stored."""
|
||||
agents = store.agents()
|
||||
agent_id = agents[0].id if agents else store.save_agent("Backend").id
|
||||
agent_id = agents[0].id if agents else store.save_agent("Backend", "python backend").id
|
||||
store.replace_scores(agent_id, [(item.uid, 1.0 - n / 10, "")
|
||||
for n, item in enumerate(items)])
|
||||
|
||||
@@ -171,8 +171,9 @@ def test_only_the_best_of_each_agent_above_the_floor_is_read(
|
||||
uid finishes it."""
|
||||
for sid in ("best", "second", "low", "unscored", "gated"):
|
||||
store.upsert_role(posting(sid))
|
||||
one = store.save_agent("One", limits=Limits(extract_top=1, extract_min=0.5))
|
||||
two = store.save_agent("Two", enabled=False)
|
||||
one = store.save_agent("One", "python backend",
|
||||
limits=Limits(extract_top=1, extract_min=0.5))
|
||||
two = store.save_agent("Two", "python backend", enabled=False)
|
||||
store.replace_scores(one.id, [(posting("best").uid, 0.9, ""),
|
||||
(posting("second").uid, 0.8, ""),
|
||||
(posting("low").uid, 0.2, ""),
|
||||
@@ -182,11 +183,14 @@ def test_only_the_best_of_each_agent_above_the_floor_is_read(
|
||||
model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles])
|
||||
|
||||
assert store.extract_pending() == 1
|
||||
store.save_agent("One", agent_id=one.id, limits=Limits(extract_top=5, extract_min=0.5))
|
||||
store.save_agent("One", "python backend", agent_id=one.id,
|
||||
limits=Limits(extract_top=5, extract_min=0.5))
|
||||
assert store.extract_pending() == 2, "the floor holds the low one back"
|
||||
store.save_agent("One", agent_id=one.id, limits=Limits(extract_top=5, extract_min=0))
|
||||
store.save_agent("One", "python backend", agent_id=one.id,
|
||||
limits=Limits(extract_top=5, extract_min=0))
|
||||
assert store.extract_pending() == 3, "the disabled agent's pick never counts"
|
||||
store.save_agent("One", agent_id=one.id, limits=Limits(extract_top=1, extract_min=0.5))
|
||||
store.save_agent("One", "python backend", agent_id=one.id,
|
||||
limits=Limits(extract_top=1, extract_min=0.5))
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
|
||||
assert [[r["uid"] for r in batch] for batch in model.asked] == [[posting("best").uid]]
|
||||
|
||||
+170
-17
@@ -3,6 +3,7 @@ by a deterministic fake. Nothing here downloads a model or reaches a board."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Iterator, Sequence
|
||||
from datetime import UTC, datetime, timedelta
|
||||
@@ -19,13 +20,17 @@ if str(ROOT) not in sys.path:
|
||||
from desk.models import Item # noqa: E402
|
||||
from pipeline import FINAL, JOBS, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, Tally # noqa: E402
|
||||
from plugins.ats import ATSSource # noqa: E402
|
||||
from app.config import AgentSeed, Config # noqa: E402
|
||||
from app.config import CALIBRATION, AgentSeed, Config, ConfigError # noqa: E402
|
||||
from app.jobs import cascade, embed, score # noqa: E402
|
||||
from app.models import Candidate, Gate, Weights # noqa: E402
|
||||
from app.models import Candidate, Gate, Limits, Weights # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
|
||||
NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
|
||||
|
||||
#: The shipped constants, which the fake is calibrated with so a test reads
|
||||
#: the rule production runs; `uncalibrated` is the min-max fallback.
|
||||
E5 = CALIBRATION["intfloat/multilingual-e5-small"]
|
||||
|
||||
#: The fake's whole vocabulary: one dimension per word, plus a constant one so
|
||||
#: no text is the zero vector.
|
||||
WORDS = ("backend", "python", "frontend", "react", "data")
|
||||
@@ -76,6 +81,12 @@ def store(tmp_path: Path, clock: Clock) -> Iterator[Store]:
|
||||
|
||||
@pytest.fixture
|
||||
def config(tmp_path: Path) -> Config:
|
||||
return Config(database=str(tmp_path / "roledesk.db"),
|
||||
embed={"model": "fake", "calibration": {"fake": E5.model_dump()}})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def uncalibrated(tmp_path: Path) -> Config:
|
||||
return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
|
||||
|
||||
|
||||
@@ -94,9 +105,17 @@ def run(config: Config, store: Store, job: Any) -> dict[str, Any]:
|
||||
notices: list[Any] = []
|
||||
result = dict(job(config, store, context=RunContext(on_notice=notices.append)))
|
||||
result["notices"] = [n.code for n in notices]
|
||||
result["notice_texts"] = [n.text for n in notices]
|
||||
return result
|
||||
|
||||
|
||||
def level(text: str, intent: str, band: tuple[float, float] = E5.fit) -> float:
|
||||
"""The calibrated fit the fake gives a role whose `document` is `text`."""
|
||||
role_vec, intent_vec = FakeEmbedder().encode([text, intent])
|
||||
lo, hi = band
|
||||
return min(max((score.cosine(role_vec, intent_vec) - lo) / (hi - lo), 0.0), 1.0)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ embed
|
||||
|
||||
def test_embed_writes_a_vector_per_open_role_once(
|
||||
@@ -431,7 +450,8 @@ def test_two_agents_rank_the_same_roles_the_other_way_round(
|
||||
result = run(config, store, score.run_score)
|
||||
|
||||
assert result["agents"] == 2
|
||||
# Fit alone, scaled per agent: each agent's closer role is 1, the other 0.
|
||||
# Fit alone, calibrated: each agent's own role is at a cosine past `hi`,
|
||||
# so 1, the other's near 0, below `lo`, so 0.
|
||||
assert store.scores(backend.id)[back][0] == 1.0
|
||||
assert store.scores(backend.id)[front][0] == 0.0
|
||||
assert store.scores(frontend.id)[front][0] == 1.0
|
||||
@@ -481,6 +501,7 @@ def test_taste_ranks_a_role_near_the_liked_one_above_one_near_the_passed(
|
||||
|
||||
scores = store.scores(agent.id)
|
||||
assert scores[near_liked][0] > scores[near_passed][0]
|
||||
# Cosines of about 0.7 and -0.7 to the direction, past both ends of the band.
|
||||
assert scores[liked][0] == 1.0 and scores[passed][0] == 0.0
|
||||
assert all(reason == "" for _, reason in scores.values())
|
||||
|
||||
@@ -513,35 +534,167 @@ def test_a_closed_liked_role_still_shapes_the_taste(
|
||||
|
||||
scores = store.scores(agent.id)
|
||||
assert liked not in scores
|
||||
assert scores[near][0] == 1.0 and scores[far][0] == 0.0
|
||||
# Near sits on the liked direction, past `hi`. Far is orthogonal to it,
|
||||
# a cosine of about 0 and so a middling level, not the 0 min-max gave it.
|
||||
assert scores[near][0] == 1.0
|
||||
assert scores[far][0] == pytest.approx(
|
||||
level("Web Engineer\nReact frontend", "Backend Engineer\nPython backend", E5.taste),
|
||||
abs=1e-5)
|
||||
assert 0 < scores[far][0] < 1
|
||||
|
||||
|
||||
# ------------------------------------------------------------ fit scaling
|
||||
|
||||
def test_scale_maps_the_closest_to_one_and_the_farthest_to_zero() -> None:
|
||||
# The spread the last real run showed under `(cos + 1) / 2`, on
|
||||
# multilingual-e5-small.
|
||||
assert score.scale({"a": 0.87, "b": 0.95, "c": 0.91}) == pytest.approx(
|
||||
def test_scale_maps_a_cosine_to_its_calibrated_level_and_clips_both_ends() -> None:
|
||||
# multilingual-e5-small's fit band: lo 0.75 is 0, hi 0.88 is 1.
|
||||
assert E5.fit == (0.75, 0.88) and E5.taste == (-0.14, 0.19)
|
||||
assert score.scale({"lo": 0.75, "mid": 0.815, "hi": 0.88, "p50": 0.7989},
|
||||
E5.fit) == pytest.approx(
|
||||
{"lo": 0.0, "mid": 0.5, "hi": 1.0, "p50": 0.0489 / 0.13})
|
||||
assert score.scale({"under": 0.6, "negative": -0.2, "over": 0.9031, "one": 1.0},
|
||||
E5.fit) == {"under": 0.0, "negative": 0.0, "over": 1.0, "one": 1.0}
|
||||
assert score.scale({"none": 0.0, "liked": 0.25}, E5.taste) == pytest.approx(
|
||||
{"none": 0.14 / 0.33, "liked": 1.0})
|
||||
# A lone role, or roles that are all equally close, get their level.
|
||||
assert score.scale({"only": 0.8}, E5.fit) == pytest.approx({"only": 0.05 / 0.13})
|
||||
assert score.scale({}, E5.fit) == {}
|
||||
|
||||
|
||||
def test_scale_without_a_band_falls_back_to_min_max() -> None:
|
||||
assert score.scale({"a": 0.87, "b": 0.95, "c": 0.91}, None) == pytest.approx(
|
||||
{"a": 0.0, "b": 1.0, "c": 0.5})
|
||||
assert score.scale({"only": 0.3}) == {"only": 1.0}
|
||||
assert score.scale({"a": 0.4, "b": 0.4}) == {"a": 1.0, "b": 1.0}
|
||||
assert score.scale({}) == {}
|
||||
assert score.scale({"only": 0.3}, None) == {"only": 1.0}
|
||||
assert score.scale({"a": 0.4, "b": 0.4}, None) == {"a": 1.0, "b": 1.0}
|
||||
assert score.scale({}, None) == {}
|
||||
|
||||
|
||||
def test_a_single_candidate_gets_full_fit(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
def test_a_single_candidate_gets_its_level_not_full_fit(
|
||||
store: Store, config: Config, uncalibrated: Config, fake: FakeEmbedder) -> None:
|
||||
"""Min-max gave the best of a poor set 1; a level gives it what it is."""
|
||||
uid = role(store, "1", "Frontend Engineer", body="React apps")
|
||||
agent = store.save_agent("Backend", "python backend developer",
|
||||
weights=Weights(fit=1, recency=0, affinity=0))
|
||||
|
||||
run(config, store, embed.run_embed)
|
||||
run(config, store, score.run_score)
|
||||
assert store.scores(agent.id)[uid] == (0.0, "")
|
||||
|
||||
run(uncalibrated, store, score.run_score)
|
||||
assert store.scores(agent.id)[uid] == (1.0, "")
|
||||
|
||||
|
||||
def test_a_gated_role_does_not_stretch_the_fit_range(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
def test_the_same_role_gets_the_same_fit_whatever_else_is_open(
|
||||
store: Store, config: Config, uncalibrated: Config, fake: FakeEmbedder) -> None:
|
||||
"""The point of a level: two runs over different candidate sets give the
|
||||
role the same fit, where min-max gave it 1 beside a worse role and 0
|
||||
beside a better one."""
|
||||
mid = role(store, "1", "Backend Engineer", body="Python backend data")
|
||||
worse = role(store, "2", "Frontend Engineer", body="React apps")
|
||||
agent = store.save_agent("Backend", "python backend",
|
||||
weights=Weights(fit=1, recency=0, affinity=0))
|
||||
expected = level("Backend Engineer\nPython backend data", "python backend")
|
||||
assert 0.5 < expected < 1
|
||||
|
||||
run(config, store, embed.run_embed)
|
||||
run(config, store, score.run_score)
|
||||
first = store.scores(agent.id)[mid][0]
|
||||
run(uncalibrated, store, score.run_score)
|
||||
assert store.scores(agent.id)[mid][0] == 1.0
|
||||
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (NOW.isoformat(), worse))
|
||||
role(store, "3", "Backend Engineer", body="Python")
|
||||
run(config, store, embed.run_embed)
|
||||
run(config, store, score.run_score)
|
||||
second = store.scores(agent.id)[mid][0]
|
||||
run(uncalibrated, store, score.run_score)
|
||||
assert store.scores(agent.id)[mid][0] == 0.0
|
||||
|
||||
assert first == second == pytest.approx(expected, abs=1e-5)
|
||||
|
||||
|
||||
def test_an_uncalibrated_model_falls_back_to_min_max_and_warns_once_a_run(
|
||||
store: Store, uncalibrated: Config, fake: FakeEmbedder,
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
close = role(store, "1", "Backend Engineer", body="Python backend data")
|
||||
far = role(store, "2", "Data Engineer", body="Data")
|
||||
only_fit = Weights(fit=1, recency=0, affinity=0)
|
||||
one = store.save_agent("One", "python backend", weights=only_fit)
|
||||
two = store.save_agent("Two", "python backend", weights=only_fit)
|
||||
run(uncalibrated, store, embed.run_embed)
|
||||
|
||||
with caplog.at_level("WARNING", logger="app.jobs.score"):
|
||||
run(uncalibrated, store, score.run_score)
|
||||
|
||||
warnings = [r.getMessage() for r in caplog.records if r.name == "app.jobs.score"]
|
||||
assert len(warnings) == 1 and warnings[0].startswith("no calibration for fake")
|
||||
for agent in (one, two):
|
||||
assert store.scores(agent.id)[close][0] == 1.0
|
||||
assert store.scores(agent.id)[far][0] == 0.0
|
||||
|
||||
|
||||
def test_the_calibration_is_the_built_in_one_unless_the_config_replaces_it() -> None:
|
||||
assert Config().embed.calibrated() == E5
|
||||
assert Config(embed={"model": "fake"}).embed.calibrated() is None
|
||||
mine = {"fit": [0.7, 0.9], "taste": [-0.2, 0.2]}
|
||||
replaced = Config(embed={"calibration": {"intfloat/multilingual-e5-small": mine}})
|
||||
assert replaced.embed.calibrated() is not None
|
||||
assert replaced.embed.calibrated().fit == (0.7, 0.9) # type: ignore[union-attr]
|
||||
# Keyed by model: another model's entry does not apply.
|
||||
other = Config(embed={"model": "fake", "calibration": {"other": mine}})
|
||||
assert other.embed.calibrated() is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entry, says", [
|
||||
({"fit": [0.88, 0.75], "taste": [-0.14, 0.19]}, "embed.calibration.fake.fit"),
|
||||
({"fit": [0.75, 0.75], "taste": [-0.14, 0.19]}, "lo < hi"),
|
||||
({"fit": [0.75, 0.88], "taste": [-2, 0.19]}, "embed.calibration.fake.taste"),
|
||||
({"fit": [0.75, 0.88]}, "embed.calibration.fake.taste"),
|
||||
({"fit": [0.75, 0.88], "taste": [-0.14, 0.19], "spread": 1}, "unknown setting"),
|
||||
])
|
||||
def test_a_bad_calibration_is_a_config_error(
|
||||
tmp_path: Path, entry: dict[str, Any], says: str) -> None:
|
||||
import yaml
|
||||
from app import config as config_mod
|
||||
|
||||
path = tmp_path / "config.yaml"
|
||||
path.write_text(yaml.safe_dump({"embed": {"model": "fake",
|
||||
"calibration": {"fake": entry}}}))
|
||||
with pytest.raises(ConfigError, match=re.escape(says)):
|
||||
config_mod.load(path)
|
||||
|
||||
|
||||
def test_an_agent_without_an_intent_feeds_neither_model_stage_and_is_named(
|
||||
store: Store, config: Config, clock: Clock, fake: FakeEmbedder) -> None:
|
||||
"""Its scores have no fit, so they rank fresh roles, not wanted ones: the
|
||||
newer role tops it, and extract and summarise must not read it on that."""
|
||||
clock.at = NOW - timedelta(days=1)
|
||||
wanted = role(store, "1", "Backend Engineer", body="Python backend")
|
||||
clock.at = NOW
|
||||
fresh = role(store, "2", "Frontend Engineer", body="React apps")
|
||||
top_one = Limits(extract_top=1, extract_min=0, summarise_top=1, summarise_min=0)
|
||||
store.save_agent("Backend", "python backend", limits=top_one)
|
||||
blind = store.save_agent("Fresh", "", limits=top_one)
|
||||
store.save_agent("Also fresh", " \n", limits=top_one)
|
||||
store.save_agent("Off", "", enabled=False)
|
||||
|
||||
run(config, store, embed.run_embed)
|
||||
result = run(config, store, score.run_score)
|
||||
|
||||
assert result["notices"] == ["score.no_intent"]
|
||||
assert result["notice_texts"][0].endswith(": Also fresh, Fresh.")
|
||||
ranked = sorted(store.scores(blind.id).items(), key=lambda kv: -kv[1][0])
|
||||
assert ranked[0][0] == fresh, "the intent-less agent would pick the newer role"
|
||||
assert [r.uid for r in store.extract_waiting(10)] == [wanted]
|
||||
assert [r.uid for r in store.summarise_waiting(10)] == [wanted]
|
||||
assert store.extract_pending() == store.summarise_pending() == 1
|
||||
|
||||
|
||||
def test_a_gated_role_does_not_stretch_the_fit_range_under_the_fallback(
|
||||
store: Store, uncalibrated: Config, fake: FakeEmbedder) -> None:
|
||||
"""Calibrated, no other role moves a role's fit. Min-max, the fallback,
|
||||
must still leave a gated role out of the range."""
|
||||
config = uncalibrated
|
||||
close = role(store, "1", "Backend Engineer", body="Python backend", place="Berlin")
|
||||
far = role(store, "2", "Data Engineer", body="Data", place="Berlin")
|
||||
# Farther from the intent than either, but gated out by place.
|
||||
@@ -562,8 +715,8 @@ def test_a_small_prefer_gate_no_longer_outweighs_fit(
|
||||
"""The flaw the last real run showed: real cosines sit close together, so
|
||||
under `(cos + 1) / 2` a prefer gate decided the order over fit. Here the
|
||||
cosines are about 0.95 and 0.87: the old fit terms differ by 0.04, which
|
||||
at weight 0.6 is less than the gate's 0.05, so the remote role won. Scaled,
|
||||
the closer role wins."""
|
||||
at weight 0.6 is less than the gate's 0.05, so the remote role won.
|
||||
Calibrated, they are 1 and about 0.89, and the closer role wins."""
|
||||
close = role(store, "1", "Backend Engineer", body="Python backend", remote="onsite")
|
||||
far = role(store, "2", "Backend Engineer", body="Python backend data", remote="remote")
|
||||
agent = store.save_agent("Backend", "python backend",
|
||||
|
||||
+11
-6
@@ -1033,24 +1033,29 @@ def test_summarise_waits_on_the_top_of_each_enabled_agent_by_its_own_limits(
|
||||
a, b, c, d = (posting(s).uid for s in "abcd")
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (store.now(), d))
|
||||
one = store.save_agent("One", limits=Limits(summarise_top=1, summarise_min=0))
|
||||
two = store.save_agent("Two", limits=Limits(summarise_top=1, summarise_min=0))
|
||||
one = store.save_agent("One", "python backend",
|
||||
limits=Limits(summarise_top=1, summarise_min=0))
|
||||
two = store.save_agent("Two", "python backend",
|
||||
limits=Limits(summarise_top=1, summarise_min=0))
|
||||
store.replace_scores(one.id, [(d, 0.99, ""), (a, 0.9, ""), (b, 0.5, "")])
|
||||
store.replace_scores(two.id, [(a, 0.95, "remote"), (c, 0.7, ""), (b, 0.8, "")])
|
||||
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b}
|
||||
|
||||
# Two goes deeper: its second, c, joins; One's second, b, was in already.
|
||||
store.save_agent("Two", agent_id=two.id, limits=Limits(summarise_top=2, summarise_min=0))
|
||||
store.save_agent("Two", "python backend", agent_id=two.id,
|
||||
limits=Limits(summarise_top=2, summarise_min=0))
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b, c}
|
||||
assert store.summarise_pending() == 3
|
||||
assert [r.uid for r in store.summarise_waiting(10, uids=[d])] == [d]
|
||||
# The floor is Two's own: c at 0.7 drops, One's b at 0.5 stays.
|
||||
store.save_agent("Two", agent_id=two.id,
|
||||
store.save_agent("Two", "python backend", agent_id=two.id,
|
||||
limits=Limits(summarise_top=2, summarise_min=0.75))
|
||||
store.save_agent("One", agent_id=one.id, limits=Limits(summarise_top=2, summarise_min=0))
|
||||
store.save_agent("One", "python backend", agent_id=one.id,
|
||||
limits=Limits(summarise_top=2, summarise_min=0))
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b}
|
||||
# '{}' is the defaults, 30 at 0.5: One's b at 0.5 stays, Two's c at 0.7 is back.
|
||||
# '{}' is the defaults, 30 at 0.65: One's b at 0.5 drops, but Two's b at
|
||||
# 0.8 and c at 0.7 clear it.
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE agents SET limits = '{}'")
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b, c}
|
||||
|
||||
+3
-3
@@ -659,7 +659,7 @@ def test_the_agents_page_lists_and_edits_search_agents(client: TestClient) -> No
|
||||
for limit in ("a-extract-top", "a-extract-min", "a-summarise-top", "a-summarise-min"):
|
||||
assert f'id="{limit}"' in template
|
||||
assert 'id="a-extract-top" min="0" step="10" value="500"' in page
|
||||
assert 'id="a-summarise-min" min="0" max="1" step="0.05" value="0.5"' in page
|
||||
assert 'id="a-summarise-min" min="0" max="1" step="0.05" value="0.65"' in page
|
||||
assert 'id="a-new"' in page and 'id="a-delete"' in page
|
||||
assert "Save and score" in page
|
||||
|
||||
@@ -823,8 +823,8 @@ def test_an_agents_limits_are_saved_listed_and_checked(client: TestClient) -> No
|
||||
"""How deep the model stages read is the agent's own; a score outside
|
||||
0..1 is refused."""
|
||||
default = client.post("/api/agents", json={"name": "Wide"}).json()
|
||||
assert default["limits"] == {"extract_top": 500, "extract_min": 0.3,
|
||||
"summarise_top": 30, "summarise_min": 0.5}
|
||||
assert default["limits"] == {"extract_top": 500, "extract_min": 0.55,
|
||||
"summarise_top": 30, "summarise_min": 0.65}
|
||||
narrow = {"extract_top": 50, "extract_min": 0.6, "summarise_top": 5, "summarise_min": 0.7}
|
||||
saved = client.post("/api/agents", json={"name": "Narrow", "limits": narrow})
|
||||
assert saved.status_code == 200 and saved.json()["limits"] == narrow
|
||||
|
||||
Reference in new issue
Block a user