Files
jobboard/app/jobs/score.py
T
WolfandClaude Opus b021838959 Calibrate fit and taste per embedding model instead of min-max per run
scale() maps a cosine to clip((cos - lo) / (hi - lo), 0, 1) with constants
from CALIBRATION in app/config.py, keyed by model and replaceable under
embed.calibration. multilingual-e5-small: fit 0.75..0.88, taste -0.14..0.19
(taste from synthetic votes). An uncalibrated model falls back to min-max
and logs a warning once per run.

The extract and summarise bars default to 0.55 and 0.65, about 300 and 35
roles for the starter agent on the measured data; stored limits are kept.
An agent without an intent is left out of the model-stage scope, and each
score run names such agents in one score.no_intent notice.

Co-Authored-By: Claude Opus <noreply@anthropic.com>
2026-09-29 14:59:34 +02:00

347 lines
15 KiB
Python

"""`score`: every open role, scored for every enabled search agent.
A role is scored per agent, `(uid, agent) -> score`, because a role is only
good for a given search. For each agent:
require gates a role that fails one keeps its row, with score 0 and the
reason ("remote not remote")
any gates one group: a role that matches none of them, and has the
column filled for at least one, is gated ("none of
country DK / remote remote")
prefer gates `weight` when the role matches, 0 when it does not
A gate rejects on evidence, never on absence: a role whose column is empty
is not gated by it. `extract` is what fills an empty column, and it reads
only the roles that rank, so gating on an empty column would keep those
roles out of extract's reach for good.
fit cos(role vector, intent vector) as a level: the embedding
model's calibrated `lo` is 0 and `hi` 1, clipped outside
(see `scale`), so a role's fit does not depend on what
else is open
recency 0.5 ** (days since posted_at / 14)
affinity (the agent's affinity for the company + 1) / 3
taste cos(role vector, liked direction), scaled like fit with the
model's taste calibration, where the direction is
mean(liked vectors) - mean(passed vectors)
over this agent's votes; with only likes it is the liked mean,
with only passes the negated passed mean; no votes, no term
Every term is 0..1. The score is the weighted sum over the sum of the weights
(the agent's four and its prefer gates'), so it stays 0..1. A favourite
company (`affinity = 2`) skips the gates and is floored at 0.9; a hidden one
(`-1`) is not scored at all. A role with no vector is scored without `fit`
and `taste`, unless `fit` is all there is, when it is gated as "no vector".
An agent with no votes scores every role without `taste`; the direction is
newsdesk's `centroid(liked) - centroid(disliked)`. An agent with no intent
has no `fit` either: its scores are not a match, so it is left out of what
extract and summarise read (`Store._top_scope`), and a run names such agents
in one notice.
Recomputed in full on every run, and each agent's rows are replaced in one
transaction: nothing is incremental, so nothing can drift.
"""
from __future__ import annotations
import hashlib
import logging
import math
from collections.abc import Sequence
from datetime import datetime
from typing import Any
import numpy as np
from pipeline import NULL, Notice, RunContext, job
from app.config import Calibration, Config
from app.jobs import embed
from app.models import Agent, Candidate, Gate, Weights
from app.storage import Store
log = logging.getLogger(__name__)
HALF_LIFE_DAYS = 14.0
#: Where a favourite company's roles start, whatever else they score.
FAVOURITE_FLOOR = 0.9
#: The `state` key that records the starter agents were imported.
SEEDED = "agents:seeded"
# ------------------------------------------------------------------- terms
def recency(posted_at: str, now: datetime) -> float:
"""1.0 for a role posted now, halving every `HALF_LIFE_DAYS`. `posted_at`
is `app.models.posted_at`: the earlier of when the app first saw the role
and, for a trusted host, when the board says it was posted — so a
months-old backlog a newly added board brings in does not start at 1.0."""
try:
seen = datetime.fromisoformat(posted_at)
except ValueError:
return 0.0
days = max((now - seen).total_seconds(), 0.0) / 86400
return math.pow(0.5, days / HALF_LIFE_DAYS)
def affinity_term(affinity: int) -> float:
"""-1..2 to 0..1. -1 never gets here; neutral is 1/3."""
return (affinity + 1) / 3
def cosine(role: np.ndarray, intent: np.ndarray) -> float:
"""Raw cosine, -1..1; 0 when either vector is zero."""
norms = float(np.linalg.norm(role) * np.linalg.norm(intent))
if norms == 0:
return 0.0
return float(np.dot(role, intent)) / norms
def taste_direction(vectors: dict[str, np.ndarray],
votes: dict[str, int]) -> np.ndarray | None:
"""`mean(liked) - mean(passed)` over the voted roles that have a vector.
A side with no votes is zero, so likes alone point at the liked mean and
passes alone away from the passed mean. None when neither side has a
vector: no votes, no direction. A neutral vote says nothing."""
liked = [vectors[uid] for uid, vote in votes.items() if vote == 1 and uid in vectors]
passed = [vectors[uid] for uid, vote in votes.items() if vote == -1 and uid in vectors]
if not liked and not passed:
return None
direction = np.zeros_like(liked[0] if liked else passed[0], dtype=np.float64)
if liked:
direction += np.mean(liked, axis=0)
if passed:
direction -= np.mean(passed, axis=0)
return direction
def scale(cosines: dict[str, float],
band: tuple[float, float] | None) -> dict[str, float]:
"""Cosines to 0..1 levels: `clip((cos - lo) / (hi - lo), 0, 1)` with the
embedding model's calibrated `band` (`app.config.CALIBRATION`).
Raw cosines on multilingual-e5-small sit between about 0.75 and 0.87 for
real postings against a real intent, so unscaled a prefer gate weighted
0.2 moved a score more than the whole spread of fit. The band stretches
that spread to 0..1 with fixed constants, so a role's level is the same
in every run and a `limits` bar is an absolute threshold.
Without a band, for a model nobody calibrated, it falls back to min-max
across the run: the closest role is 1 and the farthest 0, which ranks
rather than states a level. A single role, or roles that are all equally
close, then get 1: there is nothing to tell them apart by.
"""
if not cosines:
return {}
if band is not None:
lo, hi = band
return {uid: min(max((value - lo) / (hi - lo), 0.0), 1.0)
for uid, value in cosines.items()}
low, high = min(cosines.values()), max(cosines.values())
if high - low < 1e-9:
return dict.fromkeys(cosines, 1.0)
return {uid: (value - low) / (high - low) for uid, value in cosines.items()}
def _wanted(gate: Gate) -> str:
if gate.field == "salary_min":
return f"salary_min {min(gate.values, key=lambda v: float(v))}"
return f"{gate.field} {' / '.join(str(v) for v in gate.values)}"
def _reason(gate: Gate) -> str:
if gate.field == "salary_min":
return f"salary_min under {min(gate.values, key=lambda v: float(v))}"
return f"{gate.field} not {' / '.join(str(v) for v in gate.values)}"
def failed_gates(candidate: Candidate, gates: Sequence[Gate]) -> list[str]:
"""Why this role is gated, as reasons: every require gate it fails, and
the `any` group when it matches none of them. A gate with an empty column
(state -1) neither passes nor fails, so the group fails only when some
member had evidence. None for a favourite, which skips the gates."""
if candidate.affinity == 2:
return []
states = list(zip(gates, candidate.gates, strict=True))
reasons = [_reason(g) for g, state in states if g.mode == "require" and state == 0]
group = [(g, state) for g, state in states if g.mode == "any"]
if group and not any(state == 1 for _, state in group) \
and any(state == 0 for _, state in group):
reasons.append("none of " + ", ".join(_wanted(g) for g, _ in group))
return reasons
def score_one(candidate: Candidate, gates: Sequence[Gate], weights: Weights,
fit: float | None, now: datetime, *, taste: float | None = None,
missing: str = "no vector") -> tuple[float, str]:
"""One role for one agent: `(score, gated_reason)`. `fit` and `taste` are
already scaled to 0..1 (see `scale`), or None when there is nothing to
measure: no role vector, no intent vector, no votes. A None term leaves
the denominator; `missing` says why fit is out."""
favourite = candidate.affinity == 2
if failed := failed_gates(candidate, gates):
return 0.0, ", ".join(failed)
terms = [(weights.recency, recency(candidate.posted_at, now)),
(weights.affinity, affinity_term(candidate.affinity))]
terms += [(g.weight, 1.0 if state == 1 else 0.0)
for g, state in zip(gates, candidate.gates, strict=True) if g.mode == "prefer"]
if fit is not None:
terms.append((weights.fit, fit))
if taste is not None:
terms.append((weights.taste, taste))
total = sum(w for w, _ in terms)
if not total:
if favourite:
return FAVOURITE_FLOOR, ""
# Fit was the only term with any weight, and there is no fit.
return 0.0, missing if fit is None and weights.fit else "no weights"
score = sum(w * t for w, t in terms) / total
return (max(score, FAVOURITE_FLOOR) if favourite else score), ""
# ------------------------------------------------------------------ vectors
def intent_key(model: str, intent: str) -> str:
return hashlib.sha1(f"{model}\n{intent}".encode()).hexdigest()
class _Encoder:
"""The embedder, loaded on first use only: an agent whose intent vector
is cached costs no model load at all."""
def __init__(self, model: str, context: RunContext) -> None:
self.model = model
self.context = context
self._embedder: Any = None
self.failed = False
def encode(self, text: str) -> np.ndarray | None:
if self.failed:
return None
if self._embedder is None:
try:
self._embedder = embed.load_embedder(self.model)
except ImportError as exc:
self.failed = True
self.context.notice(embed.unavailable(exc))
return None
return np.asarray(self._embedder.encode([text])[0])
def _intent(store: Store, agent: Agent, model: str,
encoder: _Encoder) -> np.ndarray | None:
if not agent.intent.strip():
return None
key = intent_key(model, agent.intent)
cached = store.intent_vector(agent.id, key)
if cached is not None:
return cached
vec = encoder.encode(embed.intent_text(model, agent.intent))
if vec is not None:
store.set_intent_vector(agent.id, key, vec)
return vec
# ------------------------------------------------------------------- the job
def no_intent(names: Sequence[str]) -> Notice:
"""The notice for enabled agents without an intent: their scores have no
fit, so `Store._top_scope` keeps them from feeding extract and summarise."""
return Notice(
code="score.no_intent", level="warning", kind="expected", key="score",
text=f"No intent, so ranked without fit and not read by extract or "
f"summarise: {', '.join(names)}.",
fix="Give the agent an intent under Settings, Search agents.",
where="app.jobs.score")
def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
intent: np.ndarray | None, now: datetime,
calibration: Calibration | None = None) -> dict[str, int]:
"""Two passes: the cosines of the roles this agent scores, to its intent
and to its taste direction, then every score with both scaled by the
model's `calibration`, or across the run without one. A gated role is
left out of the cosines, so under the fallback a role that fails a
require gate cannot stretch the range the ranked roles share. The voted
roles are scored like any other; a liked one sits near the direction it
helped make, which is what a like should do."""
candidates = store.score_candidates(agent.id, agent.gates)
direction = taste_direction(vectors, store.votes(agent.id))
cosines: dict[str, float] = {}
tastes: dict[str, float] = {}
for candidate in candidates:
role_vec = vectors.get(candidate.uid)
if role_vec is None or failed_gates(candidate, agent.gates):
continue
if intent is not None:
cosines[candidate.uid] = cosine(role_vec, intent)
if direction is not None:
tastes[candidate.uid] = cosine(role_vec, direction)
fits = scale(cosines, calibration.fit if calibration else None)
tastes = scale(tastes, calibration.taste if calibration else None)
rows: list[tuple[str, float, str]] = []
gated = 0
for candidate in candidates:
value, reason = score_one(
candidate, agent.gates, agent.weights, fits.get(candidate.uid), now,
taste=tastes.get(candidate.uid),
missing="no vector" if intent is not None else "no intent vector")
gated += bool(reason)
rows.append((candidate.uid, round(value, 6), reason))
store.replace_scores(agent.id, rows)
return {"scored": len(rows), "gated": gated}
@job("score", "Scoring roles for every agent", solo=True, resource="cpu")
def run_score(config: Config, store: Store, *,
context: RunContext = NULL) -> dict[str, Any]:
seed_agents(config, store)
agents = store.agents(enabled_only=True)
model = config.embed.model
vectors = store.vectors(model) if agents else {}
encoder = _Encoder(model, context)
now = store.clock()
calibration = config.embed.calibrated()
if agents and calibration is None:
log.warning("no calibration for %s: fit and taste are min-max scaled across "
"each agent's roles in this run, so they rank rather than state a "
"level; add it under embed.calibration", model)
if blind := [agent.name for agent in agents if not agent.intent.strip()]:
context.notice(no_intent(blind))
scored = gated = 0
for n, agent in enumerate(agents):
context.check()
context.progress(n, len(agents), f"scoring for {agent.name}, {n + 1} of {len(agents)}")
counts = score_agent(store, agent, vectors, _intent(store, agent, model, encoder), now,
calibration)
scored += counts["scored"]
gated += counts["gated"]
log.info("%-20s %6d scored, %6d gated", agent.name, counts["scored"], counts["gated"])
context.progress(len(agents), len(agents), "")
return {"agents": len(agents), "scored": scored, "gated": gated,
"vectors": len(vectors)}
# ------------------------------------------------------------------ seeding
def seed_agents(config: Config, store: Store) -> list[str]:
"""Import `config.yaml`'s `agents:` into an empty table, once. The state
marker is what makes it once: deleting every agent afterwards is a
choice, and a restart must not undo it."""
if store.state.get(SEEDED) or store.agents():
return []
added = []
for seed in config.agents:
try:
store.save_agent(seed.name, seed.intent, seed.gates, seed.weights,
seed.enabled, limits=seed.limits)
except ValueError as exc:
log.warning("agents: %s", exc)
continue
added.append(seed.name)
if added:
store.state[SEEDED] = store.now()
log.info("imported %d agents from %s", len(added), config.path or "the defaults")
return added