Files
jobboard/app/jobs/score.py
T
WolfandClaude Opus 5.5 8b246c3035 Scale fit across each agent's run, and lower the starter prefer weights
With fit as (cos + 1) / 2 the real run spanned about 0.55 to 0.79, so a
prefer gate at weight 0.2 moved a score more than the whole spread of fit.
fit is now the cosine min-max scaled across the roles an agent scores in
that run: the closest is 1, the farthest 0, a single role 1, and a role a
require gate rules out stays out of the range. The starter agents' prefer
weights drop to 0.05, enough to break a tie. A regression test pins the
case: two close cosines that the old mapping let a 0.05 gate reorder.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:51:08 +02:00

253 lines
9.7 KiB
Python

"""`score`: every open role, scored for every enabled search agent.
A role is scored per agent, `(uid, agent) -> score`, because a role is only
good for a given search. For each agent:
require gates a role that fails one keeps its row, with score 0 and the
reason ("place unknown", "remote not remote")
prefer gates `weight` when the role matches, 0 when it does not
fit cos(role vector, intent vector), min-max scaled across
the roles this agent scores in this run: the closest is 1,
the farthest 0, a lone role 1
recency 0.5 ** (days since first_seen / 14)
affinity (companies.affinity + 1) / 3
Every term is 0..1. The score is the weighted sum over the sum of the weights
(the agent's three and its prefer gates'), so it stays 0..1. A favourite
company (`affinity = 2`) skips the gates and is floored at 0.9; a muted one
(`-1`) is not scored at all. A role with no vector is scored without `fit`,
unless `fit` is all there is, when it is gated as "no vector".
Recomputed in full on every run, and each agent's rows are replaced in one
transaction: nothing is incremental, so nothing can drift.
"""
from __future__ import annotations
import hashlib
import logging
import math
from collections.abc import Sequence
from datetime import datetime
from typing import Any
import numpy as np
from pipeline import NULL, RunContext, job
from app.config import Config
from app.jobs import embed
from app.models import Agent, Candidate, Gate, Weights
from app.storage import Store
log = logging.getLogger(__name__)
HALF_LIFE_DAYS = 14.0
#: Where a favourite company's roles start, whatever else they score.
FAVOURITE_FLOOR = 0.9
#: The `state` key that records the starter agents were imported.
SEEDED = "agents:seeded"
# ------------------------------------------------------------------- terms
def recency(first_seen: str, now: datetime) -> float:
"""1.0 for a role first seen now, halving every `HALF_LIFE_DAYS`."""
try:
seen = datetime.fromisoformat(first_seen)
except ValueError:
return 0.0
days = max((now - seen).total_seconds(), 0.0) / 86400
return math.pow(0.5, days / HALF_LIFE_DAYS)
def affinity_term(affinity: int) -> float:
"""-1..2 to 0..1. -1 never gets here; neutral is 1/3."""
return (affinity + 1) / 3
def cosine(role: np.ndarray, intent: np.ndarray) -> float:
"""Raw cosine, -1..1; 0 when either vector is zero."""
norms = float(np.linalg.norm(role) * np.linalg.norm(intent))
if norms == 0:
return 0.0
return float(np.dot(role, intent)) / norms
def scale(cosines: dict[str, float]) -> dict[str, float]:
"""Min-max to 0..1 across one agent's run, so `fit` ranks roles rather
than stating a level.
`(cos + 1) / 2` was the level, and on MiniLM it spans about 0.55 to 0.79
for real postings against a real intent, so a prefer gate weighted 0.2
moved a score more than the whole spread of fit did. Scaled, the closest
role is 1 and the farthest 0. A single role, or roles that are all equally
close, get 1: there is nothing to tell them apart by.
"""
if not cosines:
return {}
low, high = min(cosines.values()), max(cosines.values())
if high - low < 1e-9:
return dict.fromkeys(cosines, 1.0)
return {uid: (value - low) / (high - low) for uid, value in cosines.items()}
def _reason(gate: Gate, state: int) -> str:
if state < 0:
return f"{gate.field} unknown"
if gate.field == "salary_min":
return f"salary_min under {min(gate.values, key=lambda v: float(v))}"
return f"{gate.field} not {' / '.join(str(v) for v in gate.values)}"
def failed_gates(candidate: Candidate, gates: Sequence[Gate]) -> list[str]:
"""Every require gate this role fails, as reasons. None for a favourite,
which skips the gates."""
if candidate.affinity == 2:
return []
return [_reason(g, state) for g, state in zip(gates, candidate.gates, strict=True)
if g.mode == "require" and state != 1]
def score_one(candidate: Candidate, gates: Sequence[Gate], weights: Weights,
fit: float | None, now: datetime, *,
missing: str = "no vector") -> tuple[float, str]:
"""One role for one agent: `(score, gated_reason)`. `fit` is already
scaled to 0..1 (see `scale`), or None when the role or the intent has no
vector; `missing` says which."""
favourite = candidate.affinity == 2
if failed := failed_gates(candidate, gates):
return 0.0, ", ".join(failed)
terms = [(weights.recency, recency(candidate.first_seen, now)),
(weights.affinity, affinity_term(candidate.affinity))]
terms += [(g.weight, 1.0 if state == 1 else 0.0)
for g, state in zip(gates, candidate.gates, strict=True) if g.mode == "prefer"]
if fit is not None:
terms.append((weights.fit, fit))
total = sum(w for w, _ in terms)
if not total:
if favourite:
return FAVOURITE_FLOOR, ""
# Fit was the only term with any weight, and there is no fit.
return 0.0, missing if fit is None and weights.fit else "no weights"
score = sum(w * t for w, t in terms) / total
return (max(score, FAVOURITE_FLOOR) if favourite else score), ""
# ------------------------------------------------------------------ vectors
def intent_key(model: str, intent: str) -> str:
return hashlib.sha1(f"{model}\n{intent}".encode()).hexdigest()
class _Encoder:
"""The embedder, loaded on first use only: an agent whose intent vector
is cached costs no model load at all."""
def __init__(self, model: str, context: RunContext) -> None:
self.model = model
self.context = context
self._embedder: Any = None
self.failed = False
def encode(self, text: str) -> np.ndarray | None:
if self.failed:
return None
if self._embedder is None:
try:
self._embedder = embed.load_embedder(self.model)
except ImportError as exc:
self.failed = True
self.context.notice(embed.unavailable(exc))
return None
return np.asarray(self._embedder.encode([text])[0])
def _intent(store: Store, agent: Agent, model: str,
encoder: _Encoder) -> np.ndarray | None:
if not agent.intent.strip():
return None
key = intent_key(model, agent.intent)
cached = store.intent_vector(agent.id, key)
if cached is not None:
return cached
vec = encoder.encode(agent.intent)
if vec is not None:
store.set_intent_vector(agent.id, key, vec)
return vec
# ------------------------------------------------------------------- the job
def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
intent: np.ndarray | None, now: datetime) -> dict[str, int]:
"""Two passes: the cosines of the roles this agent scores, then every
score with fit scaled across them. A gated role is left out of the
scaling, so a role that fails a require gate cannot stretch the range the
ranked roles share."""
candidates = store.score_candidates(agent.gates)
cosines: dict[str, float] = {}
if intent is not None:
for candidate in candidates:
role_vec = vectors.get(candidate.uid)
if role_vec is not None and not failed_gates(candidate, agent.gates):
cosines[candidate.uid] = cosine(role_vec, intent)
fits = scale(cosines)
rows: list[tuple[str, float, str]] = []
gated = 0
for candidate in candidates:
value, reason = score_one(
candidate, agent.gates, agent.weights, fits.get(candidate.uid), now,
missing="no vector" if intent is not None else "no intent vector")
gated += bool(reason)
rows.append((candidate.uid, round(value, 6), reason))
store.replace_scores(agent.id, rows)
return {"scored": len(rows), "gated": gated}
@job("score", "Scoring roles for every agent", solo=True, resource="cpu")
def run_score(config: Config, store: Store, *,
context: RunContext = NULL) -> dict[str, Any]:
seed_agents(config, store)
agents = store.agents(enabled_only=True)
model = config.embed.model
vectors = store.vectors(model) if agents else {}
encoder = _Encoder(model, context)
now = store.clock()
scored = gated = 0
for n, agent in enumerate(agents):
context.check()
context.progress(n, len(agents), f"scoring for {agent.name}, {n + 1} of {len(agents)}")
counts = score_agent(store, agent, vectors, _intent(store, agent, model, encoder), now)
scored += counts["scored"]
gated += counts["gated"]
log.info("%-20s %6d scored, %6d gated", agent.name, counts["scored"], counts["gated"])
context.progress(len(agents), len(agents), "")
return {"agents": len(agents), "scored": scored, "gated": gated,
"vectors": len(vectors)}
# ------------------------------------------------------------------ seeding
def seed_agents(config: Config, store: Store) -> list[str]:
"""Import `config.yaml`'s `agents:` into an empty table, once. The state
marker is what makes it once: deleting every agent afterwards is a
choice, and a restart must not undo it."""
if store.state.get(SEEDED) or store.agents():
return []
added = []
for seed in config.agents:
try:
store.save_agent(seed.name, seed.intent, seed.gates, seed.weights,
seed.enabled)
except ValueError as exc:
log.warning("agents: %s", exc)
continue
added.append(seed.name)
if added:
store.state[SEEDED] = store.now()
log.info("imported %d agents from %s", len(added), config.path or "the defaults")
return added