The CV is cut into chunks inside e5's window, embedded like a prompt and cached per agent under a hash of the model, chunking and text. A role's cv_fit (scores.cv_fit, migration 31) is its best chunk cosine, scaled with its own calibration band: fit's clipped 88 of agent 2's ungated roles to 1. It is no term of the Agent match score. judge_cv_top (default 50) sits beside judge_top and judge_min on the agents page and card. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
424 lines
19 KiB
Python
424 lines
19 KiB
Python
"""`score`: every open role, scored for every enabled search agent.
|
|
|
|
A role is scored per agent, `(uid, agent) -> score`, because a role is only
|
|
good for a given search. For each agent:
|
|
|
|
require gates a role that fails one keeps its row, with score 0 and the
|
|
reason ("remote not remote")
|
|
any gates one group: a role that matches none of them, and has the
|
|
column filled for at least one, is gated ("none of
|
|
country DK / remote remote")
|
|
prefer gates `weight` when the role matches, 0 when it does not
|
|
exclude gates a role that matches one of the values keeps its row, with
|
|
score 0 and the value ("title account executive")
|
|
|
|
A gate rejects on evidence, never on absence: a role whose column is empty
|
|
is not gated by it. `extract` is what fills an empty column, and it reads
|
|
only the roles that rank, so gating on an empty column would keep those
|
|
roles out of extract's reach for good.
|
|
fit cos(role vector, prompt vector) as a level: the embedding
|
|
model's calibrated `lo` is 0 and `hi` 1, clipped outside
|
|
(see `scale`), so a role's fit does not depend on what
|
|
else is open; with several prompts, the best prompt's
|
|
level, and the score names that prompt
|
|
recency 0.5 ** (days since posted_at / 14)
|
|
affinity (the agent's affinity for the company + 1) / 3
|
|
taste cos(role vector, liked direction), scaled like fit with the
|
|
model's taste calibration, where the direction is
|
|
mean(liked vectors) - mean(passed vectors)
|
|
over this agent's votes; with only likes it is the liked mean,
|
|
with only passes the negated passed mean; no votes, no term
|
|
|
|
Beside the score, for an agent with a CV, `cv_fit`: how close the role is to
|
|
the CV, the best cosine between the role and the CV's chunks
|
|
(`embed.cv_chunks`, embedded as the prompts are), scaled like fit. It is no
|
|
term of the score; the judge reads the closest roles the score leaves out
|
|
(`Store.judge_rows`) and CV match orders the roles not judged yet by it.
|
|
|
|
Every term is 0..1. The score is the weighted sum over the sum of the weights
|
|
(the agent's four and its prefer gates'), so it stays 0..1. A favourite
|
|
company (`affinity = 2`) skips the `any` group but not the `require` or
|
|
`exclude` gates, and
|
|
is scored like any other company with its affinity term at 1: nothing puts
|
|
its roles above a better fit. A hidden one (`-1`) is not scored at all. A
|
|
role with no vector is scored without `fit` and `taste`, unless `fit` is all
|
|
there is, when it is gated as "no vector".
|
|
An agent with no votes scores every role without `taste`; the direction is
|
|
newsdesk's `centroid(liked) - centroid(disliked)`. An agent with no prompt
|
|
that has text has no `fit` either: its scores are not a match, so it is left
|
|
out of what extract and summarise read (`Store._top_scope`), and a run names
|
|
such agents in one notice.
|
|
|
|
Recomputed in full on every run, and each agent's rows are replaced in one
|
|
transaction: nothing is incremental, so nothing can drift.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import logging
|
|
import math
|
|
from collections.abc import Sequence
|
|
from datetime import datetime
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
|
|
from pipeline import NULL, Notice, RunContext, job
|
|
from app.config import Calibration, Config
|
|
from app.jobs import embed
|
|
from app.models import Agent, Candidate, Gate, Weights
|
|
from app.storage import Store
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
HALF_LIFE_DAYS = 14.0
|
|
|
|
#: The `state` key that records the starter agents were imported.
|
|
SEEDED = "agents:seeded"
|
|
|
|
|
|
# ------------------------------------------------------------------- terms
|
|
|
|
def recency(posted_at: str, now: datetime) -> float:
|
|
"""1.0 for a role posted now, halving every `HALF_LIFE_DAYS`. `posted_at`
|
|
is `app.models.posted_at`: the earlier of when the app first saw the role
|
|
and, for a trusted host, when the board says it was posted — so a
|
|
months-old backlog a newly added board brings in does not start at 1.0."""
|
|
try:
|
|
seen = datetime.fromisoformat(posted_at)
|
|
except ValueError:
|
|
return 0.0
|
|
days = max((now - seen).total_seconds(), 0.0) / 86400
|
|
return math.pow(0.5, days / HALF_LIFE_DAYS)
|
|
|
|
|
|
def affinity_term(affinity: int) -> float:
|
|
"""-1..2 to 0..1. -1 never gets here; neutral is 1/3."""
|
|
return (affinity + 1) / 3
|
|
|
|
|
|
def cosine(role: np.ndarray, intent: np.ndarray) -> float:
|
|
"""Raw cosine, -1..1; 0 when either vector is zero."""
|
|
norms = float(np.linalg.norm(role) * np.linalg.norm(intent))
|
|
if norms == 0:
|
|
return 0.0
|
|
return float(np.dot(role, intent)) / norms
|
|
|
|
|
|
def taste_direction(vectors: dict[str, np.ndarray],
|
|
votes: dict[str, int]) -> np.ndarray | None:
|
|
"""`mean(liked) - mean(passed)` over the voted roles that have a vector.
|
|
A side with no votes is zero, so likes alone point at the liked mean and
|
|
passes alone away from the passed mean. None when neither side has a
|
|
vector: no votes, no direction. A neutral vote says nothing."""
|
|
liked = [vectors[uid] for uid, vote in votes.items() if vote == 1 and uid in vectors]
|
|
passed = [vectors[uid] for uid, vote in votes.items() if vote == -1 and uid in vectors]
|
|
if not liked and not passed:
|
|
return None
|
|
direction = np.zeros_like(liked[0] if liked else passed[0], dtype=np.float64)
|
|
if liked:
|
|
direction += np.mean(liked, axis=0)
|
|
if passed:
|
|
direction -= np.mean(passed, axis=0)
|
|
return direction
|
|
|
|
|
|
def scale(cosines: dict[str, float],
|
|
band: tuple[float, float] | None) -> dict[str, float]:
|
|
"""Cosines to 0..1 levels: `clip((cos - lo) / (hi - lo), 0, 1)` with the
|
|
embedding model's calibrated `band` (`app.config.CALIBRATION`).
|
|
|
|
Raw cosines on multilingual-e5-small sit between about 0.75 and 0.87 for
|
|
real postings against a real intent, so unscaled a prefer gate weighted
|
|
0.2 moved a score more than the whole spread of fit. The band stretches
|
|
that spread to 0..1 with fixed constants, so a role's level is the same
|
|
in every run and a `limits` bar is an absolute threshold.
|
|
|
|
Without a band, for a model nobody calibrated, it falls back to min-max
|
|
across the run: the closest role is 1 and the farthest 0, which ranks
|
|
rather than states a level. A single role, or roles that are all equally
|
|
close, then get 1: there is nothing to tell them apart by.
|
|
"""
|
|
if not cosines:
|
|
return {}
|
|
if band is not None:
|
|
lo, hi = band
|
|
return {uid: min(max((value - lo) / (hi - lo), 0.0), 1.0)
|
|
for uid, value in cosines.items()}
|
|
low, high = min(cosines.values()), max(cosines.values())
|
|
if high - low < 1e-9:
|
|
return dict.fromkeys(cosines, 1.0)
|
|
return {uid: (value - low) / (high - low) for uid, value in cosines.items()}
|
|
|
|
|
|
def _wanted(gate: Gate) -> str:
|
|
if gate.field == "salary_min":
|
|
return f"salary_min {min(gate.values, key=lambda v: float(v))}"
|
|
return f"{gate.field} {' / '.join(str(v) for v in gate.values)}"
|
|
|
|
|
|
def _reason(gate: Gate) -> str:
|
|
if gate.field == "salary_min":
|
|
return f"salary_min under {min(gate.values, key=lambda v: float(v))}"
|
|
return f"{gate.field} not {' / '.join(str(v) for v in gate.values)}"
|
|
|
|
|
|
def failed_gates(candidate: Candidate, gates: Sequence[Gate]) -> list[str]:
|
|
"""Why this role is gated, as reasons: every require gate it fails, every
|
|
exclude gate it matches, and the `any` group when it matches none of them.
|
|
A gate with an empty column (state -1) neither passes nor fails, so the
|
|
group fails only when some member had evidence. A favourite company's
|
|
roles skip the `any` group but not the require and exclude gates, so a
|
|
star widens a search to a company's roles within e.g. Europe, not to the
|
|
whole world, and not to the titles the agent leaves out."""
|
|
states = list(zip(gates, candidate.gates, strict=True))
|
|
reasons = [_reason(g) for g, state in states if g.mode == "require" and state == 0]
|
|
reasons += [f"{g.field} {g.values[state - 1]}" for g, state in states
|
|
if g.mode == "exclude" and state > 0]
|
|
group = [(g, state) for g, state in states if g.mode == "any"]
|
|
if candidate.affinity != 2 and group and not any(state == 1 for _, state in group) \
|
|
and any(state == 0 for _, state in group):
|
|
reasons.append("none of " + ", ".join(_wanted(g) for g, _ in group))
|
|
return reasons
|
|
|
|
|
|
def score_one(candidate: Candidate, gates: Sequence[Gate], weights: Weights,
|
|
fit: float | None, now: datetime, *, taste: float | None = None,
|
|
missing: str = "no vector") -> tuple[float, str]:
|
|
"""One role for one agent: `(score, gated_reason)`. `fit` and `taste` are
|
|
already scaled to 0..1 (see `scale`), or None when there is nothing to
|
|
measure: no role vector, no prompt vector, no votes. A None term leaves
|
|
the denominator; `missing` says why fit is out."""
|
|
if failed := failed_gates(candidate, gates):
|
|
return 0.0, ", ".join(failed)
|
|
|
|
terms = [(weights.recency, recency(candidate.posted_at, now)),
|
|
(weights.affinity, affinity_term(candidate.affinity))]
|
|
terms += [(g.weight, 1.0 if state == 1 else 0.0)
|
|
for g, state in zip(gates, candidate.gates, strict=True) if g.mode == "prefer"]
|
|
if fit is not None:
|
|
terms.append((weights.fit, fit))
|
|
if taste is not None:
|
|
terms.append((weights.taste, taste))
|
|
total = sum(w for w, _ in terms)
|
|
if not total:
|
|
# Fit was the only term with any weight, and there is no fit.
|
|
return 0.0, missing if fit is None and weights.fit else "no weights"
|
|
return sum(w * t for w, t in terms) / total, ""
|
|
|
|
|
|
# ------------------------------------------------------------------ vectors
|
|
|
|
def intent_key(model: str, intent: str) -> str:
|
|
return hashlib.sha1(f"{model}\n{intent}".encode()).hexdigest()
|
|
|
|
|
|
def cv_key(model: str, cv_text: str) -> str:
|
|
"""What the CV's vectors are cached under: the model, how the CV is cut
|
|
(`embed.CV_CHUNKING`) and the text."""
|
|
return hashlib.sha1(f"{model}\n{embed.CV_CHUNKING}\n{cv_text}".encode()).hexdigest()
|
|
|
|
|
|
def closeness(role: np.ndarray, chunks: np.ndarray) -> float:
|
|
"""The best raw cosine between a role and the CV's chunks, -1..1, with
|
|
`chunks` one unit-length row each (`_cv`); 0 for a zero role vector."""
|
|
norm = float(np.linalg.norm(role))
|
|
if norm == 0 or not len(chunks):
|
|
return 0.0
|
|
return float(np.max(chunks @ role)) / norm
|
|
|
|
|
|
class _Encoder:
|
|
"""The embedder, loaded on first use only: an agent whose prompt vectors
|
|
are cached costs no model load at all."""
|
|
|
|
def __init__(self, model: str, context: RunContext) -> None:
|
|
self.model = model
|
|
self.context = context
|
|
self._embedder: Any = None
|
|
self.failed = False
|
|
|
|
def encode(self, text: str) -> np.ndarray | None:
|
|
found = self.encode_many([text])
|
|
return None if found is None else found[0]
|
|
|
|
def encode_many(self, texts: Sequence[str]) -> np.ndarray | None:
|
|
if self.failed:
|
|
return None
|
|
if self._embedder is None:
|
|
try:
|
|
self._embedder = embed.load_embedder(self.model)
|
|
except ImportError as exc:
|
|
self.failed = True
|
|
self.context.notice(embed.unavailable(exc))
|
|
return None
|
|
return np.asarray(self._embedder.encode(list(texts)))
|
|
|
|
|
|
def _prompts(store: Store, agent: Agent, model: str,
|
|
encoder: _Encoder) -> list[tuple[str, np.ndarray]]:
|
|
"""`(name, vector)` for each of the agent's prompts that has text and
|
|
could be embedded, in order. Each vector is cached under its own key, so
|
|
an edit re-embeds only the prompt it changed."""
|
|
found = []
|
|
for prompt in agent.prompts:
|
|
if not prompt.text.strip():
|
|
continue
|
|
key = intent_key(model, prompt.text)
|
|
vec = store.prompt_vector(agent.id, prompt.name, key)
|
|
if vec is None:
|
|
vec = encoder.encode(embed.intent_text(model, prompt.text))
|
|
if vec is None:
|
|
continue
|
|
store.set_prompt_vector(agent.id, prompt.name, key, vec)
|
|
found.append((prompt.name, vec))
|
|
return found
|
|
|
|
|
|
def _cv(store: Store, agent: Agent, model: str, encoder: _Encoder) -> np.ndarray | None:
|
|
"""The agent's CV as one unit-length row per chunk (`embed.cv_chunks`),
|
|
each encoded as the prompts are ("query: " for an e5 model), or None
|
|
without a CV or an embedder. Cached under `cv_key`, so a run embeds the
|
|
CV only after it changed."""
|
|
if not agent.cv_text.strip():
|
|
return None
|
|
key = cv_key(model, agent.cv_text)
|
|
found = store.cv_vectors(agent.id, key)
|
|
if found is None:
|
|
chunks = embed.cv_chunks(agent.cv_text)
|
|
made = encoder.encode_many([embed.intent_text(model, c) for c in chunks])
|
|
if made is None or not len(made):
|
|
return None
|
|
found = list(made)
|
|
store.set_cv_vectors(agent.id, key, found)
|
|
rows = np.vstack(found).astype(np.float64)
|
|
norms = np.linalg.norm(rows, axis=1, keepdims=True)
|
|
return rows / np.where(norms == 0, 1.0, norms)
|
|
|
|
|
|
# ------------------------------------------------------------------- the job
|
|
|
|
def no_intent(names: Sequence[str]) -> Notice:
|
|
"""The notice for enabled agents without a prompt that has text: their
|
|
scores have no fit, so `Store._top_scope` keeps them from feeding extract
|
|
and summarise. The code keeps its name from when an agent had one intent."""
|
|
return Notice(
|
|
code="score.no_intent", level="warning", kind="expected", key="score",
|
|
text=f"No prompt with text, so ranked without fit and not read by extract "
|
|
f"or summarise: {', '.join(names)}.",
|
|
fix="Give the agent a prompt under Settings, Search agents.",
|
|
where="app.jobs.score")
|
|
|
|
|
|
def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
|
|
prompts: Sequence[tuple[str, np.ndarray]], now: datetime,
|
|
calibration: Calibration | None = None,
|
|
cv: np.ndarray | None = None) -> dict[str, int]:
|
|
"""Two passes: the cosines of the roles this agent scores, to each of its
|
|
`prompts` and to its taste direction, then every score with both scaled
|
|
by the model's `calibration`, or across the run without one. A role's fit
|
|
is the best of its prompts' levels, each scaled on its own, and the score
|
|
keeps that prompt's name; the first prompt wins a tie. Under the min-max
|
|
fallback each prompt is ranged over the run on its own too, so every
|
|
prompt's closest role gets 1 and one prompt equals the old single intent.
|
|
A gated role is left out of the cosines, so under the fallback a role
|
|
that fails a require gate cannot stretch the range the ranked roles
|
|
share. The voted roles are scored like any other; a liked one sits near
|
|
the direction it helped make, which is what a like should do. With `cv`
|
|
(`_cv`), each of those roles also gets its `cv_fit`, its `closeness` to
|
|
the CV scaled like fit, beside the score and not in it."""
|
|
candidates = store.score_candidates(agent.id, agent.gates)
|
|
direction = taste_direction(vectors, store.votes(agent.id))
|
|
cosines: dict[str, dict[str, float]] = {name: {} for name, _ in prompts}
|
|
tastes: dict[str, float] = {}
|
|
near: dict[str, float] = {}
|
|
for candidate in candidates:
|
|
role_vec = vectors.get(candidate.uid)
|
|
if role_vec is None or failed_gates(candidate, agent.gates):
|
|
continue
|
|
for name, prompt_vec in prompts:
|
|
cosines[name][candidate.uid] = cosine(role_vec, prompt_vec)
|
|
if direction is not None:
|
|
tastes[candidate.uid] = cosine(role_vec, direction)
|
|
if cv is not None:
|
|
near[candidate.uid] = closeness(role_vec, cv)
|
|
fits: dict[str, float] = {}
|
|
best: dict[str, str] = {}
|
|
for name, levels in cosines.items():
|
|
for uid, value in scale(levels, calibration.fit if calibration else None).items():
|
|
if uid not in fits or value > fits[uid]:
|
|
fits[uid], best[uid] = value, name
|
|
tastes = scale(tastes, calibration.taste if calibration else None)
|
|
near = scale(near, calibration.cv_band if calibration else None)
|
|
rows: list[tuple[str, float, str, str, float | None]] = []
|
|
gated = 0
|
|
for candidate in candidates:
|
|
fit = fits.get(candidate.uid)
|
|
value, reason = score_one(
|
|
candidate, agent.gates, agent.weights, fit, now,
|
|
taste=tastes.get(candidate.uid),
|
|
missing="no vector" if prompts else "no prompt vector")
|
|
gated += bool(reason)
|
|
counted = fit is not None and agent.weights.fit and not reason
|
|
cv_fit = near.get(candidate.uid)
|
|
rows.append((candidate.uid, round(value, 6), reason,
|
|
best[candidate.uid] if counted else "",
|
|
None if cv_fit is None else round(cv_fit, 6)))
|
|
store.replace_scores(agent.id, rows)
|
|
return {"scored": len(rows), "gated": gated}
|
|
|
|
|
|
@job("score", "Scoring roles for every agent", solo=True, resource="cpu")
|
|
def run_score(config: Config, store: Store, *,
|
|
context: RunContext = NULL) -> dict[str, Any]:
|
|
seed_agents(config, store)
|
|
agents = store.agents(enabled_only=True)
|
|
model = config.embed.model
|
|
vectors = store.vectors(model) if agents else {}
|
|
encoder = _Encoder(model, context)
|
|
now = store.clock()
|
|
calibration = config.embed.calibrated()
|
|
if agents and calibration is None:
|
|
log.warning("no calibration for %s: fit and taste are min-max scaled across "
|
|
"each agent's roles in this run, so they rank rather than state a "
|
|
"level; add it under embed.calibration", model)
|
|
if blind := [agent.name for agent in agents
|
|
if not any(p.text.strip() for p in agent.prompts)]:
|
|
context.notice(no_intent(blind))
|
|
scored = gated = 0
|
|
for n, agent in enumerate(agents):
|
|
context.check()
|
|
context.progress(n, len(agents), f"scoring for {agent.name}, {n + 1} of {len(agents)}")
|
|
counts = score_agent(store, agent, vectors, _prompts(store, agent, model, encoder),
|
|
now, calibration, cv=_cv(store, agent, model, encoder))
|
|
scored += counts["scored"]
|
|
gated += counts["gated"]
|
|
log.info("%-20s %6d scored, %6d gated", agent.name, counts["scored"], counts["gated"])
|
|
context.progress(len(agents), len(agents), "")
|
|
return {"agents": len(agents), "scored": scored, "gated": gated,
|
|
"vectors": len(vectors)}
|
|
|
|
|
|
# ------------------------------------------------------------------ seeding
|
|
|
|
def seed_agents(config: Config, store: Store) -> list[str]:
|
|
"""Import `config.yaml`'s `agents:` into an empty table, once. The state
|
|
marker is what makes it once: deleting every agent afterwards is a
|
|
choice, and a restart must not undo it."""
|
|
if store.state.get(SEEDED) or store.agents():
|
|
return []
|
|
added = []
|
|
for seed in config.agents:
|
|
try:
|
|
store.save_agent(seed.name, seed.intent, seed.gates, seed.weights,
|
|
seed.enabled, limits=seed.limits, prompts=seed.prompts)
|
|
except ValueError as exc:
|
|
log.warning("agents: %s", exc)
|
|
continue
|
|
added.append(seed.name)
|
|
if added:
|
|
store.state[SEEDED] = store.now()
|
|
log.info("imported %d agents from %s", len(added), config.path or "the defaults")
|
|
return added
|