"""`score`: every open role, scored for every enabled search agent. A role is scored per agent, `(uid, agent) -> score`, because a role is only good for a given search. For each agent: require gates a role that fails one keeps its row, with score 0 and the reason ("remote not remote") any gates one group: a role that matches none of them, and has the column filled for at least one, is gated ("none of country DK / remote remote") prefer gates `weight` when the role matches, 0 when it does not exclude gates a role that matches one of the values keeps its row, with score 0 and the value ("title account executive") A gate rejects on evidence, never on absence: a role whose column is empty is not gated by it. `extract` is what fills an empty column, and it reads only the roles that rank, so gating on an empty column would keep those roles out of extract's reach for good. fit cos(role vector, prompt vector) as a level: the embedding model's calibrated `lo` is 0 and `hi` 1, clipped outside (see `scale`), so a role's fit does not depend on what else is open; with several prompts, the best prompt's level, and the score names that prompt recency 0.5 ** (days since posted_at / 14) affinity (the agent's affinity for the company + 1) / 3 taste cos(role vector, liked direction), scaled like fit with the model's taste calibration, where the direction is mean(liked vectors) - mean(passed vectors) over this agent's votes; with only likes it is the liked mean, with only passes the negated passed mean; no votes, no term Beside the score, for an agent with a CV, `cv_fit`: how close the role is to the CV, the best cosine between the role and the CV's chunks (`embed.cv_chunks`, embedded as the prompts are), scaled like fit. It is no term of the score; the judge reads the closest roles the score leaves out (`Store.judge_rows`) and CV match orders the roles not judged yet by it. Every term is 0..1. The score is the weighted sum over the sum of the weights (the agent's four and its prefer gates'), so it stays 0..1. A favourite company (`affinity = 2`) skips the `any` group but not the `require` or `exclude` gates, and is scored like any other company with its affinity term at 1: nothing puts its roles above a better fit. A hidden one (`-1`) is not scored at all. A role with no vector is scored without `fit` and `taste`, unless `fit` is all there is, when it is gated as "no vector". An agent with no votes scores every role without `taste`; the direction is newsdesk's `centroid(liked) - centroid(disliked)`. An agent with no prompt that has text has no `fit` either: its scores are not a match, so it is left out of what extract and summarise read (`Store._top_scope`), and a run names such agents in one notice. Recomputed in full on every run, and each agent's rows are replaced in one transaction: nothing is incremental, so nothing can drift. """ from __future__ import annotations import hashlib import logging import math from collections.abc import Sequence from datetime import datetime from typing import Any import numpy as np from pipeline import NULL, Notice, RunContext, job from app.config import Calibration, Config from app.jobs import embed from app.models import Agent, Candidate, Gate, Weights from app.storage import Store log = logging.getLogger(__name__) HALF_LIFE_DAYS = 14.0 #: The `state` key that records the starter agents were imported. SEEDED = "agents:seeded" # ------------------------------------------------------------------- terms def recency(posted_at: str, now: datetime) -> float: """1.0 for a role posted now, halving every `HALF_LIFE_DAYS`. `posted_at` is `app.models.posted_at`: the earlier of when the app first saw the role and, for a trusted host, when the board says it was posted — so a months-old backlog a newly added board brings in does not start at 1.0.""" try: seen = datetime.fromisoformat(posted_at) except ValueError: return 0.0 days = max((now - seen).total_seconds(), 0.0) / 86400 return math.pow(0.5, days / HALF_LIFE_DAYS) def affinity_term(affinity: int) -> float: """-1..2 to 0..1. -1 never gets here; neutral is 1/3.""" return (affinity + 1) / 3 def cosine(role: np.ndarray, intent: np.ndarray) -> float: """Raw cosine, -1..1; 0 when either vector is zero.""" norms = float(np.linalg.norm(role) * np.linalg.norm(intent)) if norms == 0: return 0.0 return float(np.dot(role, intent)) / norms def taste_direction(vectors: dict[str, np.ndarray], votes: dict[str, int]) -> np.ndarray | None: """`mean(liked) - mean(passed)` over the voted roles that have a vector. A side with no votes is zero, so likes alone point at the liked mean and passes alone away from the passed mean. None when neither side has a vector: no votes, no direction. A neutral vote says nothing.""" liked = [vectors[uid] for uid, vote in votes.items() if vote == 1 and uid in vectors] passed = [vectors[uid] for uid, vote in votes.items() if vote == -1 and uid in vectors] if not liked and not passed: return None direction = np.zeros_like(liked[0] if liked else passed[0], dtype=np.float64) if liked: direction += np.mean(liked, axis=0) if passed: direction -= np.mean(passed, axis=0) return direction def scale(cosines: dict[str, float], band: tuple[float, float] | None) -> dict[str, float]: """Cosines to 0..1 levels: `clip((cos - lo) / (hi - lo), 0, 1)` with the embedding model's calibrated `band` (`app.config.CALIBRATION`). Raw cosines on multilingual-e5-small sit between about 0.75 and 0.87 for real postings against a real intent, so unscaled a prefer gate weighted 0.2 moved a score more than the whole spread of fit. The band stretches that spread to 0..1 with fixed constants, so a role's level is the same in every run and a `limits` bar is an absolute threshold. Without a band, for a model nobody calibrated, it falls back to min-max across the run: the closest role is 1 and the farthest 0, which ranks rather than states a level. A single role, or roles that are all equally close, then get 1: there is nothing to tell them apart by. """ if not cosines: return {} if band is not None: lo, hi = band return {uid: min(max((value - lo) / (hi - lo), 0.0), 1.0) for uid, value in cosines.items()} low, high = min(cosines.values()), max(cosines.values()) if high - low < 1e-9: return dict.fromkeys(cosines, 1.0) return {uid: (value - low) / (high - low) for uid, value in cosines.items()} def _wanted(gate: Gate) -> str: if gate.field == "salary_min": return f"salary_min {min(gate.values, key=lambda v: float(v))}" return f"{gate.field} {' / '.join(str(v) for v in gate.values)}" def _reason(gate: Gate) -> str: if gate.field == "salary_min": return f"salary_min under {min(gate.values, key=lambda v: float(v))}" return f"{gate.field} not {' / '.join(str(v) for v in gate.values)}" def failed_gates(candidate: Candidate, gates: Sequence[Gate]) -> list[str]: """Why this role is gated, as reasons: every require gate it fails, every exclude gate it matches, and the `any` group when it matches none of them. A gate with an empty column (state -1) neither passes nor fails, so the group fails only when some member had evidence. A favourite company's roles skip the `any` group but not the require and exclude gates, so a star widens a search to a company's roles within e.g. Europe, not to the whole world, and not to the titles the agent leaves out.""" states = list(zip(gates, candidate.gates, strict=True)) reasons = [_reason(g) for g, state in states if g.mode == "require" and state == 0] reasons += [f"{g.field} {g.values[state - 1]}" for g, state in states if g.mode == "exclude" and state > 0] group = [(g, state) for g, state in states if g.mode == "any"] if candidate.affinity != 2 and group and not any(state == 1 for _, state in group) \ and any(state == 0 for _, state in group): reasons.append("none of " + ", ".join(_wanted(g) for g, _ in group)) return reasons def score_one(candidate: Candidate, gates: Sequence[Gate], weights: Weights, fit: float | None, now: datetime, *, taste: float | None = None, missing: str = "no vector") -> tuple[float, str]: """One role for one agent: `(score, gated_reason)`. `fit` and `taste` are already scaled to 0..1 (see `scale`), or None when there is nothing to measure: no role vector, no prompt vector, no votes. A None term leaves the denominator; `missing` says why fit is out.""" if failed := failed_gates(candidate, gates): return 0.0, ", ".join(failed) terms = [(weights.recency, recency(candidate.posted_at, now)), (weights.affinity, affinity_term(candidate.affinity))] terms += [(g.weight, 1.0 if state == 1 else 0.0) for g, state in zip(gates, candidate.gates, strict=True) if g.mode == "prefer"] if fit is not None: terms.append((weights.fit, fit)) if taste is not None: terms.append((weights.taste, taste)) total = sum(w for w, _ in terms) if not total: # Fit was the only term with any weight, and there is no fit. return 0.0, missing if fit is None and weights.fit else "no weights" return sum(w * t for w, t in terms) / total, "" # ------------------------------------------------------------------ vectors def intent_key(model: str, intent: str) -> str: return hashlib.sha1(f"{model}\n{intent}".encode()).hexdigest() def cv_key(model: str, cv_text: str) -> str: """What the CV's vectors are cached under: the model, how the CV is cut (`embed.CV_CHUNKING`) and the text.""" return hashlib.sha1(f"{model}\n{embed.CV_CHUNKING}\n{cv_text}".encode()).hexdigest() def closeness(role: np.ndarray, chunks: np.ndarray) -> float: """The best raw cosine between a role and the CV's chunks, -1..1, with `chunks` one unit-length row each (`_cv`); 0 for a zero role vector.""" norm = float(np.linalg.norm(role)) if norm == 0 or not len(chunks): return 0.0 return float(np.max(chunks @ role)) / norm class _Encoder: """The embedder, loaded on first use only: an agent whose prompt vectors are cached costs no model load at all.""" def __init__(self, model: str, context: RunContext) -> None: self.model = model self.context = context self._embedder: Any = None self.failed = False def encode(self, text: str) -> np.ndarray | None: found = self.encode_many([text]) return None if found is None else found[0] def encode_many(self, texts: Sequence[str]) -> np.ndarray | None: if self.failed: return None if self._embedder is None: try: self._embedder = embed.load_embedder(self.model) except ImportError as exc: self.failed = True self.context.notice(embed.unavailable(exc)) return None return np.asarray(self._embedder.encode(list(texts))) def _prompts(store: Store, agent: Agent, model: str, encoder: _Encoder) -> list[tuple[str, np.ndarray]]: """`(name, vector)` for each of the agent's prompts that has text and could be embedded, in order. Each vector is cached under its own key, so an edit re-embeds only the prompt it changed.""" found = [] for prompt in agent.prompts: if not prompt.text.strip(): continue key = intent_key(model, prompt.text) vec = store.prompt_vector(agent.id, prompt.name, key) if vec is None: vec = encoder.encode(embed.intent_text(model, prompt.text)) if vec is None: continue store.set_prompt_vector(agent.id, prompt.name, key, vec) found.append((prompt.name, vec)) return found def _cv(store: Store, agent: Agent, model: str, encoder: _Encoder) -> np.ndarray | None: """The agent's CV as one unit-length row per chunk (`embed.cv_chunks`), each encoded as the prompts are ("query: " for an e5 model), or None without a CV or an embedder. Cached under `cv_key`, so a run embeds the CV only after it changed.""" if not agent.cv_text.strip(): return None key = cv_key(model, agent.cv_text) found = store.cv_vectors(agent.id, key) if found is None: chunks = embed.cv_chunks(agent.cv_text) made = encoder.encode_many([embed.intent_text(model, c) for c in chunks]) if made is None or not len(made): return None found = list(made) store.set_cv_vectors(agent.id, key, found) rows = np.vstack(found).astype(np.float64) norms = np.linalg.norm(rows, axis=1, keepdims=True) return rows / np.where(norms == 0, 1.0, norms) # ------------------------------------------------------------------- the job def no_intent(names: Sequence[str]) -> Notice: """The notice for enabled agents without a prompt that has text: their scores have no fit, so `Store._top_scope` keeps them from feeding extract and summarise. The code keeps its name from when an agent had one intent.""" return Notice( code="score.no_intent", level="warning", kind="expected", key="score", text=f"No prompt with text, so ranked without fit and not read by extract " f"or summarise: {', '.join(names)}.", fix="Give the agent a prompt under Settings, Search agents.", where="app.jobs.score") def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray], prompts: Sequence[tuple[str, np.ndarray]], now: datetime, calibration: Calibration | None = None, cv: np.ndarray | None = None) -> dict[str, int]: """Two passes: the cosines of the roles this agent scores, to each of its `prompts` and to its taste direction, then every score with both scaled by the model's `calibration`, or across the run without one. A role's fit is the best of its prompts' levels, each scaled on its own, and the score keeps that prompt's name; the first prompt wins a tie. Under the min-max fallback each prompt is ranged over the run on its own too, so every prompt's closest role gets 1 and one prompt equals the old single intent. A gated role is left out of the cosines, so under the fallback a role that fails a require gate cannot stretch the range the ranked roles share. The voted roles are scored like any other; a liked one sits near the direction it helped make, which is what a like should do. With `cv` (`_cv`), each of those roles also gets its `cv_fit`, its `closeness` to the CV scaled like fit, beside the score and not in it.""" candidates = store.score_candidates(agent.id, agent.gates) direction = taste_direction(vectors, store.votes(agent.id)) cosines: dict[str, dict[str, float]] = {name: {} for name, _ in prompts} tastes: dict[str, float] = {} near: dict[str, float] = {} for candidate in candidates: role_vec = vectors.get(candidate.uid) if role_vec is None or failed_gates(candidate, agent.gates): continue for name, prompt_vec in prompts: cosines[name][candidate.uid] = cosine(role_vec, prompt_vec) if direction is not None: tastes[candidate.uid] = cosine(role_vec, direction) if cv is not None: near[candidate.uid] = closeness(role_vec, cv) fits: dict[str, float] = {} best: dict[str, str] = {} for name, levels in cosines.items(): for uid, value in scale(levels, calibration.fit if calibration else None).items(): if uid not in fits or value > fits[uid]: fits[uid], best[uid] = value, name tastes = scale(tastes, calibration.taste if calibration else None) near = scale(near, calibration.cv_band if calibration else None) rows: list[tuple[str, float, str, str, float | None]] = [] gated = 0 for candidate in candidates: fit = fits.get(candidate.uid) value, reason = score_one( candidate, agent.gates, agent.weights, fit, now, taste=tastes.get(candidate.uid), missing="no vector" if prompts else "no prompt vector") gated += bool(reason) counted = fit is not None and agent.weights.fit and not reason cv_fit = near.get(candidate.uid) rows.append((candidate.uid, round(value, 6), reason, best[candidate.uid] if counted else "", None if cv_fit is None else round(cv_fit, 6))) store.replace_scores(agent.id, rows) return {"scored": len(rows), "gated": gated} @job("score", "Scoring roles for every agent", solo=True, resource="cpu") def run_score(config: Config, store: Store, *, context: RunContext = NULL) -> dict[str, Any]: seed_agents(config, store) agents = store.agents(enabled_only=True) model = config.embed.model vectors = store.vectors(model) if agents else {} encoder = _Encoder(model, context) now = store.clock() calibration = config.embed.calibrated() if agents and calibration is None: log.warning("no calibration for %s: fit and taste are min-max scaled across " "each agent's roles in this run, so they rank rather than state a " "level; add it under embed.calibration", model) if blind := [agent.name for agent in agents if not any(p.text.strip() for p in agent.prompts)]: context.notice(no_intent(blind)) scored = gated = 0 for n, agent in enumerate(agents): context.check() context.progress(n, len(agents), f"scoring for {agent.name}, {n + 1} of {len(agents)}") counts = score_agent(store, agent, vectors, _prompts(store, agent, model, encoder), now, calibration, cv=_cv(store, agent, model, encoder)) scored += counts["scored"] gated += counts["gated"] log.info("%-20s %6d scored, %6d gated", agent.name, counts["scored"], counts["gated"]) context.progress(len(agents), len(agents), "") return {"agents": len(agents), "scored": scored, "gated": gated, "vectors": len(vectors)} # ------------------------------------------------------------------ seeding def seed_agents(config: Config, store: Store) -> list[str]: """Import `config.yaml`'s `agents:` into an empty table, once. The state marker is what makes it once: deleting every agent afterwards is a choice, and a restart must not undo it.""" if store.state.get(SEEDED) or store.agents(): return [] added = [] for seed in config.agents: try: store.save_agent(seed.name, seed.intent, seed.gates, seed.weights, seed.enabled, limits=seed.limits, prompts=seed.prompts) except ValueError as exc: log.warning("agents: %s", exc) continue added.append(seed.name) if added: store.state[SEEDED] = store.now() log.info("imported %d agents from %s", len(added), config.path or "the defaults") return added