Files

499 lines
23 KiB
Python

"""`score`: every open role, scored for every enabled search agent, and each
agent's cut.
A role is scored per agent, `(uid, agent) -> score`, because a role is only
good for a given search. Each agent first applies its fetch gate:
require gates a role that fails one keeps its row, with score 0 and the
reason ("remote not remote"); the region, which nothing
lifts
any gates one group, the place part: a role that matches none of
them, and has the column filled for at least one, is
gated ("none of country DK / remote remote"), unless its
location is unknown; skipped for a company set to All
locations
exclude gates a role that matches one of the values keeps its row, with
score 0 and the value ("title account executive"); those
on `title` are skipped for a company set to All roles
A gate rejects on evidence, never on absence: a role whose column is empty
is not gated by it, and a place the gazetteer could not resolve, or one
that holds the agent's places (Øresund for Copenhagen), is no evidence.
`extract` is what fills an empty column, and it reads only the roles in
the cut, so gating on an empty column would keep those roles out of
extract's reach for good. A hidden company (`affinity = -1`)
is not scored at all; a favourite passes the gate like any other company.
The final score, the feed's order, is the weighted sum of four terms over
the sum of the weights, so it stays 0..1:
fit cos(role vector, prompt vector) as a level: the embedding
model's calibrated `lo` is 0 and `hi` 1, clipped outside
(see `scale`), so a role's fit does not depend on what
else is open; with several prompts, the best prompt's
level, and the score names that prompt
recency 0.5 ** (days since posted_at / 14)
affinity (the agent's affinity for the company + 1) / 3
taste cos(role vector, liked direction), scaled like fit with the
model's taste calibration, where the direction is
mean(liked vectors) - mean(passed vectors)
over this agent's votes; with only likes it is the liked mean,
with only passes the negated passed mean; no votes, no term
A role with no vector is scored without `fit` and `taste`, unless `fit` is
all there is, when it is gated as "no vector". An agent with no votes scores
every role without `taste`; the direction is newsdesk's
`centroid(liked) - centroid(disliked)`. An agent with no prompt that has
text has no `fit` either, so nothing of it enters the cut by fit and its
want order does not decide what summarise reads (`Store._summarise_top`); a
run names such agents in one notice.
Beside the score, `fit` is stored as it is, and for an agent with a CV,
`cv_fit`: how close the role is to the CV, the best cosine between the role
and the CV's chunks (`embed.cv_chunks`, embedded as the prompts are), scaled
like fit. Neither carries the affinity term.
The cut is the roles that get fetch text, extract and judge (`cut_of`). It
is computed from two levels, each with the company's affinity as a bonus of
`w.affinity * affinity / 3` over the sum of the weights, the same size as in
the final score (a liked company about 0.05 above neutral, a favourite about
0.1): the fit level `fit + bonus` and the CV level `cv_fit + bonus`. Over
the roles that pass the fetch gate, the best `cut_fit_top` by fit level at
or above `cut_fit_min` are `fit`; the best `cut_cv_top` by CV level at or
above `cut_cv_min` that are not in yet are `cv`; every role this agent
liked or anybody saved that is not in yet is `kept`. Newness and taste are
no part of it, and the two lists are not merged into one number.
Recomputed in full on every run, and each agent's rows are replaced in one
transaction: nothing is incremental, so nothing can drift.
"""
from __future__ import annotations
import hashlib
import logging
import math
from collections.abc import Sequence
from datetime import datetime
from typing import Any
import numpy as np
from pipeline import NULL, Notice, RunContext, job
from app.config import Calibration, Config
from app.jobs import embed
from app.models import Agent, Candidate, Gate, Limits, Weights
from app.storage import Store
log = logging.getLogger(__name__)
HALF_LIFE_DAYS = 14.0
#: The `state` key that records the starter agents were imported.
SEEDED = "agents:seeded"
# ------------------------------------------------------------------- terms
def recency(posted_at: str, now: datetime) -> float:
"""1.0 for a role posted now, halving every `HALF_LIFE_DAYS`. `posted_at`
is `app.models.posted_at`: the earlier of when the app first saw the role
and, for a trusted host, when the board says it was posted — so a
months-old backlog a newly added board brings in does not start at 1.0."""
try:
seen = datetime.fromisoformat(posted_at)
except ValueError:
return 0.0
days = max((now - seen).total_seconds(), 0.0) / 86400
return math.pow(0.5, days / HALF_LIFE_DAYS)
def affinity_term(affinity: int) -> float:
"""-1..2 to 0..1. -1 never gets here; neutral is 1/3."""
return (affinity + 1) / 3
def cosine(role: np.ndarray, intent: np.ndarray) -> float:
"""Raw cosine, -1..1; 0 when either vector is zero."""
norms = float(np.linalg.norm(role) * np.linalg.norm(intent))
if norms == 0:
return 0.0
return float(np.dot(role, intent)) / norms
def taste_direction(vectors: dict[str, np.ndarray],
votes: dict[str, int]) -> np.ndarray | None:
"""`mean(liked) - mean(passed)` over the voted roles that have a vector.
A side with no votes is zero, so likes alone point at the liked mean and
passes alone away from the passed mean. None when neither side has a
vector: no votes, no direction. A neutral vote says nothing."""
liked = [vectors[uid] for uid, vote in votes.items() if vote == 1 and uid in vectors]
passed = [vectors[uid] for uid, vote in votes.items() if vote == -1 and uid in vectors]
if not liked and not passed:
return None
direction = np.zeros_like(liked[0] if liked else passed[0], dtype=np.float64)
if liked:
direction += np.mean(liked, axis=0)
if passed:
direction -= np.mean(passed, axis=0)
return direction
def scale(cosines: dict[str, float],
band: tuple[float, float] | None) -> dict[str, float]:
"""Cosines to 0..1 levels: `clip((cos - lo) / (hi - lo), 0, 1)` with the
embedding model's calibrated `band` (`app.config.CALIBRATION`).
Raw cosines on multilingual-e5-small sit between about 0.75 and 0.87 for
real postings against a real intent, so unscaled the whole spread of fit
was smaller than a recency step. The band stretches that spread to 0..1
with fixed constants, so a role's level is the same in every run and a
minimum of the cut is an absolute threshold.
Without a band, for a model nobody calibrated, it falls back to min-max
across the run: the closest role is 1 and the farthest 0, which ranks
rather than states a level. A single role, or roles that are all equally
close, then get 1: there is nothing to tell them apart by.
"""
if not cosines:
return {}
if band is not None:
lo, hi = band
return {uid: min(max((value - lo) / (hi - lo), 0.0), 1.0)
for uid, value in cosines.items()}
low, high = min(cosines.values()), max(cosines.values())
if high - low < 1e-9:
return dict.fromkeys(cosines, 1.0)
return {uid: (value - low) / (high - low) for uid, value in cosines.items()}
def _wanted(gate: Gate) -> str:
if gate.field == "salary_min":
return f"salary_min {min(gate.values, key=lambda v: float(v))}"
return f"{gate.field} {' / '.join(str(v) for v in gate.values)}"
def _reason(gate: Gate) -> str:
if gate.field == "salary_min":
return f"salary_min under {min(gate.values, key=lambda v: float(v))}"
return f"{gate.field} not {' / '.join(str(v) for v in gate.values)}"
def failed_gates(candidate: Candidate, gates: Sequence[Gate]) -> list[str]:
"""Why the fetch gate rejects this role, as reasons: every require gate
it fails, every exclude gate it matches, and the `any` group when it
matches none of them. A gate with an empty column (state -1) neither
passes nor fails, so the group fails only when some member had evidence.
A role whose location is unknown, every `country` and `place` member
empty, passes the group whatever its remote says: a hybrid role with no
place may still be in one of the agent's towns, and extract reads it
only in the cut. A group of remote alone still fails a hybrid role.
A company set to All locations skips the `any` group, the place part,
and one set to All roles the exclude gates on `title`, the titles to
leave out; the require gates, the region, apply to every company."""
states = list(zip(gates, candidate.gates, strict=True))
reasons = [_reason(g) for g, state in states if g.mode == "require" and state == 0]
reasons += [f"{g.field} {g.values[state - 1]}" for g, state in states
if g.mode == "exclude" and state > 0
and not (candidate.all_roles and g.field == "title")]
group = [(g, state) for g, state in states if g.mode == "any"]
located = [state for g, state in group if g.field in ("country", "place")]
unknown = bool(located) and all(state == -1 for state in located)
if not candidate.all_countries and group and not unknown \
and not any(state == 1 for _, state in group) \
and any(state == 0 for _, state in group):
reasons.append("none of " + ", ".join(_wanted(g) for g, _ in group))
return reasons
def score_one(candidate: Candidate, gates: Sequence[Gate], weights: Weights,
fit: float | None, now: datetime, *, taste: float | None = None,
missing: str = "no vector") -> tuple[float, str]:
"""One role for one agent: `(final score, gated_reason)`. `fit` and
`taste` are already scaled to 0..1 (see `scale`), or None when there is
nothing to measure: no role vector, no prompt vector, no votes. A None
term leaves the denominator; `missing` says why fit is out."""
if failed := failed_gates(candidate, gates):
return 0.0, ", ".join(failed)
terms = [(weights.recency, recency(candidate.posted_at, now)),
(weights.affinity, affinity_term(candidate.affinity))]
if fit is not None:
terms.append((weights.fit, fit))
if taste is not None:
terms.append((weights.taste, taste))
total = sum(w for w, _ in terms)
if not total:
# Fit was the only term with any weight, and there is no fit.
return 0.0, missing if fit is None and weights.fit else "no weights"
return sum(w * t for w, t in terms) / total, ""
def bonus(affinity: int, weights: Weights) -> float:
"""What a company's affinity adds to both levels of the cut: the share
the affinity term gives it in the final score above a neutral one, so
a liked company sits about 0.05 above a neutral one and a favourite
about 0.1 on the default weights. 0 when every weight is 0."""
total = weights.fit + weights.recency + weights.affinity + weights.taste
return weights.affinity * affinity / 3 / total if total else 0.0
def cut_of(candidates: Sequence[Candidate], passed: set[str], fits: dict[str, float],
near: dict[str, float], kept: set[str], weights: Weights,
limits: Limits) -> dict[str, str]:
"""`uid -> why` for the roles in the cut, among `candidates` whose uid
is in `passed` (the fetch gate let them through): `fit` for the best
`limits.cut_fit_top` by fit level at or above `cut_fit_min`, `cv` for
the best `cut_cv_top` by CV level at or above `cut_cv_min` not in yet,
and `kept` for those in `kept` (liked or saved) not in yet. A level is
`fits` or `near` plus the company's `bonus`; a role without one is in
no list by it. Ties go to the newer role, then the lower uid."""
by_uid = {c.uid: c for c in candidates if c.uid in passed}
# Newest first, then uid, as the sort below keeps equal levels in order.
ordered = sorted(sorted(by_uid.values(), key=lambda c: c.uid),
key=lambda c: c.posted_at, reverse=True)
cut: dict[str, str] = {}
for why, levels, top, floor in (("fit", fits, limits.cut_fit_top, limits.cut_fit_min),
("cv", near, limits.cut_cv_top, limits.cut_cv_min)):
scored = [(levels[c.uid] + bonus(c.affinity, weights), c.uid) for c in ordered
if c.uid in levels and c.uid not in cut]
scored = [(level, uid) for level, uid in scored if level >= floor]
scored.sort(key=lambda pair: pair[0], reverse=True)
for _, uid in scored[:top]:
cut[uid] = why
for c in ordered:
if c.uid in kept and c.uid not in cut:
cut[c.uid] = "kept"
return cut
# ------------------------------------------------------------------ vectors
def intent_key(model: str, intent: str) -> str:
return hashlib.sha1(f"{model}\n{intent}".encode()).hexdigest()
def cv_key(model: str, cv_text: str) -> str:
"""What the CV's vectors are cached under: the model, how the CV is cut
(`embed.CV_CHUNKING`) and the text."""
return hashlib.sha1(f"{model}\n{embed.CV_CHUNKING}\n{cv_text}".encode()).hexdigest()
def closeness(role: np.ndarray, chunks: np.ndarray) -> float:
"""The best raw cosine between a role and the CV's chunks, -1..1, with
`chunks` one unit-length row each (`_cv`); 0 for a zero role vector."""
norm = float(np.linalg.norm(role))
if norm == 0 or not len(chunks):
return 0.0
return float(np.max(chunks @ role)) / norm
class _Encoder:
"""The embedder, loaded on first use only: an agent whose prompt vectors
are cached costs no model load at all."""
def __init__(self, model: str, context: RunContext) -> None:
self.model = model
self.context = context
self._embedder: Any = None
self.failed = False
def encode(self, text: str) -> np.ndarray | None:
found = self.encode_many([text])
return None if found is None else found[0]
def encode_many(self, texts: Sequence[str]) -> np.ndarray | None:
if self.failed:
return None
if self._embedder is None:
try:
self._embedder = embed.load_embedder(self.model)
except ImportError as exc:
self.failed = True
self.context.notice(embed.unavailable(exc))
return None
return np.asarray(self._embedder.encode(list(texts)))
def _prompts(store: Store, agent: Agent, model: str,
encoder: _Encoder) -> list[tuple[str, np.ndarray]]:
"""`(name, vector)` for each of the agent's prompts that has text and
could be embedded, in order. Each vector is cached under its own key, so
an edit re-embeds only the prompt it changed."""
found = []
for prompt in agent.prompts:
if not prompt.text.strip():
continue
key = intent_key(model, prompt.text)
vec = store.prompt_vector(agent.id, prompt.name, key)
if vec is None:
vec = encoder.encode(embed.intent_text(model, prompt.text))
if vec is None:
continue
store.set_prompt_vector(agent.id, prompt.name, key, vec)
found.append((prompt.name, vec))
return found
def _cv(store: Store, agent: Agent, model: str, encoder: _Encoder) -> np.ndarray | None:
"""The agent's CV as one unit-length row per chunk (`embed.cv_chunks`),
each encoded as the prompts are ("query: " for an e5 model), or None
without a CV or an embedder. Cached under `cv_key`, so a run embeds the
CV only after it changed."""
if not agent.cv_text.strip():
return None
key = cv_key(model, agent.cv_text)
found = store.cv_vectors(agent.id, key)
if found is None:
chunks = embed.cv_chunks(agent.cv_text)
made = encoder.encode_many([embed.intent_text(model, c) for c in chunks])
if made is None or not len(made):
return None
found = list(made)
store.set_cv_vectors(agent.id, key, found)
rows = np.vstack(found).astype(np.float64)
norms = np.linalg.norm(rows, axis=1, keepdims=True)
return rows / np.where(norms == 0, 1.0, norms)
# ------------------------------------------------------------------- the job
def no_intent(names: Sequence[str]) -> Notice:
"""The notice for enabled agents without a prompt that has text: their
scores have no fit, so nothing of theirs enters the cut by fit and
their final scores do not choose what summarise reads. The code keeps
its name from when an agent had one intent."""
return Notice(
code="score.no_intent", level="warning", kind="expected", key="score",
text=f"No prompt with text, so ranked without fit and nothing in the cut by fit: "
f"{', '.join(names)}.",
fix="Give the agent a prompt under Settings, Search agents.",
where="app.jobs.score")
def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
prompts: Sequence[tuple[str, np.ndarray]], now: datetime,
calibration: Calibration | None = None,
cv: np.ndarray | None = None,
saved: set[str] | None = None) -> dict[str, int]:
"""Two passes: the cosines of the roles this agent scores, to each of its
`prompts` and to its taste direction, then every score with both scaled
by the model's `calibration`, or across the run without one. A role's fit
is the best of its prompts' levels, each scaled on its own, and the score
keeps that prompt's name; the first prompt wins a tie. Under the min-max
fallback each prompt is ranged over the run on its own too, so every
prompt's closest role gets 1 and one prompt equals the old single intent.
A gated role is left out of the cosines, so under the fallback a role
that fails a require gate cannot stretch the range the ranked roles
share. The voted roles are scored like any other; a liked one sits near
the direction it helped make, which is what a like should do. With `cv`
(`_cv`), each of those roles also gets its `cv_fit`, its `closeness` to
the CV scaled like fit. Then the cut (`cut_of`), with this agent's likes
and the `saved` roles (`Store.saved_uids`, read here when not given)
kept."""
candidates = store.score_candidates(agent.id, agent.gates)
votes = store.votes(agent.id)
direction = taste_direction(vectors, votes)
cosines: dict[str, dict[str, float]] = {name: {} for name, _ in prompts}
tastes: dict[str, float] = {}
near: dict[str, float] = {}
for candidate in candidates:
role_vec = vectors.get(candidate.uid)
if role_vec is None or failed_gates(candidate, agent.gates):
continue
for name, prompt_vec in prompts:
cosines[name][candidate.uid] = cosine(role_vec, prompt_vec)
if direction is not None:
tastes[candidate.uid] = cosine(role_vec, direction)
if cv is not None:
near[candidate.uid] = closeness(role_vec, cv)
fits: dict[str, float] = {}
best: dict[str, str] = {}
for name, levels in cosines.items():
for uid, value in scale(levels, calibration.fit if calibration else None).items():
if uid not in fits or value > fits[uid]:
fits[uid], best[uid] = value, name
tastes = scale(tastes, calibration.taste if calibration else None)
near = scale(near, calibration.cv_band if calibration else None)
scored: list[tuple[str, float, str, float | None]] = []
passed: set[str] = set()
for candidate in candidates:
fit = fits.get(candidate.uid)
value, reason = score_one(
candidate, agent.gates, agent.weights, fit, now,
taste=tastes.get(candidate.uid),
missing="no vector" if prompts else "no prompt vector")
if not reason:
passed.add(candidate.uid)
scored.append((candidate.uid, value, reason, fit))
kept = {uid for uid, vote in votes.items() if vote == 1}
kept |= store.saved_uids() if saved is None else saved
cut = cut_of(candidates, passed, fits, near, kept, agent.weights, agent.limits)
rows = []
for uid, value, reason, fit in scored:
counted = fit is not None and agent.weights.fit and not reason
cv_fit = near.get(uid)
rows.append((uid, round(value, 6), reason, best[uid] if counted else "",
None if cv_fit is None else round(cv_fit, 6),
None if fit is None else round(fit, 6), cut.get(uid, "")))
store.replace_scores(agent.id, rows)
return {"scored": len(rows), "gated": len(rows) - len(passed), "cut": len(cut)}
@job("score", "Scoring roles for every agent", solo=True, resource="cpu")
def run_score(config: Config, store: Store, *,
context: RunContext = NULL) -> dict[str, Any]:
seed_agents(config, store)
agents = store.agents(enabled_only=True)
model = config.embed.model
vectors = store.vectors(model) if agents else {}
encoder = _Encoder(model, context)
now = store.clock()
calibration = config.embed.calibrated()
if agents and calibration is None:
log.warning("no calibration for %s: fit and taste are min-max scaled across "
"each agent's roles in this run, so they rank rather than state a "
"level; add it under embed.calibration", model)
if blind := [agent.name for agent in agents
if not any(p.text.strip() for p in agent.prompts)]:
context.notice(no_intent(blind))
scored = gated = cut = 0
saved = store.saved_uids()
for n, agent in enumerate(agents):
context.check()
context.progress(n, len(agents), f"scoring for {agent.name}, {n + 1} of {len(agents)}")
counts = score_agent(store, agent, vectors, _prompts(store, agent, model, encoder),
now, calibration, cv=_cv(store, agent, model, encoder),
saved=saved)
scored += counts["scored"]
gated += counts["gated"]
cut += counts["cut"]
log.info("%-20s %6d scored, %6d gated, %5d in the cut", agent.name, counts["scored"],
counts["gated"], counts["cut"])
context.progress(len(agents), len(agents), "")
return {"agents": len(agents), "scored": scored, "gated": gated, "cut": cut,
"vectors": len(vectors)}
# ------------------------------------------------------------------ seeding
def seed_agents(config: Config, store: Store) -> list[str]:
"""Import `config.yaml`'s `agents:` into an empty table, once. The state
marker is what makes it once: deleting every agent afterwards is a
choice, and a restart must not undo it."""
if store.state.get(SEEDED) or store.agents():
return []
added = []
for seed in config.agents:
try:
store.save_agent(seed.name, seed.intent, seed.gates, seed.weights,
seed.enabled, limits=seed.limits, prompts=seed.prompts)
except ValueError as exc:
log.warning("agents: %s", exc)
continue
added.append(seed.name)
if added:
store.state[SEEDED] = store.now()
log.info("imported %d agents from %s", len(added), config.path or "the defaults")
return added