489 lines
23 KiB
Python
489 lines
23 KiB
Python
"""`score`: every open role, scored for every enabled search agent, and each
|
|
agent's cut.
|
|
|
|
A role is scored per agent, `(uid, agent) -> score`, because a role is only
|
|
good for a given search. Each agent first applies its fetch gate:
|
|
|
|
require gates a role that fails one keeps its row, with score 0 and the
|
|
reason ("remote not remote"); the region, which nothing
|
|
lifts
|
|
any gates one group, the place part: a role that matches none of
|
|
them, and has the column filled for at least one, is
|
|
gated ("none of country DK / remote remote"); skipped for
|
|
a company set to All countries
|
|
exclude gates a role that matches one of the values keeps its row, with
|
|
score 0 and the value ("title account executive"); those
|
|
on `title` are skipped for a company set to All roles
|
|
|
|
A gate rejects on evidence, never on absence: a role whose column is empty
|
|
is not gated by it. `extract` is what fills an empty column, and it reads
|
|
only the roles in the cut, so gating on an empty column would keep those
|
|
roles out of extract's reach for good. A hidden company (`affinity = -1`)
|
|
is not scored at all; a favourite passes the gate like any other company.
|
|
|
|
The final score, the feed's order, is the weighted sum of four terms over
|
|
the sum of the weights, so it stays 0..1:
|
|
|
|
fit cos(role vector, prompt vector) as a level: the embedding
|
|
model's calibrated `lo` is 0 and `hi` 1, clipped outside
|
|
(see `scale`), so a role's fit does not depend on what
|
|
else is open; with several prompts, the best prompt's
|
|
level, and the score names that prompt
|
|
recency 0.5 ** (days since posted_at / 14)
|
|
affinity (the agent's affinity for the company + 1) / 3
|
|
taste cos(role vector, liked direction), scaled like fit with the
|
|
model's taste calibration, where the direction is
|
|
mean(liked vectors) - mean(passed vectors)
|
|
over this agent's votes; with only likes it is the liked mean,
|
|
with only passes the negated passed mean; no votes, no term
|
|
|
|
A role with no vector is scored without `fit` and `taste`, unless `fit` is
|
|
all there is, when it is gated as "no vector". An agent with no votes scores
|
|
every role without `taste`; the direction is newsdesk's
|
|
`centroid(liked) - centroid(disliked)`. An agent with no prompt that has
|
|
text has no `fit` either, so nothing of it enters the cut by fit and its
|
|
scores do not decide what summarise reads (`Store._summarise_top`); a run
|
|
names such agents in one notice.
|
|
|
|
Beside the score, `fit` is stored as it is, and for an agent with a CV,
|
|
`cv_fit`: how close the role is to the CV, the best cosine between the role
|
|
and the CV's chunks (`embed.cv_chunks`, embedded as the prompts are), scaled
|
|
like fit. Neither carries the affinity term.
|
|
|
|
The cut is the roles that get fetch text, extract and judge (`cut_of`). It
|
|
is computed from two levels, each with the company's affinity as a bonus of
|
|
`w.affinity * affinity / 3` over the sum of the weights, the same size as in
|
|
the final score (a liked company about 0.05 above neutral, a favourite about
|
|
0.1): the fit level `fit + bonus` and the CV level `cv_fit + bonus`. Over
|
|
the roles that pass the fetch gate, the best `cut_fit_top` by fit level at
|
|
or above `cut_fit_min` are `fit`; the best `cut_cv_top` by CV level at or
|
|
above `cut_cv_min` that are not in yet are `cv`; every role this agent
|
|
liked or anybody saved that is not in yet is `kept`. Newness and taste are
|
|
no part of it, and the two lists are not merged into one number.
|
|
|
|
Recomputed in full on every run, and each agent's rows are replaced in one
|
|
transaction: nothing is incremental, so nothing can drift.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import logging
|
|
import math
|
|
from collections.abc import Sequence
|
|
from datetime import datetime
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
|
|
from pipeline import NULL, Notice, RunContext, job
|
|
from app.config import Calibration, Config
|
|
from app.jobs import embed
|
|
from app.models import Agent, Candidate, Gate, Limits, Weights
|
|
from app.storage import Store
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
HALF_LIFE_DAYS = 14.0
|
|
|
|
#: The `state` key that records the starter agents were imported.
|
|
SEEDED = "agents:seeded"
|
|
|
|
|
|
# ------------------------------------------------------------------- terms
|
|
|
|
def recency(posted_at: str, now: datetime) -> float:
|
|
"""1.0 for a role posted now, halving every `HALF_LIFE_DAYS`. `posted_at`
|
|
is `app.models.posted_at`: the earlier of when the app first saw the role
|
|
and, for a trusted host, when the board says it was posted — so a
|
|
months-old backlog a newly added board brings in does not start at 1.0."""
|
|
try:
|
|
seen = datetime.fromisoformat(posted_at)
|
|
except ValueError:
|
|
return 0.0
|
|
days = max((now - seen).total_seconds(), 0.0) / 86400
|
|
return math.pow(0.5, days / HALF_LIFE_DAYS)
|
|
|
|
|
|
def affinity_term(affinity: int) -> float:
|
|
"""-1..2 to 0..1. -1 never gets here; neutral is 1/3."""
|
|
return (affinity + 1) / 3
|
|
|
|
|
|
def cosine(role: np.ndarray, intent: np.ndarray) -> float:
|
|
"""Raw cosine, -1..1; 0 when either vector is zero."""
|
|
norms = float(np.linalg.norm(role) * np.linalg.norm(intent))
|
|
if norms == 0:
|
|
return 0.0
|
|
return float(np.dot(role, intent)) / norms
|
|
|
|
|
|
def taste_direction(vectors: dict[str, np.ndarray],
|
|
votes: dict[str, int]) -> np.ndarray | None:
|
|
"""`mean(liked) - mean(passed)` over the voted roles that have a vector.
|
|
A side with no votes is zero, so likes alone point at the liked mean and
|
|
passes alone away from the passed mean. None when neither side has a
|
|
vector: no votes, no direction. A neutral vote says nothing."""
|
|
liked = [vectors[uid] for uid, vote in votes.items() if vote == 1 and uid in vectors]
|
|
passed = [vectors[uid] for uid, vote in votes.items() if vote == -1 and uid in vectors]
|
|
if not liked and not passed:
|
|
return None
|
|
direction = np.zeros_like(liked[0] if liked else passed[0], dtype=np.float64)
|
|
if liked:
|
|
direction += np.mean(liked, axis=0)
|
|
if passed:
|
|
direction -= np.mean(passed, axis=0)
|
|
return direction
|
|
|
|
|
|
def scale(cosines: dict[str, float],
|
|
band: tuple[float, float] | None) -> dict[str, float]:
|
|
"""Cosines to 0..1 levels: `clip((cos - lo) / (hi - lo), 0, 1)` with the
|
|
embedding model's calibrated `band` (`app.config.CALIBRATION`).
|
|
|
|
Raw cosines on multilingual-e5-small sit between about 0.75 and 0.87 for
|
|
real postings against a real intent, so unscaled the whole spread of fit
|
|
was smaller than a recency step. The band stretches that spread to 0..1
|
|
with fixed constants, so a role's level is the same in every run and a
|
|
minimum of the cut is an absolute threshold.
|
|
|
|
Without a band, for a model nobody calibrated, it falls back to min-max
|
|
across the run: the closest role is 1 and the farthest 0, which ranks
|
|
rather than states a level. A single role, or roles that are all equally
|
|
close, then get 1: there is nothing to tell them apart by.
|
|
"""
|
|
if not cosines:
|
|
return {}
|
|
if band is not None:
|
|
lo, hi = band
|
|
return {uid: min(max((value - lo) / (hi - lo), 0.0), 1.0)
|
|
for uid, value in cosines.items()}
|
|
low, high = min(cosines.values()), max(cosines.values())
|
|
if high - low < 1e-9:
|
|
return dict.fromkeys(cosines, 1.0)
|
|
return {uid: (value - low) / (high - low) for uid, value in cosines.items()}
|
|
|
|
|
|
def _wanted(gate: Gate) -> str:
|
|
if gate.field == "salary_min":
|
|
return f"salary_min {min(gate.values, key=lambda v: float(v))}"
|
|
return f"{gate.field} {' / '.join(str(v) for v in gate.values)}"
|
|
|
|
|
|
def _reason(gate: Gate) -> str:
|
|
if gate.field == "salary_min":
|
|
return f"salary_min under {min(gate.values, key=lambda v: float(v))}"
|
|
return f"{gate.field} not {' / '.join(str(v) for v in gate.values)}"
|
|
|
|
|
|
def failed_gates(candidate: Candidate, gates: Sequence[Gate]) -> list[str]:
|
|
"""Why the fetch gate rejects this role, as reasons: every require gate
|
|
it fails, every exclude gate it matches, and the `any` group when it
|
|
matches none of them. A gate with an empty column (state -1) neither
|
|
passes nor fails, so the group fails only when some member had evidence.
|
|
A company set to All countries skips the `any` group, the place part,
|
|
and one set to All roles the exclude gates on `title`, the titles to
|
|
leave out; the require gates, the region, apply to every company."""
|
|
states = list(zip(gates, candidate.gates, strict=True))
|
|
reasons = [_reason(g) for g, state in states if g.mode == "require" and state == 0]
|
|
reasons += [f"{g.field} {g.values[state - 1]}" for g, state in states
|
|
if g.mode == "exclude" and state > 0
|
|
and not (candidate.all_roles and g.field == "title")]
|
|
group = [(g, state) for g, state in states if g.mode == "any"]
|
|
if not candidate.all_countries and group and not any(state == 1 for _, state in group) \
|
|
and any(state == 0 for _, state in group):
|
|
reasons.append("none of " + ", ".join(_wanted(g) for g, _ in group))
|
|
return reasons
|
|
|
|
|
|
def score_one(candidate: Candidate, gates: Sequence[Gate], weights: Weights,
|
|
fit: float | None, now: datetime, *, taste: float | None = None,
|
|
missing: str = "no vector") -> tuple[float, str]:
|
|
"""One role for one agent: `(final score, gated_reason)`. `fit` and
|
|
`taste` are already scaled to 0..1 (see `scale`), or None when there is
|
|
nothing to measure: no role vector, no prompt vector, no votes. A None
|
|
term leaves the denominator; `missing` says why fit is out."""
|
|
if failed := failed_gates(candidate, gates):
|
|
return 0.0, ", ".join(failed)
|
|
|
|
terms = [(weights.recency, recency(candidate.posted_at, now)),
|
|
(weights.affinity, affinity_term(candidate.affinity))]
|
|
if fit is not None:
|
|
terms.append((weights.fit, fit))
|
|
if taste is not None:
|
|
terms.append((weights.taste, taste))
|
|
total = sum(w for w, _ in terms)
|
|
if not total:
|
|
# Fit was the only term with any weight, and there is no fit.
|
|
return 0.0, missing if fit is None and weights.fit else "no weights"
|
|
return sum(w * t for w, t in terms) / total, ""
|
|
|
|
|
|
def bonus(affinity: int, weights: Weights) -> float:
|
|
"""What a company's affinity adds to both levels of the cut: the share
|
|
the affinity term gives it in the final score above a neutral one, so
|
|
a liked company sits about 0.05 above a neutral one and a favourite
|
|
about 0.1 on the default weights. 0 when every weight is 0."""
|
|
total = weights.fit + weights.recency + weights.affinity + weights.taste
|
|
return weights.affinity * affinity / 3 / total if total else 0.0
|
|
|
|
|
|
def cut_of(candidates: Sequence[Candidate], passed: set[str], fits: dict[str, float],
|
|
near: dict[str, float], kept: set[str], weights: Weights,
|
|
limits: Limits) -> dict[str, str]:
|
|
"""`uid -> why` for the roles in the cut, among `candidates` whose uid
|
|
is in `passed` (the fetch gate let them through): `fit` for the best
|
|
`limits.cut_fit_top` by fit level at or above `cut_fit_min`, `cv` for
|
|
the best `cut_cv_top` by CV level at or above `cut_cv_min` not in yet,
|
|
and `kept` for those in `kept` (liked or saved) not in yet. A level is
|
|
`fits` or `near` plus the company's `bonus`; a role without one is in
|
|
no list by it. Ties go to the newer role, then the lower uid."""
|
|
by_uid = {c.uid: c for c in candidates if c.uid in passed}
|
|
# Newest first, then uid, as the sort below keeps equal levels in order.
|
|
ordered = sorted(sorted(by_uid.values(), key=lambda c: c.uid),
|
|
key=lambda c: c.posted_at, reverse=True)
|
|
cut: dict[str, str] = {}
|
|
for why, levels, top, floor in (("fit", fits, limits.cut_fit_top, limits.cut_fit_min),
|
|
("cv", near, limits.cut_cv_top, limits.cut_cv_min)):
|
|
scored = [(levels[c.uid] + bonus(c.affinity, weights), c.uid) for c in ordered
|
|
if c.uid in levels and c.uid not in cut]
|
|
scored = [(level, uid) for level, uid in scored if level >= floor]
|
|
scored.sort(key=lambda pair: pair[0], reverse=True)
|
|
for _, uid in scored[:top]:
|
|
cut[uid] = why
|
|
for c in ordered:
|
|
if c.uid in kept and c.uid not in cut:
|
|
cut[c.uid] = "kept"
|
|
return cut
|
|
|
|
|
|
# ------------------------------------------------------------------ vectors
|
|
|
|
def intent_key(model: str, intent: str) -> str:
|
|
return hashlib.sha1(f"{model}\n{intent}".encode()).hexdigest()
|
|
|
|
|
|
def cv_key(model: str, cv_text: str) -> str:
|
|
"""What the CV's vectors are cached under: the model, how the CV is cut
|
|
(`embed.CV_CHUNKING`) and the text."""
|
|
return hashlib.sha1(f"{model}\n{embed.CV_CHUNKING}\n{cv_text}".encode()).hexdigest()
|
|
|
|
|
|
def closeness(role: np.ndarray, chunks: np.ndarray) -> float:
|
|
"""The best raw cosine between a role and the CV's chunks, -1..1, with
|
|
`chunks` one unit-length row each (`_cv`); 0 for a zero role vector."""
|
|
norm = float(np.linalg.norm(role))
|
|
if norm == 0 or not len(chunks):
|
|
return 0.0
|
|
return float(np.max(chunks @ role)) / norm
|
|
|
|
|
|
class _Encoder:
|
|
"""The embedder, loaded on first use only: an agent whose prompt vectors
|
|
are cached costs no model load at all."""
|
|
|
|
def __init__(self, model: str, context: RunContext) -> None:
|
|
self.model = model
|
|
self.context = context
|
|
self._embedder: Any = None
|
|
self.failed = False
|
|
|
|
def encode(self, text: str) -> np.ndarray | None:
|
|
found = self.encode_many([text])
|
|
return None if found is None else found[0]
|
|
|
|
def encode_many(self, texts: Sequence[str]) -> np.ndarray | None:
|
|
if self.failed:
|
|
return None
|
|
if self._embedder is None:
|
|
try:
|
|
self._embedder = embed.load_embedder(self.model)
|
|
except ImportError as exc:
|
|
self.failed = True
|
|
self.context.notice(embed.unavailable(exc))
|
|
return None
|
|
return np.asarray(self._embedder.encode(list(texts)))
|
|
|
|
|
|
def _prompts(store: Store, agent: Agent, model: str,
|
|
encoder: _Encoder) -> list[tuple[str, np.ndarray]]:
|
|
"""`(name, vector)` for each of the agent's prompts that has text and
|
|
could be embedded, in order. Each vector is cached under its own key, so
|
|
an edit re-embeds only the prompt it changed."""
|
|
found = []
|
|
for prompt in agent.prompts:
|
|
if not prompt.text.strip():
|
|
continue
|
|
key = intent_key(model, prompt.text)
|
|
vec = store.prompt_vector(agent.id, prompt.name, key)
|
|
if vec is None:
|
|
vec = encoder.encode(embed.intent_text(model, prompt.text))
|
|
if vec is None:
|
|
continue
|
|
store.set_prompt_vector(agent.id, prompt.name, key, vec)
|
|
found.append((prompt.name, vec))
|
|
return found
|
|
|
|
|
|
def _cv(store: Store, agent: Agent, model: str, encoder: _Encoder) -> np.ndarray | None:
|
|
"""The agent's CV as one unit-length row per chunk (`embed.cv_chunks`),
|
|
each encoded as the prompts are ("query: " for an e5 model), or None
|
|
without a CV or an embedder. Cached under `cv_key`, so a run embeds the
|
|
CV only after it changed."""
|
|
if not agent.cv_text.strip():
|
|
return None
|
|
key = cv_key(model, agent.cv_text)
|
|
found = store.cv_vectors(agent.id, key)
|
|
if found is None:
|
|
chunks = embed.cv_chunks(agent.cv_text)
|
|
made = encoder.encode_many([embed.intent_text(model, c) for c in chunks])
|
|
if made is None or not len(made):
|
|
return None
|
|
found = list(made)
|
|
store.set_cv_vectors(agent.id, key, found)
|
|
rows = np.vstack(found).astype(np.float64)
|
|
norms = np.linalg.norm(rows, axis=1, keepdims=True)
|
|
return rows / np.where(norms == 0, 1.0, norms)
|
|
|
|
|
|
# ------------------------------------------------------------------- the job
|
|
|
|
def no_intent(names: Sequence[str]) -> Notice:
|
|
"""The notice for enabled agents without a prompt that has text: their
|
|
scores have no fit, so nothing of theirs enters the cut by fit and
|
|
their final scores do not choose what summarise reads. The code keeps
|
|
its name from when an agent had one intent."""
|
|
return Notice(
|
|
code="score.no_intent", level="warning", kind="expected", key="score",
|
|
text=f"No prompt with text, so ranked without fit and nothing in the cut by fit: "
|
|
f"{', '.join(names)}.",
|
|
fix="Give the agent a prompt under Settings, Search agents.",
|
|
where="app.jobs.score")
|
|
|
|
|
|
def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
|
|
prompts: Sequence[tuple[str, np.ndarray]], now: datetime,
|
|
calibration: Calibration | None = None,
|
|
cv: np.ndarray | None = None,
|
|
saved: set[str] | None = None) -> dict[str, int]:
|
|
"""Two passes: the cosines of the roles this agent scores, to each of its
|
|
`prompts` and to its taste direction, then every score with both scaled
|
|
by the model's `calibration`, or across the run without one. A role's fit
|
|
is the best of its prompts' levels, each scaled on its own, and the score
|
|
keeps that prompt's name; the first prompt wins a tie. Under the min-max
|
|
fallback each prompt is ranged over the run on its own too, so every
|
|
prompt's closest role gets 1 and one prompt equals the old single intent.
|
|
A gated role is left out of the cosines, so under the fallback a role
|
|
that fails a require gate cannot stretch the range the ranked roles
|
|
share. The voted roles are scored like any other; a liked one sits near
|
|
the direction it helped make, which is what a like should do. With `cv`
|
|
(`_cv`), each of those roles also gets its `cv_fit`, its `closeness` to
|
|
the CV scaled like fit. Then the cut (`cut_of`), with this agent's likes
|
|
and the `saved` roles (`Store.saved_uids`, read here when not given)
|
|
kept."""
|
|
candidates = store.score_candidates(agent.id, agent.gates)
|
|
votes = store.votes(agent.id)
|
|
direction = taste_direction(vectors, votes)
|
|
cosines: dict[str, dict[str, float]] = {name: {} for name, _ in prompts}
|
|
tastes: dict[str, float] = {}
|
|
near: dict[str, float] = {}
|
|
for candidate in candidates:
|
|
role_vec = vectors.get(candidate.uid)
|
|
if role_vec is None or failed_gates(candidate, agent.gates):
|
|
continue
|
|
for name, prompt_vec in prompts:
|
|
cosines[name][candidate.uid] = cosine(role_vec, prompt_vec)
|
|
if direction is not None:
|
|
tastes[candidate.uid] = cosine(role_vec, direction)
|
|
if cv is not None:
|
|
near[candidate.uid] = closeness(role_vec, cv)
|
|
fits: dict[str, float] = {}
|
|
best: dict[str, str] = {}
|
|
for name, levels in cosines.items():
|
|
for uid, value in scale(levels, calibration.fit if calibration else None).items():
|
|
if uid not in fits or value > fits[uid]:
|
|
fits[uid], best[uid] = value, name
|
|
tastes = scale(tastes, calibration.taste if calibration else None)
|
|
near = scale(near, calibration.cv_band if calibration else None)
|
|
scored: list[tuple[str, float, str, float | None]] = []
|
|
passed: set[str] = set()
|
|
for candidate in candidates:
|
|
fit = fits.get(candidate.uid)
|
|
value, reason = score_one(
|
|
candidate, agent.gates, agent.weights, fit, now,
|
|
taste=tastes.get(candidate.uid),
|
|
missing="no vector" if prompts else "no prompt vector")
|
|
if not reason:
|
|
passed.add(candidate.uid)
|
|
scored.append((candidate.uid, value, reason, fit))
|
|
kept = {uid for uid, vote in votes.items() if vote == 1}
|
|
kept |= store.saved_uids() if saved is None else saved
|
|
cut = cut_of(candidates, passed, fits, near, kept, agent.weights, agent.limits)
|
|
rows = []
|
|
for uid, value, reason, fit in scored:
|
|
counted = fit is not None and agent.weights.fit and not reason
|
|
cv_fit = near.get(uid)
|
|
rows.append((uid, round(value, 6), reason, best[uid] if counted else "",
|
|
None if cv_fit is None else round(cv_fit, 6),
|
|
None if fit is None else round(fit, 6), cut.get(uid, "")))
|
|
store.replace_scores(agent.id, rows)
|
|
return {"scored": len(rows), "gated": len(rows) - len(passed), "cut": len(cut)}
|
|
|
|
|
|
@job("score", "Scoring roles for every agent", solo=True, resource="cpu")
|
|
def run_score(config: Config, store: Store, *,
|
|
context: RunContext = NULL) -> dict[str, Any]:
|
|
seed_agents(config, store)
|
|
agents = store.agents(enabled_only=True)
|
|
model = config.embed.model
|
|
vectors = store.vectors(model) if agents else {}
|
|
encoder = _Encoder(model, context)
|
|
now = store.clock()
|
|
calibration = config.embed.calibrated()
|
|
if agents and calibration is None:
|
|
log.warning("no calibration for %s: fit and taste are min-max scaled across "
|
|
"each agent's roles in this run, so they rank rather than state a "
|
|
"level; add it under embed.calibration", model)
|
|
if blind := [agent.name for agent in agents
|
|
if not any(p.text.strip() for p in agent.prompts)]:
|
|
context.notice(no_intent(blind))
|
|
scored = gated = cut = 0
|
|
saved = store.saved_uids()
|
|
for n, agent in enumerate(agents):
|
|
context.check()
|
|
context.progress(n, len(agents), f"scoring for {agent.name}, {n + 1} of {len(agents)}")
|
|
counts = score_agent(store, agent, vectors, _prompts(store, agent, model, encoder),
|
|
now, calibration, cv=_cv(store, agent, model, encoder),
|
|
saved=saved)
|
|
scored += counts["scored"]
|
|
gated += counts["gated"]
|
|
cut += counts["cut"]
|
|
log.info("%-20s %6d scored, %6d gated, %5d in the cut", agent.name, counts["scored"],
|
|
counts["gated"], counts["cut"])
|
|
context.progress(len(agents), len(agents), "")
|
|
return {"agents": len(agents), "scored": scored, "gated": gated, "cut": cut,
|
|
"vectors": len(vectors)}
|
|
|
|
|
|
# ------------------------------------------------------------------ seeding
|
|
|
|
def seed_agents(config: Config, store: Store) -> list[str]:
|
|
"""Import `config.yaml`'s `agents:` into an empty table, once. The state
|
|
marker is what makes it once: deleting every agent afterwards is a
|
|
choice, and a restart must not undo it."""
|
|
if store.state.get(SEEDED) or store.agents():
|
|
return []
|
|
added = []
|
|
for seed in config.agents:
|
|
try:
|
|
store.save_agent(seed.name, seed.intent, seed.gates, seed.weights,
|
|
seed.enabled, limits=seed.limits, prompts=seed.prompts)
|
|
except ValueError as exc:
|
|
log.warning("agents: %s", exc)
|
|
continue
|
|
added.append(seed.name)
|
|
if added:
|
|
store.state[SEEDED] = store.now()
|
|
log.info("imported %d agents from %s", len(added), config.path or "the defaults")
|
|
return added
|