Score each role's closeness to the agent's CV, let the judge also read the 50 closest the score leaves out, and order CV match's unjudged roles by it

The CV is cut into chunks inside e5's window, embedded like a prompt and
cached per agent under a hash of the model, chunking and text. A role's
cv_fit (scores.cv_fit, migration 31) is its best chunk cosine, scaled with
its own calibration band: fit's clipped 88 of agent 2's ungated roles to 1.
It is no term of the Agent match score. judge_cv_top (default 50) sits
beside judge_top and judge_min on the agents page and card.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-10-03 23:35:21 +02:00
1 parent b481cb5692
commit c23594507a
23 files changed
+603 -85

No files matched your search

+56
View File
@@ -77,6 +77,62 @@ def intent_text(model: str, intent: str) -> str:
return f"query: {intent}" if wants_e5_prefix(model) else intent
#: The most of a CV one vector reads. multilingual-e5-small reads 512 tokens
#: and cuts the rest; on the owner's English CV a paragraph runs about four
#: characters a token (1,896 characters were 471 tokens with "query: "), so
#: 1,200 characters stay inside the window with room for a denser language.
CV_CHUNK_CHARS = 1200
#: The version of `cv_chunks`, part of the CV vectors' cache key
#: (`score.cv_key`): a change to how a CV is cut embeds every CV again.
CV_CHUNKING = 1
#: Where a long paragraph is cut, coarsest first, and what joins its pieces
#: again: lines, then sentences.
_CUTS = ((re.compile(r"\n+"), "\n"), (re.compile(r"(?<=[.!?;])\s+"), " "))
def _pieces(text: str, limit: int) -> list[str]:
"""`text` cut to pieces of at most `limit` characters: by line, then by
sentence, then at the last space before the limit, else at the limit."""
if len(text) <= limit:
return [text]
for pattern, glue in _CUTS:
parts = [p.strip() for p in pattern.split(text) if p.strip()]
if len(parts) > 1:
return _merge([piece for part in parts for piece in _pieces(part, limit)],
limit, glue)
cut = text.rfind(" ", 0, limit + 1)
cut = cut if cut > 0 else limit
return [text[:cut].strip(), *_pieces(text[cut:].strip(), limit)]
def _merge(pieces: list[str], limit: int, glue: str) -> list[str]:
"""Neighbouring pieces joined with `glue` while the join fits `limit`."""
merged: list[str] = []
for piece in pieces:
if merged and len(merged[-1]) + len(glue) + len(piece) <= limit:
merged[-1] += glue + piece
else:
merged.append(piece)
return merged
def cv_chunks(cv_text: str, limit: int | None = None) -> list[str]:
"""A CV as the texts its vectors are made from: its paragraphs (blank
lines apart), a longer one cut by line, then by sentence, and neighbours
joined again while they fit `limit` (`CV_CHUNK_CHARS`), so a heading
travels with what follows it and no chunk runs past what the model
reads."""
limit = limit or CV_CHUNK_CHARS
pieces = []
for paragraph in re.split(r"\n\s*\n", cv_text):
lines = "\n".join(" ".join(line.split()) for line in paragraph.splitlines()).strip()
if lines:
pieces += _pieces(lines, limit)
return _merge(pieces, limit, "\n\n")
def load_embedder(model: str) -> embedding.Embedder:
"""The encoder. A seam of its own so a test hands in a fake and nothing
downloads a model; `score` loads through it too."""
+4 -3
View File
@@ -14,9 +14,10 @@ is guessed from ad text, the second is what the gates do.
The work is pairs, `(agent, role)`: an enabled agent with a CV, and a role
among its best ungated scores as deep and as low as its own `limits` say
(`judge_top`, `judge_min`) with some text, that has no judgement yet, or one
read from another CV, prompts, like, avoid or must text or role text than today's
(`input_key`). A pair is named by the composite key `"<agent_id>:<uid>"`,
(`judge_top`, `judge_min`), or among the `judge_cv_top` ungated roles closest
to its CV (`scores.cv_fit`) that those leave out, with some text, that has
no judgement yet, or one read from another CV, prompts, like, avoid or must
text or role text than today's (`input_key`). A pair is named by the composite key `"<agent_id>:<uid>"`,
which is what `desk.modeltask` hands round as a uid, so `desk/` is untouched.
A batch holds one agent's pairs, since the profile is in the system prompt;
the finder cuts per agent (`_finder`) and the job runs one call per agent it
+61 -6
View File
@@ -29,6 +29,12 @@ roles out of extract's reach for good.
over this agent's votes; with only likes it is the liked mean,
with only passes the negated passed mean; no votes, no term
Beside the score, for an agent with a CV, `cv_fit`: how close the role is to
the CV, the best cosine between the role and the CV's chunks
(`embed.cv_chunks`, embedded as the prompts are), scaled like fit. It is no
term of the score; the judge reads the closest roles the score leaves out
(`Store.judge_rows`) and CV match orders the roles not judged yet by it.
Every term is 0..1. The score is the weighted sum over the sum of the weights
(the agent's four and its prefer gates'), so it stays 0..1. A favourite
company (`affinity = 2`) skips the `any` group but not the `require` or
@@ -208,6 +214,21 @@ def intent_key(model: str, intent: str) -> str:
return hashlib.sha1(f"{model}\n{intent}".encode()).hexdigest()
def cv_key(model: str, cv_text: str) -> str:
"""What the CV's vectors are cached under: the model, how the CV is cut
(`embed.CV_CHUNKING`) and the text."""
return hashlib.sha1(f"{model}\n{embed.CV_CHUNKING}\n{cv_text}".encode()).hexdigest()
def closeness(role: np.ndarray, chunks: np.ndarray) -> float:
"""The best raw cosine between a role and the CV's chunks, -1..1, with
`chunks` one unit-length row each (`_cv`); 0 for a zero role vector."""
norm = float(np.linalg.norm(role))
if norm == 0 or not len(chunks):
return 0.0
return float(np.max(chunks @ role)) / norm
class _Encoder:
"""The embedder, loaded on first use only: an agent whose prompt vectors
are cached costs no model load at all."""
@@ -219,6 +240,10 @@ class _Encoder:
self.failed = False
def encode(self, text: str) -> np.ndarray | None:
found = self.encode_many([text])
return None if found is None else found[0]
def encode_many(self, texts: Sequence[str]) -> np.ndarray | None:
if self.failed:
return None
if self._embedder is None:
@@ -228,7 +253,7 @@ class _Encoder:
self.failed = True
self.context.notice(embed.unavailable(exc))
return None
return np.asarray(self._embedder.encode([text])[0])
return np.asarray(self._embedder.encode(list(texts)))
def _prompts(store: Store, agent: Agent, model: str,
@@ -251,6 +276,27 @@ def _prompts(store: Store, agent: Agent, model: str,
return found
def _cv(store: Store, agent: Agent, model: str, encoder: _Encoder) -> np.ndarray | None:
"""The agent's CV as one unit-length row per chunk (`embed.cv_chunks`),
each encoded as the prompts are ("query: " for an e5 model), or None
without a CV or an embedder. Cached under `cv_key`, so a run embeds the
CV only after it changed."""
if not agent.cv_text.strip():
return None
key = cv_key(model, agent.cv_text)
found = store.cv_vectors(agent.id, key)
if found is None:
chunks = embed.cv_chunks(agent.cv_text)
made = encoder.encode_many([embed.intent_text(model, c) for c in chunks])
if made is None or not len(made):
return None
found = list(made)
store.set_cv_vectors(agent.id, key, found)
rows = np.vstack(found).astype(np.float64)
norms = np.linalg.norm(rows, axis=1, keepdims=True)
return rows / np.where(norms == 0, 1.0, norms)
# ------------------------------------------------------------------- the job
def no_intent(names: Sequence[str]) -> Notice:
@@ -267,7 +313,8 @@ def no_intent(names: Sequence[str]) -> Notice:
def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
prompts: Sequence[tuple[str, np.ndarray]], now: datetime,
calibration: Calibration | None = None) -> dict[str, int]:
calibration: Calibration | None = None,
cv: np.ndarray | None = None) -> dict[str, int]:
"""Two passes: the cosines of the roles this agent scores, to each of its
`prompts` and to its taste direction, then every score with both scaled
by the model's `calibration`, or across the run without one. A role's fit
@@ -278,11 +325,14 @@ def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
A gated role is left out of the cosines, so under the fallback a role
that fails a require gate cannot stretch the range the ranked roles
share. The voted roles are scored like any other; a liked one sits near
the direction it helped make, which is what a like should do."""
the direction it helped make, which is what a like should do. With `cv`
(`_cv`), each of those roles also gets its `cv_fit`, its `closeness` to
the CV scaled like fit, beside the score and not in it."""
candidates = store.score_candidates(agent.id, agent.gates)
direction = taste_direction(vectors, store.votes(agent.id))
cosines: dict[str, dict[str, float]] = {name: {} for name, _ in prompts}
tastes: dict[str, float] = {}
near: dict[str, float] = {}
for candidate in candidates:
role_vec = vectors.get(candidate.uid)
if role_vec is None or failed_gates(candidate, agent.gates):
@@ -291,6 +341,8 @@ def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
cosines[name][candidate.uid] = cosine(role_vec, prompt_vec)
if direction is not None:
tastes[candidate.uid] = cosine(role_vec, direction)
if cv is not None:
near[candidate.uid] = closeness(role_vec, cv)
fits: dict[str, float] = {}
best: dict[str, str] = {}
for name, levels in cosines.items():
@@ -298,7 +350,8 @@ def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
if uid not in fits or value > fits[uid]:
fits[uid], best[uid] = value, name
tastes = scale(tastes, calibration.taste if calibration else None)
rows: list[tuple[str, float, str, str]] = []
near = scale(near, calibration.cv_band if calibration else None)
rows: list[tuple[str, float, str, str, float | None]] = []
gated = 0
for candidate in candidates:
fit = fits.get(candidate.uid)
@@ -308,8 +361,10 @@ def score_agent(store: Store, agent: Agent, vectors: dict[str, np.ndarray],
missing="no vector" if prompts else "no prompt vector")
gated += bool(reason)
counted = fit is not None and agent.weights.fit and not reason
cv_fit = near.get(candidate.uid)
rows.append((candidate.uid, round(value, 6), reason,
best[candidate.uid] if counted else ""))
best[candidate.uid] if counted else "",
None if cv_fit is None else round(cv_fit, 6)))
store.replace_scores(agent.id, rows)
return {"scored": len(rows), "gated": gated}
@@ -336,7 +391,7 @@ def run_score(config: Config, store: Store, *,
context.check()
context.progress(n, len(agents), f"scoring for {agent.name}, {n + 1} of {len(agents)}")
counts = score_agent(store, agent, vectors, _prompts(store, agent, model, encoder),
now, calibration)
now, calibration, cv=_cv(store, agent, model, encoder))
scored += counts["scored"]
gated += counts["gated"]
log.info("%-20s %6d scored, %6d gated", agent.name, counts["scored"], counts["gated"])