Files
WolfandClaude Opus 5.5 c23594507a Score each role's closeness to the agent's CV, let the judge also read the 50 closest the score leaves out, and order CV match's unjudged roles by it
The CV is cut into chunks inside e5's window, embedded like a prompt and
cached per agent under a hash of the model, chunking and text. A role's
cv_fit (scores.cv_fit, migration 31) is its best chunk cosine, scaled with
its own calibration band: fit's clipped 88 of agent 2's ungated roles to 1.
It is no term of the Agent match score. judge_cv_top (default 50) sits
beside judge_top and judge_min on the agents page and card.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-03 23:35:21 +02:00

179 lines
7.6 KiB
Python

"""`embed`: one vector per open role, for the `fit` term of a score.
The encoder is `desk.tools.embedding`; the table is roledesk's own `vectors`
(see `Store.put_vectors`). What gets encoded is the title and the opening of
the description, which is what says what a role is; while a role has no
description, the teaser its host's list carried, and while it has neither,
the title alone. `Store.upsert_role` drops the vector when a better text
arrives, so the role is encoded again.
sentence-transformers is optional, as in newsdesk. Without it this task puts
one notice on screen and does nothing; `score` then leaves `fit` out and ranks
on what it has.
The default, `intfloat/multilingual-e5-small`, was picked over the English-only
MiniLM once the corpus turned European: measured on the live roles against the
"Copenhagen engineering" agent's intent, MiniLM separated engineering from
non-engineering roles with an AUC of 0.68 (0.74 restricted to Danish-country
roles), multilingual-e5-small with an AUC of 0.70 (1.00 restricted to
Danish-country roles), for about 3x the CPU time. `paraphrase-multilingual-
MiniLM-L12-v2`, the other multilingual candidate measured, scored *worse*
than plain MiniLM (AUC 0.60) and was not chosen.
"""
from __future__ import annotations
import logging
import re
from typing import Any
from desk.tools import embedding
from pipeline import NULL, Notice, RunContext, job
from app.config import Config
from app.models import Role
from app.storage import Store
log = logging.getLogger(__name__)
#: How much of the description the encoder sees. MiniLM truncates at 256
#: tokens anyway; 2000 characters is past that for any language.
BODY_CHARS = 2000
#: Roles encoded per call, and per commit: a kill loses at most one chunk.
CHUNK = 64
#: intfloat's e5 family is trained with a "query: " / "passage: " prefix baked
#: into the pretraining objective; encoding without it measurably hurts
#: retrieval. Matched against the model id itself, not a second config field,
#: so switching `embed.model` to an e5 variant can't leave a stale prefix
#: setting behind it -- the one place that says which model to use is also
#: the one place that decides whether it wants a prefix. Every other model
#: (MiniLM and friends) has no such convention, so the match is narrow: the
#: model id's own name must contain "e5" set off by "-", "_" or "/".
_E5_NAME = re.compile(r"(?:^|[-_/])e5(?:[-_]|$)", re.IGNORECASE)
def wants_e5_prefix(model: str) -> bool:
return bool(_E5_NAME.search(model))
def document(role: Role) -> str:
"""The title and the description, else the teaser."""
return f"{role.title}\n{(role.description or role.teaser)[:BODY_CHARS]}"
def role_text(model: str, role: Role) -> str:
"""What gets encoded for a role: `document(role)`, prefixed "passage: "
for an e5 model since that is the side e5 calls the passage."""
text = document(role)
return f"passage: {text}" if wants_e5_prefix(model) else text
def intent_text(model: str, intent: str) -> str:
"""What gets encoded for an agent's intent: prefixed "query: " for an e5
model, since the intent is what a role is matched against, e5's "query"
side. `score._intent` calls this so the cached vector and a freshly
encoded one are always the same text for a given model."""
return f"query: {intent}" if wants_e5_prefix(model) else intent
#: The most of a CV one vector reads. multilingual-e5-small reads 512 tokens
#: and cuts the rest; on the owner's English CV a paragraph runs about four
#: characters a token (1,896 characters were 471 tokens with "query: "), so
#: 1,200 characters stay inside the window with room for a denser language.
CV_CHUNK_CHARS = 1200
#: The version of `cv_chunks`, part of the CV vectors' cache key
#: (`score.cv_key`): a change to how a CV is cut embeds every CV again.
CV_CHUNKING = 1
#: Where a long paragraph is cut, coarsest first, and what joins its pieces
#: again: lines, then sentences.
_CUTS = ((re.compile(r"\n+"), "\n"), (re.compile(r"(?<=[.!?;])\s+"), " "))
def _pieces(text: str, limit: int) -> list[str]:
"""`text` cut to pieces of at most `limit` characters: by line, then by
sentence, then at the last space before the limit, else at the limit."""
if len(text) <= limit:
return [text]
for pattern, glue in _CUTS:
parts = [p.strip() for p in pattern.split(text) if p.strip()]
if len(parts) > 1:
return _merge([piece for part in parts for piece in _pieces(part, limit)],
limit, glue)
cut = text.rfind(" ", 0, limit + 1)
cut = cut if cut > 0 else limit
return [text[:cut].strip(), *_pieces(text[cut:].strip(), limit)]
def _merge(pieces: list[str], limit: int, glue: str) -> list[str]:
"""Neighbouring pieces joined with `glue` while the join fits `limit`."""
merged: list[str] = []
for piece in pieces:
if merged and len(merged[-1]) + len(glue) + len(piece) <= limit:
merged[-1] += glue + piece
else:
merged.append(piece)
return merged
def cv_chunks(cv_text: str, limit: int | None = None) -> list[str]:
"""A CV as the texts its vectors are made from: its paragraphs (blank
lines apart), a longer one cut by line, then by sentence, and neighbours
joined again while they fit `limit` (`CV_CHUNK_CHARS`), so a heading
travels with what follows it and no chunk runs past what the model
reads."""
limit = limit or CV_CHUNK_CHARS
pieces = []
for paragraph in re.split(r"\n\s*\n", cv_text):
lines = "\n".join(" ".join(line.split()) for line in paragraph.splitlines()).strip()
if lines:
pieces += _pieces(lines, limit)
return _merge(pieces, limit, "\n\n")
def load_embedder(model: str) -> embedding.Embedder:
"""The encoder. A seam of its own so a test hands in a fake and nothing
downloads a model; `score` loads through it too."""
return embedding.load(model)
def unavailable(exc: BaseException) -> Notice:
"""The notice for a machine without sentence-transformers. One code and
key for `embed` and `score`, so the two say it as one line."""
return Notice(
code="embed.unavailable", level="warning", kind="expected", key="embed",
text="sentence-transformers is not installed, so roles are not embedded "
"and scores leave fit out.",
fix="Install it into roledesk's environment: "
"`uv pip install sentence-transformers`.",
detail=str(exc), where="app.jobs.embed")
@job("embed", "Embedding roles", solo=True, resource="cpu")
def run_embed(config: Config, store: Store, *,
context: RunContext = NULL) -> dict[str, Any]:
"""Embed every open role that has no vector from the configured model."""
model = config.embed.model
try:
embedder = load_embedder(model)
except ImportError as exc:
log.warning("%s", exc)
context.notice(unavailable(exc))
return {"embedded": 0, "skipped": True, "reason": "sentence-transformers missing"}
dropped = store.drop_vectors(keep=model)
if dropped:
log.info("dropped %d vectors from a previous embedding model", dropped)
roles = store.roles_to_embed(model, config.embed.per_run)
for at in range(0, len(roles), CHUNK):
context.check()
context.progress(at, len(roles), f"embedding roles, {at} of {len(roles)}")
chunk = roles[at:at + CHUNK]
vectors = embedder.encode([role_text(model, r) for r in chunk])
store.put_vectors(model, [(r.uid, v) for r, v in zip(chunk, vectors, strict=True)])
context.progress(len(roles), len(roles),
f"embedding roles, {len(roles)} of {len(roles)}")
return {"embedded": len(roles), "dropped": dropped}