Files
jobboard/app/jobs/embed.py
T
WolfandClaude Sonnet b560cdce2a Switch the embed model to multilingual-e5-small for the European corpus
Measured on the live roles against the "Copenhagen engineering" agent's
intent: the English-only all-MiniLM-L6-v2 separates engineering from
non-engineering roles with an AUC of 0.68 overall but only 0.74 restricted
to Danish-country roles (a Danish engineering role beats the rest's
non-engineering median only 68% of the time). multilingual-e5-small reaches
0.70 overall and 1.00 on Danish-country roles, for about 3x the CPU time.
paraphrase-multilingual-MiniLM-L12-v2 measured worse than the English-only
baseline (AUC 0.60) and was dropped.

e5 needs "query: "/"passage: " prefixes; app/jobs/embed.py adds them by
matching the model name so a model change can't leave a stale prefix
setting behind. Added tests for the prefixing and for the two invariants
the model change relies on: embed drops the old model's vectors before
re-embedding, and an agent's cached intent vector is keyed by model so a
model change is a cache miss, never a stale-dimension read.

Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
2026-09-29 13:59:47 +02:00

119 lines
5.0 KiB
Python

"""`embed`: one vector per open role, for the `fit` term of a score.
The encoder is `desk.tools.embedding`; the table is roledesk's own `vectors`
(see `Store.put_vectors`). What gets encoded is the title and the opening of
the description, which is what says what a role is.
sentence-transformers is optional, as in newsdesk. Without it this task puts
one notice on screen and does nothing; `score` then leaves `fit` out and ranks
on what it has.
The default, `intfloat/multilingual-e5-small`, was picked over the English-only
MiniLM once the corpus turned European: measured on the live roles against the
"Copenhagen engineering" agent's intent, MiniLM separated engineering from
non-engineering roles with an AUC of 0.68 (0.74 restricted to Danish-country
roles), multilingual-e5-small with an AUC of 0.70 (1.00 restricted to
Danish-country roles), for about 3x the CPU time. `paraphrase-multilingual-
MiniLM-L12-v2`, the other multilingual candidate measured, scored *worse*
than plain MiniLM (AUC 0.60) and was not chosen.
"""
from __future__ import annotations
import logging
import re
from typing import Any
from desk.tools import embedding
from pipeline import NULL, Notice, RunContext, job
from app.config import Config
from app.models import Role
from app.storage import Store
log = logging.getLogger(__name__)
#: How much of the description the encoder sees. MiniLM truncates at 256
#: tokens anyway; 2000 characters is past that for any language.
BODY_CHARS = 2000
#: Roles encoded per call, and per commit: a kill loses at most one chunk.
CHUNK = 64
#: intfloat's e5 family is trained with a "query: " / "passage: " prefix baked
#: into the pretraining objective; encoding without it measurably hurts
#: retrieval. Matched against the model id itself, not a second config field,
#: so switching `embed.model` to an e5 variant can't leave a stale prefix
#: setting behind it -- the one place that says which model to use is also
#: the one place that decides whether it wants a prefix. Every other model
#: (MiniLM and friends) has no such convention, so the match is narrow: the
#: model id's own name must contain "e5" set off by "-", "_" or "/".
_E5_NAME = re.compile(r"(?:^|[-_/])e5(?:[-_]|$)", re.IGNORECASE)
def wants_e5_prefix(model: str) -> bool:
return bool(_E5_NAME.search(model))
def document(role: Role) -> str:
return f"{role.title}\n{role.description[:BODY_CHARS]}"
def role_text(model: str, role: Role) -> str:
"""What gets encoded for a role: `document(role)`, prefixed "passage: "
for an e5 model since that is the side e5 calls the passage."""
text = document(role)
return f"passage: {text}" if wants_e5_prefix(model) else text
def intent_text(model: str, intent: str) -> str:
"""What gets encoded for an agent's intent: prefixed "query: " for an e5
model, since the intent is what a role is matched against, e5's "query"
side. `score._intent` calls this so the cached vector and a freshly
encoded one are always the same text for a given model."""
return f"query: {intent}" if wants_e5_prefix(model) else intent
def load_embedder(model: str) -> embedding.Embedder:
"""The encoder. A seam of its own so a test hands in a fake and nothing
downloads a model; `score` loads through it too."""
return embedding.load(model)
def unavailable(exc: BaseException) -> Notice:
"""The notice for a machine without sentence-transformers. One code and
key for `embed` and `score`, so the two say it as one line."""
return Notice(
code="embed.unavailable", level="warning", kind="expected", key="embed",
text="sentence-transformers is not installed, so roles are not embedded "
"and scores leave fit out.",
fix="Install it into roledesk's environment: "
"`uv pip install sentence-transformers`.",
detail=str(exc), where="app.jobs.embed")
@job("embed", "Embedding roles", solo=True, resource="cpu")
def run_embed(config: Config, store: Store, *,
context: RunContext = NULL) -> dict[str, Any]:
"""Embed every open role that has no vector from the configured model."""
model = config.embed.model
try:
embedder = load_embedder(model)
except ImportError as exc:
log.warning("%s", exc)
context.notice(unavailable(exc))
return {"embedded": 0, "skipped": True, "reason": "sentence-transformers missing"}
dropped = store.drop_vectors(keep=model)
if dropped:
log.info("dropped %d vectors from a previous embedding model", dropped)
roles = store.roles_to_embed(model, config.embed.per_run)
for at in range(0, len(roles), CHUNK):
context.check()
context.progress(at, len(roles), f"embedding roles, {at} of {len(roles)}")
chunk = roles[at:at + CHUNK]
vectors = embedder.encode([role_text(model, r) for r in chunk])
store.put_vectors(model, [(r.uid, v) for r, v in zip(chunk, vectors, strict=True)])
context.progress(len(roles), len(roles),
f"embedding roles, {len(roles)} of {len(roles)}")
return {"embedded": len(roles), "dropped": dropped}