Measured on the live roles against the "Copenhagen engineering" agent's intent: the English-only all-MiniLM-L6-v2 separates engineering from non-engineering roles with an AUC of 0.68 overall but only 0.74 restricted to Danish-country roles (a Danish engineering role beats the rest's non-engineering median only 68% of the time). multilingual-e5-small reaches 0.70 overall and 1.00 on Danish-country roles, for about 3x the CPU time. paraphrase-multilingual-MiniLM-L12-v2 measured worse than the English-only baseline (AUC 0.60) and was dropped. e5 needs "query: "/"passage: " prefixes; app/jobs/embed.py adds them by matching the model name so a model change can't leave a stale prefix setting behind. Added tests for the prefixing and for the two invariants the model change relies on: embed drops the old model's vectors before re-embedding, and an agent's cached intent vector is keyed by model so a model change is a cache miss, never a stale-dimension read. Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
119 lines
5.0 KiB
Python
119 lines
5.0 KiB
Python
"""`embed`: one vector per open role, for the `fit` term of a score.
|
|
|
|
The encoder is `desk.tools.embedding`; the table is roledesk's own `vectors`
|
|
(see `Store.put_vectors`). What gets encoded is the title and the opening of
|
|
the description, which is what says what a role is.
|
|
|
|
sentence-transformers is optional, as in newsdesk. Without it this task puts
|
|
one notice on screen and does nothing; `score` then leaves `fit` out and ranks
|
|
on what it has.
|
|
|
|
The default, `intfloat/multilingual-e5-small`, was picked over the English-only
|
|
MiniLM once the corpus turned European: measured on the live roles against the
|
|
"Copenhagen engineering" agent's intent, MiniLM separated engineering from
|
|
non-engineering roles with an AUC of 0.68 (0.74 restricted to Danish-country
|
|
roles), multilingual-e5-small with an AUC of 0.70 (1.00 restricted to
|
|
Danish-country roles), for about 3x the CPU time. `paraphrase-multilingual-
|
|
MiniLM-L12-v2`, the other multilingual candidate measured, scored *worse*
|
|
than plain MiniLM (AUC 0.60) and was not chosen.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import re
|
|
from typing import Any
|
|
|
|
from desk.tools import embedding
|
|
from pipeline import NULL, Notice, RunContext, job
|
|
from app.config import Config
|
|
from app.models import Role
|
|
from app.storage import Store
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: How much of the description the encoder sees. MiniLM truncates at 256
|
|
#: tokens anyway; 2000 characters is past that for any language.
|
|
BODY_CHARS = 2000
|
|
|
|
#: Roles encoded per call, and per commit: a kill loses at most one chunk.
|
|
CHUNK = 64
|
|
|
|
#: intfloat's e5 family is trained with a "query: " / "passage: " prefix baked
|
|
#: into the pretraining objective; encoding without it measurably hurts
|
|
#: retrieval. Matched against the model id itself, not a second config field,
|
|
#: so switching `embed.model` to an e5 variant can't leave a stale prefix
|
|
#: setting behind it -- the one place that says which model to use is also
|
|
#: the one place that decides whether it wants a prefix. Every other model
|
|
#: (MiniLM and friends) has no such convention, so the match is narrow: the
|
|
#: model id's own name must contain "e5" set off by "-", "_" or "/".
|
|
_E5_NAME = re.compile(r"(?:^|[-_/])e5(?:[-_]|$)", re.IGNORECASE)
|
|
|
|
|
|
def wants_e5_prefix(model: str) -> bool:
|
|
return bool(_E5_NAME.search(model))
|
|
|
|
|
|
def document(role: Role) -> str:
|
|
return f"{role.title}\n{role.description[:BODY_CHARS]}"
|
|
|
|
|
|
def role_text(model: str, role: Role) -> str:
|
|
"""What gets encoded for a role: `document(role)`, prefixed "passage: "
|
|
for an e5 model since that is the side e5 calls the passage."""
|
|
text = document(role)
|
|
return f"passage: {text}" if wants_e5_prefix(model) else text
|
|
|
|
|
|
def intent_text(model: str, intent: str) -> str:
|
|
"""What gets encoded for an agent's intent: prefixed "query: " for an e5
|
|
model, since the intent is what a role is matched against, e5's "query"
|
|
side. `score._intent` calls this so the cached vector and a freshly
|
|
encoded one are always the same text for a given model."""
|
|
return f"query: {intent}" if wants_e5_prefix(model) else intent
|
|
|
|
|
|
def load_embedder(model: str) -> embedding.Embedder:
|
|
"""The encoder. A seam of its own so a test hands in a fake and nothing
|
|
downloads a model; `score` loads through it too."""
|
|
return embedding.load(model)
|
|
|
|
|
|
def unavailable(exc: BaseException) -> Notice:
|
|
"""The notice for a machine without sentence-transformers. One code and
|
|
key for `embed` and `score`, so the two say it as one line."""
|
|
return Notice(
|
|
code="embed.unavailable", level="warning", kind="expected", key="embed",
|
|
text="sentence-transformers is not installed, so roles are not embedded "
|
|
"and scores leave fit out.",
|
|
fix="Install it into roledesk's environment: "
|
|
"`uv pip install sentence-transformers`.",
|
|
detail=str(exc), where="app.jobs.embed")
|
|
|
|
|
|
@job("embed", "Embedding roles", solo=True, resource="cpu")
|
|
def run_embed(config: Config, store: Store, *,
|
|
context: RunContext = NULL) -> dict[str, Any]:
|
|
"""Embed every open role that has no vector from the configured model."""
|
|
model = config.embed.model
|
|
try:
|
|
embedder = load_embedder(model)
|
|
except ImportError as exc:
|
|
log.warning("%s", exc)
|
|
context.notice(unavailable(exc))
|
|
return {"embedded": 0, "skipped": True, "reason": "sentence-transformers missing"}
|
|
|
|
dropped = store.drop_vectors(keep=model)
|
|
if dropped:
|
|
log.info("dropped %d vectors from a previous embedding model", dropped)
|
|
roles = store.roles_to_embed(model, config.embed.per_run)
|
|
for at in range(0, len(roles), CHUNK):
|
|
context.check()
|
|
context.progress(at, len(roles), f"embedding roles, {at} of {len(roles)}")
|
|
chunk = roles[at:at + CHUNK]
|
|
vectors = embedder.encode([role_text(model, r) for r in chunk])
|
|
store.put_vectors(model, [(r.uid, v) for r, v in zip(chunk, vectors, strict=True)])
|
|
context.progress(len(roles), len(roles),
|
|
f"embedding roles, {len(roles)} of {len(roles)}")
|
|
return {"embedded": len(roles), "dropped": dropped}
|