"""`embed`: one vector per open role, for the `fit` term of a score. The encoder is `desk.tools.embedding`; the table is roledesk's own `vectors` (see `Store.put_vectors`). What gets encoded is the title and the opening of the description, which is what says what a role is; while a role has no description, the teaser its host's list carried, and while it has neither, the title alone. `Store.upsert_role` drops the vector when a better text arrives, so the role is encoded again. sentence-transformers is optional, as in newsdesk. Without it this task puts one notice on screen and does nothing; `score` then leaves `fit` out and ranks on what it has. The default, `intfloat/multilingual-e5-small`, was picked over the English-only MiniLM once the corpus turned European: measured on the live roles against the "Copenhagen engineering" agent's intent, MiniLM separated engineering from non-engineering roles with an AUC of 0.68 (0.74 restricted to Danish-country roles), multilingual-e5-small with an AUC of 0.70 (1.00 restricted to Danish-country roles), for about 3x the CPU time. `paraphrase-multilingual- MiniLM-L12-v2`, the other multilingual candidate measured, scored *worse* than plain MiniLM (AUC 0.60) and was not chosen. """ from __future__ import annotations import logging import re from typing import Any from desk.tools import embedding from pipeline import NULL, Notice, RunContext, job from app.config import Config from app.models import Role from app.storage import Store log = logging.getLogger(__name__) #: How much of the description the encoder sees. MiniLM truncates at 256 #: tokens anyway; 2000 characters is past that for any language. BODY_CHARS = 2000 #: Roles encoded per call, and per commit: a kill loses at most one chunk. CHUNK = 64 #: intfloat's e5 family is trained with a "query: " / "passage: " prefix baked #: into the pretraining objective; encoding without it measurably hurts #: retrieval. Matched against the model id itself, not a second config field, #: so switching `embed.model` to an e5 variant can't leave a stale prefix #: setting behind it -- the one place that says which model to use is also #: the one place that decides whether it wants a prefix. Every other model #: (MiniLM and friends) has no such convention, so the match is narrow: the #: model id's own name must contain "e5" set off by "-", "_" or "/". _E5_NAME = re.compile(r"(?:^|[-_/])e5(?:[-_]|$)", re.IGNORECASE) def wants_e5_prefix(model: str) -> bool: return bool(_E5_NAME.search(model)) def document(role: Role) -> str: """The title and the description, else the teaser.""" return f"{role.title}\n{(role.description or role.teaser)[:BODY_CHARS]}" def role_text(model: str, role: Role) -> str: """What gets encoded for a role: `document(role)`, prefixed "passage: " for an e5 model since that is the side e5 calls the passage.""" text = document(role) return f"passage: {text}" if wants_e5_prefix(model) else text def intent_text(model: str, intent: str) -> str: """What gets encoded for an agent's intent: prefixed "query: " for an e5 model, since the intent is what a role is matched against, e5's "query" side. `score._intent` calls this so the cached vector and a freshly encoded one are always the same text for a given model.""" return f"query: {intent}" if wants_e5_prefix(model) else intent def load_embedder(model: str) -> embedding.Embedder: """The encoder. A seam of its own so a test hands in a fake and nothing downloads a model; `score` loads through it too.""" return embedding.load(model) def unavailable(exc: BaseException) -> Notice: """The notice for a machine without sentence-transformers. One code and key for `embed` and `score`, so the two say it as one line.""" return Notice( code="embed.unavailable", level="warning", kind="expected", key="embed", text="sentence-transformers is not installed, so roles are not embedded " "and scores leave fit out.", fix="Install it into roledesk's environment: " "`uv pip install sentence-transformers`.", detail=str(exc), where="app.jobs.embed") @job("embed", "Embedding roles", solo=True, resource="cpu") def run_embed(config: Config, store: Store, *, context: RunContext = NULL) -> dict[str, Any]: """Embed every open role that has no vector from the configured model.""" model = config.embed.model try: embedder = load_embedder(model) except ImportError as exc: log.warning("%s", exc) context.notice(unavailable(exc)) return {"embedded": 0, "skipped": True, "reason": "sentence-transformers missing"} dropped = store.drop_vectors(keep=model) if dropped: log.info("dropped %d vectors from a previous embedding model", dropped) roles = store.roles_to_embed(model, config.embed.per_run) for at in range(0, len(roles), CHUNK): context.check() context.progress(at, len(roles), f"embedding roles, {at} of {len(roles)}") chunk = roles[at:at + CHUNK] vectors = embedder.encode([role_text(model, r) for r in chunk]) store.put_vectors(model, [(r.uid, v) for r, v in zip(chunk, vectors, strict=True)]) context.progress(len(roles), len(roles), f"embedding roles, {len(roles)} of {len(roles)}") return {"embedded": len(roles), "dropped": dropped}