"""`embed`: one vector per open role, for the `fit` term of a score. The encoder is `desk.tools.embedding`; the table is roledesk's own `vectors` (see `Store.put_vectors`). What gets encoded is the title and the opening of the description, which is what says what a role is; while a role has no description, the teaser its host's list carried, and while it has neither, the title alone. `Store.upsert_role` drops the vector when a better text arrives, so the role is encoded again. sentence-transformers is optional, as in newsdesk. Without it this task puts one notice on screen and does nothing; `score` then leaves `fit` out and ranks on what it has. The default, `intfloat/multilingual-e5-small`, was picked over the English-only MiniLM once the corpus turned European: measured on the live roles against the "Copenhagen engineering" agent's intent, MiniLM separated engineering from non-engineering roles with an AUC of 0.68 (0.74 restricted to Danish-country roles), multilingual-e5-small with an AUC of 0.70 (1.00 restricted to Danish-country roles), for about 3x the CPU time. `paraphrase-multilingual- MiniLM-L12-v2`, the other multilingual candidate measured, scored *worse* than plain MiniLM (AUC 0.60) and was not chosen. """ from __future__ import annotations import logging import re from typing import Any from desk.tools import embedding from pipeline import NULL, Notice, RunContext, job from app.config import Config from app.models import Role from app.storage import Store log = logging.getLogger(__name__) #: How much of the description the encoder sees. MiniLM truncates at 256 #: tokens anyway; 2000 characters is past that for any language. BODY_CHARS = 2000 #: Roles encoded per call, and per commit: a kill loses at most one chunk. CHUNK = 64 #: intfloat's e5 family is trained with a "query: " / "passage: " prefix baked #: into the pretraining objective; encoding without it measurably hurts #: retrieval. Matched against the model id itself, not a second config field, #: so switching `embed.model` to an e5 variant can't leave a stale prefix #: setting behind it -- the one place that says which model to use is also #: the one place that decides whether it wants a prefix. Every other model #: (MiniLM and friends) has no such convention, so the match is narrow: the #: model id's own name must contain "e5" set off by "-", "_" or "/". _E5_NAME = re.compile(r"(?:^|[-_/])e5(?:[-_]|$)", re.IGNORECASE) def wants_e5_prefix(model: str) -> bool: return bool(_E5_NAME.search(model)) def document(role: Role) -> str: """The title and the description, else the teaser.""" return f"{role.title}\n{(role.description or role.teaser)[:BODY_CHARS]}" def role_text(model: str, role: Role) -> str: """What gets encoded for a role: `document(role)`, prefixed "passage: " for an e5 model since that is the side e5 calls the passage.""" text = document(role) return f"passage: {text}" if wants_e5_prefix(model) else text def intent_text(model: str, intent: str) -> str: """What gets encoded for an agent's intent: prefixed "query: " for an e5 model, since the intent is what a role is matched against, e5's "query" side. `score._intent` calls this so the cached vector and a freshly encoded one are always the same text for a given model.""" return f"query: {intent}" if wants_e5_prefix(model) else intent #: The most of a CV one vector reads. multilingual-e5-small reads 512 tokens #: and cuts the rest; on the owner's English CV a paragraph runs about four #: characters a token (1,896 characters were 471 tokens with "query: "), so #: 1,200 characters stay inside the window with room for a denser language. CV_CHUNK_CHARS = 1200 #: The version of `cv_chunks`, part of the CV vectors' cache key #: (`score.cv_key`): a change to how a CV is cut embeds every CV again. CV_CHUNKING = 1 #: Where a long paragraph is cut, coarsest first, and what joins its pieces #: again: lines, then sentences. _CUTS = ((re.compile(r"\n+"), "\n"), (re.compile(r"(?<=[.!?;])\s+"), " ")) def _pieces(text: str, limit: int) -> list[str]: """`text` cut to pieces of at most `limit` characters: by line, then by sentence, then at the last space before the limit, else at the limit.""" if len(text) <= limit: return [text] for pattern, glue in _CUTS: parts = [p.strip() for p in pattern.split(text) if p.strip()] if len(parts) > 1: return _merge([piece for part in parts for piece in _pieces(part, limit)], limit, glue) cut = text.rfind(" ", 0, limit + 1) cut = cut if cut > 0 else limit return [text[:cut].strip(), *_pieces(text[cut:].strip(), limit)] def _merge(pieces: list[str], limit: int, glue: str) -> list[str]: """Neighbouring pieces joined with `glue` while the join fits `limit`.""" merged: list[str] = [] for piece in pieces: if merged and len(merged[-1]) + len(glue) + len(piece) <= limit: merged[-1] += glue + piece else: merged.append(piece) return merged def cv_chunks(cv_text: str, limit: int | None = None) -> list[str]: """A CV as the texts its vectors are made from: its paragraphs (blank lines apart), a longer one cut by line, then by sentence, and neighbours joined again while they fit `limit` (`CV_CHUNK_CHARS`), so a heading travels with what follows it and no chunk runs past what the model reads.""" limit = limit or CV_CHUNK_CHARS pieces = [] for paragraph in re.split(r"\n\s*\n", cv_text): lines = "\n".join(" ".join(line.split()) for line in paragraph.splitlines()).strip() if lines: pieces += _pieces(lines, limit) return _merge(pieces, limit, "\n\n") def load_embedder(model: str) -> embedding.Embedder: """The encoder. A seam of its own so a test hands in a fake and nothing downloads a model; `score` loads through it too.""" return embedding.load(model) def unavailable(exc: BaseException) -> Notice: """The notice for a machine without sentence-transformers. One code and key for `embed` and `score`, so the two say it as one line.""" return Notice( code="embed.unavailable", level="warning", kind="expected", key="embed", text="sentence-transformers is not installed, so roles are not embedded " "and scores leave fit out.", fix="Install it into roledesk's environment: " "`uv pip install sentence-transformers`.", detail=str(exc), where="app.jobs.embed") @job("embed", "Embedding roles", solo=True, resource="cpu") def run_embed(config: Config, store: Store, *, context: RunContext = NULL) -> dict[str, Any]: """Embed every open role that has no vector from the configured model.""" model = config.embed.model try: embedder = load_embedder(model) except ImportError as exc: log.warning("%s", exc) context.notice(unavailable(exc)) return {"embedded": 0, "skipped": True, "reason": "sentence-transformers missing"} dropped = store.drop_vectors(keep=model) if dropped: log.info("dropped %d vectors from a previous embedding model", dropped) roles = store.roles_to_embed(model, config.embed.per_run) for at in range(0, len(roles), CHUNK): context.check() context.progress(at, len(roles), f"embedding roles, {at} of {len(roles)}") chunk = roles[at:at + CHUNK] vectors = embedder.encode([role_text(model, r) for r in chunk]) store.put_vectors(model, [(r.uid, v) for r, v in zip(chunk, vectors, strict=True)]) context.progress(len(roles), len(roles), f"embedding roles, {len(roles)} of {len(roles)}") return {"embedded": len(roles), "dropped": dropped}