"""`embed`: one vector per open role, for the `fit` term of a score. The encoder is `desk.tools.embedding`; the table is roledesk's own `vectors` (see `Store.put_vectors`). What gets encoded is the title and the opening of the description, which is what says what a role is. sentence-transformers is optional, as in newsdesk. Without it this task puts one notice on screen and does nothing; `score` then leaves `fit` out and ranks on what it has. """ from __future__ import annotations import logging from typing import Any from desk.tools import embedding from pipeline import NULL, Notice, RunContext, job from app.config import Config from app.models import Role from app.storage import Store log = logging.getLogger(__name__) #: How much of the description the encoder sees. MiniLM truncates at 256 #: tokens anyway; 2000 characters is past that for any language. BODY_CHARS = 2000 #: Roles encoded per call, and per commit: a kill loses at most one chunk. CHUNK = 64 def document(role: Role) -> str: return f"{role.title}\n{role.description[:BODY_CHARS]}" def load_embedder(model: str) -> embedding.Embedder: """The encoder. A seam of its own so a test hands in a fake and nothing downloads a model; `score` loads through it too.""" return embedding.load(model) def unavailable(exc: BaseException) -> Notice: """The notice for a machine without sentence-transformers. One code and key for `embed` and `score`, so the two say it as one line.""" return Notice( code="embed.unavailable", level="warning", kind="expected", key="embed", text="sentence-transformers is not installed, so roles are not embedded " "and scores leave fit out.", fix="Install it into roledesk's environment: " "`uv pip install sentence-transformers`.", detail=str(exc), where="app.jobs.embed") @job("embed", "Embedding roles", solo=True, resource="cpu") def run_embed(config: Config, store: Store, *, context: RunContext = NULL) -> dict[str, Any]: """Embed every open role that has no vector from the configured model.""" model = config.embed.model try: embedder = load_embedder(model) except ImportError as exc: log.warning("%s", exc) context.notice(unavailable(exc)) return {"embedded": 0, "skipped": True, "reason": "sentence-transformers missing"} dropped = store.drop_vectors(keep=model) if dropped: log.info("dropped %d vectors from a previous embedding model", dropped) roles = store.roles_to_embed(model, config.embed.per_run) for at in range(0, len(roles), CHUNK): context.check() context.progress(at, len(roles), f"embedding roles, {at} of {len(roles)}") chunk = roles[at:at + CHUNK] vectors = embedder.encode([document(r) for r in chunk]) store.put_vectors(model, [(r.uid, v) for r, v in zip(chunk, vectors, strict=True)]) context.progress(len(roles), len(roles), f"embedding roles, {len(roles)} of {len(roles)}") return {"embedded": len(roles), "dropped": dropped}