Migration 19 adds places, place_aliases, place_includes (seeded from the dictionaries in app/models.py and app/places.py, Europe as a continent over the European countries) and role_places, one row per place of a role, resolved by app.places.Gazetteer, NULL when unknown. upsert_role writes them from raw["places"], else the place line cut at ";"; the migration builds them from each role's place column. The Place filter and the country and place gates are joins over the tree: country:DK is any place under Denmark, region:skane any town in Skane, a country gate is met by any place in its countries, and the EU fallback of the word matcher is no longer used. A place: token the gazetteer does not know still matches whole words. Store gains gazetteer, role_places, unresolved_places, add_place_alias, add_place and relink_places. roles.teaser is kept-filled and embedded while there is no description; a vector is dropped when a better text arrives. seniority_model, _stated and _title sit behind roles.seniority, which stays the first filled of them. TRUSTED_POSTED_HOSTS gives way to trusted_posted(host), which reads READS.fields.posted and falls back to the old set for undeclared readers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
123 lines
5.3 KiB
Python
123 lines
5.3 KiB
Python
"""`embed`: one vector per open role, for the `fit` term of a score.
|
|
|
|
The encoder is `desk.tools.embedding`; the table is roledesk's own `vectors`
|
|
(see `Store.put_vectors`). What gets encoded is the title and the opening of
|
|
the description, which is what says what a role is; while a role has no
|
|
description, the teaser its host's list carried, and while it has neither,
|
|
the title alone. `Store.upsert_role` drops the vector when a better text
|
|
arrives, so the role is encoded again.
|
|
|
|
sentence-transformers is optional, as in newsdesk. Without it this task puts
|
|
one notice on screen and does nothing; `score` then leaves `fit` out and ranks
|
|
on what it has.
|
|
|
|
The default, `intfloat/multilingual-e5-small`, was picked over the English-only
|
|
MiniLM once the corpus turned European: measured on the live roles against the
|
|
"Copenhagen engineering" agent's intent, MiniLM separated engineering from
|
|
non-engineering roles with an AUC of 0.68 (0.74 restricted to Danish-country
|
|
roles), multilingual-e5-small with an AUC of 0.70 (1.00 restricted to
|
|
Danish-country roles), for about 3x the CPU time. `paraphrase-multilingual-
|
|
MiniLM-L12-v2`, the other multilingual candidate measured, scored *worse*
|
|
than plain MiniLM (AUC 0.60) and was not chosen.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import re
|
|
from typing import Any
|
|
|
|
from desk.tools import embedding
|
|
from pipeline import NULL, Notice, RunContext, job
|
|
from app.config import Config
|
|
from app.models import Role
|
|
from app.storage import Store
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: How much of the description the encoder sees. MiniLM truncates at 256
|
|
#: tokens anyway; 2000 characters is past that for any language.
|
|
BODY_CHARS = 2000
|
|
|
|
#: Roles encoded per call, and per commit: a kill loses at most one chunk.
|
|
CHUNK = 64
|
|
|
|
#: intfloat's e5 family is trained with a "query: " / "passage: " prefix baked
|
|
#: into the pretraining objective; encoding without it measurably hurts
|
|
#: retrieval. Matched against the model id itself, not a second config field,
|
|
#: so switching `embed.model` to an e5 variant can't leave a stale prefix
|
|
#: setting behind it -- the one place that says which model to use is also
|
|
#: the one place that decides whether it wants a prefix. Every other model
|
|
#: (MiniLM and friends) has no such convention, so the match is narrow: the
|
|
#: model id's own name must contain "e5" set off by "-", "_" or "/".
|
|
_E5_NAME = re.compile(r"(?:^|[-_/])e5(?:[-_]|$)", re.IGNORECASE)
|
|
|
|
|
|
def wants_e5_prefix(model: str) -> bool:
|
|
return bool(_E5_NAME.search(model))
|
|
|
|
|
|
def document(role: Role) -> str:
|
|
"""The title and the description, else the teaser."""
|
|
return f"{role.title}\n{(role.description or role.teaser)[:BODY_CHARS]}"
|
|
|
|
|
|
def role_text(model: str, role: Role) -> str:
|
|
"""What gets encoded for a role: `document(role)`, prefixed "passage: "
|
|
for an e5 model since that is the side e5 calls the passage."""
|
|
text = document(role)
|
|
return f"passage: {text}" if wants_e5_prefix(model) else text
|
|
|
|
|
|
def intent_text(model: str, intent: str) -> str:
|
|
"""What gets encoded for an agent's intent: prefixed "query: " for an e5
|
|
model, since the intent is what a role is matched against, e5's "query"
|
|
side. `score._intent` calls this so the cached vector and a freshly
|
|
encoded one are always the same text for a given model."""
|
|
return f"query: {intent}" if wants_e5_prefix(model) else intent
|
|
|
|
|
|
def load_embedder(model: str) -> embedding.Embedder:
|
|
"""The encoder. A seam of its own so a test hands in a fake and nothing
|
|
downloads a model; `score` loads through it too."""
|
|
return embedding.load(model)
|
|
|
|
|
|
def unavailable(exc: BaseException) -> Notice:
|
|
"""The notice for a machine without sentence-transformers. One code and
|
|
key for `embed` and `score`, so the two say it as one line."""
|
|
return Notice(
|
|
code="embed.unavailable", level="warning", kind="expected", key="embed",
|
|
text="sentence-transformers is not installed, so roles are not embedded "
|
|
"and scores leave fit out.",
|
|
fix="Install it into roledesk's environment: "
|
|
"`uv pip install sentence-transformers`.",
|
|
detail=str(exc), where="app.jobs.embed")
|
|
|
|
|
|
@job("embed", "Embedding roles", solo=True, resource="cpu")
|
|
def run_embed(config: Config, store: Store, *,
|
|
context: RunContext = NULL) -> dict[str, Any]:
|
|
"""Embed every open role that has no vector from the configured model."""
|
|
model = config.embed.model
|
|
try:
|
|
embedder = load_embedder(model)
|
|
except ImportError as exc:
|
|
log.warning("%s", exc)
|
|
context.notice(unavailable(exc))
|
|
return {"embedded": 0, "skipped": True, "reason": "sentence-transformers missing"}
|
|
|
|
dropped = store.drop_vectors(keep=model)
|
|
if dropped:
|
|
log.info("dropped %d vectors from a previous embedding model", dropped)
|
|
roles = store.roles_to_embed(model, config.embed.per_run)
|
|
for at in range(0, len(roles), CHUNK):
|
|
context.check()
|
|
context.progress(at, len(roles), f"embedding roles, {at} of {len(roles)}")
|
|
chunk = roles[at:at + CHUNK]
|
|
vectors = embedder.encode([role_text(model, r) for r in chunk])
|
|
store.put_vectors(model, [(r.uid, v) for r, v in zip(chunk, vectors, strict=True)])
|
|
context.progress(len(roles), len(roles),
|
|
f"embedding roles, {len(roles)} of {len(roles)}")
|
|
return {"embedded": len(roles), "dropped": dropped}
|