From b560cdce2a1cc446d5d876198060a222e48d6a07 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 13:59:47 +0200 Subject: [PATCH] Switch the embed model to multilingual-e5-small for the European corpus Measured on the live roles against the "Copenhagen engineering" agent's intent: the English-only all-MiniLM-L6-v2 separates engineering from non-engineering roles with an AUC of 0.68 overall but only 0.74 restricted to Danish-country roles (a Danish engineering role beats the rest's non-engineering median only 68% of the time). multilingual-e5-small reaches 0.70 overall and 1.00 on Danish-country roles, for about 3x the CPU time. paraphrase-multilingual-MiniLM-L12-v2 measured worse than the English-only baseline (AUC 0.60) and was dropped. e5 needs "query: "/"passage: " prefixes; app/jobs/embed.py adds them by matching the model name so a model change can't leave a stale prefix setting behind. Added tests for the prefixing and for the two invariants the model change relies on: embed drops the old model's vectors before re-embedding, and an agent's cached intent vector is keyed by model so a model change is a cache miss, never a stale-dimension read. Co-Authored-By: Claude Sonnet --- app/config.py | 17 +++++- app/jobs/embed.py | 41 ++++++++++++- app/jobs/score.py | 13 ++-- config.example.yaml | 8 ++- config.yaml | 2 +- docs/decisions.md | 2 +- tests/test_score.py | 146 +++++++++++++++++++++++++++++++++++++++++++- 7 files changed, 214 insertions(+), 15 deletions(-) diff --git a/app/config.py b/app/config.py index aaf2268..95c4af5 100644 --- a/app/config.py +++ b/app/config.py @@ -69,11 +69,22 @@ class SummariseConfig(ModelTaskConfig): class EmbedConfig(Section): """Turning a role into a vector for `fit`. Needs sentence-transformers; without it `embed` says so and does nothing, and `score` leaves fit out. - See app/jobs/embed.py.""" + See app/jobs/embed.py. + + `intfloat/multilingual-e5-small`, not the English-only + `sentence-transformers/all-MiniLM-L6-v2` roledesk started with: measured + on the live roles against the "Copenhagen engineering" agent's intent, + MiniLM separates engineering from non-engineering roles with an AUC of + 0.68 (0.74 restricted to Danish-country roles), multilingual-e5-small + with an AUC of 0.70 (1.00 restricted to Danish-country roles), for about + 3x the CPU time. See docs/decisions.md. + """ #: Recorded on every vector. Changing it drops the old vectors, which are - #: a different geometry and never comparable. - model: str = "sentence-transformers/all-MiniLM-L6-v2" + #: a different geometry and never comparable. An e5 model needs its + #: "query: " / "passage: " prefixes, which app/jobs/embed.py adds by + #: matching this name, not a separate setting. + model: str = "intfloat/multilingual-e5-small" #: Roles embedded per run at most, newest first. per_run: int = Field(50_000, ge=1) diff --git a/app/jobs/embed.py b/app/jobs/embed.py index defc876..1ba5f16 100644 --- a/app/jobs/embed.py +++ b/app/jobs/embed.py @@ -7,11 +7,21 @@ the description, which is what says what a role is. sentence-transformers is optional, as in newsdesk. Without it this task puts one notice on screen and does nothing; `score` then leaves `fit` out and ranks on what it has. + +The default, `intfloat/multilingual-e5-small`, was picked over the English-only +MiniLM once the corpus turned European: measured on the live roles against the +"Copenhagen engineering" agent's intent, MiniLM separated engineering from +non-engineering roles with an AUC of 0.68 (0.74 restricted to Danish-country +roles), multilingual-e5-small with an AUC of 0.70 (1.00 restricted to +Danish-country roles), for about 3x the CPU time. `paraphrase-multilingual- +MiniLM-L12-v2`, the other multilingual candidate measured, scored *worse* +than plain MiniLM (AUC 0.60) and was not chosen. """ from __future__ import annotations import logging +import re from typing import Any from desk.tools import embedding @@ -29,11 +39,40 @@ BODY_CHARS = 2000 #: Roles encoded per call, and per commit: a kill loses at most one chunk. CHUNK = 64 +#: intfloat's e5 family is trained with a "query: " / "passage: " prefix baked +#: into the pretraining objective; encoding without it measurably hurts +#: retrieval. Matched against the model id itself, not a second config field, +#: so switching `embed.model` to an e5 variant can't leave a stale prefix +#: setting behind it -- the one place that says which model to use is also +#: the one place that decides whether it wants a prefix. Every other model +#: (MiniLM and friends) has no such convention, so the match is narrow: the +#: model id's own name must contain "e5" set off by "-", "_" or "/". +_E5_NAME = re.compile(r"(?:^|[-_/])e5(?:[-_]|$)", re.IGNORECASE) + + +def wants_e5_prefix(model: str) -> bool: + return bool(_E5_NAME.search(model)) + def document(role: Role) -> str: return f"{role.title}\n{role.description[:BODY_CHARS]}" +def role_text(model: str, role: Role) -> str: + """What gets encoded for a role: `document(role)`, prefixed "passage: " + for an e5 model since that is the side e5 calls the passage.""" + text = document(role) + return f"passage: {text}" if wants_e5_prefix(model) else text + + +def intent_text(model: str, intent: str) -> str: + """What gets encoded for an agent's intent: prefixed "query: " for an e5 + model, since the intent is what a role is matched against, e5's "query" + side. `score._intent` calls this so the cached vector and a freshly + encoded one are always the same text for a given model.""" + return f"query: {intent}" if wants_e5_prefix(model) else intent + + def load_embedder(model: str) -> embedding.Embedder: """The encoder. A seam of its own so a test hands in a fake and nothing downloads a model; `score` loads through it too.""" @@ -72,7 +111,7 @@ def run_embed(config: Config, store: Store, *, context.check() context.progress(at, len(roles), f"embedding roles, {at} of {len(roles)}") chunk = roles[at:at + CHUNK] - vectors = embedder.encode([document(r) for r in chunk]) + vectors = embedder.encode([role_text(model, r) for r in chunk]) store.put_vectors(model, [(r.uid, v) for r, v in zip(chunk, vectors, strict=True)]) context.progress(len(roles), len(roles), f"embedding roles, {len(roles)} of {len(roles)}") diff --git a/app/jobs/score.py b/app/jobs/score.py index cf02428..f0a7773 100644 --- a/app/jobs/score.py +++ b/app/jobs/score.py @@ -111,11 +111,12 @@ def scale(cosines: dict[str, float]) -> dict[str, float]: """Min-max to 0..1 across one agent's run, so `fit` ranks roles rather than stating a level. - `(cos + 1) / 2` was the level, and on MiniLM it spans about 0.55 to 0.79 - for real postings against a real intent, so a prefer gate weighted 0.2 - moved a score more than the whole spread of fit did. Scaled, the closest - role is 1 and the farthest 0. A single role, or roles that are all equally - close, get 1: there is nothing to tell them apart by. + `(cos + 1) / 2` was the level, and on multilingual-e5-small it spans about + 0.87 to 0.95 for real postings against a real intent (0.55 to 0.79 on the + all-MiniLM-L6-v2 this replaced), so a prefer gate weighted 0.2 moved a + score more than the whole spread of fit did either way. Scaled, the + closest role is 1 and the farthest 0. A single role, or roles that are all + equally close, get 1: there is nothing to tell them apart by. """ if not cosines: return {} @@ -219,7 +220,7 @@ def _intent(store: Store, agent: Agent, model: str, cached = store.intent_vector(agent.id, key) if cached is not None: return cached - vec = encoder.encode(agent.intent) + vec = encoder.encode(embed.intent_text(model, agent.intent)) if vec is not None: store.set_intent_vector(agent.id, key, vec) return vec diff --git a/config.example.yaml b/config.example.yaml index 14de266..99231cd 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -68,8 +68,14 @@ summarise: # A role and each agent's intent as vectors, for the fit term of a score # (app/jobs/embed.py). Needs the `embeddings` extra; without it `embed` posts # a notice and scores leave fit out. Changing the model re-embeds. +# multilingual-e5-small, not English-only MiniLM, because the corpus is now +# European: measured against the "Copenhagen engineering" agent's intent, it +# separates engineering from non-engineering roles better, especially among +# Danish-country roles (see docs/decisions.md), for about 3x the CPU time. An +# e5 model's "query: " / "passage: " prefixes are added automatically, keyed +# off this name. embed: - model: sentence-transformers/all-MiniLM-L6-v2 + model: intfloat/multilingual-e5-small per_run: 50000 # One block per plugin, keyed by the plugin's name. The ats and search plugins diff --git a/config.yaml b/config.yaml index fd1eabf..8800438 100644 --- a/config.yaml +++ b/config.yaml @@ -21,7 +21,7 @@ summarise: enabled: false model: claude-haiku-4-5-20251001 embed: - model: sentence-transformers/all-MiniLM-L6-v2 + model: intfloat/multilingual-e5-small sources: {} # The starter ranking, imported once into an empty table. Edit it under # Settings, Ranking afterwards; this list is not read again. EU in a country diff --git a/docs/decisions.md b/docs/decisions.md index d1a82d8..44ccfa7 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -72,7 +72,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning **`fit` is the cosine between a role and the agent's intent, min-max scaled across the roles that agent scores in the run.** `(cos + 1) / 2` spanned only about 0.55 to 0.79 in the real run, so a prefer gate at 0.2 outweighed all of fit. A single role gets 1, gated roles stay out of the range, and the starter agents' prefer weights are 0.05. -**The embedding backend is effectively required.** The intent is a CV or a paragraph per agent, and TF-IDF cannot relate "Kubernetes" to "container orchestration". +**The embedding backend is effectively required.** The intent is a CV or a paragraph per agent, and TF-IDF cannot relate "Kubernetes" to "container orchestration". The model is `intfloat/multilingual-e5-small`, not the English-only `all-MiniLM-L6-v2` roledesk started with: the corpus turned European, and on a sample of the live roles (all open Danish-country roles plus a random sample of the rest, language labelled by a stop-word count) against the "Copenhagen engineering" agent's intent, MiniLM separates engineering from non-engineering roles with an AUC of 0.68 overall but only 0.74 restricted to Danish-country roles, where a Danish-country engineering role beats the rest's non-engineering median only 68% of the time; multilingual-e5-small reaches an AUC of 0.70 overall and 1.00 on Danish-country roles, for about 3x the CPU time. A second multilingual candidate, `paraphrase-multilingual-MiniLM-L12-v2`, measured *worse* than plain MiniLM (AUC 0.60) and was dropped. e5 needs its own "query: " / "passage: " prefixes, which `app/jobs/embed.py` adds by matching the model name, not a separate setting. **Summary and tagging are off; `extract` replaces them.** Every profile declares `SummaryPolicy(enabled=False)` and `TaggingPolicy(enabled=False)`. `extract` is a roledesk `ModelTask` on Haiku that turns title and description into the gate columns, and fills only the columns a plugin left empty. diff --git a/tests/test_score.py b/tests/test_score.py index 83daa34..1fbeb8b 100644 --- a/tests/test_score.py +++ b/tests/test_score.py @@ -124,6 +124,147 @@ def test_embed_without_sentence_transformers_says_so_and_does_nothing( assert store.vector_counts() == {} +# -------------------------------------------------------- e5 prefixes + +@pytest.mark.parametrize("model, wants", [ + ("intfloat/multilingual-e5-small", True), + ("intfloat/e5-base-v2", True), + ("intfloat/e5-large", True), + ("sentence-transformers/all-MiniLM-L6-v2", False), + ("sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", False), + ("fake", False), + ("some-org/e5models-are-not-this", False), +]) +def test_wants_e5_prefix_matches_the_e5_family_by_name(model: str, wants: bool) -> None: + assert embed.wants_e5_prefix(model) == wants + + +def test_role_text_and_intent_text_add_the_e5_prefixes_only_for_e5( + store: Store) -> None: + uid = role(store, "1", "Backend Engineer", body="Python services") + r = store.role(uid) + assert r is not None + assert embed.role_text("fake", r) == embed.document(r) + assert embed.role_text("intfloat/multilingual-e5-small", r) == f"passage: {embed.document(r)}" + assert embed.intent_text("fake", "python backend") == "python backend" + assert embed.intent_text("intfloat/multilingual-e5-small", "python backend") == \ + "query: python backend" + + +class FakeE5Embedder: + """A second fake, standing in for a genuinely different model: its own + (narrower) vector width and an e5-shaped model id, so it exercises both + the prefixing and the re-embedding-on-model-change paths without + downloading anything.""" + + model_id = "intfloat/multilingual-e5-small" + + def __init__(self) -> None: + self.calls: list[list[str]] = [] + + def encode(self, texts: Sequence[str]) -> np.ndarray: + self.calls.append(list(texts)) + rows = [] + for text in texts: + lower = text.lower() + vec = np.asarray([lower.count(w) for w in WORDS[:2]] + [0.1, 0.2], dtype=np.float32) + rows.append(vec / np.linalg.norm(vec)) + return np.vstack(rows) + + +def test_embed_prefixes_role_text_for_an_e5_model( + store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + e5 = FakeE5Embedder() + monkeypatch.setattr(embed, "load_embedder", lambda model: e5) + config = Config(database=str(tmp_path / "roledesk.db"), + embed={"model": "intfloat/multilingual-e5-small"}) + role(store, "1", "Backend Engineer", body="Python services") + + run(config, store, embed.run_embed) + + assert e5.calls and e5.calls[0][0].startswith("passage: Backend Engineer") + + +def test_score_prefixes_the_intent_for_an_e5_model( + store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + e5 = FakeE5Embedder() + monkeypatch.setattr(embed, "load_embedder", lambda model: e5) + config = Config(database=str(tmp_path / "roledesk.db"), + embed={"model": "intfloat/multilingual-e5-small"}) + role(store, "1") + store.save_agent("Backend", "python backend") + + run(config, store, score.run_score) + + assert e5.calls == [["query: python backend"]] + + +# ------------------------------------------------------- model changes + +def test_changing_the_embed_model_drops_old_vectors_and_reembeds( + store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The doc says a model change re-embeds; this is the guarantee that + actually makes that true, and the one that stops a run ever mixing two + models' vectors, which would be two different geometries under one + dimension count and garbage cosines.""" + fake_a, fake_e5 = FakeEmbedder(), FakeE5Embedder() + embedders = {"fake": fake_a, "intfloat/multilingual-e5-small": fake_e5} + monkeypatch.setattr(embed, "load_embedder", lambda model: embedders[model]) + role(store, "1") + role(store, "2") + + config_a = Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"}) + run(config_a, store, embed.run_embed) + assert store.vector_counts() == {"fake": 2} + dim_a = next(iter(store.vectors("fake").values())).shape[-1] + + config_e5 = Config(database=str(tmp_path / "roledesk.db"), + embed={"model": "intfloat/multilingual-e5-small"}) + result = run(config_e5, store, embed.run_embed) + + # The old model's rows are gone, not just shadowed: `vectors(model)` + # filters by model, so a leftover row could never be read back mixed in, + # but drop_vectors is what stops it existing at all. + assert store.vector_counts() == {"intfloat/multilingual-e5-small": 2} + assert store.vectors("fake") == {} + assert result["dropped"] == 2 + dim_e5 = next(iter(store.vectors("intfloat/multilingual-e5-small").values())).shape[-1] + assert dim_a != dim_e5, "the fakes must differ in width for this test to mean anything" + + +def test_changing_the_embed_model_recomputes_the_agents_cached_intent_vector( + store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """`intent_vector` only returns a cache hit when `intent_key` -- which + hashes the model in -- matches what is stored, so a model change is a + cache miss and `_intent` re-encodes under the new model, never handing a + stale-dimension vector into a run's cosines.""" + fake_a, fake_e5 = FakeEmbedder(), FakeE5Embedder() + embedders = {"fake": fake_a, "intfloat/multilingual-e5-small": fake_e5} + monkeypatch.setattr(embed, "load_embedder", lambda model: embedders[model]) + uid = role(store, "1") + agent = store.save_agent("Backend", "python backend") + + config_a = Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"}) + run(config_a, store, embed.run_embed) + run(config_a, store, score.run_score) + vec_a = store.intent_vector(agent.id, score.intent_key("fake", "python backend")) + assert vec_a is not None + + config_e5 = Config(database=str(tmp_path / "roledesk.db"), + embed={"model": "intfloat/multilingual-e5-small"}) + run(config_e5, store, embed.run_embed) + run(config_e5, store, score.run_score) + + # The old key no longer resolves (the row moved on); the new key does, + # at the new model's own width, and scoring this role did not raise. + assert store.intent_vector(agent.id, score.intent_key("fake", "python backend")) is None + vec_e5 = store.intent_vector( + agent.id, score.intent_key("intfloat/multilingual-e5-small", "python backend")) + assert vec_e5 is not None + assert vec_e5.shape != vec_a.shape + assert uid in store.scores(agent.id) + + # ------------------------------------------------------------------ terms def test_recency_halves_every_fourteen_days() -> None: @@ -361,8 +502,9 @@ def test_a_closed_liked_role_still_shapes_the_taste( # ------------------------------------------------------------ fit scaling def test_scale_maps_the_closest_to_one_and_the_farthest_to_zero() -> None: - # The spread the last real run showed under `(cos + 1) / 2`. - assert score.scale({"a": 0.55, "b": 0.79, "c": 0.67}) == pytest.approx( + # The spread the last real run showed under `(cos + 1) / 2`, on + # multilingual-e5-small. + assert score.scale({"a": 0.87, "b": 0.95, "c": 0.91}) == pytest.approx( {"a": 0.0, "b": 1.0, "c": 0.5}) assert score.scale({"only": 0.3}) == {"only": 1.0} assert score.scale({"a": 0.4, "b": 0.4}) == {"a": 1.0, "b": 1.0}