Switch the embed model to multilingual-e5-small for the European corpus
Measured on the live roles against the "Copenhagen engineering" agent's intent: the English-only all-MiniLM-L6-v2 separates engineering from non-engineering roles with an AUC of 0.68 overall but only 0.74 restricted to Danish-country roles (a Danish engineering role beats the rest's non-engineering median only 68% of the time). multilingual-e5-small reaches 0.70 overall and 1.00 on Danish-country roles, for about 3x the CPU time. paraphrase-multilingual-MiniLM-L12-v2 measured worse than the English-only baseline (AUC 0.60) and was dropped. e5 needs "query: "/"passage: " prefixes; app/jobs/embed.py adds them by matching the model name so a model change can't leave a stale prefix setting behind. Added tests for the prefixing and for the two invariants the model change relies on: embed drops the old model's vectors before re-embedding, and an agent's cached intent vector is keyed by model so a model change is a cache miss, never a stale-dimension read. Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
This commit is contained in:
1 parent
0558ef19ed
commit
b560cdce2a
7 files changed
+214
-15
No files matched your search
+144
-2
@@ -124,6 +124,147 @@ def test_embed_without_sentence_transformers_says_so_and_does_nothing(
|
||||
assert store.vector_counts() == {}
|
||||
|
||||
|
||||
# -------------------------------------------------------- e5 prefixes
|
||||
|
||||
@pytest.mark.parametrize("model, wants", [
|
||||
("intfloat/multilingual-e5-small", True),
|
||||
("intfloat/e5-base-v2", True),
|
||||
("intfloat/e5-large", True),
|
||||
("sentence-transformers/all-MiniLM-L6-v2", False),
|
||||
("sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", False),
|
||||
("fake", False),
|
||||
("some-org/e5models-are-not-this", False),
|
||||
])
|
||||
def test_wants_e5_prefix_matches_the_e5_family_by_name(model: str, wants: bool) -> None:
|
||||
assert embed.wants_e5_prefix(model) == wants
|
||||
|
||||
|
||||
def test_role_text_and_intent_text_add_the_e5_prefixes_only_for_e5(
|
||||
store: Store) -> None:
|
||||
uid = role(store, "1", "Backend Engineer", body="Python services")
|
||||
r = store.role(uid)
|
||||
assert r is not None
|
||||
assert embed.role_text("fake", r) == embed.document(r)
|
||||
assert embed.role_text("intfloat/multilingual-e5-small", r) == f"passage: {embed.document(r)}"
|
||||
assert embed.intent_text("fake", "python backend") == "python backend"
|
||||
assert embed.intent_text("intfloat/multilingual-e5-small", "python backend") == \
|
||||
"query: python backend"
|
||||
|
||||
|
||||
class FakeE5Embedder:
|
||||
"""A second fake, standing in for a genuinely different model: its own
|
||||
(narrower) vector width and an e5-shaped model id, so it exercises both
|
||||
the prefixing and the re-embedding-on-model-change paths without
|
||||
downloading anything."""
|
||||
|
||||
model_id = "intfloat/multilingual-e5-small"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[list[str]] = []
|
||||
|
||||
def encode(self, texts: Sequence[str]) -> np.ndarray:
|
||||
self.calls.append(list(texts))
|
||||
rows = []
|
||||
for text in texts:
|
||||
lower = text.lower()
|
||||
vec = np.asarray([lower.count(w) for w in WORDS[:2]] + [0.1, 0.2], dtype=np.float32)
|
||||
rows.append(vec / np.linalg.norm(vec))
|
||||
return np.vstack(rows)
|
||||
|
||||
|
||||
def test_embed_prefixes_role_text_for_an_e5_model(
|
||||
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
e5 = FakeE5Embedder()
|
||||
monkeypatch.setattr(embed, "load_embedder", lambda model: e5)
|
||||
config = Config(database=str(tmp_path / "roledesk.db"),
|
||||
embed={"model": "intfloat/multilingual-e5-small"})
|
||||
role(store, "1", "Backend Engineer", body="Python services")
|
||||
|
||||
run(config, store, embed.run_embed)
|
||||
|
||||
assert e5.calls and e5.calls[0][0].startswith("passage: Backend Engineer")
|
||||
|
||||
|
||||
def test_score_prefixes_the_intent_for_an_e5_model(
|
||||
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
e5 = FakeE5Embedder()
|
||||
monkeypatch.setattr(embed, "load_embedder", lambda model: e5)
|
||||
config = Config(database=str(tmp_path / "roledesk.db"),
|
||||
embed={"model": "intfloat/multilingual-e5-small"})
|
||||
role(store, "1")
|
||||
store.save_agent("Backend", "python backend")
|
||||
|
||||
run(config, store, score.run_score)
|
||||
|
||||
assert e5.calls == [["query: python backend"]]
|
||||
|
||||
|
||||
# ------------------------------------------------------- model changes
|
||||
|
||||
def test_changing_the_embed_model_drops_old_vectors_and_reembeds(
|
||||
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The doc says a model change re-embeds; this is the guarantee that
|
||||
actually makes that true, and the one that stops a run ever mixing two
|
||||
models' vectors, which would be two different geometries under one
|
||||
dimension count and garbage cosines."""
|
||||
fake_a, fake_e5 = FakeEmbedder(), FakeE5Embedder()
|
||||
embedders = {"fake": fake_a, "intfloat/multilingual-e5-small": fake_e5}
|
||||
monkeypatch.setattr(embed, "load_embedder", lambda model: embedders[model])
|
||||
role(store, "1")
|
||||
role(store, "2")
|
||||
|
||||
config_a = Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
|
||||
run(config_a, store, embed.run_embed)
|
||||
assert store.vector_counts() == {"fake": 2}
|
||||
dim_a = next(iter(store.vectors("fake").values())).shape[-1]
|
||||
|
||||
config_e5 = Config(database=str(tmp_path / "roledesk.db"),
|
||||
embed={"model": "intfloat/multilingual-e5-small"})
|
||||
result = run(config_e5, store, embed.run_embed)
|
||||
|
||||
# The old model's rows are gone, not just shadowed: `vectors(model)`
|
||||
# filters by model, so a leftover row could never be read back mixed in,
|
||||
# but drop_vectors is what stops it existing at all.
|
||||
assert store.vector_counts() == {"intfloat/multilingual-e5-small": 2}
|
||||
assert store.vectors("fake") == {}
|
||||
assert result["dropped"] == 2
|
||||
dim_e5 = next(iter(store.vectors("intfloat/multilingual-e5-small").values())).shape[-1]
|
||||
assert dim_a != dim_e5, "the fakes must differ in width for this test to mean anything"
|
||||
|
||||
|
||||
def test_changing_the_embed_model_recomputes_the_agents_cached_intent_vector(
|
||||
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""`intent_vector` only returns a cache hit when `intent_key` -- which
|
||||
hashes the model in -- matches what is stored, so a model change is a
|
||||
cache miss and `_intent` re-encodes under the new model, never handing a
|
||||
stale-dimension vector into a run's cosines."""
|
||||
fake_a, fake_e5 = FakeEmbedder(), FakeE5Embedder()
|
||||
embedders = {"fake": fake_a, "intfloat/multilingual-e5-small": fake_e5}
|
||||
monkeypatch.setattr(embed, "load_embedder", lambda model: embedders[model])
|
||||
uid = role(store, "1")
|
||||
agent = store.save_agent("Backend", "python backend")
|
||||
|
||||
config_a = Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
|
||||
run(config_a, store, embed.run_embed)
|
||||
run(config_a, store, score.run_score)
|
||||
vec_a = store.intent_vector(agent.id, score.intent_key("fake", "python backend"))
|
||||
assert vec_a is not None
|
||||
|
||||
config_e5 = Config(database=str(tmp_path / "roledesk.db"),
|
||||
embed={"model": "intfloat/multilingual-e5-small"})
|
||||
run(config_e5, store, embed.run_embed)
|
||||
run(config_e5, store, score.run_score)
|
||||
|
||||
# The old key no longer resolves (the row moved on); the new key does,
|
||||
# at the new model's own width, and scoring this role did not raise.
|
||||
assert store.intent_vector(agent.id, score.intent_key("fake", "python backend")) is None
|
||||
vec_e5 = store.intent_vector(
|
||||
agent.id, score.intent_key("intfloat/multilingual-e5-small", "python backend"))
|
||||
assert vec_e5 is not None
|
||||
assert vec_e5.shape != vec_a.shape
|
||||
assert uid in store.scores(agent.id)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ terms
|
||||
|
||||
def test_recency_halves_every_fourteen_days() -> None:
|
||||
@@ -361,8 +502,9 @@ def test_a_closed_liked_role_still_shapes_the_taste(
|
||||
# ------------------------------------------------------------ fit scaling
|
||||
|
||||
def test_scale_maps_the_closest_to_one_and_the_farthest_to_zero() -> None:
|
||||
# The spread the last real run showed under `(cos + 1) / 2`.
|
||||
assert score.scale({"a": 0.55, "b": 0.79, "c": 0.67}) == pytest.approx(
|
||||
# The spread the last real run showed under `(cos + 1) / 2`, on
|
||||
# multilingual-e5-small.
|
||||
assert score.scale({"a": 0.87, "b": 0.95, "c": 0.91}) == pytest.approx(
|
||||
{"a": 0.0, "b": 1.0, "c": 0.5})
|
||||
assert score.scale({"only": 0.3}) == {"only": 1.0}
|
||||
assert score.scale({"a": 0.4, "b": 0.4}) == {"a": 1.0, "b": 1.0}
|
||||
|
||||
Reference in new issue
Block a user