Score each role's closeness to the agent's CV, let the judge also read the 50 closest the score leaves out, and order CV match's unjudged roles by it
The CV is cut into chunks inside e5's window, embedded like a prompt and cached per agent under a hash of the model, chunking and text. A role's cv_fit (scores.cv_fit, migration 31) is its best chunk cosine, scaled with its own calibration band: fit's clipped 88 of agent 2's ungated roles to 1. It is no term of the Agent match score. judge_cv_top (default 50) sits beside judge_top and judge_min on the agents page and card. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b481cb5692
commit
c23594507a
23 files changed
+603
-85
No files matched your search
@@ -0,0 +1,286 @@
|
||||
"""CV closeness: the CV cut into chunks, embedded and cached like the
|
||||
prompts, each role's `cv_fit` beside its score, the judge window it widens and
|
||||
the CV match order of the roles not judged yet. The embedder is a fake;
|
||||
nothing here downloads a model or reaches a board."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from collections.abc import Iterator, Sequence
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from app.config import CALIBRATION, Calibration, Config
|
||||
from app.jobs import embed, judge, score
|
||||
from app.models import Gate, Limits, Prompt
|
||||
from app.storage import Store
|
||||
from app.storage.schema import MIGRATIONS
|
||||
from desk.models import Item
|
||||
from pipeline import RunContext
|
||||
from prepped import upsert
|
||||
|
||||
NOW = datetime(2026, 10, 3, 9, 0, tzinfo=UTC)
|
||||
E5 = CALIBRATION["intfloat/multilingual-e5-small"]
|
||||
WORDS = ("backend", "python", "frontend", "react", "data")
|
||||
|
||||
|
||||
class FakeEmbedder:
|
||||
"""One dimension per word of `WORDS` and a constant one, as in
|
||||
test_score; records what it was asked to encode."""
|
||||
|
||||
model_id = "fake"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[list[str]] = []
|
||||
|
||||
def encode(self, texts: Sequence[str]) -> np.ndarray:
|
||||
self.calls.append(list(texts))
|
||||
rows = []
|
||||
for text in texts:
|
||||
lower = text.lower()
|
||||
vec = np.asarray([lower.count(w) for w in WORDS] + [0.1], dtype=np.float32)
|
||||
rows.append(vec / np.linalg.norm(vec))
|
||||
return np.vstack(rows)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake(monkeypatch: pytest.MonkeyPatch) -> FakeEmbedder:
|
||||
embedder = FakeEmbedder()
|
||||
monkeypatch.setattr(embed, "load_embedder", lambda model: embedder)
|
||||
return embedder
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> Iterator[Store]:
|
||||
s = Store(tmp_path / "roledesk.db")
|
||||
s.clock = lambda: NOW
|
||||
yield s
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def config(tmp_path: Path) -> Config:
|
||||
return Config(database=str(tmp_path / "roledesk.db"),
|
||||
embed={"model": "fake", "calibration": {"fake": E5.model_dump()}})
|
||||
|
||||
|
||||
def role(store: Store, source_id: str, title: str, body: str = "", **raw: Any) -> str:
|
||||
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
|
||||
title=title, body=body or title, published_at=datetime(2026, 9, 30, tzinfo=UTC),
|
||||
raw={"company": "Acme", **raw})
|
||||
item.source = "ats"
|
||||
upsert(store, item)
|
||||
return item.uid
|
||||
|
||||
|
||||
def run(config: Config, store: Store, job: Any) -> dict[str, Any]:
|
||||
return dict(job(config, store, context=RunContext()))
|
||||
|
||||
|
||||
def cv_fits(store: Store, agent_id: int) -> dict[str, float | None]:
|
||||
with store.transaction() as conn:
|
||||
return dict(conn.execute(
|
||||
"SELECT uid, cv_fit FROM scores WHERE agent_id = ?", (agent_id,)).fetchall())
|
||||
|
||||
|
||||
def level(text: str, against: str) -> float:
|
||||
role_vec, other = FakeEmbedder().encode([text, against])
|
||||
lo, hi = E5.cv_band
|
||||
return min(max((score.cosine(role_vec, other) - lo) / (hi - lo), 0.0), 1.0)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ chunks
|
||||
|
||||
def test_a_cv_is_cut_at_paragraphs_and_joined_again_up_to_the_limit() -> None:
|
||||
cv = "Experience\n\nLed data.\n\n\n Skills: Python \n\n"
|
||||
assert embed.cv_chunks(cv, limit=40) == ["Experience\n\nLed data.\n\nSkills: Python"]
|
||||
assert embed.cv_chunks(cv, limit=21) == ["Experience\n\nLed data.", "Skills: Python"]
|
||||
assert embed.cv_chunks(" \n\n ") == []
|
||||
|
||||
|
||||
def test_a_long_paragraph_is_cut_by_line_then_sentence_then_word() -> None:
|
||||
lines = "\n".join(["a" * 30, "b" * 30, "c" * 30])
|
||||
assert embed.cv_chunks(lines, limit=65) == ["a" * 30 + "\n" + "b" * 30, "c" * 30]
|
||||
sentences = "One two three. Four five six. Seven eight nine."
|
||||
assert embed.cv_chunks(sentences, limit=30) == ["One two three. Four five six.",
|
||||
"Seven eight nine."]
|
||||
words = " ".join(["word"] * 20)
|
||||
found = embed.cv_chunks(words, limit=24)
|
||||
assert all(len(c) <= 24 for c in found) and " ".join(found) == words
|
||||
assert embed.cv_chunks("x" * 50, limit=20) == ["x" * 20, "x" * 20, "x" * 10]
|
||||
|
||||
|
||||
def test_no_chunk_of_a_long_cv_runs_past_the_model_window() -> None:
|
||||
cv = "\n\n".join(f"Role {n}. " + "Built Python data platforms. " * 60 for n in range(5))
|
||||
found = embed.cv_chunks(cv)
|
||||
assert found and all(len(c) <= embed.CV_CHUNK_CHARS for c in found)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- score
|
||||
|
||||
def test_the_cv_band_is_its_own_and_falls_back_to_fits() -> None:
|
||||
assert E5.cv is not None and E5.cv_band == E5.cv != E5.fit
|
||||
assert Calibration(fit=(0.7, 0.9), taste=(-0.1, 0.2)).cv_band == (0.7, 0.9)
|
||||
with pytest.raises(ValueError):
|
||||
Calibration(fit=(0.7, 0.9), taste=(-0.1, 0.2), cv=(0.9, 0.8))
|
||||
|
||||
|
||||
def test_cv_fit_is_the_best_chunk_scaled_with_the_cv_band_and_no_term_of_the_score(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
backend = role(store, "1", "Backend Python")
|
||||
frontend = role(store, "2", "Frontend React")
|
||||
cv = "Backend engineer, python services.\n\nSome react on the side."
|
||||
plain = store.save_agent("Plain", prompts=[Prompt(name="Data", text="data")])
|
||||
with_cv = store.save_agent("With CV", prompts=[Prompt(name="Data", text="data")],
|
||||
cv_text=cv)
|
||||
run(config, store, embed.run_embed)
|
||||
run(config, store, score.run_score)
|
||||
|
||||
texts = {uid: embed.document(r) for uid in (backend, frontend)
|
||||
if (r := store.role(uid)) is not None}
|
||||
chunks = embed.cv_chunks(cv)
|
||||
assert len(chunks) == 1, "two short paragraphs are one chunk"
|
||||
fits = cv_fits(store, with_cv.id)
|
||||
for uid in (backend, frontend):
|
||||
assert fits[uid] == pytest.approx(max(level(texts[uid], c) for c in chunks), abs=1e-6)
|
||||
assert fits[backend] > fits[frontend]
|
||||
assert set(cv_fits(store, plain.id).values()) == {None}, "no CV, no closeness"
|
||||
assert store.scores(plain.id) == store.scores(with_cv.id), "not a term of the score"
|
||||
|
||||
|
||||
def test_the_best_chunk_wins(store: Store, config: Config, fake: FakeEmbedder,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(embed, "CV_CHUNK_CHARS", 20)
|
||||
uid = role(store, "1", "Frontend React")
|
||||
owner = store.save_agent("CV", prompts=[Prompt(name="P", text="data")],
|
||||
cv_text="Backend python python.\n\nFrontend react work.")
|
||||
run(config, store, embed.run_embed)
|
||||
run(config, store, score.run_score)
|
||||
text = embed.document(store.role(uid)) # type: ignore[arg-type]
|
||||
assert cv_fits(store, owner.id)[uid] == pytest.approx(
|
||||
level(text, "Frontend react work."), abs=1e-6)
|
||||
|
||||
|
||||
def test_a_gated_role_and_a_role_without_a_vector_have_no_cv_fit(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
kept = role(store, "1", "Backend Python", country="DK")
|
||||
gated = role(store, "2", "Backend Python", country="SE")
|
||||
owner = store.save_agent("CV", prompts=[Prompt(name="P", text="python")],
|
||||
gates=[Gate(field="country", values=["DK"])],
|
||||
cv_text="Backend python.")
|
||||
run(config, store, embed.run_embed)
|
||||
later = role(store, "3", "Backend Python later", country="DK")
|
||||
run(config, store, score.run_score)
|
||||
fits = cv_fits(store, owner.id)
|
||||
assert fits[kept] is not None and fits[gated] is None and fits[later] is None
|
||||
|
||||
|
||||
def test_the_cv_is_embedded_once_per_text_with_the_query_prefix(
|
||||
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
embedder = FakeEmbedder()
|
||||
monkeypatch.setattr(embed, "load_embedder", lambda model: embedder)
|
||||
config = Config(database=str(tmp_path / "roledesk.db"),
|
||||
embed={"model": "intfloat/multilingual-e5-small"})
|
||||
role(store, "1", "Backend Python")
|
||||
owner = store.save_agent("CV", prompts=[Prompt(name="P", text="python")],
|
||||
cv_text="Backend python.")
|
||||
run(config, store, score.run_score)
|
||||
assert embedder.calls == [["query: python"], ["query: Backend python."]]
|
||||
|
||||
embedder.calls.clear()
|
||||
run(config, store, score.run_score)
|
||||
assert embedder.calls == [], "an unchanged CV is read from the cache"
|
||||
|
||||
store.save_agent("CV", prompts=owner.prompts, agent_id=owner.id, cv_text="Frontend react.")
|
||||
run(config, store, score.run_score)
|
||||
assert embedder.calls == [["query: Frontend react."]]
|
||||
key = score.cv_key("intfloat/multilingual-e5-small", "Frontend react.")
|
||||
assert store.cv_vectors(owner.id, key) is not None
|
||||
assert store.cv_vectors(owner.id, "stale") is None
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- judge
|
||||
|
||||
def ranked(store: Store, owner: Any, rows: Sequence[tuple[str, float, str, float | None]]
|
||||
) -> list[str]:
|
||||
uids = [role(store, sid, f"Head of AI {sid}", "Lead the data science team.")
|
||||
for sid, _, _, _ in rows]
|
||||
store.replace_scores(owner.id, [(uid, value, reason, "", fit)
|
||||
for uid, (_, value, reason, fit) in zip(uids, rows,
|
||||
strict=True)])
|
||||
return uids
|
||||
|
||||
|
||||
def window(store: Store) -> list[str]:
|
||||
return [row["uid"] for _, row, _, _ in judge.waiting(store)]
|
||||
|
||||
|
||||
def test_the_judge_also_reads_the_closest_to_the_cv_the_score_leaves_out(
|
||||
store: Store) -> None:
|
||||
owner = store.save_agent("Data", prompts=[Prompt(name="AI", text="Head of AI")],
|
||||
cv_text="VP Data.",
|
||||
limits=Limits(judge_top=2, judge_min=0.5, judge_cv_top=2))
|
||||
top, second, low, near, nearer, nearest_gated, far = ranked(store, owner, [
|
||||
("top", 0.9, "", 0.1), ("second", 0.8, "", 0.99), ("low", 0.4, "", 0.2),
|
||||
("near", 0.3, "", 0.6), ("nearer", 0.2, "", 0.7), ("gated", 0.0, "country SE", None),
|
||||
("far", 0.1, "", 0.05)])
|
||||
assert window(store) == [top, second, nearer, near]
|
||||
|
||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
||||
limits=Limits(judge_top=2, judge_min=0.85, judge_cv_top=2))
|
||||
assert window(store) == [top, second, nearer], \
|
||||
"a role under the bar is no part of the first set, so it may come in by the CV"
|
||||
|
||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
||||
limits=Limits(judge_top=2, judge_min=0.5, judge_cv_top=0))
|
||||
assert window(store) == [top, second]
|
||||
assert low and far and nearest_gated
|
||||
|
||||
|
||||
def test_a_gated_role_stays_out_of_the_window_however_close(store: Store) -> None:
|
||||
owner = store.save_agent("Data", prompts=[Prompt(name="AI", text="Head of AI")],
|
||||
cv_text="VP Data.",
|
||||
limits=Limits(judge_top=1, judge_min=0.5, judge_cv_top=5))
|
||||
top, _gated = ranked(store, owner, [("top", 0.9, "", 0.1), ("gated", 0.0, "remote", 1.0)])
|
||||
assert window(store) == [top]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- order
|
||||
|
||||
def test_cv_match_orders_the_unjudged_by_closeness_then_score(store: Store) -> None:
|
||||
owner = store.save_agent("Data", prompts=[Prompt(name="AI", text="Head of AI")],
|
||||
cv_text="VP Data.")
|
||||
judged, close, far, none_a, none_b = ranked(store, owner, [
|
||||
("judged", 0.1, "", 0.1), ("close", 0.2, "", 0.9), ("far", 0.9, "", 0.3),
|
||||
("none_a", 0.5, "", None), ("none_b", 0.6, "", None)])
|
||||
store.set_judgement(owner.id, judged, "k", 50, 50, 50, ["One."], ["Two."])
|
||||
page = store.roles_page(agent=owner.id, by_cv=True)
|
||||
assert [r.uid for r in page] == [judged, close, far, none_b, none_a]
|
||||
assert [r.cv_fit for r in page] == [0.1, 0.9, 0.3, None, None]
|
||||
assert [r.uid for r in store.roles(agent=owner.id, by_cv=True)] == [
|
||||
judged, close, far, none_b, none_a]
|
||||
assert [r.uid for r in store.roles_page(agent=owner.id)] == [
|
||||
far, none_b, none_a, close, judged], "Agent match ignores it"
|
||||
found = store.role(close, owner.id)
|
||||
assert found is not None and found.cv_fit == 0.9
|
||||
|
||||
|
||||
# --------------------------------------------------------------- migration
|
||||
|
||||
def test_migration_31_adds_the_column_and_the_table_and_runs_twice(tmp_path: Path) -> None:
|
||||
path = tmp_path / "old.db"
|
||||
Store(path)
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("DROP TABLE agent_cv_vectors")
|
||||
conn.execute("ALTER TABLE scores DROP COLUMN cv_fit")
|
||||
conn.execute("PRAGMA user_version = 30")
|
||||
assert Store(path).migrations_applied == [31]
|
||||
step = {v: s for v, _, s in MIGRATIONS}[31]
|
||||
with sqlite3.connect(path) as conn:
|
||||
assert callable(step)
|
||||
step(conn)
|
||||
assert "cv_fit" in {r[1] for r in conn.execute("PRAGMA table_info(scores)")}
|
||||
assert conn.execute("SELECT COUNT(*) FROM agent_cv_vectors").fetchone() == (0,)
|
||||
Reference in new issue
Block a user