Files
jobboard/tests/test_cv_fit.py
T

292 lines
13 KiB
Python

"""CV closeness: the CV cut into chunks, embedded and cached like the
prompts, each role's `cv_fit` beside its score, the judge window it widens and
the CV match order of the roles not judged yet. The embedder is a fake;
nothing here downloads a model or reaches a board."""
from __future__ import annotations
import sqlite3
from collections.abc import Iterator, Sequence
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import numpy as np
import pytest
from app.config import CALIBRATION, Calibration, Config
from app.jobs import embed, judge, score
from app.models import Gate, Limits, Prompt
from app.storage import Store
from app.storage.schema import MIGRATIONS
from desk.models import Item
from pipeline import RunContext
from prepped import upsert
NOW = datetime(2026, 10, 3, 9, 0, tzinfo=UTC)
E5 = CALIBRATION["intfloat/multilingual-e5-small"]
WORDS = ("backend", "python", "frontend", "react", "data")
class FakeEmbedder:
"""One dimension per word of `WORDS` and a constant one, as in
test_score; records what it was asked to encode."""
model_id = "fake"
def __init__(self) -> None:
self.calls: list[list[str]] = []
def encode(self, texts: Sequence[str]) -> np.ndarray:
self.calls.append(list(texts))
rows = []
for text in texts:
lower = text.lower()
vec = np.asarray([lower.count(w) for w in WORDS] + [0.1], dtype=np.float32)
rows.append(vec / np.linalg.norm(vec))
return np.vstack(rows)
@pytest.fixture
def fake(monkeypatch: pytest.MonkeyPatch) -> FakeEmbedder:
embedder = FakeEmbedder()
monkeypatch.setattr(embed, "load_embedder", lambda model: embedder)
return embedder
@pytest.fixture
def store(tmp_path: Path) -> Iterator[Store]:
s = Store(tmp_path / "roledesk.db")
s.clock = lambda: NOW
yield s
@pytest.fixture
def config(tmp_path: Path) -> Config:
return Config(database=str(tmp_path / "roledesk.db"),
embed={"model": "fake", "calibration": {"fake": E5.model_dump()}})
def role(store: Store, source_id: str, title: str, body: str = "", **raw: Any) -> str:
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
title=title, body=body or title, published_at=datetime(2026, 9, 30, tzinfo=UTC),
raw={"company": "Acme", **raw})
item.source = "ats"
upsert(store, item)
return item.uid
def run(config: Config, store: Store, job: Any) -> dict[str, Any]:
return dict(job(config, store, context=RunContext()))
def cv_fits(store: Store, agent_id: int) -> dict[str, float | None]:
with store.transaction() as conn:
return dict(conn.execute(
"SELECT uid, cv_fit FROM scores WHERE agent_id = ?", (agent_id,)).fetchall())
def level(text: str, against: str) -> float:
role_vec, other = FakeEmbedder().encode([text, against])
lo, hi = E5.cv_band
return min(max((score.cosine(role_vec, other) - lo) / (hi - lo), 0.0), 1.0)
# ------------------------------------------------------------------ chunks
def test_a_cv_is_cut_at_paragraphs_and_joined_again_up_to_the_limit() -> None:
cv = "Experience\n\nLed data.\n\n\n Skills: Python \n\n"
assert embed.cv_chunks(cv, limit=40) == ["Experience\n\nLed data.\n\nSkills: Python"]
assert embed.cv_chunks(cv, limit=21) == ["Experience\n\nLed data.", "Skills: Python"]
assert embed.cv_chunks(" \n\n ") == []
def test_a_long_paragraph_is_cut_by_line_then_sentence_then_word() -> None:
lines = "\n".join(["a" * 30, "b" * 30, "c" * 30])
assert embed.cv_chunks(lines, limit=65) == ["a" * 30 + "\n" + "b" * 30, "c" * 30]
sentences = "One two three. Four five six. Seven eight nine."
assert embed.cv_chunks(sentences, limit=30) == ["One two three. Four five six.",
"Seven eight nine."]
words = " ".join(["word"] * 20)
found = embed.cv_chunks(words, limit=24)
assert all(len(c) <= 24 for c in found) and " ".join(found) == words
assert embed.cv_chunks("x" * 50, limit=20) == ["x" * 20, "x" * 20, "x" * 10]
def test_no_chunk_of_a_long_cv_runs_past_the_model_window() -> None:
cv = "\n\n".join(f"Role {n}. " + "Built Python data platforms. " * 60 for n in range(5))
found = embed.cv_chunks(cv)
assert found and all(len(c) <= embed.CV_CHUNK_CHARS for c in found)
# ------------------------------------------------------------------- score
def test_the_cv_band_is_its_own_and_falls_back_to_fits() -> None:
assert E5.cv is not None and E5.cv_band == E5.cv != E5.fit
assert Calibration(fit=(0.7, 0.9), taste=(-0.1, 0.2)).cv_band == (0.7, 0.9)
with pytest.raises(ValueError):
Calibration(fit=(0.7, 0.9), taste=(-0.1, 0.2), cv=(0.9, 0.8))
def test_cv_fit_is_the_best_chunk_scaled_with_the_cv_band_and_no_term_of_the_score(
store: Store, config: Config, fake: FakeEmbedder) -> None:
backend = role(store, "1", "Backend Python")
frontend = role(store, "2", "Frontend React")
cv = "Backend engineer, python services.\n\nSome react on the side."
plain = store.save_agent("Plain", prompts=[Prompt(name="Data", text="data")])
with_cv = store.save_agent("With CV", prompts=[Prompt(name="Data", text="data")],
cv_text=cv)
run(config, store, embed.run_embed)
run(config, store, score.run_score)
texts = {uid: embed.document(r) for uid in (backend, frontend)
if (r := store.role(uid)) is not None}
chunks = embed.cv_chunks(cv)
assert len(chunks) == 1, "two short paragraphs are one chunk"
fits = cv_fits(store, with_cv.id)
for uid in (backend, frontend):
assert fits[uid] == pytest.approx(max(level(texts[uid], c) for c in chunks), abs=1e-6)
assert fits[backend] > fits[frontend]
assert set(cv_fits(store, plain.id).values()) == {None}, "no CV, no closeness"
assert store.scores(plain.id) == store.scores(with_cv.id), "not a term of the score"
def test_the_best_chunk_wins(store: Store, config: Config, fake: FakeEmbedder,
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(embed, "CV_CHUNK_CHARS", 20)
uid = role(store, "1", "Frontend React")
owner = store.save_agent("CV", prompts=[Prompt(name="P", text="data")],
cv_text="Backend python python.\n\nFrontend react work.")
run(config, store, embed.run_embed)
run(config, store, score.run_score)
text = embed.document(store.role(uid)) # type: ignore[arg-type]
assert cv_fits(store, owner.id)[uid] == pytest.approx(
level(text, "Frontend react work."), abs=1e-6)
def test_a_gated_role_and_a_role_without_a_vector_have_no_cv_fit(
store: Store, config: Config, fake: FakeEmbedder) -> None:
kept = role(store, "1", "Backend Python", country="DK")
gated = role(store, "2", "Backend Python", country="SE")
owner = store.save_agent("CV", prompts=[Prompt(name="P", text="python")],
gates=[Gate(field="country", values=["DK"])],
cv_text="Backend python.")
run(config, store, embed.run_embed)
later = role(store, "3", "Backend Python later", country="DK")
run(config, store, score.run_score)
fits = cv_fits(store, owner.id)
assert fits[kept] is not None and fits[gated] is None and fits[later] is None
def test_the_cv_is_embedded_once_per_text_with_the_query_prefix(
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
embedder = FakeEmbedder()
monkeypatch.setattr(embed, "load_embedder", lambda model: embedder)
config = Config(database=str(tmp_path / "roledesk.db"),
embed={"model": "intfloat/multilingual-e5-small"})
role(store, "1", "Backend Python")
owner = store.save_agent("CV", prompts=[Prompt(name="P", text="python")],
cv_text="Backend python.")
run(config, store, score.run_score)
assert embedder.calls == [["query: python"], ["query: Backend python."]]
embedder.calls.clear()
run(config, store, score.run_score)
assert embedder.calls == [], "an unchanged CV is read from the cache"
store.save_agent("CV", prompts=owner.prompts, agent_id=owner.id, cv_text="Frontend react.")
run(config, store, score.run_score)
assert embedder.calls == [["query: Frontend react."]]
key = score.cv_key("intfloat/multilingual-e5-small", "Frontend react.")
assert store.cv_vectors(owner.id, key) is not None
assert store.cv_vectors(owner.id, "stale") is None
# ------------------------------------------------------------------- judge
def ranked(store: Store, owner: Any, rows: Sequence[tuple[str, float, str, float | None]]
) -> list[str]:
uids = [role(store, sid, f"Head of AI {sid}", "Lead the data science team.")
for sid, _, _, _ in rows]
store.replace_scores(owner.id, [(uid, value, reason, "", fit)
for uid, (_, value, reason, fit) in zip(uids, rows,
strict=True)])
return uids
def window(store: Store) -> list[str]:
return [row["uid"] for _, row, _, _ in judge.waiting(store)]
def test_the_cut_takes_the_closest_to_the_cv_the_fit_list_leaves_out_and_judge_reads_it(
store: Store, config: Config, fake: FakeEmbedder) -> None:
"""One role in by fit to the prompt, the one closest to the CV that the
fit list leaves out by CV closeness, a liked one kept; a gated role is in
no list however close, liked or not."""
backend = role(store, "1", "Backend Python", "Backend python python.", country="DK")
frontend = role(store, "2", "Frontend React", "Frontend react react.", country="DK")
data = role(store, "3", "Data", "Data.", country="DK")
other = role(store, "4", "Data again", "Data data.", country="DK")
gated = role(store, "5", "Frontend React", "Frontend react react.", country="SE")
owner = store.save_agent("Data", prompts=[Prompt(name="Backend", text="python backend")],
gates=[Gate(field="country", values=["DK"])],
cv_text="Frontend react developer.",
limits=Limits(cut_fit_top=1, cut_fit_min=0, cut_cv_top=1,
cut_cv_min=0))
store.set_vote(owner.id, data, 1)
store.set_vote(owner.id, gated, 1)
run(config, store, embed.run_embed)
result = run(config, store, score.run_score)
assert store.cut(owner.id) == {backend: "fit", frontend: "cv", data: "kept"}
assert result["cut"] == 3
with store.transaction() as conn:
fits = dict(conn.execute("SELECT uid, fit FROM scores WHERE agent_id = ?",
(owner.id,)).fetchall())
assert fits[backend] > fits[other] and fits[gated] is None
assert window(store) == [backend, frontend, data]
# No count, no list: what is liked is kept all the same.
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
gates=owner.gates, limits=Limits(cut_fit_top=0, cut_cv_top=0))
run(config, store, score.run_score)
assert store.cut(owner.id) == {data: "kept"}
# ------------------------------------------------------------------- order
def test_cv_match_orders_the_unjudged_by_closeness_then_score(store: Store) -> None:
owner = store.save_agent("Data", prompts=[Prompt(name="AI", text="Head of AI")],
cv_text="VP Data.")
judged, close, far, none_a, none_b = ranked(store, owner, [
("judged", 0.1, "", 0.1), ("close", 0.2, "", 0.9), ("far", 0.9, "", 0.3),
("none_a", 0.5, "", None), ("none_b", 0.6, "", None)])
store.set_judgement(owner.id, judged, "k", 50, 50, 50, ["One."], ["Two."])
page = store.roles_page(agent=owner.id, by_cv=True)
assert [r.uid for r in page] == [judged, close, far, none_b, none_a]
assert [r.cv_fit for r in page] == [0.1, 0.9, 0.3, None, None]
assert [r.uid for r in store.roles(agent=owner.id, by_cv=True)] == [
judged, close, far, none_b, none_a]
assert [r.uid for r in store.roles_page(agent=owner.id)] == [
far, none_b, none_a, close, judged], "Agent match ignores it"
found = store.role(close, owner.id)
assert found is not None and found.cv_fit == 0.9
# --------------------------------------------------------------- migration
def test_migration_31_adds_the_column_and_the_table_and_runs_twice(tmp_path: Path) -> None:
path = tmp_path / "old.db"
Store(path)
with sqlite3.connect(path) as conn:
conn.execute("DROP TABLE agent_cv_vectors")
conn.execute("ALTER TABLE scores DROP COLUMN cv_fit")
conn.execute("PRAGMA user_version = 30")
assert Store(path).migrations_applied == [31, 32, 33]
step = {v: s for v, _, s in MIGRATIONS}[31]
with sqlite3.connect(path) as conn:
assert callable(step)
step(conn)
assert "cv_fit" in {r[1] for r in conn.execute("PRAGMA table_info(scores)")}
assert conn.execute("SELECT COUNT(*) FROM agent_cv_vectors").fetchone() == (0,)