"""CV closeness: the CV cut into chunks, embedded and cached like the prompts, each role's `cv_fit` beside its score, the judge window it widens and the CV match order of the roles not judged yet. The embedder is a fake; nothing here downloads a model or reaches a board.""" from __future__ import annotations import sqlite3 from collections.abc import Iterator, Sequence from datetime import UTC, datetime from pathlib import Path from typing import Any import numpy as np import pytest from app.config import CALIBRATION, Calibration, Config from app.jobs import embed, judge, score from app.models import Gate, Limits, Prompt from app.storage import Store from app.storage.schema import MIGRATIONS from desk.models import Item from pipeline import RunContext from prepped import upsert NOW = datetime(2026, 10, 3, 9, 0, tzinfo=UTC) E5 = CALIBRATION["intfloat/multilingual-e5-small"] WORDS = ("backend", "python", "frontend", "react", "data") class FakeEmbedder: """One dimension per word of `WORDS` and a constant one, as in test_score; records what it was asked to encode.""" model_id = "fake" def __init__(self) -> None: self.calls: list[list[str]] = [] def encode(self, texts: Sequence[str]) -> np.ndarray: self.calls.append(list(texts)) rows = [] for text in texts: lower = text.lower() vec = np.asarray([lower.count(w) for w in WORDS] + [0.1], dtype=np.float32) rows.append(vec / np.linalg.norm(vec)) return np.vstack(rows) @pytest.fixture def fake(monkeypatch: pytest.MonkeyPatch) -> FakeEmbedder: embedder = FakeEmbedder() monkeypatch.setattr(embed, "load_embedder", lambda model: embedder) return embedder @pytest.fixture def store(tmp_path: Path) -> Iterator[Store]: s = Store(tmp_path / "roledesk.db") s.clock = lambda: NOW yield s @pytest.fixture def config(tmp_path: Path) -> Config: return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake", "calibration": {"fake": E5.model_dump()}}) def role(store: Store, source_id: str, title: str, body: str = "", **raw: Any) -> str: item = Item(source_id=source_id, location=f"https://boards.example/{source_id}", title=title, body=body or title, published_at=datetime(2026, 9, 30, tzinfo=UTC), raw={"company": "Acme", **raw}) item.source = "ats" upsert(store, item) return item.uid def run(config: Config, store: Store, job: Any) -> dict[str, Any]: return dict(job(config, store, context=RunContext())) def cv_fits(store: Store, agent_id: int) -> dict[str, float | None]: with store.transaction() as conn: return dict(conn.execute( "SELECT uid, cv_fit FROM scores WHERE agent_id = ?", (agent_id,)).fetchall()) def level(text: str, against: str) -> float: role_vec, other = FakeEmbedder().encode([text, against]) lo, hi = E5.cv_band return min(max((score.cosine(role_vec, other) - lo) / (hi - lo), 0.0), 1.0) # ------------------------------------------------------------------ chunks def test_a_cv_is_cut_at_paragraphs_and_joined_again_up_to_the_limit() -> None: cv = "Experience\n\nLed data.\n\n\n Skills: Python \n\n" assert embed.cv_chunks(cv, limit=40) == ["Experience\n\nLed data.\n\nSkills: Python"] assert embed.cv_chunks(cv, limit=21) == ["Experience\n\nLed data.", "Skills: Python"] assert embed.cv_chunks(" \n\n ") == [] def test_a_long_paragraph_is_cut_by_line_then_sentence_then_word() -> None: lines = "\n".join(["a" * 30, "b" * 30, "c" * 30]) assert embed.cv_chunks(lines, limit=65) == ["a" * 30 + "\n" + "b" * 30, "c" * 30] sentences = "One two three. Four five six. Seven eight nine." assert embed.cv_chunks(sentences, limit=30) == ["One two three. Four five six.", "Seven eight nine."] words = " ".join(["word"] * 20) found = embed.cv_chunks(words, limit=24) assert all(len(c) <= 24 for c in found) and " ".join(found) == words assert embed.cv_chunks("x" * 50, limit=20) == ["x" * 20, "x" * 20, "x" * 10] def test_no_chunk_of_a_long_cv_runs_past_the_model_window() -> None: cv = "\n\n".join(f"Role {n}. " + "Built Python data platforms. " * 60 for n in range(5)) found = embed.cv_chunks(cv) assert found and all(len(c) <= embed.CV_CHUNK_CHARS for c in found) # ------------------------------------------------------------------- score def test_the_cv_band_is_its_own_and_falls_back_to_fits() -> None: assert E5.cv is not None and E5.cv_band == E5.cv != E5.fit assert Calibration(fit=(0.7, 0.9), taste=(-0.1, 0.2)).cv_band == (0.7, 0.9) with pytest.raises(ValueError): Calibration(fit=(0.7, 0.9), taste=(-0.1, 0.2), cv=(0.9, 0.8)) def test_cv_fit_is_the_best_chunk_scaled_with_the_cv_band_and_no_term_of_the_score( store: Store, config: Config, fake: FakeEmbedder) -> None: backend = role(store, "1", "Backend Python") frontend = role(store, "2", "Frontend React") cv = "Backend engineer, python services.\n\nSome react on the side." plain = store.save_agent("Plain", prompts=[Prompt(name="Data", text="data")]) with_cv = store.save_agent("With CV", prompts=[Prompt(name="Data", text="data")], cv_text=cv) run(config, store, embed.run_embed) run(config, store, score.run_score) texts = {uid: embed.document(r) for uid in (backend, frontend) if (r := store.role(uid)) is not None} chunks = embed.cv_chunks(cv) assert len(chunks) == 1, "two short paragraphs are one chunk" fits = cv_fits(store, with_cv.id) for uid in (backend, frontend): assert fits[uid] == pytest.approx(max(level(texts[uid], c) for c in chunks), abs=1e-6) assert fits[backend] > fits[frontend] assert set(cv_fits(store, plain.id).values()) == {None}, "no CV, no closeness" assert store.scores(plain.id) == store.scores(with_cv.id), "not a term of the score" def test_the_best_chunk_wins(store: Store, config: Config, fake: FakeEmbedder, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(embed, "CV_CHUNK_CHARS", 20) uid = role(store, "1", "Frontend React") owner = store.save_agent("CV", prompts=[Prompt(name="P", text="data")], cv_text="Backend python python.\n\nFrontend react work.") run(config, store, embed.run_embed) run(config, store, score.run_score) text = embed.document(store.role(uid)) # type: ignore[arg-type] assert cv_fits(store, owner.id)[uid] == pytest.approx( level(text, "Frontend react work."), abs=1e-6) def test_a_gated_role_and_a_role_without_a_vector_have_no_cv_fit( store: Store, config: Config, fake: FakeEmbedder) -> None: kept = role(store, "1", "Backend Python", country="DK") gated = role(store, "2", "Backend Python", country="SE") owner = store.save_agent("CV", prompts=[Prompt(name="P", text="python")], gates=[Gate(field="country", values=["DK"])], cv_text="Backend python.") run(config, store, embed.run_embed) later = role(store, "3", "Backend Python later", country="DK") run(config, store, score.run_score) fits = cv_fits(store, owner.id) assert fits[kept] is not None and fits[gated] is None and fits[later] is None def test_the_cv_is_embedded_once_per_text_with_the_query_prefix( store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: embedder = FakeEmbedder() monkeypatch.setattr(embed, "load_embedder", lambda model: embedder) config = Config(database=str(tmp_path / "roledesk.db"), embed={"model": "intfloat/multilingual-e5-small"}) role(store, "1", "Backend Python") owner = store.save_agent("CV", prompts=[Prompt(name="P", text="python")], cv_text="Backend python.") run(config, store, score.run_score) assert embedder.calls == [["query: python"], ["query: Backend python."]] embedder.calls.clear() run(config, store, score.run_score) assert embedder.calls == [], "an unchanged CV is read from the cache" store.save_agent("CV", prompts=owner.prompts, agent_id=owner.id, cv_text="Frontend react.") run(config, store, score.run_score) assert embedder.calls == [["query: Frontend react."]] key = score.cv_key("intfloat/multilingual-e5-small", "Frontend react.") assert store.cv_vectors(owner.id, key) is not None assert store.cv_vectors(owner.id, "stale") is None # ------------------------------------------------------------------- judge def ranked(store: Store, owner: Any, rows: Sequence[tuple[str, float, str, float | None]] ) -> list[str]: uids = [role(store, sid, f"Head of AI {sid}", "Lead the data science team.") for sid, _, _, _ in rows] store.replace_scores(owner.id, [(uid, value, reason, "", fit) for uid, (_, value, reason, fit) in zip(uids, rows, strict=True)]) return uids def window(store: Store) -> list[str]: return [row["uid"] for _, row, _, _ in judge.waiting(store)] def test_the_cut_takes_the_closest_to_the_cv_the_fit_list_leaves_out_and_judge_reads_it( store: Store, config: Config, fake: FakeEmbedder) -> None: """One role in by fit to the prompt, the one closest to the CV that the fit list leaves out by CV closeness, a liked one kept; a gated role is in no list however close, liked or not.""" backend = role(store, "1", "Backend Python", "Backend python python.", country="DK") frontend = role(store, "2", "Frontend React", "Frontend react react.", country="DK") data = role(store, "3", "Data", "Data.", country="DK") other = role(store, "4", "Data again", "Data data.", country="DK") gated = role(store, "5", "Frontend React", "Frontend react react.", country="SE") owner = store.save_agent("Data", prompts=[Prompt(name="Backend", text="python backend")], gates=[Gate(field="country", values=["DK"])], cv_text="Frontend react developer.", limits=Limits(cut_fit_top=1, cut_fit_min=0, cut_cv_top=1, cut_cv_min=0)) store.set_vote(owner.id, data, 1) store.set_vote(owner.id, gated, 1) run(config, store, embed.run_embed) result = run(config, store, score.run_score) assert store.cut(owner.id) == {backend: "fit", frontend: "cv", data: "kept"} assert result["cut"] == 3 with store.transaction() as conn: fits = dict(conn.execute("SELECT uid, fit FROM scores WHERE agent_id = ?", (owner.id,)).fetchall()) assert fits[backend] > fits[other] and fits[gated] is None assert window(store) == [backend, frontend, data] # No count, no list: what is liked is kept all the same. store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, gates=owner.gates, limits=Limits(cut_fit_top=0, cut_cv_top=0)) run(config, store, score.run_score) assert store.cut(owner.id) == {data: "kept"} # ------------------------------------------------------------------- order def test_cv_match_orders_the_unjudged_by_closeness_then_score(store: Store) -> None: owner = store.save_agent("Data", prompts=[Prompt(name="AI", text="Head of AI")], cv_text="VP Data.") judged, close, far, none_a, none_b = ranked(store, owner, [ ("judged", 0.1, "", 0.1), ("close", 0.2, "", 0.9), ("far", 0.9, "", 0.3), ("none_a", 0.5, "", None), ("none_b", 0.6, "", None)]) store.set_judgement(owner.id, judged, "k", 50, 50, 50, ["One."], ["Two."]) page = store.roles_page(agent=owner.id, by_cv=True) assert [r.uid for r in page] == [judged, close, far, none_b, none_a] assert [r.cv_fit for r in page] == [0.1, 0.9, 0.3, None, None] assert [r.uid for r in store.roles(agent=owner.id, by_cv=True)] == [ judged, close, far, none_b, none_a] assert [r.uid for r in store.roles_page(agent=owner.id)] == [ far, none_b, none_a, close, judged], "What I'm looking for ignores it" found = store.role(close, owner.id) assert found is not None and found.cv_fit == 0.9 # --------------------------------------------------------------- migration def test_migration_31_adds_the_column_and_the_table_and_runs_twice(tmp_path: Path) -> None: path = tmp_path / "old.db" Store(path) with sqlite3.connect(path) as conn: conn.execute("DROP TABLE agent_cv_vectors") conn.execute("ALTER TABLE scores DROP COLUMN cv_fit") conn.execute("PRAGMA user_version = 30") assert Store(path).migrations_applied == [31, 32, 33, 34, 35] step = {v: s for v, _, s in MIGRATIONS}[31] with sqlite3.connect(path) as conn: assert callable(step) step(conn) assert "cv_fit" in {r[1] for r in conn.execute("PRAGMA table_info(scores)")} assert conn.execute("SELECT COUNT(*) FROM agent_cv_vectors").fetchone() == (0,)