292 lines
13 KiB
Python
292 lines
13 KiB
Python
"""CV closeness: the CV cut into chunks, embedded and cached like the
|
|
prompts, each role's `cv_fit` beside its score, the judge window it widens and
|
|
the CV match order of the roles not judged yet. The embedder is a fake;
|
|
nothing here downloads a model or reaches a board."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
from collections.abc import Iterator, Sequence
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from app.config import CALIBRATION, Calibration, Config
|
|
from app.jobs import embed, judge, score
|
|
from app.models import Gate, Limits, Prompt
|
|
from app.storage import Store
|
|
from app.storage.schema import MIGRATIONS
|
|
from desk.models import Item
|
|
from pipeline import RunContext
|
|
from prepped import upsert
|
|
|
|
NOW = datetime(2026, 10, 3, 9, 0, tzinfo=UTC)
|
|
E5 = CALIBRATION["intfloat/multilingual-e5-small"]
|
|
WORDS = ("backend", "python", "frontend", "react", "data")
|
|
|
|
|
|
class FakeEmbedder:
|
|
"""One dimension per word of `WORDS` and a constant one, as in
|
|
test_score; records what it was asked to encode."""
|
|
|
|
model_id = "fake"
|
|
|
|
def __init__(self) -> None:
|
|
self.calls: list[list[str]] = []
|
|
|
|
def encode(self, texts: Sequence[str]) -> np.ndarray:
|
|
self.calls.append(list(texts))
|
|
rows = []
|
|
for text in texts:
|
|
lower = text.lower()
|
|
vec = np.asarray([lower.count(w) for w in WORDS] + [0.1], dtype=np.float32)
|
|
rows.append(vec / np.linalg.norm(vec))
|
|
return np.vstack(rows)
|
|
|
|
|
|
@pytest.fixture
|
|
def fake(monkeypatch: pytest.MonkeyPatch) -> FakeEmbedder:
|
|
embedder = FakeEmbedder()
|
|
monkeypatch.setattr(embed, "load_embedder", lambda model: embedder)
|
|
return embedder
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path) -> Iterator[Store]:
|
|
s = Store(tmp_path / "roledesk.db")
|
|
s.clock = lambda: NOW
|
|
yield s
|
|
|
|
|
|
@pytest.fixture
|
|
def config(tmp_path: Path) -> Config:
|
|
return Config(database=str(tmp_path / "roledesk.db"),
|
|
embed={"model": "fake", "calibration": {"fake": E5.model_dump()}})
|
|
|
|
|
|
def role(store: Store, source_id: str, title: str, body: str = "", **raw: Any) -> str:
|
|
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
|
|
title=title, body=body or title, published_at=datetime(2026, 9, 30, tzinfo=UTC),
|
|
raw={"company": "Acme", **raw})
|
|
item.source = "ats"
|
|
upsert(store, item)
|
|
return item.uid
|
|
|
|
|
|
def run(config: Config, store: Store, job: Any) -> dict[str, Any]:
|
|
return dict(job(config, store, context=RunContext()))
|
|
|
|
|
|
def cv_fits(store: Store, agent_id: int) -> dict[str, float | None]:
|
|
with store.transaction() as conn:
|
|
return dict(conn.execute(
|
|
"SELECT uid, cv_fit FROM scores WHERE agent_id = ?", (agent_id,)).fetchall())
|
|
|
|
|
|
def level(text: str, against: str) -> float:
|
|
role_vec, other = FakeEmbedder().encode([text, against])
|
|
lo, hi = E5.cv_band
|
|
return min(max((score.cosine(role_vec, other) - lo) / (hi - lo), 0.0), 1.0)
|
|
|
|
|
|
# ------------------------------------------------------------------ chunks
|
|
|
|
def test_a_cv_is_cut_at_paragraphs_and_joined_again_up_to_the_limit() -> None:
|
|
cv = "Experience\n\nLed data.\n\n\n Skills: Python \n\n"
|
|
assert embed.cv_chunks(cv, limit=40) == ["Experience\n\nLed data.\n\nSkills: Python"]
|
|
assert embed.cv_chunks(cv, limit=21) == ["Experience\n\nLed data.", "Skills: Python"]
|
|
assert embed.cv_chunks(" \n\n ") == []
|
|
|
|
|
|
def test_a_long_paragraph_is_cut_by_line_then_sentence_then_word() -> None:
|
|
lines = "\n".join(["a" * 30, "b" * 30, "c" * 30])
|
|
assert embed.cv_chunks(lines, limit=65) == ["a" * 30 + "\n" + "b" * 30, "c" * 30]
|
|
sentences = "One two three. Four five six. Seven eight nine."
|
|
assert embed.cv_chunks(sentences, limit=30) == ["One two three. Four five six.",
|
|
"Seven eight nine."]
|
|
words = " ".join(["word"] * 20)
|
|
found = embed.cv_chunks(words, limit=24)
|
|
assert all(len(c) <= 24 for c in found) and " ".join(found) == words
|
|
assert embed.cv_chunks("x" * 50, limit=20) == ["x" * 20, "x" * 20, "x" * 10]
|
|
|
|
|
|
def test_no_chunk_of_a_long_cv_runs_past_the_model_window() -> None:
|
|
cv = "\n\n".join(f"Role {n}. " + "Built Python data platforms. " * 60 for n in range(5))
|
|
found = embed.cv_chunks(cv)
|
|
assert found and all(len(c) <= embed.CV_CHUNK_CHARS for c in found)
|
|
|
|
|
|
# ------------------------------------------------------------------- score
|
|
|
|
def test_the_cv_band_is_its_own_and_falls_back_to_fits() -> None:
|
|
assert E5.cv is not None and E5.cv_band == E5.cv != E5.fit
|
|
assert Calibration(fit=(0.7, 0.9), taste=(-0.1, 0.2)).cv_band == (0.7, 0.9)
|
|
with pytest.raises(ValueError):
|
|
Calibration(fit=(0.7, 0.9), taste=(-0.1, 0.2), cv=(0.9, 0.8))
|
|
|
|
|
|
def test_cv_fit_is_the_best_chunk_scaled_with_the_cv_band_and_no_term_of_the_score(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
backend = role(store, "1", "Backend Python")
|
|
frontend = role(store, "2", "Frontend React")
|
|
cv = "Backend engineer, python services.\n\nSome react on the side."
|
|
plain = store.save_agent("Plain", prompts=[Prompt(name="Data", text="data")])
|
|
with_cv = store.save_agent("With CV", prompts=[Prompt(name="Data", text="data")],
|
|
cv_text=cv)
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
|
|
texts = {uid: embed.document(r) for uid in (backend, frontend)
|
|
if (r := store.role(uid)) is not None}
|
|
chunks = embed.cv_chunks(cv)
|
|
assert len(chunks) == 1, "two short paragraphs are one chunk"
|
|
fits = cv_fits(store, with_cv.id)
|
|
for uid in (backend, frontend):
|
|
assert fits[uid] == pytest.approx(max(level(texts[uid], c) for c in chunks), abs=1e-6)
|
|
assert fits[backend] > fits[frontend]
|
|
assert set(cv_fits(store, plain.id).values()) == {None}, "no CV, no closeness"
|
|
assert store.scores(plain.id) == store.scores(with_cv.id), "not a term of the score"
|
|
|
|
|
|
def test_the_best_chunk_wins(store: Store, config: Config, fake: FakeEmbedder,
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(embed, "CV_CHUNK_CHARS", 20)
|
|
uid = role(store, "1", "Frontend React")
|
|
owner = store.save_agent("CV", prompts=[Prompt(name="P", text="data")],
|
|
cv_text="Backend python python.\n\nFrontend react work.")
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
text = embed.document(store.role(uid)) # type: ignore[arg-type]
|
|
assert cv_fits(store, owner.id)[uid] == pytest.approx(
|
|
level(text, "Frontend react work."), abs=1e-6)
|
|
|
|
|
|
def test_a_gated_role_and_a_role_without_a_vector_have_no_cv_fit(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
kept = role(store, "1", "Backend Python", country="DK")
|
|
gated = role(store, "2", "Backend Python", country="SE")
|
|
owner = store.save_agent("CV", prompts=[Prompt(name="P", text="python")],
|
|
gates=[Gate(field="country", values=["DK"])],
|
|
cv_text="Backend python.")
|
|
run(config, store, embed.run_embed)
|
|
later = role(store, "3", "Backend Python later", country="DK")
|
|
run(config, store, score.run_score)
|
|
fits = cv_fits(store, owner.id)
|
|
assert fits[kept] is not None and fits[gated] is None and fits[later] is None
|
|
|
|
|
|
def test_the_cv_is_embedded_once_per_text_with_the_query_prefix(
|
|
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
embedder = FakeEmbedder()
|
|
monkeypatch.setattr(embed, "load_embedder", lambda model: embedder)
|
|
config = Config(database=str(tmp_path / "roledesk.db"),
|
|
embed={"model": "intfloat/multilingual-e5-small"})
|
|
role(store, "1", "Backend Python")
|
|
owner = store.save_agent("CV", prompts=[Prompt(name="P", text="python")],
|
|
cv_text="Backend python.")
|
|
run(config, store, score.run_score)
|
|
assert embedder.calls == [["query: python"], ["query: Backend python."]]
|
|
|
|
embedder.calls.clear()
|
|
run(config, store, score.run_score)
|
|
assert embedder.calls == [], "an unchanged CV is read from the cache"
|
|
|
|
store.save_agent("CV", prompts=owner.prompts, agent_id=owner.id, cv_text="Frontend react.")
|
|
run(config, store, score.run_score)
|
|
assert embedder.calls == [["query: Frontend react."]]
|
|
key = score.cv_key("intfloat/multilingual-e5-small", "Frontend react.")
|
|
assert store.cv_vectors(owner.id, key) is not None
|
|
assert store.cv_vectors(owner.id, "stale") is None
|
|
|
|
|
|
# ------------------------------------------------------------------- judge
|
|
|
|
def ranked(store: Store, owner: Any, rows: Sequence[tuple[str, float, str, float | None]]
|
|
) -> list[str]:
|
|
uids = [role(store, sid, f"Head of AI {sid}", "Lead the data science team.")
|
|
for sid, _, _, _ in rows]
|
|
store.replace_scores(owner.id, [(uid, value, reason, "", fit)
|
|
for uid, (_, value, reason, fit) in zip(uids, rows,
|
|
strict=True)])
|
|
return uids
|
|
|
|
|
|
def window(store: Store) -> list[str]:
|
|
return [row["uid"] for _, row, _, _ in judge.waiting(store)]
|
|
|
|
|
|
def test_the_cut_takes_the_closest_to_the_cv_the_fit_list_leaves_out_and_judge_reads_it(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
"""One role in by fit to the prompt, the one closest to the CV that the
|
|
fit list leaves out by CV closeness, a liked one kept; a gated role is in
|
|
no list however close, liked or not."""
|
|
backend = role(store, "1", "Backend Python", "Backend python python.", country="DK")
|
|
frontend = role(store, "2", "Frontend React", "Frontend react react.", country="DK")
|
|
data = role(store, "3", "Data", "Data.", country="DK")
|
|
other = role(store, "4", "Data again", "Data data.", country="DK")
|
|
gated = role(store, "5", "Frontend React", "Frontend react react.", country="SE")
|
|
owner = store.save_agent("Data", prompts=[Prompt(name="Backend", text="python backend")],
|
|
gates=[Gate(field="country", values=["DK"])],
|
|
cv_text="Frontend react developer.",
|
|
limits=Limits(cut_fit_top=1, cut_fit_min=0, cut_cv_top=1,
|
|
cut_cv_min=0))
|
|
store.set_vote(owner.id, data, 1)
|
|
store.set_vote(owner.id, gated, 1)
|
|
run(config, store, embed.run_embed)
|
|
result = run(config, store, score.run_score)
|
|
|
|
assert store.cut(owner.id) == {backend: "fit", frontend: "cv", data: "kept"}
|
|
assert result["cut"] == 3
|
|
with store.transaction() as conn:
|
|
fits = dict(conn.execute("SELECT uid, fit FROM scores WHERE agent_id = ?",
|
|
(owner.id,)).fetchall())
|
|
assert fits[backend] > fits[other] and fits[gated] is None
|
|
assert window(store) == [backend, frontend, data]
|
|
|
|
# No count, no list: what is liked is kept all the same.
|
|
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
|
gates=owner.gates, limits=Limits(cut_fit_top=0, cut_cv_top=0))
|
|
run(config, store, score.run_score)
|
|
assert store.cut(owner.id) == {data: "kept"}
|
|
|
|
|
|
# ------------------------------------------------------------------- order
|
|
|
|
def test_cv_match_orders_the_unjudged_by_closeness_then_score(store: Store) -> None:
|
|
owner = store.save_agent("Data", prompts=[Prompt(name="AI", text="Head of AI")],
|
|
cv_text="VP Data.")
|
|
judged, close, far, none_a, none_b = ranked(store, owner, [
|
|
("judged", 0.1, "", 0.1), ("close", 0.2, "", 0.9), ("far", 0.9, "", 0.3),
|
|
("none_a", 0.5, "", None), ("none_b", 0.6, "", None)])
|
|
store.set_judgement(owner.id, judged, "k", 50, 50, 50, ["One."], ["Two."])
|
|
page = store.roles_page(agent=owner.id, by_cv=True)
|
|
assert [r.uid for r in page] == [judged, close, far, none_b, none_a]
|
|
assert [r.cv_fit for r in page] == [0.1, 0.9, 0.3, None, None]
|
|
assert [r.uid for r in store.roles(agent=owner.id, by_cv=True)] == [
|
|
judged, close, far, none_b, none_a]
|
|
assert [r.uid for r in store.roles_page(agent=owner.id)] == [
|
|
far, none_b, none_a, close, judged], "What I'm looking for ignores it"
|
|
found = store.role(close, owner.id)
|
|
assert found is not None and found.cv_fit == 0.9
|
|
|
|
|
|
# --------------------------------------------------------------- migration
|
|
|
|
def test_migration_31_adds_the_column_and_the_table_and_runs_twice(tmp_path: Path) -> None:
|
|
path = tmp_path / "old.db"
|
|
Store(path)
|
|
with sqlite3.connect(path) as conn:
|
|
conn.execute("DROP TABLE agent_cv_vectors")
|
|
conn.execute("ALTER TABLE scores DROP COLUMN cv_fit")
|
|
conn.execute("PRAGMA user_version = 30")
|
|
assert Store(path).migrations_applied == [31, 32, 33, 34, 35]
|
|
step = {v: s for v, _, s in MIGRATIONS}[31]
|
|
with sqlite3.connect(path) as conn:
|
|
assert callable(step)
|
|
step(conn)
|
|
assert "cv_fit" in {r[1] for r in conn.execute("PRAGMA table_info(scores)")}
|
|
assert conn.execute("SELECT COUNT(*) FROM agent_cv_vectors").fetchone() == (0,)
|