748 lines
32 KiB
Python
748 lines
32 KiB
Python
"""`embed`, `score` and the pull cascade, with the embedding backend replaced
|
|
by a deterministic fake. Nothing here downloads a model or reaches a board."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from collections.abc import Iterator, Sequence
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from desk.models import Item # noqa: E402
|
|
from pipeline import FINAL, JOBS, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, Tally # noqa: E402
|
|
from plugins.ats import ATSSource # noqa: E402
|
|
from app.config import AgentSeed, Config # noqa: E402
|
|
from app.jobs import cascade, embed, score # noqa: E402
|
|
from app.models import Candidate, Gate, Weights # noqa: E402
|
|
from app.storage import Store # noqa: E402
|
|
|
|
NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
|
|
|
|
#: The fake's whole vocabulary: one dimension per word, plus a constant one so
|
|
#: no text is the zero vector.
|
|
WORDS = ("backend", "python", "frontend", "react", "data")
|
|
|
|
|
|
class FakeEmbedder:
|
|
model_id = "fake"
|
|
|
|
def __init__(self) -> None:
|
|
self.calls: list[list[str]] = []
|
|
|
|
def encode(self, texts: Sequence[str]) -> np.ndarray:
|
|
self.calls.append(list(texts))
|
|
rows = []
|
|
for text in texts:
|
|
lower = text.lower()
|
|
vec = np.asarray([lower.count(w) for w in WORDS] + [0.1], dtype=np.float32)
|
|
rows.append(vec / np.linalg.norm(vec))
|
|
return np.vstack(rows)
|
|
|
|
|
|
class Clock:
|
|
def __init__(self) -> None:
|
|
self.at = NOW
|
|
|
|
def __call__(self) -> datetime:
|
|
return self.at
|
|
|
|
|
|
@pytest.fixture
|
|
def fake(monkeypatch: pytest.MonkeyPatch) -> FakeEmbedder:
|
|
embedder = FakeEmbedder()
|
|
monkeypatch.setattr(embed, "load_embedder", lambda model: embedder)
|
|
return embedder
|
|
|
|
|
|
@pytest.fixture
|
|
def clock() -> Clock:
|
|
return Clock()
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path, clock: Clock) -> Iterator[Store]:
|
|
s = Store(tmp_path / "roledesk.db")
|
|
s.clock = clock
|
|
yield s
|
|
|
|
|
|
@pytest.fixture
|
|
def config(tmp_path: Path) -> Config:
|
|
return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
|
|
|
|
|
|
def role(store: Store, source_id: str, title: str = "Backend Engineer", *,
|
|
company: str = "Acme", body: str = "",
|
|
published_at: datetime = datetime(2026, 9, 20, tzinfo=UTC), **raw: Any) -> str:
|
|
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
|
|
title=title, body=body, published_at=published_at,
|
|
raw={"company": company, **raw})
|
|
item.source = "ats"
|
|
store.upsert_role(item)
|
|
return item.uid
|
|
|
|
|
|
def run(config: Config, store: Store, job: Any) -> dict[str, Any]:
|
|
notices: list[Any] = []
|
|
result = dict(job(config, store, context=RunContext(on_notice=notices.append)))
|
|
result["notices"] = [n.code for n in notices]
|
|
return result
|
|
|
|
|
|
# ------------------------------------------------------------------ embed
|
|
|
|
def test_embed_writes_a_vector_per_open_role_once(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
role(store, "1")
|
|
closed = role(store, "2")
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (NOW.isoformat(), closed))
|
|
|
|
assert run(config, store, embed.run_embed)["embedded"] == 1
|
|
assert run(config, store, embed.run_embed)["embedded"] == 0
|
|
assert store.vector_counts() == {"fake": 1}
|
|
|
|
|
|
def test_embed_without_sentence_transformers_says_so_and_does_nothing(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
def missing(model: str) -> Any:
|
|
raise ImportError("no sentence_transformers")
|
|
|
|
monkeypatch.setattr(embed, "load_embedder", missing)
|
|
role(store, "1")
|
|
|
|
result = run(config, store, embed.run_embed)
|
|
|
|
assert result["embedded"] == 0 and result["notices"] == ["embed.unavailable"]
|
|
assert store.vector_counts() == {}
|
|
|
|
|
|
# -------------------------------------------------------- e5 prefixes
|
|
|
|
@pytest.mark.parametrize("model, wants", [
|
|
("intfloat/multilingual-e5-small", True),
|
|
("intfloat/e5-base-v2", True),
|
|
("intfloat/e5-large", True),
|
|
("sentence-transformers/all-MiniLM-L6-v2", False),
|
|
("sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", False),
|
|
("fake", False),
|
|
("some-org/e5models-are-not-this", False),
|
|
])
|
|
def test_wants_e5_prefix_matches_the_e5_family_by_name(model: str, wants: bool) -> None:
|
|
assert embed.wants_e5_prefix(model) == wants
|
|
|
|
|
|
def test_role_text_and_intent_text_add_the_e5_prefixes_only_for_e5(
|
|
store: Store) -> None:
|
|
uid = role(store, "1", "Backend Engineer", body="Python services")
|
|
r = store.role(uid)
|
|
assert r is not None
|
|
assert embed.role_text("fake", r) == embed.document(r)
|
|
assert embed.role_text("intfloat/multilingual-e5-small", r) == f"passage: {embed.document(r)}"
|
|
assert embed.intent_text("fake", "python backend") == "python backend"
|
|
assert embed.intent_text("intfloat/multilingual-e5-small", "python backend") == \
|
|
"query: python backend"
|
|
|
|
|
|
class FakeE5Embedder:
|
|
"""A second fake, standing in for a genuinely different model: its own
|
|
(narrower) vector width and an e5-shaped model id, so it exercises both
|
|
the prefixing and the re-embedding-on-model-change paths without
|
|
downloading anything."""
|
|
|
|
model_id = "intfloat/multilingual-e5-small"
|
|
|
|
def __init__(self) -> None:
|
|
self.calls: list[list[str]] = []
|
|
|
|
def encode(self, texts: Sequence[str]) -> np.ndarray:
|
|
self.calls.append(list(texts))
|
|
rows = []
|
|
for text in texts:
|
|
lower = text.lower()
|
|
vec = np.asarray([lower.count(w) for w in WORDS[:2]] + [0.1, 0.2], dtype=np.float32)
|
|
rows.append(vec / np.linalg.norm(vec))
|
|
return np.vstack(rows)
|
|
|
|
|
|
def test_embed_prefixes_role_text_for_an_e5_model(
|
|
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
e5 = FakeE5Embedder()
|
|
monkeypatch.setattr(embed, "load_embedder", lambda model: e5)
|
|
config = Config(database=str(tmp_path / "roledesk.db"),
|
|
embed={"model": "intfloat/multilingual-e5-small"})
|
|
role(store, "1", "Backend Engineer", body="Python services")
|
|
|
|
run(config, store, embed.run_embed)
|
|
|
|
assert e5.calls and e5.calls[0][0].startswith("passage: Backend Engineer")
|
|
|
|
|
|
def test_score_prefixes_the_intent_for_an_e5_model(
|
|
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
e5 = FakeE5Embedder()
|
|
monkeypatch.setattr(embed, "load_embedder", lambda model: e5)
|
|
config = Config(database=str(tmp_path / "roledesk.db"),
|
|
embed={"model": "intfloat/multilingual-e5-small"})
|
|
role(store, "1")
|
|
store.save_agent("Backend", "python backend")
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
assert e5.calls == [["query: python backend"]]
|
|
|
|
|
|
# ------------------------------------------------------- model changes
|
|
|
|
def test_changing_the_embed_model_drops_old_vectors_and_reembeds(
|
|
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The doc says a model change re-embeds; this is the guarantee that
|
|
actually makes that true, and the one that stops a run ever mixing two
|
|
models' vectors, which would be two different geometries under one
|
|
dimension count and garbage cosines."""
|
|
fake_a, fake_e5 = FakeEmbedder(), FakeE5Embedder()
|
|
embedders = {"fake": fake_a, "intfloat/multilingual-e5-small": fake_e5}
|
|
monkeypatch.setattr(embed, "load_embedder", lambda model: embedders[model])
|
|
role(store, "1")
|
|
role(store, "2")
|
|
|
|
config_a = Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
|
|
run(config_a, store, embed.run_embed)
|
|
assert store.vector_counts() == {"fake": 2}
|
|
dim_a = next(iter(store.vectors("fake").values())).shape[-1]
|
|
|
|
config_e5 = Config(database=str(tmp_path / "roledesk.db"),
|
|
embed={"model": "intfloat/multilingual-e5-small"})
|
|
result = run(config_e5, store, embed.run_embed)
|
|
|
|
# The old model's rows are gone, not just shadowed: `vectors(model)`
|
|
# filters by model, so a leftover row could never be read back mixed in,
|
|
# but drop_vectors is what stops it existing at all.
|
|
assert store.vector_counts() == {"intfloat/multilingual-e5-small": 2}
|
|
assert store.vectors("fake") == {}
|
|
assert result["dropped"] == 2
|
|
dim_e5 = next(iter(store.vectors("intfloat/multilingual-e5-small").values())).shape[-1]
|
|
assert dim_a != dim_e5, "the fakes must differ in width for this test to mean anything"
|
|
|
|
|
|
def test_changing_the_embed_model_recomputes_the_agents_cached_intent_vector(
|
|
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""`intent_vector` only returns a cache hit when `intent_key` -- which
|
|
hashes the model in -- matches what is stored, so a model change is a
|
|
cache miss and `_intent` re-encodes under the new model, never handing a
|
|
stale-dimension vector into a run's cosines."""
|
|
fake_a, fake_e5 = FakeEmbedder(), FakeE5Embedder()
|
|
embedders = {"fake": fake_a, "intfloat/multilingual-e5-small": fake_e5}
|
|
monkeypatch.setattr(embed, "load_embedder", lambda model: embedders[model])
|
|
uid = role(store, "1")
|
|
agent = store.save_agent("Backend", "python backend")
|
|
|
|
config_a = Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
|
|
run(config_a, store, embed.run_embed)
|
|
run(config_a, store, score.run_score)
|
|
vec_a = store.intent_vector(agent.id, score.intent_key("fake", "python backend"))
|
|
assert vec_a is not None
|
|
|
|
config_e5 = Config(database=str(tmp_path / "roledesk.db"),
|
|
embed={"model": "intfloat/multilingual-e5-small"})
|
|
run(config_e5, store, embed.run_embed)
|
|
run(config_e5, store, score.run_score)
|
|
|
|
# The old key no longer resolves (the row moved on); the new key does,
|
|
# at the new model's own width, and scoring this role did not raise.
|
|
assert store.intent_vector(agent.id, score.intent_key("fake", "python backend")) is None
|
|
vec_e5 = store.intent_vector(
|
|
agent.id, score.intent_key("intfloat/multilingual-e5-small", "python backend"))
|
|
assert vec_e5 is not None
|
|
assert vec_e5.shape != vec_a.shape
|
|
assert uid in store.scores(agent.id)
|
|
|
|
|
|
# ------------------------------------------------------------------ terms
|
|
|
|
def test_recency_halves_every_fourteen_days() -> None:
|
|
assert score.recency(NOW.isoformat(), NOW) == 1.0
|
|
assert score.recency((NOW - timedelta(days=14)).isoformat(), NOW) == pytest.approx(0.5)
|
|
assert score.recency((NOW - timedelta(days=28)).isoformat(), NOW) == pytest.approx(0.25)
|
|
|
|
|
|
def test_a_role_without_a_vector_is_scored_without_fit_unless_fit_is_all() -> None:
|
|
candidate = Candidate(uid="u", posted_at=NOW.isoformat(), affinity=0)
|
|
value, reason = score.score_one(candidate, [], Weights(), None, NOW)
|
|
assert reason == "" and value == pytest.approx((0.2 * 1.0 + 0.2 / 3) / 0.4)
|
|
|
|
only_fit = Weights(fit=1, recency=0, affinity=0)
|
|
assert score.score_one(candidate, [], only_fit, None, NOW) == (0.0, "no vector")
|
|
|
|
|
|
# ------------------------------------------------------------------ score
|
|
|
|
def test_a_require_gate_keeps_the_row_with_a_reason(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
berlin = role(store, "1", place="Berlin, Germany")
|
|
munich = role(store, "2", place="Munich")
|
|
nowhere = role(store, "3")
|
|
agent = store.save_agent("Berlin", "python backend",
|
|
[Gate(field="place", values=["Berlin"])])
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[berlin][1] == "" and scores[berlin][0] > 0
|
|
assert scores[munich] == (0.0, "place not Berlin")
|
|
# An empty column is no evidence: the role ranks, and extract can reach it.
|
|
assert scores[nowhere][1] == "" and scores[nowhere][0] > 0
|
|
shown = store.roles(agent=agent.id)
|
|
assert {r.uid for r in shown} == {berlin, nowhere}
|
|
assert {r.uid for r in store.roles(agent=agent.id, gated=True)} == {berlin, munich, nowhere}
|
|
|
|
|
|
def test_any_gates_are_one_group_a_role_must_match_one_of(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
""""In Denmark, or remote": a role passes on either, is gated when it has
|
|
evidence against both, and ranks when both columns are empty."""
|
|
copenhagen = role(store, "1", place="Copenhagen, Denmark")
|
|
remote_de = role(store, "2", place="Berlin, Germany", remote="remote")
|
|
onsite_de = role(store, "3", place="Berlin, Germany", remote="onsite")
|
|
hybrid_nowhere = role(store, "4", place="Hybrid") # remote from place, no country
|
|
blank = role(store, "5")
|
|
agent = store.save_agent("Copenhagen or remote", "", [
|
|
Gate(field="country", values=["DK"], mode="any"),
|
|
Gate(field="remote", values=["remote"], mode="any")])
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[copenhagen][1] == "" and scores[remote_de][1] == ""
|
|
assert scores[onsite_de] == (0.0, "none of country DK, remote remote")
|
|
assert scores[blank][1] == ""
|
|
# "Hybrid" is a country-less line the remote lookup reads as hybrid, so
|
|
# the group has evidence against remote and none for country: gated.
|
|
assert scores[hybrid_nowhere][1].startswith("none of")
|
|
|
|
|
|
def test_failed_gates_reads_evidence_only() -> None:
|
|
gates = [Gate(field="country", values=["DK"]),
|
|
Gate(field="remote", values=["remote"], mode="any"),
|
|
Gate(field="seniority", values=["senior"], mode="any"),
|
|
Gate(field="function", values=["engineering"], mode="prefer")]
|
|
def cand(*states: int) -> Candidate:
|
|
return Candidate(uid="u", posted_at=NOW.isoformat(), affinity=0, gates=list(states))
|
|
assert score.failed_gates(cand(1, 1, 0, 0), gates) == []
|
|
assert score.failed_gates(cand(-1, -1, -1, 0), gates) == []
|
|
assert score.failed_gates(cand(0, 0, -1, 1), gates) == [
|
|
"country not DK", "none of remote remote, seniority senior"]
|
|
assert score.failed_gates(cand(1, 0, 0, 1), gates) == [
|
|
"none of remote remote, seniority senior"]
|
|
|
|
|
|
def test_a_prefer_gate_adds_its_weight_to_the_score(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
remote = role(store, "1", remote="remote")
|
|
office = role(store, "2", remote="onsite")
|
|
agent = store.save_agent("Remote", "", [Gate(field="remote", values=["remote"],
|
|
mode="prefer", weight=0.5)])
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
# No intent, so fit is out: recency 0.2 + affinity 0.2 + the gate's 0.5.
|
|
assert scores[remote][0] - scores[office][0] == pytest.approx(0.5 / 0.9, abs=1e-5)
|
|
assert scores[office][1] == ""
|
|
|
|
|
|
def test_a_favourite_skips_the_gates_and_is_floored(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
store.add_company("Fave")
|
|
fave = role(store, "1", company="Fave", place="Munich")
|
|
agent = store.save_agent("Berlin", "", [Gate(field="place", values=["Berlin"])])
|
|
store.set_affinity(agent.id, "fave", 2)
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
value, reason = store.scores(agent.id)[fave]
|
|
assert reason == "" and value >= 0.9
|
|
|
|
|
|
def test_a_hidden_company_is_never_scored_for_the_agent_that_hid_it(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
store.add_company("Muted")
|
|
muted = role(store, "1", company="Muted")
|
|
kept = role(store, "2")
|
|
agent = store.save_agent("Any")
|
|
other = store.save_agent("Other")
|
|
store.set_affinity(agent.id, "muted", -1)
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
assert set(store.scores(agent.id)) == {kept}
|
|
assert muted not in {r.uid for r in store.roles(agent=agent.id, gated=True)}
|
|
# The other agent did not hide it, so it is scored for that one.
|
|
assert set(store.scores(other.id)) == {muted, kept}
|
|
|
|
|
|
def test_an_older_role_scores_lower(
|
|
store: Store, config: Config, clock: Clock, fake: FakeEmbedder) -> None:
|
|
clock.at = NOW - timedelta(days=14)
|
|
old = role(store, "1")
|
|
clock.at = NOW
|
|
new = role(store, "2")
|
|
agent = store.save_agent("Any")
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[new][0] > scores[old][0]
|
|
|
|
|
|
def test_recency_counts_from_posted_at_not_first_seen(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
"""A trusted board's backlog, first seen today, must not score as fresh:
|
|
recency has to read the board's own (trusted) posted date, not the
|
|
moment the app happened to pull it in."""
|
|
stale = role(store, "1", board="greenhouse",
|
|
published_at=NOW - timedelta(days=28))
|
|
fresh = role(store, "2") # untrusted host: posted_at falls back to first_seen (NOW)
|
|
agent = store.save_agent("Any")
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[fresh][0] > scores[stale][0]
|
|
|
|
|
|
def test_two_agents_rank_the_same_roles_the_other_way_round(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
back = role(store, "1", "Backend Engineer", body="Python services")
|
|
front = role(store, "2", "Frontend Engineer", body="React apps")
|
|
only_fit = Weights(fit=1, recency=0, affinity=0)
|
|
backend = store.save_agent("Backend", "python backend developer", weights=only_fit)
|
|
frontend = store.save_agent("Frontend", "react frontend developer", weights=only_fit)
|
|
|
|
run(config, store, embed.run_embed)
|
|
result = run(config, store, score.run_score)
|
|
|
|
assert result["agents"] == 2
|
|
# Fit alone, scaled per agent: each agent's closer role is 1, the other 0.
|
|
assert store.scores(backend.id)[back][0] == 1.0
|
|
assert store.scores(backend.id)[front][0] == 0.0
|
|
assert store.scores(frontend.id)[front][0] == 1.0
|
|
assert store.scores(frontend.id)[back][0] == 0.0
|
|
|
|
|
|
# ------------------------------------------------------------------ taste
|
|
|
|
def _unit(*words: str) -> np.ndarray:
|
|
return FakeEmbedder().encode([" ".join(words)])[0]
|
|
|
|
|
|
def test_taste_direction_is_liked_minus_passed_and_none_without_votes() -> None:
|
|
vectors = {"a": _unit("backend"), "b": _unit("frontend"), "c": _unit("data")}
|
|
|
|
assert score.taste_direction(vectors, {}) is None
|
|
assert score.taste_direction(vectors, {"a": 0}) is None # neutral says nothing
|
|
assert score.taste_direction(vectors, {"gone": 1}) is None # no vector to point at
|
|
|
|
liked_only = score.taste_direction(vectors, {"a": 1, "c": 1})
|
|
assert liked_only is not None
|
|
assert liked_only == pytest.approx((vectors["a"] + vectors["c"]) / 2)
|
|
|
|
passed_only = score.taste_direction(vectors, {"b": -1})
|
|
assert passed_only is not None
|
|
assert passed_only == pytest.approx(-vectors["b"])
|
|
|
|
both = score.taste_direction(vectors, {"a": 1, "b": -1, "c": 0})
|
|
assert both is not None
|
|
assert both == pytest.approx(vectors["a"] - vectors["b"])
|
|
|
|
|
|
def test_taste_ranks_a_role_near_the_liked_one_above_one_near_the_passed(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
liked = role(store, "1", "Backend Engineer", body="Python backend")
|
|
passed = role(store, "2", "Frontend Engineer", body="React frontend")
|
|
near_liked = role(store, "3", "Platform Engineer", body="Python backend services")
|
|
near_passed = role(store, "4", "Web Engineer", body="React frontend apps")
|
|
# No intent: with taste alone weighted, the votes are all that orders them.
|
|
agent = store.save_agent("Taste", "", weights=Weights(fit=0, recency=0, affinity=0,
|
|
taste=1))
|
|
store.set_vote(agent.id, liked, 1)
|
|
store.set_vote(agent.id, passed, -1)
|
|
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[near_liked][0] > scores[near_passed][0]
|
|
assert scores[liked][0] == 1.0 and scores[passed][0] == 0.0
|
|
assert all(reason == "" for _, reason in scores.values())
|
|
|
|
|
|
def test_without_votes_taste_leaves_the_denominator(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
uid = role(store, "1")
|
|
agent = store.save_agent("Any", weights=Weights(fit=0, recency=0, affinity=0, taste=1))
|
|
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
|
|
# Taste is the only weighted term and there is no taste, so no weights.
|
|
assert store.scores(agent.id)[uid] == (0.0, "no weights")
|
|
|
|
|
|
def test_a_closed_liked_role_still_shapes_the_taste(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
liked = role(store, "1", "Backend Engineer", body="Python backend")
|
|
near = role(store, "2", "Platform Engineer", body="Python backend")
|
|
far = role(store, "3", "Web Engineer", body="React frontend")
|
|
agent = store.save_agent("Taste", "", weights=Weights(fit=0, recency=0, affinity=0,
|
|
taste=1))
|
|
run(config, store, embed.run_embed)
|
|
store.set_vote(agent.id, liked, 1)
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (NOW.isoformat(), liked))
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert liked not in scores
|
|
assert scores[near][0] == 1.0 and scores[far][0] == 0.0
|
|
|
|
|
|
# ------------------------------------------------------------ fit scaling
|
|
|
|
def test_scale_maps_the_closest_to_one_and_the_farthest_to_zero() -> None:
|
|
# The spread the last real run showed under `(cos + 1) / 2`, on
|
|
# multilingual-e5-small.
|
|
assert score.scale({"a": 0.87, "b": 0.95, "c": 0.91}) == pytest.approx(
|
|
{"a": 0.0, "b": 1.0, "c": 0.5})
|
|
assert score.scale({"only": 0.3}) == {"only": 1.0}
|
|
assert score.scale({"a": 0.4, "b": 0.4}) == {"a": 1.0, "b": 1.0}
|
|
assert score.scale({}) == {}
|
|
|
|
|
|
def test_a_single_candidate_gets_full_fit(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
uid = role(store, "1", "Frontend Engineer", body="React apps")
|
|
agent = store.save_agent("Backend", "python backend developer",
|
|
weights=Weights(fit=1, recency=0, affinity=0))
|
|
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
|
|
assert store.scores(agent.id)[uid] == (1.0, "")
|
|
|
|
|
|
def test_a_gated_role_does_not_stretch_the_fit_range(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
close = role(store, "1", "Backend Engineer", body="Python backend", place="Berlin")
|
|
far = role(store, "2", "Data Engineer", body="Data", place="Berlin")
|
|
# Farther from the intent than either, but gated out by place.
|
|
role(store, "3", "Frontend Engineer", body="React react", place="Munich")
|
|
agent = store.save_agent("Berlin", "python backend",
|
|
[Gate(field="place", values=["Berlin"])],
|
|
weights=Weights(fit=1, recency=0, affinity=0))
|
|
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[close][0] == 1.0 and scores[far][0] == 0.0
|
|
|
|
|
|
def test_a_small_prefer_gate_no_longer_outweighs_fit(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
"""The flaw the last real run showed: real cosines sit close together, so
|
|
under `(cos + 1) / 2` a prefer gate decided the order over fit. Here the
|
|
cosines are about 0.95 and 0.87: the old fit terms differ by 0.04, which
|
|
at weight 0.6 is less than the gate's 0.05, so the remote role won. Scaled,
|
|
the closer role wins."""
|
|
close = role(store, "1", "Backend Engineer", body="Python backend", remote="onsite")
|
|
far = role(store, "2", "Backend Engineer", body="Python backend data", remote="remote")
|
|
agent = store.save_agent("Backend", "python backend",
|
|
[Gate(field="remote", values=["remote"], mode="prefer",
|
|
weight=0.05)])
|
|
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[close][0] > scores[far][0]
|
|
|
|
|
|
def test_the_intent_vector_is_cached_until_the_intent_changes(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
role(store, "1")
|
|
agent = store.save_agent("Backend", "python backend")
|
|
|
|
run(config, store, score.run_score)
|
|
run(config, store, score.run_score)
|
|
assert fake.calls == [["python backend"]]
|
|
|
|
store.save_agent("Backend", "react frontend", agent_id=agent.id)
|
|
run(config, store, score.run_score)
|
|
assert fake.calls[-1] == ["react frontend"]
|
|
|
|
|
|
def test_a_disabled_agent_is_not_scored(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
role(store, "1")
|
|
agent = store.save_agent("Off", enabled=False)
|
|
assert run(config, store, score.run_score)["agents"] == 0
|
|
assert store.scores(agent.id) == {}
|
|
|
|
|
|
def test_starter_agents_are_imported_once(store: Store, tmp_path: Path) -> None:
|
|
config = Config(database=str(tmp_path / "roledesk.db"),
|
|
agents=[AgentSeed(name="A"), AgentSeed(name="B")])
|
|
assert score.seed_agents(config, store) == ["A", "B"]
|
|
for agent in store.agents():
|
|
store.delete_agent(agent.id)
|
|
assert score.seed_agents(config, store) == []
|
|
assert store.agents() == []
|
|
|
|
|
|
def test_the_shipped_starter_agents_validate() -> None:
|
|
from app import config as config_mod
|
|
|
|
agents = config_mod.load(ROOT / "config.yaml").agents
|
|
assert [a.name for a in agents] == ["Copenhagen engineering"]
|
|
# The two commented examples in config.example.yaml, uncommented, so they
|
|
# still validate when somebody copies them.
|
|
import yaml
|
|
lines = (ROOT / "config.example.yaml").read_text(encoding="utf-8").splitlines()
|
|
start = lines.index("agents: []") + 1
|
|
block = [line[2:] for line in lines[start:] if line.startswith("# ")]
|
|
examples = [AgentSeed.model_validate(a) for a in yaml.safe_load("\n".join(block))]
|
|
assert [a.name for a in examples] == ["Remote from Europe", "Europe hybrid"]
|
|
# Small enough that a prefer gate breaks ties rather than overrides fit.
|
|
assert {g.weight for a in [*agents, *examples]
|
|
for g in a.gates if g.mode == "prefer"} == {0.05}
|
|
|
|
|
|
def test_the_shipped_starter_searches_validate() -> None:
|
|
from app import config as config_mod
|
|
|
|
searches = config_mod.load(ROOT / "config.yaml").searches
|
|
assert searches and {s.provider for s in searches} == {"jobindex"}
|
|
assert all(s.params.get("q") for s in searches)
|
|
|
|
|
|
# ---------------------------------------------------------------- cascade
|
|
|
|
def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
|
|
store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
|
board="greenhouse")
|
|
agent = store.save_agent("Backend", "python backend")
|
|
|
|
def fake_fetch(self: ATSSource) -> Iterator[Item]:
|
|
item = Item(source_id="greenhouse:acme:1", location="https://x.example/1",
|
|
title="Backend Engineer", body="Python",
|
|
published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
|
raw={"company": "Acme", "board": "greenhouse", "slug": "acme"})
|
|
yield item
|
|
self.state["ats:ok:greenhouse:acme"] = self.options["run"]
|
|
self.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
|
|
|
|
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
|
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config, store,
|
|
workers={"cpu": 1, "http": 1, "llm": 1})
|
|
|
|
# The `pull` job, which is what the schedule fires: its own work is to ask
|
|
# for the chain, which then runs to its end in the same drain.
|
|
started = queue.enqueue("pull")
|
|
queue.drain(timeout=60)
|
|
|
|
assert started is not None and queue.jobs.get(started.id).state == "done" # type: ignore[union-attr]
|
|
batch = queue.jobs.last("pull_batch")
|
|
assert batch is not None
|
|
jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id)
|
|
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
|
|
assert stages == list(cascade.CASCADES["pull"])
|
|
work = [j.kind for j in jobs if j.kind != FINAL]
|
|
# The `pull` job and the chain it asked for are one cascade, so the Jobs
|
|
# tab shows a scheduled pull as one run. Extract and summarise are off and
|
|
# find nothing; score runs twice, before and after extract.
|
|
assert work == ["pull", "pull_batch", "embed", "score", "score"]
|
|
assert all(j.state == "done" for j in jobs)
|
|
assert len(store.scores(agent.id)) == 1
|
|
# The board was stamped with the run the `pull` job claimed, and the
|
|
# settle let that run go, so the next pull may start.
|
|
run = queue.jobs.get(started.id).result["run"] # type: ignore[union-attr]
|
|
assert store.state["ats:ok:greenhouse:acme"] == run == store.state["pull:settled"]
|
|
|
|
|
|
def test_the_pull_job_queues_nothing_while_another_pull_runs(
|
|
store: Store, config: Config, tmp_path: Path) -> None:
|
|
from app.jobs import pull
|
|
|
|
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
|
board="greenhouse")
|
|
running = pull.claim_run(store)
|
|
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config, store,
|
|
workers={"cpu": 1, "http": 1, "llm": 1})
|
|
started = queue.enqueue("pull")
|
|
queue.drain(timeout=60)
|
|
|
|
done = queue.jobs.get(started.id) # type: ignore[union-attr]
|
|
assert done is not None and done.state == "done"
|
|
assert done.result == {"skipped": pull.ALREADY_RUNNING, "running": running}
|
|
assert [j.kind for j in queue.jobs.of_cascade(done.cascade_id)] == ["pull"]
|
|
|
|
|
|
def test_pull_is_a_job_on_a_six_hour_schedule_and_the_stages_are_tasks() -> None:
|
|
"""A schedule enqueues a job kind, so `pull` is a job; were it a task too,
|
|
the Jobs tab would draw two `pull` buttons."""
|
|
assert "pull" not in TASKS
|
|
assert {"extract", "embed", "score"} <= set(TASKS)
|
|
assert SCHEDULES["pull"].kind == "pull"
|
|
assert SCHEDULES["pull"].cadence == "every 6h"
|
|
|
|
|
|
def test_the_model_batches_are_isolated_with_a_timeout() -> None:
|
|
"""A hung model call is otherwise unkillable short of restarting the
|
|
server: `context.check()` runs between calls, not inside one, so only a
|
|
process boundary can end a batch that never comes back."""
|
|
for kind in ("extract_batch", "summarise_batch"):
|
|
spec = JOBS[kind]
|
|
assert spec.isolate, f"{kind} must run in its own process to be killable"
|
|
assert spec.timeout > 0, f"{kind} must have a deadline now that it can"
|
|
|
|
|
|
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
|
|
"""By position, not by name: `score` is in the chain twice, once before
|
|
extract and once after, and the second must lead on to summarise."""
|
|
assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "extract", "score",
|
|
"summarise")
|
|
assert [cascade.follows("pull", at) for at in range(6)] == \
|
|
["embed", "score", "extract", "score", "summarise", ""]
|
|
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 9) == ""
|
|
assert cascade.follows("nope", 0) == ""
|
|
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
|
|
|
|
|
|
def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
made: list[Any] = []
|
|
|
|
class Context:
|
|
def task(self, t: Any) -> Any:
|
|
made.append(t)
|
|
return t
|
|
|
|
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 2},
|
|
context=Context())
|
|
assert out == {"task": "score", "next": "extract"}
|
|
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 3}
|
|
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 4},
|
|
context=Context())
|
|
assert out["next"] == "summarise" and made[1].carries["at"] == 5
|