1045 lines
46 KiB
Python
1045 lines
46 KiB
Python
"""`embed`, `score` and the pull cascade, with the embedding backend replaced
|
|
by a deterministic fake. Nothing here downloads a model or reaches a board."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import sys
|
|
from collections.abc import Iterator, Sequence
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from desk.models import Item # noqa: E402
|
|
from pipeline import FINAL, JOBS, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, Tally # noqa: E402
|
|
from plugins.ats import ATSSource # noqa: E402
|
|
from app.config import CALIBRATION, AgentSeed, Config, ConfigError # noqa: E402
|
|
from app.jobs import cascade, embed, score # noqa: E402
|
|
from app.models import Candidate, Gate, Limits, Prompt, Weights # noqa: E402
|
|
from app.storage import Store # noqa: E402
|
|
|
|
NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
|
|
|
|
#: The shipped constants, which the fake is calibrated with so a test reads
|
|
#: the rule production runs; `uncalibrated` is the min-max fallback.
|
|
E5 = CALIBRATION["intfloat/multilingual-e5-small"]
|
|
|
|
#: The fake's whole vocabulary: one dimension per word, plus a constant one so
|
|
#: no text is the zero vector.
|
|
WORDS = ("backend", "python", "frontend", "react", "data")
|
|
|
|
|
|
class FakeEmbedder:
|
|
model_id = "fake"
|
|
|
|
def __init__(self) -> None:
|
|
self.calls: list[list[str]] = []
|
|
|
|
def encode(self, texts: Sequence[str]) -> np.ndarray:
|
|
self.calls.append(list(texts))
|
|
rows = []
|
|
for text in texts:
|
|
lower = text.lower()
|
|
vec = np.asarray([lower.count(w) for w in WORDS] + [0.1], dtype=np.float32)
|
|
rows.append(vec / np.linalg.norm(vec))
|
|
return np.vstack(rows)
|
|
|
|
|
|
class Clock:
|
|
def __init__(self) -> None:
|
|
self.at = NOW
|
|
|
|
def __call__(self) -> datetime:
|
|
return self.at
|
|
|
|
|
|
@pytest.fixture
|
|
def fake(monkeypatch: pytest.MonkeyPatch) -> FakeEmbedder:
|
|
embedder = FakeEmbedder()
|
|
monkeypatch.setattr(embed, "load_embedder", lambda model: embedder)
|
|
return embedder
|
|
|
|
|
|
@pytest.fixture
|
|
def clock() -> Clock:
|
|
return Clock()
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path, clock: Clock) -> Iterator[Store]:
|
|
s = Store(tmp_path / "roledesk.db")
|
|
s.clock = clock
|
|
yield s
|
|
|
|
|
|
@pytest.fixture
|
|
def config(tmp_path: Path) -> Config:
|
|
return Config(database=str(tmp_path / "roledesk.db"),
|
|
embed={"model": "fake", "calibration": {"fake": E5.model_dump()}})
|
|
|
|
|
|
@pytest.fixture
|
|
def uncalibrated(tmp_path: Path) -> Config:
|
|
return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
|
|
|
|
|
|
def role(store: Store, source_id: str, title: str = "Backend Engineer", *,
|
|
company: str = "Acme", body: str = "",
|
|
published_at: datetime = datetime(2026, 9, 20, tzinfo=UTC), **raw: Any) -> str:
|
|
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
|
|
title=title, body=body, published_at=published_at,
|
|
raw={"company": company, **raw})
|
|
item.source = "ats"
|
|
store.upsert_role(item)
|
|
return item.uid
|
|
|
|
|
|
def run(config: Config, store: Store, job: Any) -> dict[str, Any]:
|
|
notices: list[Any] = []
|
|
result = dict(job(config, store, context=RunContext(on_notice=notices.append)))
|
|
result["notices"] = [n.code for n in notices]
|
|
result["notice_texts"] = [n.text for n in notices]
|
|
return result
|
|
|
|
|
|
def level(text: str, intent: str, band: tuple[float, float] = E5.fit) -> float:
|
|
"""The calibrated fit the fake gives a role whose `document` is `text`."""
|
|
role_vec, intent_vec = FakeEmbedder().encode([text, intent])
|
|
lo, hi = band
|
|
return min(max((score.cosine(role_vec, intent_vec) - lo) / (hi - lo), 0.0), 1.0)
|
|
|
|
|
|
# ------------------------------------------------------------------ embed
|
|
|
|
def test_embed_writes_a_vector_per_open_role_once(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
role(store, "1")
|
|
closed = role(store, "2")
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (NOW.isoformat(), closed))
|
|
|
|
assert run(config, store, embed.run_embed)["embedded"] == 1
|
|
assert run(config, store, embed.run_embed)["embedded"] == 0
|
|
assert store.vector_counts() == {"fake": 1}
|
|
|
|
|
|
def test_embed_without_sentence_transformers_says_so_and_does_nothing(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
def missing(model: str) -> Any:
|
|
raise ImportError("no sentence_transformers")
|
|
|
|
monkeypatch.setattr(embed, "load_embedder", missing)
|
|
role(store, "1")
|
|
|
|
result = run(config, store, embed.run_embed)
|
|
|
|
assert result["embedded"] == 0 and result["notices"] == ["embed.unavailable"]
|
|
assert store.vector_counts() == {}
|
|
|
|
|
|
# -------------------------------------------------------- e5 prefixes
|
|
|
|
@pytest.mark.parametrize("model, wants", [
|
|
("intfloat/multilingual-e5-small", True),
|
|
("intfloat/e5-base-v2", True),
|
|
("intfloat/e5-large", True),
|
|
("sentence-transformers/all-MiniLM-L6-v2", False),
|
|
("sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", False),
|
|
("fake", False),
|
|
("some-org/e5models-are-not-this", False),
|
|
])
|
|
def test_wants_e5_prefix_matches_the_e5_family_by_name(model: str, wants: bool) -> None:
|
|
assert embed.wants_e5_prefix(model) == wants
|
|
|
|
|
|
def test_role_text_and_intent_text_add_the_e5_prefixes_only_for_e5(
|
|
store: Store) -> None:
|
|
uid = role(store, "1", "Backend Engineer", body="Python services")
|
|
r = store.role(uid)
|
|
assert r is not None
|
|
assert embed.role_text("fake", r) == embed.document(r)
|
|
assert embed.role_text("intfloat/multilingual-e5-small", r) == f"passage: {embed.document(r)}"
|
|
assert embed.intent_text("fake", "python backend") == "python backend"
|
|
assert embed.intent_text("intfloat/multilingual-e5-small", "python backend") == \
|
|
"query: python backend"
|
|
|
|
|
|
class FakeE5Embedder:
|
|
"""A second fake, standing in for a genuinely different model: its own
|
|
(narrower) vector width and an e5-shaped model id, so it exercises both
|
|
the prefixing and the re-embedding-on-model-change paths without
|
|
downloading anything."""
|
|
|
|
model_id = "intfloat/multilingual-e5-small"
|
|
|
|
def __init__(self) -> None:
|
|
self.calls: list[list[str]] = []
|
|
|
|
def encode(self, texts: Sequence[str]) -> np.ndarray:
|
|
self.calls.append(list(texts))
|
|
rows = []
|
|
for text in texts:
|
|
lower = text.lower()
|
|
vec = np.asarray([lower.count(w) for w in WORDS[:2]] + [0.1, 0.2], dtype=np.float32)
|
|
rows.append(vec / np.linalg.norm(vec))
|
|
return np.vstack(rows)
|
|
|
|
|
|
def test_embed_prefixes_role_text_for_an_e5_model(
|
|
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
e5 = FakeE5Embedder()
|
|
monkeypatch.setattr(embed, "load_embedder", lambda model: e5)
|
|
config = Config(database=str(tmp_path / "roledesk.db"),
|
|
embed={"model": "intfloat/multilingual-e5-small"})
|
|
role(store, "1", "Backend Engineer", body="Python services")
|
|
|
|
run(config, store, embed.run_embed)
|
|
|
|
assert e5.calls and e5.calls[0][0].startswith("passage: Backend Engineer")
|
|
|
|
|
|
def test_score_prefixes_the_intent_for_an_e5_model(
|
|
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
e5 = FakeE5Embedder()
|
|
monkeypatch.setattr(embed, "load_embedder", lambda model: e5)
|
|
config = Config(database=str(tmp_path / "roledesk.db"),
|
|
embed={"model": "intfloat/multilingual-e5-small"})
|
|
role(store, "1")
|
|
store.save_agent("Backend", "python backend")
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
assert e5.calls == [["query: python backend"]]
|
|
|
|
|
|
# ------------------------------------------------------- model changes
|
|
|
|
def test_changing_the_embed_model_drops_old_vectors_and_reembeds(
|
|
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The doc says a model change re-embeds; this is the guarantee that
|
|
actually makes that true, and the one that stops a run ever mixing two
|
|
models' vectors, which would be two different geometries under one
|
|
dimension count and garbage cosines."""
|
|
fake_a, fake_e5 = FakeEmbedder(), FakeE5Embedder()
|
|
embedders = {"fake": fake_a, "intfloat/multilingual-e5-small": fake_e5}
|
|
monkeypatch.setattr(embed, "load_embedder", lambda model: embedders[model])
|
|
role(store, "1")
|
|
role(store, "2")
|
|
|
|
config_a = Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
|
|
run(config_a, store, embed.run_embed)
|
|
assert store.vector_counts() == {"fake": 2}
|
|
dim_a = next(iter(store.vectors("fake").values())).shape[-1]
|
|
|
|
config_e5 = Config(database=str(tmp_path / "roledesk.db"),
|
|
embed={"model": "intfloat/multilingual-e5-small"})
|
|
result = run(config_e5, store, embed.run_embed)
|
|
|
|
# The old model's rows are gone, not just shadowed: `vectors(model)`
|
|
# filters by model, so a leftover row could never be read back mixed in,
|
|
# but drop_vectors is what stops it existing at all.
|
|
assert store.vector_counts() == {"intfloat/multilingual-e5-small": 2}
|
|
assert store.vectors("fake") == {}
|
|
assert result["dropped"] == 2
|
|
dim_e5 = next(iter(store.vectors("intfloat/multilingual-e5-small").values())).shape[-1]
|
|
assert dim_a != dim_e5, "the fakes must differ in width for this test to mean anything"
|
|
|
|
|
|
def test_changing_the_embed_model_recomputes_the_agents_cached_prompt_vector(
|
|
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""`prompt_vector` only returns a cache hit when `intent_key` -- which
|
|
hashes the model in -- matches what is stored, so a model change is a
|
|
cache miss and `_prompts` re-encodes under the new model, never handing a
|
|
stale-dimension vector into a run's cosines."""
|
|
fake_a, fake_e5 = FakeEmbedder(), FakeE5Embedder()
|
|
embedders = {"fake": fake_a, "intfloat/multilingual-e5-small": fake_e5}
|
|
monkeypatch.setattr(embed, "load_embedder", lambda model: embedders[model])
|
|
uid = role(store, "1")
|
|
agent = store.save_agent("Backend", "python backend")
|
|
|
|
config_a = Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
|
|
run(config_a, store, embed.run_embed)
|
|
run(config_a, store, score.run_score)
|
|
vec_a = store.prompt_vector(agent.id, "Backend", score.intent_key("fake", "python backend"))
|
|
assert vec_a is not None
|
|
|
|
config_e5 = Config(database=str(tmp_path / "roledesk.db"),
|
|
embed={"model": "intfloat/multilingual-e5-small"})
|
|
run(config_e5, store, embed.run_embed)
|
|
run(config_e5, store, score.run_score)
|
|
|
|
# The old key no longer resolves (the row moved on); the new key does,
|
|
# at the new model's own width, and scoring this role did not raise.
|
|
assert store.prompt_vector(
|
|
agent.id, "Backend", score.intent_key("fake", "python backend")) is None
|
|
vec_e5 = store.prompt_vector(
|
|
agent.id, "Backend", score.intent_key("intfloat/multilingual-e5-small", "python backend"))
|
|
assert vec_e5 is not None
|
|
assert vec_e5.shape != vec_a.shape
|
|
assert uid in store.scores(agent.id)
|
|
|
|
|
|
# ------------------------------------------------------------------ terms
|
|
|
|
def test_recency_halves_every_fourteen_days() -> None:
|
|
assert score.recency(NOW.isoformat(), NOW) == 1.0
|
|
assert score.recency((NOW - timedelta(days=14)).isoformat(), NOW) == pytest.approx(0.5)
|
|
assert score.recency((NOW - timedelta(days=28)).isoformat(), NOW) == pytest.approx(0.25)
|
|
|
|
|
|
def test_a_role_without_a_vector_is_scored_without_fit_unless_fit_is_all() -> None:
|
|
candidate = Candidate(uid="u", posted_at=NOW.isoformat(), affinity=0)
|
|
value, reason = score.score_one(candidate, [], Weights(), None, NOW)
|
|
assert reason == "" and value == pytest.approx((0.2 * 1.0 + 0.2 / 3) / 0.4)
|
|
|
|
only_fit = Weights(fit=1, recency=0, affinity=0)
|
|
assert score.score_one(candidate, [], only_fit, None, NOW) == (0.0, "no vector")
|
|
|
|
|
|
# ------------------------------------------------------------------ score
|
|
|
|
def test_a_require_gate_keeps_the_row_with_a_reason(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
berlin = role(store, "1", place="Berlin, Germany")
|
|
munich = role(store, "2", place="Munich")
|
|
nowhere = role(store, "3")
|
|
agent = store.save_agent("Berlin", "python backend",
|
|
[Gate(field="place", values=["Berlin"])])
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[berlin][1] == "" and scores[berlin][0] > 0
|
|
assert scores[munich] == (0.0, "place not Berlin")
|
|
# An empty column is no evidence: the role ranks, and extract can reach it.
|
|
assert scores[nowhere][1] == "" and scores[nowhere][0] > 0
|
|
shown = store.roles(agent=agent.id)
|
|
assert {r.uid for r in shown} == {berlin, nowhere}
|
|
assert {r.uid for r in store.roles(agent=agent.id, gated=True)} == {berlin, munich, nowhere}
|
|
|
|
|
|
def test_any_gates_are_one_group_a_role_must_match_one_of(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
""""In Denmark, or remote": a role passes on either, is gated when it has
|
|
evidence against both, and ranks when both columns are empty."""
|
|
copenhagen = role(store, "1", place="Copenhagen, Denmark")
|
|
remote_de = role(store, "2", place="Berlin, Germany", remote="remote")
|
|
onsite_de = role(store, "3", place="Berlin, Germany", remote="onsite")
|
|
hybrid_nowhere = role(store, "4", place="Hybrid") # remote from place, no country
|
|
blank = role(store, "5")
|
|
agent = store.save_agent("Copenhagen or remote", "", [
|
|
Gate(field="country", values=["DK"], mode="any"),
|
|
Gate(field="remote", values=["remote"], mode="any")])
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[copenhagen][1] == "" and scores[remote_de][1] == ""
|
|
assert scores[onsite_de] == (0.0, "none of country DK, remote remote")
|
|
assert scores[blank][1] == ""
|
|
# "Hybrid" is a country-less line the remote lookup reads as hybrid, so
|
|
# the group has evidence against remote and none for country: gated.
|
|
assert scores[hybrid_nowhere][1].startswith("none of")
|
|
|
|
|
|
def test_failed_gates_reads_evidence_only() -> None:
|
|
gates = [Gate(field="country", values=["DK"]),
|
|
Gate(field="remote", values=["remote"], mode="any"),
|
|
Gate(field="seniority", values=["senior"], mode="any"),
|
|
Gate(field="function", values=["engineering"], mode="prefer")]
|
|
def cand(*states: int) -> Candidate:
|
|
return Candidate(uid="u", posted_at=NOW.isoformat(), affinity=0, gates=list(states))
|
|
assert score.failed_gates(cand(1, 1, 0, 0), gates) == []
|
|
assert score.failed_gates(cand(-1, -1, -1, 0), gates) == []
|
|
assert score.failed_gates(cand(0, 0, -1, 1), gates) == [
|
|
"country not DK", "none of remote remote, seniority senior"]
|
|
assert score.failed_gates(cand(1, 0, 0, 1), gates) == [
|
|
"none of remote remote, seniority senior"]
|
|
|
|
|
|
def test_a_prefer_gate_adds_its_weight_to_the_score(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
remote = role(store, "1", remote="remote")
|
|
office = role(store, "2", remote="onsite")
|
|
agent = store.save_agent("Remote", "", [Gate(field="remote", values=["remote"],
|
|
mode="prefer", weight=0.5)])
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
# No intent, so fit is out: recency 0.2 + affinity 0.2 + the gate's 0.5.
|
|
assert scores[remote][0] - scores[office][0] == pytest.approx(0.5 / 0.9, abs=1e-5)
|
|
assert scores[office][1] == ""
|
|
|
|
|
|
def test_a_favourite_skips_the_gates_and_is_floored(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
store.add_company("Fave")
|
|
fave = role(store, "1", company="Fave", place="Munich")
|
|
agent = store.save_agent("Berlin", "", [Gate(field="place", values=["Berlin"])])
|
|
store.set_affinity(agent.id, "fave", 2)
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
value, reason = store.scores(agent.id)[fave]
|
|
assert reason == "" and value >= 0.9
|
|
|
|
|
|
def test_a_hidden_company_is_never_scored_for_the_agent_that_hid_it(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
store.add_company("Muted")
|
|
muted = role(store, "1", company="Muted")
|
|
kept = role(store, "2")
|
|
agent = store.save_agent("Any")
|
|
other = store.save_agent("Other")
|
|
store.set_affinity(agent.id, "muted", -1)
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
assert set(store.scores(agent.id)) == {kept}
|
|
assert muted not in {r.uid for r in store.roles(agent=agent.id, gated=True)}
|
|
# The other agent did not hide it, so it is scored for that one.
|
|
assert set(store.scores(other.id)) == {muted, kept}
|
|
|
|
|
|
def test_an_older_role_scores_lower(
|
|
store: Store, config: Config, clock: Clock, fake: FakeEmbedder) -> None:
|
|
clock.at = NOW - timedelta(days=14)
|
|
old = role(store, "1")
|
|
clock.at = NOW
|
|
new = role(store, "2")
|
|
agent = store.save_agent("Any")
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[new][0] > scores[old][0]
|
|
|
|
|
|
def test_recency_counts_from_posted_at_not_first_seen(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
"""A trusted board's backlog, first seen today, must not score as fresh:
|
|
recency has to read the board's own (trusted) posted date, not the
|
|
moment the app happened to pull it in."""
|
|
stale = role(store, "1", board="greenhouse",
|
|
published_at=NOW - timedelta(days=28))
|
|
fresh = role(store, "2") # untrusted host: posted_at falls back to first_seen (NOW)
|
|
agent = store.save_agent("Any")
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[fresh][0] > scores[stale][0]
|
|
|
|
|
|
def test_two_agents_rank_the_same_roles_the_other_way_round(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
back = role(store, "1", "Backend Engineer", body="Python services")
|
|
front = role(store, "2", "Frontend Engineer", body="React apps")
|
|
only_fit = Weights(fit=1, recency=0, affinity=0)
|
|
backend = store.save_agent("Backend", "python backend developer", weights=only_fit)
|
|
frontend = store.save_agent("Frontend", "react frontend developer", weights=only_fit)
|
|
|
|
run(config, store, embed.run_embed)
|
|
result = run(config, store, score.run_score)
|
|
|
|
assert result["agents"] == 2
|
|
# Fit alone, calibrated: each agent's own role is at a cosine past `hi`,
|
|
# so 1, the other's near 0, below `lo`, so 0.
|
|
assert store.scores(backend.id)[back][0] == 1.0
|
|
assert store.scores(backend.id)[front][0] == 0.0
|
|
assert store.scores(frontend.id)[front][0] == 1.0
|
|
assert store.scores(frontend.id)[back][0] == 0.0
|
|
|
|
|
|
# ------------------------------------------------------------------ taste
|
|
|
|
def _unit(*words: str) -> np.ndarray:
|
|
return FakeEmbedder().encode([" ".join(words)])[0]
|
|
|
|
|
|
def test_taste_direction_is_liked_minus_passed_and_none_without_votes() -> None:
|
|
vectors = {"a": _unit("backend"), "b": _unit("frontend"), "c": _unit("data")}
|
|
|
|
assert score.taste_direction(vectors, {}) is None
|
|
assert score.taste_direction(vectors, {"a": 0}) is None # neutral says nothing
|
|
assert score.taste_direction(vectors, {"gone": 1}) is None # no vector to point at
|
|
|
|
liked_only = score.taste_direction(vectors, {"a": 1, "c": 1})
|
|
assert liked_only is not None
|
|
assert liked_only == pytest.approx((vectors["a"] + vectors["c"]) / 2)
|
|
|
|
passed_only = score.taste_direction(vectors, {"b": -1})
|
|
assert passed_only is not None
|
|
assert passed_only == pytest.approx(-vectors["b"])
|
|
|
|
both = score.taste_direction(vectors, {"a": 1, "b": -1, "c": 0})
|
|
assert both is not None
|
|
assert both == pytest.approx(vectors["a"] - vectors["b"])
|
|
|
|
|
|
def test_taste_ranks_a_role_near_the_liked_one_above_one_near_the_passed(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
liked = role(store, "1", "Backend Engineer", body="Python backend")
|
|
passed = role(store, "2", "Frontend Engineer", body="React frontend")
|
|
near_liked = role(store, "3", "Platform Engineer", body="Python backend services")
|
|
near_passed = role(store, "4", "Web Engineer", body="React frontend apps")
|
|
# No intent: with taste alone weighted, the votes are all that orders them.
|
|
agent = store.save_agent("Taste", "", weights=Weights(fit=0, recency=0, affinity=0,
|
|
taste=1))
|
|
store.set_vote(agent.id, liked, 1)
|
|
store.set_vote(agent.id, passed, -1)
|
|
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[near_liked][0] > scores[near_passed][0]
|
|
# Cosines of about 0.7 and -0.7 to the direction, past both ends of the band.
|
|
assert scores[liked][0] == 1.0 and scores[passed][0] == 0.0
|
|
assert all(reason == "" for _, reason in scores.values())
|
|
|
|
|
|
def test_without_votes_taste_leaves_the_denominator(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
uid = role(store, "1")
|
|
agent = store.save_agent("Any", weights=Weights(fit=0, recency=0, affinity=0, taste=1))
|
|
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
|
|
# Taste is the only weighted term and there is no taste, so no weights.
|
|
assert store.scores(agent.id)[uid] == (0.0, "no weights")
|
|
|
|
|
|
def test_a_closed_liked_role_still_shapes_the_taste(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
liked = role(store, "1", "Backend Engineer", body="Python backend")
|
|
near = role(store, "2", "Platform Engineer", body="Python backend")
|
|
far = role(store, "3", "Web Engineer", body="React frontend")
|
|
agent = store.save_agent("Taste", "", weights=Weights(fit=0, recency=0, affinity=0,
|
|
taste=1))
|
|
run(config, store, embed.run_embed)
|
|
store.set_vote(agent.id, liked, 1)
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (NOW.isoformat(), liked))
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert liked not in scores
|
|
# Near sits on the liked direction, past `hi`. Far is orthogonal to it,
|
|
# a cosine of about 0 and so a middling level, not the 0 min-max gave it.
|
|
assert scores[near][0] == 1.0
|
|
assert scores[far][0] == pytest.approx(
|
|
level("Web Engineer\nReact frontend", "Backend Engineer\nPython backend", E5.taste),
|
|
abs=1e-5)
|
|
assert 0 < scores[far][0] < 1
|
|
|
|
|
|
# ------------------------------------------------------------ fit scaling
|
|
|
|
def test_scale_maps_a_cosine_to_its_calibrated_level_and_clips_both_ends() -> None:
|
|
# multilingual-e5-small's fit band: lo 0.75 is 0, hi 0.88 is 1.
|
|
assert E5.fit == (0.75, 0.88) and E5.taste == (-0.14, 0.19)
|
|
assert score.scale({"lo": 0.75, "mid": 0.815, "hi": 0.88, "p50": 0.7989},
|
|
E5.fit) == pytest.approx(
|
|
{"lo": 0.0, "mid": 0.5, "hi": 1.0, "p50": 0.0489 / 0.13})
|
|
assert score.scale({"under": 0.6, "negative": -0.2, "over": 0.9031, "one": 1.0},
|
|
E5.fit) == {"under": 0.0, "negative": 0.0, "over": 1.0, "one": 1.0}
|
|
assert score.scale({"none": 0.0, "liked": 0.25}, E5.taste) == pytest.approx(
|
|
{"none": 0.14 / 0.33, "liked": 1.0})
|
|
# A lone role, or roles that are all equally close, get their level.
|
|
assert score.scale({"only": 0.8}, E5.fit) == pytest.approx({"only": 0.05 / 0.13})
|
|
assert score.scale({}, E5.fit) == {}
|
|
|
|
|
|
def test_scale_without_a_band_falls_back_to_min_max() -> None:
|
|
assert score.scale({"a": 0.87, "b": 0.95, "c": 0.91}, None) == pytest.approx(
|
|
{"a": 0.0, "b": 1.0, "c": 0.5})
|
|
assert score.scale({"only": 0.3}, None) == {"only": 1.0}
|
|
assert score.scale({"a": 0.4, "b": 0.4}, None) == {"a": 1.0, "b": 1.0}
|
|
assert score.scale({}, None) == {}
|
|
|
|
|
|
def test_a_single_candidate_gets_its_level_not_full_fit(
|
|
store: Store, config: Config, uncalibrated: Config, fake: FakeEmbedder) -> None:
|
|
"""Min-max gave the best of a poor set 1; a level gives it what it is."""
|
|
uid = role(store, "1", "Frontend Engineer", body="React apps")
|
|
agent = store.save_agent("Backend", "python backend developer",
|
|
weights=Weights(fit=1, recency=0, affinity=0))
|
|
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
assert store.scores(agent.id)[uid] == (0.0, "")
|
|
|
|
run(uncalibrated, store, score.run_score)
|
|
assert store.scores(agent.id)[uid] == (1.0, "")
|
|
|
|
|
|
def test_the_same_role_gets_the_same_fit_whatever_else_is_open(
|
|
store: Store, config: Config, uncalibrated: Config, fake: FakeEmbedder) -> None:
|
|
"""The point of a level: two runs over different candidate sets give the
|
|
role the same fit, where min-max gave it 1 beside a worse role and 0
|
|
beside a better one."""
|
|
mid = role(store, "1", "Backend Engineer", body="Python backend data")
|
|
worse = role(store, "2", "Frontend Engineer", body="React apps")
|
|
agent = store.save_agent("Backend", "python backend",
|
|
weights=Weights(fit=1, recency=0, affinity=0))
|
|
expected = level("Backend Engineer\nPython backend data", "python backend")
|
|
assert 0.5 < expected < 1
|
|
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
first = store.scores(agent.id)[mid][0]
|
|
run(uncalibrated, store, score.run_score)
|
|
assert store.scores(agent.id)[mid][0] == 1.0
|
|
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (NOW.isoformat(), worse))
|
|
role(store, "3", "Backend Engineer", body="Python")
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
second = store.scores(agent.id)[mid][0]
|
|
run(uncalibrated, store, score.run_score)
|
|
assert store.scores(agent.id)[mid][0] == 0.0
|
|
|
|
assert first == second == pytest.approx(expected, abs=1e-5)
|
|
|
|
|
|
def test_an_uncalibrated_model_falls_back_to_min_max_and_warns_once_a_run(
|
|
store: Store, uncalibrated: Config, fake: FakeEmbedder,
|
|
caplog: pytest.LogCaptureFixture) -> None:
|
|
close = role(store, "1", "Backend Engineer", body="Python backend data")
|
|
far = role(store, "2", "Data Engineer", body="Data")
|
|
only_fit = Weights(fit=1, recency=0, affinity=0)
|
|
one = store.save_agent("One", "python backend", weights=only_fit)
|
|
two = store.save_agent("Two", "python backend", weights=only_fit)
|
|
run(uncalibrated, store, embed.run_embed)
|
|
|
|
with caplog.at_level("WARNING", logger="app.jobs.score"):
|
|
run(uncalibrated, store, score.run_score)
|
|
|
|
warnings = [r.getMessage() for r in caplog.records if r.name == "app.jobs.score"]
|
|
assert len(warnings) == 1 and warnings[0].startswith("no calibration for fake")
|
|
for agent in (one, two):
|
|
assert store.scores(agent.id)[close][0] == 1.0
|
|
assert store.scores(agent.id)[far][0] == 0.0
|
|
|
|
|
|
def test_the_calibration_is_the_built_in_one_unless_the_config_replaces_it() -> None:
|
|
assert Config().embed.calibrated() == E5
|
|
assert Config(embed={"model": "fake"}).embed.calibrated() is None
|
|
mine = {"fit": [0.7, 0.9], "taste": [-0.2, 0.2]}
|
|
replaced = Config(embed={"calibration": {"intfloat/multilingual-e5-small": mine}})
|
|
assert replaced.embed.calibrated() is not None
|
|
assert replaced.embed.calibrated().fit == (0.7, 0.9) # type: ignore[union-attr]
|
|
# Keyed by model: another model's entry does not apply.
|
|
other = Config(embed={"model": "fake", "calibration": {"other": mine}})
|
|
assert other.embed.calibrated() is None
|
|
|
|
|
|
@pytest.mark.parametrize("entry, says", [
|
|
({"fit": [0.88, 0.75], "taste": [-0.14, 0.19]}, "embed.calibration.fake.fit"),
|
|
({"fit": [0.75, 0.75], "taste": [-0.14, 0.19]}, "lo < hi"),
|
|
({"fit": [0.75, 0.88], "taste": [-2, 0.19]}, "embed.calibration.fake.taste"),
|
|
({"fit": [0.75, 0.88]}, "embed.calibration.fake.taste"),
|
|
({"fit": [0.75, 0.88], "taste": [-0.14, 0.19], "spread": 1}, "unknown setting"),
|
|
])
|
|
def test_a_bad_calibration_is_a_config_error(
|
|
tmp_path: Path, entry: dict[str, Any], says: str) -> None:
|
|
import yaml
|
|
from app import config as config_mod
|
|
|
|
path = tmp_path / "config.yaml"
|
|
path.write_text(yaml.safe_dump({"embed": {"model": "fake",
|
|
"calibration": {"fake": entry}}}))
|
|
with pytest.raises(ConfigError, match=re.escape(says)):
|
|
config_mod.load(path)
|
|
|
|
|
|
def test_an_agent_without_an_intent_feeds_neither_model_stage_and_is_named(
|
|
store: Store, config: Config, clock: Clock, fake: FakeEmbedder) -> None:
|
|
"""Its scores have no fit, so they rank fresh roles, not wanted ones: the
|
|
newer role tops it, and extract and summarise must not read it on that."""
|
|
clock.at = NOW - timedelta(days=1)
|
|
wanted = role(store, "1", "Backend Engineer", body="Python backend")
|
|
clock.at = NOW
|
|
fresh = role(store, "2", "Frontend Engineer", body="React apps")
|
|
top_one = Limits(extract_top=1, extract_min=0, summarise_top=1, summarise_min=0)
|
|
store.save_agent("Backend", "python backend", limits=top_one)
|
|
blind = store.save_agent("Fresh", "", limits=top_one)
|
|
store.save_agent("Also fresh", " \n", limits=top_one)
|
|
store.save_agent("Blank prompts", limits=top_one,
|
|
prompts=[Prompt(name="One"), Prompt(name="Two", text=" ")])
|
|
store.save_agent("Off", "", enabled=False)
|
|
|
|
run(config, store, embed.run_embed)
|
|
result = run(config, store, score.run_score)
|
|
|
|
assert result["notices"] == ["score.no_intent"]
|
|
assert result["notice_texts"][0].startswith("No prompt with text")
|
|
assert result["notice_texts"][0].endswith(": Also fresh, Blank prompts, Fresh.")
|
|
ranked = sorted(store.scores(blind.id).items(), key=lambda kv: -kv[1][0])
|
|
assert ranked[0][0] == fresh, "the intent-less agent would pick the newer role"
|
|
assert [r.uid for r in store.extract_waiting(10)] == [wanted]
|
|
assert [r.uid for r in store.summarise_waiting(10)] == [wanted]
|
|
assert store.extract_pending() == store.summarise_pending() == 1
|
|
|
|
|
|
def test_a_gated_role_does_not_stretch_the_fit_range_under_the_fallback(
|
|
store: Store, uncalibrated: Config, fake: FakeEmbedder) -> None:
|
|
"""Calibrated, no other role moves a role's fit. Min-max, the fallback,
|
|
must still leave a gated role out of the range."""
|
|
config = uncalibrated
|
|
close = role(store, "1", "Backend Engineer", body="Python backend", place="Berlin")
|
|
far = role(store, "2", "Data Engineer", body="Data", place="Berlin")
|
|
# Farther from the intent than either, but gated out by place.
|
|
role(store, "3", "Frontend Engineer", body="React react", place="Munich")
|
|
agent = store.save_agent("Berlin", "python backend",
|
|
[Gate(field="place", values=["Berlin"])],
|
|
weights=Weights(fit=1, recency=0, affinity=0))
|
|
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[close][0] == 1.0 and scores[far][0] == 0.0
|
|
|
|
|
|
def test_a_small_prefer_gate_no_longer_outweighs_fit(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
"""The flaw the last real run showed: real cosines sit close together, so
|
|
under `(cos + 1) / 2` a prefer gate decided the order over fit. Here the
|
|
cosines are about 0.95 and 0.87: the old fit terms differ by 0.04, which
|
|
at weight 0.6 is less than the gate's 0.05, so the remote role won.
|
|
Calibrated, they are 1 and about 0.89, and the closer role wins."""
|
|
close = role(store, "1", "Backend Engineer", body="Python backend", remote="onsite")
|
|
far = role(store, "2", "Backend Engineer", body="Python backend data", remote="remote")
|
|
agent = store.save_agent("Backend", "python backend",
|
|
[Gate(field="remote", values=["remote"], mode="prefer",
|
|
weight=0.05)])
|
|
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[close][0] > scores[far][0]
|
|
|
|
|
|
def test_the_intent_vector_is_cached_until_the_intent_changes(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
role(store, "1")
|
|
agent = store.save_agent("Backend", "python backend")
|
|
|
|
run(config, store, score.run_score)
|
|
run(config, store, score.run_score)
|
|
assert fake.calls == [["python backend"]]
|
|
|
|
store.save_agent("Backend", "react frontend", agent_id=agent.id)
|
|
run(config, store, score.run_score)
|
|
assert fake.calls[-1] == ["react frontend"]
|
|
|
|
|
|
# ----------------------------------------------------------------- prompts
|
|
|
|
def test_fit_is_the_best_prompts_level_and_the_score_names_that_prompt(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
backend = role(store, "1", "Backend Engineer", body="Python backend")
|
|
frontend = role(store, "2", "Frontend Engineer", body="React frontend")
|
|
gated = role(store, "3", "Backend Engineer", body="Python backend", place="Munich")
|
|
agent = store.save_agent(
|
|
"Two kinds", gates=[Gate(field="place", values=["Berlin"])],
|
|
weights=Weights(fit=1, recency=0, affinity=0),
|
|
prompts=[Prompt(name="Web", text="react frontend"),
|
|
Prompt(name="Empty"),
|
|
Prompt(name="Server", text="python backend")])
|
|
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
|
|
texts = {backend: "Backend Engineer\nPython backend",
|
|
frontend: "Frontend Engineer\nReact frontend"}
|
|
served = {r.uid: r for r in store.roles(agent=agent.id, gated=True)}
|
|
assert served[backend].prompt == "Server" and served[frontend].prompt == "Web"
|
|
for uid, text in texts.items():
|
|
best = max(level(text, "react frontend"), level(text, "python backend"))
|
|
assert served[uid].score == pytest.approx(best, abs=1e-5)
|
|
# A gated role has no fit that counted, so no prompt.
|
|
assert (served[gated].gated_reason, served[gated].prompt) == ("place not Berlin", "")
|
|
assert fake.calls[1:] == [["react frontend"], ["python backend"]]
|
|
|
|
|
|
def test_one_prompt_scores_exactly_as_the_intent_did(
|
|
store: Store, config: Config, uncalibrated: Config, fake: FakeEmbedder) -> None:
|
|
"""The legacy intent, one named prompt, and one prompt beside an empty
|
|
one are the same agent, calibrated and under the min-max fallback."""
|
|
for n, body in enumerate(["Python backend", "Python backend data", "React apps", "Data"]):
|
|
role(store, str(n), "Engineer", body=body)
|
|
legacy = store.save_agent("Legacy", "python backend")
|
|
named = store.save_agent("Named", prompts=[Prompt(name="Backend", text="python backend")])
|
|
padded = store.save_agent("Padded", prompts=[Prompt(name="Later"),
|
|
Prompt(name="Backend", text="python backend")])
|
|
run(config, store, embed.run_embed)
|
|
for cfg in (config, uncalibrated):
|
|
run(cfg, store, score.run_score)
|
|
assert store.scores(legacy.id) == store.scores(named.id) == store.scores(padded.id)
|
|
assert {r.prompt for r in store.roles(agent=legacy.id)} == {"Legacy"}
|
|
assert {r.prompt for r in store.roles(agent=padded.id)} == {"Backend"}
|
|
|
|
|
|
def test_under_the_fallback_each_prompt_is_ranged_on_its_own(
|
|
store: Store, uncalibrated: Config, fake: FakeEmbedder) -> None:
|
|
"""Min-max per prompt: each prompt's closest role gets 1, however far
|
|
that prompt's roles sit from it in absolute terms."""
|
|
backend = role(store, "1", "Backend Engineer", body="Python backend")
|
|
frontend = role(store, "2", "Frontend Engineer", body="React frontend")
|
|
agent = store.save_agent("Two", weights=Weights(fit=1, recency=0, affinity=0),
|
|
prompts=[Prompt(name="Server", text="python backend"),
|
|
Prompt(name="Web", text="react")])
|
|
run(uncalibrated, store, embed.run_embed)
|
|
run(uncalibrated, store, score.run_score)
|
|
|
|
served = {r.uid: (r.score, r.prompt) for r in store.roles(agent=agent.id)}
|
|
assert served == {backend: (1.0, "Server"), frontend: (1.0, "Web")}
|
|
|
|
|
|
def test_an_edit_re_embeds_only_the_prompt_it_changed(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
role(store, "1")
|
|
agent = store.save_agent("Two", prompts=[Prompt(name="Server", text="python backend"),
|
|
Prompt(name="Web", text="react frontend")])
|
|
run(config, store, score.run_score)
|
|
run(config, store, score.run_score)
|
|
assert fake.calls == [["python backend"], ["react frontend"]]
|
|
|
|
# Reordered, one renamed and one rewritten: only the new text is encoded.
|
|
store.save_agent("Two", agent_id=agent.id,
|
|
prompts=[Prompt(name="Frontend", text="react frontend"),
|
|
Prompt(name="Server", text="python data")])
|
|
run(config, store, score.run_score)
|
|
assert fake.calls[2:] == [["python data"]]
|
|
|
|
|
|
def test_without_an_embedder_the_prompts_are_skipped_and_fit_is_out(
|
|
store: Store, config: Config, fake: FakeEmbedder,
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
uid = role(store, "1")
|
|
run(config, store, embed.run_embed)
|
|
|
|
def missing(model: str) -> Any:
|
|
raise ImportError("no sentence_transformers")
|
|
|
|
monkeypatch.setattr(embed, "load_embedder", missing)
|
|
agent = store.save_agent("Fit only", weights=Weights(fit=1, recency=0, affinity=0),
|
|
prompts=[Prompt(name="Server", text="python backend")])
|
|
|
|
result = run(config, store, score.run_score)
|
|
|
|
assert result["notices"] == ["embed.unavailable"]
|
|
assert store.scores(agent.id) == {uid: (0.0, "no prompt vector")}
|
|
|
|
|
|
def test_a_disabled_agent_is_not_scored(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
role(store, "1")
|
|
agent = store.save_agent("Off", enabled=False)
|
|
assert run(config, store, score.run_score)["agents"] == 0
|
|
assert store.scores(agent.id) == {}
|
|
|
|
|
|
def test_starter_agents_are_imported_once(store: Store, tmp_path: Path) -> None:
|
|
config = Config(database=str(tmp_path / "roledesk.db"),
|
|
agents=[AgentSeed(name="A"), AgentSeed(name="B")])
|
|
assert score.seed_agents(config, store) == ["A", "B"]
|
|
for agent in store.agents():
|
|
store.delete_agent(agent.id)
|
|
assert score.seed_agents(config, store) == []
|
|
assert store.agents() == []
|
|
|
|
|
|
def test_the_shipped_starter_agents_validate() -> None:
|
|
from app import config as config_mod
|
|
|
|
agents = config_mod.load(ROOT / "config.yaml").agents
|
|
assert [a.name for a in agents] == ["Copenhagen engineering"]
|
|
# The two commented examples in config.example.yaml, uncommented, so they
|
|
# still validate when somebody copies them.
|
|
import yaml
|
|
lines = (ROOT / "config.example.yaml").read_text(encoding="utf-8").splitlines()
|
|
start = lines.index("agents: []") + 1
|
|
block = [line[2:] for line in lines[start:] if line.startswith("# ")]
|
|
examples = [AgentSeed.model_validate(a) for a in yaml.safe_load("\n".join(block))]
|
|
assert [a.name for a in examples] == ["Remote from Europe", "Europe hybrid"]
|
|
# Small enough that a prefer gate breaks ties rather than overrides fit.
|
|
assert {g.weight for a in [*agents, *examples]
|
|
for g in a.gates if g.mode == "prefer"} == {0.05}
|
|
|
|
|
|
def test_a_starter_agent_may_list_prompts_or_give_an_intent(
|
|
store: Store, tmp_path: Path) -> None:
|
|
config = Config(database=str(tmp_path / "roledesk.db"), agents=[
|
|
AgentSeed(name="Old", intent="python backend"),
|
|
AgentSeed.model_validate({"name": "New", "prompts": [
|
|
{"name": "AI", "text": "Head of AI"}, {"name": "FDE", "text": "FDE lead"}]})])
|
|
score.seed_agents(config, store)
|
|
assert {a.name: a.prompts for a in store.agents()} == {
|
|
"New": [Prompt(name="AI", text="Head of AI"), Prompt(name="FDE", text="FDE lead")],
|
|
"Old": [Prompt(name="Old", text="python backend")]}
|
|
with pytest.raises(ValueError, match="two prompts are named"):
|
|
AgentSeed.model_validate({"name": "Bad", "prompts": [{"name": "A"}, {"name": "a"}]})
|
|
|
|
|
|
def test_the_cli_adds_replaces_removes_and_lists_prompts(
|
|
tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
|
import yaml
|
|
from app import cli
|
|
|
|
db = tmp_path / "roledesk.db"
|
|
path = tmp_path / "config.yaml"
|
|
path.write_text(yaml.safe_dump({"database": str(db)}))
|
|
agent = Store(db).save_agent("Wolf", "Head of data")
|
|
|
|
def app(*argv: str) -> int:
|
|
return cli.main(["-c", str(path), "agents", *argv])
|
|
|
|
assert app("prompt", "Wolf", "FDE", "Forward deployed lead") == 0
|
|
assert app("prompt", "Wolf", "wolf", "Head of AI") == 0 # replaced in place
|
|
assert app("prompt", "Wolf", "Research") == 0 # no text yet
|
|
assert Store(db).agent(agent.id).prompts == [ # type: ignore[union-attr]
|
|
Prompt(name="wolf", text="Head of AI"), Prompt(name="FDE", text="Forward deployed lead"),
|
|
Prompt(name="Research")]
|
|
assert app("prompt", "Wolf", "fde", "--remove") == 0
|
|
assert app("prompt", "Wolf", "Nothing", "--remove") == 1
|
|
assert app("prompt", "Nobody", "X", "y") == 1
|
|
capsys.readouterr()
|
|
assert app("list") == 0
|
|
assert capsys.readouterr().out == "Wolf\n wolf: Head of AI\n Research: \n"
|
|
|
|
|
|
def test_the_shipped_starter_searches_validate() -> None:
|
|
from app import config as config_mod
|
|
|
|
searches = config_mod.load(ROOT / "config.yaml").searches
|
|
assert searches and {s.provider for s in searches} == {"jobindex"}
|
|
assert all(s.params.get("q") for s in searches)
|
|
|
|
|
|
# ---------------------------------------------------------------- cascade
|
|
|
|
def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
|
|
store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
|
board="greenhouse")
|
|
agent = store.save_agent("Backend", "python backend")
|
|
|
|
def fake_fetch(self: ATSSource) -> Iterator[Item]:
|
|
item = Item(source_id="greenhouse:acme:1", location="https://x.example/1",
|
|
title="Backend Engineer", body="Python",
|
|
published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
|
raw={"company": "Acme", "board": "greenhouse", "slug": "acme"})
|
|
yield item
|
|
self.state["ats:ok:greenhouse:acme"] = self.options["run"]
|
|
self.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
|
|
|
|
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
|
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config, store,
|
|
workers={"cpu": 1, "http": 1, "llm": 1})
|
|
|
|
# The `pull` job, which is what the schedule fires: its own work is to ask
|
|
# for the chain, which then runs to its end in the same drain.
|
|
started = queue.enqueue("pull")
|
|
queue.drain(timeout=60)
|
|
|
|
assert started is not None and queue.jobs.get(started.id).state == "done" # type: ignore[union-attr]
|
|
batch = queue.jobs.last("pull_batch")
|
|
assert batch is not None
|
|
jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id)
|
|
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
|
|
assert stages == list(cascade.CASCADES["pull"])
|
|
work = [j.kind for j in jobs if j.kind != FINAL]
|
|
# The `pull` job and the chain it asked for are one cascade, so the Jobs
|
|
# tab shows a scheduled pull as one run. Extract and summarise are off and
|
|
# find nothing; score runs twice, before and after extract.
|
|
assert work == ["pull", "pull_batch", "embed", "score", "score"]
|
|
assert all(j.state == "done" for j in jobs)
|
|
assert len(store.scores(agent.id)) == 1
|
|
# The board was stamped with the run the `pull` job claimed, and the
|
|
# settle let that run go, so the next pull may start.
|
|
run = queue.jobs.get(started.id).result["run"] # type: ignore[union-attr]
|
|
assert store.state["ats:ok:greenhouse:acme"] == run == store.state["pull:settled"]
|
|
|
|
|
|
def test_the_pull_job_queues_nothing_while_another_pull_runs(
|
|
store: Store, config: Config, tmp_path: Path) -> None:
|
|
from app.jobs import pull
|
|
|
|
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
|
board="greenhouse")
|
|
running = pull.claim_run(store)
|
|
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config, store,
|
|
workers={"cpu": 1, "http": 1, "llm": 1})
|
|
started = queue.enqueue("pull")
|
|
queue.drain(timeout=60)
|
|
|
|
done = queue.jobs.get(started.id) # type: ignore[union-attr]
|
|
assert done is not None and done.state == "done"
|
|
assert done.result == {"skipped": pull.ALREADY_RUNNING, "running": running}
|
|
assert [j.kind for j in queue.jobs.of_cascade(done.cascade_id)] == ["pull"]
|
|
|
|
|
|
def test_pull_is_a_job_on_a_six_hour_schedule_and_the_stages_are_tasks() -> None:
|
|
"""A schedule enqueues a job kind, so `pull` is a job; were it a task too,
|
|
the Jobs tab would draw two `pull` buttons."""
|
|
assert "pull" not in TASKS
|
|
assert {"extract", "embed", "score"} <= set(TASKS)
|
|
assert SCHEDULES["pull"].kind == "pull"
|
|
assert SCHEDULES["pull"].cadence == "every 6h"
|
|
|
|
|
|
def test_the_model_batches_are_isolated_with_a_timeout() -> None:
|
|
"""A hung model call is otherwise unkillable short of restarting the
|
|
server: `context.check()` runs between calls, not inside one, so only a
|
|
process boundary can end a batch that never comes back."""
|
|
for kind in ("extract_batch", "summarise_batch"):
|
|
spec = JOBS[kind]
|
|
assert spec.isolate, f"{kind} must run in its own process to be killable"
|
|
assert spec.timeout > 0, f"{kind} must have a deadline now that it can"
|
|
|
|
|
|
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
|
|
"""By position, not by name: `score` is in the chain twice, once before
|
|
extract and once after, and the second must lead on to summarise."""
|
|
assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "extract", "score",
|
|
"summarise")
|
|
assert [cascade.follows("pull", at) for at in range(6)] == \
|
|
["embed", "score", "extract", "score", "summarise", ""]
|
|
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 9) == ""
|
|
assert cascade.follows("nope", 0) == ""
|
|
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
|
|
|
|
|
|
def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
made: list[Any] = []
|
|
|
|
class Context:
|
|
def task(self, t: Any) -> Any:
|
|
made.append(t)
|
|
return t
|
|
|
|
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 2},
|
|
context=Context())
|
|
assert out == {"task": "score", "next": "extract"}
|
|
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 3}
|
|
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 4},
|
|
context=Context())
|
|
assert out["next"] == "summarise" and made[1].carries["at"] == 5
|