Files
jobboard/tests/test_score.py
T

1192 lines
54 KiB
Python

"""`embed`, `score` and the pull cascade, with the embedding backend replaced
by a deterministic fake. Nothing here downloads a model or reaches a board."""
from __future__ import annotations
import re
import sys
from collections.abc import Iterator, Sequence
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
import numpy as np
import pytest
from pydantic import ValidationError
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from desk.models import Item # noqa: E402
from pipeline import FINAL, JOBS, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, Tally # noqa: E402
from plugins.ats import ATSSource # noqa: E402
from app.config import CALIBRATION, AgentSeed, Config, ConfigError # noqa: E402
from app.jobs import cascade, embed, score # noqa: E402
from app.models import Candidate, Gate, Limits, Prompt, Weights # noqa: E402
from app.storage import Store # noqa: E402
from prepped import upsert # noqa: E402
NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
#: The shipped constants, which the fake is calibrated with so a test reads
#: the rule production runs; `uncalibrated` is the min-max fallback.
E5 = CALIBRATION["intfloat/multilingual-e5-small"]
#: The fake's whole vocabulary: one dimension per word, plus a constant one so
#: no text is the zero vector.
WORDS = ("backend", "python", "frontend", "react", "data")
class FakeEmbedder:
model_id = "fake"
def __init__(self) -> None:
self.calls: list[list[str]] = []
def encode(self, texts: Sequence[str]) -> np.ndarray:
self.calls.append(list(texts))
rows = []
for text in texts:
lower = text.lower()
vec = np.asarray([lower.count(w) for w in WORDS] + [0.1], dtype=np.float32)
rows.append(vec / np.linalg.norm(vec))
return np.vstack(rows)
class Clock:
def __init__(self) -> None:
self.at = NOW
def __call__(self) -> datetime:
return self.at
@pytest.fixture
def fake(monkeypatch: pytest.MonkeyPatch) -> FakeEmbedder:
embedder = FakeEmbedder()
monkeypatch.setattr(embed, "load_embedder", lambda model: embedder)
return embedder
@pytest.fixture
def clock() -> Clock:
return Clock()
@pytest.fixture
def store(tmp_path: Path, clock: Clock) -> Iterator[Store]:
s = Store(tmp_path / "roledesk.db")
s.clock = clock
yield s
@pytest.fixture
def config(tmp_path: Path) -> Config:
return Config(database=str(tmp_path / "roledesk.db"),
embed={"model": "fake", "calibration": {"fake": E5.model_dump()}})
@pytest.fixture
def uncalibrated(tmp_path: Path) -> Config:
return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
def role(store: Store, source_id: str, title: str = "Backend Engineer", *,
company: str = "Acme", body: str = "",
published_at: datetime = datetime(2026, 9, 20, tzinfo=UTC), **raw: Any) -> str:
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
title=title, body=body, published_at=published_at,
raw={"company": company, **raw})
item.source = "ats"
upsert(store, item)
return item.uid
def run(config: Config, store: Store, job: Any) -> dict[str, Any]:
notices: list[Any] = []
result = dict(job(config, store, context=RunContext(on_notice=notices.append)))
result["notices"] = [n.code for n in notices]
result["notice_texts"] = [n.text for n in notices]
return result
def level(text: str, intent: str, band: tuple[float, float] = E5.fit) -> float:
"""The calibrated fit the fake gives a role whose `document` is `text`."""
role_vec, intent_vec = FakeEmbedder().encode([text, intent])
lo, hi = band
return min(max((score.cosine(role_vec, intent_vec) - lo) / (hi - lo), 0.0), 1.0)
# ------------------------------------------------------------------ embed
def test_embed_writes_a_vector_per_open_role_once(
store: Store, config: Config, fake: FakeEmbedder) -> None:
role(store, "1")
closed = role(store, "2")
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (NOW.isoformat(), closed))
assert run(config, store, embed.run_embed)["embedded"] == 1
assert run(config, store, embed.run_embed)["embedded"] == 0
assert store.vector_counts() == {"fake": 1}
def test_embed_without_sentence_transformers_says_so_and_does_nothing(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
def missing(model: str) -> Any:
raise ImportError("no sentence_transformers")
monkeypatch.setattr(embed, "load_embedder", missing)
role(store, "1")
result = run(config, store, embed.run_embed)
assert result["embedded"] == 0 and result["notices"] == ["embed.unavailable"]
assert store.vector_counts() == {}
# -------------------------------------------------------- e5 prefixes
@pytest.mark.parametrize("model, wants", [
("intfloat/multilingual-e5-small", True),
("intfloat/e5-base-v2", True),
("intfloat/e5-large", True),
("sentence-transformers/all-MiniLM-L6-v2", False),
("sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", False),
("fake", False),
("some-org/e5models-are-not-this", False),
])
def test_wants_e5_prefix_matches_the_e5_family_by_name(model: str, wants: bool) -> None:
assert embed.wants_e5_prefix(model) == wants
def test_role_text_and_intent_text_add_the_e5_prefixes_only_for_e5(
store: Store) -> None:
uid = role(store, "1", "Backend Engineer", body="Python services")
r = store.role(uid)
assert r is not None
assert embed.role_text("fake", r) == embed.document(r)
assert embed.role_text("intfloat/multilingual-e5-small", r) == f"passage: {embed.document(r)}"
assert embed.intent_text("fake", "python backend") == "python backend"
assert embed.intent_text("intfloat/multilingual-e5-small", "python backend") == \
"query: python backend"
class FakeE5Embedder:
"""A second fake, standing in for a genuinely different model: its own
(narrower) vector width and an e5-shaped model id, so it exercises both
the prefixing and the re-embedding-on-model-change paths without
downloading anything."""
model_id = "intfloat/multilingual-e5-small"
def __init__(self) -> None:
self.calls: list[list[str]] = []
def encode(self, texts: Sequence[str]) -> np.ndarray:
self.calls.append(list(texts))
rows = []
for text in texts:
lower = text.lower()
vec = np.asarray([lower.count(w) for w in WORDS[:2]] + [0.1, 0.2], dtype=np.float32)
rows.append(vec / np.linalg.norm(vec))
return np.vstack(rows)
def test_embed_prefixes_role_text_for_an_e5_model(
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
e5 = FakeE5Embedder()
monkeypatch.setattr(embed, "load_embedder", lambda model: e5)
config = Config(database=str(tmp_path / "roledesk.db"),
embed={"model": "intfloat/multilingual-e5-small"})
role(store, "1", "Backend Engineer", body="Python services")
run(config, store, embed.run_embed)
assert e5.calls and e5.calls[0][0].startswith("passage: Backend Engineer")
def test_score_prefixes_the_intent_for_an_e5_model(
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
e5 = FakeE5Embedder()
monkeypatch.setattr(embed, "load_embedder", lambda model: e5)
config = Config(database=str(tmp_path / "roledesk.db"),
embed={"model": "intfloat/multilingual-e5-small"})
role(store, "1")
store.save_agent("Backend", "python backend")
run(config, store, score.run_score)
assert e5.calls == [["query: python backend"]]
# ------------------------------------------------------- model changes
def test_changing_the_embed_model_drops_old_vectors_and_reembeds(
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""The doc says a model change re-embeds; this is the guarantee that
actually makes that true, and the one that stops a run ever mixing two
models' vectors, which would be two different geometries under one
dimension count and garbage cosines."""
fake_a, fake_e5 = FakeEmbedder(), FakeE5Embedder()
embedders = {"fake": fake_a, "intfloat/multilingual-e5-small": fake_e5}
monkeypatch.setattr(embed, "load_embedder", lambda model: embedders[model])
role(store, "1")
role(store, "2")
config_a = Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
run(config_a, store, embed.run_embed)
assert store.vector_counts() == {"fake": 2}
dim_a = next(iter(store.vectors("fake").values())).shape[-1]
config_e5 = Config(database=str(tmp_path / "roledesk.db"),
embed={"model": "intfloat/multilingual-e5-small"})
result = run(config_e5, store, embed.run_embed)
# The old model's rows are gone, not just shadowed: `vectors(model)`
# filters by model, so a leftover row could never be read back mixed in,
# but drop_vectors is what stops it existing at all.
assert store.vector_counts() == {"intfloat/multilingual-e5-small": 2}
assert store.vectors("fake") == {}
assert result["dropped"] == 2
dim_e5 = next(iter(store.vectors("intfloat/multilingual-e5-small").values())).shape[-1]
assert dim_a != dim_e5, "the fakes must differ in width for this test to mean anything"
def test_changing_the_embed_model_recomputes_the_agents_cached_prompt_vector(
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""`prompt_vector` only returns a cache hit when `intent_key` -- which
hashes the model in -- matches what is stored, so a model change is a
cache miss and `_prompts` re-encodes under the new model, never handing a
stale-dimension vector into a run's cosines."""
fake_a, fake_e5 = FakeEmbedder(), FakeE5Embedder()
embedders = {"fake": fake_a, "intfloat/multilingual-e5-small": fake_e5}
monkeypatch.setattr(embed, "load_embedder", lambda model: embedders[model])
uid = role(store, "1")
agent = store.save_agent("Backend", "python backend")
config_a = Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
run(config_a, store, embed.run_embed)
run(config_a, store, score.run_score)
vec_a = store.prompt_vector(agent.id, "Backend", score.intent_key("fake", "python backend"))
assert vec_a is not None
config_e5 = Config(database=str(tmp_path / "roledesk.db"),
embed={"model": "intfloat/multilingual-e5-small"})
run(config_e5, store, embed.run_embed)
run(config_e5, store, score.run_score)
# The old key no longer resolves (the row moved on); the new key does,
# at the new model's own width, and scoring this role did not raise.
assert store.prompt_vector(
agent.id, "Backend", score.intent_key("fake", "python backend")) is None
vec_e5 = store.prompt_vector(
agent.id, "Backend", score.intent_key("intfloat/multilingual-e5-small", "python backend"))
assert vec_e5 is not None
assert vec_e5.shape != vec_a.shape
assert uid in store.scores(agent.id)
# ------------------------------------------------------------------ terms
def test_recency_halves_every_fourteen_days() -> None:
assert score.recency(NOW.isoformat(), NOW) == 1.0
assert score.recency((NOW - timedelta(days=14)).isoformat(), NOW) == pytest.approx(0.5)
assert score.recency((NOW - timedelta(days=28)).isoformat(), NOW) == pytest.approx(0.25)
def test_a_role_without_a_vector_is_scored_without_fit_unless_fit_is_all() -> None:
candidate = Candidate(uid="u", posted_at=NOW.isoformat(), affinity=0)
value, reason = score.score_one(candidate, [], Weights(), None, NOW)
assert reason == "" and value == pytest.approx((0.2 * 1.0 + 0.2 / 3) / 0.4)
only_fit = Weights(fit=1, recency=0, affinity=0)
assert score.score_one(candidate, [], only_fit, None, NOW) == (0.0, "no vector")
# ------------------------------------------------------------------ score
def test_a_require_gate_keeps_the_row_with_a_reason(
store: Store, config: Config, fake: FakeEmbedder) -> None:
berlin = role(store, "1", place="Berlin, Germany")
munich = role(store, "2", place="Munich")
nowhere = role(store, "3")
agent = store.save_agent("Berlin", "python backend",
[Gate(field="place", values=["Berlin"])])
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[berlin][1] == "" and scores[berlin][0] > 0
assert scores[munich] == (0.0, "place not Berlin")
# An empty column is no evidence: the role ranks, and extract can reach it.
assert scores[nowhere][1] == "" and scores[nowhere][0] > 0
shown = store.roles(agent=agent.id)
assert {r.uid for r in shown} == {berlin, nowhere}
assert {r.uid for r in store.roles(agent=agent.id, gated=True)} == {berlin, munich, nowhere}
def test_any_gates_are_one_group_a_role_must_match_one_of(
store: Store, config: Config, fake: FakeEmbedder) -> None:
""""In Denmark, or remote": a role passes on either, is gated when it has
evidence against both, and ranks when both columns are empty."""
copenhagen = role(store, "1", place="Copenhagen, Denmark")
remote_de = role(store, "2", place="Berlin, Germany", remote="remote")
onsite_de = role(store, "3", place="Berlin, Germany", remote="onsite")
hybrid_nowhere = role(store, "4", place="Hybrid") # remote from place, no country
blank = role(store, "5")
agent = store.save_agent("Copenhagen or remote", "", [
Gate(field="country", values=["DK"], mode="any"),
Gate(field="remote", values=["remote"], mode="any")])
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[copenhagen][1] == "" and scores[remote_de][1] == ""
assert scores[onsite_de] == (0.0, "none of country DK, remote remote")
assert scores[blank][1] == ""
# "Hybrid" is a country-less line the remote lookup reads as hybrid, so
# the group has evidence against remote and none for country: gated.
assert scores[hybrid_nowhere][1].startswith("none of")
def test_failed_gates_reads_evidence_only() -> None:
gates = [Gate(field="country", values=["DK"]),
Gate(field="remote", values=["remote"], mode="any"),
Gate(field="seniority", values=["senior"], mode="any")]
def cand(*states: int) -> Candidate:
return Candidate(uid="u", posted_at=NOW.isoformat(), affinity=0, gates=list(states))
assert score.failed_gates(cand(1, 1, 0), gates) == []
assert score.failed_gates(cand(-1, -1, -1), gates) == []
assert score.failed_gates(cand(0, 0, -1), gates) == [
"country not DK", "none of remote remote, seniority senior"]
assert score.failed_gates(cand(1, 0, 0), gates) == [
"none of remote remote, seniority senior"]
def test_a_gate_is_require_any_or_exclude_and_has_no_weight() -> None:
"""Prefer gates left the agent: hybrid is a feed filter now."""
with pytest.raises(ValidationError):
Gate(field="remote", values=["hybrid"], mode="prefer") # type: ignore[arg-type]
with pytest.raises(ValidationError):
Gate.model_validate({"field": "remote", "values": ["remote"], "weight": 1.0})
def test_an_exclude_gate_on_title_gates_whole_words_and_names_the_value(
store: Store, config: Config, fake: FakeEmbedder) -> None:
"""The sales and data engineering titles the owner does not want: a
phrase matches whole words of the folded title, so "Data Engineering
Manager" and "Forward Deployed Engineer" stay, and the reason names the
phrase that matched."""
ae = role(store, "1", "Enterprise Account Executive, UK")
de = role(store, "2", "Senior MLOps/Data Engineer")
manager = role(store, "3", "Data Engineering Manager")
fde = role(store, "4", "Forward Deployed Engineer")
se = role(store, "5", "Sales Engineer (SE)")
agent = store.save_agent("No sales", "", [
Gate(field="title", values=["Account Executive", "data engineer"], mode="exclude")])
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[ae] == (0.0, "title Account Executive")
assert scores[de] == (0.0, "title data engineer")
assert all(scores[uid][1] == "" and scores[uid][0] > 0 for uid in (manager, fde, se))
def test_a_favourite_is_gated_by_an_exclude_gate_it_matches(
store: Store, config: Config, fake: FakeEmbedder) -> None:
"""A star raises the level only: it skips no part of the fetch gate."""
store.add_company("Fave")
director = role(store, "1", "Account Director, Nordics", company="Fave")
engineer = role(store, "2", "Forward Deployed Engineer", company="Fave")
agent = store.save_agent("No sales", "", [
Gate(field="title", values=["account director"], mode="exclude")])
store.set_affinity(agent.id, "fave", 2)
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[director] == (0.0, "title account director")
assert scores[engineer][1] == ""
def test_failed_gates_names_the_value_an_exclude_gate_matched() -> None:
gates = [Gate(field="country", values=["DK"]),
Gate(field="title", values=["account executive", "rvp"], mode="exclude")]
def cand(*states: int) -> Candidate:
return Candidate(uid="u", posted_at=NOW.isoformat(), affinity=2, gates=list(states))
assert score.failed_gates(cand(1, 0), gates) == []
assert score.failed_gates(cand(1, -1), gates) == []
assert score.failed_gates(cand(1, 2), gates) == ["title rvp"]
assert score.failed_gates(cand(0, 1), gates) == ["country not DK", "title account executive"]
def test_a_favourite_is_gated_by_a_require_gate_it_fails(
store: Store, config: Config, fake: FakeEmbedder) -> None:
"""A star widens the search to the company, not past a require gate: the
Munich role of a starred company is gated like anyone's."""
store.add_company("Fave")
munich = role(store, "1", company="Fave", place="Munich")
berlin = role(store, "2", company="Fave", place="Berlin")
agent = store.save_agent("Berlin", "", [Gate(field="place", values=["Berlin"])])
store.set_affinity(agent.id, "fave", 2)
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[munich] == (0.0, "place not Berlin")
assert scores[berlin][1] == "" and scores[berlin][0] > 0
def test_all_countries_passes_the_any_group_a_favourite_does_not(
store: Store, config: Config, fake: FakeEmbedder) -> None:
""""Denmark or remote" gates the onsite German role of a neutral company
and of a favourite, and lets the same role of an All countries company
through."""
for name in ("Fave", "Wide", "Other"):
store.add_company(name)
fave = role(store, "1", company="Fave", place="Berlin, Germany", remote="onsite")
wide = role(store, "2", company="Wide", place="Berlin, Germany", remote="onsite")
other = role(store, "3", company="Other", place="Berlin, Germany", remote="onsite")
agent = store.save_agent("Copenhagen or remote", "", [
Gate(field="country", values=["DK"], mode="any"),
Gate(field="remote", values=["remote"], mode="any")])
store.set_affinity(agent.id, "fave", 2)
store.set_all_countries("wide", True)
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[wide][1] == "" and scores[wide][0] > 0
assert scores[fave] == (0.0, "none of country DK, remote remote")
assert scores[other] == (0.0, "none of country DK, remote remote")
def test_all_countries_skips_the_place_part_and_all_roles_the_titles() -> None:
"""The region applies to every company; All countries lifts the `any`
group, All roles the exclude gates on title and no other exclude gate."""
gates = [Gate(field="country", values=["DK", "SE", "EU"]),
Gate(field="country", values=["DK"], mode="any"),
Gate(field="remote", values=["remote"], mode="any"),
Gate(field="title", values=["account executive"], mode="exclude"),
Gate(field="company", values=["acme"], mode="exclude")]
def cand(*states: int, wide: bool = False, every: bool = False) -> Candidate:
return Candidate(uid="u", posted_at=NOW.isoformat(), all_countries=wide,
all_roles=every, gates=list(states))
assert score.failed_gates(cand(1, 0, 0, 0, 0), gates) == [
"none of country DK, remote remote"]
assert score.failed_gates(cand(1, 0, 0, 0, 0, wide=True), gates) == []
assert score.failed_gates(cand(0, 0, 0, 0, 0, wide=True, every=True), gates) == [
"country not DK / SE / EU"]
assert score.failed_gates(cand(1, 1, 0, 1, 0), gates) == ["title account executive"]
assert score.failed_gates(cand(1, 1, 0, 1, 0, wide=True), gates) == [
"title account executive"]
assert score.failed_gates(cand(1, 1, 0, 1, 0, every=True), gates) == []
assert score.failed_gates(cand(1, 1, 0, 1, 1, every=True), gates) == ["company acme"]
assert score.failed_gates(cand(1, 0, 0, 1, 0, every=True), gates) == [
"none of country DK, remote remote"]
def test_all_roles_keeps_a_title_the_agent_leaves_out(
store: Store, config: Config, fake: FakeEmbedder) -> None:
for name in ("Small", "Other"):
store.add_company(name)
small = role(store, "1", "Account Executive", company="Small")
other = role(store, "2", "Account Executive", company="Other")
agent = store.save_agent("No sales", "", [
Gate(field="title", values=["account executive"], mode="exclude")])
store.set_all_roles("small", True)
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[small][1] == ""
assert scores[other] == (0.0, "title account executive")
def test_a_favourite_with_an_empty_column_is_not_gated() -> None:
"""A require gate rejects on evidence only, for a favourite as for anyone."""
gates = [Gate(field="country", values=["DK"])]
candidate = Candidate(uid="u", posted_at=NOW.isoformat(), affinity=2, gates=[-1])
assert score.failed_gates(candidate, gates) == []
value, reason = score.score_one(candidate, gates, Weights(), 0.5, NOW)
assert reason == "" and value > 0
def test_a_favourite_ranks_above_the_same_role_of_a_neutral_company(
store: Store, config: Config, fake: FakeEmbedder) -> None:
store.add_company("Fave")
fave = role(store, "1", company="Fave", body="python backend")
other = role(store, "2", company="Other", body="python backend")
agent = store.save_agent("Backend", "python backend")
store.set_affinity(agent.id, "fave", 2)
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[fave][1] == "" == scores[other][1]
# Same age, and no vectors here, so fit is out and recency and affinity
# share the weight: the affinity term alone (1 against 1/3) is the gap.
assert scores[fave][0] - scores[other][0] == pytest.approx(0.5 * (1 - 1 / 3), abs=1e-5)
def test_a_favourite_is_the_weighted_sum_with_its_affinity_at_one() -> None:
"""No floor: a stale, poorly fitting role of a favourite scores what its
terms say, well under the old 0.9."""
stale = (NOW - timedelta(days=28)).isoformat()
candidate = Candidate(uid="u", posted_at=stale, affinity=2)
value, reason = score.score_one(candidate, [], Weights(), 0.0, NOW)
assert reason == ""
assert value == pytest.approx((0.2 * 0.25 + 0.2 * 1.0) / 1.0)
assert value < 0.9
# A perfect role is not lifted to anything but 1.
fresh = Candidate(uid="u", posted_at=NOW.isoformat(), affinity=2)
assert score.score_one(fresh, [], Weights(), 1.0, NOW)[0] == pytest.approx(1.0)
def test_a_favourite_with_no_weights_is_scored_like_any_other_company() -> None:
"""Weights that sum to 0 leave nothing to average: 0 with a reason, not a
number a favourite alone would get."""
none = Weights(fit=0, recency=0, affinity=0, taste=0)
for affinity in (0, 2):
candidate = Candidate(uid="u", posted_at=NOW.isoformat(), affinity=affinity)
assert score.score_one(candidate, [], none, None, NOW) == (0.0, "no weights")
def test_a_hidden_company_is_never_scored_for_the_agent_that_hid_it(
store: Store, config: Config, fake: FakeEmbedder) -> None:
store.add_company("Muted")
muted = role(store, "1", company="Muted")
kept = role(store, "2")
agent = store.save_agent("Any")
other = store.save_agent("Other")
store.set_affinity(agent.id, "muted", -1)
run(config, store, score.run_score)
assert set(store.scores(agent.id)) == {kept}
assert muted not in {r.uid for r in store.roles(agent=agent.id, gated=True)}
# The other agent did not hide it, so it is scored for that one.
assert set(store.scores(other.id)) == {muted, kept}
def test_an_older_role_scores_lower(
store: Store, config: Config, clock: Clock, fake: FakeEmbedder) -> None:
clock.at = NOW - timedelta(days=14)
old = role(store, "1")
clock.at = NOW
new = role(store, "2")
agent = store.save_agent("Any")
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[new][0] > scores[old][0]
def test_recency_counts_from_posted_at_not_first_seen(
store: Store, config: Config, fake: FakeEmbedder) -> None:
"""A trusted board's backlog, first seen today, must not score as fresh:
recency has to read the board's own (trusted) posted date, not the
moment the app happened to pull it in."""
stale = role(store, "1", board="greenhouse",
published_at=NOW - timedelta(days=28))
fresh = role(store, "2") # untrusted host: posted_at falls back to first_seen (NOW)
agent = store.save_agent("Any")
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[fresh][0] > scores[stale][0]
def test_two_agents_rank_the_same_roles_the_other_way_round(
store: Store, config: Config, fake: FakeEmbedder) -> None:
back = role(store, "1", "Backend Engineer", body="Python services")
front = role(store, "2", "Frontend Engineer", body="React apps")
only_fit = Weights(fit=1, recency=0, affinity=0)
backend = store.save_agent("Backend", "python backend developer", weights=only_fit)
frontend = store.save_agent("Frontend", "react frontend developer", weights=only_fit)
run(config, store, embed.run_embed)
result = run(config, store, score.run_score)
assert result["agents"] == 2
# Fit alone, calibrated: each agent's own role is at a cosine past `hi`,
# so 1, the other's near 0, below `lo`, so 0.
assert store.scores(backend.id)[back][0] == 1.0
assert store.scores(backend.id)[front][0] == 0.0
assert store.scores(frontend.id)[front][0] == 1.0
assert store.scores(frontend.id)[back][0] == 0.0
# ------------------------------------------------------------------ taste
def _unit(*words: str) -> np.ndarray:
return FakeEmbedder().encode([" ".join(words)])[0]
def test_taste_direction_is_liked_minus_passed_and_none_without_votes() -> None:
vectors = {"a": _unit("backend"), "b": _unit("frontend"), "c": _unit("data")}
assert score.taste_direction(vectors, {}) is None
assert score.taste_direction(vectors, {"a": 0}) is None # neutral says nothing
assert score.taste_direction(vectors, {"gone": 1}) is None # no vector to point at
liked_only = score.taste_direction(vectors, {"a": 1, "c": 1})
assert liked_only is not None
assert liked_only == pytest.approx((vectors["a"] + vectors["c"]) / 2)
passed_only = score.taste_direction(vectors, {"b": -1})
assert passed_only is not None
assert passed_only == pytest.approx(-vectors["b"])
both = score.taste_direction(vectors, {"a": 1, "b": -1, "c": 0})
assert both is not None
assert both == pytest.approx(vectors["a"] - vectors["b"])
def test_taste_ranks_a_role_near_the_liked_one_above_one_near_the_passed(
store: Store, config: Config, fake: FakeEmbedder) -> None:
liked = role(store, "1", "Backend Engineer", body="Python backend")
passed = role(store, "2", "Frontend Engineer", body="React frontend")
near_liked = role(store, "3", "Platform Engineer", body="Python backend services")
near_passed = role(store, "4", "Web Engineer", body="React frontend apps")
# No intent: with taste alone weighted, the votes are all that orders them.
agent = store.save_agent("Taste", "", weights=Weights(fit=0, recency=0, affinity=0,
taste=1))
store.set_vote(agent.id, liked, 1)
store.set_vote(agent.id, passed, -1)
run(config, store, embed.run_embed)
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[near_liked][0] > scores[near_passed][0]
# Cosines of about 0.7 and -0.7 to the direction, past both ends of the band.
assert scores[liked][0] == 1.0 and scores[passed][0] == 0.0
assert all(reason == "" for _, reason in scores.values())
def test_without_votes_taste_leaves_the_denominator(
store: Store, config: Config, fake: FakeEmbedder) -> None:
uid = role(store, "1")
agent = store.save_agent("Any", weights=Weights(fit=0, recency=0, affinity=0, taste=1))
run(config, store, embed.run_embed)
run(config, store, score.run_score)
# Taste is the only weighted term and there is no taste, so no weights.
assert store.scores(agent.id)[uid] == (0.0, "no weights")
def test_a_closed_liked_role_still_shapes_the_taste(
store: Store, config: Config, fake: FakeEmbedder) -> None:
liked = role(store, "1", "Backend Engineer", body="Python backend")
near = role(store, "2", "Platform Engineer", body="Python backend")
far = role(store, "3", "Web Engineer", body="React frontend")
agent = store.save_agent("Taste", "", weights=Weights(fit=0, recency=0, affinity=0,
taste=1))
run(config, store, embed.run_embed)
store.set_vote(agent.id, liked, 1)
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (NOW.isoformat(), liked))
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert liked not in scores
# Near sits on the liked direction, past `hi`. Far is orthogonal to it,
# a cosine of about 0 and so a middling level, not the 0 min-max gave it.
assert scores[near][0] == 1.0
assert scores[far][0] == pytest.approx(
level("Web Engineer\nReact frontend", "Backend Engineer\nPython backend", E5.taste),
abs=1e-5)
assert 0 < scores[far][0] < 1
# ------------------------------------------------------------ fit scaling
def test_scale_maps_a_cosine_to_its_calibrated_level_and_clips_both_ends() -> None:
# multilingual-e5-small's fit band: lo 0.75 is 0, hi 0.88 is 1.
assert E5.fit == (0.75, 0.88) and E5.taste == (-0.14, 0.19)
assert score.scale({"lo": 0.75, "mid": 0.815, "hi": 0.88, "p50": 0.7989},
E5.fit) == pytest.approx(
{"lo": 0.0, "mid": 0.5, "hi": 1.0, "p50": 0.0489 / 0.13})
assert score.scale({"under": 0.6, "negative": -0.2, "over": 0.9031, "one": 1.0},
E5.fit) == {"under": 0.0, "negative": 0.0, "over": 1.0, "one": 1.0}
assert score.scale({"none": 0.0, "liked": 0.25}, E5.taste) == pytest.approx(
{"none": 0.14 / 0.33, "liked": 1.0})
# A lone role, or roles that are all equally close, get their level.
assert score.scale({"only": 0.8}, E5.fit) == pytest.approx({"only": 0.05 / 0.13})
assert score.scale({}, E5.fit) == {}
def test_scale_without_a_band_falls_back_to_min_max() -> None:
assert score.scale({"a": 0.87, "b": 0.95, "c": 0.91}, None) == pytest.approx(
{"a": 0.0, "b": 1.0, "c": 0.5})
assert score.scale({"only": 0.3}, None) == {"only": 1.0}
assert score.scale({"a": 0.4, "b": 0.4}, None) == {"a": 1.0, "b": 1.0}
assert score.scale({}, None) == {}
def test_a_single_candidate_gets_its_level_not_full_fit(
store: Store, config: Config, uncalibrated: Config, fake: FakeEmbedder) -> None:
"""Min-max gave the best of a poor set 1; a level gives it what it is."""
uid = role(store, "1", "Frontend Engineer", body="React apps")
agent = store.save_agent("Backend", "python backend developer",
weights=Weights(fit=1, recency=0, affinity=0))
run(config, store, embed.run_embed)
run(config, store, score.run_score)
assert store.scores(agent.id)[uid] == (0.0, "")
run(uncalibrated, store, score.run_score)
assert store.scores(agent.id)[uid] == (1.0, "")
def test_the_same_role_gets_the_same_fit_whatever_else_is_open(
store: Store, config: Config, uncalibrated: Config, fake: FakeEmbedder) -> None:
"""The point of a level: two runs over different candidate sets give the
role the same fit, where min-max gave it 1 beside a worse role and 0
beside a better one."""
mid = role(store, "1", "Backend Engineer", body="Python backend data")
worse = role(store, "2", "Frontend Engineer", body="React apps")
agent = store.save_agent("Backend", "python backend",
weights=Weights(fit=1, recency=0, affinity=0))
expected = level("Backend Engineer\nPython backend data", "python backend")
assert 0.5 < expected < 1
run(config, store, embed.run_embed)
run(config, store, score.run_score)
first = store.scores(agent.id)[mid][0]
run(uncalibrated, store, score.run_score)
assert store.scores(agent.id)[mid][0] == 1.0
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (NOW.isoformat(), worse))
role(store, "3", "Backend Engineer", body="Python")
run(config, store, embed.run_embed)
run(config, store, score.run_score)
second = store.scores(agent.id)[mid][0]
run(uncalibrated, store, score.run_score)
assert store.scores(agent.id)[mid][0] == 0.0
assert first == second == pytest.approx(expected, abs=1e-5)
def test_an_uncalibrated_model_falls_back_to_min_max_and_warns_once_a_run(
store: Store, uncalibrated: Config, fake: FakeEmbedder,
caplog: pytest.LogCaptureFixture) -> None:
close = role(store, "1", "Backend Engineer", body="Python backend data")
far = role(store, "2", "Data Engineer", body="Data")
only_fit = Weights(fit=1, recency=0, affinity=0)
one = store.save_agent("One", "python backend", weights=only_fit)
two = store.save_agent("Two", "python backend", weights=only_fit)
run(uncalibrated, store, embed.run_embed)
with caplog.at_level("WARNING", logger="app.jobs.score"):
run(uncalibrated, store, score.run_score)
warnings = [r.getMessage() for r in caplog.records if r.name == "app.jobs.score"]
assert len(warnings) == 1 and warnings[0].startswith("no calibration for fake")
for agent in (one, two):
assert store.scores(agent.id)[close][0] == 1.0
assert store.scores(agent.id)[far][0] == 0.0
def test_the_calibration_is_the_built_in_one_unless_the_config_replaces_it() -> None:
assert Config().embed.calibrated() == E5
assert Config(embed={"model": "fake"}).embed.calibrated() is None
mine = {"fit": [0.7, 0.9], "taste": [-0.2, 0.2]}
replaced = Config(embed={"calibration": {"intfloat/multilingual-e5-small": mine}})
assert replaced.embed.calibrated() is not None
assert replaced.embed.calibrated().fit == (0.7, 0.9) # type: ignore[union-attr]
# Keyed by model: another model's entry does not apply.
other = Config(embed={"model": "fake", "calibration": {"other": mine}})
assert other.embed.calibrated() is None
@pytest.mark.parametrize("entry, says", [
({"fit": [0.88, 0.75], "taste": [-0.14, 0.19]}, "embed.calibration.fake.fit"),
({"fit": [0.75, 0.75], "taste": [-0.14, 0.19]}, "lo < hi"),
({"fit": [0.75, 0.88], "taste": [-2, 0.19]}, "embed.calibration.fake.taste"),
({"fit": [0.75, 0.88]}, "embed.calibration.fake.taste"),
({"fit": [0.75, 0.88], "taste": [-0.14, 0.19], "spread": 1}, "unknown setting"),
])
def test_a_bad_calibration_is_a_config_error(
tmp_path: Path, entry: dict[str, Any], says: str) -> None:
import yaml
from app import config as config_mod
path = tmp_path / "config.yaml"
path.write_text(yaml.safe_dump({"embed": {"model": "fake",
"calibration": {"fake": entry}}}))
with pytest.raises(ConfigError, match=re.escape(says)):
config_mod.load(path)
def test_an_agent_without_an_intent_feeds_neither_model_stage_and_is_named(
store: Store, config: Config, clock: Clock, fake: FakeEmbedder) -> None:
"""Its scores have no fit, so they rank fresh roles, not wanted ones: the
newer role tops it, and nothing of it enters the cut by fit, nor does its
top choose what summarise reads."""
clock.at = NOW - timedelta(days=1)
wanted = role(store, "1", "Backend Engineer", body="Python backend")
clock.at = NOW
fresh = role(store, "2", "Frontend Engineer", body="React apps")
top_one = Limits(cut_fit_top=1, cut_fit_min=0, summarise_top=1)
store.save_agent("Backend", "python backend", limits=top_one)
blind = store.save_agent("Fresh", "", limits=top_one)
store.save_agent("Also fresh", " \n", limits=top_one)
store.save_agent("Blank prompts", limits=top_one,
prompts=[Prompt(name="One"), Prompt(name="Two", text=" ")])
store.save_agent("Off", "", enabled=False)
run(config, store, embed.run_embed)
result = run(config, store, score.run_score)
assert result["notices"] == ["score.no_intent"]
assert result["notice_texts"][0].startswith("No prompt with text")
assert result["notice_texts"][0].endswith(": Also fresh, Blank prompts, Fresh.")
ranked = sorted(store.scores(blind.id).items(), key=lambda kv: -kv[1][0])
assert ranked[0][0] == fresh, "the intent-less agent would pick the newer role"
assert [r.uid for r in store.extract_waiting(10)] == [wanted]
assert [r.uid for r in store.summarise_waiting(10)] == [wanted]
assert store.extract_pending() == store.summarise_pending() == 1
def test_a_gated_role_does_not_stretch_the_fit_range_under_the_fallback(
store: Store, uncalibrated: Config, fake: FakeEmbedder) -> None:
"""Calibrated, no other role moves a role's fit. Min-max, the fallback,
must still leave a gated role out of the range."""
config = uncalibrated
close = role(store, "1", "Backend Engineer", body="Python backend", place="Berlin")
far = role(store, "2", "Data Engineer", body="Data", place="Berlin")
# Farther from the intent than either, but gated out by place.
role(store, "3", "Frontend Engineer", body="React react", place="Munich")
agent = store.save_agent("Berlin", "python backend",
[Gate(field="place", values=["Berlin"])],
weights=Weights(fit=1, recency=0, affinity=0))
run(config, store, embed.run_embed)
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[close][0] == 1.0 and scores[far][0] == 0.0
def test_the_intent_vector_is_cached_until_the_intent_changes(
store: Store, config: Config, fake: FakeEmbedder) -> None:
role(store, "1")
agent = store.save_agent("Backend", "python backend")
run(config, store, score.run_score)
run(config, store, score.run_score)
assert fake.calls == [["python backend"]]
store.save_agent("Backend", "react frontend", agent_id=agent.id)
run(config, store, score.run_score)
assert fake.calls[-1] == ["react frontend"]
# ----------------------------------------------------------------- prompts
def test_fit_is_the_best_prompts_level_and_the_score_names_that_prompt(
store: Store, config: Config, fake: FakeEmbedder) -> None:
backend = role(store, "1", "Backend Engineer", body="Python backend")
frontend = role(store, "2", "Frontend Engineer", body="React frontend")
gated = role(store, "3", "Backend Engineer", body="Python backend", place="Munich")
agent = store.save_agent(
"Two kinds", gates=[Gate(field="place", values=["Berlin"])],
weights=Weights(fit=1, recency=0, affinity=0),
prompts=[Prompt(name="Web", text="react frontend"),
Prompt(name="Empty"),
Prompt(name="Server", text="python backend")])
run(config, store, embed.run_embed)
run(config, store, score.run_score)
texts = {backend: "Backend Engineer\nPython backend",
frontend: "Frontend Engineer\nReact frontend"}
served = {r.uid: r for r in store.roles(agent=agent.id, gated=True)}
assert served[backend].prompt == "Server" and served[frontend].prompt == "Web"
for uid, text in texts.items():
best = max(level(text, "react frontend"), level(text, "python backend"))
assert served[uid].score == pytest.approx(best, abs=1e-5)
# A gated role has no fit that counted, so no prompt.
assert (served[gated].gated_reason, served[gated].prompt) == ("place not Berlin", "")
assert fake.calls[1:] == [["react frontend"], ["python backend"]]
def test_one_prompt_scores_exactly_as_the_intent_did(
store: Store, config: Config, uncalibrated: Config, fake: FakeEmbedder) -> None:
"""The legacy intent, one named prompt, and one prompt beside an empty
one are the same agent, calibrated and under the min-max fallback."""
for n, body in enumerate(["Python backend", "Python backend data", "React apps", "Data"]):
role(store, str(n), "Engineer", body=body)
legacy = store.save_agent("Legacy", "python backend")
named = store.save_agent("Named", prompts=[Prompt(name="Backend", text="python backend")])
padded = store.save_agent("Padded", prompts=[Prompt(name="Later"),
Prompt(name="Backend", text="python backend")])
run(config, store, embed.run_embed)
for cfg in (config, uncalibrated):
run(cfg, store, score.run_score)
assert store.scores(legacy.id) == store.scores(named.id) == store.scores(padded.id)
assert {r.prompt for r in store.roles(agent=legacy.id)} == {"Legacy"}
assert {r.prompt for r in store.roles(agent=padded.id)} == {"Backend"}
def test_under_the_fallback_each_prompt_is_ranged_on_its_own(
store: Store, uncalibrated: Config, fake: FakeEmbedder) -> None:
"""Min-max per prompt: each prompt's closest role gets 1, however far
that prompt's roles sit from it in absolute terms."""
backend = role(store, "1", "Backend Engineer", body="Python backend")
frontend = role(store, "2", "Frontend Engineer", body="React frontend")
agent = store.save_agent("Two", weights=Weights(fit=1, recency=0, affinity=0),
prompts=[Prompt(name="Server", text="python backend"),
Prompt(name="Web", text="react")])
run(uncalibrated, store, embed.run_embed)
run(uncalibrated, store, score.run_score)
served = {r.uid: (r.score, r.prompt) for r in store.roles(agent=agent.id)}
assert served == {backend: (1.0, "Server"), frontend: (1.0, "Web")}
def test_an_edit_re_embeds_only_the_prompt_it_changed(
store: Store, config: Config, fake: FakeEmbedder) -> None:
role(store, "1")
agent = store.save_agent("Two", prompts=[Prompt(name="Server", text="python backend"),
Prompt(name="Web", text="react frontend")])
run(config, store, score.run_score)
run(config, store, score.run_score)
assert fake.calls == [["python backend"], ["react frontend"]]
# Reordered, one renamed and one rewritten: only the new text is encoded.
store.save_agent("Two", agent_id=agent.id,
prompts=[Prompt(name="Frontend", text="react frontend"),
Prompt(name="Server", text="python data")])
run(config, store, score.run_score)
assert fake.calls[2:] == [["python data"]]
def test_without_an_embedder_the_prompts_are_skipped_and_fit_is_out(
store: Store, config: Config, fake: FakeEmbedder,
monkeypatch: pytest.MonkeyPatch) -> None:
uid = role(store, "1")
run(config, store, embed.run_embed)
def missing(model: str) -> Any:
raise ImportError("no sentence_transformers")
monkeypatch.setattr(embed, "load_embedder", missing)
agent = store.save_agent("Fit only", weights=Weights(fit=1, recency=0, affinity=0),
prompts=[Prompt(name="Server", text="python backend")])
result = run(config, store, score.run_score)
assert result["notices"] == ["embed.unavailable"]
assert store.scores(agent.id) == {uid: (0.0, "no prompt vector")}
def test_a_disabled_agent_is_not_scored(
store: Store, config: Config, fake: FakeEmbedder) -> None:
role(store, "1")
agent = store.save_agent("Off", enabled=False)
assert run(config, store, score.run_score)["agents"] == 0
assert store.scores(agent.id) == {}
def test_starter_agents_are_imported_once(store: Store, tmp_path: Path) -> None:
config = Config(database=str(tmp_path / "roledesk.db"),
agents=[AgentSeed(name="A"), AgentSeed(name="B")])
assert score.seed_agents(config, store) == ["A", "B"]
for agent in store.agents():
store.delete_agent(agent.id)
assert score.seed_agents(config, store) == []
assert store.agents() == []
def test_the_shipped_starter_agents_validate() -> None:
from app import config as config_mod
agents = config_mod.load(ROOT / "config.yaml").agents
assert [a.name for a in agents] == ["Copenhagen engineering"]
# The two commented examples in config.example.yaml, uncommented, so they
# still validate when somebody copies them.
import yaml
lines = (ROOT / "config.example.yaml").read_text(encoding="utf-8").splitlines()
start = lines.index("agents: []") + 1
block = [line[2:] for line in lines[start:] if line.startswith("# ")]
examples = [AgentSeed.model_validate(a) for a in yaml.safe_load("\n".join(block))]
assert [a.name for a in examples] == ["Remote from Europe", "Europe hybrid"]
assert examples[0].limits.cut_fit_top == 1000
def test_a_starter_agent_may_list_prompts_or_give_an_intent(
store: Store, tmp_path: Path) -> None:
config = Config(database=str(tmp_path / "roledesk.db"), agents=[
AgentSeed(name="Old", intent="python backend"),
AgentSeed.model_validate({"name": "New", "prompts": [
{"name": "AI", "text": "Head of AI"}, {"name": "FDE", "text": "FDE lead"}]})])
score.seed_agents(config, store)
assert {a.name: a.prompts for a in store.agents()} == {
"New": [Prompt(name="AI", text="Head of AI"), Prompt(name="FDE", text="FDE lead")],
"Old": [Prompt(name="Old", text="python backend")]}
with pytest.raises(ValueError, match="two prompts are named"):
AgentSeed.model_validate({"name": "Bad", "prompts": [{"name": "A"}, {"name": "a"}]})
def test_the_cli_adds_replaces_removes_and_lists_prompts(
tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
import yaml
from app import cli
db = tmp_path / "roledesk.db"
path = tmp_path / "config.yaml"
path.write_text(yaml.safe_dump({"database": str(db)}))
agent = Store(db).save_agent("Wolf", "Head of data")
def app(*argv: str) -> int:
return cli.main(["-c", str(path), "agents", *argv])
assert app("prompt", "Wolf", "FDE", "Forward deployed lead") == 0
assert app("prompt", "Wolf", "wolf", "Head of AI") == 0 # replaced in place
assert app("prompt", "Wolf", "Research") == 0 # no text yet
assert Store(db).agent(agent.id).prompts == [ # type: ignore[union-attr]
Prompt(name="wolf", text="Head of AI"), Prompt(name="FDE", text="Forward deployed lead"),
Prompt(name="Research")]
assert app("prompt", "Wolf", "fde", "--remove") == 0
assert app("prompt", "Wolf", "Nothing", "--remove") == 1
assert app("prompt", "Nobody", "X", "y") == 1
capsys.readouterr()
assert app("list") == 0
assert capsys.readouterr().out == "Wolf\n wolf: Head of AI\n Research: \n"
def test_the_shipped_starter_searches_validate() -> None:
from app import config as config_mod
searches = config_mod.load(ROOT / "config.yaml").searches
assert searches and {s.provider for s in searches} == {"jobindex"}
assert all(s.params.get("q") for s in searches)
# ---------------------------------------------------------------- cascade
def test_the_pull_job_runs_pull_prep_embed_score_extract_score_judge_then_summarise(
store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
board="greenhouse")
agent = store.save_agent("Backend", "python backend")
def fake_fetch(self: ATSSource) -> Iterator[Item]:
item = Item(source_id="greenhouse:acme:1", location="https://x.example/1",
title="Backend Engineer", body="Python",
published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"company": "Acme", "board": "greenhouse", "slug": "acme"})
yield item
self.state["ats:ok:greenhouse:acme"] = self.options["run"]
self.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config, store,
workers={"cpu": 1, "http": 1, "llm": 1})
# The `pull` job, which is what the schedule fires: its own work is to ask
# for the chain, which then runs to its end in the same drain.
started = queue.enqueue("pull")
queue.drain(timeout=60)
assert started is not None and queue.jobs.get(started.id).state == "done" # type: ignore[union-attr]
batch = queue.jobs.last("pull_batch")
assert batch is not None
jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id)
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
# Fetch text found no board with role pages, so the prep, embed and score
# behind it are passed over.
assert stages == ["pull", "prep", "embed", "score", "fetch_text", "extract", "score",
"judge", "summarise"]
work = [j.kind for j in jobs if j.kind != FINAL]
# The `pull` job and the chain it asked for are one cascade, so the Jobs
# tab shows a scheduled pull as one run. Extract and summarise are off and
# find nothing; score runs twice, before and after extract.
assert work == ["pull", "pull_batch", "prep", "embed", "score", "score"]
assert all(j.state == "done" for j in jobs)
assert len(store.scores(agent.id)) == 1
# The board was stamped with the run the `pull` job claimed, and the
# settle let that run go, so the next pull may start.
run = queue.jobs.get(started.id).result["run"] # type: ignore[union-attr]
assert store.state["ats:ok:greenhouse:acme"] == run == store.state["pull:settled"]
def test_the_pull_job_queues_nothing_while_another_pull_runs(
store: Store, config: Config, tmp_path: Path) -> None:
from app.jobs import pull
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
board="greenhouse")
running = pull.claim_run(store)
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config, store,
workers={"cpu": 1, "http": 1, "llm": 1})
started = queue.enqueue("pull")
queue.drain(timeout=60)
done = queue.jobs.get(started.id) # type: ignore[union-attr]
assert done is not None and done.state == "done"
assert done.result == {"skipped": pull.ALREADY_RUNNING, "running": running}
assert [j.kind for j in queue.jobs.of_cascade(done.cascade_id)] == ["pull"]
def test_pull_is_a_job_on_a_six_hour_schedule_and_the_stages_are_tasks() -> None:
"""A schedule enqueues a job kind, so `pull` is a job; were it a task too,
the Jobs tab would draw two `pull` buttons."""
assert "pull" not in TASKS
assert {"extract", "embed", "score"} <= set(TASKS)
assert SCHEDULES["pull"].kind == "pull"
assert SCHEDULES["pull"].cadence == "every 6h"
def test_the_model_batches_are_isolated_with_a_timeout() -> None:
"""A hung model call is otherwise unkillable short of restarting the
server: `context.check()` runs between calls, not inside one, so only a
process boundary can end a batch that never comes back."""
for kind in ("extract_batch", "summarise_batch"):
spec = JOBS[kind]
assert spec.isolate, f"{kind} must run in its own process to be killable"
assert spec.timeout > 0, f"{kind} must have a deadline now that it can"
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
"""By position, not by name: `score` is in the chain three times, and the
one after extract must lead on to judge."""
assert cascade.CASCADES["pull"] == ("pull", "prep", "embed", "score", "fetch_text", "prep",
"embed", "score", "extract", "score", "judge",
"summarise")
assert [cascade.follows("pull", at) for at in range(12)] == \
["prep", "embed", "score", "fetch_text", "prep", "embed", "score", "extract", "score",
"judge", "summarise", ""]
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 11) == ""
assert cascade.follows("nope", 0) == ""
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None:
made: list[Any] = []
class Context:
def task(self, t: Any) -> Any:
made.append(t)
return t
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 7},
context=Context())
assert out == {"task": "score", "next": "extract"}
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 8}
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 9},
context=Context())
assert out["next"] == "judge" and made[1].carries["at"] == 10