Score roles per search agent, and run pull into extract, embed and score
embed stores one vector per open role; score rates every open role for every enabled agent from require and prefer gates, fit against the agent's intent, recency and company affinity, keeping gated rows with a reason. Schema v3 adds vectors and the cached intent vector. The page gets an agent order, the score on each row, and an Agents tab. Two starter agents in roles.yaml are imported once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4dfb8831aa
commit
bc60324fab
22 files changed
+1457
-44
No files matched your search
@@ -0,0 +1,313 @@
|
||||
"""`embed`, `score` and the pull cascade, with the embedding backend replaced
|
||||
by a deterministic fake. Nothing here downloads a model or reaches a board."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from collections.abc import Iterator, Sequence
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from desk.models import Item # noqa: E402
|
||||
from pipeline import FINAL, TASKS, JobQueue, Jobs, RunContext # noqa: E402
|
||||
from plugins.ats import ATSSource # noqa: E402
|
||||
from roledesk.config import AgentSeed, Config # noqa: E402
|
||||
from roledesk.jobs import cascade, embed, score # noqa: E402
|
||||
from roledesk.models import Candidate, Gate, Weights # noqa: E402
|
||||
from roledesk.storage import Store # noqa: E402
|
||||
|
||||
NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
|
||||
|
||||
#: The fake's whole vocabulary: one dimension per word, plus a constant one so
|
||||
#: no text is the zero vector.
|
||||
WORDS = ("backend", "python", "frontend", "react", "data")
|
||||
|
||||
|
||||
class FakeEmbedder:
|
||||
model_id = "fake"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[list[str]] = []
|
||||
|
||||
def encode(self, texts: Sequence[str]) -> np.ndarray:
|
||||
self.calls.append(list(texts))
|
||||
rows = []
|
||||
for text in texts:
|
||||
lower = text.lower()
|
||||
vec = np.asarray([lower.count(w) for w in WORDS] + [0.1], dtype=np.float32)
|
||||
rows.append(vec / np.linalg.norm(vec))
|
||||
return np.vstack(rows)
|
||||
|
||||
|
||||
class Clock:
|
||||
def __init__(self) -> None:
|
||||
self.at = NOW
|
||||
|
||||
def __call__(self) -> datetime:
|
||||
return self.at
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake(monkeypatch: pytest.MonkeyPatch) -> FakeEmbedder:
|
||||
embedder = FakeEmbedder()
|
||||
monkeypatch.setattr(embed, "load_embedder", lambda model: embedder)
|
||||
return embedder
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clock() -> Clock:
|
||||
return Clock()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path, clock: Clock) -> Iterator[Store]:
|
||||
s = Store(tmp_path / "roledesk.db")
|
||||
s.clock = clock
|
||||
yield s
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def config(tmp_path: Path) -> Config:
|
||||
return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
|
||||
|
||||
|
||||
def role(store: Store, source_id: str, title: str = "Backend Engineer", *,
|
||||
company: str = "Acme", body: str = "", **raw: Any) -> str:
|
||||
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
|
||||
title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
||||
raw={"company": company, **raw})
|
||||
item.source = "ats"
|
||||
store.upsert_role(item)
|
||||
return item.uid
|
||||
|
||||
|
||||
def run(config: Config, store: Store, job: Any) -> dict[str, Any]:
|
||||
notices: list[Any] = []
|
||||
result = dict(job(config, store, context=RunContext(on_notice=notices.append)))
|
||||
result["notices"] = [n.code for n in notices]
|
||||
return result
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ embed
|
||||
|
||||
def test_embed_writes_a_vector_per_open_role_once(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
role(store, "1")
|
||||
closed = role(store, "2")
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (NOW.isoformat(), closed))
|
||||
|
||||
assert run(config, store, embed.run_embed)["embedded"] == 1
|
||||
assert run(config, store, embed.run_embed)["embedded"] == 0
|
||||
assert store.vector_counts() == {"fake": 1}
|
||||
|
||||
|
||||
def test_embed_without_sentence_transformers_says_so_and_does_nothing(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def missing(model: str) -> Any:
|
||||
raise ImportError("no sentence_transformers")
|
||||
|
||||
monkeypatch.setattr(embed, "load_embedder", missing)
|
||||
role(store, "1")
|
||||
|
||||
result = run(config, store, embed.run_embed)
|
||||
|
||||
assert result["embedded"] == 0 and result["notices"] == ["embed.unavailable"]
|
||||
assert store.vector_counts() == {}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ terms
|
||||
|
||||
def test_recency_halves_every_fourteen_days() -> None:
|
||||
assert score.recency(NOW.isoformat(), NOW) == 1.0
|
||||
assert score.recency((NOW - timedelta(days=14)).isoformat(), NOW) == pytest.approx(0.5)
|
||||
assert score.recency((NOW - timedelta(days=28)).isoformat(), NOW) == pytest.approx(0.25)
|
||||
|
||||
|
||||
def test_a_role_without_a_vector_is_scored_without_fit_unless_fit_is_all() -> None:
|
||||
candidate = Candidate(uid="u", first_seen=NOW.isoformat(), affinity=0)
|
||||
value, reason = score.score_one(candidate, [], Weights(), None, NOW)
|
||||
assert reason == "" and value == pytest.approx((0.2 * 1.0 + 0.2 / 3) / 0.4)
|
||||
|
||||
only_fit = Weights(fit=1, recency=0, affinity=0)
|
||||
assert score.score_one(candidate, [], only_fit, None, NOW) == (0.0, "no vector")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ score
|
||||
|
||||
def test_a_require_gate_keeps_the_row_with_a_reason(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
berlin = role(store, "1", place="Berlin, Germany")
|
||||
munich = role(store, "2", place="Munich")
|
||||
nowhere = role(store, "3")
|
||||
agent = store.save_agent("Berlin", "python backend",
|
||||
[Gate(field="place", values=["Berlin"])])
|
||||
|
||||
run(config, store, score.run_score)
|
||||
|
||||
scores = store.scores(agent.id)
|
||||
assert scores[berlin][1] == "" and scores[berlin][0] > 0
|
||||
assert scores[munich] == (0.0, "place not Berlin")
|
||||
assert scores[nowhere] == (0.0, "place unknown")
|
||||
shown = store.roles(agent=agent.id)
|
||||
assert [r.uid for r in shown] == [berlin]
|
||||
assert {r.uid for r in store.roles(agent=agent.id, gated=True)} == {berlin, munich, nowhere}
|
||||
|
||||
|
||||
def test_a_prefer_gate_adds_its_weight_to_the_score(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
remote = role(store, "1", remote="remote")
|
||||
office = role(store, "2", remote="onsite")
|
||||
agent = store.save_agent("Remote", "", [Gate(field="remote", values=["remote"],
|
||||
mode="prefer", weight=0.5)])
|
||||
|
||||
run(config, store, score.run_score)
|
||||
|
||||
scores = store.scores(agent.id)
|
||||
# No intent, so fit is out: recency 0.2 + affinity 0.2 + the gate's 0.5.
|
||||
assert scores[remote][0] - scores[office][0] == pytest.approx(0.5 / 0.9, abs=1e-5)
|
||||
assert scores[office][1] == ""
|
||||
|
||||
|
||||
def test_a_favourite_skips_the_gates_and_is_floored(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
store.add_company("Fave", affinity=2)
|
||||
fave = role(store, "1", company="Fave", place="Munich")
|
||||
agent = store.save_agent("Berlin", "", [Gate(field="place", values=["Berlin"])])
|
||||
|
||||
run(config, store, score.run_score)
|
||||
|
||||
value, reason = store.scores(agent.id)[fave]
|
||||
assert reason == "" and value >= 0.9
|
||||
|
||||
|
||||
def test_a_muted_company_is_never_scored(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
store.add_company("Muted", affinity=-1)
|
||||
muted = role(store, "1", company="Muted")
|
||||
kept = role(store, "2")
|
||||
agent = store.save_agent("Any")
|
||||
|
||||
run(config, store, score.run_score)
|
||||
|
||||
assert set(store.scores(agent.id)) == {kept}
|
||||
assert muted not in {r.uid for r in store.roles(agent=agent.id, gated=True)}
|
||||
|
||||
|
||||
def test_an_older_role_scores_lower(
|
||||
store: Store, config: Config, clock: Clock, fake: FakeEmbedder) -> None:
|
||||
clock.at = NOW - timedelta(days=14)
|
||||
old = role(store, "1")
|
||||
clock.at = NOW
|
||||
new = role(store, "2")
|
||||
agent = store.save_agent("Any")
|
||||
|
||||
run(config, store, score.run_score)
|
||||
|
||||
scores = store.scores(agent.id)
|
||||
assert scores[new][0] > scores[old][0]
|
||||
|
||||
|
||||
def test_two_agents_score_the_same_role_differently(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
uid = role(store, "1", "Backend Engineer", body="Python services")
|
||||
backend = store.save_agent("Backend", "python backend developer",
|
||||
weights=Weights(fit=1, recency=0, affinity=0))
|
||||
frontend = store.save_agent("Frontend", "react frontend developer",
|
||||
weights=Weights(fit=1, recency=0, affinity=0))
|
||||
|
||||
run(config, store, embed.run_embed)
|
||||
result = run(config, store, score.run_score)
|
||||
|
||||
assert result["agents"] == 2
|
||||
assert store.scores(backend.id)[uid][0] > store.scores(frontend.id)[uid][0]
|
||||
|
||||
|
||||
def test_the_intent_vector_is_cached_until_the_intent_changes(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
role(store, "1")
|
||||
agent = store.save_agent("Backend", "python backend")
|
||||
|
||||
run(config, store, score.run_score)
|
||||
run(config, store, score.run_score)
|
||||
assert fake.calls == [["python backend"]]
|
||||
|
||||
store.save_agent("Backend", "react frontend", agent_id=agent.id)
|
||||
run(config, store, score.run_score)
|
||||
assert fake.calls[-1] == ["react frontend"]
|
||||
|
||||
|
||||
def test_a_disabled_agent_is_not_scored(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
role(store, "1")
|
||||
agent = store.save_agent("Off", enabled=False)
|
||||
assert run(config, store, score.run_score)["agents"] == 0
|
||||
assert store.scores(agent.id) == {}
|
||||
|
||||
|
||||
def test_starter_agents_are_imported_once(store: Store, tmp_path: Path) -> None:
|
||||
config = Config(database=str(tmp_path / "roledesk.db"),
|
||||
agents=[AgentSeed(name="A"), AgentSeed(name="B")])
|
||||
assert score.seed_agents(config, store) == ["A", "B"]
|
||||
for agent in store.agents():
|
||||
store.delete_agent(agent.id)
|
||||
assert score.seed_agents(config, store) == []
|
||||
assert store.agents() == []
|
||||
|
||||
|
||||
def test_the_shipped_starter_agents_validate() -> None:
|
||||
from roledesk import config as config_mod
|
||||
|
||||
names = [a.name for a in config_mod.load(ROOT / "roles.yaml").agents]
|
||||
assert names == ["Berlin backend", "Remote EU stretch"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- cascade
|
||||
|
||||
def test_the_pull_cascade_runs_extract_embed_then_score(
|
||||
store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
||||
board="greenhouse")
|
||||
agent = store.save_agent("Backend", "python backend")
|
||||
|
||||
def fake_fetch(self: ATSSource) -> Iterator[Item]:
|
||||
item = Item(source_id="greenhouse:acme:1", location="https://x.example/1",
|
||||
title="Backend Engineer", body="Python",
|
||||
published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
||||
raw={"company": "Acme", "board": "greenhouse", "slug": "acme"})
|
||||
yield item
|
||||
self.state["ats:ok:greenhouse:acme"] = self.state.get("ats:run", "")
|
||||
self.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
|
||||
|
||||
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
||||
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config, store,
|
||||
workers={"cpu": 1, "http": 1, "llm": 1})
|
||||
|
||||
task_id = queue.start_task(TASKS["pull"]())
|
||||
queue.drain(timeout=60)
|
||||
|
||||
cascade_id = queue.jobs.of_task(task_id)[0].cascade_id
|
||||
jobs = sorted(queue.jobs.of_cascade(cascade_id), key=lambda j: j.id)
|
||||
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
|
||||
assert stages == list(cascade.CASCADES["pull"])
|
||||
work = [j.kind for j in jobs if j.kind != FINAL]
|
||||
assert work == ["pull_batch", "embed", "score"], "extract is off, so it finds nothing"
|
||||
assert all(j.state == "done" for j in jobs)
|
||||
assert len(store.scores(agent.id)) == 1
|
||||
|
||||
|
||||
def test_follows_walks_the_cascade_and_stops_at_its_end() -> None:
|
||||
assert cascade.follows("pull", "pull") == "extract"
|
||||
assert cascade.follows("pull", "extract") == "embed"
|
||||
assert cascade.follows("pull", "embed") == "score"
|
||||
assert cascade.follows("pull", "score") == ""
|
||||
assert cascade.follows("pull", "nope") == ""
|
||||
@@ -87,11 +87,15 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None:
|
||||
with sqlite3.connect(old) as conn:
|
||||
for column in ("permit", "languages", "extract_tries"):
|
||||
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
|
||||
for column in ("intent_key", "intent_vec"):
|
||||
conn.execute(f"ALTER TABLE agents DROP COLUMN {column}")
|
||||
conn.execute("DROP TABLE vectors")
|
||||
conn.execute("PRAGMA user_version = 1")
|
||||
|
||||
assert Store(old).migrations_applied == list(range(2, LATEST + 1))
|
||||
# ALTER appends, so the order differs; every read is by column name.
|
||||
assert sorted(_columns(old, "roles")) == sorted(_columns(fresh, "roles"))
|
||||
for table in ("roles", "agents", "vectors"):
|
||||
assert sorted(_columns(old, table)) == sorted(_columns(fresh, table))
|
||||
|
||||
|
||||
def test_state_is_a_mapping(store: Store) -> None:
|
||||
|
||||
@@ -203,3 +203,49 @@ def test_index_renders_the_shell(client: TestClient) -> None:
|
||||
assert "roledesk" in res.text
|
||||
assert 'id="roles-list"' in res.text
|
||||
assert "<desk-jobs>" in res.text
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ agents
|
||||
|
||||
def test_agents_are_created_updated_listed_and_deleted(client: TestClient) -> None:
|
||||
body = {"name": "Berlin", "intent": "python backend",
|
||||
"gates": [{"field": "place", "values": ["Berlin"], "mode": "require"}]}
|
||||
created = client.post("/api/agents", json=body)
|
||||
assert created.status_code == 200
|
||||
agent = created.json()
|
||||
assert agent["weights"] == {"fit": 0.6, "recency": 0.2, "affinity": 0.2}
|
||||
assert agent["gates"][0]["weight"] == 1.0
|
||||
|
||||
updated = client.post("/api/agents", json={**body, "id": agent["id"], "enabled": False,
|
||||
"weights": {"fit": 1, "recency": 0,
|
||||
"affinity": 0}})
|
||||
assert updated.json()["enabled"] is False and updated.json()["weights"]["fit"] == 1
|
||||
|
||||
assert [a["name"] for a in client.get("/api/agents").json()] == ["Berlin"]
|
||||
assert client.post("/api/agents", json=body).status_code == 409
|
||||
assert client.delete(f"/api/agents/{agent['id']}").status_code == 204
|
||||
assert client.get("/api/agents").json() == []
|
||||
assert client.delete(f"/api/agents/{agent['id']}").status_code == 404
|
||||
|
||||
|
||||
def test_a_gate_on_an_unknown_field_is_refused(client: TestClient) -> None:
|
||||
body = {"name": "Bad", "gates": [{"field": "salary", "values": [1]}]}
|
||||
assert client.post("/api/agents", json=body).status_code == 422
|
||||
|
||||
|
||||
def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated(
|
||||
client: TestClient, store: Store) -> None:
|
||||
for sid in ("low", "high", "gated"):
|
||||
store.upsert_role(posting(sid))
|
||||
uids = {r.source_id: r.uid for r in store.roles()}
|
||||
agent = store.save_agent("A")
|
||||
store.replace_scores(agent.id, [(uids["low"], 0.2, ""), (uids["high"], 0.8, ""),
|
||||
(uids["gated"], 0.0, "place unknown")])
|
||||
|
||||
shown = client.get(f"/api/roles?agent={agent.id}").json()
|
||||
assert [r["source_id"] for r in shown] == ["high", "low"]
|
||||
assert shown[0]["score"] == 0.8
|
||||
everything = client.get(f"/api/roles?agent={agent.id}&gated=1").json()
|
||||
assert everything[-1]["gated_reason"] == "place unknown"
|
||||
assert client.get("/api/roles").json()[0]["score"] is None
|
||||
assert client.get("/api/roles?agent=999").status_code == 404
|
||||
Reference in new issue
Block a user