The ats boards only read companies someone listed, and the seed list is US-centred: 31 of 29,000 open roles were in Denmark. A `search` plugin runs saved queries on boards that span companies, pulled in the same task as the boards: Jobindex (Denmark, RSS, no key) and Adzuna (UK and most of the EU, free key, built to its documented API). Jobnet and Jooble were checked and left out. `country` is now filled from the board's location line by a lookup, with EU for Europe-wide roles, so country gates work without extract. `app companies prune --in` unsubscribes boards with no open role in a region. The starter agents and searches are Copenhagen and Europe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
404 lines
16 KiB
Python
404 lines
16 KiB
Python
"""`embed`, `score` and the pull cascade, with the embedding backend replaced
|
|
by a deterministic fake. Nothing here downloads a model or reaches a board."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from collections.abc import Iterator, Sequence
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from desk.models import Item # noqa: E402
|
|
from pipeline import FINAL, SCHEDULES, TASKS, JobQueue, Jobs, RunContext # noqa: E402
|
|
from plugins.ats import ATSSource # noqa: E402
|
|
from app.config import AgentSeed, Config # noqa: E402
|
|
from app.jobs import cascade, embed, score # noqa: E402
|
|
from app.models import Candidate, Gate, Weights # noqa: E402
|
|
from app.storage import Store # noqa: E402
|
|
|
|
NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
|
|
|
|
#: The fake's whole vocabulary: one dimension per word, plus a constant one so
|
|
#: no text is the zero vector.
|
|
WORDS = ("backend", "python", "frontend", "react", "data")
|
|
|
|
|
|
class FakeEmbedder:
|
|
model_id = "fake"
|
|
|
|
def __init__(self) -> None:
|
|
self.calls: list[list[str]] = []
|
|
|
|
def encode(self, texts: Sequence[str]) -> np.ndarray:
|
|
self.calls.append(list(texts))
|
|
rows = []
|
|
for text in texts:
|
|
lower = text.lower()
|
|
vec = np.asarray([lower.count(w) for w in WORDS] + [0.1], dtype=np.float32)
|
|
rows.append(vec / np.linalg.norm(vec))
|
|
return np.vstack(rows)
|
|
|
|
|
|
class Clock:
|
|
def __init__(self) -> None:
|
|
self.at = NOW
|
|
|
|
def __call__(self) -> datetime:
|
|
return self.at
|
|
|
|
|
|
@pytest.fixture
|
|
def fake(monkeypatch: pytest.MonkeyPatch) -> FakeEmbedder:
|
|
embedder = FakeEmbedder()
|
|
monkeypatch.setattr(embed, "load_embedder", lambda model: embedder)
|
|
return embedder
|
|
|
|
|
|
@pytest.fixture
|
|
def clock() -> Clock:
|
|
return Clock()
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path, clock: Clock) -> Iterator[Store]:
|
|
s = Store(tmp_path / "roledesk.db")
|
|
s.clock = clock
|
|
yield s
|
|
|
|
|
|
@pytest.fixture
|
|
def config(tmp_path: Path) -> Config:
|
|
return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"})
|
|
|
|
|
|
def role(store: Store, source_id: str, title: str = "Backend Engineer", *,
|
|
company: str = "Acme", body: str = "", **raw: Any) -> str:
|
|
item = Item(source_id=source_id, location=f"https://boards.example/{source_id}",
|
|
title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
|
raw={"company": company, **raw})
|
|
item.source = "ats"
|
|
store.upsert_role(item)
|
|
return item.uid
|
|
|
|
|
|
def run(config: Config, store: Store, job: Any) -> dict[str, Any]:
|
|
notices: list[Any] = []
|
|
result = dict(job(config, store, context=RunContext(on_notice=notices.append)))
|
|
result["notices"] = [n.code for n in notices]
|
|
return result
|
|
|
|
|
|
# ------------------------------------------------------------------ embed
|
|
|
|
def test_embed_writes_a_vector_per_open_role_once(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
role(store, "1")
|
|
closed = role(store, "2")
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (NOW.isoformat(), closed))
|
|
|
|
assert run(config, store, embed.run_embed)["embedded"] == 1
|
|
assert run(config, store, embed.run_embed)["embedded"] == 0
|
|
assert store.vector_counts() == {"fake": 1}
|
|
|
|
|
|
def test_embed_without_sentence_transformers_says_so_and_does_nothing(
|
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
def missing(model: str) -> Any:
|
|
raise ImportError("no sentence_transformers")
|
|
|
|
monkeypatch.setattr(embed, "load_embedder", missing)
|
|
role(store, "1")
|
|
|
|
result = run(config, store, embed.run_embed)
|
|
|
|
assert result["embedded"] == 0 and result["notices"] == ["embed.unavailable"]
|
|
assert store.vector_counts() == {}
|
|
|
|
|
|
# ------------------------------------------------------------------ terms
|
|
|
|
def test_recency_halves_every_fourteen_days() -> None:
|
|
assert score.recency(NOW.isoformat(), NOW) == 1.0
|
|
assert score.recency((NOW - timedelta(days=14)).isoformat(), NOW) == pytest.approx(0.5)
|
|
assert score.recency((NOW - timedelta(days=28)).isoformat(), NOW) == pytest.approx(0.25)
|
|
|
|
|
|
def test_a_role_without_a_vector_is_scored_without_fit_unless_fit_is_all() -> None:
|
|
candidate = Candidate(uid="u", first_seen=NOW.isoformat(), affinity=0)
|
|
value, reason = score.score_one(candidate, [], Weights(), None, NOW)
|
|
assert reason == "" and value == pytest.approx((0.2 * 1.0 + 0.2 / 3) / 0.4)
|
|
|
|
only_fit = Weights(fit=1, recency=0, affinity=0)
|
|
assert score.score_one(candidate, [], only_fit, None, NOW) == (0.0, "no vector")
|
|
|
|
|
|
# ------------------------------------------------------------------ score
|
|
|
|
def test_a_require_gate_keeps_the_row_with_a_reason(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
berlin = role(store, "1", place="Berlin, Germany")
|
|
munich = role(store, "2", place="Munich")
|
|
nowhere = role(store, "3")
|
|
agent = store.save_agent("Berlin", "python backend",
|
|
[Gate(field="place", values=["Berlin"])])
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[berlin][1] == "" and scores[berlin][0] > 0
|
|
assert scores[munich] == (0.0, "place not Berlin")
|
|
assert scores[nowhere] == (0.0, "place unknown")
|
|
shown = store.roles(agent=agent.id)
|
|
assert [r.uid for r in shown] == [berlin]
|
|
assert {r.uid for r in store.roles(agent=agent.id, gated=True)} == {berlin, munich, nowhere}
|
|
|
|
|
|
def test_a_prefer_gate_adds_its_weight_to_the_score(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
remote = role(store, "1", remote="remote")
|
|
office = role(store, "2", remote="onsite")
|
|
agent = store.save_agent("Remote", "", [Gate(field="remote", values=["remote"],
|
|
mode="prefer", weight=0.5)])
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
# No intent, so fit is out: recency 0.2 + affinity 0.2 + the gate's 0.5.
|
|
assert scores[remote][0] - scores[office][0] == pytest.approx(0.5 / 0.9, abs=1e-5)
|
|
assert scores[office][1] == ""
|
|
|
|
|
|
def test_a_favourite_skips_the_gates_and_is_floored(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
store.add_company("Fave", affinity=2)
|
|
fave = role(store, "1", company="Fave", place="Munich")
|
|
agent = store.save_agent("Berlin", "", [Gate(field="place", values=["Berlin"])])
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
value, reason = store.scores(agent.id)[fave]
|
|
assert reason == "" and value >= 0.9
|
|
|
|
|
|
def test_a_muted_company_is_never_scored(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
store.add_company("Muted", affinity=-1)
|
|
muted = role(store, "1", company="Muted")
|
|
kept = role(store, "2")
|
|
agent = store.save_agent("Any")
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
assert set(store.scores(agent.id)) == {kept}
|
|
assert muted not in {r.uid for r in store.roles(agent=agent.id, gated=True)}
|
|
|
|
|
|
def test_an_older_role_scores_lower(
|
|
store: Store, config: Config, clock: Clock, fake: FakeEmbedder) -> None:
|
|
clock.at = NOW - timedelta(days=14)
|
|
old = role(store, "1")
|
|
clock.at = NOW
|
|
new = role(store, "2")
|
|
agent = store.save_agent("Any")
|
|
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[new][0] > scores[old][0]
|
|
|
|
|
|
def test_two_agents_rank_the_same_roles_the_other_way_round(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
back = role(store, "1", "Backend Engineer", body="Python services")
|
|
front = role(store, "2", "Frontend Engineer", body="React apps")
|
|
only_fit = Weights(fit=1, recency=0, affinity=0)
|
|
backend = store.save_agent("Backend", "python backend developer", weights=only_fit)
|
|
frontend = store.save_agent("Frontend", "react frontend developer", weights=only_fit)
|
|
|
|
run(config, store, embed.run_embed)
|
|
result = run(config, store, score.run_score)
|
|
|
|
assert result["agents"] == 2
|
|
# Fit alone, scaled per agent: each agent's closer role is 1, the other 0.
|
|
assert store.scores(backend.id)[back][0] == 1.0
|
|
assert store.scores(backend.id)[front][0] == 0.0
|
|
assert store.scores(frontend.id)[front][0] == 1.0
|
|
assert store.scores(frontend.id)[back][0] == 0.0
|
|
|
|
|
|
# ------------------------------------------------------------ fit scaling
|
|
|
|
def test_scale_maps_the_closest_to_one_and_the_farthest_to_zero() -> None:
|
|
# The spread the last real run showed under `(cos + 1) / 2`.
|
|
assert score.scale({"a": 0.55, "b": 0.79, "c": 0.67}) == pytest.approx(
|
|
{"a": 0.0, "b": 1.0, "c": 0.5})
|
|
assert score.scale({"only": 0.3}) == {"only": 1.0}
|
|
assert score.scale({"a": 0.4, "b": 0.4}) == {"a": 1.0, "b": 1.0}
|
|
assert score.scale({}) == {}
|
|
|
|
|
|
def test_a_single_candidate_gets_full_fit(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
uid = role(store, "1", "Frontend Engineer", body="React apps")
|
|
agent = store.save_agent("Backend", "python backend developer",
|
|
weights=Weights(fit=1, recency=0, affinity=0))
|
|
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
|
|
assert store.scores(agent.id)[uid] == (1.0, "")
|
|
|
|
|
|
def test_a_gated_role_does_not_stretch_the_fit_range(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
close = role(store, "1", "Backend Engineer", body="Python backend", place="Berlin")
|
|
far = role(store, "2", "Data Engineer", body="Data", place="Berlin")
|
|
# Farther from the intent than either, but gated out by place.
|
|
role(store, "3", "Frontend Engineer", body="React react", place="Munich")
|
|
agent = store.save_agent("Berlin", "python backend",
|
|
[Gate(field="place", values=["Berlin"])],
|
|
weights=Weights(fit=1, recency=0, affinity=0))
|
|
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[close][0] == 1.0 and scores[far][0] == 0.0
|
|
|
|
|
|
def test_a_small_prefer_gate_no_longer_outweighs_fit(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
"""The flaw the last real run showed: real cosines sit close together, so
|
|
under `(cos + 1) / 2` a prefer gate decided the order over fit. Here the
|
|
cosines are about 0.95 and 0.87: the old fit terms differ by 0.04, which
|
|
at weight 0.6 is less than the gate's 0.05, so the remote role won. Scaled,
|
|
the closer role wins."""
|
|
close = role(store, "1", "Backend Engineer", body="Python backend", remote="onsite")
|
|
far = role(store, "2", "Backend Engineer", body="Python backend data", remote="remote")
|
|
agent = store.save_agent("Backend", "python backend",
|
|
[Gate(field="remote", values=["remote"], mode="prefer",
|
|
weight=0.05)])
|
|
|
|
run(config, store, embed.run_embed)
|
|
run(config, store, score.run_score)
|
|
|
|
scores = store.scores(agent.id)
|
|
assert scores[close][0] > scores[far][0]
|
|
|
|
|
|
def test_the_intent_vector_is_cached_until_the_intent_changes(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
role(store, "1")
|
|
agent = store.save_agent("Backend", "python backend")
|
|
|
|
run(config, store, score.run_score)
|
|
run(config, store, score.run_score)
|
|
assert fake.calls == [["python backend"]]
|
|
|
|
store.save_agent("Backend", "react frontend", agent_id=agent.id)
|
|
run(config, store, score.run_score)
|
|
assert fake.calls[-1] == ["react frontend"]
|
|
|
|
|
|
def test_a_disabled_agent_is_not_scored(
|
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
|
role(store, "1")
|
|
agent = store.save_agent("Off", enabled=False)
|
|
assert run(config, store, score.run_score)["agents"] == 0
|
|
assert store.scores(agent.id) == {}
|
|
|
|
|
|
def test_starter_agents_are_imported_once(store: Store, tmp_path: Path) -> None:
|
|
config = Config(database=str(tmp_path / "roledesk.db"),
|
|
agents=[AgentSeed(name="A"), AgentSeed(name="B")])
|
|
assert score.seed_agents(config, store) == ["A", "B"]
|
|
for agent in store.agents():
|
|
store.delete_agent(agent.id)
|
|
assert score.seed_agents(config, store) == []
|
|
assert store.agents() == []
|
|
|
|
|
|
def test_the_shipped_starter_agents_validate() -> None:
|
|
from app import config as config_mod
|
|
|
|
agents = config_mod.load(ROOT / "config.yaml").agents
|
|
assert [a.name for a in agents] == ["Copenhagen engineering", "Remote from Europe",
|
|
"Europe hybrid"]
|
|
# Small enough that a prefer gate breaks ties rather than overrides fit.
|
|
assert {g.weight for a in agents for g in a.gates if g.mode == "prefer"} == {0.05}
|
|
|
|
|
|
def test_the_shipped_starter_searches_validate() -> None:
|
|
from app import config as config_mod
|
|
|
|
searches = config_mod.load(ROOT / "config.yaml").searches
|
|
assert searches and {s.provider for s in searches} == {"jobindex"}
|
|
assert all(s.params.get("q") for s in searches)
|
|
|
|
|
|
# ---------------------------------------------------------------- cascade
|
|
|
|
def test_the_pull_job_runs_pull_extract_embed_then_score(
|
|
store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
|
board="greenhouse")
|
|
agent = store.save_agent("Backend", "python backend")
|
|
|
|
def fake_fetch(self: ATSSource) -> Iterator[Item]:
|
|
item = Item(source_id="greenhouse:acme:1", location="https://x.example/1",
|
|
title="Backend Engineer", body="Python",
|
|
published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
|
raw={"company": "Acme", "board": "greenhouse", "slug": "acme"})
|
|
yield item
|
|
self.state["ats:ok:greenhouse:acme"] = self.state.get("ats:run", "")
|
|
self.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
|
|
|
|
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
|
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config, store,
|
|
workers={"cpu": 1, "http": 1, "llm": 1})
|
|
|
|
# The `pull` job, which is what the schedule fires: its own work is to ask
|
|
# for the chain, which then runs to its end in the same drain.
|
|
started = queue.enqueue("pull")
|
|
queue.drain(timeout=60)
|
|
|
|
assert started is not None and queue.jobs.get(started.id).state == "done" # type: ignore[union-attr]
|
|
batch = queue.jobs.last("pull_batch")
|
|
assert batch is not None
|
|
jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id)
|
|
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
|
|
assert stages == list(cascade.CASCADES["pull"])
|
|
work = [j.kind for j in jobs if j.kind != FINAL]
|
|
# The `pull` job and the chain it asked for are one cascade, so the Jobs
|
|
# tab shows a scheduled pull as one run. Extract is off and finds nothing.
|
|
assert work == ["pull", "pull_batch", "embed", "score"]
|
|
assert all(j.state == "done" for j in jobs)
|
|
assert len(store.scores(agent.id)) == 1
|
|
|
|
|
|
def test_pull_is_a_job_on_a_six_hour_schedule_and_the_stages_are_tasks() -> None:
|
|
"""A schedule enqueues a job kind, so `pull` is a job; were it a task too,
|
|
the Jobs tab would draw two `pull` buttons."""
|
|
assert "pull" not in TASKS
|
|
assert {"extract", "embed", "score"} <= set(TASKS)
|
|
assert SCHEDULES["pull"].kind == "pull"
|
|
assert SCHEDULES["pull"].cadence == "every 6h"
|
|
|
|
|
|
def test_follows_walks_the_cascade_and_stops_at_its_end() -> None:
|
|
assert cascade.follows("pull", "pull") == "extract"
|
|
assert cascade.follows("pull", "extract") == "embed"
|
|
assert cascade.follows("pull", "embed") == "score"
|
|
assert cascade.follows("pull", "score") == ""
|
|
assert cascade.follows("pull", "nope") == ""
|