diff --git a/config.example.yaml b/config.example.yaml index b190caf..1d96271 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -26,5 +26,23 @@ extract: enabled: false model: claude-haiku-4-5-20251001 +# A role and each agent's intent as vectors, for the fit term of a score +# (roledesk/jobs/embed.py). Needs sentence-transformers; without it `embed` +# posts a notice and scores leave fit out. Changing the model re-embeds. +embed: + model: sentence-transformers/all-MiniLM-L6-v2 + per_run: 50000 + +# Starter search agents, imported once into an empty `agents` table. Each gate +# is {field, values, mode: require|prefer, weight}; fields are place, country, +# remote, seniority, function, salary_min, currency, permit, languages and +# company. weights default to {fit: 0.6, recency: 0.2, affinity: 0.2}. +agents: + - name: Berlin backend + intent: Senior backend engineer, Python services and PostgreSQL. + gates: + - {field: place, values: [Berlin], mode: require} + - {field: remote, values: [hybrid, remote], mode: prefer, weight: 0.2} + # One block per plugin. Empty until the ats plugin lands. sources: {} diff --git a/docs/decisions.md b/docs/decisions.md index e03c27a..dd0270a 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -63,6 +63,20 @@ at least the listing. **Summary and tagging are off; `extract` replaces them.** Every profile declares `SummaryPolicy(enabled=False)` and `TaggingPolicy(enabled=False)`. `extract` is a roledesk `ModelTask` on Haiku that turns title and description into the gate columns, and fills only the columns a plugin left empty. +**A score is the weighted sum over the sum of the weights.** The agent's `fit`, `recency` and `affinity` weights and every prefer gate's `weight` form one denominator, so a score stays in 0..1 and the 0.9 floor for a favourite means the same thing whatever the weights are. A prefer gate normalised by the prefer weights alone, then weighted as one term, comes to the same number. + +**Gates match loosely where the column is free text.** `place` is a case-insensitive substring, like the place filter, so `Berlin` matches "Berlin, Germany". `company` compares company keys. `languages` passes when every language the role requires is one of the values. `salary_min` is at least the smallest value. The rest are case-insensitive equality. The gates are evaluated in SQL as one `CASE` column each, so a `require` gate is the `WHERE` it reads as while the gated row and its reason survive. The reason lists every failed require gate, not just the first. + +**A role with no vector is scored without `fit`.** The fit weight leaves the denominator, so a role `embed` has not reached yet ranks on recency, affinity and prefer gates instead of sinking to the bottom. It is gated as "no vector" only when fit is the only weighted term. An agent whose intent could not be embedded (empty, or no sentence-transformers) says "no intent vector" instead. + +**Each run replaces an agent's scores.** `replace_scores` deletes and inserts in one transaction, so a role closed or muted since the last run drops out rather than keeping a stale score, and a reader sees one list or the other. + +**Starter agents are imported once, by a state marker.** newsdesk seeds whenever its table is empty. Here `agents:seeded` in `state` is set after the import, so deleting every agent survives a restart. + +**`embed` and `score` run in the server process.** newsdesk isolates `embed` in a child process with a timeout. roledesk's run is one bounded pass over open roles, and in-process keeps the embedder replaceable in tests. The cost is torch's memory staying in the server after the first run; isolate it if that becomes the problem. + +**The pull cascade is `pull → extract → embed → score`.** Every stage is always in the chain and a stage that is off finds nothing, as in newsdesk. Extract precedes score so the gates read what it filled, and reads `on_ingest` only as a stage of a pull, so with `auto` and no model reachable a pull passes it quietly. + ## The alpha page **The template renders an empty shell; `static/app.js` fills it and drives every action.** `index.html` carries only the filter bar's ` + + @@ -43,6 +47,9 @@ +

@@ -59,6 +66,33 @@ + +

Jobs

@@ -76,6 +110,8 @@ + + diff --git a/roles.yaml b/roles.yaml index c62a74a..3505af9 100644 --- a/roles.yaml +++ b/roles.yaml @@ -10,3 +10,24 @@ extract: enabled: false model: claude-haiku-4-5-20251001 sources: {} +embed: + model: sentence-transformers/all-MiniLM-L6-v2 +# Starter search agents, imported once into an empty table. Edit them on the +# page afterwards; this list is not read again. +agents: + - name: Berlin backend + intent: >- + Senior backend engineer. Python and Go services, PostgreSQL, event-driven + systems, APIs and data pipelines on Kubernetes in the cloud. Wants + ownership of production systems in a product company. + gates: + - {field: place, values: [Berlin], mode: require} + - {field: remote, values: [hybrid, remote], mode: prefer, weight: 0.2} + - name: Remote EU stretch + intent: >- + Staff or lead engineer role, fully remote from Europe. Platform and + infrastructure work, distributed systems, developer tooling, technical + leadership across teams. + gates: + - {field: remote, values: [remote], mode: require} + - {field: country, values: [DE, NL, AT, CH], mode: prefer, weight: 0.2} diff --git a/tests/test_score.py b/tests/test_score.py new file mode 100644 index 0000000..5f645f2 --- /dev/null +++ b/tests/test_score.py @@ -0,0 +1,313 @@ +"""`embed`, `score` and the pull cascade, with the embedding backend replaced +by a deterministic fake. Nothing here downloads a model or reaches a board.""" + +from __future__ import annotations + +import sys +from collections.abc import Iterator, Sequence +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import numpy as np +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from desk.models import Item # noqa: E402 +from pipeline import FINAL, TASKS, JobQueue, Jobs, RunContext # noqa: E402 +from plugins.ats import ATSSource # noqa: E402 +from roledesk.config import AgentSeed, Config # noqa: E402 +from roledesk.jobs import cascade, embed, score # noqa: E402 +from roledesk.models import Candidate, Gate, Weights # noqa: E402 +from roledesk.storage import Store # noqa: E402 + +NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC) + +#: The fake's whole vocabulary: one dimension per word, plus a constant one so +#: no text is the zero vector. +WORDS = ("backend", "python", "frontend", "react", "data") + + +class FakeEmbedder: + model_id = "fake" + + def __init__(self) -> None: + self.calls: list[list[str]] = [] + + def encode(self, texts: Sequence[str]) -> np.ndarray: + self.calls.append(list(texts)) + rows = [] + for text in texts: + lower = text.lower() + vec = np.asarray([lower.count(w) for w in WORDS] + [0.1], dtype=np.float32) + rows.append(vec / np.linalg.norm(vec)) + return np.vstack(rows) + + +class Clock: + def __init__(self) -> None: + self.at = NOW + + def __call__(self) -> datetime: + return self.at + + +@pytest.fixture +def fake(monkeypatch: pytest.MonkeyPatch) -> FakeEmbedder: + embedder = FakeEmbedder() + monkeypatch.setattr(embed, "load_embedder", lambda model: embedder) + return embedder + + +@pytest.fixture +def clock() -> Clock: + return Clock() + + +@pytest.fixture +def store(tmp_path: Path, clock: Clock) -> Iterator[Store]: + s = Store(tmp_path / "roledesk.db") + s.clock = clock + yield s + + +@pytest.fixture +def config(tmp_path: Path) -> Config: + return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"}) + + +def role(store: Store, source_id: str, title: str = "Backend Engineer", *, + company: str = "Acme", body: str = "", **raw: Any) -> str: + item = Item(source_id=source_id, location=f"https://boards.example/{source_id}", + title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC), + raw={"company": company, **raw}) + item.source = "ats" + store.upsert_role(item) + return item.uid + + +def run(config: Config, store: Store, job: Any) -> dict[str, Any]: + notices: list[Any] = [] + result = dict(job(config, store, context=RunContext(on_notice=notices.append))) + result["notices"] = [n.code for n in notices] + return result + + +# ------------------------------------------------------------------ embed + +def test_embed_writes_a_vector_per_open_role_once( + store: Store, config: Config, fake: FakeEmbedder) -> None: + role(store, "1") + closed = role(store, "2") + with store.transaction() as conn: + conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (NOW.isoformat(), closed)) + + assert run(config, store, embed.run_embed)["embedded"] == 1 + assert run(config, store, embed.run_embed)["embedded"] == 0 + assert store.vector_counts() == {"fake": 1} + + +def test_embed_without_sentence_transformers_says_so_and_does_nothing( + store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: + def missing(model: str) -> Any: + raise ImportError("no sentence_transformers") + + monkeypatch.setattr(embed, "load_embedder", missing) + role(store, "1") + + result = run(config, store, embed.run_embed) + + assert result["embedded"] == 0 and result["notices"] == ["embed.unavailable"] + assert store.vector_counts() == {} + + +# ------------------------------------------------------------------ terms + +def test_recency_halves_every_fourteen_days() -> None: + assert score.recency(NOW.isoformat(), NOW) == 1.0 + assert score.recency((NOW - timedelta(days=14)).isoformat(), NOW) == pytest.approx(0.5) + assert score.recency((NOW - timedelta(days=28)).isoformat(), NOW) == pytest.approx(0.25) + + +def test_a_role_without_a_vector_is_scored_without_fit_unless_fit_is_all() -> None: + candidate = Candidate(uid="u", first_seen=NOW.isoformat(), affinity=0) + value, reason = score.score_one(candidate, [], Weights(), None, NOW) + assert reason == "" and value == pytest.approx((0.2 * 1.0 + 0.2 / 3) / 0.4) + + only_fit = Weights(fit=1, recency=0, affinity=0) + assert score.score_one(candidate, [], only_fit, None, NOW) == (0.0, "no vector") + + +# ------------------------------------------------------------------ score + +def test_a_require_gate_keeps_the_row_with_a_reason( + store: Store, config: Config, fake: FakeEmbedder) -> None: + berlin = role(store, "1", place="Berlin, Germany") + munich = role(store, "2", place="Munich") + nowhere = role(store, "3") + agent = store.save_agent("Berlin", "python backend", + [Gate(field="place", values=["Berlin"])]) + + run(config, store, score.run_score) + + scores = store.scores(agent.id) + assert scores[berlin][1] == "" and scores[berlin][0] > 0 + assert scores[munich] == (0.0, "place not Berlin") + assert scores[nowhere] == (0.0, "place unknown") + shown = store.roles(agent=agent.id) + assert [r.uid for r in shown] == [berlin] + assert {r.uid for r in store.roles(agent=agent.id, gated=True)} == {berlin, munich, nowhere} + + +def test_a_prefer_gate_adds_its_weight_to_the_score( + store: Store, config: Config, fake: FakeEmbedder) -> None: + remote = role(store, "1", remote="remote") + office = role(store, "2", remote="onsite") + agent = store.save_agent("Remote", "", [Gate(field="remote", values=["remote"], + mode="prefer", weight=0.5)]) + + run(config, store, score.run_score) + + scores = store.scores(agent.id) + # No intent, so fit is out: recency 0.2 + affinity 0.2 + the gate's 0.5. + assert scores[remote][0] - scores[office][0] == pytest.approx(0.5 / 0.9, abs=1e-5) + assert scores[office][1] == "" + + +def test_a_favourite_skips_the_gates_and_is_floored( + store: Store, config: Config, fake: FakeEmbedder) -> None: + store.add_company("Fave", affinity=2) + fave = role(store, "1", company="Fave", place="Munich") + agent = store.save_agent("Berlin", "", [Gate(field="place", values=["Berlin"])]) + + run(config, store, score.run_score) + + value, reason = store.scores(agent.id)[fave] + assert reason == "" and value >= 0.9 + + +def test_a_muted_company_is_never_scored( + store: Store, config: Config, fake: FakeEmbedder) -> None: + store.add_company("Muted", affinity=-1) + muted = role(store, "1", company="Muted") + kept = role(store, "2") + agent = store.save_agent("Any") + + run(config, store, score.run_score) + + assert set(store.scores(agent.id)) == {kept} + assert muted not in {r.uid for r in store.roles(agent=agent.id, gated=True)} + + +def test_an_older_role_scores_lower( + store: Store, config: Config, clock: Clock, fake: FakeEmbedder) -> None: + clock.at = NOW - timedelta(days=14) + old = role(store, "1") + clock.at = NOW + new = role(store, "2") + agent = store.save_agent("Any") + + run(config, store, score.run_score) + + scores = store.scores(agent.id) + assert scores[new][0] > scores[old][0] + + +def test_two_agents_score_the_same_role_differently( + store: Store, config: Config, fake: FakeEmbedder) -> None: + uid = role(store, "1", "Backend Engineer", body="Python services") + backend = store.save_agent("Backend", "python backend developer", + weights=Weights(fit=1, recency=0, affinity=0)) + frontend = store.save_agent("Frontend", "react frontend developer", + weights=Weights(fit=1, recency=0, affinity=0)) + + run(config, store, embed.run_embed) + result = run(config, store, score.run_score) + + assert result["agents"] == 2 + assert store.scores(backend.id)[uid][0] > store.scores(frontend.id)[uid][0] + + +def test_the_intent_vector_is_cached_until_the_intent_changes( + store: Store, config: Config, fake: FakeEmbedder) -> None: + role(store, "1") + agent = store.save_agent("Backend", "python backend") + + run(config, store, score.run_score) + run(config, store, score.run_score) + assert fake.calls == [["python backend"]] + + store.save_agent("Backend", "react frontend", agent_id=agent.id) + run(config, store, score.run_score) + assert fake.calls[-1] == ["react frontend"] + + +def test_a_disabled_agent_is_not_scored( + store: Store, config: Config, fake: FakeEmbedder) -> None: + role(store, "1") + agent = store.save_agent("Off", enabled=False) + assert run(config, store, score.run_score)["agents"] == 0 + assert store.scores(agent.id) == {} + + +def test_starter_agents_are_imported_once(store: Store, tmp_path: Path) -> None: + config = Config(database=str(tmp_path / "roledesk.db"), + agents=[AgentSeed(name="A"), AgentSeed(name="B")]) + assert score.seed_agents(config, store) == ["A", "B"] + for agent in store.agents(): + store.delete_agent(agent.id) + assert score.seed_agents(config, store) == [] + assert store.agents() == [] + + +def test_the_shipped_starter_agents_validate() -> None: + from roledesk import config as config_mod + + names = [a.name for a in config_mod.load(ROOT / "roles.yaml").agents] + assert names == ["Berlin backend", "Remote EU stretch"] + + +# ---------------------------------------------------------------- cascade + +def test_the_pull_cascade_runs_extract_embed_then_score( + store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path, + monkeypatch: pytest.MonkeyPatch) -> None: + store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme", + board="greenhouse") + agent = store.save_agent("Backend", "python backend") + + def fake_fetch(self: ATSSource) -> Iterator[Item]: + item = Item(source_id="greenhouse:acme:1", location="https://x.example/1", + title="Backend Engineer", body="Python", + published_at=datetime(2026, 9, 20, tzinfo=UTC), + raw={"company": "Acme", "board": "greenhouse", "slug": "acme"}) + yield item + self.state["ats:ok:greenhouse:acme"] = self.state.get("ats:run", "") + self.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]' + + monkeypatch.setattr(ATSSource, "fetch", fake_fetch) + queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config, store, + workers={"cpu": 1, "http": 1, "llm": 1}) + + task_id = queue.start_task(TASKS["pull"]()) + queue.drain(timeout=60) + + cascade_id = queue.jobs.of_task(task_id)[0].cascade_id + jobs = sorted(queue.jobs.of_cascade(cascade_id), key=lambda j: j.id) + stages = [j.result.get("task") for j in jobs if j.kind == FINAL] + assert stages == list(cascade.CASCADES["pull"]) + work = [j.kind for j in jobs if j.kind != FINAL] + assert work == ["pull_batch", "embed", "score"], "extract is off, so it finds nothing" + assert all(j.state == "done" for j in jobs) + assert len(store.scores(agent.id)) == 1 + + +def test_follows_walks_the_cascade_and_stops_at_its_end() -> None: + assert cascade.follows("pull", "pull") == "extract" + assert cascade.follows("pull", "extract") == "embed" + assert cascade.follows("pull", "embed") == "score" + assert cascade.follows("pull", "score") == "" + assert cascade.follows("pull", "nope") == "" diff --git a/tests/test_storage.py b/tests/test_storage.py index 1703ca3..40d6593 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -87,11 +87,15 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None: with sqlite3.connect(old) as conn: for column in ("permit", "languages", "extract_tries"): conn.execute(f"ALTER TABLE roles DROP COLUMN {column}") + for column in ("intent_key", "intent_vec"): + conn.execute(f"ALTER TABLE agents DROP COLUMN {column}") + conn.execute("DROP TABLE vectors") conn.execute("PRAGMA user_version = 1") assert Store(old).migrations_applied == list(range(2, LATEST + 1)) # ALTER appends, so the order differs; every read is by column name. - assert sorted(_columns(old, "roles")) == sorted(_columns(fresh, "roles")) + for table in ("roles", "agents", "vectors"): + assert sorted(_columns(old, table)) == sorted(_columns(fresh, table)) def test_state_is_a_mapping(store: Store) -> None: diff --git a/tests/test_web.py b/tests/test_web.py index e6e3c9d..d2764b3 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -203,3 +203,49 @@ def test_index_renders_the_shell(client: TestClient) -> None: assert "roledesk" in res.text assert 'id="roles-list"' in res.text assert "" in res.text + + +# ------------------------------------------------------------------ agents + +def test_agents_are_created_updated_listed_and_deleted(client: TestClient) -> None: + body = {"name": "Berlin", "intent": "python backend", + "gates": [{"field": "place", "values": ["Berlin"], "mode": "require"}]} + created = client.post("/api/agents", json=body) + assert created.status_code == 200 + agent = created.json() + assert agent["weights"] == {"fit": 0.6, "recency": 0.2, "affinity": 0.2} + assert agent["gates"][0]["weight"] == 1.0 + + updated = client.post("/api/agents", json={**body, "id": agent["id"], "enabled": False, + "weights": {"fit": 1, "recency": 0, + "affinity": 0}}) + assert updated.json()["enabled"] is False and updated.json()["weights"]["fit"] == 1 + + assert [a["name"] for a in client.get("/api/agents").json()] == ["Berlin"] + assert client.post("/api/agents", json=body).status_code == 409 + assert client.delete(f"/api/agents/{agent['id']}").status_code == 204 + assert client.get("/api/agents").json() == [] + assert client.delete(f"/api/agents/{agent['id']}").status_code == 404 + + +def test_a_gate_on_an_unknown_field_is_refused(client: TestClient) -> None: + body = {"name": "Bad", "gates": [{"field": "salary", "values": [1]}]} + assert client.post("/api/agents", json=body).status_code == 422 + + +def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated( + client: TestClient, store: Store) -> None: + for sid in ("low", "high", "gated"): + store.upsert_role(posting(sid)) + uids = {r.source_id: r.uid for r in store.roles()} + agent = store.save_agent("A") + store.replace_scores(agent.id, [(uids["low"], 0.2, ""), (uids["high"], 0.8, ""), + (uids["gated"], 0.0, "place unknown")]) + + shown = client.get(f"/api/roles?agent={agent.id}").json() + assert [r["source_id"] for r in shown] == ["high", "low"] + assert shown[0]["score"] == 0.8 + everything = client.get(f"/api/roles?agent={agent.id}&gated=1").json() + assert everything[-1]["gated_reason"] == "place unknown" + assert client.get("/api/roles").json()[0]["score"] is None + assert client.get("/api/roles?agent=999").status_code == 404