The pull reads listings only. `enrich` (app/jobs/enrich.py) takes the open roles without text that an enabled agent ranks ungated in its best `enrich.top` at `enrich.min_score` or above, best score first, on boards whose reader has `detail`, within `enrich.per_board` and `enrich.per_run`, one batch per board. `plugins.ats.read_board_details` reads them and keeps the per-board state; a read that gave no text is read again after `enrich.retry_days` (finding 18). Filled roles are upserted, which drops their vectors, and embed and score follow; `cascade.step` passes over both when nothing was filled. The pull cascade is pull, embed, score, enrich, embed, score, extract, score, summarise. Saving an agent and changing a company's affinity queue the `enrich` job (enrich, embed, score) behind their rescore; a vote does not. The stage has a switch, `stage:enrich`, over `enrich.enabled`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
369 lines
16 KiB
Python
369 lines
16 KiB
Python
"""The `enrich` stage: the best roles without text, read on their own pages
|
|
after scoring. Which roles and in what order, the budgets, what is stored,
|
|
the cascade around it, and what triggers it. No network: a fake host stands
|
|
in, answering from a dict."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
import types
|
|
from collections.abc import Iterator
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import numpy as np
|
|
import pytest
|
|
from fastapi import FastAPI
|
|
from fastapi.testclient import TestClient
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from app.config import Config, ServerConfig # noqa: E402
|
|
from app.jobs import cascade, embed, enrich, stages # noqa: E402
|
|
from app.main import create_app # noqa: E402
|
|
from app.storage import Store # noqa: E402
|
|
from desk.models import Item # noqa: E402
|
|
from pipeline import FINAL, JobQueue, Jobs, Tally # noqa: E402
|
|
from plugins import ats, readers # noqa: E402
|
|
|
|
BASE = "https://jobs.fake.example"
|
|
|
|
|
|
def _fake_host(name: str = "fake", *, detail: bool = True) -> types.ModuleType:
|
|
"""A host whose role pages are `BASE/<slug>/<id>`, answered by `web`."""
|
|
module: Any = types.ModuleType(name)
|
|
module.NAME, module.LABEL, module.PATTERNS = name, name.title(), ()
|
|
module.slug_ok = lambda slug: slug.isalnum()
|
|
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
|
module.careers_url = lambda slug: f"{BASE}/{slug}"
|
|
module.feed_url = lambda slug: f"{BASE}/{slug}/list"
|
|
module.parse = lambda body, slug: []
|
|
if detail:
|
|
module.detail = lambda get, slug, row: {
|
|
"description": get(f"{BASE}/{slug}/{row['source_id']}").decode(),
|
|
"location": "Copenhagen"}
|
|
return module
|
|
|
|
|
|
@pytest.fixture
|
|
def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
|
|
"""`ats._get` answering from `answers` (URL -> body), 404 otherwise,
|
|
remembering what it was asked; hosts `fake` and `plain` (no detail)."""
|
|
answers: dict[str, Any] = {}
|
|
asked: list[str] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None):
|
|
asked.append(url)
|
|
answer = answers.get(url)
|
|
if answer is None:
|
|
request = httpx.Request("GET", url)
|
|
raise httpx.HTTPStatusError("404", request=request,
|
|
response=httpx.Response(404, request=request))
|
|
return answer
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
|
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host())
|
|
monkeypatch.setitem(readers.HOSTS, "plain", _fake_host("plain", detail=False))
|
|
return {"answers": answers, "asked": asked}
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path) -> Store:
|
|
return Store(tmp_path / "roledesk.db")
|
|
|
|
|
|
def config(tmp_path: Path, **enrich_settings: Any) -> Config:
|
|
return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"},
|
|
enrich=enrich_settings)
|
|
|
|
|
|
def board(store: Store, target: str) -> Any:
|
|
host = target.partition(":")[0]
|
|
return store.add_subscription("ats", target, label=target, company=target, board=host)
|
|
|
|
|
|
def role(store: Store, target: str, native: str, *, body: str = "",
|
|
title: str = "Backend Engineer") -> str:
|
|
"""A role of board `target` as a pull stores it: without text on a host
|
|
with `detail`."""
|
|
sub = next(s for s in store.subscriptions(source="ats") if s.target == target)
|
|
host, _, slug = target.partition(":")
|
|
item = Item(source_id=f"{target}:{native}", location=f"{BASE}/{slug}/{native}",
|
|
title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
|
raw={"company": target, "place": "2 Locations", "board": host, "slug": slug})
|
|
item.source = "ats"
|
|
store.upsert_role(item, sub)
|
|
return item.uid
|
|
|
|
|
|
def scored(store: Store, name: str, scores: dict[str, float], *,
|
|
gated: tuple[str, ...] = ()) -> Any:
|
|
agent = store.save_agent(name, f"{name} python backend")
|
|
store.replace_scores(agent.id, [(uid, value, "gated" if uid in gated else "")
|
|
for uid, value in scores.items()])
|
|
return agent
|
|
|
|
|
|
# -------------------------------------------------------------- which roles
|
|
|
|
def test_roles_go_best_first_by_their_best_agent_score(
|
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
|
board(store, "fake:acme")
|
|
board(store, "fake:beta")
|
|
a1, a2 = role(store, "fake:acme", "1"), role(store, "fake:acme", "2")
|
|
b1 = role(store, "fake:beta", "1")
|
|
scored(store, "One", {a1: 0.6, a2: 0.9, b1: 0.55})
|
|
scored(store, "Two", {a1: 0.95, b1: 0.7})
|
|
|
|
assert [(r["uid"], r["score"]) for r in store.enrich_waiting(500, 0.5)] == [
|
|
(a1, 0.95), (a2, 0.9), (b1, 0.7)]
|
|
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [a1, a2]),
|
|
("fake:beta", [b1])]
|
|
|
|
|
|
def test_a_role_no_agent_ranks_high_enough_gets_no_call(
|
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
|
board(store, "fake:acme")
|
|
low, gated, deep, kept = (role(store, "fake:acme", n) for n in ("1", "2", "3", "4"))
|
|
scored(store, "One", {low: 0.4, gated: 0.0, deep: 0.8, kept: 0.9}, gated=(gated,))
|
|
# `top` 1 per agent: only the best ungated role of each agent is in reach.
|
|
assert enrich.plan(config(tmp_path, top=1), store) == [("fake:acme", [kept])]
|
|
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [kept, deep])]
|
|
# A disabled agent ranks nothing.
|
|
agent = store.agents()[0]
|
|
store.save_agent(agent.name, gates=agent.gates, weights=agent.weights, enabled=False,
|
|
agent_id=agent.id, limits=agent.limits, prompts=agent.prompts)
|
|
assert enrich.plan(config(tmp_path), store) == []
|
|
|
|
|
|
def test_roles_with_text_hosts_without_pages_and_failing_boards_are_left_out(
|
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
|
for target in ("fake:acme", "plain:acme", "fake:down"):
|
|
board(store, target)
|
|
wanted = role(store, "fake:acme", "1")
|
|
has_text = role(store, "fake:acme", "2", body="Python in full")
|
|
plain = role(store, "plain:acme", "1")
|
|
down = role(store, "fake:down", "1")
|
|
store.state["ats:failing:fake:down"] = json.dumps({"since": "2026-09-29", "reason": "500"})
|
|
scored(store, "One", {wanted: 0.9, has_text: 0.95, plain: 0.9, down: 0.9})
|
|
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [wanted])]
|
|
|
|
|
|
def test_a_role_read_already_waits_for_the_retry_and_a_failed_one_goes_last(
|
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
|
board(store, "fake:acme")
|
|
read, failed, fresh = (role(store, "fake:acme", n) for n in ("1", "2", "3"))
|
|
now = datetime.now(UTC).isoformat(timespec="seconds")
|
|
store.state["ats:detailed:fake:acme"] = json.dumps(["fake:acme:1"])
|
|
store.state["ats:detailempty:fake:acme"] = json.dumps({"fake:acme:1": now})
|
|
store.state["ats:detailfailed:fake:acme"] = json.dumps(["fake:acme:2"])
|
|
scored(store, "One", {read: 0.9, failed: 0.8, fresh: 0.6})
|
|
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [fresh, failed])]
|
|
|
|
|
|
# ------------------------------------------------------------------ budgets
|
|
|
|
def test_the_budgets_per_board_and_per_run(
|
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
|
board(store, "fake:acme")
|
|
board(store, "fake:beta")
|
|
acme = [role(store, "fake:acme", str(n)) for n in range(4)]
|
|
beta = [role(store, "fake:beta", str(n)) for n in range(4)]
|
|
# Interleaved: acme 0.99, beta 0.98, acme 0.97, ...
|
|
scored(store, "One", {uid: 0.99 - 0.01 * n
|
|
for n, uid in enumerate(u for pair in zip(acme, beta, strict=True)
|
|
for u in pair)})
|
|
|
|
assert enrich.plan(config(tmp_path, per_board=2), store) == [
|
|
("fake:acme", acme[:2]), ("fake:beta", beta[:2])]
|
|
assert enrich.plan(config(tmp_path, per_run=3), store) == [
|
|
("fake:acme", acme[:2]), ("fake:beta", beta[:1])]
|
|
assert enrich.plan(config(tmp_path, per_board=1, per_run=5), store) == [
|
|
("fake:acme", acme[:1]), ("fake:beta", beta[:1])]
|
|
|
|
|
|
def test_the_stage_off_finds_nothing(
|
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
|
board(store, "fake:acme")
|
|
scored(store, "One", {role(store, "fake:acme", "1"): 0.9})
|
|
assert len(enrich.batches(config(tmp_path), store)) == 1
|
|
assert enrich.batches(config(tmp_path, enabled=False), store) == []
|
|
stages.set_stage(store, "enrich", True)
|
|
assert len(enrich.batches(config(tmp_path, enabled=False), store)) == 1
|
|
stages.set_stage(store, "enrich", False)
|
|
assert enrich.batches(config(tmp_path), store) == []
|
|
|
|
|
|
# ------------------------------------------------------------ what is stored
|
|
|
|
def test_a_batch_stores_what_the_pages_filled_and_drops_those_vectors(
|
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
|
board(store, "fake:acme")
|
|
one, two, gone = (role(store, "fake:acme", n) for n in ("1", "2", "3"))
|
|
store.put_vectors("fake", [(uid, np.ones(3, dtype=np.float32)) for uid in (one, two, gone)])
|
|
web["answers"].update({f"{BASE}/acme/1": b"Python in full", f"{BASE}/acme/2": b"Go too"})
|
|
|
|
out = enrich.run_batch(config(tmp_path), store, target="fake:acme", uids=[two, gone, one])
|
|
|
|
assert out == {"target": "fake:acme", "read": 3, "filled": 2, "texts": 2}
|
|
assert web["asked"] == [f"{BASE}/acme/2", f"{BASE}/acme/3", f"{BASE}/acme/1"]
|
|
first = store.role(one)
|
|
assert first is not None and first.description == "Python in full"
|
|
assert first.place == "Copenhagen" and first.title == "Backend Engineer"
|
|
assert set(store.vectors("fake")) == {gone}
|
|
assert json.loads(store.state["ats:detailed:fake:acme"]) == [
|
|
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
|
# A later pull of the same listing keeps the text and the page's place.
|
|
item = ats.item_of({"source_id": "1", "title": "Backend Engineer", "url": f"{BASE}/acme/1",
|
|
"description": "", "location": "", "places": [], "remote": None},
|
|
"fake:acme", "fake:acme")
|
|
item.source = "ats"
|
|
store.upsert_role(item, store.subscriptions(source="ats")[0])
|
|
again = store.role(one)
|
|
assert again is not None and again.description == "Python in full"
|
|
assert again.place == "Copenhagen"
|
|
|
|
|
|
def test_a_board_switched_off_since_the_plan_reads_nothing(
|
|
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
|
sub = board(store, "fake:acme")
|
|
uid = role(store, "fake:acme", "1")
|
|
store.update_subscription(sub.id, fetch=0)
|
|
assert enrich.run_batch(config(tmp_path), store, target="fake:acme", uids=[uid])["filled"] == 0
|
|
assert web["asked"] == []
|
|
|
|
|
|
# ------------------------------------------------------------------ cascade
|
|
|
|
class _Context:
|
|
def __init__(self) -> None:
|
|
self.made: list[Any] = []
|
|
|
|
def task(self, t: Any) -> Any:
|
|
self.made.append(t)
|
|
return t
|
|
|
|
|
|
def test_nothing_filled_passes_over_the_embed_and_score_behind_enrich() -> None:
|
|
context = _Context()
|
|
empty = Tally(results=({"filled": 0},))
|
|
out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 3},
|
|
context=context)
|
|
assert out["next"] == "extract" and context.made[0].carries["at"] == 6
|
|
|
|
out = cascade.step(None, None, Tally(results=({"filled": 2}, {"filled": 0})),
|
|
{"cascade": "pull", "task": "enrich", "at": 3}, context=context)
|
|
assert out["next"] == "embed" and context.made[1].carries["at"] == 4
|
|
|
|
out = cascade.step(None, None, empty, {"cascade": "enrich", "task": "enrich", "at": 0},
|
|
context=context)
|
|
assert out["next"] == "" and len(context.made) == 2
|
|
|
|
|
|
class _Embedder:
|
|
def encode(self, texts: list[str]) -> np.ndarray:
|
|
return np.vstack([np.asarray([t.lower().count("python"), 0.1], dtype=np.float32)
|
|
for t in texts])
|
|
|
|
|
|
def test_the_enrich_job_reads_the_pages_then_embeds_and_scores_those_roles(
|
|
store: Store, tmp_path: Path, web: dict[str, Any],
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(embed, "load_embedder", lambda model: _Embedder())
|
|
board(store, "fake:acme")
|
|
one, two = role(store, "fake:acme", "1"), role(store, "fake:acme", "2")
|
|
store.put_vectors("fake", [(uid, np.ones(2, dtype=np.float32)) for uid in (one, two)])
|
|
agent = scored(store, "One", {one: 0.9, two: 0.1})
|
|
web["answers"][f"{BASE}/acme/1"] = b"Python in full"
|
|
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config(tmp_path), store,
|
|
workers={"cpu": 1, "http": 1, "llm": 1})
|
|
|
|
started = queue.enqueue("enrich")
|
|
queue.drain(timeout=60)
|
|
|
|
assert started is not None
|
|
jobs = sorted(queue.jobs.of_cascade(queue.jobs.last(enrich.KIND).cascade_id), # type: ignore[union-attr]
|
|
key=lambda j: j.id)
|
|
assert [j.result.get("task") for j in jobs if j.kind == FINAL] == ["enrich", "embed", "score"]
|
|
assert [j.kind for j in jobs if j.kind != FINAL] == ["enrich", enrich.KIND, "embed", "score"]
|
|
assert web["asked"] == [f"{BASE}/acme/1"]
|
|
embedded = next(j for j in jobs if j.kind == "embed")
|
|
assert embedded.result["embedded"] == 1
|
|
assert set(store.vectors("fake")) == {one, two}
|
|
assert store.scores(agent.id)
|
|
|
|
|
|
# ----------------------------------------------------------------- triggers
|
|
|
|
@pytest.fixture
|
|
def app(tmp_path: Path) -> FastAPI:
|
|
return create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
|
|
database=tmp_path / "roledesk.db"))
|
|
|
|
|
|
@pytest.fixture
|
|
def client(app: FastAPI) -> Iterator[TestClient]:
|
|
with TestClient(app) as c:
|
|
yield c
|
|
|
|
|
|
ANY_STATE = ("queued", "running", "done", "failed", "cancelled")
|
|
|
|
|
|
def _enrich_behind_a_score(queue: Any) -> Any:
|
|
found = queue.jobs.last("enrich", states=ANY_STATE)
|
|
assert found is not None
|
|
assert [queue.jobs.get(i).kind for i in found.depends_on] == ["score"]
|
|
assert found.after == "done"
|
|
return found
|
|
|
|
|
|
def test_saving_an_agent_queues_a_score_then_enrich(client: TestClient, app: FastAPI) -> None:
|
|
res = client.post("/api/agents", json={"name": "Backend", "intent": "python backend"})
|
|
assert res.status_code == 200
|
|
_enrich_behind_a_score(app.state.queue)
|
|
res = client.post("/api/agents", json={"id": res.json()["id"], "name": "Backend",
|
|
"intent": "go backend"})
|
|
assert res.status_code == 200
|
|
|
|
|
|
def test_a_company_affinity_queues_a_score_then_enrich(
|
|
client: TestClient, app: FastAPI) -> None:
|
|
store: Store = app.state.store
|
|
store.save_agent("Backend", "python backend")
|
|
store.add_company("Acme")
|
|
res = client.post("/api/companies/acme/affinity", json={"affinity": 2})
|
|
assert res.status_code == 200
|
|
found = _enrich_behind_a_score(app.state.queue)
|
|
assert found.depends_on == [res.json()["score_job"]]
|
|
|
|
|
|
def test_one_score_is_one_enrich(app: FastAPI) -> None:
|
|
queue = app.state.queue
|
|
score = queue.enqueue("score")
|
|
first = cascade.enrich_after(queue, score)
|
|
assert cascade.enrich_after(queue, score).id == first.id
|
|
assert cascade.enrich_after(queue, None) is None
|
|
|
|
|
|
def test_a_vote_queues_neither_score_nor_enrich(client: TestClient, app: FastAPI) -> None:
|
|
store: Store = app.state.store
|
|
store.save_agent("Backend", "python backend")
|
|
item = Item(source_id="fake:acme:1", location=f"{BASE}/acme/1", title="Backend Engineer",
|
|
body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw={"company": "Acme"})
|
|
item.source = "ats"
|
|
store.upsert_role(item)
|
|
|
|
assert client.post(f"/api/roles/{item.uid}/vote", json={"value": 1}).status_code == 200
|
|
assert client.delete(f"/api/roles/{item.uid}/vote").status_code == 200
|
|
|
|
queue = app.state.queue
|
|
assert queue.jobs.last("enrich", states=ANY_STATE) is None
|
|
assert queue.jobs.last("score", states=ANY_STATE) is None
|