"""The `fetch_text` stage: the roles in the cut without text, read on their own pages after scoring. Which roles and in what order, the budgets, what is stored, the cascade around it, and what triggers it. No network: a fake host stands in, answering from a dict.""" from __future__ import annotations import json import sys import types from collections.abc import Iterator from datetime import UTC, datetime from pathlib import Path from typing import Any import httpx import numpy as np import pytest from fastapi import FastAPI from fastapi.testclient import TestClient ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from app.config import Config, ServerConfig # noqa: E402 from app.jobs import cascade, embed, fetch_text, stages # noqa: E402 from app.main import create_app # noqa: E402 from app.storage import Store # noqa: E402 from desk.models import Item # noqa: E402 from pipeline import FINAL, JobQueue, Jobs, Tally # noqa: E402 from plugins import ats, readers # noqa: E402 BASE = "https://jobs.fake.example" def _fake_host(name: str = "fake", *, detail: bool = True) -> types.ModuleType: """A host whose role pages are `BASE//`, answered by `web`.""" module: Any = types.ModuleType(name) module.NAME, module.LABEL, module.PATTERNS = name, name.title(), () module.slug_ok = lambda slug: slug.isalnum() module.allowed = lambda slug: lambda h: h == "jobs.fake.example" module.careers_url = lambda slug: f"{BASE}/{slug}" module.feed_url = lambda slug: f"{BASE}/{slug}/list" module.parse = lambda body, slug: [] if detail: module.detail = lambda get, slug, row: { "description": get(f"{BASE}/{slug}/{row['source_id']}").decode(), "location": "Copenhagen"} return module @pytest.fixture def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: """`ats._get` answering from `answers` (URL -> body), 404 otherwise, remembering what it was asked; hosts `fake` and `plain` (no detail).""" answers: dict[str, Any] = {} asked: list[str] = [] def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None): asked.append(url) answer = answers.get(url) if answer is None: request = httpx.Request("GET", url) raise httpx.HTTPStatusError("404", request=request, response=httpx.Response(404, request=request)) return answer monkeypatch.setattr(ats, "_get", fake_get) monkeypatch.setattr(ats, "PAGE_DELAY", 0) monkeypatch.setitem(readers.HOSTS, "fake", _fake_host()) monkeypatch.setitem(readers.HOSTS, "plain", _fake_host("plain", detail=False)) return {"answers": answers, "asked": asked} @pytest.fixture def store(tmp_path: Path) -> Store: return Store(tmp_path / "roledesk.db") def config(tmp_path: Path, **settings: Any) -> Config: return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"}, fetch_text=settings) def board(store: Store, target: str) -> Any: host = target.partition(":")[0] return store.add_subscription("ats", target, label=target, company=target, board=host) def role(store: Store, target: str, native: str, *, body: str = "", title: str = "Backend Engineer") -> str: """A role of board `target` as a pull stores it: without text on a host with `detail`.""" sub = next(s for s in store.subscriptions(source="ats") if s.target == target) host, _, slug = target.partition(":") item = Item(source_id=f"{target}:{native}", location=f"{BASE}/{slug}/{native}", title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC), raw={"company": target, "place": "2 Locations", "board": host, "slug": slug}) item.source = "ats" store.upsert_role(item, sub) return item.uid def scored(store: Store, name: str, scores: dict[str, float], *, gated: tuple[str, ...] = (), out: tuple[str, ...] = ()) -> Any: """An agent whose scores are these, every ungated role in its cut by fit but those in `out`.""" agent = store.save_agent(name, f"{name} python backend") store.replace_scores(agent.id, [ (uid, value, "gated" if uid in gated else "", "", None, value, "" if uid in gated or uid in out else "fit") for uid, value in scores.items()]) return agent # -------------------------------------------------------------- which roles def test_roles_in_the_cut_go_best_first_by_their_best_agent_score( store: Store, tmp_path: Path, web: dict[str, Any]) -> None: board(store, "fake:acme") board(store, "fake:beta") a1, a2 = role(store, "fake:acme", "1"), role(store, "fake:acme", "2") b1 = role(store, "fake:beta", "1") scored(store, "One", {a1: 0.6, a2: 0.9, b1: 0.55}) scored(store, "Two", {a1: 0.95, b1: 0.7}) assert [(r["uid"], r["score"]) for r in store.fetch_text_waiting()] == [ (a1, 0.95), (a2, 0.9), (b1, 0.7)] assert fetch_text.plan(config(tmp_path), store) == [("fake:acme", [a1, a2]), ("fake:beta", [b1])] def test_a_role_in_no_cut_gets_no_call( store: Store, tmp_path: Path, web: dict[str, Any]) -> None: board(store, "fake:acme") low, gated, deep, kept = (role(store, "fake:acme", n) for n in ("1", "2", "3", "4")) scored(store, "One", {low: 0.4, gated: 0.0, deep: 0.8, kept: 0.9}, gated=(gated,), out=(low,)) assert fetch_text.plan(config(tmp_path), store) == [("fake:acme", [kept, deep])] # In another agent's cut, it is read. scored(store, "Two", {low: 0.4}) assert fetch_text.plan(config(tmp_path), store) == [("fake:acme", [kept, deep, low])] store.delete_agent(store.agents()[1].id) # A disabled agent ranks nothing. agent = store.agents()[0] store.save_agent(agent.name, gates=agent.gates, weights=agent.weights, enabled=False, agent_id=agent.id, limits=agent.limits, prompts=agent.prompts) assert fetch_text.plan(config(tmp_path), store) == [] def test_roles_with_text_hosts_without_pages_and_failing_boards_are_left_out( store: Store, tmp_path: Path, web: dict[str, Any]) -> None: for target in ("fake:acme", "plain:acme", "fake:down"): board(store, target) wanted = role(store, "fake:acme", "1") has_text = role(store, "fake:acme", "2", body="Python in full") plain = role(store, "plain:acme", "1") down = role(store, "fake:down", "1") store.state["ats:failing:fake:down"] = json.dumps({"since": "2026-09-29", "reason": "500"}) scored(store, "One", {wanted: 0.9, has_text: 0.95, plain: 0.9, down: 0.9}) assert fetch_text.plan(config(tmp_path), store) == [("fake:acme", [wanted])] def test_a_role_read_already_waits_for_the_retry_and_a_failed_one_goes_last( store: Store, tmp_path: Path, web: dict[str, Any]) -> None: board(store, "fake:acme") read, failed, fresh = (role(store, "fake:acme", n) for n in ("1", "2", "3")) now = datetime.now(UTC).isoformat(timespec="seconds") store.state["ats:detailed:fake:acme"] = json.dumps(["fake:acme:1"]) store.state["ats:detailempty:fake:acme"] = json.dumps({"fake:acme:1": now}) store.state["ats:detailfailed:fake:acme"] = json.dumps(["fake:acme:2"]) scored(store, "One", {read: 0.9, failed: 0.8, fresh: 0.6}) assert fetch_text.plan(config(tmp_path), store) == [("fake:acme", [fresh, failed])] # ------------------------------------------------------------------ budgets def test_the_budgets_per_board_and_per_run( store: Store, tmp_path: Path, web: dict[str, Any]) -> None: board(store, "fake:acme") board(store, "fake:beta") acme = [role(store, "fake:acme", str(n)) for n in range(4)] beta = [role(store, "fake:beta", str(n)) for n in range(4)] # Interleaved: acme 0.99, beta 0.98, acme 0.97, ... scored(store, "One", {uid: 0.99 - 0.01 * n for n, uid in enumerate(u for pair in zip(acme, beta, strict=True) for u in pair)}) assert fetch_text.plan(config(tmp_path, per_board=2), store) == [ ("fake:acme", acme[:2]), ("fake:beta", beta[:2])] assert fetch_text.plan(config(tmp_path, per_run=3), store) == [ ("fake:acme", acme[:2]), ("fake:beta", beta[:1])] assert fetch_text.plan(config(tmp_path, per_board=1, per_run=5), store) == [ ("fake:acme", acme[:1]), ("fake:beta", beta[:1])] def test_the_stage_off_finds_nothing( store: Store, tmp_path: Path, web: dict[str, Any]) -> None: board(store, "fake:acme") scored(store, "One", {role(store, "fake:acme", "1"): 0.9}) assert len(fetch_text.batches(config(tmp_path), store)) == 1 assert fetch_text.batches(config(tmp_path, enabled=False), store) == [] stages.set_stage(store, "fetch_text", True) assert len(fetch_text.batches(config(tmp_path, enabled=False), store)) == 1 stages.set_stage(store, "fetch_text", False) assert fetch_text.batches(config(tmp_path), store) == [] # ------------------------------------------------------------ what is stored def test_a_batch_stores_what_the_pages_filled_and_drops_those_vectors( store: Store, tmp_path: Path, web: dict[str, Any]) -> None: board(store, "fake:acme") one, two, gone = (role(store, "fake:acme", n) for n in ("1", "2", "3")) store.put_vectors("fake", [(uid, np.ones(3, dtype=np.float32)) for uid in (one, two, gone)]) web["answers"].update({f"{BASE}/acme/1": b"Python in full", f"{BASE}/acme/2": b"Go too"}) out = fetch_text.run_batch(config(tmp_path), store, target="fake:acme", uids=[two, gone, one]) assert out == {"target": "fake:acme", "read": 3, "filled": 2, "texts": 2} assert web["asked"] == [f"{BASE}/acme/2", f"{BASE}/acme/3", f"{BASE}/acme/1"] first = store.role(one) assert first is not None and first.description == "Python in full" assert first.place == "Copenhagen" and first.title == "Backend Engineer" assert set(store.vectors("fake")) == {gone} assert json.loads(store.state["ats:detailed:fake:acme"]) == [ "fake:acme:1", "fake:acme:2", "fake:acme:3"] # A later pull of the same listing keeps the text and the page's place. item = ats.item_of({"source_id": "1", "title": "Backend Engineer", "url": f"{BASE}/acme/1", "description": "", "location": "", "places": [], "remote": None}, "fake:acme", "fake:acme") item.source = "ats" store.upsert_role(item, store.subscriptions(source="ats")[0]) again = store.role(one) assert again is not None and again.description == "Python in full" assert again.place == "Copenhagen" def test_a_board_switched_off_since_the_plan_reads_nothing( store: Store, tmp_path: Path, web: dict[str, Any]) -> None: sub = board(store, "fake:acme") uid = role(store, "fake:acme", "1") store.update_subscription(sub.id, fetch=0) assert fetch_text.run_batch(config(tmp_path), store, target="fake:acme", uids=[uid])["filled"] == 0 assert web["asked"] == [] # ----------------------------------------------------------- search roles AD = (ROOT / "tests" / "fixtures" / "search" / "jobbank-detail.html").read_bytes() AD_URL = "https://jobbank.dk/job/1234/acme/role" def search(store: Store, target: str = "jobbank:cph") -> Any: return store.add_subscription("search", target, label=target, params={"q": "x"}) def search_role(store: Store, target: str, native: str, *, provider: str = "") -> str: """A role of search `target` as a pull stores it: `provider:id`, the employer in `raw` and no text.""" provider = provider or target.partition(":")[0] sub = next(s for s in store.subscriptions(source="search") if s.target == target) row = {"source_id": native, "title": "Researcher", "url": AD_URL.replace("1234", native), "description": "", "company": "Acme A/S", "places": [{"text": "Fyn", "remote": None}]} item = ats.item_of(row, target, sub.label) item.source = "search" assert item.source_id == f"{provider}:{native}" store.upsert_role(item, sub) return item.uid def test_a_search_whose_reader_has_detail_is_planned_like_a_board( store: Store, tmp_path: Path, web: dict[str, Any]) -> None: search(store) search(store, "jobindex:cph") # no `detail`: left out board(store, "fake:acme") a, b = search_role(store, "jobbank:cph", "1234"), search_role(store, "jobbank:cph", "99") j = search_role(store, "jobindex:cph", "7") f = role(store, "fake:acme", "1") scored(store, "One", {a: 0.9, b: 0.6, j: 0.95, f: 0.7}) assert fetch_text.plan(config(tmp_path), store) == [("jobbank:cph", [a, b]), ("fake:acme", [f])] # The budgets are a board's: `per_board` and `per_run` count the search as one. assert fetch_text.plan(config(tmp_path, per_board=1), store) == [("jobbank:cph", [a]), ("fake:acme", [f])] assert fetch_text.plan(config(tmp_path, per_run=2), store) == [("jobbank:cph", [a]), ("fake:acme", [f])] # A failing search is left out, as a failing board is. store.state["search:failing:jobbank:cph"] = "{}" assert fetch_text.plan(config(tmp_path), store) == [("fake:acme", [f])] def test_a_search_role_read_waits_for_the_retry_and_a_failed_one_goes_last( store: Store, tmp_path: Path, web: dict[str, Any]) -> None: search(store) one, two, three = (search_role(store, "jobbank:cph", n) for n in ("1", "2", "3")) scored(store, "One", {one: 0.9, two: 0.8, three: 0.7}) store.state["ats:detailed:jobbank:cph"] = json.dumps(["jobbank:cph:1", "jobbank:cph:2"]) store.state["ats:detailfailed:jobbank:cph"] = json.dumps(["jobbank:cph:2"]) store.state["ats:detailempty:jobbank:cph"] = json.dumps( {"jobbank:cph:1": datetime.now(UTC).isoformat(timespec="seconds"), "jobbank:cph:2": "2020-01-01T00:00:00+00:00"}) # 1 gave no text a moment ago and waits; 2 failed and is due, after 3. assert fetch_text.plan(config(tmp_path), store) == [("jobbank:cph", [three, two])] def test_a_batch_reads_a_search_roles_page_and_stores_its_text( store: Store, tmp_path: Path, web: dict[str, Any]) -> None: search(store) uid = search_role(store, "jobbank:cph", "1234") store.put_vectors("fake", [(uid, np.ones(3, dtype=np.float32))]) web["answers"][AD_URL] = AD out = fetch_text.run_batch(config(tmp_path), store, target="jobbank:cph", uids=[uid]) assert out == {"target": "jobbank:cph", "read": 1, "filled": 1, "texts": 1} assert web["asked"] == [AD_URL] got = store.role(uid) assert got is not None and got.description assert (got.source, got.source_id, got.company) == ("search", "jobbank:1234", "Acme A/S") assert got.raw["search"] == "jobbank:cph" and got.raw["provider"] == "jobbank" assert store.vectors("fake") == {} assert json.loads(store.state["ats:detailed:jobbank:cph"]) == ["jobbank:cph:1234"] # Read, it is not planned again, and has no text left to wait for. scored(store, "One", {uid: 0.9}) assert fetch_text.plan(config(tmp_path), store) == [] def test_a_search_batch_counts_a_404_as_read_as_a_board_does( store: Store, tmp_path: Path, web: dict[str, Any]) -> None: search(store) one, two = search_role(store, "jobbank:cph", "1"), search_role(store, "jobbank:cph", "2") out = fetch_text.run_batch(config(tmp_path), store, target="jobbank:cph", uids=[one, two]) assert out["filled"] == 0 and out["read"] == 2 assert json.loads(store.state["ats:detailed:jobbank:cph"]) == [ "jobbank:cph:1", "jobbank:cph:2"] def test_a_reader_is_found_for_a_board_and_a_search_and_one_without_detail_is_refused( web: dict[str, Any]) -> None: assert ats.reader_of("fake:acme") is readers.HOSTS["fake"] assert ats.reader_of("jobbank:cph") is readers.PROVIDERS["jobbank"] assert ats.reader_of("nowhere:x") is None with httpx.Client() as client: for target in ("jobindex:cph", "plain:acme", "nowhere:x"): with pytest.raises(ValueError, match="no reader"): ats.read_detail(client, target, {"source_id": "1", "url": BASE}) with pytest.raises(ValueError, match="no reader"): ats.read_board_details({}, target, [{"source_id": "1", "url": BASE}], client=client) # ------------------------------------------------------------------ cascade class _Context: def __init__(self) -> None: self.made: list[Any] = [] def task(self, t: Any) -> Any: self.made.append(t) return t def test_nothing_filled_passes_over_the_prep_embed_and_score_behind_fetch_text() -> None: context = _Context() empty = Tally(results=({"filled": 0},)) out = cascade.step(None, None, empty, {"cascade": "pull", "task": "fetch_text", "at": 4}, context=context) assert out["next"] == "extract" and context.made[0].carries["at"] == 8 out = cascade.step(None, None, Tally(results=({"filled": 2}, {"filled": 0})), {"cascade": "pull", "task": "fetch_text", "at": 4}, context=context) assert out["next"] == "prep" and context.made[1].carries["at"] == 5 out = cascade.step(None, None, empty, {"cascade": "fetch_text", "task": "fetch_text", "at": 0}, context=context) assert out["next"] == "extract" and context.made[2].carries["at"] == 4 class _Embedder: def encode(self, texts: list[str]) -> np.ndarray: return np.vstack([np.asarray([t.lower().count("python"), 0.1], dtype=np.float32) for t in texts]) def test_the_fetch_text_job_reads_the_pages_then_embeds_and_scores_those_roles( store: Store, tmp_path: Path, web: dict[str, Any], monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(embed, "load_embedder", lambda model: _Embedder()) board(store, "fake:acme") one, two = role(store, "fake:acme", "1"), role(store, "fake:acme", "2") store.put_vectors("fake", [(uid, np.ones(2, dtype=np.float32)) for uid in (one, two)]) agent = scored(store, "One", {one: 0.9, two: 0.1}, out=(two,)) web["answers"][f"{BASE}/acme/1"] = b"Python in full" queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config(tmp_path), store, workers={"cpu": 1, "http": 1, "llm": 1}) started = queue.enqueue("fetch_text") queue.drain(timeout=60) assert started is not None jobs = sorted(queue.jobs.of_cascade(queue.jobs.last(fetch_text.KIND).cascade_id), # type: ignore[union-attr] key=lambda j: j.id) assert [j.result.get("task") for j in jobs if j.kind == FINAL] == \ ["fetch_text", "prep", "embed", "score", "extract", "score", "judge", "summarise"] assert [j.kind for j in jobs if j.kind != FINAL] == \ ["fetch_text", fetch_text.KIND, "prep", "embed", "score", "score"] assert web["asked"] == [f"{BASE}/acme/1"] embedded = next(j for j in jobs if j.kind == "embed") assert embedded.result["embedded"] == 1 assert set(store.vectors("fake")) == {one, two} assert store.scores(agent.id) # ----------------------------------------------------------------- triggers @pytest.fixture def app(tmp_path: Path) -> FastAPI: return create_app(Config(server=ServerConfig(host="127.0.0.1", port=0), database=tmp_path / "roledesk.db")) @pytest.fixture def client(app: FastAPI) -> Iterator[TestClient]: with TestClient(app) as c: yield c ANY_STATE = ("queued", "running", "done", "failed", "cancelled") def _fetch_text_behind_a_score(queue: Any) -> Any: found = queue.jobs.last("fetch_text", states=ANY_STATE) assert found is not None assert [queue.jobs.get(i).kind for i in found.depends_on] == ["score"] assert found.after == "done" return found def test_saving_an_agent_queues_a_score_then_fetch_text(client: TestClient, app: FastAPI) -> None: res = client.post("/api/agents", json={"name": "Backend", "intent": "python backend"}) assert res.status_code == 200 _fetch_text_behind_a_score(app.state.queue) res = client.post("/api/agents", json={"id": res.json()["id"], "name": "Backend", "intent": "go backend"}) assert res.status_code == 200 def test_a_company_affinity_queues_a_score_then_fetch_text( client: TestClient, app: FastAPI) -> None: store: Store = app.state.store store.save_agent("Backend", "python backend") store.add_company("Acme") res = client.post("/api/companies/acme/affinity", json={"affinity": 2}) assert res.status_code == 200 found = _fetch_text_behind_a_score(app.state.queue) assert found.depends_on == [res.json()["score_job"]] def test_one_score_is_one_fetch_text(app: FastAPI) -> None: queue = app.state.queue score = queue.enqueue("score") first = cascade.fetch_text_after(queue, score) assert cascade.fetch_text_after(queue, score).id == first.id assert cascade.fetch_text_after(queue, None) is None def test_a_vote_queues_neither_score_nor_fetch_text(client: TestClient, app: FastAPI) -> None: store: Store = app.state.store store.save_agent("Backend", "python backend") item = Item(source_id="fake:acme:1", location=f"{BASE}/acme/1", title="Backend Engineer", body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw={"company": "Acme"}) item.source = "ats" store.upsert_role(item) assert client.post(f"/api/roles/{item.uid}/vote", json={"value": 1}).status_code == 200 assert client.delete(f"/api/roles/{item.uid}/vote").status_code == 200 queue = app.state.queue assert queue.jobs.last("fetch_text", states=ANY_STATE) is None assert queue.jobs.last("score", states=ANY_STATE) is None