Merge branch 'wp-g-enrich'

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:53:57 +02:00
commit 38a80d5b55
10 files changed
+956 -219

No files matched your search

+368
View File
@@ -0,0 +1,368 @@
"""The `enrich` stage: the best roles without text, read on their own pages
after scoring. Which roles and in what order, the budgets, what is stored,
the cascade around it, and what triggers it. No network: a fake host stands
in, answering from a dict."""
from __future__ import annotations
import json
import sys
import types
from collections.abc import Iterator
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import httpx
import numpy as np
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.config import Config, ServerConfig # noqa: E402
from app.jobs import cascade, embed, enrich, stages # noqa: E402
from app.main import create_app # noqa: E402
from app.storage import Store # noqa: E402
from desk.models import Item # noqa: E402
from pipeline import FINAL, JobQueue, Jobs, Tally # noqa: E402
from plugins import ats, readers # noqa: E402
BASE = "https://jobs.fake.example"
def _fake_host(name: str = "fake", *, detail: bool = True) -> types.ModuleType:
"""A host whose role pages are `BASE/<slug>/<id>`, answered by `web`."""
module: Any = types.ModuleType(name)
module.NAME, module.LABEL, module.PATTERNS = name, name.title(), ()
module.slug_ok = lambda slug: slug.isalnum()
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
module.careers_url = lambda slug: f"{BASE}/{slug}"
module.feed_url = lambda slug: f"{BASE}/{slug}/list"
module.parse = lambda body, slug: []
if detail:
module.detail = lambda get, slug, row: {
"description": get(f"{BASE}/{slug}/{row['source_id']}").decode(),
"location": "Copenhagen"}
return module
@pytest.fixture
def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
"""`ats._get` answering from `answers` (URL -> body), 404 otherwise,
remembering what it was asked; hosts `fake` and `plain` (no detail)."""
answers: dict[str, Any] = {}
asked: list[str] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None):
asked.append(url)
answer = answers.get(url)
if answer is None:
request = httpx.Request("GET", url)
raise httpx.HTTPStatusError("404", request=request,
response=httpx.Response(404, request=request))
return answer
monkeypatch.setattr(ats, "_get", fake_get)
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host())
monkeypatch.setitem(readers.HOSTS, "plain", _fake_host("plain", detail=False))
return {"answers": answers, "asked": asked}
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
def config(tmp_path: Path, **enrich_settings: Any) -> Config:
return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"},
enrich=enrich_settings)
def board(store: Store, target: str) -> Any:
host = target.partition(":")[0]
return store.add_subscription("ats", target, label=target, company=target, board=host)
def role(store: Store, target: str, native: str, *, body: str = "",
title: str = "Backend Engineer") -> str:
"""A role of board `target` as a pull stores it: without text on a host
with `detail`."""
sub = next(s for s in store.subscriptions(source="ats") if s.target == target)
host, _, slug = target.partition(":")
item = Item(source_id=f"{target}:{native}", location=f"{BASE}/{slug}/{native}",
title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"company": target, "place": "2 Locations", "board": host, "slug": slug})
item.source = "ats"
store.upsert_role(item, sub)
return item.uid
def scored(store: Store, name: str, scores: dict[str, float], *,
gated: tuple[str, ...] = ()) -> Any:
agent = store.save_agent(name, f"{name} python backend")
store.replace_scores(agent.id, [(uid, value, "gated" if uid in gated else "")
for uid, value in scores.items()])
return agent
# -------------------------------------------------------------- which roles
def test_roles_go_best_first_by_their_best_agent_score(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
board(store, "fake:acme")
board(store, "fake:beta")
a1, a2 = role(store, "fake:acme", "1"), role(store, "fake:acme", "2")
b1 = role(store, "fake:beta", "1")
scored(store, "One", {a1: 0.6, a2: 0.9, b1: 0.55})
scored(store, "Two", {a1: 0.95, b1: 0.7})
assert [(r["uid"], r["score"]) for r in store.enrich_waiting(500, 0.5)] == [
(a1, 0.95), (a2, 0.9), (b1, 0.7)]
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [a1, a2]),
("fake:beta", [b1])]
def test_a_role_no_agent_ranks_high_enough_gets_no_call(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
board(store, "fake:acme")
low, gated, deep, kept = (role(store, "fake:acme", n) for n in ("1", "2", "3", "4"))
scored(store, "One", {low: 0.4, gated: 0.0, deep: 0.8, kept: 0.9}, gated=(gated,))
# `top` 1 per agent: only the best ungated role of each agent is in reach.
assert enrich.plan(config(tmp_path, top=1), store) == [("fake:acme", [kept])]
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [kept, deep])]
# A disabled agent ranks nothing.
agent = store.agents()[0]
store.save_agent(agent.name, gates=agent.gates, weights=agent.weights, enabled=False,
agent_id=agent.id, limits=agent.limits, prompts=agent.prompts)
assert enrich.plan(config(tmp_path), store) == []
def test_roles_with_text_hosts_without_pages_and_failing_boards_are_left_out(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
for target in ("fake:acme", "plain:acme", "fake:down"):
board(store, target)
wanted = role(store, "fake:acme", "1")
has_text = role(store, "fake:acme", "2", body="Python in full")
plain = role(store, "plain:acme", "1")
down = role(store, "fake:down", "1")
store.state["ats:failing:fake:down"] = json.dumps({"since": "2026-09-29", "reason": "500"})
scored(store, "One", {wanted: 0.9, has_text: 0.95, plain: 0.9, down: 0.9})
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [wanted])]
def test_a_role_read_already_waits_for_the_retry_and_a_failed_one_goes_last(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
board(store, "fake:acme")
read, failed, fresh = (role(store, "fake:acme", n) for n in ("1", "2", "3"))
now = datetime.now(UTC).isoformat(timespec="seconds")
store.state["ats:detailed:fake:acme"] = json.dumps(["fake:acme:1"])
store.state["ats:detailempty:fake:acme"] = json.dumps({"fake:acme:1": now})
store.state["ats:detailfailed:fake:acme"] = json.dumps(["fake:acme:2"])
scored(store, "One", {read: 0.9, failed: 0.8, fresh: 0.6})
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [fresh, failed])]
# ------------------------------------------------------------------ budgets
def test_the_budgets_per_board_and_per_run(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
board(store, "fake:acme")
board(store, "fake:beta")
acme = [role(store, "fake:acme", str(n)) for n in range(4)]
beta = [role(store, "fake:beta", str(n)) for n in range(4)]
# Interleaved: acme 0.99, beta 0.98, acme 0.97, ...
scored(store, "One", {uid: 0.99 - 0.01 * n
for n, uid in enumerate(u for pair in zip(acme, beta, strict=True)
for u in pair)})
assert enrich.plan(config(tmp_path, per_board=2), store) == [
("fake:acme", acme[:2]), ("fake:beta", beta[:2])]
assert enrich.plan(config(tmp_path, per_run=3), store) == [
("fake:acme", acme[:2]), ("fake:beta", beta[:1])]
assert enrich.plan(config(tmp_path, per_board=1, per_run=5), store) == [
("fake:acme", acme[:1]), ("fake:beta", beta[:1])]
def test_the_stage_off_finds_nothing(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
board(store, "fake:acme")
scored(store, "One", {role(store, "fake:acme", "1"): 0.9})
assert len(enrich.batches(config(tmp_path), store)) == 1
assert enrich.batches(config(tmp_path, enabled=False), store) == []
stages.set_stage(store, "enrich", True)
assert len(enrich.batches(config(tmp_path, enabled=False), store)) == 1
stages.set_stage(store, "enrich", False)
assert enrich.batches(config(tmp_path), store) == []
# ------------------------------------------------------------ what is stored
def test_a_batch_stores_what_the_pages_filled_and_drops_those_vectors(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
board(store, "fake:acme")
one, two, gone = (role(store, "fake:acme", n) for n in ("1", "2", "3"))
store.put_vectors("fake", [(uid, np.ones(3, dtype=np.float32)) for uid in (one, two, gone)])
web["answers"].update({f"{BASE}/acme/1": b"Python in full", f"{BASE}/acme/2": b"Go too"})
out = enrich.run_batch(config(tmp_path), store, target="fake:acme", uids=[two, gone, one])
assert out == {"target": "fake:acme", "read": 3, "filled": 2, "texts": 2}
assert web["asked"] == [f"{BASE}/acme/2", f"{BASE}/acme/3", f"{BASE}/acme/1"]
first = store.role(one)
assert first is not None and first.description == "Python in full"
assert first.place == "Copenhagen" and first.title == "Backend Engineer"
assert set(store.vectors("fake")) == {gone}
assert json.loads(store.state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
# A later pull of the same listing keeps the text and the page's place.
item = ats.item_of({"source_id": "1", "title": "Backend Engineer", "url": f"{BASE}/acme/1",
"description": "", "location": "", "places": [], "remote": None},
"fake:acme", "fake:acme")
item.source = "ats"
store.upsert_role(item, store.subscriptions(source="ats")[0])
again = store.role(one)
assert again is not None and again.description == "Python in full"
assert again.place == "Copenhagen"
def test_a_board_switched_off_since_the_plan_reads_nothing(
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
sub = board(store, "fake:acme")
uid = role(store, "fake:acme", "1")
store.update_subscription(sub.id, fetch=0)
assert enrich.run_batch(config(tmp_path), store, target="fake:acme", uids=[uid])["filled"] == 0
assert web["asked"] == []
# ------------------------------------------------------------------ cascade
class _Context:
def __init__(self) -> None:
self.made: list[Any] = []
def task(self, t: Any) -> Any:
self.made.append(t)
return t
def test_nothing_filled_passes_over_the_embed_and_score_behind_enrich() -> None:
context = _Context()
empty = Tally(results=({"filled": 0},))
out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 3},
context=context)
assert out["next"] == "extract" and context.made[0].carries["at"] == 6
out = cascade.step(None, None, Tally(results=({"filled": 2}, {"filled": 0})),
{"cascade": "pull", "task": "enrich", "at": 3}, context=context)
assert out["next"] == "embed" and context.made[1].carries["at"] == 4
out = cascade.step(None, None, empty, {"cascade": "enrich", "task": "enrich", "at": 0},
context=context)
assert out["next"] == "" and len(context.made) == 2
class _Embedder:
def encode(self, texts: list[str]) -> np.ndarray:
return np.vstack([np.asarray([t.lower().count("python"), 0.1], dtype=np.float32)
for t in texts])
def test_the_enrich_job_reads_the_pages_then_embeds_and_scores_those_roles(
store: Store, tmp_path: Path, web: dict[str, Any],
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(embed, "load_embedder", lambda model: _Embedder())
board(store, "fake:acme")
one, two = role(store, "fake:acme", "1"), role(store, "fake:acme", "2")
store.put_vectors("fake", [(uid, np.ones(2, dtype=np.float32)) for uid in (one, two)])
agent = scored(store, "One", {one: 0.9, two: 0.1})
web["answers"][f"{BASE}/acme/1"] = b"Python in full"
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config(tmp_path), store,
workers={"cpu": 1, "http": 1, "llm": 1})
started = queue.enqueue("enrich")
queue.drain(timeout=60)
assert started is not None
jobs = sorted(queue.jobs.of_cascade(queue.jobs.last(enrich.KIND).cascade_id), # type: ignore[union-attr]
key=lambda j: j.id)
assert [j.result.get("task") for j in jobs if j.kind == FINAL] == ["enrich", "embed", "score"]
assert [j.kind for j in jobs if j.kind != FINAL] == ["enrich", enrich.KIND, "embed", "score"]
assert web["asked"] == [f"{BASE}/acme/1"]
embedded = next(j for j in jobs if j.kind == "embed")
assert embedded.result["embedded"] == 1
assert set(store.vectors("fake")) == {one, two}
assert store.scores(agent.id)
# ----------------------------------------------------------------- triggers
@pytest.fixture
def app(tmp_path: Path) -> FastAPI:
return create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
database=tmp_path / "roledesk.db"))
@pytest.fixture
def client(app: FastAPI) -> Iterator[TestClient]:
with TestClient(app) as c:
yield c
ANY_STATE = ("queued", "running", "done", "failed", "cancelled")
def _enrich_behind_a_score(queue: Any) -> Any:
found = queue.jobs.last("enrich", states=ANY_STATE)
assert found is not None
assert [queue.jobs.get(i).kind for i in found.depends_on] == ["score"]
assert found.after == "done"
return found
def test_saving_an_agent_queues_a_score_then_enrich(client: TestClient, app: FastAPI) -> None:
res = client.post("/api/agents", json={"name": "Backend", "intent": "python backend"})
assert res.status_code == 200
_enrich_behind_a_score(app.state.queue)
res = client.post("/api/agents", json={"id": res.json()["id"], "name": "Backend",
"intent": "go backend"})
assert res.status_code == 200
def test_a_company_affinity_queues_a_score_then_enrich(
client: TestClient, app: FastAPI) -> None:
store: Store = app.state.store
store.save_agent("Backend", "python backend")
store.add_company("Acme")
res = client.post("/api/companies/acme/affinity", json={"affinity": 2})
assert res.status_code == 200
found = _enrich_behind_a_score(app.state.queue)
assert found.depends_on == [res.json()["score_job"]]
def test_one_score_is_one_enrich(app: FastAPI) -> None:
queue = app.state.queue
score = queue.enqueue("score")
first = cascade.enrich_after(queue, score)
assert cascade.enrich_after(queue, score).id == first.id
assert cascade.enrich_after(queue, None) is None
def test_a_vote_queues_neither_score_nor_enrich(client: TestClient, app: FastAPI) -> None:
store: Store = app.state.store
store.save_agent("Backend", "python backend")
item = Item(source_id="fake:acme:1", location=f"{BASE}/acme/1", title="Backend Engineer",
body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw={"company": "Acme"})
item.source = "ats"
store.upsert_role(item)
assert client.post(f"/api/roles/{item.uid}/vote", json={"value": 1}).status_code == 200
assert client.delete(f"/api/roles/{item.uid}/vote").status_code == 200
queue = app.state.queue
assert queue.jobs.last("enrich", states=ANY_STATE) is None
assert queue.jobs.last("score", states=ANY_STATE) is None
+136 -163
View File
@@ -1,6 +1,7 @@
"""The host modules of `plugins/readers/` and what `plugins/ats.py` does for
every host: the registry and its contract check, the fetch a host is handed,
the per-role detail reads, and Lever's EU instance. No network —
the per-role detail reads the `enrich` stage makes, and Lever's EU
instance. No network —
a fake host stands in where a behaviour belongs to no real host yet."""
from __future__ import annotations
@@ -11,7 +12,7 @@ import subprocess
import sys
import types
from collections.abc import Callable
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
@@ -308,34 +309,26 @@ def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
# ---------------------------------------------------------------- detail
def test_detail_is_read_for_unread_roles_only(
def test_a_pull_reads_no_role_page(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""The pages are `enrich`'s: the pull reads the listing only, and sends
every row of a host with `detail` without a description."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one in full",
f"{base}/2": b"two in full", f"{base}/3": b"three in full"})
module = _fake_host(detail=_detail)
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"])}
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/3"]
assert [(i.source_id, i.body) for i in items] == [
("fake:acme:1", "one in full"), ("fake:acme:2", ""), ("fake:acme:3", "three in full")]
# What detail returns replaces the listing's field; a role not read keeps the listing's.
assert items[0].raw["place"] == "Copenhagen" and items[1].raw["place"] == ""
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one in full"})
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert web["asked"] == [f"{base}/list"]
assert [(i.source_id, i.body) for i in items] == [("fake:acme:1", ""), ("fake:acme:2", "")]
assert "ats:detailed:fake:acme" not in state
assert state["ats:complete:fake:acme"] == "1"
def test_a_role_read_on_an_earlier_pull_keeps_the_place_its_page_gave(
def test_a_role_read_on_its_page_keeps_the_place_its_page_gave(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""The listing states a place on every pull, and a filled value replaces
a filled column, so for a role already read it is withheld."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": b"two in full",
f"{base}/3": RuntimeError("down")})
module = _fake_host(detail=lambda get, slug, row: {
"description": get(f"{base}/{row['source_id']}").decode()})
web["answers"].update({f"{base}/list": b'["1", "3"]'})
module = _fake_host(detail=_detail)
listing = module.parse
module.parse = lambda body, slug: [row | {"location": "2 Locations", "remote": "hybrid"}
for row in listing(body, slug)]
@@ -345,70 +338,25 @@ def test_a_role_read_on_an_earlier_pull_keeps_the_place_its_page_gave(
by_id = {i.source_id: (i.body, i.raw["place"], i.raw["remote"]) for i in items}
assert by_id["fake:acme:1"] == ("", "", None)
# Read now, and its page stated neither: the listing's stand.
assert by_id["fake:acme:2"] == ("two in full", "2 Locations", "hybrid")
# Never read: the listing's stand until its page is.
assert by_id["fake:acme:3"] == ("", "2 Locations", "hybrid")
def test_detail_reads_stop_at_the_cap_and_the_rest_wait(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
ids = [str(n) for n in range(5)]
web["answers"].update({f"{base}/list": json.dumps(ids).encode()}
| {f"{base}/{i}": f"full {i}".encode() for i in ids})
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
module = _fake_host(detail=_detail)
items, state = _run(monkeypatch, module)
assert [i.body for i in items] == ["full 0", "full 1", "", "", ""]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:0", "fake:acme:1"]
web["asked"].clear()
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/2", f"{base}/3"]
assert [i.body for i in items] == ["", "", "full 2", "full 3", ""]
def test_a_detail_that_fails_skips_the_role_and_not_the_board(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
f"{base}/2": ValueError("no such page shape"), f"{base}/3": b"three"})
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert [(i.source_id, i.body) for i in items] == [
("fake:acme:1", "one"), ("fake:acme:2", ""), ("fake:acme:3", "three")]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:3"]
assert "ats:failing:fake:acme" not in state and state["ats:ok:fake:acme"] == "r"
assert state["ats:complete:fake:acme"] == "1"
def test_a_429_on_a_detail_stops_the_reads_for_the_pull(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
f"{base}/2": _status(429), f"{base}/3": b"three"})
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
assert [i.body for i in items] == ["one", "", ""]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
assert state["ats:complete:fake:acme"] == "1"
def test_ids_no_longer_listed_leave_the_detailed_list(
def test_ids_no_longer_listed_leave_the_detail_lists(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]'})
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"]),
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""A listing cut short says nothing about the roles it left out, so their
ids stay, and the next pull does not read them in detail again."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]'})
ids stay, and they are not read on their pages again."""
monkeypatch.setattr(_runner, "read_listing", lambda *args, **kwargs: _runner.Listing(
readers.HOSTS["fake"].parse(b'["2"]', "acme"), False, ["2"]))
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
@@ -417,6 +365,19 @@ def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
assert state["ats:complete:fake:acme"] == ""
def test_a_pull_stopped_half_way_through_a_board_drops_no_detail_ids(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]'})
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
source.state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])}
roles: Any = source.fetch() # a generator, which `close` stops
assert next(roles).body == ""
roles.close()
assert json.loads(source.state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
def test_fetch_board_reads_the_listing_only(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
@@ -435,63 +396,137 @@ def test_a_host_without_detail_keeps_the_listings_description(
assert "ats:detailed:fake:acme" not in state
def test_a_detail_answering_404_or_410_counts_as_read(
# -------------------------------------------------- read_board_details
NOW = datetime(2026, 9, 30, 12, tzinfo=UTC)
def _details(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType,
state: dict[str, str], ids: list[str], **kwargs: Any) -> list[dict[str, Any]]:
"""`ats.read_board_details` over the fake host's roles `ids`, stored
without text, in that order."""
monkeypatch.setitem(readers.HOSTS, "fake", module)
rows = [row | {"description": ""}
for row in module.parse(json.dumps(ids).encode(), "acme")]
return ats.read_board_details(state, "fake:acme", rows, client=object(), # type: ignore[arg-type]
now=kwargs.pop("now", NOW), **kwargs)
def test_details_are_read_in_the_order_given_for_unread_roles_only(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": _status(410),
f"{base}/3": b"three"}) # 1 answers 404
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert [i.body for i in items] == ["", "", "three"]
web["answers"].update({f"{base}/1": b"one in full", f"{base}/2": b"two in full",
f"{base}/3": b"three in full"})
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
"ats:detailempty:fake:acme": json.dumps({"fake:acme:2": NOW.isoformat()})}
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["3", "2", "1"])
assert web["asked"] == [f"{base}/3", f"{base}/1"]
assert [(r["source_id"], r["description"], r["location"]) for r in filled] == [
("3", "three in full", "Copenhagen"), ("1", "one in full", "Copenhagen")]
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
def test_details_stop_at_the_cap_and_the_rest_wait(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
ids = [str(n) for n in range(5)]
web["answers"].update({f"{base}/{i}": f"full {i}".encode() for i in ids})
module = _fake_host(detail=_detail)
state: dict[str, str] = {}
filled = _details(monkeypatch, module, state, ids, cap=2)
assert [r["description"] for r in filled] == ["full 0", "full 1"]
web["asked"].clear()
filled = _details(monkeypatch, module, state, ids[2:], cap=2)
assert web["asked"] == [f"{base}/2", f"{base}/3"]
def test_a_detail_that_fails_skips_the_role_and_a_429_stops_the_board(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/1": b"one", f"{base}/2": ValueError("no such page shape"),
f"{base}/3": _status(429), f"{base}/4": b"four"})
state: dict[str, str] = {}
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3", "4"])
assert web["asked"] == [f"{base}/1", f"{base}/2", f"{base}/3"]
assert [r["source_id"] for r in filled] == ["1"]
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
# A 429 is not the role's fault.
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
@pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")])
def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
"""Two roles that always fail and a cap of two: the third is read on the
second pull, since roles never tried go before those that failed."""
"""Roles never tried go before those that failed, within the cap."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3", "4"]', f"{base}/1": failure,
f"{base}/2": failure, f"{base}/3": b"three", f"{base}/4": b"four"})
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
web["answers"].update({f"{base}/1": failure, f"{base}/2": failure,
f"{base}/3": b"three", f"{base}/4": b"four"})
module = _fake_host(detail=_detail)
_items, state = _run(monkeypatch, module)
state: dict[str, str] = {}
_details(monkeypatch, module, state, ["1", "2"])
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
web["asked"].clear()
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/3", f"{base}/4"]
assert [i.body for i in items] == ["", "", "three", "four"]
_details(monkeypatch, module, state, ["1", "2", "3", "4"], cap=2)
assert web["asked"] == [f"{base}/3", f"{base}/4"]
# With nothing new left, the failed ones are tried again, within the cap.
web["asked"].clear()
web["answers"][f"{base}/1"] = b"one at last"
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
assert [i.body for i in items] == ["one at last", "", "", ""]
filled = _details(monkeypatch, module, state, ["1", "2"], cap=2)
assert [r["description"] for r in filled] == ["one at last"]
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:3", "fake:acme:4"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
def test_a_429_on_a_detail_does_not_mark_the_role_as_failing(
def test_a_detail_answering_404_or_410_counts_as_read(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": _status(429)})
_items, state = _run(monkeypatch, _fake_host(detail=_detail))
web["answers"].update({f"{base}/2": _status(410), f"{base}/3": b"three"}) # 1 answers 404
state: dict[str, str] = {}
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3"])
assert [r["source_id"] for r in filled] == ["3"]
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
assert json.loads(state["ats:detailed:fake:acme"]) == []
def test_failed_ids_no_longer_listed_leave_the_failed_list(
def test_a_read_without_text_is_read_again_after_some_days(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""Finding 18: an Oracle requisition not yet published has an empty page
today and a full one next week."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]', f"{base}/2": ValueError("odd page")})
state = {"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
web["answers"][f"{base}/1"] = b""
module = _fake_host(detail=lambda get, slug, row: {
"description": get(f"{base}/{row['source_id']}").decode()})
state: dict[str, str] = {}
assert _details(monkeypatch, module, state, ["1"]) == []
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
assert json.loads(state["ats:detailempty:fake:acme"]) == {
"fake:acme:1": NOW.isoformat(timespec="seconds")}
web["asked"].clear()
later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS - 1)
assert _details(monkeypatch, module, state, ["1"], now=later) == []
assert web["asked"] == []
web["answers"][f"{base}/1"] = b"published now"
later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS)
[filled] = _details(monkeypatch, module, state, ["1"], now=later)
assert filled["description"] == "published now"
assert json.loads(state["ats:detailempty:fake:acme"]) == {}
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
def test_a_role_read_before_the_dates_were_kept_is_read_again(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"]["https://jobs.fake.example/acme/1"] = b"one"
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])}
[filled] = _details(monkeypatch, _fake_host(detail=_detail), state, ["1"])
assert filled["description"] == "one"
def test_detail_fills_only_the_fields_a_later_pull_keeps(
@@ -499,9 +534,6 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
caplog: pytest.LogCaptureFixture) -> None:
"""`salary_text`, `department` and `title` would be rewritten from the
listing on the next pull, so detail does not fill them."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1"]'})
def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
return {"description": "in full", "remote": "hybrid", "currency": "EUR",
"salary_min": 60000, "salary_max": 70000,
@@ -510,7 +542,8 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
"title": "Another title"}
caplog.set_level("DEBUG", logger=_runner.log.name)
[item], _state = _run(monkeypatch, _fake_host(detail=detail))
[row] = _details(monkeypatch, _fake_host(detail=detail), {}, ["1"])
item = ats.item_of(row, "fake:acme", "Acme")
assert (item.body, item.title, item.published_at) == (
"in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC))
assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"],
@@ -522,66 +555,6 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
"salary_max", "posted"}
def _paged_host() -> types.ModuleType:
"""A host `fake` that pages its listing, one id a page, and has `detail`."""
module: Any = _fake_host(detail=_detail)
del module.feed_url, module.parse
def page(get: Any, slug: str, cursor: Any) -> tuple[Any, list[dict[str, Any]], Any, Any]:
index = int(cursor or 0)
ids = json.loads(get(f"https://jobs.fake.example/{slug}/list?page={index}"))
return ids, [{"id": i} for i in ids], 2, index + 1 if ids else None
module.page = page
module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug)
assert not readers.problems(module, "fake")
return module
def test_a_paged_host_reads_every_page_then_each_role_once(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": b'["2"]',
f"{base}/1": b"one", f"{base}/2": b"two"})
items, state = _run(monkeypatch, _paged_host())
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1",
f"{base}/1", f"{base}/2"]
assert [i.body for i in items] == ["one", "two"]
assert state["ats:complete:fake:acme"] == "1"
web["asked"].clear()
items, state = _run(monkeypatch, _paged_host(), state)
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1"]
assert [i.body for i in items] == ["", ""]
def test_a_paged_listing_cut_short_keeps_the_ids_it_did_not_reach(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": _status(429),
f"{base}/1": b"one"})
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])}
_items, state = _run(monkeypatch, _paged_host(), state)
assert state["ats:complete:fake:acme"] == ""
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:3"]
def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""The consumer stops after the first role, so the second, read but not
stored, is read again on the next pull."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"})
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
roles: Any = source.fetch() # a generator, which `close` stops
assert next(roles).body == "one"
roles.close()
assert "ats:detailed:fake:acme" not in source.state
# ----------------------------------------------------------------- Lever EU
def test_a_lever_slug_may_name_the_eu_instance() -> None:
+11 -9
View File
@@ -1053,7 +1053,9 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
assert batch is not None
jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id)
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
assert stages == list(cascade.CASCADES["pull"])
# Enrich found no board with role pages, so the embed and score behind it
# are passed over.
assert stages == ["pull", "embed", "score", "enrich", "extract", "score", "summarise"]
work = [j.kind for j in jobs if j.kind != FINAL]
# The `pull` job and the chain it asked for are one cascade, so the Jobs
# tab shows a scheduled pull as one run. Extract and summarise are off and
@@ -1107,10 +1109,10 @@ def test_the_model_batches_are_isolated_with_a_timeout() -> None:
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
"""By position, not by name: `score` is in the chain twice, once before
extract and once after, and the second must lead on to summarise."""
assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "extract", "score",
"summarise")
assert [cascade.follows("pull", at) for at in range(6)] == \
["embed", "score", "extract", "score", "summarise", ""]
assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "enrich", "embed", "score",
"extract", "score", "summarise")
assert [cascade.follows("pull", at) for at in range(9)] == \
["embed", "score", "enrich", "embed", "score", "extract", "score", "summarise", ""]
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 9) == ""
assert cascade.follows("nope", 0) == ""
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
@@ -1124,10 +1126,10 @@ def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None:
made.append(t)
return t
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 2},
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 5},
context=Context())
assert out == {"task": "score", "next": "extract"}
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 3}
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 4},
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 6}
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 7},
context=Context())
assert out["next"] == "summarise" and made[1].carries["at"] == 5
assert out["next"] == "summarise" and made[1].carries["at"] == 8