Merge branch 'wp-g-enrich'
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
38a80d5b55
10 files changed
+956
-219
No files matched your search
@@ -0,0 +1,368 @@
|
||||
"""The `enrich` stage: the best roles without text, read on their own pages
|
||||
after scoring. Which roles and in what order, the budgets, what is stored,
|
||||
the cascade around it, and what triggers it. No network: a fake host stands
|
||||
in, answering from a dict."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import numpy as np
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app.config import Config, ServerConfig # noqa: E402
|
||||
from app.jobs import cascade, embed, enrich, stages # noqa: E402
|
||||
from app.main import create_app # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
from desk.models import Item # noqa: E402
|
||||
from pipeline import FINAL, JobQueue, Jobs, Tally # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
|
||||
BASE = "https://jobs.fake.example"
|
||||
|
||||
|
||||
def _fake_host(name: str = "fake", *, detail: bool = True) -> types.ModuleType:
|
||||
"""A host whose role pages are `BASE/<slug>/<id>`, answered by `web`."""
|
||||
module: Any = types.ModuleType(name)
|
||||
module.NAME, module.LABEL, module.PATTERNS = name, name.title(), ()
|
||||
module.slug_ok = lambda slug: slug.isalnum()
|
||||
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
||||
module.careers_url = lambda slug: f"{BASE}/{slug}"
|
||||
module.feed_url = lambda slug: f"{BASE}/{slug}/list"
|
||||
module.parse = lambda body, slug: []
|
||||
if detail:
|
||||
module.detail = lambda get, slug, row: {
|
||||
"description": get(f"{BASE}/{slug}/{row['source_id']}").decode(),
|
||||
"location": "Copenhagen"}
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
|
||||
"""`ats._get` answering from `answers` (URL -> body), 404 otherwise,
|
||||
remembering what it was asked; hosts `fake` and `plain` (no detail)."""
|
||||
answers: dict[str, Any] = {}
|
||||
asked: list[str] = []
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None):
|
||||
asked.append(url)
|
||||
answer = answers.get(url)
|
||||
if answer is None:
|
||||
request = httpx.Request("GET", url)
|
||||
raise httpx.HTTPStatusError("404", request=request,
|
||||
response=httpx.Response(404, request=request))
|
||||
return answer
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host())
|
||||
monkeypatch.setitem(readers.HOSTS, "plain", _fake_host("plain", detail=False))
|
||||
return {"answers": answers, "asked": asked}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> Store:
|
||||
return Store(tmp_path / "roledesk.db")
|
||||
|
||||
|
||||
def config(tmp_path: Path, **enrich_settings: Any) -> Config:
|
||||
return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"},
|
||||
enrich=enrich_settings)
|
||||
|
||||
|
||||
def board(store: Store, target: str) -> Any:
|
||||
host = target.partition(":")[0]
|
||||
return store.add_subscription("ats", target, label=target, company=target, board=host)
|
||||
|
||||
|
||||
def role(store: Store, target: str, native: str, *, body: str = "",
|
||||
title: str = "Backend Engineer") -> str:
|
||||
"""A role of board `target` as a pull stores it: without text on a host
|
||||
with `detail`."""
|
||||
sub = next(s for s in store.subscriptions(source="ats") if s.target == target)
|
||||
host, _, slug = target.partition(":")
|
||||
item = Item(source_id=f"{target}:{native}", location=f"{BASE}/{slug}/{native}",
|
||||
title=title, body=body, published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
||||
raw={"company": target, "place": "2 Locations", "board": host, "slug": slug})
|
||||
item.source = "ats"
|
||||
store.upsert_role(item, sub)
|
||||
return item.uid
|
||||
|
||||
|
||||
def scored(store: Store, name: str, scores: dict[str, float], *,
|
||||
gated: tuple[str, ...] = ()) -> Any:
|
||||
agent = store.save_agent(name, f"{name} python backend")
|
||||
store.replace_scores(agent.id, [(uid, value, "gated" if uid in gated else "")
|
||||
for uid, value in scores.items()])
|
||||
return agent
|
||||
|
||||
|
||||
# -------------------------------------------------------------- which roles
|
||||
|
||||
def test_roles_go_best_first_by_their_best_agent_score(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
board(store, "fake:acme")
|
||||
board(store, "fake:beta")
|
||||
a1, a2 = role(store, "fake:acme", "1"), role(store, "fake:acme", "2")
|
||||
b1 = role(store, "fake:beta", "1")
|
||||
scored(store, "One", {a1: 0.6, a2: 0.9, b1: 0.55})
|
||||
scored(store, "Two", {a1: 0.95, b1: 0.7})
|
||||
|
||||
assert [(r["uid"], r["score"]) for r in store.enrich_waiting(500, 0.5)] == [
|
||||
(a1, 0.95), (a2, 0.9), (b1, 0.7)]
|
||||
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [a1, a2]),
|
||||
("fake:beta", [b1])]
|
||||
|
||||
|
||||
def test_a_role_no_agent_ranks_high_enough_gets_no_call(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
board(store, "fake:acme")
|
||||
low, gated, deep, kept = (role(store, "fake:acme", n) for n in ("1", "2", "3", "4"))
|
||||
scored(store, "One", {low: 0.4, gated: 0.0, deep: 0.8, kept: 0.9}, gated=(gated,))
|
||||
# `top` 1 per agent: only the best ungated role of each agent is in reach.
|
||||
assert enrich.plan(config(tmp_path, top=1), store) == [("fake:acme", [kept])]
|
||||
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [kept, deep])]
|
||||
# A disabled agent ranks nothing.
|
||||
agent = store.agents()[0]
|
||||
store.save_agent(agent.name, gates=agent.gates, weights=agent.weights, enabled=False,
|
||||
agent_id=agent.id, limits=agent.limits, prompts=agent.prompts)
|
||||
assert enrich.plan(config(tmp_path), store) == []
|
||||
|
||||
|
||||
def test_roles_with_text_hosts_without_pages_and_failing_boards_are_left_out(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
for target in ("fake:acme", "plain:acme", "fake:down"):
|
||||
board(store, target)
|
||||
wanted = role(store, "fake:acme", "1")
|
||||
has_text = role(store, "fake:acme", "2", body="Python in full")
|
||||
plain = role(store, "plain:acme", "1")
|
||||
down = role(store, "fake:down", "1")
|
||||
store.state["ats:failing:fake:down"] = json.dumps({"since": "2026-09-29", "reason": "500"})
|
||||
scored(store, "One", {wanted: 0.9, has_text: 0.95, plain: 0.9, down: 0.9})
|
||||
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [wanted])]
|
||||
|
||||
|
||||
def test_a_role_read_already_waits_for_the_retry_and_a_failed_one_goes_last(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
board(store, "fake:acme")
|
||||
read, failed, fresh = (role(store, "fake:acme", n) for n in ("1", "2", "3"))
|
||||
now = datetime.now(UTC).isoformat(timespec="seconds")
|
||||
store.state["ats:detailed:fake:acme"] = json.dumps(["fake:acme:1"])
|
||||
store.state["ats:detailempty:fake:acme"] = json.dumps({"fake:acme:1": now})
|
||||
store.state["ats:detailfailed:fake:acme"] = json.dumps(["fake:acme:2"])
|
||||
scored(store, "One", {read: 0.9, failed: 0.8, fresh: 0.6})
|
||||
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [fresh, failed])]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ budgets
|
||||
|
||||
def test_the_budgets_per_board_and_per_run(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
board(store, "fake:acme")
|
||||
board(store, "fake:beta")
|
||||
acme = [role(store, "fake:acme", str(n)) for n in range(4)]
|
||||
beta = [role(store, "fake:beta", str(n)) for n in range(4)]
|
||||
# Interleaved: acme 0.99, beta 0.98, acme 0.97, ...
|
||||
scored(store, "One", {uid: 0.99 - 0.01 * n
|
||||
for n, uid in enumerate(u for pair in zip(acme, beta, strict=True)
|
||||
for u in pair)})
|
||||
|
||||
assert enrich.plan(config(tmp_path, per_board=2), store) == [
|
||||
("fake:acme", acme[:2]), ("fake:beta", beta[:2])]
|
||||
assert enrich.plan(config(tmp_path, per_run=3), store) == [
|
||||
("fake:acme", acme[:2]), ("fake:beta", beta[:1])]
|
||||
assert enrich.plan(config(tmp_path, per_board=1, per_run=5), store) == [
|
||||
("fake:acme", acme[:1]), ("fake:beta", beta[:1])]
|
||||
|
||||
|
||||
def test_the_stage_off_finds_nothing(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
board(store, "fake:acme")
|
||||
scored(store, "One", {role(store, "fake:acme", "1"): 0.9})
|
||||
assert len(enrich.batches(config(tmp_path), store)) == 1
|
||||
assert enrich.batches(config(tmp_path, enabled=False), store) == []
|
||||
stages.set_stage(store, "enrich", True)
|
||||
assert len(enrich.batches(config(tmp_path, enabled=False), store)) == 1
|
||||
stages.set_stage(store, "enrich", False)
|
||||
assert enrich.batches(config(tmp_path), store) == []
|
||||
|
||||
|
||||
# ------------------------------------------------------------ what is stored
|
||||
|
||||
def test_a_batch_stores_what_the_pages_filled_and_drops_those_vectors(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
board(store, "fake:acme")
|
||||
one, two, gone = (role(store, "fake:acme", n) for n in ("1", "2", "3"))
|
||||
store.put_vectors("fake", [(uid, np.ones(3, dtype=np.float32)) for uid in (one, two, gone)])
|
||||
web["answers"].update({f"{BASE}/acme/1": b"Python in full", f"{BASE}/acme/2": b"Go too"})
|
||||
|
||||
out = enrich.run_batch(config(tmp_path), store, target="fake:acme", uids=[two, gone, one])
|
||||
|
||||
assert out == {"target": "fake:acme", "read": 3, "filled": 2, "texts": 2}
|
||||
assert web["asked"] == [f"{BASE}/acme/2", f"{BASE}/acme/3", f"{BASE}/acme/1"]
|
||||
first = store.role(one)
|
||||
assert first is not None and first.description == "Python in full"
|
||||
assert first.place == "Copenhagen" and first.title == "Backend Engineer"
|
||||
assert set(store.vectors("fake")) == {gone}
|
||||
assert json.loads(store.state["ats:detailed:fake:acme"]) == [
|
||||
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
||||
# A later pull of the same listing keeps the text and the page's place.
|
||||
item = ats.item_of({"source_id": "1", "title": "Backend Engineer", "url": f"{BASE}/acme/1",
|
||||
"description": "", "location": "", "places": [], "remote": None},
|
||||
"fake:acme", "fake:acme")
|
||||
item.source = "ats"
|
||||
store.upsert_role(item, store.subscriptions(source="ats")[0])
|
||||
again = store.role(one)
|
||||
assert again is not None and again.description == "Python in full"
|
||||
assert again.place == "Copenhagen"
|
||||
|
||||
|
||||
def test_a_board_switched_off_since_the_plan_reads_nothing(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
sub = board(store, "fake:acme")
|
||||
uid = role(store, "fake:acme", "1")
|
||||
store.update_subscription(sub.id, fetch=0)
|
||||
assert enrich.run_batch(config(tmp_path), store, target="fake:acme", uids=[uid])["filled"] == 0
|
||||
assert web["asked"] == []
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ cascade
|
||||
|
||||
class _Context:
|
||||
def __init__(self) -> None:
|
||||
self.made: list[Any] = []
|
||||
|
||||
def task(self, t: Any) -> Any:
|
||||
self.made.append(t)
|
||||
return t
|
||||
|
||||
|
||||
def test_nothing_filled_passes_over_the_embed_and_score_behind_enrich() -> None:
|
||||
context = _Context()
|
||||
empty = Tally(results=({"filled": 0},))
|
||||
out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 3},
|
||||
context=context)
|
||||
assert out["next"] == "extract" and context.made[0].carries["at"] == 6
|
||||
|
||||
out = cascade.step(None, None, Tally(results=({"filled": 2}, {"filled": 0})),
|
||||
{"cascade": "pull", "task": "enrich", "at": 3}, context=context)
|
||||
assert out["next"] == "embed" and context.made[1].carries["at"] == 4
|
||||
|
||||
out = cascade.step(None, None, empty, {"cascade": "enrich", "task": "enrich", "at": 0},
|
||||
context=context)
|
||||
assert out["next"] == "" and len(context.made) == 2
|
||||
|
||||
|
||||
class _Embedder:
|
||||
def encode(self, texts: list[str]) -> np.ndarray:
|
||||
return np.vstack([np.asarray([t.lower().count("python"), 0.1], dtype=np.float32)
|
||||
for t in texts])
|
||||
|
||||
|
||||
def test_the_enrich_job_reads_the_pages_then_embeds_and_scores_those_roles(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any],
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(embed, "load_embedder", lambda model: _Embedder())
|
||||
board(store, "fake:acme")
|
||||
one, two = role(store, "fake:acme", "1"), role(store, "fake:acme", "2")
|
||||
store.put_vectors("fake", [(uid, np.ones(2, dtype=np.float32)) for uid in (one, two)])
|
||||
agent = scored(store, "One", {one: 0.9, two: 0.1})
|
||||
web["answers"][f"{BASE}/acme/1"] = b"Python in full"
|
||||
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config(tmp_path), store,
|
||||
workers={"cpu": 1, "http": 1, "llm": 1})
|
||||
|
||||
started = queue.enqueue("enrich")
|
||||
queue.drain(timeout=60)
|
||||
|
||||
assert started is not None
|
||||
jobs = sorted(queue.jobs.of_cascade(queue.jobs.last(enrich.KIND).cascade_id), # type: ignore[union-attr]
|
||||
key=lambda j: j.id)
|
||||
assert [j.result.get("task") for j in jobs if j.kind == FINAL] == ["enrich", "embed", "score"]
|
||||
assert [j.kind for j in jobs if j.kind != FINAL] == ["enrich", enrich.KIND, "embed", "score"]
|
||||
assert web["asked"] == [f"{BASE}/acme/1"]
|
||||
embedded = next(j for j in jobs if j.kind == "embed")
|
||||
assert embedded.result["embedded"] == 1
|
||||
assert set(store.vectors("fake")) == {one, two}
|
||||
assert store.scores(agent.id)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- triggers
|
||||
|
||||
@pytest.fixture
|
||||
def app(tmp_path: Path) -> FastAPI:
|
||||
return create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
|
||||
database=tmp_path / "roledesk.db"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(app: FastAPI) -> Iterator[TestClient]:
|
||||
with TestClient(app) as c:
|
||||
yield c
|
||||
|
||||
|
||||
ANY_STATE = ("queued", "running", "done", "failed", "cancelled")
|
||||
|
||||
|
||||
def _enrich_behind_a_score(queue: Any) -> Any:
|
||||
found = queue.jobs.last("enrich", states=ANY_STATE)
|
||||
assert found is not None
|
||||
assert [queue.jobs.get(i).kind for i in found.depends_on] == ["score"]
|
||||
assert found.after == "done"
|
||||
return found
|
||||
|
||||
|
||||
def test_saving_an_agent_queues_a_score_then_enrich(client: TestClient, app: FastAPI) -> None:
|
||||
res = client.post("/api/agents", json={"name": "Backend", "intent": "python backend"})
|
||||
assert res.status_code == 200
|
||||
_enrich_behind_a_score(app.state.queue)
|
||||
res = client.post("/api/agents", json={"id": res.json()["id"], "name": "Backend",
|
||||
"intent": "go backend"})
|
||||
assert res.status_code == 200
|
||||
|
||||
|
||||
def test_a_company_affinity_queues_a_score_then_enrich(
|
||||
client: TestClient, app: FastAPI) -> None:
|
||||
store: Store = app.state.store
|
||||
store.save_agent("Backend", "python backend")
|
||||
store.add_company("Acme")
|
||||
res = client.post("/api/companies/acme/affinity", json={"affinity": 2})
|
||||
assert res.status_code == 200
|
||||
found = _enrich_behind_a_score(app.state.queue)
|
||||
assert found.depends_on == [res.json()["score_job"]]
|
||||
|
||||
|
||||
def test_one_score_is_one_enrich(app: FastAPI) -> None:
|
||||
queue = app.state.queue
|
||||
score = queue.enqueue("score")
|
||||
first = cascade.enrich_after(queue, score)
|
||||
assert cascade.enrich_after(queue, score).id == first.id
|
||||
assert cascade.enrich_after(queue, None) is None
|
||||
|
||||
|
||||
def test_a_vote_queues_neither_score_nor_enrich(client: TestClient, app: FastAPI) -> None:
|
||||
store: Store = app.state.store
|
||||
store.save_agent("Backend", "python backend")
|
||||
item = Item(source_id="fake:acme:1", location=f"{BASE}/acme/1", title="Backend Engineer",
|
||||
body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw={"company": "Acme"})
|
||||
item.source = "ats"
|
||||
store.upsert_role(item)
|
||||
|
||||
assert client.post(f"/api/roles/{item.uid}/vote", json={"value": 1}).status_code == 200
|
||||
assert client.delete(f"/api/roles/{item.uid}/vote").status_code == 200
|
||||
|
||||
queue = app.state.queue
|
||||
assert queue.jobs.last("enrich", states=ANY_STATE) is None
|
||||
assert queue.jobs.last("score", states=ANY_STATE) is None
|
||||
+136
-163
@@ -1,6 +1,7 @@
|
||||
"""The host modules of `plugins/readers/` and what `plugins/ats.py` does for
|
||||
every host: the registry and its contract check, the fetch a host is handed,
|
||||
the per-role detail reads, and Lever's EU instance. No network —
|
||||
the per-role detail reads the `enrich` stage makes, and Lever's EU
|
||||
instance. No network —
|
||||
a fake host stands in where a behaviour belongs to no real host yet."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -11,7 +12,7 @@ import subprocess
|
||||
import sys
|
||||
import types
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -308,34 +309,26 @@ def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
|
||||
|
||||
# ---------------------------------------------------------------- detail
|
||||
|
||||
def test_detail_is_read_for_unread_roles_only(
|
||||
def test_a_pull_reads_no_role_page(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
"""The pages are `enrich`'s: the pull reads the listing only, and sends
|
||||
every row of a host with `detail` without a description."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one in full",
|
||||
f"{base}/2": b"two in full", f"{base}/3": b"three in full"})
|
||||
module = _fake_host(detail=_detail)
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"])}
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/3"]
|
||||
assert [(i.source_id, i.body) for i in items] == [
|
||||
("fake:acme:1", "one in full"), ("fake:acme:2", ""), ("fake:acme:3", "three in full")]
|
||||
# What detail returns replaces the listing's field; a role not read keeps the listing's.
|
||||
assert items[0].raw["place"] == "Copenhagen" and items[1].raw["place"] == ""
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
||||
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one in full"})
|
||||
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
assert web["asked"] == [f"{base}/list"]
|
||||
assert [(i.source_id, i.body) for i in items] == [("fake:acme:1", ""), ("fake:acme:2", "")]
|
||||
assert "ats:detailed:fake:acme" not in state
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
|
||||
|
||||
def test_a_role_read_on_an_earlier_pull_keeps_the_place_its_page_gave(
|
||||
def test_a_role_read_on_its_page_keeps_the_place_its_page_gave(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
"""The listing states a place on every pull, and a filled value replaces
|
||||
a filled column, so for a role already read it is withheld."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": b"two in full",
|
||||
f"{base}/3": RuntimeError("down")})
|
||||
module = _fake_host(detail=lambda get, slug, row: {
|
||||
"description": get(f"{base}/{row['source_id']}").decode()})
|
||||
web["answers"].update({f"{base}/list": b'["1", "3"]'})
|
||||
module = _fake_host(detail=_detail)
|
||||
listing = module.parse
|
||||
module.parse = lambda body, slug: [row | {"location": "2 Locations", "remote": "hybrid"}
|
||||
for row in listing(body, slug)]
|
||||
@@ -345,70 +338,25 @@ def test_a_role_read_on_an_earlier_pull_keeps_the_place_its_page_gave(
|
||||
|
||||
by_id = {i.source_id: (i.body, i.raw["place"], i.raw["remote"]) for i in items}
|
||||
assert by_id["fake:acme:1"] == ("", "", None)
|
||||
# Read now, and its page stated neither: the listing's stand.
|
||||
assert by_id["fake:acme:2"] == ("two in full", "2 Locations", "hybrid")
|
||||
# Never read: the listing's stand until its page is.
|
||||
assert by_id["fake:acme:3"] == ("", "2 Locations", "hybrid")
|
||||
|
||||
|
||||
def test_detail_reads_stop_at_the_cap_and_the_rest_wait(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
ids = [str(n) for n in range(5)]
|
||||
web["answers"].update({f"{base}/list": json.dumps(ids).encode()}
|
||||
| {f"{base}/{i}": f"full {i}".encode() for i in ids})
|
||||
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
|
||||
module = _fake_host(detail=_detail)
|
||||
items, state = _run(monkeypatch, module)
|
||||
assert [i.body for i in items] == ["full 0", "full 1", "", "", ""]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:0", "fake:acme:1"]
|
||||
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/2", f"{base}/3"]
|
||||
assert [i.body for i in items] == ["", "", "full 2", "full 3", ""]
|
||||
|
||||
|
||||
def test_a_detail_that_fails_skips_the_role_and_not_the_board(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
|
||||
f"{base}/2": ValueError("no such page shape"), f"{base}/3": b"three"})
|
||||
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
assert [(i.source_id, i.body) for i in items] == [
|
||||
("fake:acme:1", "one"), ("fake:acme:2", ""), ("fake:acme:3", "three")]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:3"]
|
||||
assert "ats:failing:fake:acme" not in state and state["ats:ok:fake:acme"] == "r"
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
|
||||
|
||||
def test_a_429_on_a_detail_stops_the_reads_for_the_pull(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
|
||||
f"{base}/2": _status(429), f"{base}/3": b"three"})
|
||||
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
|
||||
assert [i.body for i in items] == ["one", "", ""]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
|
||||
|
||||
def test_ids_no_longer_listed_leave_the_detailed_list(
|
||||
def test_ids_no_longer_listed_leave_the_detail_lists(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"]),
|
||||
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"]
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
||||
|
||||
|
||||
def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
"""A listing cut short says nothing about the roles it left out, so their
|
||||
ids stay, and the next pull does not read them in detail again."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||
ids stay, and they are not read on their pages again."""
|
||||
monkeypatch.setattr(_runner, "read_listing", lambda *args, **kwargs: _runner.Listing(
|
||||
readers.HOSTS["fake"].parse(b'["2"]', "acme"), False, ["2"]))
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||
@@ -417,6 +365,19 @@ def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
|
||||
assert state["ats:complete:fake:acme"] == ""
|
||||
|
||||
|
||||
def test_a_pull_stopped_half_way_through_a_board_drops_no_detail_ids(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
|
||||
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
||||
source.state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])}
|
||||
roles: Any = source.fetch() # a generator, which `close` stops
|
||||
assert next(roles).body == ""
|
||||
roles.close()
|
||||
assert json.loads(source.state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
||||
|
||||
|
||||
def test_fetch_board_reads_the_listing_only(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
@@ -435,63 +396,137 @@ def test_a_host_without_detail_keeps_the_listings_description(
|
||||
assert "ats:detailed:fake:acme" not in state
|
||||
|
||||
|
||||
def test_a_detail_answering_404_or_410_counts_as_read(
|
||||
# -------------------------------------------------- read_board_details
|
||||
|
||||
NOW = datetime(2026, 9, 30, 12, tzinfo=UTC)
|
||||
|
||||
|
||||
def _details(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType,
|
||||
state: dict[str, str], ids: list[str], **kwargs: Any) -> list[dict[str, Any]]:
|
||||
"""`ats.read_board_details` over the fake host's roles `ids`, stored
|
||||
without text, in that order."""
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", module)
|
||||
rows = [row | {"description": ""}
|
||||
for row in module.parse(json.dumps(ids).encode(), "acme")]
|
||||
return ats.read_board_details(state, "fake:acme", rows, client=object(), # type: ignore[arg-type]
|
||||
now=kwargs.pop("now", NOW), **kwargs)
|
||||
|
||||
|
||||
def test_details_are_read_in_the_order_given_for_unread_roles_only(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": _status(410),
|
||||
f"{base}/3": b"three"}) # 1 answers 404
|
||||
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
assert [i.body for i in items] == ["", "", "three"]
|
||||
web["answers"].update({f"{base}/1": b"one in full", f"{base}/2": b"two in full",
|
||||
f"{base}/3": b"three in full"})
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
|
||||
"ats:detailempty:fake:acme": json.dumps({"fake:acme:2": NOW.isoformat()})}
|
||||
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["3", "2", "1"])
|
||||
|
||||
assert web["asked"] == [f"{base}/3", f"{base}/1"]
|
||||
assert [(r["source_id"], r["description"], r["location"]) for r in filled] == [
|
||||
("3", "three in full", "Copenhagen"), ("1", "one in full", "Copenhagen")]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
|
||||
|
||||
|
||||
def test_details_stop_at_the_cap_and_the_rest_wait(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
ids = [str(n) for n in range(5)]
|
||||
web["answers"].update({f"{base}/{i}": f"full {i}".encode() for i in ids})
|
||||
module = _fake_host(detail=_detail)
|
||||
state: dict[str, str] = {}
|
||||
filled = _details(monkeypatch, module, state, ids, cap=2)
|
||||
assert [r["description"] for r in filled] == ["full 0", "full 1"]
|
||||
|
||||
web["asked"].clear()
|
||||
filled = _details(monkeypatch, module, state, ids[2:], cap=2)
|
||||
assert web["asked"] == [f"{base}/2", f"{base}/3"]
|
||||
|
||||
|
||||
def test_a_detail_that_fails_skips_the_role_and_a_429_stops_the_board(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/1": b"one", f"{base}/2": ValueError("no such page shape"),
|
||||
f"{base}/3": _status(429), f"{base}/4": b"four"})
|
||||
state: dict[str, str] = {}
|
||||
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3", "4"])
|
||||
assert web["asked"] == [f"{base}/1", f"{base}/2", f"{base}/3"]
|
||||
assert [r["source_id"] for r in filled] == ["1"]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
||||
# A 429 is not the role's fault.
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")])
|
||||
def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
|
||||
"""Two roles that always fail and a cap of two: the third is read on the
|
||||
second pull, since roles never tried go before those that failed."""
|
||||
"""Roles never tried go before those that failed, within the cap."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3", "4"]', f"{base}/1": failure,
|
||||
f"{base}/2": failure, f"{base}/3": b"three", f"{base}/4": b"four"})
|
||||
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
|
||||
web["answers"].update({f"{base}/1": failure, f"{base}/2": failure,
|
||||
f"{base}/3": b"three", f"{base}/4": b"four"})
|
||||
module = _fake_host(detail=_detail)
|
||||
_items, state = _run(monkeypatch, module)
|
||||
state: dict[str, str] = {}
|
||||
_details(monkeypatch, module, state, ["1", "2"])
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
||||
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/3", f"{base}/4"]
|
||||
assert [i.body for i in items] == ["", "", "three", "four"]
|
||||
_details(monkeypatch, module, state, ["1", "2", "3", "4"], cap=2)
|
||||
assert web["asked"] == [f"{base}/3", f"{base}/4"]
|
||||
|
||||
# With nothing new left, the failed ones are tried again, within the cap.
|
||||
web["asked"].clear()
|
||||
web["answers"][f"{base}/1"] = b"one at last"
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
|
||||
assert [i.body for i in items] == ["one at last", "", "", ""]
|
||||
filled = _details(monkeypatch, module, state, ["1", "2"], cap=2)
|
||||
assert [r["description"] for r in filled] == ["one at last"]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||
"fake:acme:1", "fake:acme:3", "fake:acme:4"]
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
||||
|
||||
|
||||
def test_a_429_on_a_detail_does_not_mark_the_role_as_failing(
|
||||
def test_a_detail_answering_404_or_410_counts_as_read(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": _status(429)})
|
||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
web["answers"].update({f"{base}/2": _status(410), f"{base}/3": b"three"}) # 1 answers 404
|
||||
state: dict[str, str] = {}
|
||||
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3"])
|
||||
assert [r["source_id"] for r in filled] == ["3"]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == []
|
||||
|
||||
|
||||
def test_failed_ids_no_longer_listed_leave_the_failed_list(
|
||||
def test_a_read_without_text_is_read_again_after_some_days(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
"""Finding 18: an Oracle requisition not yet published has an empty page
|
||||
today and a full one next week."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["2"]', f"{base}/2": ValueError("odd page")})
|
||||
state = {"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
||||
web["answers"][f"{base}/1"] = b""
|
||||
module = _fake_host(detail=lambda get, slug, row: {
|
||||
"description": get(f"{base}/{row['source_id']}").decode()})
|
||||
state: dict[str, str] = {}
|
||||
assert _details(monkeypatch, module, state, ["1"]) == []
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
||||
assert json.loads(state["ats:detailempty:fake:acme"]) == {
|
||||
"fake:acme:1": NOW.isoformat(timespec="seconds")}
|
||||
|
||||
web["asked"].clear()
|
||||
later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS - 1)
|
||||
assert _details(monkeypatch, module, state, ["1"], now=later) == []
|
||||
assert web["asked"] == []
|
||||
|
||||
web["answers"][f"{base}/1"] = b"published now"
|
||||
later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS)
|
||||
[filled] = _details(monkeypatch, module, state, ["1"], now=later)
|
||||
assert filled["description"] == "published now"
|
||||
assert json.loads(state["ats:detailempty:fake:acme"]) == {}
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
||||
|
||||
|
||||
def test_a_role_read_before_the_dates_were_kept_is_read_again(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"]["https://jobs.fake.example/acme/1"] = b"one"
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])}
|
||||
[filled] = _details(monkeypatch, _fake_host(detail=_detail), state, ["1"])
|
||||
assert filled["description"] == "one"
|
||||
|
||||
|
||||
def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
||||
@@ -499,9 +534,6 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
"""`salary_text`, `department` and `title` would be rewritten from the
|
||||
listing on the next pull, so detail does not fill them."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1"]'})
|
||||
|
||||
def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {"description": "in full", "remote": "hybrid", "currency": "EUR",
|
||||
"salary_min": 60000, "salary_max": 70000,
|
||||
@@ -510,7 +542,8 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
||||
"title": "Another title"}
|
||||
|
||||
caplog.set_level("DEBUG", logger=_runner.log.name)
|
||||
[item], _state = _run(monkeypatch, _fake_host(detail=detail))
|
||||
[row] = _details(monkeypatch, _fake_host(detail=detail), {}, ["1"])
|
||||
item = ats.item_of(row, "fake:acme", "Acme")
|
||||
assert (item.body, item.title, item.published_at) == (
|
||||
"in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC))
|
||||
assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"],
|
||||
@@ -522,66 +555,6 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
||||
"salary_max", "posted"}
|
||||
|
||||
|
||||
def _paged_host() -> types.ModuleType:
|
||||
"""A host `fake` that pages its listing, one id a page, and has `detail`."""
|
||||
module: Any = _fake_host(detail=_detail)
|
||||
del module.feed_url, module.parse
|
||||
|
||||
def page(get: Any, slug: str, cursor: Any) -> tuple[Any, list[dict[str, Any]], Any, Any]:
|
||||
index = int(cursor or 0)
|
||||
ids = json.loads(get(f"https://jobs.fake.example/{slug}/list?page={index}"))
|
||||
return ids, [{"id": i} for i in ids], 2, index + 1 if ids else None
|
||||
|
||||
module.page = page
|
||||
module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug)
|
||||
assert not readers.problems(module, "fake")
|
||||
return module
|
||||
|
||||
|
||||
def test_a_paged_host_reads_every_page_then_each_role_once(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": b'["2"]',
|
||||
f"{base}/1": b"one", f"{base}/2": b"two"})
|
||||
items, state = _run(monkeypatch, _paged_host())
|
||||
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1",
|
||||
f"{base}/1", f"{base}/2"]
|
||||
assert [i.body for i in items] == ["one", "two"]
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, _paged_host(), state)
|
||||
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1"]
|
||||
assert [i.body for i in items] == ["", ""]
|
||||
|
||||
|
||||
def test_a_paged_listing_cut_short_keeps_the_ids_it_did_not_reach(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": _status(429),
|
||||
f"{base}/1": b"one"})
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
|
||||
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])}
|
||||
_items, state = _run(monkeypatch, _paged_host(), state)
|
||||
assert state["ats:complete:fake:acme"] == ""
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:3"]
|
||||
|
||||
|
||||
def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
"""The consumer stops after the first role, so the second, read but not
|
||||
stored, is read again on the next pull."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"})
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
|
||||
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
||||
roles: Any = source.fetch() # a generator, which `close` stops
|
||||
assert next(roles).body == "one"
|
||||
roles.close()
|
||||
assert "ats:detailed:fake:acme" not in source.state
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- Lever EU
|
||||
|
||||
def test_a_lever_slug_may_name_the_eu_instance() -> None:
|
||||
|
||||
+11
-9
@@ -1053,7 +1053,9 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
|
||||
assert batch is not None
|
||||
jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id)
|
||||
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
|
||||
assert stages == list(cascade.CASCADES["pull"])
|
||||
# Enrich found no board with role pages, so the embed and score behind it
|
||||
# are passed over.
|
||||
assert stages == ["pull", "embed", "score", "enrich", "extract", "score", "summarise"]
|
||||
work = [j.kind for j in jobs if j.kind != FINAL]
|
||||
# The `pull` job and the chain it asked for are one cascade, so the Jobs
|
||||
# tab shows a scheduled pull as one run. Extract and summarise are off and
|
||||
@@ -1107,10 +1109,10 @@ def test_the_model_batches_are_isolated_with_a_timeout() -> None:
|
||||
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
|
||||
"""By position, not by name: `score` is in the chain twice, once before
|
||||
extract and once after, and the second must lead on to summarise."""
|
||||
assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "extract", "score",
|
||||
"summarise")
|
||||
assert [cascade.follows("pull", at) for at in range(6)] == \
|
||||
["embed", "score", "extract", "score", "summarise", ""]
|
||||
assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "enrich", "embed", "score",
|
||||
"extract", "score", "summarise")
|
||||
assert [cascade.follows("pull", at) for at in range(9)] == \
|
||||
["embed", "score", "enrich", "embed", "score", "extract", "score", "summarise", ""]
|
||||
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 9) == ""
|
||||
assert cascade.follows("nope", 0) == ""
|
||||
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
|
||||
@@ -1124,10 +1126,10 @@ def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
made.append(t)
|
||||
return t
|
||||
|
||||
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 2},
|
||||
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 5},
|
||||
context=Context())
|
||||
assert out == {"task": "score", "next": "extract"}
|
||||
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 3}
|
||||
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 4},
|
||||
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 6}
|
||||
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 7},
|
||||
context=Context())
|
||||
assert out["next"] == "summarise" and made[1].carries["at"] == 5
|
||||
assert out["next"] == "summarise" and made[1].carries["at"] == 8
|
||||
Reference in new issue
Block a user