Extract is bounded to the ranking: pull → embed → score → extract → score → summarise

The free stages run first and their score bounds the two model stages.
Extract reads only the best extract.per_agent roles of each agent at
extract.min_score or more, then score runs again so the gates read what it
filled; summarise gets a min_score of its own. The cascade steps by
position, since score is in it twice.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-28 15:26:58 +02:00
1 parent c4f2bcc0c9
commit 7db77e83a3
15 files changed
+215 -89

No files matched your search

+44 -2
View File
@@ -57,6 +57,15 @@ def config(tmp_path: Path) -> Config:
return Config(database=str(tmp_path / "roledesk.db"), extract={"enabled": True})
def ranked(store: Store, *items: Item) -> None:
"""Score the roles for one enabled agent, best first, so they are in the
top extract reads. Every role given must already be stored."""
agents = store.agents()
agent_id = agents[0].id if agents else store.save_agent("Backend").id
store.replace_scores(agent_id, [(item.uid, 1.0 - n / 10, "")
for n, item in enumerate(items)])
def stub(monkeypatch: pytest.MonkeyPatch, reply: Any) -> Model:
model = Model(reply)
monkeypatch.setattr(agent, "backend", lambda settings: "api")
@@ -68,6 +77,7 @@ def test_a_batch_writes_the_columns_and_stamps_extracted_at(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
uid = posting("1").uid
store.upsert_role(posting("1"))
ranked(store, posting("1"))
stub(monkeypatch, lambda roles: [answer(
r["uid"], place="Berlin", country="de", remote="hybrid", seniority="senior",
salary_min=80000, salary_max=95000, currency="eur", function="engineering",
@@ -90,6 +100,7 @@ def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
item = posting("1", place="Berlin", remote="hybrid", salary_min=70000)
store.upsert_role(item)
ranked(store, item)
stub(monkeypatch, lambda roles: [answer(
r["uid"], place="Munich", remote="", salary_min=None, salary_max=90000,
country="DE") for r in roles])
@@ -105,6 +116,7 @@ def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
def test_the_next_upsert_keeps_what_extract_filled(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
store.upsert_role(posting("1"))
ranked(store, posting("1"))
stub(monkeypatch, lambda roles: [answer(r["uid"], country="DE", seniority="mid")
for r in roles])
modeltask.run(extract.TASK, config, store)
@@ -119,6 +131,7 @@ def test_closed_and_answered_roles_are_not_offered(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
for sid in ("open", "closed", "done"):
store.upsert_role(posting(sid))
ranked(store, posting("open"), posting("closed"), posting("done"))
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' "
"WHERE source_id = 'closed'")
@@ -134,6 +147,7 @@ def test_closed_and_answered_roles_are_not_offered(
def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
store.upsert_role(posting("1"))
ranked(store, posting("1"))
stub(monkeypatch, lambda roles: [answer("not-in-the-batch", country="DE")])
result = modeltask.run(extract.TASK, config, store)
@@ -141,11 +155,39 @@ def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged(
role = store.role(posting("1").uid)
assert role is not None and role.country == "" and role.extracted_at is None
assert result["rejected"] == ["not-in-the-batch"]
assert store.extract_pending() == 1
assert store.extract_pending(500) == 1
modeltask.run(extract.TASK, config, store)
modeltask.run(extract.TASK, config, store)
assert store.extract_pending() == 0, "given up after three unanswered replies"
assert store.extract_pending(500) == 0, "given up after three unanswered replies"
def test_only_the_best_of_each_agent_above_the_floor_is_read(
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""The bound that keeps extract cheap: the top `per_agent` of an enabled
agent's scores at `min_score` or more, whatever the boards produced. A
role nobody ranks is never sent; a batch dealt a uid finishes it."""
for sid in ("best", "second", "low", "unscored", "gated"):
store.upsert_role(posting(sid))
one = store.save_agent("One")
two = store.save_agent("Two", enabled=False)
store.replace_scores(one.id, [(posting("best").uid, 0.9, ""),
(posting("second").uid, 0.8, ""),
(posting("low").uid, 0.2, ""),
(posting("gated").uid, 0.95, "country unknown")])
store.replace_scores(two.id, [(posting("unscored").uid, 0.99, "")])
config = Config(database=str(tmp_path / "x.db"),
extract={"enabled": True, "per_agent": 1, "min_score": 0.5})
model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles])
assert store.extract_pending(1, 0.5) == 1
assert store.extract_pending(5, 0.5) == 2, "the floor holds the low one back"
assert store.extract_pending(5, 0.0) == 3, "the disabled agent's pick never counts"
modeltask.run(extract.TASK, config, store)
assert [[r["uid"] for r in batch] for batch in model.asked] == [[posting("best").uid]]
assert [r.uid for r in store.extract_waiting(10, 1, 0.5, uids=[posting("low").uid])] == \
[posting("low").uid], "a dealt uid is read whatever its rank"
def test_a_value_outside_the_vocabulary_is_dropped() -> None:
+32 -11
View File
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from desk.models import Item # noqa: E402
from pipeline import FINAL, SCHEDULES, TASKS, JobQueue, Jobs, RunContext # noqa: E402
from pipeline import FINAL, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, Tally # noqa: E402
from plugins.ats import ATSSource # noqa: E402
from app.config import AgentSeed, Config # noqa: E402
from app.jobs import cascade, embed, score # noqa: E402
@@ -438,7 +438,7 @@ def test_the_shipped_starter_searches_validate() -> None:
# ---------------------------------------------------------------- cascade
def test_the_pull_job_runs_pull_extract_embed_then_score(
def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
@@ -471,8 +471,9 @@ def test_the_pull_job_runs_pull_extract_embed_then_score(
assert stages == list(cascade.CASCADES["pull"])
work = [j.kind for j in jobs if j.kind != FINAL]
# The `pull` job and the chain it asked for are one cascade, so the Jobs
# tab shows a scheduled pull as one run. Extract is off and finds nothing.
assert work == ["pull", "pull_batch", "embed", "score"]
# tab shows a scheduled pull as one run. Extract and summarise are off and
# find nothing; score runs twice, before and after extract.
assert work == ["pull", "pull_batch", "embed", "score", "score"]
assert all(j.state == "done" for j in jobs)
assert len(store.scores(agent.id)) == 1
@@ -486,10 +487,30 @@ def test_pull_is_a_job_on_a_six_hour_schedule_and_the_stages_are_tasks() -> None
assert SCHEDULES["pull"].cadence == "every 6h"
def test_follows_walks_the_cascade_and_stops_at_its_end() -> None:
assert cascade.follows("pull", "pull") == "extract"
assert cascade.follows("pull", "extract") == "embed"
assert cascade.follows("pull", "embed") == "score"
assert cascade.follows("pull", "score") == "summarise"
assert cascade.follows("pull", "summarise") == ""
assert cascade.follows("pull", "nope") == ""
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
"""By position, not by name: `score` is in the chain twice, once before
extract and once after, and the second must lead on to summarise."""
assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "extract", "score",
"summarise")
assert [cascade.follows("pull", at) for at in range(6)] == \
["embed", "score", "extract", "score", "summarise", ""]
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 9) == ""
assert cascade.follows("nope", 0) == ""
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None:
made: list[Any] = []
class Context:
def task(self, t: Any) -> Any:
made.append(t)
return t
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 2},
context=Context())
assert out == {"task": "score", "next": "extract"}
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 3}
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 4},
context=Context())
assert out["next"] == "summarise" and made[1].carries["at"] == 5
+3
View File
@@ -750,3 +750,6 @@ def test_summarise_waits_on_the_top_of_each_enabled_agent(store: Store) -> None:
assert {r.uid for r in store.summarise_waiting(10, per_agent=2)} == {a, b, c}
assert store.summarise_pending(per_agent=2) == 3
assert [r.uid for r in store.summarise_waiting(10, 1, uids=[d])] == [d]
# The floor: b's best score is 0.8 under Two, c's 0.7, a's 0.9.
assert {r.uid for r in store.summarise_waiting(10, 2, min_score=0.75)} == {a, b}
assert store.summarise_pending(2, min_score=0.85) == 1
+1 -1
View File
@@ -201,7 +201,7 @@ def test_disabled_finds_nothing_and_asks_nobody(
def test_summarise_follows_score_and_finds_nothing_without_a_model(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
ranked(store, "1", "2")
assert cascade.follows("pull", "score") == "summarise"
assert cascade.follows("pull", cascade.CASCADES["pull"].index("summarise") - 1) == "summarise"
assert "summarise" in TASKS
stage = cascade.task("summarise", cascade="pull")