Extract is bounded to the ranking: pull → embed → score → extract → score → summarise
The free stages run first and their score bounds the two model stages. Extract reads only the best extract.per_agent roles of each agent at extract.min_score or more, then score runs again so the gates read what it filled; summarise gets a min_score of its own. The cascade steps by position, since score is in it twice. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
c4f2bcc0c9
commit
7db77e83a3
15 files changed
+215
-89
No files matched your search
+44
-2
@@ -57,6 +57,15 @@ def config(tmp_path: Path) -> Config:
|
||||
return Config(database=str(tmp_path / "roledesk.db"), extract={"enabled": True})
|
||||
|
||||
|
||||
def ranked(store: Store, *items: Item) -> None:
|
||||
"""Score the roles for one enabled agent, best first, so they are in the
|
||||
top extract reads. Every role given must already be stored."""
|
||||
agents = store.agents()
|
||||
agent_id = agents[0].id if agents else store.save_agent("Backend").id
|
||||
store.replace_scores(agent_id, [(item.uid, 1.0 - n / 10, "")
|
||||
for n, item in enumerate(items)])
|
||||
|
||||
|
||||
def stub(monkeypatch: pytest.MonkeyPatch, reply: Any) -> Model:
|
||||
model = Model(reply)
|
||||
monkeypatch.setattr(agent, "backend", lambda settings: "api")
|
||||
@@ -68,6 +77,7 @@ def test_a_batch_writes_the_columns_and_stamps_extracted_at(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
uid = posting("1").uid
|
||||
store.upsert_role(posting("1"))
|
||||
ranked(store, posting("1"))
|
||||
stub(monkeypatch, lambda roles: [answer(
|
||||
r["uid"], place="Berlin", country="de", remote="hybrid", seniority="senior",
|
||||
salary_min=80000, salary_max=95000, currency="eur", function="engineering",
|
||||
@@ -90,6 +100,7 @@ def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
item = posting("1", place="Berlin", remote="hybrid", salary_min=70000)
|
||||
store.upsert_role(item)
|
||||
ranked(store, item)
|
||||
stub(monkeypatch, lambda roles: [answer(
|
||||
r["uid"], place="Munich", remote="", salary_min=None, salary_max=90000,
|
||||
country="DE") for r in roles])
|
||||
@@ -105,6 +116,7 @@ def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
|
||||
def test_the_next_upsert_keeps_what_extract_filled(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.upsert_role(posting("1"))
|
||||
ranked(store, posting("1"))
|
||||
stub(monkeypatch, lambda roles: [answer(r["uid"], country="DE", seniority="mid")
|
||||
for r in roles])
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
@@ -119,6 +131,7 @@ def test_closed_and_answered_roles_are_not_offered(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
for sid in ("open", "closed", "done"):
|
||||
store.upsert_role(posting(sid))
|
||||
ranked(store, posting("open"), posting("closed"), posting("done"))
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' "
|
||||
"WHERE source_id = 'closed'")
|
||||
@@ -134,6 +147,7 @@ def test_closed_and_answered_roles_are_not_offered(
|
||||
def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.upsert_role(posting("1"))
|
||||
ranked(store, posting("1"))
|
||||
stub(monkeypatch, lambda roles: [answer("not-in-the-batch", country="DE")])
|
||||
|
||||
result = modeltask.run(extract.TASK, config, store)
|
||||
@@ -141,11 +155,39 @@ def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged(
|
||||
role = store.role(posting("1").uid)
|
||||
assert role is not None and role.country == "" and role.extracted_at is None
|
||||
assert result["rejected"] == ["not-in-the-batch"]
|
||||
assert store.extract_pending() == 1
|
||||
assert store.extract_pending(500) == 1
|
||||
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
assert store.extract_pending() == 0, "given up after three unanswered replies"
|
||||
assert store.extract_pending(500) == 0, "given up after three unanswered replies"
|
||||
|
||||
|
||||
def test_only_the_best_of_each_agent_above_the_floor_is_read(
|
||||
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The bound that keeps extract cheap: the top `per_agent` of an enabled
|
||||
agent's scores at `min_score` or more, whatever the boards produced. A
|
||||
role nobody ranks is never sent; a batch dealt a uid finishes it."""
|
||||
for sid in ("best", "second", "low", "unscored", "gated"):
|
||||
store.upsert_role(posting(sid))
|
||||
one = store.save_agent("One")
|
||||
two = store.save_agent("Two", enabled=False)
|
||||
store.replace_scores(one.id, [(posting("best").uid, 0.9, ""),
|
||||
(posting("second").uid, 0.8, ""),
|
||||
(posting("low").uid, 0.2, ""),
|
||||
(posting("gated").uid, 0.95, "country unknown")])
|
||||
store.replace_scores(two.id, [(posting("unscored").uid, 0.99, "")])
|
||||
config = Config(database=str(tmp_path / "x.db"),
|
||||
extract={"enabled": True, "per_agent": 1, "min_score": 0.5})
|
||||
model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles])
|
||||
|
||||
assert store.extract_pending(1, 0.5) == 1
|
||||
assert store.extract_pending(5, 0.5) == 2, "the floor holds the low one back"
|
||||
assert store.extract_pending(5, 0.0) == 3, "the disabled agent's pick never counts"
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
|
||||
assert [[r["uid"] for r in batch] for batch in model.asked] == [[posting("best").uid]]
|
||||
assert [r.uid for r in store.extract_waiting(10, 1, 0.5, uids=[posting("low").uid])] == \
|
||||
[posting("low").uid], "a dealt uid is read whatever its rank"
|
||||
|
||||
|
||||
def test_a_value_outside_the_vocabulary_is_dropped() -> None:
|
||||
|
||||
+32
-11
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from desk.models import Item # noqa: E402
|
||||
from pipeline import FINAL, SCHEDULES, TASKS, JobQueue, Jobs, RunContext # noqa: E402
|
||||
from pipeline import FINAL, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, Tally # noqa: E402
|
||||
from plugins.ats import ATSSource # noqa: E402
|
||||
from app.config import AgentSeed, Config # noqa: E402
|
||||
from app.jobs import cascade, embed, score # noqa: E402
|
||||
@@ -438,7 +438,7 @@ def test_the_shipped_starter_searches_validate() -> None:
|
||||
|
||||
# ---------------------------------------------------------------- cascade
|
||||
|
||||
def test_the_pull_job_runs_pull_extract_embed_then_score(
|
||||
def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
|
||||
store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
||||
@@ -471,8 +471,9 @@ def test_the_pull_job_runs_pull_extract_embed_then_score(
|
||||
assert stages == list(cascade.CASCADES["pull"])
|
||||
work = [j.kind for j in jobs if j.kind != FINAL]
|
||||
# The `pull` job and the chain it asked for are one cascade, so the Jobs
|
||||
# tab shows a scheduled pull as one run. Extract is off and finds nothing.
|
||||
assert work == ["pull", "pull_batch", "embed", "score"]
|
||||
# tab shows a scheduled pull as one run. Extract and summarise are off and
|
||||
# find nothing; score runs twice, before and after extract.
|
||||
assert work == ["pull", "pull_batch", "embed", "score", "score"]
|
||||
assert all(j.state == "done" for j in jobs)
|
||||
assert len(store.scores(agent.id)) == 1
|
||||
|
||||
@@ -486,10 +487,30 @@ def test_pull_is_a_job_on_a_six_hour_schedule_and_the_stages_are_tasks() -> None
|
||||
assert SCHEDULES["pull"].cadence == "every 6h"
|
||||
|
||||
|
||||
def test_follows_walks_the_cascade_and_stops_at_its_end() -> None:
|
||||
assert cascade.follows("pull", "pull") == "extract"
|
||||
assert cascade.follows("pull", "extract") == "embed"
|
||||
assert cascade.follows("pull", "embed") == "score"
|
||||
assert cascade.follows("pull", "score") == "summarise"
|
||||
assert cascade.follows("pull", "summarise") == ""
|
||||
assert cascade.follows("pull", "nope") == ""
|
||||
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
|
||||
"""By position, not by name: `score` is in the chain twice, once before
|
||||
extract and once after, and the second must lead on to summarise."""
|
||||
assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "extract", "score",
|
||||
"summarise")
|
||||
assert [cascade.follows("pull", at) for at in range(6)] == \
|
||||
["embed", "score", "extract", "score", "summarise", ""]
|
||||
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 9) == ""
|
||||
assert cascade.follows("nope", 0) == ""
|
||||
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
|
||||
|
||||
|
||||
def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
made: list[Any] = []
|
||||
|
||||
class Context:
|
||||
def task(self, t: Any) -> Any:
|
||||
made.append(t)
|
||||
return t
|
||||
|
||||
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 2},
|
||||
context=Context())
|
||||
assert out == {"task": "score", "next": "extract"}
|
||||
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 3}
|
||||
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 4},
|
||||
context=Context())
|
||||
assert out["next"] == "summarise" and made[1].carries["at"] == 5
|
||||
@@ -750,3 +750,6 @@ def test_summarise_waits_on_the_top_of_each_enabled_agent(store: Store) -> None:
|
||||
assert {r.uid for r in store.summarise_waiting(10, per_agent=2)} == {a, b, c}
|
||||
assert store.summarise_pending(per_agent=2) == 3
|
||||
assert [r.uid for r in store.summarise_waiting(10, 1, uids=[d])] == [d]
|
||||
# The floor: b's best score is 0.8 under Two, c's 0.7, a's 0.9.
|
||||
assert {r.uid for r in store.summarise_waiting(10, 2, min_score=0.75)} == {a, b}
|
||||
assert store.summarise_pending(2, min_score=0.85) == 1
|
||||
@@ -201,7 +201,7 @@ def test_disabled_finds_nothing_and_asks_nobody(
|
||||
def test_summarise_follows_score_and_finds_nothing_without_a_model(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
ranked(store, "1", "2")
|
||||
assert cascade.follows("pull", "score") == "summarise"
|
||||
assert cascade.follows("pull", cascade.CASCADES["pull"].index("summarise") - 1) == "summarise"
|
||||
assert "summarise" in TASKS
|
||||
stage = cascade.task("summarise", cascade="pull")
|
||||
|
||||
|
||||
Reference in new issue
Block a user