Extract and summarise bounds are each agent's own limits, set on its form
agents.limits holds extract_top, extract_min, summarise_top and summarise_min; the scope query reads them per ranked row, and a row saved before this behaves like a fresh agent. The four global config keys go. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
e602eb3388
commit
cebe62ddc9
20 files changed
+229
-147
No files matched your search
+16
-12
@@ -14,6 +14,7 @@ from desk.models import Item
|
||||
from desk.tools import agent
|
||||
from app.config import Config
|
||||
from app.jobs import extract
|
||||
from app.models import Limits
|
||||
from app.storage import Store
|
||||
|
||||
|
||||
@@ -155,38 +156,41 @@ def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged(
|
||||
role = store.role(posting("1").uid)
|
||||
assert role is not None and role.country == "" and role.extracted_at is None
|
||||
assert result["rejected"] == ["not-in-the-batch"]
|
||||
assert store.extract_pending(500) == 1
|
||||
assert store.extract_pending() == 1
|
||||
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
assert store.extract_pending(500) == 0, "given up after three unanswered replies"
|
||||
assert store.extract_pending() == 0, "given up after three unanswered replies"
|
||||
|
||||
|
||||
def test_only_the_best_of_each_agent_above_the_floor_is_read(
|
||||
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The bound that keeps extract cheap: the top `per_agent` of an enabled
|
||||
agent's scores at `min_score` or more, whatever the boards produced. A
|
||||
role nobody ranks is never sent; a batch dealt a uid finishes it."""
|
||||
"""The bound that keeps extract cheap: the top of an enabled agent's
|
||||
scores, as deep and as low as that agent's own `limits` say, whatever
|
||||
the boards produced. A role nobody ranks is never sent; a batch dealt a
|
||||
uid finishes it."""
|
||||
for sid in ("best", "second", "low", "unscored", "gated"):
|
||||
store.upsert_role(posting(sid))
|
||||
one = store.save_agent("One")
|
||||
one = store.save_agent("One", limits=Limits(extract_top=1, extract_min=0.5))
|
||||
two = store.save_agent("Two", enabled=False)
|
||||
store.replace_scores(one.id, [(posting("best").uid, 0.9, ""),
|
||||
(posting("second").uid, 0.8, ""),
|
||||
(posting("low").uid, 0.2, ""),
|
||||
(posting("gated").uid, 0.95, "country unknown")])
|
||||
store.replace_scores(two.id, [(posting("unscored").uid, 0.99, "")])
|
||||
config = Config(database=str(tmp_path / "x.db"),
|
||||
extract={"enabled": True, "per_agent": 1, "min_score": 0.5})
|
||||
config = Config(database=str(tmp_path / "x.db"), extract={"enabled": True})
|
||||
model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles])
|
||||
|
||||
assert store.extract_pending(1, 0.5) == 1
|
||||
assert store.extract_pending(5, 0.5) == 2, "the floor holds the low one back"
|
||||
assert store.extract_pending(5, 0.0) == 3, "the disabled agent's pick never counts"
|
||||
assert store.extract_pending() == 1
|
||||
store.save_agent("One", agent_id=one.id, limits=Limits(extract_top=5, extract_min=0.5))
|
||||
assert store.extract_pending() == 2, "the floor holds the low one back"
|
||||
store.save_agent("One", agent_id=one.id, limits=Limits(extract_top=5, extract_min=0))
|
||||
assert store.extract_pending() == 3, "the disabled agent's pick never counts"
|
||||
store.save_agent("One", agent_id=one.id, limits=Limits(extract_top=1, extract_min=0.5))
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
|
||||
assert [[r["uid"] for r in batch] for batch in model.asked] == [[posting("best").uid]]
|
||||
assert [r.uid for r in store.extract_waiting(10, 1, 0.5, uids=[posting("low").uid])] == \
|
||||
assert [r.uid for r in store.extract_waiting(10, uids=[posting("low").uid])] == \
|
||||
[posting("low").uid], "a dealt uid is read whatever its rank"
|
||||
|
||||
|
||||
|
||||
+29
-16
@@ -11,7 +11,7 @@ import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from desk.models import Item
|
||||
from app.models import Application, company_key, place_key, title_key
|
||||
from app.models import Application, Limits, company_key, place_key, title_key
|
||||
from app.storage import LATEST, Store
|
||||
|
||||
|
||||
@@ -62,6 +62,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
|
||||
"application_events", "agents", "scores", "state"} <= tables
|
||||
assert "idx_roles_cluster" in indexes
|
||||
assert store.migrations_applied == []
|
||||
assert LATEST == 8
|
||||
# A second open finds it current and applies nothing.
|
||||
assert Store(path).migrations_applied == []
|
||||
|
||||
@@ -88,7 +89,7 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None:
|
||||
for column in ("permit", "languages", "extract_tries", "summary", "summarised_at",
|
||||
"article_text", "article_at", "read_at"):
|
||||
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
|
||||
for column in ("intent_key", "intent_vec"):
|
||||
for column in ("intent_key", "intent_vec", "limits"): # v3, v8
|
||||
conn.execute(f"ALTER TABLE agents DROP COLUMN {column}")
|
||||
# v1 had affinity on the company; the vote columns v4 added and v6
|
||||
# drops again are not in a fresh file, so there is nothing to put back.
|
||||
@@ -117,6 +118,7 @@ def _downgrade_to_v4(path: Path) -> None:
|
||||
conn.execute("ALTER TABLE companies ADD COLUMN affinity INTEGER NOT NULL DEFAULT 0")
|
||||
for column in ("ats", "country", "city"): # v7
|
||||
conn.execute(f"ALTER TABLE companies DROP COLUMN {column}")
|
||||
conn.execute("ALTER TABLE agents DROP COLUMN limits") # v8
|
||||
conn.execute("DROP TABLE agent_companies")
|
||||
conn.execute("DROP TABLE role_votes")
|
||||
conn.execute("PRAGMA user_version = 4")
|
||||
@@ -734,26 +736,37 @@ def test_a_summarys_tags_land_with_it_and_not_on_a_rejected_late_answer(
|
||||
assert role is not None and role.tags == ["backend", "python"]
|
||||
|
||||
|
||||
def test_summarise_waits_on_the_top_of_each_enabled_agent(store: Store) -> None:
|
||||
"""Two agents each contribute their own top one; a closed role and a
|
||||
gated score are not in anybody's top. A hidden company needs no case:
|
||||
`score` writes no rows for it under the agent that hid it."""
|
||||
def test_summarise_waits_on_the_top_of_each_enabled_agent_by_its_own_limits(
|
||||
store: Store) -> None:
|
||||
"""Each agent contributes its own top, as deep and as low as its own
|
||||
`limits` say; a closed role and a gated score are in nobody's top. A
|
||||
hidden company needs no case: `score` writes no rows for it under the
|
||||
agent that hid it. An agent saved before v8 has `limits` of '{}', which
|
||||
reads as the defaults."""
|
||||
for sid in "abcd":
|
||||
store.upsert_role(posting(sid))
|
||||
a, b, c, d = (posting(s).uid for s in "abcd")
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (store.now(), d))
|
||||
one = store.save_agent("One")
|
||||
two = store.save_agent("Two")
|
||||
one = store.save_agent("One", limits=Limits(summarise_top=1, summarise_min=0))
|
||||
two = store.save_agent("Two", limits=Limits(summarise_top=1, summarise_min=0))
|
||||
store.replace_scores(one.id, [(d, 0.99, ""), (a, 0.9, ""), (b, 0.5, "")])
|
||||
store.replace_scores(two.id, [(a, 0.95, "remote"), (c, 0.7, ""), (b, 0.8, "")])
|
||||
|
||||
waiting = {r.uid for r in store.summarise_waiting(10, per_agent=1)}
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b}
|
||||
|
||||
assert waiting == {a, b}
|
||||
assert {r.uid for r in store.summarise_waiting(10, per_agent=2)} == {a, b, c}
|
||||
assert store.summarise_pending(per_agent=2) == 3
|
||||
assert [r.uid for r in store.summarise_waiting(10, 1, uids=[d])] == [d]
|
||||
# The floor: b's best score is 0.8 under Two, c's 0.7, a's 0.9.
|
||||
assert {r.uid for r in store.summarise_waiting(10, 2, min_score=0.75)} == {a, b}
|
||||
assert store.summarise_pending(2, min_score=0.85) == 1
|
||||
# Two goes deeper: its second, c, joins; One's second, b, was in already.
|
||||
store.save_agent("Two", agent_id=two.id, limits=Limits(summarise_top=2, summarise_min=0))
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b, c}
|
||||
assert store.summarise_pending() == 3
|
||||
assert [r.uid for r in store.summarise_waiting(10, uids=[d])] == [d]
|
||||
# The floor is Two's own: c at 0.7 drops, One's b at 0.5 stays.
|
||||
store.save_agent("Two", agent_id=two.id,
|
||||
limits=Limits(summarise_top=2, summarise_min=0.75))
|
||||
store.save_agent("One", agent_id=one.id, limits=Limits(summarise_top=2, summarise_min=0))
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b}
|
||||
# '{}' is the defaults, 30 at 0.5: One's b at 0.5 stays, Two's c at 0.7 is back.
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE agents SET limits = '{}'")
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b, c}
|
||||
assert store.agents()[0].limits == Limits()
|
||||
@@ -12,6 +12,7 @@ import pytest
|
||||
|
||||
from app.config import Config
|
||||
from app.jobs import cascade, summarise
|
||||
from app.models import Limits
|
||||
from app.storage import Store
|
||||
from desk import modeltask
|
||||
from desk.models import Item
|
||||
@@ -63,18 +64,19 @@ def store(tmp_path: Path) -> Store:
|
||||
|
||||
@pytest.fixture
|
||||
def config(tmp_path: Path) -> Config:
|
||||
return Config(database=str(tmp_path / "roledesk.db"),
|
||||
summarise={"enabled": True, "per_agent": 2})
|
||||
return Config(database=str(tmp_path / "roledesk.db"), summarise={"enabled": True})
|
||||
|
||||
|
||||
def ranked(store: Store, *ids: str, agent_name: str = "Backend",
|
||||
enabled: bool = True) -> list[str]:
|
||||
"""Store a role per id and score them for one agent, best first."""
|
||||
"""Store a role per id and score them for one agent, best first. The
|
||||
agent summarises its top two: the bound is the agent's own."""
|
||||
uids = []
|
||||
for sid in ids:
|
||||
store.upsert_role(posting(sid))
|
||||
uids.append(posting(sid).uid)
|
||||
one = store.save_agent(agent_name, "python backend", enabled=enabled)
|
||||
one = store.save_agent(agent_name, "python backend", enabled=enabled,
|
||||
limits=Limits(summarise_top=2))
|
||||
store.replace_scores(one.id, [(uid, 1.0 - n / 10, "") for n, uid in enumerate(uids)])
|
||||
return uids
|
||||
|
||||
@@ -154,7 +156,7 @@ def test_a_role_with_no_text_is_skipped_not_sent(
|
||||
summarise.run_batch(config, store, uids=[empty])
|
||||
|
||||
assert model.asked == []
|
||||
assert store.summarise_pending(config.summarise.per_agent) == 0
|
||||
assert store.summarise_pending() == 0
|
||||
|
||||
|
||||
def test_a_named_uid_is_summarised_whatever_its_rank_or_the_switch(
|
||||
@@ -182,10 +184,10 @@ def test_an_unknown_or_empty_answer_is_rejected_and_the_role_given_up_after_thre
|
||||
|
||||
assert result["rejected"] == ["not-in-the-batch", uid]
|
||||
assert store.role(uid).summary == "" # type: ignore[union-attr]
|
||||
assert store.summarise_pending(2) == 1
|
||||
assert store.summarise_pending() == 1
|
||||
modeltask.run(summarise.TASK, config, store)
|
||||
modeltask.run(summarise.TASK, config, store)
|
||||
assert store.summarise_pending(2) == 0, "given up after three unanswered replies"
|
||||
assert store.summarise_pending() == 0, "given up after three unanswered replies"
|
||||
|
||||
|
||||
def test_disabled_finds_nothing_and_asks_nobody(
|
||||
|
||||
@@ -621,6 +621,12 @@ def test_the_agents_page_lists_and_edits_search_agents(client: TestClient) -> No
|
||||
for weight in ("a-fit", "a-recency", "a-affinity", "a-taste"):
|
||||
assert f'id="{weight}"' in page
|
||||
assert 'id="a-taste" min="0" step="0.05" value="0.2"' in page
|
||||
# What the model reads after a pull: a count and a score floor per stage.
|
||||
assert "<legend>What the model reads</legend>" in template
|
||||
for limit in ("a-extract-top", "a-extract-min", "a-summarise-top", "a-summarise-min"):
|
||||
assert f'id="{limit}"' in template
|
||||
assert 'id="a-extract-top" min="0" step="10" value="500"' in page
|
||||
assert 'id="a-summarise-min" min="0" max="1" step="0.05" value="0.5"' in page
|
||||
assert 'id="a-new"' in page and 'id="a-delete"' in page
|
||||
assert "Save and score" in page
|
||||
|
||||
@@ -756,6 +762,20 @@ def test_a_gate_on_an_unknown_field_is_refused(client: TestClient) -> None:
|
||||
assert client.post("/api/agents", json=body).status_code == 422
|
||||
|
||||
|
||||
def test_an_agents_limits_are_saved_listed_and_checked(client: TestClient) -> None:
|
||||
"""How deep the model stages read is the agent's own; a score outside
|
||||
0..1 is refused."""
|
||||
default = client.post("/api/agents", json={"name": "Wide"}).json()
|
||||
assert default["limits"] == {"extract_top": 500, "extract_min": 0.3,
|
||||
"summarise_top": 30, "summarise_min": 0.5}
|
||||
narrow = {"extract_top": 50, "extract_min": 0.6, "summarise_top": 5, "summarise_min": 0.7}
|
||||
saved = client.post("/api/agents", json={"name": "Narrow", "limits": narrow})
|
||||
assert saved.status_code == 200 and saved.json()["limits"] == narrow
|
||||
assert [a["limits"]["extract_top"] for a in client.get("/api/agents").json()] == [50, 500]
|
||||
bad = client.post("/api/agents", json={"name": "Bad", "limits": {"extract_min": 2}})
|
||||
assert bad.status_code == 422
|
||||
|
||||
|
||||
def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated(
|
||||
client: TestClient, store: Store) -> None:
|
||||
for sid in ("low", "high", "gated"):
|
||||
|
||||
Reference in new issue
Block a user