Rank by the cut: fetch gate overrides in scoring, the cut per agent, stages read the cut, fetch text renamed from enrich, the feed kept to the fetch gate, and migration 32
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
060ccf66e6
commit
617356e60c
35 files changed
+1054
-751
No files matched your search
@@ -131,7 +131,7 @@ def test_a_stage_shows_the_model_that_will_answer(tmp_path: Path) -> None:
|
||||
config.agent.model = "claude-opus-4-1"
|
||||
by_name = {s["name"]: s for s in client.get("/api/stages").json()}
|
||||
assert by_name["extract"]["model"] == by_name["summarise"]["model"] == "claude-opus-4-1"
|
||||
assert by_name["enrich"]["model"] == ""
|
||||
assert by_name["fetch_text"]["model"] == ""
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- config
|
||||
@@ -162,7 +162,7 @@ def test_the_example_config_names_every_key_of_the_sections_it_documents() -> No
|
||||
import yaml
|
||||
|
||||
raw = yaml.safe_load((ROOT / "config.example.yaml").read_text("utf-8"))
|
||||
for name, allowed_missing in (("agent", set()), ("extract", set()), ("enrich", set()),
|
||||
for name, allowed_missing in (("agent", set()), ("extract", set()), ("fetch_text", set()),
|
||||
("embed", set()), ("workers", set()),
|
||||
("summarise", {"on_ingest"})): # accepted, ignored
|
||||
fields = set(type(getattr(Config(), name)).model_fields)
|
||||
|
||||
@@ -148,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
|
||||
"VALUES (?, ?, '', '')", (key, name))
|
||||
conn.execute("PRAGMA user_version = 19")
|
||||
reopened = Store(path)
|
||||
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
|
||||
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]
|
||||
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
|
||||
assert len(reopened.companies()) == 3
|
||||
|
||||
@@ -186,7 +186,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
|
||||
conn.execute("ALTER TABLE companies DROP COLUMN all_countries")
|
||||
conn.execute("PRAGMA user_version = 20")
|
||||
reopened = Store(path)
|
||||
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
|
||||
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]
|
||||
# The tag of 21 is All countries and All roles since 24.
|
||||
tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
|
||||
assert {c.key: c.all_roles for c in reopened.companies()} == tagged
|
||||
|
||||
@@ -183,10 +183,15 @@ def test_closing_soon_keeps_today_to_four_days_on(client: TestClient, store: Sto
|
||||
owner, uid = setup(store)
|
||||
params = {"agent": owner.id, "order": "newest", "closing": 1}
|
||||
page = client.get("/api/roles", params=params).json()
|
||||
# d is gated, so newest leaves it out as every order does.
|
||||
assert sorted(r["uid"] for r in page["items"]) == sorted([uid["a"], uid["b"]])
|
||||
assert page["total"] == 2
|
||||
facets = client.get("/api/roles/facets", params=params).json()
|
||||
assert sum(f["count"] for f in facets["pay"]) == 2
|
||||
params["gated"] = 1
|
||||
page = client.get("/api/roles", params=params).json()
|
||||
assert sorted(r["uid"] for r in page["items"]) == sorted([uid["a"], uid["b"], uid["d"]])
|
||||
assert page["total"] == 3
|
||||
facets = client.get("/api/roles/facets", params=params).json()
|
||||
assert sum(f["count"] for f in facets["pay"]) == 3
|
||||
|
||||
|
||||
def test_the_role_carries_its_judgement_and_languages(
|
||||
|
||||
+12
-10
@@ -71,9 +71,11 @@ def test_a_cv_language_is_a_code_and_a_level_of_the_ladder() -> None:
|
||||
CvLanguage(language="da", level="good")
|
||||
|
||||
|
||||
def test_new_agents_have_judge_limits_by_default(store: Store) -> None:
|
||||
def test_new_agents_have_the_cut_by_default(store: Store) -> None:
|
||||
made = store.save_agent("Data")
|
||||
assert (made.limits.judge_top, made.limits.judge_min) == (150, 0.55)
|
||||
assert made.limits.model_dump() == {"cut_fit_top": 500, "cut_fit_min": 0.6,
|
||||
"cut_cv_top": 150, "cut_cv_min": 0.6,
|
||||
"summarise_top": 50}
|
||||
|
||||
|
||||
# --------------------------------------------------------------- migration
|
||||
@@ -96,7 +98,7 @@ def test_v28_adds_the_cv_columns_and_the_host_deadline_and_runs_twice(tmp_path:
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [28, 29, 30, 31]
|
||||
assert migrated.migrations_applied == [28, 29, 30, 31, 32]
|
||||
rows = {r["source_id"]: r for r in migrated._rows(
|
||||
"SELECT source_id, languages_required, deadline_host, deadline FROM roles")}
|
||||
assert (rows["a"]["deadline_host"], rows["a"]["deadline"]) == ("2026-10-18", None)
|
||||
@@ -108,7 +110,7 @@ def test_v28_adds_the_cv_columns_and_the_host_deadline_and_runs_twice(tmp_path:
|
||||
assert migrated.role(posting("a").uid).deadline == "2026-10-18"
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("PRAGMA user_version = 27")
|
||||
assert Store(path).migrations_applied == [28, 29, 30, 31]
|
||||
assert Store(path).migrations_applied == [28, 29, 30, 31, 32]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- deadline
|
||||
@@ -159,7 +161,7 @@ def test_extract_reads_levels_kinds_and_the_deadline(
|
||||
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
item = posting("1")
|
||||
upsert(store, item)
|
||||
store.replace_scores(store.save_agent("Data", "data").id, [(item.uid, 0.9, "")])
|
||||
store.replace_scores(store.save_agent("Data", "data").id, [(item.uid, 0.9, "", "", None, 0.9, "fit")])
|
||||
model = Model(lambda roles: [answer(r["uid"], deadline="2026-10-30", languages_required=[
|
||||
{"language": "DA", "level": "fluent", "kind": "required"},
|
||||
{"language": "en", "level": "", "kind": "required"},
|
||||
@@ -186,7 +188,7 @@ def test_extract_reads_levels_kinds_and_the_deadline(
|
||||
def test_a_role_extracted_before_is_read_once_more_for_the_new_fields(store: Store) -> None:
|
||||
item = posting("1")
|
||||
upsert(store, item)
|
||||
store.replace_scores(store.save_agent("Data", "data").id, [(item.uid, 0.9, "")])
|
||||
store.replace_scores(store.save_agent("Data", "data").id, [(item.uid, 0.9, "", "", None, 0.9, "fit")])
|
||||
store.set_extracted(item.uid, {"seniority": "director"})
|
||||
assert store.extract_pending() == 0
|
||||
with store.transaction() as conn:
|
||||
@@ -249,12 +251,12 @@ def test_v29_adds_the_musts_and_broken_columns_and_runs_twice(tmp_path: Path) ->
|
||||
conn.execute(f"ALTER TABLE role_judgements DROP COLUMN {column}")
|
||||
conn.execute("PRAGMA user_version = 28")
|
||||
|
||||
assert Store(path).migrations_applied == [29, 30, 31]
|
||||
assert Store(path).migrations_applied == [29, 30, 31, 32]
|
||||
with sqlite3.connect(path) as conn:
|
||||
columns = {r[1] for r in conn.execute("PRAGMA table_info(role_judgements)")}
|
||||
assert {"musts", "broken"} <= columns
|
||||
conn.execute("PRAGMA user_version = 28")
|
||||
assert Store(path).migrations_applied == [29, 30, 31]
|
||||
assert Store(path).migrations_applied == [29, 30, 31, 32]
|
||||
|
||||
|
||||
def test_v30_adds_like_text_and_runs_twice(tmp_path: Path) -> None:
|
||||
@@ -266,12 +268,12 @@ def test_v30_adds_like_text_and_runs_twice(tmp_path: Path) -> None:
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [30, 31]
|
||||
assert migrated.migrations_applied == [30, 31, 32]
|
||||
[data] = migrated.agents()
|
||||
assert (data.like_text, data.avoid_text) == ("", "Reporting.")
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("PRAGMA user_version = 29")
|
||||
assert Store(path).migrations_applied == [30, 31]
|
||||
assert Store(path).migrations_applied == [30, 31, 32]
|
||||
|
||||
|
||||
def test_the_musts_text_is_one_must_have_per_line() -> None:
|
||||
|
||||
+32
-27
@@ -218,34 +218,39 @@ def window(store: Store) -> list[str]:
|
||||
return [row["uid"] for _, row, _, _ in judge.waiting(store)]
|
||||
|
||||
|
||||
def test_the_judge_also_reads_the_closest_to_the_cv_the_score_leaves_out(
|
||||
store: Store) -> None:
|
||||
owner = store.save_agent("Data", prompts=[Prompt(name="AI", text="Head of AI")],
|
||||
cv_text="VP Data.",
|
||||
limits=Limits(judge_top=2, judge_min=0.5, judge_cv_top=2))
|
||||
top, second, low, near, nearer, nearest_gated, far = ranked(store, owner, [
|
||||
("top", 0.9, "", 0.1), ("second", 0.8, "", 0.99), ("low", 0.4, "", 0.2),
|
||||
("near", 0.3, "", 0.6), ("nearer", 0.2, "", 0.7), ("gated", 0.0, "country SE", None),
|
||||
("far", 0.1, "", 0.05)])
|
||||
assert window(store) == [top, second, nearer, near]
|
||||
def test_the_cut_takes_the_closest_to_the_cv_the_fit_list_leaves_out_and_judge_reads_it(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
"""One role in by fit to the prompt, the one closest to the CV that the
|
||||
fit list leaves out by CV closeness, a liked one kept; a gated role is in
|
||||
no list however close, liked or not."""
|
||||
backend = role(store, "1", "Backend Python", "Backend python python.", country="DK")
|
||||
frontend = role(store, "2", "Frontend React", "Frontend react react.", country="DK")
|
||||
data = role(store, "3", "Data", "Data.", country="DK")
|
||||
other = role(store, "4", "Data again", "Data data.", country="DK")
|
||||
gated = role(store, "5", "Frontend React", "Frontend react react.", country="SE")
|
||||
owner = store.save_agent("Data", prompts=[Prompt(name="Backend", text="python backend")],
|
||||
gates=[Gate(field="country", values=["DK"])],
|
||||
cv_text="Frontend react developer.",
|
||||
limits=Limits(cut_fit_top=1, cut_fit_min=0, cut_cv_top=1,
|
||||
cut_cv_min=0))
|
||||
store.set_vote(owner.id, data, 1)
|
||||
store.set_vote(owner.id, gated, 1)
|
||||
run(config, store, embed.run_embed)
|
||||
result = run(config, store, score.run_score)
|
||||
|
||||
assert store.cut(owner.id) == {backend: "fit", frontend: "cv", data: "kept"}
|
||||
assert result["cut"] == 3
|
||||
with store.transaction() as conn:
|
||||
fits = dict(conn.execute("SELECT uid, fit FROM scores WHERE agent_id = ?",
|
||||
(owner.id,)).fetchall())
|
||||
assert fits[backend] > fits[other] and fits[gated] is None
|
||||
assert window(store) == [backend, frontend, data]
|
||||
|
||||
# No count, no list: what is liked is kept all the same.
|
||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
||||
limits=Limits(judge_top=2, judge_min=0.85, judge_cv_top=2))
|
||||
assert window(store) == [top, second, nearer], \
|
||||
"a role under the bar is no part of the first set, so it may come in by the CV"
|
||||
|
||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
||||
limits=Limits(judge_top=2, judge_min=0.5, judge_cv_top=0))
|
||||
assert window(store) == [top, second]
|
||||
assert low and far and nearest_gated
|
||||
|
||||
|
||||
def test_a_gated_role_stays_out_of_the_window_however_close(store: Store) -> None:
|
||||
owner = store.save_agent("Data", prompts=[Prompt(name="AI", text="Head of AI")],
|
||||
cv_text="VP Data.",
|
||||
limits=Limits(judge_top=1, judge_min=0.5, judge_cv_top=5))
|
||||
top, _gated = ranked(store, owner, [("top", 0.9, "", 0.1), ("gated", 0.0, "remote", 1.0)])
|
||||
assert window(store) == [top]
|
||||
gates=owner.gates, limits=Limits(cut_fit_top=0, cut_cv_top=0))
|
||||
run(config, store, score.run_score)
|
||||
assert store.cut(owner.id) == {data: "kept"}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- order
|
||||
@@ -277,7 +282,7 @@ def test_migration_31_adds_the_column_and_the_table_and_runs_twice(tmp_path: Pat
|
||||
conn.execute("DROP TABLE agent_cv_vectors")
|
||||
conn.execute("ALTER TABLE scores DROP COLUMN cv_fit")
|
||||
conn.execute("PRAGMA user_version = 30")
|
||||
assert Store(path).migrations_applied == [31]
|
||||
assert Store(path).migrations_applied == [31, 32]
|
||||
step = {v: s for v, _, s in MIGRATIONS}[31]
|
||||
with sqlite3.connect(path) as conn:
|
||||
assert callable(step)
|
||||
|
||||
@@ -246,7 +246,8 @@ def test_a_copy_is_left_out_of_the_feed_counts_and_model_stages(store: Store) ->
|
||||
assert {r.uid for r in store.roles_to_embed("m", 10)} == {kept.uid, other.uid}
|
||||
assert store.open_role_counts() == {"sap": 2}
|
||||
assert [count for _, _, count, _ in store.open_companies()] == [2]
|
||||
store.replace_scores(agent.id, [(u, 0.9, "") for u in (kept.uid, copy.uid, other.uid)])
|
||||
store.replace_scores(agent.id, [(u, 0.9, "", "", None, 0.9, "fit")
|
||||
for u in (kept.uid, copy.uid, other.uid)])
|
||||
assert {r.uid for r in store.extract_waiting(10)} == {kept.uid, other.uid}
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {kept.uid, other.uid}
|
||||
|
||||
@@ -278,7 +279,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
|
||||
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]
|
||||
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
|
||||
with sqlite3.connect(path) as conn:
|
||||
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
|
||||
|
||||
+22
-30
@@ -15,7 +15,7 @@ from desk.models import Item
|
||||
from desk.tools import agent
|
||||
from app.config import Config
|
||||
from app.jobs import extract
|
||||
from app.models import LEVELS, Limits
|
||||
from app.models import LEVELS
|
||||
from app.storage import Store
|
||||
from prepped import upsert
|
||||
|
||||
@@ -61,11 +61,11 @@ def config(tmp_path: Path) -> Config:
|
||||
|
||||
|
||||
def ranked(store: Store, *items: Item) -> None:
|
||||
"""Score the roles for one enabled agent, best first, so they are in the
|
||||
top extract reads. Every role given must already be stored."""
|
||||
"""Score the roles for one enabled agent, best first, each in its cut, so
|
||||
extract reads them. Every role given must already be stored."""
|
||||
agents = store.agents()
|
||||
agent_id = agents[0].id if agents else store.save_agent("Backend", "python backend").id
|
||||
store.replace_scores(agent_id, [(item.uid, 1.0 - n / 10, "")
|
||||
store.replace_scores(agent_id, [(item.uid, 1.0 - n / 10, "", "", None, None, "fit")
|
||||
for n, item in enumerate(items)])
|
||||
|
||||
|
||||
@@ -167,37 +167,28 @@ def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged(
|
||||
assert store.extract_pending() == 0, "given up after three unanswered replies"
|
||||
|
||||
|
||||
def test_only_the_best_of_each_agent_above_the_floor_is_read(
|
||||
def test_only_the_cut_of_an_enabled_agent_is_read(
|
||||
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The bound that keeps extract cheap: the top of an enabled agent's
|
||||
scores, as deep and as low as that agent's own `limits` say, whatever
|
||||
the boards produced. A role nobody ranks is never sent; a batch dealt a
|
||||
uid finishes it."""
|
||||
for sid in ("best", "second", "low", "unscored", "gated"):
|
||||
"""The bound that keeps extract cheap: the cut of an enabled agent,
|
||||
whatever the boards produced. A role in no cut is never sent; a batch
|
||||
dealt a uid finishes it."""
|
||||
for sid in ("best", "kept", "low", "unscored", "gated"):
|
||||
upsert(store, posting(sid))
|
||||
one = store.save_agent("One", "python backend",
|
||||
limits=Limits(extract_top=1, extract_min=0.5))
|
||||
one = store.save_agent("One", "python backend")
|
||||
two = store.save_agent("Two", "python backend", enabled=False)
|
||||
store.replace_scores(one.id, [(posting("best").uid, 0.9, ""),
|
||||
(posting("second").uid, 0.8, ""),
|
||||
(posting("low").uid, 0.2, ""),
|
||||
(posting("gated").uid, 0.95, "country unknown")])
|
||||
store.replace_scores(two.id, [(posting("unscored").uid, 0.99, "")])
|
||||
store.replace_scores(one.id, [(posting("best").uid, 0.9, "", "", None, 0.9, "fit"),
|
||||
(posting("kept").uid, 0.3, "", "", None, 0.1, "kept"),
|
||||
(posting("low").uid, 0.2, "", "", None, 0.2, ""),
|
||||
(posting("gated").uid, 0.0, "country unknown")])
|
||||
store.replace_scores(two.id, [(posting("unscored").uid, 0.99, "", "", None, 0.99, "fit")])
|
||||
config = Config(database=str(tmp_path / "x.db"), extract={"enabled": True})
|
||||
model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles])
|
||||
|
||||
assert store.extract_pending() == 1
|
||||
store.save_agent("One", "python backend", agent_id=one.id,
|
||||
limits=Limits(extract_top=5, extract_min=0.5))
|
||||
assert store.extract_pending() == 2, "the floor holds the low one back"
|
||||
store.save_agent("One", "python backend", agent_id=one.id,
|
||||
limits=Limits(extract_top=5, extract_min=0))
|
||||
assert store.extract_pending() == 3, "the disabled agent's pick never counts"
|
||||
store.save_agent("One", "python backend", agent_id=one.id,
|
||||
limits=Limits(extract_top=1, extract_min=0.5))
|
||||
assert store.extract_pending() == 2, "the disabled agent's cut never counts"
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
|
||||
assert [[r["uid"] for r in batch] for batch in model.asked] == [[posting("best").uid]]
|
||||
assert [[r["uid"] for r in batch] for batch in model.asked] == [
|
||||
[posting("best").uid, posting("kept").uid]]
|
||||
assert [r.uid for r in store.extract_waiting(10, uids=[posting("low").uid])] == \
|
||||
[posting("low").uid], "a dealt uid is read whatever its rank"
|
||||
|
||||
@@ -293,9 +284,10 @@ def test_each_agents_best_come_first_then_the_rest(store: Store) -> None:
|
||||
upsert(store, item)
|
||||
one = store.save_agent("One", "python backend")
|
||||
two = store.save_agent("Two", "python backend")
|
||||
store.replace_scores(one.id, [(items[0].uid, 0.9, ""), (items[1].uid, 0.8, ""),
|
||||
(items[2].uid, 0.7, "")])
|
||||
store.replace_scores(two.id, [(items[3].uid, 0.6, "")])
|
||||
store.replace_scores(one.id, [(items[0].uid, 0.9, "", "", None, None, "fit"),
|
||||
(items[1].uid, 0.8, "", "", None, None, "fit"),
|
||||
(items[2].uid, 0.7, "", "", None, None, "cv")])
|
||||
store.replace_scores(two.id, [(items[3].uid, 0.6, "", "", None, None, "kept")])
|
||||
|
||||
got = [r.uid for r in store.extract_waiting(10)]
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""The `enrich` stage: the best roles without text, read on their own pages
|
||||
after scoring. Which roles and in what order, the budgets, what is stored,
|
||||
the cascade around it, and what triggers it. No network: a fake host stands
|
||||
in, answering from a dict."""
|
||||
"""The `fetch_text` stage: the roles in the cut without text, read on their
|
||||
own pages after scoring. Which roles and in what order, the budgets, what is
|
||||
stored, the cascade around it, and what triggers it. No network: a fake host
|
||||
stands in, answering from a dict."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -24,7 +24,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app.config import Config, ServerConfig # noqa: E402
|
||||
from app.jobs import cascade, embed, enrich, stages # noqa: E402
|
||||
from app.jobs import cascade, embed, fetch_text, stages # noqa: E402
|
||||
from app.main import create_app # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
from desk.models import Item # noqa: E402
|
||||
@@ -78,9 +78,9 @@ def store(tmp_path: Path) -> Store:
|
||||
return Store(tmp_path / "roledesk.db")
|
||||
|
||||
|
||||
def config(tmp_path: Path, **enrich_settings: Any) -> Config:
|
||||
def config(tmp_path: Path, **settings: Any) -> Config:
|
||||
return Config(database=str(tmp_path / "roledesk.db"), embed={"model": "fake"},
|
||||
enrich=enrich_settings)
|
||||
fetch_text=settings)
|
||||
|
||||
|
||||
def board(store: Store, target: str) -> Any:
|
||||
@@ -103,16 +103,20 @@ def role(store: Store, target: str, native: str, *, body: str = "",
|
||||
|
||||
|
||||
def scored(store: Store, name: str, scores: dict[str, float], *,
|
||||
gated: tuple[str, ...] = ()) -> Any:
|
||||
gated: tuple[str, ...] = (), out: tuple[str, ...] = ()) -> Any:
|
||||
"""An agent whose scores are these, every ungated role in its cut by fit
|
||||
but those in `out`."""
|
||||
agent = store.save_agent(name, f"{name} python backend")
|
||||
store.replace_scores(agent.id, [(uid, value, "gated" if uid in gated else "")
|
||||
for uid, value in scores.items()])
|
||||
store.replace_scores(agent.id, [
|
||||
(uid, value, "gated" if uid in gated else "", "", None, value,
|
||||
"" if uid in gated or uid in out else "fit")
|
||||
for uid, value in scores.items()])
|
||||
return agent
|
||||
|
||||
|
||||
# -------------------------------------------------------------- which roles
|
||||
|
||||
def test_roles_go_best_first_by_their_best_agent_score(
|
||||
def test_roles_in_the_cut_go_best_first_by_their_best_agent_score(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
board(store, "fake:acme")
|
||||
board(store, "fake:beta")
|
||||
@@ -121,25 +125,28 @@ def test_roles_go_best_first_by_their_best_agent_score(
|
||||
scored(store, "One", {a1: 0.6, a2: 0.9, b1: 0.55})
|
||||
scored(store, "Two", {a1: 0.95, b1: 0.7})
|
||||
|
||||
assert [(r["uid"], r["score"]) for r in store.enrich_waiting(500, 0.5)] == [
|
||||
assert [(r["uid"], r["score"]) for r in store.fetch_text_waiting()] == [
|
||||
(a1, 0.95), (a2, 0.9), (b1, 0.7)]
|
||||
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [a1, a2]),
|
||||
assert fetch_text.plan(config(tmp_path), store) == [("fake:acme", [a1, a2]),
|
||||
("fake:beta", [b1])]
|
||||
|
||||
|
||||
def test_a_role_no_agent_ranks_high_enough_gets_no_call(
|
||||
def test_a_role_in_no_cut_gets_no_call(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
board(store, "fake:acme")
|
||||
low, gated, deep, kept = (role(store, "fake:acme", n) for n in ("1", "2", "3", "4"))
|
||||
scored(store, "One", {low: 0.4, gated: 0.0, deep: 0.8, kept: 0.9}, gated=(gated,))
|
||||
# `top` 1 per agent: only the best ungated role of each agent is in reach.
|
||||
assert enrich.plan(config(tmp_path, top=1), store) == [("fake:acme", [kept])]
|
||||
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [kept, deep])]
|
||||
scored(store, "One", {low: 0.4, gated: 0.0, deep: 0.8, kept: 0.9}, gated=(gated,),
|
||||
out=(low,))
|
||||
assert fetch_text.plan(config(tmp_path), store) == [("fake:acme", [kept, deep])]
|
||||
# In another agent's cut, it is read.
|
||||
scored(store, "Two", {low: 0.4})
|
||||
assert fetch_text.plan(config(tmp_path), store) == [("fake:acme", [kept, deep, low])]
|
||||
store.delete_agent(store.agents()[1].id)
|
||||
# A disabled agent ranks nothing.
|
||||
agent = store.agents()[0]
|
||||
store.save_agent(agent.name, gates=agent.gates, weights=agent.weights, enabled=False,
|
||||
agent_id=agent.id, limits=agent.limits, prompts=agent.prompts)
|
||||
assert enrich.plan(config(tmp_path), store) == []
|
||||
assert fetch_text.plan(config(tmp_path), store) == []
|
||||
|
||||
|
||||
def test_roles_with_text_hosts_without_pages_and_failing_boards_are_left_out(
|
||||
@@ -152,7 +159,7 @@ def test_roles_with_text_hosts_without_pages_and_failing_boards_are_left_out(
|
||||
down = role(store, "fake:down", "1")
|
||||
store.state["ats:failing:fake:down"] = json.dumps({"since": "2026-09-29", "reason": "500"})
|
||||
scored(store, "One", {wanted: 0.9, has_text: 0.95, plain: 0.9, down: 0.9})
|
||||
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [wanted])]
|
||||
assert fetch_text.plan(config(tmp_path), store) == [("fake:acme", [wanted])]
|
||||
|
||||
|
||||
def test_a_role_read_already_waits_for_the_retry_and_a_failed_one_goes_last(
|
||||
@@ -164,7 +171,7 @@ def test_a_role_read_already_waits_for_the_retry_and_a_failed_one_goes_last(
|
||||
store.state["ats:detailempty:fake:acme"] = json.dumps({"fake:acme:1": now})
|
||||
store.state["ats:detailfailed:fake:acme"] = json.dumps(["fake:acme:2"])
|
||||
scored(store, "One", {read: 0.9, failed: 0.8, fresh: 0.6})
|
||||
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [fresh, failed])]
|
||||
assert fetch_text.plan(config(tmp_path), store) == [("fake:acme", [fresh, failed])]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ budgets
|
||||
@@ -180,11 +187,11 @@ def test_the_budgets_per_board_and_per_run(
|
||||
for n, uid in enumerate(u for pair in zip(acme, beta, strict=True)
|
||||
for u in pair)})
|
||||
|
||||
assert enrich.plan(config(tmp_path, per_board=2), store) == [
|
||||
assert fetch_text.plan(config(tmp_path, per_board=2), store) == [
|
||||
("fake:acme", acme[:2]), ("fake:beta", beta[:2])]
|
||||
assert enrich.plan(config(tmp_path, per_run=3), store) == [
|
||||
assert fetch_text.plan(config(tmp_path, per_run=3), store) == [
|
||||
("fake:acme", acme[:2]), ("fake:beta", beta[:1])]
|
||||
assert enrich.plan(config(tmp_path, per_board=1, per_run=5), store) == [
|
||||
assert fetch_text.plan(config(tmp_path, per_board=1, per_run=5), store) == [
|
||||
("fake:acme", acme[:1]), ("fake:beta", beta[:1])]
|
||||
|
||||
|
||||
@@ -192,12 +199,12 @@ def test_the_stage_off_finds_nothing(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
board(store, "fake:acme")
|
||||
scored(store, "One", {role(store, "fake:acme", "1"): 0.9})
|
||||
assert len(enrich.batches(config(tmp_path), store)) == 1
|
||||
assert enrich.batches(config(tmp_path, enabled=False), store) == []
|
||||
stages.set_stage(store, "enrich", True)
|
||||
assert len(enrich.batches(config(tmp_path, enabled=False), store)) == 1
|
||||
stages.set_stage(store, "enrich", False)
|
||||
assert enrich.batches(config(tmp_path), store) == []
|
||||
assert len(fetch_text.batches(config(tmp_path), store)) == 1
|
||||
assert fetch_text.batches(config(tmp_path, enabled=False), store) == []
|
||||
stages.set_stage(store, "fetch_text", True)
|
||||
assert len(fetch_text.batches(config(tmp_path, enabled=False), store)) == 1
|
||||
stages.set_stage(store, "fetch_text", False)
|
||||
assert fetch_text.batches(config(tmp_path), store) == []
|
||||
|
||||
|
||||
# ------------------------------------------------------------ what is stored
|
||||
@@ -209,7 +216,7 @@ def test_a_batch_stores_what_the_pages_filled_and_drops_those_vectors(
|
||||
store.put_vectors("fake", [(uid, np.ones(3, dtype=np.float32)) for uid in (one, two, gone)])
|
||||
web["answers"].update({f"{BASE}/acme/1": b"Python in full", f"{BASE}/acme/2": b"Go too"})
|
||||
|
||||
out = enrich.run_batch(config(tmp_path), store, target="fake:acme", uids=[two, gone, one])
|
||||
out = fetch_text.run_batch(config(tmp_path), store, target="fake:acme", uids=[two, gone, one])
|
||||
|
||||
assert out == {"target": "fake:acme", "read": 3, "filled": 2, "texts": 2}
|
||||
assert web["asked"] == [f"{BASE}/acme/2", f"{BASE}/acme/3", f"{BASE}/acme/1"]
|
||||
@@ -235,7 +242,7 @@ def test_a_board_switched_off_since_the_plan_reads_nothing(
|
||||
sub = board(store, "fake:acme")
|
||||
uid = role(store, "fake:acme", "1")
|
||||
store.update_subscription(sub.id, fetch=0)
|
||||
assert enrich.run_batch(config(tmp_path), store, target="fake:acme", uids=[uid])["filled"] == 0
|
||||
assert fetch_text.run_batch(config(tmp_path), store, target="fake:acme", uids=[uid])["filled"] == 0
|
||||
assert web["asked"] == []
|
||||
|
||||
|
||||
@@ -273,15 +280,15 @@ def test_a_search_whose_reader_has_detail_is_planned_like_a_board(
|
||||
f = role(store, "fake:acme", "1")
|
||||
scored(store, "One", {a: 0.9, b: 0.6, j: 0.95, f: 0.7})
|
||||
|
||||
assert enrich.plan(config(tmp_path), store) == [("jobbank:cph", [a, b]), ("fake:acme", [f])]
|
||||
assert fetch_text.plan(config(tmp_path), store) == [("jobbank:cph", [a, b]), ("fake:acme", [f])]
|
||||
# The budgets are a board's: `per_board` and `per_run` count the search as one.
|
||||
assert enrich.plan(config(tmp_path, per_board=1), store) == [("jobbank:cph", [a]),
|
||||
assert fetch_text.plan(config(tmp_path, per_board=1), store) == [("jobbank:cph", [a]),
|
||||
("fake:acme", [f])]
|
||||
assert enrich.plan(config(tmp_path, per_run=2), store) == [("jobbank:cph", [a]),
|
||||
assert fetch_text.plan(config(tmp_path, per_run=2), store) == [("jobbank:cph", [a]),
|
||||
("fake:acme", [f])]
|
||||
# A failing search is left out, as a failing board is.
|
||||
store.state["search:failing:jobbank:cph"] = "{}"
|
||||
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [f])]
|
||||
assert fetch_text.plan(config(tmp_path), store) == [("fake:acme", [f])]
|
||||
|
||||
|
||||
def test_a_search_role_read_waits_for_the_retry_and_a_failed_one_goes_last(
|
||||
@@ -296,7 +303,7 @@ def test_a_search_role_read_waits_for_the_retry_and_a_failed_one_goes_last(
|
||||
"jobbank:cph:2": "2020-01-01T00:00:00+00:00"})
|
||||
|
||||
# 1 gave no text a moment ago and waits; 2 failed and is due, after 3.
|
||||
assert enrich.plan(config(tmp_path), store) == [("jobbank:cph", [three, two])]
|
||||
assert fetch_text.plan(config(tmp_path), store) == [("jobbank:cph", [three, two])]
|
||||
|
||||
|
||||
def test_a_batch_reads_a_search_roles_page_and_stores_its_text(
|
||||
@@ -306,7 +313,7 @@ def test_a_batch_reads_a_search_roles_page_and_stores_its_text(
|
||||
store.put_vectors("fake", [(uid, np.ones(3, dtype=np.float32))])
|
||||
web["answers"][AD_URL] = AD
|
||||
|
||||
out = enrich.run_batch(config(tmp_path), store, target="jobbank:cph", uids=[uid])
|
||||
out = fetch_text.run_batch(config(tmp_path), store, target="jobbank:cph", uids=[uid])
|
||||
|
||||
assert out == {"target": "jobbank:cph", "read": 1, "filled": 1, "texts": 1}
|
||||
assert web["asked"] == [AD_URL]
|
||||
@@ -318,14 +325,14 @@ def test_a_batch_reads_a_search_roles_page_and_stores_its_text(
|
||||
assert json.loads(store.state["ats:detailed:jobbank:cph"]) == ["jobbank:cph:1234"]
|
||||
# Read, it is not planned again, and has no text left to wait for.
|
||||
scored(store, "One", {uid: 0.9})
|
||||
assert enrich.plan(config(tmp_path), store) == []
|
||||
assert fetch_text.plan(config(tmp_path), store) == []
|
||||
|
||||
|
||||
def test_a_search_batch_counts_a_404_as_read_as_a_board_does(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
search(store)
|
||||
one, two = search_role(store, "jobbank:cph", "1"), search_role(store, "jobbank:cph", "2")
|
||||
out = enrich.run_batch(config(tmp_path), store, target="jobbank:cph", uids=[one, two])
|
||||
out = fetch_text.run_batch(config(tmp_path), store, target="jobbank:cph", uids=[one, two])
|
||||
assert out["filled"] == 0 and out["read"] == 2
|
||||
assert json.loads(store.state["ats:detailed:jobbank:cph"]) == [
|
||||
"jobbank:cph:1", "jobbank:cph:2"]
|
||||
@@ -356,20 +363,21 @@ class _Context:
|
||||
return t
|
||||
|
||||
|
||||
def test_nothing_filled_passes_over_the_prep_embed_and_score_behind_enrich() -> None:
|
||||
def test_nothing_filled_passes_over_the_prep_embed_and_score_behind_fetch_text() -> None:
|
||||
context = _Context()
|
||||
empty = Tally(results=({"filled": 0},))
|
||||
out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 4},
|
||||
out = cascade.step(None, None, empty, {"cascade": "pull", "task": "fetch_text", "at": 4},
|
||||
context=context)
|
||||
assert out["next"] == "extract" and context.made[0].carries["at"] == 8
|
||||
|
||||
out = cascade.step(None, None, Tally(results=({"filled": 2}, {"filled": 0})),
|
||||
{"cascade": "pull", "task": "enrich", "at": 4}, context=context)
|
||||
{"cascade": "pull", "task": "fetch_text", "at": 4}, context=context)
|
||||
assert out["next"] == "prep" and context.made[1].carries["at"] == 5
|
||||
|
||||
out = cascade.step(None, None, empty, {"cascade": "enrich", "task": "enrich", "at": 0},
|
||||
out = cascade.step(None, None, empty,
|
||||
{"cascade": "fetch_text", "task": "fetch_text", "at": 0},
|
||||
context=context)
|
||||
assert out["next"] == "judge" and context.made[2].carries["at"] == 4
|
||||
assert out["next"] == "extract" and context.made[2].carries["at"] == 4
|
||||
|
||||
|
||||
class _Embedder:
|
||||
@@ -378,28 +386,28 @@ class _Embedder:
|
||||
for t in texts])
|
||||
|
||||
|
||||
def test_the_enrich_job_reads_the_pages_then_embeds_and_scores_those_roles(
|
||||
def test_the_fetch_text_job_reads_the_pages_then_embeds_and_scores_those_roles(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any],
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(embed, "load_embedder", lambda model: _Embedder())
|
||||
board(store, "fake:acme")
|
||||
one, two = role(store, "fake:acme", "1"), role(store, "fake:acme", "2")
|
||||
store.put_vectors("fake", [(uid, np.ones(2, dtype=np.float32)) for uid in (one, two)])
|
||||
agent = scored(store, "One", {one: 0.9, two: 0.1})
|
||||
agent = scored(store, "One", {one: 0.9, two: 0.1}, out=(two,))
|
||||
web["answers"][f"{BASE}/acme/1"] = b"Python in full"
|
||||
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config(tmp_path), store,
|
||||
workers={"cpu": 1, "http": 1, "llm": 1})
|
||||
|
||||
started = queue.enqueue("enrich")
|
||||
started = queue.enqueue("fetch_text")
|
||||
queue.drain(timeout=60)
|
||||
|
||||
assert started is not None
|
||||
jobs = sorted(queue.jobs.of_cascade(queue.jobs.last(enrich.KIND).cascade_id), # type: ignore[union-attr]
|
||||
jobs = sorted(queue.jobs.of_cascade(queue.jobs.last(fetch_text.KIND).cascade_id), # type: ignore[union-attr]
|
||||
key=lambda j: j.id)
|
||||
assert [j.result.get("task") for j in jobs if j.kind == FINAL] == \
|
||||
["enrich", "prep", "embed", "score", "judge"]
|
||||
["fetch_text", "prep", "embed", "score", "extract", "score", "judge", "summarise"]
|
||||
assert [j.kind for j in jobs if j.kind != FINAL] == \
|
||||
["enrich", enrich.KIND, "prep", "embed", "score"]
|
||||
["fetch_text", fetch_text.KIND, "prep", "embed", "score", "score"]
|
||||
assert web["asked"] == [f"{BASE}/acme/1"]
|
||||
embedded = next(j for j in jobs if j.kind == "embed")
|
||||
assert embedded.result["embedded"] == 1
|
||||
@@ -424,43 +432,43 @@ def client(app: FastAPI) -> Iterator[TestClient]:
|
||||
ANY_STATE = ("queued", "running", "done", "failed", "cancelled")
|
||||
|
||||
|
||||
def _enrich_behind_a_score(queue: Any) -> Any:
|
||||
found = queue.jobs.last("enrich", states=ANY_STATE)
|
||||
def _fetch_text_behind_a_score(queue: Any) -> Any:
|
||||
found = queue.jobs.last("fetch_text", states=ANY_STATE)
|
||||
assert found is not None
|
||||
assert [queue.jobs.get(i).kind for i in found.depends_on] == ["score"]
|
||||
assert found.after == "done"
|
||||
return found
|
||||
|
||||
|
||||
def test_saving_an_agent_queues_a_score_then_enrich(client: TestClient, app: FastAPI) -> None:
|
||||
def test_saving_an_agent_queues_a_score_then_fetch_text(client: TestClient, app: FastAPI) -> None:
|
||||
res = client.post("/api/agents", json={"name": "Backend", "intent": "python backend"})
|
||||
assert res.status_code == 200
|
||||
_enrich_behind_a_score(app.state.queue)
|
||||
_fetch_text_behind_a_score(app.state.queue)
|
||||
res = client.post("/api/agents", json={"id": res.json()["id"], "name": "Backend",
|
||||
"intent": "go backend"})
|
||||
assert res.status_code == 200
|
||||
|
||||
|
||||
def test_a_company_affinity_queues_a_score_then_enrich(
|
||||
def test_a_company_affinity_queues_a_score_then_fetch_text(
|
||||
client: TestClient, app: FastAPI) -> None:
|
||||
store: Store = app.state.store
|
||||
store.save_agent("Backend", "python backend")
|
||||
store.add_company("Acme")
|
||||
res = client.post("/api/companies/acme/affinity", json={"affinity": 2})
|
||||
assert res.status_code == 200
|
||||
found = _enrich_behind_a_score(app.state.queue)
|
||||
found = _fetch_text_behind_a_score(app.state.queue)
|
||||
assert found.depends_on == [res.json()["score_job"]]
|
||||
|
||||
|
||||
def test_one_score_is_one_enrich(app: FastAPI) -> None:
|
||||
def test_one_score_is_one_fetch_text(app: FastAPI) -> None:
|
||||
queue = app.state.queue
|
||||
score = queue.enqueue("score")
|
||||
first = cascade.enrich_after(queue, score)
|
||||
assert cascade.enrich_after(queue, score).id == first.id
|
||||
assert cascade.enrich_after(queue, None) is None
|
||||
first = cascade.fetch_text_after(queue, score)
|
||||
assert cascade.fetch_text_after(queue, score).id == first.id
|
||||
assert cascade.fetch_text_after(queue, None) is None
|
||||
|
||||
|
||||
def test_a_vote_queues_neither_score_nor_enrich(client: TestClient, app: FastAPI) -> None:
|
||||
def test_a_vote_queues_neither_score_nor_fetch_text(client: TestClient, app: FastAPI) -> None:
|
||||
store: Store = app.state.store
|
||||
store.save_agent("Backend", "python backend")
|
||||
item = Item(source_id="fake:acme:1", location=f"{BASE}/acme/1", title="Backend Engineer",
|
||||
@@ -472,5 +480,5 @@ def test_a_vote_queues_neither_score_nor_enrich(client: TestClient, app: FastAPI
|
||||
assert client.delete(f"/api/roles/{item.uid}/vote").status_code == 200
|
||||
|
||||
queue = app.state.queue
|
||||
assert queue.jobs.last("enrich", states=ANY_STATE) is None
|
||||
assert queue.jobs.last("fetch_text", states=ANY_STATE) is None
|
||||
assert queue.jobs.last("score", states=ANY_STATE) is None
|
||||
@@ -440,7 +440,6 @@ EUROPE_DK_SE = [
|
||||
Gate(field="country", values=["DK", "SE"], mode="any"),
|
||||
Gate(field="remote", values=["remote"], mode="any"),
|
||||
Gate(field="place", values=["Malmö", "copenhagen"], mode="any"),
|
||||
Gate(field="remote", values=["hybrid"], mode="prefer", weight=0.05),
|
||||
]
|
||||
|
||||
|
||||
|
||||
+22
-14
@@ -14,7 +14,7 @@ import pytest
|
||||
|
||||
from app.config import Config
|
||||
from app.jobs import judge, stages
|
||||
from app.models import CvLanguage, Limits, Prompt
|
||||
from app.models import CvLanguage, Prompt
|
||||
from app.storage import Store
|
||||
from desk import modeltask
|
||||
from desk.models import Item
|
||||
@@ -85,7 +85,8 @@ def ranked(store: Store, owner: Any, *ids: str) -> list[str]:
|
||||
for sid in ids:
|
||||
store.upsert_role(posting(sid))
|
||||
uids.append(posting(sid).uid)
|
||||
store.replace_scores(owner.id, [(uid, 0.9 - n / 100, "") for n, uid in enumerate(uids)])
|
||||
store.replace_scores(owner.id, [(uid, 0.9 - n / 100, "", "", None, 0.9 - n / 100, "fit")
|
||||
for n, uid in enumerate(uids)])
|
||||
return uids
|
||||
|
||||
|
||||
@@ -192,20 +193,27 @@ def test_only_enabled_agents_with_a_cv_and_a_prompt_are_judged(store: Store) ->
|
||||
assert judge.waiting(store) == []
|
||||
|
||||
|
||||
def test_the_agents_judge_limits_bound_what_is_read(store: Store) -> None:
|
||||
owner = hunter(store, limits=Limits(judge_top=2, judge_min=0.885))
|
||||
uids = ranked(store, owner, "a", "b", "c")
|
||||
assert [row["uid"] for _, row, _, _ in judge.waiting(store)] == uids[:2]
|
||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
||||
limits=Limits(judge_top=2, judge_min=0.895))
|
||||
assert [row["uid"] for _, row, _, _ in judge.waiting(store)] == uids[:1]
|
||||
def test_the_agents_cut_is_what_is_read_by_fit_then_cv_then_kept(store: Store) -> None:
|
||||
owner = hunter(store)
|
||||
for sid in "abcdef":
|
||||
store.upsert_role(posting(sid))
|
||||
uid = {sid: posting(sid).uid for sid in "abcdef"}
|
||||
store.replace_scores(owner.id, [
|
||||
(uid["a"], 0.5, "", "", 0.9, 0.5, "kept"),
|
||||
(uid["b"], 0.6, "", "", 0.7, 0.4, "cv"),
|
||||
(uid["c"], 0.7, "", "", 0.1, 0.7, "fit"),
|
||||
(uid["d"], 0.8, "", "", 0.1, 0.8, "fit"),
|
||||
(uid["e"], 0.9, "", "", 0.9, 0.9, ""),
|
||||
(uid["f"], 0.9, "", "", 0.8, 0.3, "cv")])
|
||||
assert [row["uid"] for _, row, _, _ in judge.waiting(store)] == [
|
||||
uid["d"], uid["c"], uid["f"], uid["b"], uid["a"]]
|
||||
|
||||
|
||||
def test_a_role_without_text_is_not_judged(store: Store) -> None:
|
||||
owner = hunter(store)
|
||||
item = posting("empty", body="")
|
||||
store.upsert_role(item)
|
||||
store.replace_scores(owner.id, [(item.uid, 0.9, "")])
|
||||
store.replace_scores(owner.id, [(item.uid, 0.9, "", "", None, 0.9, "fit")])
|
||||
assert judge.waiting(store) == []
|
||||
|
||||
|
||||
@@ -218,7 +226,7 @@ def test_a_judgement_is_read_again_only_when_what_it_read_changed(
|
||||
assert judge.pending(store) == 0
|
||||
|
||||
# A rescore and the role closing leave it be.
|
||||
store.replace_scores(owner.id, [(uid, 0.8, "")])
|
||||
store.replace_scores(owner.id, [(uid, 0.8, "", "", None, 0.8, "fit")])
|
||||
assert judge.pending(store) == 0
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET closed_at = '2026-10-01' WHERE uid = ?", (uid,))
|
||||
@@ -274,7 +282,7 @@ def test_a_batch_holds_one_agents_pairs_with_that_agents_profile(
|
||||
first = hunter(store, "First", cv="First CV.")
|
||||
second = hunter(store, "Second", cv="Second CV.")
|
||||
shared = ranked(store, first, "a", "b")
|
||||
store.replace_scores(second.id, [(uid, 0.9, "") for uid in shared])
|
||||
store.replace_scores(second.id, [(uid, 0.9, "", "", None, 0.9, "fit") for uid in shared])
|
||||
model = stub(monkeypatch)
|
||||
|
||||
modeltask.run(judge.TASK, config, store)
|
||||
@@ -295,7 +303,7 @@ def test_the_finder_cuts_per_agent_and_finds_nothing_while_the_switch_is_off(
|
||||
first = hunter(store, "First")
|
||||
second = hunter(store, "Second")
|
||||
uids = ranked(store, first, *"abcdefg")
|
||||
store.replace_scores(second.id, [(uids[0], 0.9, "")])
|
||||
store.replace_scores(second.id, [(uids[0], 0.9, "", "", None, 0.9, "fit")])
|
||||
found = judge.judge_task().finder(config, store)
|
||||
assert [len(b.payload["uids"]) for b in found] == [5, 2, 1]
|
||||
assert found[0].label == "Judging 5 roles for First, 1 of 3"
|
||||
@@ -309,7 +317,7 @@ def test_a_job_given_two_agents_keys_asks_once_for_each(
|
||||
first = hunter(store, "First", cv="First CV.")
|
||||
second = hunter(store, "Second", cv="Second CV.")
|
||||
[uid] = ranked(store, first, "a")
|
||||
store.replace_scores(second.id, [(uid, 0.9, "")])
|
||||
store.replace_scores(second.id, [(uid, 0.9, "", "", None, 0.9, "fit")])
|
||||
model = stub(monkeypatch)
|
||||
|
||||
result = judge.run_batch(config, store, uids=[f"{first.id}:{uid}", f"{second.id}:{uid}"])
|
||||
|
||||
+1
-1
@@ -316,7 +316,7 @@ def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
|
||||
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]
|
||||
one = row(migrated, "1")
|
||||
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
|
||||
one["country_host"]) == ("senior", "full_time", "true", "DK")
|
||||
|
||||
@@ -366,7 +366,7 @@ def test_extract_reads_the_area_once_more_and_maps_the_role_before_the_rescore(
|
||||
uid = add(store, "1", "Freiburg, Germany", board="ashby", remote="OnSite")
|
||||
prep.run(store, dupes=False)
|
||||
agent = store.save_agent("Backend", "python backend")
|
||||
store.replace_scores(agent.id, [(uid, 1.0, "")])
|
||||
store.replace_scores(agent.id, [(uid, 1.0, "", "", None, 1.0, "fit")])
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
|
||||
"seniority_model = 'senior' WHERE uid = ?", (uid,))
|
||||
@@ -530,7 +530,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path:
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [27, 28, 29, 30, 31]
|
||||
assert migrated.migrations_applied == [27, 28, 29, 30, 31, 32]
|
||||
assert migrated.gazetteer().anywhere is not None
|
||||
row = migrated._row("SELECT remote_host, remote_model, remote_where_model FROM roles "
|
||||
"WHERE uid = ?", (board,))
|
||||
@@ -540,7 +540,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path:
|
||||
assert area(migrated, search.uid) == ("", "")
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("PRAGMA user_version = 26")
|
||||
assert Store(path).migrations_applied == [27, 28, 29, 30, 31]
|
||||
assert Store(path).migrations_applied == [27, 28, 29, 30, 31, 32]
|
||||
|
||||
|
||||
def test_a_role_waits_for_extract_until_it_was_asked_where(store: Store) -> None:
|
||||
|
||||
+80
-78
@@ -12,6 +12,7 @@ from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
@@ -353,18 +354,25 @@ def test_any_gates_are_one_group_a_role_must_match_one_of(
|
||||
def test_failed_gates_reads_evidence_only() -> None:
|
||||
gates = [Gate(field="country", values=["DK"]),
|
||||
Gate(field="remote", values=["remote"], mode="any"),
|
||||
Gate(field="seniority", values=["senior"], mode="any"),
|
||||
Gate(field="function", values=["engineering"], mode="prefer")]
|
||||
Gate(field="seniority", values=["senior"], mode="any")]
|
||||
def cand(*states: int) -> Candidate:
|
||||
return Candidate(uid="u", posted_at=NOW.isoformat(), affinity=0, gates=list(states))
|
||||
assert score.failed_gates(cand(1, 1, 0, 0), gates) == []
|
||||
assert score.failed_gates(cand(-1, -1, -1, 0), gates) == []
|
||||
assert score.failed_gates(cand(0, 0, -1, 1), gates) == [
|
||||
assert score.failed_gates(cand(1, 1, 0), gates) == []
|
||||
assert score.failed_gates(cand(-1, -1, -1), gates) == []
|
||||
assert score.failed_gates(cand(0, 0, -1), gates) == [
|
||||
"country not DK", "none of remote remote, seniority senior"]
|
||||
assert score.failed_gates(cand(1, 0, 0, 1), gates) == [
|
||||
assert score.failed_gates(cand(1, 0, 0), gates) == [
|
||||
"none of remote remote, seniority senior"]
|
||||
|
||||
|
||||
def test_a_gate_is_require_any_or_exclude_and_has_no_weight() -> None:
|
||||
"""Prefer gates left the agent: hybrid is a feed filter now."""
|
||||
with pytest.raises(ValidationError):
|
||||
Gate(field="remote", values=["hybrid"], mode="prefer") # type: ignore[arg-type]
|
||||
with pytest.raises(ValidationError):
|
||||
Gate.model_validate({"field": "remote", "values": ["remote"], "weight": 1.0})
|
||||
|
||||
|
||||
def test_an_exclude_gate_on_title_gates_whole_words_and_names_the_value(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
"""The sales and data engineering titles the owner does not want: a
|
||||
@@ -389,7 +397,7 @@ def test_an_exclude_gate_on_title_gates_whole_words_and_names_the_value(
|
||||
|
||||
def test_a_favourite_is_gated_by_an_exclude_gate_it_matches(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
"""A star skips the `any` group, not the titles the agent leaves out."""
|
||||
"""A star raises the level only: it skips no part of the fetch gate."""
|
||||
store.add_company("Fave")
|
||||
director = role(store, "1", "Account Director, Nordics", company="Fave")
|
||||
engineer = role(store, "2", "Forward Deployed Engineer", company="Fave")
|
||||
@@ -415,21 +423,6 @@ def test_failed_gates_names_the_value_an_exclude_gate_matched() -> None:
|
||||
assert score.failed_gates(cand(0, 1), gates) == ["country not DK", "title account executive"]
|
||||
|
||||
|
||||
def test_a_prefer_gate_adds_its_weight_to_the_score(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
remote = role(store, "1", remote="remote")
|
||||
office = role(store, "2", remote="onsite")
|
||||
agent = store.save_agent("Remote", "", [Gate(field="remote", values=["remote"],
|
||||
mode="prefer", weight=0.5)])
|
||||
|
||||
run(config, store, score.run_score)
|
||||
|
||||
scores = store.scores(agent.id)
|
||||
# No intent, so fit is out: recency 0.2 + affinity 0.2 + the gate's 0.5.
|
||||
assert scores[remote][0] - scores[office][0] == pytest.approx(0.5 / 0.9, abs=1e-5)
|
||||
assert scores[office][1] == ""
|
||||
|
||||
|
||||
def test_a_favourite_is_gated_by_a_require_gate_it_fails(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
"""A star widens the search to the company, not past a require gate: the
|
||||
@@ -447,40 +440,70 @@ def test_a_favourite_is_gated_by_a_require_gate_it_fails(
|
||||
assert scores[berlin][1] == "" and scores[berlin][0] > 0
|
||||
|
||||
|
||||
def test_a_favourite_passes_the_any_group_others_fail(
|
||||
def test_all_countries_passes_the_any_group_a_favourite_does_not(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
""""Denmark or remote" gates the onsite German role of a neutral company
|
||||
and lets the same role of a starred one through."""
|
||||
store.add_company("Fave")
|
||||
and of a favourite, and lets the same role of an All countries company
|
||||
through."""
|
||||
for name in ("Fave", "Wide", "Other"):
|
||||
store.add_company(name)
|
||||
fave = role(store, "1", company="Fave", place="Berlin, Germany", remote="onsite")
|
||||
other = role(store, "2", company="Other", place="Berlin, Germany", remote="onsite")
|
||||
wide = role(store, "2", company="Wide", place="Berlin, Germany", remote="onsite")
|
||||
other = role(store, "3", company="Other", place="Berlin, Germany", remote="onsite")
|
||||
agent = store.save_agent("Copenhagen or remote", "", [
|
||||
Gate(field="country", values=["DK"], mode="any"),
|
||||
Gate(field="remote", values=["remote"], mode="any")])
|
||||
store.set_affinity(agent.id, "fave", 2)
|
||||
store.set_all_countries("wide", True)
|
||||
|
||||
run(config, store, score.run_score)
|
||||
|
||||
scores = store.scores(agent.id)
|
||||
assert scores[fave][1] == "" and scores[fave][0] > 0
|
||||
assert scores[wide][1] == "" and scores[wide][0] > 0
|
||||
assert scores[fave] == (0.0, "none of country DK, remote remote")
|
||||
assert scores[other] == (0.0, "none of country DK, remote remote")
|
||||
|
||||
|
||||
def test_a_favourite_takes_both_halves_of_the_rule_in_one_agent() -> None:
|
||||
gates = [Gate(field="country", values=["DK"]),
|
||||
def test_all_countries_skips_the_place_part_and_all_roles_the_titles() -> None:
|
||||
"""The region applies to every company; All countries lifts the `any`
|
||||
group, All roles the exclude gates on title and no other exclude gate."""
|
||||
gates = [Gate(field="country", values=["DK", "SE", "EU"]),
|
||||
Gate(field="country", values=["DK"], mode="any"),
|
||||
Gate(field="remote", values=["remote"], mode="any"),
|
||||
Gate(field="seniority", values=["senior"], mode="any")]
|
||||
def cand(affinity: int, *states: int) -> Candidate:
|
||||
return Candidate(uid="u", posted_at=NOW.isoformat(), affinity=affinity,
|
||||
gates=list(states))
|
||||
# The any group fails for both; only the neutral company is gated by it.
|
||||
assert score.failed_gates(cand(2, 1, 0, 0), gates) == []
|
||||
assert score.failed_gates(cand(0, 1, 0, 0), gates) == [
|
||||
"none of remote remote, seniority senior"]
|
||||
# The require gate fails for both.
|
||||
assert score.failed_gates(cand(2, 0, 0, 0), gates) == ["country not DK"]
|
||||
assert score.failed_gates(cand(0, 0, 0, 0), gates) == [
|
||||
"country not DK", "none of remote remote, seniority senior"]
|
||||
Gate(field="title", values=["account executive"], mode="exclude"),
|
||||
Gate(field="company", values=["acme"], mode="exclude")]
|
||||
def cand(*states: int, wide: bool = False, every: bool = False) -> Candidate:
|
||||
return Candidate(uid="u", posted_at=NOW.isoformat(), all_countries=wide,
|
||||
all_roles=every, gates=list(states))
|
||||
assert score.failed_gates(cand(1, 0, 0, 0, 0), gates) == [
|
||||
"none of country DK, remote remote"]
|
||||
assert score.failed_gates(cand(1, 0, 0, 0, 0, wide=True), gates) == []
|
||||
assert score.failed_gates(cand(0, 0, 0, 0, 0, wide=True, every=True), gates) == [
|
||||
"country not DK / SE / EU"]
|
||||
assert score.failed_gates(cand(1, 1, 0, 1, 0), gates) == ["title account executive"]
|
||||
assert score.failed_gates(cand(1, 1, 0, 1, 0, wide=True), gates) == [
|
||||
"title account executive"]
|
||||
assert score.failed_gates(cand(1, 1, 0, 1, 0, every=True), gates) == []
|
||||
assert score.failed_gates(cand(1, 1, 0, 1, 1, every=True), gates) == ["company acme"]
|
||||
assert score.failed_gates(cand(1, 0, 0, 1, 0, every=True), gates) == [
|
||||
"none of country DK, remote remote"]
|
||||
|
||||
|
||||
def test_all_roles_keeps_a_title_the_agent_leaves_out(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
for name in ("Small", "Other"):
|
||||
store.add_company(name)
|
||||
small = role(store, "1", "Account Executive", company="Small")
|
||||
other = role(store, "2", "Account Executive", company="Other")
|
||||
agent = store.save_agent("No sales", "", [
|
||||
Gate(field="title", values=["account executive"], mode="exclude")])
|
||||
store.set_all_roles("small", True)
|
||||
|
||||
run(config, store, score.run_score)
|
||||
|
||||
scores = store.scores(agent.id)
|
||||
assert scores[small][1] == ""
|
||||
assert scores[other] == (0.0, "title account executive")
|
||||
|
||||
|
||||
def test_a_favourite_with_an_empty_column_is_not_gated() -> None:
|
||||
@@ -808,12 +831,13 @@ def test_a_bad_calibration_is_a_config_error(
|
||||
def test_an_agent_without_an_intent_feeds_neither_model_stage_and_is_named(
|
||||
store: Store, config: Config, clock: Clock, fake: FakeEmbedder) -> None:
|
||||
"""Its scores have no fit, so they rank fresh roles, not wanted ones: the
|
||||
newer role tops it, and extract and summarise must not read it on that."""
|
||||
newer role tops it, and nothing of it enters the cut by fit, nor does its
|
||||
top choose what summarise reads."""
|
||||
clock.at = NOW - timedelta(days=1)
|
||||
wanted = role(store, "1", "Backend Engineer", body="Python backend")
|
||||
clock.at = NOW
|
||||
fresh = role(store, "2", "Frontend Engineer", body="React apps")
|
||||
top_one = Limits(extract_top=1, extract_min=0, summarise_top=1, summarise_min=0)
|
||||
top_one = Limits(cut_fit_top=1, cut_fit_min=0, summarise_top=1)
|
||||
store.save_agent("Backend", "python backend", limits=top_one)
|
||||
blind = store.save_agent("Fresh", "", limits=top_one)
|
||||
store.save_agent("Also fresh", " \n", limits=top_one)
|
||||
@@ -854,26 +878,6 @@ def test_a_gated_role_does_not_stretch_the_fit_range_under_the_fallback(
|
||||
assert scores[close][0] == 1.0 and scores[far][0] == 0.0
|
||||
|
||||
|
||||
def test_a_small_prefer_gate_no_longer_outweighs_fit(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
"""The flaw the last real run showed: real cosines sit close together, so
|
||||
under `(cos + 1) / 2` a prefer gate decided the order over fit. Here the
|
||||
cosines are about 0.95 and 0.87: the old fit terms differ by 0.04, which
|
||||
at weight 0.6 is less than the gate's 0.05, so the remote role won.
|
||||
Calibrated, they are 1 and about 0.89, and the closer role wins."""
|
||||
close = role(store, "1", "Backend Engineer", body="Python backend", remote="onsite")
|
||||
far = role(store, "2", "Backend Engineer", body="Python backend data", remote="remote")
|
||||
agent = store.save_agent("Backend", "python backend",
|
||||
[Gate(field="remote", values=["remote"], mode="prefer",
|
||||
weight=0.05)])
|
||||
|
||||
run(config, store, embed.run_embed)
|
||||
run(config, store, score.run_score)
|
||||
|
||||
scores = store.scores(agent.id)
|
||||
assert scores[close][0] > scores[far][0]
|
||||
|
||||
|
||||
def test_the_intent_vector_is_cached_until_the_intent_changes(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
role(store, "1")
|
||||
@@ -1018,9 +1022,7 @@ def test_the_shipped_starter_agents_validate() -> None:
|
||||
block = [line[2:] for line in lines[start:] if line.startswith("# ")]
|
||||
examples = [AgentSeed.model_validate(a) for a in yaml.safe_load("\n".join(block))]
|
||||
assert [a.name for a in examples] == ["Remote from Europe", "Europe hybrid"]
|
||||
# Small enough that a prefer gate breaks ties rather than overrides fit.
|
||||
assert {g.weight for a in [*agents, *examples]
|
||||
for g in a.gates if g.mode == "prefer"} == {0.05}
|
||||
assert examples[0].limits.cut_fit_top == 1000
|
||||
|
||||
|
||||
def test_a_starter_agent_may_list_prompts_or_give_an_intent(
|
||||
@@ -1074,7 +1076,7 @@ def test_the_shipped_starter_searches_validate() -> None:
|
||||
|
||||
# ---------------------------------------------------------------- cascade
|
||||
|
||||
def test_the_pull_job_runs_pull_prep_embed_score_extract_score_then_summarise(
|
||||
def test_the_pull_job_runs_pull_prep_embed_score_extract_score_judge_then_summarise(
|
||||
store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
||||
@@ -1104,10 +1106,10 @@ def test_the_pull_job_runs_pull_prep_embed_score_extract_score_then_summarise(
|
||||
assert batch is not None
|
||||
jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id)
|
||||
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
|
||||
# Enrich found no board with role pages, so the prep, embed and score
|
||||
# Fetch text found no board with role pages, so the prep, embed and score
|
||||
# behind it are passed over.
|
||||
assert stages == ["pull", "prep", "embed", "score", "enrich", "extract", "score",
|
||||
"summarise", "judge"]
|
||||
assert stages == ["pull", "prep", "embed", "score", "fetch_text", "extract", "score",
|
||||
"judge", "summarise"]
|
||||
work = [j.kind for j in jobs if j.kind != FINAL]
|
||||
# The `pull` job and the chain it asked for are one cascade, so the Jobs
|
||||
# tab shows a scheduled pull as one run. Extract and summarise are off and
|
||||
@@ -1159,14 +1161,14 @@ def test_the_model_batches_are_isolated_with_a_timeout() -> None:
|
||||
|
||||
|
||||
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
|
||||
"""By position, not by name: `score` is in the chain twice, once before
|
||||
extract and once after, and the second must lead on to summarise."""
|
||||
assert cascade.CASCADES["pull"] == ("pull", "prep", "embed", "score", "enrich", "prep",
|
||||
"embed", "score", "extract", "score", "summarise",
|
||||
"judge")
|
||||
"""By position, not by name: `score` is in the chain three times, and the
|
||||
one after extract must lead on to judge."""
|
||||
assert cascade.CASCADES["pull"] == ("pull", "prep", "embed", "score", "fetch_text", "prep",
|
||||
"embed", "score", "extract", "score", "judge",
|
||||
"summarise")
|
||||
assert [cascade.follows("pull", at) for at in range(12)] == \
|
||||
["prep", "embed", "score", "enrich", "prep", "embed", "score", "extract", "score",
|
||||
"summarise", "judge", ""]
|
||||
["prep", "embed", "score", "fetch_text", "prep", "embed", "score", "extract", "score",
|
||||
"judge", "summarise", ""]
|
||||
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 11) == ""
|
||||
assert cascade.follows("nope", 0) == ""
|
||||
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
|
||||
@@ -1186,4 +1188,4 @@ def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 8}
|
||||
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 9},
|
||||
context=Context())
|
||||
assert out["next"] == "summarise" and made[1].carries["at"] == 10
|
||||
assert out["next"] == "judge" and made[1].carries["at"] == 10
|
||||
+11
-10
@@ -45,11 +45,12 @@ def store(tmp_path: Path) -> Store:
|
||||
|
||||
|
||||
def ranked(store: Store) -> None:
|
||||
"""Two roles scored well for one enabled agent: inside both stages' bounds."""
|
||||
"""Two roles scored well for one enabled agent, both in its cut."""
|
||||
for sid in ("1", "2"):
|
||||
store.upsert_role(posting(sid))
|
||||
one = store.save_agent("Backend", "python backend")
|
||||
store.replace_scores(one.id, [(posting("1").uid, 1.0, ""), (posting("2").uid, 0.9, "")])
|
||||
store.replace_scores(one.id, [(posting("1").uid, 1.0, "", "", None, 1.0, "fit"),
|
||||
(posting("2").uid, 0.9, "", "", None, 0.9, "fit")])
|
||||
|
||||
|
||||
def answered_by(monkeypatch: pytest.MonkeyPatch, backend: str) -> None:
|
||||
@@ -159,7 +160,7 @@ def test_get_stages_lists_all_four_with_their_source_model_and_who_answers(
|
||||
{"name": "judge", "on": False, "source": "config",
|
||||
"model": "claude-haiku-4-5-20251001", "backend": "api",
|
||||
"answer": "answered by the Claude API"},
|
||||
{"name": "enrich", "on": True, "source": "config", "model": "", "backend": "",
|
||||
{"name": "fetch_text", "on": True, "source": "config", "model": "", "backend": "",
|
||||
"answer": "reads role pages, asks no model"},
|
||||
]
|
||||
|
||||
@@ -169,7 +170,7 @@ def test_who_answers_is_said_for_the_api_claude_code_and_nobody(
|
||||
answers = {}
|
||||
for backend in ("api", "claude_code", ""):
|
||||
answered_by(monkeypatch, backend)
|
||||
[first, second, third, _enrich] = client.get("/api/stages").json()
|
||||
[first, second, third, _fetch_text] = client.get("/api/stages").json()
|
||||
assert first["backend"] == second["backend"] == third["backend"] == backend
|
||||
answers[backend] = first["answer"]
|
||||
|
||||
@@ -196,16 +197,16 @@ def test_put_stores_the_setting_returns_the_same_shape_and_starts_nothing(
|
||||
assert res.json()["on"] is False and store.state["stage:summarise"] == "off"
|
||||
|
||||
|
||||
def test_the_enrich_switch_is_listed_and_stored_like_the_others(
|
||||
def test_the_fetch_text_switch_is_listed_and_stored_like_the_others(
|
||||
client: TestClient, store: Store) -> None:
|
||||
res = client.put("/api/stages/enrich", json={"on": False})
|
||||
res = client.put("/api/stages/fetch_text", json={"on": False})
|
||||
|
||||
assert res.status_code == 200
|
||||
assert res.json()["name"] == "enrich" and res.json()["on"] is False
|
||||
assert store.state["stage:enrich"] == "off"
|
||||
assert res.json()["name"] == "fetch_text" and res.json()["on"] is False
|
||||
assert store.state["stage:fetch_text"] == "off"
|
||||
listed = {stage["name"]: stage for stage in client.get("/api/stages").json()}
|
||||
assert list(listed) == ["extract", "summarise", "judge", "enrich"]
|
||||
assert listed["enrich"]["on"] is False and listed["enrich"]["source"] == "stored"
|
||||
assert list(listed) == ["extract", "summarise", "judge", "fetch_text"]
|
||||
assert listed["fetch_text"]["on"] is False and listed["fetch_text"]["source"] == "stored"
|
||||
|
||||
|
||||
def test_an_unknown_stage_is_404_and_a_bad_body_is_422(client: TestClient) -> None:
|
||||
|
||||
+42
-39
@@ -70,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
|
||||
"application_events", "agents", "scores", "state"} <= tables
|
||||
assert "idx_roles_cluster" in indexes
|
||||
assert store.migrations_applied == []
|
||||
assert LATEST == 31
|
||||
assert LATEST == 32
|
||||
# A second open finds it current and applies nothing.
|
||||
assert Store(path).migrations_applied == []
|
||||
|
||||
@@ -190,7 +190,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
|
||||
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]
|
||||
acme = migrated.company("acme")
|
||||
assert acme is not None
|
||||
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
|
||||
@@ -395,7 +395,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
|
||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]
|
||||
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
|
||||
Prompt(name="Backend", text="python backend")]
|
||||
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
|
||||
@@ -419,7 +419,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
|
||||
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]
|
||||
acme = migrated.company("acme")
|
||||
assert acme is not None
|
||||
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
|
||||
@@ -448,7 +448,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
|
||||
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]
|
||||
with sqlite3.connect(path) as conn:
|
||||
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
|
||||
"SELECT uid, place_fold, seniority FROM roles")}
|
||||
@@ -494,22 +494,21 @@ def test_prompts_are_saved_replaced_reordered_and_deleted(store: Store) -> None:
|
||||
assert conn.execute("SELECT COUNT(*) FROM agent_prompts").fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_an_agent_whose_prompts_are_all_empty_feeds_neither_model_stage(
|
||||
def test_an_agent_whose_prompts_are_all_empty_does_not_choose_what_summarise_reads(
|
||||
store: Store) -> None:
|
||||
upsert(store, posting("a"))
|
||||
upsert(store, posting("b"))
|
||||
a, b = posting("a").uid, posting("b").uid
|
||||
top = Limits(extract_top=1, extract_min=0, summarise_top=1, summarise_min=0)
|
||||
top = Limits(summarise_top=1)
|
||||
blank = store.save_agent("Blank", limits=top,
|
||||
prompts=[Prompt(name="One"), Prompt(name="Two", text=" \n")])
|
||||
store.replace_scores(blank.id, [(a, 0.9, "")])
|
||||
assert store.extract_waiting(10) == [] and store.summarise_waiting(10) == []
|
||||
store.replace_scores(blank.id, [(a, 0.9, ""), (b, 0.8, "")])
|
||||
assert store.summarise_waiting(10) == []
|
||||
|
||||
# One prompt with text is enough.
|
||||
store.save_agent("Blank", agent_id=blank.id, limits=top,
|
||||
prompts=[Prompt(name="One"), Prompt(name="Two", text="python")])
|
||||
assert [r.uid for r in store.extract_waiting(10)] == [a]
|
||||
assert b not in {r.uid for r in store.summarise_waiting(10)}
|
||||
assert [r.uid for r in store.summarise_waiting(10)] == [a]
|
||||
|
||||
|
||||
def test_state_is_a_mapping(store: Store) -> None:
|
||||
@@ -691,7 +690,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
|
||||
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]
|
||||
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
|
||||
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
|
||||
assert migrated.role_places(stated.uid) == []
|
||||
@@ -733,7 +732,7 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [25, 26, 27, 28, 29, 30, 31]
|
||||
assert migrated.migrations_applied == [25, 26, 27, 28, 29, 30, 31, 32]
|
||||
with sqlite3.connect(path) as conn:
|
||||
found = {sid: rest for sid, *rest in conn.execute(
|
||||
"SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority "
|
||||
@@ -751,12 +750,12 @@ def test_v26_adds_posting_url_empty_and_runs_twice_without_harm(tmp_path: Path)
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [26, 27, 28, 29, 30, 31]
|
||||
assert migrated.migrations_applied == [26, 27, 28, 29, 30, 31, 32]
|
||||
role = migrated.role(posting("a").uid)
|
||||
assert role is not None and role.posting_url == ""
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("PRAGMA user_version = 25") # the column is there already
|
||||
assert Store(path).migrations_applied == [26, 27, 28, 29, 30, 31]
|
||||
assert Store(path).migrations_applied == [26, 27, 28, 29, 30, 31, 32]
|
||||
|
||||
|
||||
def test_a_pull_keeps_the_posting_url(store: Store) -> None:
|
||||
@@ -1277,21 +1276,34 @@ def test_votes_are_per_agent(store: Store) -> None:
|
||||
|
||||
def test_roles_with_an_agent_but_not_by_score_are_newest_first_with_votes(
|
||||
store: Store, clock: Clock) -> None:
|
||||
for sid in ("1", "2", "3"):
|
||||
for sid in ("1", "2", "3", "4"):
|
||||
upsert(store, posting(sid))
|
||||
clock.tick()
|
||||
uid = {r.source_id: r.uid for r in store.roles()}
|
||||
agent = store.save_agent("A")
|
||||
store.replace_scores(agent.id, [(uid["1"], 0.9, ""), (uid["2"], 0.1, "")]) # 3 unscored
|
||||
store.replace_scores(agent.id, [(uid["1"], 0.9, ""), (uid["2"], 0.1, ""),
|
||||
(uid["4"], 0.0, "country not DK")]) # 3 unscored
|
||||
store.set_vote(agent.id, uid["1"], 1)
|
||||
|
||||
newest = store.roles(agent=agent.id, by_score=False)
|
||||
assert [r.source_id for r in newest] == ["3", "2", "1"]
|
||||
assert [r.score for r in newest] == [None, None, None]
|
||||
assert [r.vote for r in newest] == [None, None, 1]
|
||||
assert store.count_roles(agent=agent.id, by_score=False) == 3
|
||||
assert [r.source_id for r in newest] == ["4", "3", "2", "1"]
|
||||
assert [r.score for r in newest] == [None, None, None, None]
|
||||
assert [r.vote for r in newest] == [None, None, None, 1]
|
||||
assert store.count_roles(agent=agent.id, by_score=False) == 4
|
||||
assert [r.source_id for r in store.roles(agent=agent.id)] == ["1", "2"]
|
||||
|
||||
# Kept to the agent's fetch gate, newest lists only what it scored and
|
||||
# passed, as the score order does; `gated` shows the rest with the reason.
|
||||
fenced = store.roles(agent=agent.id, by_score=False, fetch_gate=True)
|
||||
assert [r.source_id for r in fenced] == ["2", "1"]
|
||||
assert [r.score for r in fenced] == [None, None]
|
||||
assert store.count_roles(agent=agent.id, by_score=False, fetch_gate=True) == 2
|
||||
page = store.roles_page(agent=agent.id, by_score=False, fetch_gate=True, gated=True)
|
||||
assert [(r.uid, r.gated_reason) for r in page] == [
|
||||
(uid["4"], "country not DK"), (uid["2"], ""), (uid["1"], "")]
|
||||
assert store.count_roles(agent=agent.id, by_score=False, fetch_gate=True,
|
||||
gated=True) == 3
|
||||
|
||||
|
||||
def test_read_state_is_set_cleared_and_a_vote_marks_read(
|
||||
store: Store, clock: Clock) -> None:
|
||||
@@ -1738,39 +1750,30 @@ def test_a_summarys_tags_land_with_it_and_not_on_a_rejected_late_answer(
|
||||
|
||||
def test_summarise_waits_on_the_top_of_each_enabled_agent_by_its_own_limits(
|
||||
store: Store) -> None:
|
||||
"""Each agent contributes its own top, as deep and as low as its own
|
||||
`limits` say; a closed role and a gated score are in nobody's top. A
|
||||
"""Each agent contributes the top of its feed, its best `summarise_top`
|
||||
final scores; a closed role and a gated score are in nobody's top. A
|
||||
hidden company needs no case: `score` writes no rows for it under the
|
||||
agent that hid it. An agent saved before v8 has `limits` of '{}', which
|
||||
reads as the defaults."""
|
||||
agent that hid it. `limits` of '{}' reads as the defaults."""
|
||||
for sid in "abcd":
|
||||
upsert(store, posting(sid))
|
||||
a, b, c, d = (posting(s).uid for s in "abcd")
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (store.now(), d))
|
||||
one = store.save_agent("One", "python backend",
|
||||
limits=Limits(summarise_top=1, summarise_min=0))
|
||||
two = store.save_agent("Two", "python backend",
|
||||
limits=Limits(summarise_top=1, summarise_min=0))
|
||||
one = store.save_agent("One", "python backend", limits=Limits(summarise_top=1))
|
||||
two = store.save_agent("Two", "python backend", limits=Limits(summarise_top=1))
|
||||
store.replace_scores(one.id, [(d, 0.99, ""), (a, 0.9, ""), (b, 0.5, "")])
|
||||
store.replace_scores(two.id, [(a, 0.95, "remote"), (c, 0.7, ""), (b, 0.8, "")])
|
||||
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b}
|
||||
|
||||
# Two goes deeper: its second, c, joins; One's second, b, was in already.
|
||||
store.save_agent("Two", "python backend", agent_id=two.id,
|
||||
limits=Limits(summarise_top=2, summarise_min=0))
|
||||
store.save_agent("Two", "python backend", agent_id=two.id, limits=Limits(summarise_top=2))
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b, c}
|
||||
assert store.summarise_pending() == 3
|
||||
assert [r.uid for r in store.summarise_waiting(10, uids=[d])] == [d]
|
||||
# The floor is Two's own: c at 0.7 drops, One's b at 0.5 stays.
|
||||
store.save_agent("Two", "python backend", agent_id=two.id,
|
||||
limits=Limits(summarise_top=2, summarise_min=0.75))
|
||||
store.save_agent("One", "python backend", agent_id=one.id,
|
||||
limits=Limits(summarise_top=2, summarise_min=0))
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b}
|
||||
# '{}' is the defaults, 30 at 0.65: One's b at 0.5 drops, but Two's b at
|
||||
# 0.8 and c at 0.7 clear it.
|
||||
store.save_agent("Two", "python backend", agent_id=two.id, limits=Limits(summarise_top=0))
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a}
|
||||
# '{}' is the defaults, 50 deep.
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE agents SET limits = '{}'")
|
||||
assert {r.uid for r in store.summarise_waiting(10)} == {a, b, c}
|
||||
|
||||
+23
-13
@@ -351,6 +351,7 @@ def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store)
|
||||
upsert(store, posting(sid))
|
||||
uid = {r.source_id: r.uid for r in store.roles()}
|
||||
agent = store.save_agent("A")
|
||||
store.replace_scores(agent.id, [(u, 0.5, "") for u in uid.values()])
|
||||
|
||||
# No `agent` in the body: the default agent's vote.
|
||||
res = client.post(f"/api/roles/{uid['liked']}/vote", json={"value": 1})
|
||||
@@ -394,6 +395,8 @@ def test_votes_are_per_agent_over_the_api(client: TestClient, store: Store) -> N
|
||||
uid = posting("1").uid
|
||||
one = store.save_agent("One")
|
||||
two = store.save_agent("Two")
|
||||
for agent in (one, two):
|
||||
store.replace_scores(agent.id, [(uid, 0.5, "")])
|
||||
|
||||
assert client.post(f"/api/roles/{uid}/vote", json={"value": 1, "agent": one.id}).json()["vote"] == 1
|
||||
assert client.post(f"/api/roles/{uid}/vote", json={"value": -1, "agent": two.id}).json()["vote"] == -1
|
||||
@@ -439,11 +442,15 @@ def test_roles_with_an_agent_can_be_ordered_newest_and_keep_the_votes(
|
||||
store.replace_scores(agent.id, [(uid["1"], 0.9, ""), (uid["2"], 0.1, "")]) # 3 unscored
|
||||
store.set_vote(agent.id, uid["1"], 1)
|
||||
|
||||
# Newest keeps to the agent's fetch gate: only what it scored and passed.
|
||||
page = client.get("/api/roles", params={"agent": agent.id, "order": "newest"}).json()
|
||||
assert [r["uid"] for r in page["items"]] == [uid["2"], uid["1"]]
|
||||
assert [r["score"] for r in page["items"]] == [None, None]
|
||||
assert [r["vote"] for r in page["items"]] == [None, 1]
|
||||
assert page["total"] == 2
|
||||
# Without an agent named, newest is every open role.
|
||||
page = client.get("/api/roles", params={"order": "newest"}).json()
|
||||
assert [r["uid"] for r in page["items"]] == [uid["3"], uid["2"], uid["1"]]
|
||||
assert [r["score"] for r in page["items"]] == [None, None, None]
|
||||
assert [r["vote"] for r in page["items"]] == [None, None, 1]
|
||||
assert page["total"] == 3
|
||||
scored = client.get("/api/roles", params={"agent": agent.id, "order": "score"}).json()
|
||||
assert [r["uid"] for r in scored["items"]] == [uid["1"], uid["2"]]
|
||||
assert client.get("/api/roles", params={"agent": agent.id,
|
||||
@@ -1086,7 +1093,9 @@ def test_agents_are_created_updated_listed_and_deleted(client: TestClient) -> No
|
||||
agent = created.json()
|
||||
assert agent["weights"] == {"fit": 0.6, "recency": 0.2, "affinity": 0.2, "taste": 0.2,
|
||||
"cv_skills": 30, "cv_experience": 25, "cv_career": 30}
|
||||
assert agent["gates"][0]["weight"] == 1.0
|
||||
assert agent["gates"] == body["gates"]
|
||||
prefer = {**body, "gates": [{"field": "remote", "values": ["hybrid"], "mode": "prefer"}]}
|
||||
assert client.post("/api/agents", json=prefer).status_code == 422
|
||||
|
||||
updated = client.post("/api/agents", json={**body, "id": agent["id"], "enabled": False,
|
||||
"weights": {"fit": 1, "recency": 0,
|
||||
@@ -1179,19 +1188,20 @@ def test_a_gate_on_an_unknown_field_is_refused(client: TestClient) -> None:
|
||||
|
||||
|
||||
def test_an_agents_limits_are_saved_listed_and_checked(client: TestClient) -> None:
|
||||
"""How deep the model stages read is the agent's own; a score outside
|
||||
0..1 is refused."""
|
||||
"""The cut and the depth of summarise are the agent's own; a minimum
|
||||
outside 0..1 and a key from before the cut are refused."""
|
||||
default = client.post("/api/agents", json={"name": "Wide"}).json()
|
||||
assert default["limits"] == {"extract_top": 500, "extract_min": 0.55,
|
||||
"summarise_top": 30, "summarise_min": 0.65,
|
||||
"judge_top": 150, "judge_min": 0.55, "judge_cv_top": 50}
|
||||
narrow = {"extract_top": 50, "extract_min": 0.6, "summarise_top": 5, "summarise_min": 0.7,
|
||||
"judge_top": 20, "judge_min": 0.6, "judge_cv_top": 10}
|
||||
assert default["limits"] == {"cut_fit_top": 500, "cut_fit_min": 0.6,
|
||||
"cut_cv_top": 150, "cut_cv_min": 0.6, "summarise_top": 50}
|
||||
narrow = {"cut_fit_top": 50, "cut_fit_min": 0.7, "cut_cv_top": 10, "cut_cv_min": 0.75,
|
||||
"summarise_top": 5}
|
||||
saved = client.post("/api/agents", json={"name": "Narrow", "limits": narrow})
|
||||
assert saved.status_code == 200 and saved.json()["limits"] == narrow
|
||||
assert [a["limits"]["extract_top"] for a in client.get("/api/agents").json()] == [50, 500]
|
||||
bad = client.post("/api/agents", json={"name": "Bad", "limits": {"extract_min": 2}})
|
||||
assert [a["limits"]["cut_fit_top"] for a in client.get("/api/agents").json()] == [50, 500]
|
||||
bad = client.post("/api/agents", json={"name": "Bad", "limits": {"cut_fit_min": 2}})
|
||||
assert bad.status_code == 422
|
||||
old = client.post("/api/agents", json={"name": "Old", "limits": {"extract_top": 5}})
|
||||
assert old.status_code == 422
|
||||
|
||||
|
||||
def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated(
|
||||
|
||||
Reference in new issue
Block a user