Extract is bounded to the ranking: pull → embed → score → extract → score → summarise

The free stages run first and their score bounds the two model stages.
Extract reads only the best extract.per_agent roles of each agent at
extract.min_score or more, then score runs again so the gates read what it
filled; summarise gets a min_score of its own. The cascade steps by
position, since score is in it twice.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-28 15:26:58 +02:00
1 parent c4f2bcc0c9
commit 7db77e83a3
15 files changed
+215 -89

No files matched your search

+1 -1
View File
@@ -9,7 +9,7 @@ Built on the [boilerplate](../boilerplate): its job queue and scheduler (`pipeli
```bash
uv sync --all-extras
uv run app companies seed # 186 companies from seed/companies.json, one board each
uv run app job pull # read every board and search, then extract, embed and score
uv run app job pull # read every board and search, embed and score them, extract the best, rescore, summarise
uv run app serve # http://127.0.0.1:8790
```
+12 -1
View File
@@ -42,11 +42,19 @@ class ServerConfig(Section):
class ExtractConfig(ModelTaskConfig):
"""Turning a posting's title and text into the gate columns. See
app/jobs/extract.py. Off by default, so nobody is billed by surprise:
on the API backend every batch is a paid call."""
on the API backend every batch is a paid call. On, it reads only the
best `per_agent` roles of each enabled agent that score at least
`min_score`, so the cost follows the ranking rather than the boards."""
enabled: bool = False
model: str = "claude-haiku-4-5-20251001"
batch_size: int = Field(20, ge=1)
#: How deep into each enabled agent's ranking extract reads: the backstop.
per_agent: int = Field(500, ge=0)
#: Below this score a role is not read, however few are above it. Loose,
#: since a fresh role with the worst fit scores about 0.27 on the default
#: weights and a good one 0.6 or more.
min_score: float = Field(0.3, ge=0, le=1)
class SummariseConfig(ModelTaskConfig):
@@ -63,6 +71,9 @@ class SummariseConfig(ModelTaskConfig):
batch_size: int = Field(10, ge=1)
#: How deep into each enabled agent's ranking a pull summarises.
per_agent: int = Field(30, ge=0)
#: Below this score a role is not summarised. Stricter than extract's,
#: since a summary is the expensive answer.
min_score: float = Field(0.5, ge=0, le=1)
class EmbedConfig(Section):
+1 -1
View File
@@ -6,7 +6,7 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
embed.py one vector per open role, for fit
score.py every open role scored for every enabled search agent
summarise.py about 60 words on each of the best roles, on Haiku
cascade.py pull → extract → embed → score → summarise, what `TASKS`
cascade.py pull → embed → score → extract → score → summarise, what `TASKS`
offers, the `pull` job a schedule fires, and that schedule
sources.py what a pull reads, changed by hand: boards and searches
added, switched on or off, pruned by region
+22 -20
View File
@@ -1,18 +1,21 @@
"""The tasks roledesk offers, and the order a pull runs them in.
pull → extract → embed → score → summarise
pull → embed → score → extract → score → summarise
newsdesk's cascade convention: a cascade is a tuple of task names, and the
final job of one task starts the next (`step` is the callback it carries).
Nothing is alive between two tasks.
Nothing is alive between two tasks. A stage is addressed by its position in
the tuple, not its name, since `score` runs twice.
Every stage is always in the chain, and a stage that is switched off finds
nothing: `extract.enabled: false`, or an `on_ingest` this pull does not
satisfy, is an extract stage with no batches whose final job moves straight
on. Extract runs before score so the gates read what it filled; before embed
too, since embedding does not read its columns and the order costs nothing.
Summarise runs last because it reads the scores: it takes only the best
`summarise.per_agent` roles of each enabled agent.
on. The free stages go first: embed and score cost no tokens, and the score
they produce is what bounds the two that do. Extract reads only the best
`extract.per_agent` roles of each enabled agent, not below
`extract.min_score`; the second score then applies the gates to the columns
it filled. Summarise runs last, on the best `summarise.per_agent` roles not
below `summarise.min_score`.
A batch that failed, or a stage somebody killed, does not stop the chain: the
work comes round on the next pull, and a rescore of what did arrive is worth
@@ -37,7 +40,7 @@ from pipeline import NULL, TASKS, RunContext, Tally, Task, declare, job
STEP = "app.jobs.cascade:step"
CASCADES: dict[str, tuple[str, ...]] = {
"pull": ("pull", "extract", "embed", "score", "summarise"),
"pull": ("pull", "embed", "score", "extract", "score", "summarise"),
}
@@ -92,21 +95,19 @@ STAGES: dict[str, Callable[[str], Task]] = {
}
def task(name: str, *, cascade: str = "") -> Task:
def task(name: str, *, cascade: str = "", at: int = 0) -> Task:
"""One stage as a `Task`. With `cascade` its final job carries `step`,
and the next stage follows; without, it stops when its batches do."""
and the stage after position `at` follows; without, it stops when its
batches do."""
return replace(STAGES[name](cascade), name=name,
callback=STEP if cascade else "",
carries={"cascade": cascade, "task": name})
carries={"cascade": cascade, "task": name, "at": at})
def follows(cascade: str, name: str) -> str:
"""What runs after `name`, or "" at the end or for an unknown name."""
def follows(cascade: str, at: int) -> str:
"""The stage after position `at`, or "" at the end or off the cascade."""
tasks = CASCADES.get(cascade, ())
if name not in tasks:
return ""
at = tasks.index(name)
return tasks[at + 1] if at + 1 < len(tasks) else ""
return tasks[at + 1] if 0 <= at < len(tasks) - 1 else ""
def step(config: Any, store: Any, tally: Tally, payload: dict[str, Any], *,
@@ -114,9 +115,10 @@ def step(config: Any, store: Any, tally: Tally, payload: dict[str, Any], *,
"""The callback: make the next task. A killed stage is skipped and the
chain goes on, as in newsdesk's cascade."""
name, done = payload.get("cascade", ""), payload.get("task", "")
following = follows(name, done)
at = int(payload.get("at", -1))
following = follows(name, at)
if following:
context.task(task(following, cascade=name))
context.task(task(following, cascade=name, at=at + 1))
out: dict[str, Any] = {"task": done, "next": following}
if tally.killed:
out["skipped"] = "cancelled"
@@ -125,7 +127,7 @@ def step(config: Any, store: Any, tally: Tally, payload: dict[str, Any], *,
def chain(name: str) -> Task:
"""The first stage of cascade `name`, carrying `step`, so the rest follow."""
return task(CASCADES[name][0], cascade=name)
return task(CASCADES[name][0], cascade=name, at=0)
@job("pull", "Pulling every board", resource="http")
@@ -154,4 +156,4 @@ for _name in ("extract", "embed", "score", "summarise"):
# setting: a cadence edited or switched off on the Pulls page survives
# every restart (see pipeline/schedule.py).
declare("pull", "pull", "every 6h",
label="Pull every board, then extract, embed, score and summarise")
label="Pull every board, then embed, score, extract, rescore and summarise")
+15 -7
View File
@@ -8,12 +8,17 @@ refused batch, and failing with a notice when no model is reachable are
`desk.modeltask`'s.
A role is waiting while it is open, has never been answered (`extracted_at IS
NULL`) and has not been given up on. The answer only fills columns that are
still empty: a board's structured field wins over what a model read out of the
prose. See `Store.set_extracted`.
NULL`), has not been given up on, and is in the best `extract.per_agent` of an
enabled agent's ungated scores at `extract.min_score` or more. That bound is
what keeps the cost proportional to the rankings rather than to the boards:
the pull scores every role for free first, then reads the few hundred that
rank, and scores again so the gates see what was filled. A batch built for
named uids finishes what it was dealt whatever their rank. The answer only
fills columns that are still empty: a board's structured field wins over what
a model read out of the prose. See `Store.set_extracted`.
Off by default (`extract.enabled: false`), because on the API backend every
batch is a paid call and ~30k open roles is ~1,500 of them.
batch is a paid call.
"""
from __future__ import annotations
@@ -128,7 +133,8 @@ def _item(role: Role) -> dict[str, Any]:
def build_batch(config: Config, store: Store, limit: int = 20,
uids: Sequence[str] | None = None) -> dict[str, Any]:
return {"instructions": SYSTEM,
"roles": [_item(r) for r in store.extract_waiting(limit, uids)]}
"roles": [_item(r) for r in store.extract_waiting(
limit, config.extract.per_agent, config.extract.min_score, uids)]}
# -------------------------------------------------------------------- answer
@@ -199,9 +205,11 @@ TASK: ModelTask[ExtractResult] = ModelTask(
kind="extract", label="Reading postings for gate fields",
verb="extracting", noun="fields", idle="nothing to extract", items="roles",
settings=lambda config: config.extract,
pending=lambda config, store: store.extract_pending(),
pending=lambda config, store: store.extract_pending(
config.extract.per_agent, config.extract.min_score),
waiting=lambda config, store, limit: [
r.uid for r in store.extract_waiting(limit)],
r.uid for r in store.extract_waiting(
limit, config.extract.per_agent, config.extract.min_score)],
build_batch=build_batch,
system=lambda store: SYSTEM,
schema=SCHEMA,
+7 -5
View File
@@ -6,9 +6,9 @@ what the model is told, the schema it answers in, what to do with the
answer), and running it is `desk.modeltask`'s.
A role is waiting while it has no summary, has some text, and is in the top
`summarise.per_agent` of an enabled agent's ungated scores. That bound is what
keeps the cost flat: a pull summarises the roles somebody will look at, not the
30k that are open. A batch built for named uids (the read page's Summarise
`summarise.per_agent` of an enabled agent's ungated scores at
`summarise.min_score` or more. That bound is what keeps the cost flat: a pull
summarises the roles somebody will look at, not the 30k that are open. A batch built for named uids (the read page's Summarise
button) takes those roles whatever their rank. See `Store.summarise_waiting`.
The text is `article_text` when the posting page was fetched, else
@@ -121,7 +121,8 @@ def _item(role: Role) -> dict[str, Any]:
def _waiting(config: Config, store: Store, limit: int,
uids: Sequence[str] | None = None) -> list[Role]:
return store.summarise_waiting(limit, config.summarise.per_agent, uids)
return store.summarise_waiting(limit, config.summarise.per_agent, uids,
min_score=config.summarise.min_score)
def build_batch(config: Config, store: Store, limit: int = 10,
@@ -165,7 +166,8 @@ TASK: ModelTask[SummariseResult] = ModelTask(
verb="summarising", noun="summaries", idle="nothing to summarise",
items="summaries",
settings=lambda config: config.summarise,
pending=lambda config, store: store.summarise_pending(config.summarise.per_agent),
pending=lambda config, store: store.summarise_pending(
config.summarise.per_agent, config.summarise.min_score),
waiting=lambda config, store, limit: [r.uid for r in _waiting(config, store, limit)],
build_batch=build_batch,
system=lambda store: SYSTEM,
+56 -33
View File
@@ -619,21 +619,51 @@ class Store:
# -------------------------------------------------------------- extract
def extract_pending(self) -> int:
row = self._row(f"SELECT COUNT(*) AS n FROM roles WHERE {_EXTRACT_WAITING}")
def _top_scope(self, per_agent: int, min_score: float) -> tuple[str, list[Any]]:
"""The `WHERE` fragment for a role among the `per_agent` best ungated
scores of an enabled agent, not below `min_score`: what the two model
stages, extract and summarise, are bounded to. The role is `r`. A
company an agent hid needs no clause: `score` writes no rows for it
under that agent."""
return ("""r.uid IN (SELECT uid FROM (
SELECT s.uid, s.score, ROW_NUMBER() OVER (
PARTITION BY s.agent_id
ORDER BY s.score DESC, top.first_seen DESC, s.uid) AS n
FROM scores s
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
WHERE s.gated_reason = '')
WHERE n <= ? AND score >= ?)""", [per_agent, min_score])
def _extract_scope(self, per_agent: int, min_score: float,
uids: Sequence[str] | None) -> tuple[str, list[Any]]:
"""The `WHERE` for a role `extract` may read: open, never answered,
not given up. Given `uids`, those of them and nothing else, since a
batch asks again when it runs and must finish what it was dealt;
otherwise only the roles in `_top_scope`."""
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")]
if uids is not None:
where.append("r.uid IN (SELECT value FROM json_each(?))")
return " AND ".join(where), [json.dumps(list(uids))]
top, params = self._top_scope(per_agent, min_score)
return " AND ".join([*where, top]), params
def extract_pending(self, per_agent: int, min_score: float = 0.0) -> int:
where, params = self._extract_scope(per_agent, min_score, None)
row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params)
return int(row["n"]) if row else 0
def extract_waiting(self, limit: int,
def extract_waiting(self, limit: int, per_agent: int, min_score: float = 0.0,
uids: Sequence[str] | None = None) -> list[Role]:
"""Roles `extract` has yet to read, newest first. Given `uids`, only
those of them still waiting: a batch asks again when it runs, since a
role may have been answered since the task shared the work out."""
sql, params = f"SELECT * FROM roles WHERE {_EXTRACT_WAITING}", []
if uids is not None:
sql += " AND uid IN (SELECT value FROM json_each(?))"
params.append(json.dumps(list(uids)))
sql += " ORDER BY first_seen DESC, uid LIMIT ?"
return [Role.from_row(r) for r in self._rows(sql, [*params, limit])]
"""Roles `extract` has yet to read, newest first: the best
`per_agent` of every enabled agent's scores not below `min_score`, so
the backlog follows the rankings rather than the boards. Given
`uids`, only those of them still waiting, whatever their rank."""
where, params = self._extract_scope(per_agent, min_score, uids)
return [Role.from_row(r) for r in self._rows(
f"SELECT r.* FROM roles r WHERE {where} "
"ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
def set_extracted(self, uid: str, fields: dict[str, Any]) -> int | None:
"""Write what `extract` read for one role and stamp `extracted_at`.
@@ -692,48 +722,41 @@ class Store:
#: roles in some agent's top list.
_SUMMARY_MISSED = "summarise.missed:"
def _summary_scope(self, per_agent: int,
def _summary_scope(self, per_agent: int, min_score: float,
uids: Sequence[str] | None) -> tuple[str, list[Any]]:
"""The `WHERE` for a role `summarise` may be sent: no summary yet and
some text to summarise. Given `uids`, those of them and nothing else,
open or not and given up or not, because somebody asked for them.
Otherwise open, not given up, and among the `per_agent` best ungated
scores of an enabled agent. A company an agent hid needs no clause:
`score` writes no rows for it under that agent."""
Otherwise open, not given up, and in `_top_scope`."""
where = ["r.summary = ''",
"(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"]
if uids is not None:
where.append("r.uid IN (SELECT value FROM json_each(?))")
return " AND ".join(where), [json.dumps(list(uids))]
top, params = self._top_scope(per_agent, min_score)
where += [
"r.closed_at IS NULL",
f"""NOT EXISTS (SELECT 1 FROM state
WHERE key = '{self._SUMMARY_MISSED}' || r.uid
AND CAST(value AS INTEGER) >= {GIVE_UP_AFTER})""",
"""r.uid IN (SELECT uid FROM (
SELECT s.uid, ROW_NUMBER() OVER (
PARTITION BY s.agent_id
ORDER BY s.score DESC, top.first_seen DESC, s.uid) AS n
FROM scores s
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
WHERE s.gated_reason = '')
WHERE n <= ?)""",
top,
]
return " AND ".join(where), [per_agent]
return " AND ".join(where), params
def summarise_pending(self, per_agent: int) -> int:
where, params = self._summary_scope(per_agent, None)
def summarise_pending(self, per_agent: int, min_score: float = 0.0) -> int:
where, params = self._summary_scope(per_agent, min_score, None)
row = self._row(f"SELECT COUNT(*) AS n FROM roles r WHERE {where}", params)
return int(row["n"]) if row else 0
def summarise_waiting(self, limit: int, per_agent: int,
uids: Sequence[str] | None = None) -> list[Role]:
uids: Sequence[str] | None = None,
min_score: float = 0.0) -> list[Role]:
"""Roles `summarise` has yet to write, newest first. The top
`per_agent` of every enabled agent's scores, so the backlog is bounded
by the number of agents rather than by the number of open roles; given
`uids`, those still without a summary. A batch asks again when it runs."""
where, params = self._summary_scope(per_agent, uids)
`per_agent` of every enabled agent's scores not below `min_score`, so
the backlog is bounded by the number of agents rather than by the
number of open roles; given `uids`, those still without a summary,
whatever their rank. A batch asks again when it runs."""
where, params = self._summary_scope(per_agent, min_score, uids)
return [Role.from_row(r) for r in self._rows(
f"SELECT r.* FROM roles r WHERE {where} "
"ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
+1 -1
View File
@@ -126,7 +126,7 @@
<h2>Runs</h2>
<button type="button" id="pull-now" class="secondary">Pull now</button>
</div>
<p class="lede">Every 6 hours the pull reads every board, then extract fills in what each posting says, embed and score rank them, and summarise writes the summaries. Change the cadence below.</p>
<p class="lede">Every 6 hours the pull reads every board, embed and score rank every role, extract fills in what the best postings say, score runs again with what it filled, and summarise writes the summaries. Change the cadence below.</p>
<h3>What is running</h3>
{# The heading and the idle line are the app's: the element takes up no
+13 -4
View File
@@ -48,20 +48,29 @@ agent:
backend: auto # auto | api | claude_code | off
# Title and description in, gate columns out (app/jobs/extract.py). Off by
# default: on the API backend every batch of 20 roles is a paid call.
# default: on the API backend every batch of 20 roles is a paid call. On, a
# pull reads only the best per_agent roles of every enabled agent (the
# backstop) that score at least min_score (the real stop), after a first
# score; a second score then applies the gates to what was filled. A fresh
# role with the worst fit scores about 0.27 on the default weights, a good
# one 0.6 or more, so 0.3 is loose.
extract:
enabled: false
model: claude-haiku-4-5-20251001
per_agent: 500
min_score: 0.3
# About 60 words on a role: what it is, the stack, where, the pay if stated
# (app/jobs/summarise.py). Off by default. On, each pull summarises the best
# per_agent roles of every enabled agent that have none yet, so the cost stays
# bounded however many roles are open. The Summarise button on a role's page
# asks for that one role either way.
# per_agent roles of every enabled agent that have none yet and score at least
# min_score, so the cost stays bounded however many roles are open; stricter
# than extract's floor, since a summary is the long answer. The Summarise
# button on a role's page asks for that one role either way.
summarise:
enabled: false
model: claude-haiku-4-5-20251001
per_agent: 30
min_score: 0.5
# A role and each agent's intent as vectors, for the fit term of a score
# (app/jobs/embed.py). Needs the `embeddings` extra; without it `embed` posts
+3
View File
@@ -17,10 +17,13 @@ workers:
extract:
enabled: false
model: claude-haiku-4-5-20251001
per_agent: 500
min_score: 0.3
summarise:
enabled: false
model: claude-haiku-4-5-20251001
per_agent: 30
min_score: 0.5
embed:
model: sentence-transformers/all-MiniLM-L6-v2
sources: {}
+4 -2
View File
@@ -84,7 +84,9 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**`embed` and `score` run in the server process.** newsdesk isolates `embed` in a child process with a timeout. roledesk's run is one bounded pass over open roles, and in-process keeps the embedder replaceable in tests. The cost is torch's memory staying in the server after the first run; isolate it if that becomes the problem.
**The pull cascade is `pull → extract → embed → score`.** Every stage is always in the chain and a stage that is off finds nothing, as in newsdesk. Extract precedes score so the gates read what it filled, and reads `on_ingest` only as a stage of a pull, so with `auto` and no model reachable a pull passes it quietly.
**The pull cascade is `pull → embed → score → extract → score → summarise`.** Every stage is always in the chain and a stage that is off finds nothing, as in newsdesk. The free stages run first: embed and score cost no tokens, and their score is what bounds the two model stages. Extract reads only what ranks, then score runs again so the gates read what it filled. A stage is addressed by its position in the tuple, since `score` is in it twice. Extract reads `on_ingest` only as a stage of a pull, so with `auto` and no model reachable a pull passes it quietly.
**Extract is bounded like summarise: the top `per_agent` roles of each agent at `min_score` or more.** Extract has short answers but would otherwise touch every open role, and the seed boards alone produced 29,000; at Haiku's rates that is tens of dollars for a first pass over roles nobody will look at. Bounded to the ranking, a pull reads a few hundred. `per_agent` is the backstop (default 500) and `min_score` the real stop (default 0.3, loose): a fixed count stays flat whatever the boards produce, where a percentage would not. A batch dealt named uids finishes them whatever their rank, so a running batch is not cut short by a rescore.
## The pages
@@ -128,7 +130,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**A search role's posting text is fetched on demand, not in the pull.** `POST /api/roles/{uid}/text` scrapes `original_url` only when the read page asks for it, because following Jobindex's redirect for every role during a pull would count as a click in Jobindex's own statistics; one click for a posting somebody actually opened is fair.
**Summarising is bounded to the top `per_agent` roles of each agent, and off by default.** The pull's `summarise` stage only considers roles with no summary yet that are in the top `summarise.per_agent` (default 30) of an enabled agent's scores, so the cost stays flat whatever the backlog of open roles is. `summarise.enabled: false` matches `extract`'s own default; the read page's Summarise button asks for one role's summary either way.
**Summarising is bounded to the top `per_agent` roles of each agent at `min_score` or more, and off by default.** The pull's `summarise` stage only considers roles with no summary yet that are in the top `summarise.per_agent` (default 30) of an enabled agent's scores and at `summarise.min_score` (default 0.5, stricter than extract's since a summary is the long answer), so the cost stays flat whatever the backlog of open roles is. `summarise.enabled: false` matches `extract`'s own default; the read page's Summarise button asks for one role's summary either way.
**The schedules element's narrow layout is fixed upstream, in `../boilerplate`.** `<desk-schedules>` is a shadow root the app cannot lay out from outside, and `pipeline/` is a downstream copy refreshed with `git archive`, so a fix landing only in roledesk would be reverted by the next refresh.
+44 -2
View File
@@ -57,6 +57,15 @@ def config(tmp_path: Path) -> Config:
return Config(database=str(tmp_path / "roledesk.db"), extract={"enabled": True})
def ranked(store: Store, *items: Item) -> None:
"""Score the roles for one enabled agent, best first, so they are in the
top extract reads. Every role given must already be stored."""
agents = store.agents()
agent_id = agents[0].id if agents else store.save_agent("Backend").id
store.replace_scores(agent_id, [(item.uid, 1.0 - n / 10, "")
for n, item in enumerate(items)])
def stub(monkeypatch: pytest.MonkeyPatch, reply: Any) -> Model:
model = Model(reply)
monkeypatch.setattr(agent, "backend", lambda settings: "api")
@@ -68,6 +77,7 @@ def test_a_batch_writes_the_columns_and_stamps_extracted_at(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
uid = posting("1").uid
store.upsert_role(posting("1"))
ranked(store, posting("1"))
stub(monkeypatch, lambda roles: [answer(
r["uid"], place="Berlin", country="de", remote="hybrid", seniority="senior",
salary_min=80000, salary_max=95000, currency="eur", function="engineering",
@@ -90,6 +100,7 @@ def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
item = posting("1", place="Berlin", remote="hybrid", salary_min=70000)
store.upsert_role(item)
ranked(store, item)
stub(monkeypatch, lambda roles: [answer(
r["uid"], place="Munich", remote="", salary_min=None, salary_max=90000,
country="DE") for r in roles])
@@ -105,6 +116,7 @@ def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
def test_the_next_upsert_keeps_what_extract_filled(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
store.upsert_role(posting("1"))
ranked(store, posting("1"))
stub(monkeypatch, lambda roles: [answer(r["uid"], country="DE", seniority="mid")
for r in roles])
modeltask.run(extract.TASK, config, store)
@@ -119,6 +131,7 @@ def test_closed_and_answered_roles_are_not_offered(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
for sid in ("open", "closed", "done"):
store.upsert_role(posting(sid))
ranked(store, posting("open"), posting("closed"), posting("done"))
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' "
"WHERE source_id = 'closed'")
@@ -134,6 +147,7 @@ def test_closed_and_answered_roles_are_not_offered(
def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
store.upsert_role(posting("1"))
ranked(store, posting("1"))
stub(monkeypatch, lambda roles: [answer("not-in-the-batch", country="DE")])
result = modeltask.run(extract.TASK, config, store)
@@ -141,11 +155,39 @@ def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged(
role = store.role(posting("1").uid)
assert role is not None and role.country == "" and role.extracted_at is None
assert result["rejected"] == ["not-in-the-batch"]
assert store.extract_pending() == 1
assert store.extract_pending(500) == 1
modeltask.run(extract.TASK, config, store)
modeltask.run(extract.TASK, config, store)
assert store.extract_pending() == 0, "given up after three unanswered replies"
assert store.extract_pending(500) == 0, "given up after three unanswered replies"
def test_only_the_best_of_each_agent_above_the_floor_is_read(
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""The bound that keeps extract cheap: the top `per_agent` of an enabled
agent's scores at `min_score` or more, whatever the boards produced. A
role nobody ranks is never sent; a batch dealt a uid finishes it."""
for sid in ("best", "second", "low", "unscored", "gated"):
store.upsert_role(posting(sid))
one = store.save_agent("One")
two = store.save_agent("Two", enabled=False)
store.replace_scores(one.id, [(posting("best").uid, 0.9, ""),
(posting("second").uid, 0.8, ""),
(posting("low").uid, 0.2, ""),
(posting("gated").uid, 0.95, "country unknown")])
store.replace_scores(two.id, [(posting("unscored").uid, 0.99, "")])
config = Config(database=str(tmp_path / "x.db"),
extract={"enabled": True, "per_agent": 1, "min_score": 0.5})
model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles])
assert store.extract_pending(1, 0.5) == 1
assert store.extract_pending(5, 0.5) == 2, "the floor holds the low one back"
assert store.extract_pending(5, 0.0) == 3, "the disabled agent's pick never counts"
modeltask.run(extract.TASK, config, store)
assert [[r["uid"] for r in batch] for batch in model.asked] == [[posting("best").uid]]
assert [r.uid for r in store.extract_waiting(10, 1, 0.5, uids=[posting("low").uid])] == \
[posting("low").uid], "a dealt uid is read whatever its rank"
def test_a_value_outside_the_vocabulary_is_dropped() -> None:
+32 -11
View File
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from desk.models import Item # noqa: E402
from pipeline import FINAL, SCHEDULES, TASKS, JobQueue, Jobs, RunContext # noqa: E402
from pipeline import FINAL, SCHEDULES, TASKS, JobQueue, Jobs, RunContext, Tally # noqa: E402
from plugins.ats import ATSSource # noqa: E402
from app.config import AgentSeed, Config # noqa: E402
from app.jobs import cascade, embed, score # noqa: E402
@@ -438,7 +438,7 @@ def test_the_shipped_starter_searches_validate() -> None:
# ---------------------------------------------------------------- cascade
def test_the_pull_job_runs_pull_extract_embed_then_score(
def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
@@ -471,8 +471,9 @@ def test_the_pull_job_runs_pull_extract_embed_then_score(
assert stages == list(cascade.CASCADES["pull"])
work = [j.kind for j in jobs if j.kind != FINAL]
# The `pull` job and the chain it asked for are one cascade, so the Jobs
# tab shows a scheduled pull as one run. Extract is off and finds nothing.
assert work == ["pull", "pull_batch", "embed", "score"]
# tab shows a scheduled pull as one run. Extract and summarise are off and
# find nothing; score runs twice, before and after extract.
assert work == ["pull", "pull_batch", "embed", "score", "score"]
assert all(j.state == "done" for j in jobs)
assert len(store.scores(agent.id)) == 1
@@ -486,10 +487,30 @@ def test_pull_is_a_job_on_a_six_hour_schedule_and_the_stages_are_tasks() -> None
assert SCHEDULES["pull"].cadence == "every 6h"
def test_follows_walks_the_cascade_and_stops_at_its_end() -> None:
assert cascade.follows("pull", "pull") == "extract"
assert cascade.follows("pull", "extract") == "embed"
assert cascade.follows("pull", "embed") == "score"
assert cascade.follows("pull", "score") == "summarise"
assert cascade.follows("pull", "summarise") == ""
assert cascade.follows("pull", "nope") == ""
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
"""By position, not by name: `score` is in the chain twice, once before
extract and once after, and the second must lead on to summarise."""
assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "extract", "score",
"summarise")
assert [cascade.follows("pull", at) for at in range(6)] == \
["embed", "score", "extract", "score", "summarise", ""]
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 9) == ""
assert cascade.follows("nope", 0) == ""
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None:
made: list[Any] = []
class Context:
def task(self, t: Any) -> Any:
made.append(t)
return t
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 2},
context=Context())
assert out == {"task": "score", "next": "extract"}
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 3}
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 4},
context=Context())
assert out["next"] == "summarise" and made[1].carries["at"] == 5
+3
View File
@@ -750,3 +750,6 @@ def test_summarise_waits_on_the_top_of_each_enabled_agent(store: Store) -> None:
assert {r.uid for r in store.summarise_waiting(10, per_agent=2)} == {a, b, c}
assert store.summarise_pending(per_agent=2) == 3
assert [r.uid for r in store.summarise_waiting(10, 1, uids=[d])] == [d]
# The floor: b's best score is 0.8 under Two, c's 0.7, a's 0.9.
assert {r.uid for r in store.summarise_waiting(10, 2, min_score=0.75)} == {a, b}
assert store.summarise_pending(2, min_score=0.85) == 1
+1 -1
View File
@@ -201,7 +201,7 @@ def test_disabled_finds_nothing_and_asks_nobody(
def test_summarise_follows_score_and_finds_nothing_without_a_model(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
ranked(store, "1", "2")
assert cascade.follows("pull", "score") == "summarise"
assert cascade.follows("pull", cascade.CASCADES["pull"].index("summarise") - 1) == "summarise"
assert "summarise" in TASKS
stage = cascade.task("summarise", cascade="pull")