Read role pages in an enrich stage after scoring, best roles first

The pull reads listings only. `enrich` (app/jobs/enrich.py) takes the open
roles without text that an enabled agent ranks ungated in its best
`enrich.top` at `enrich.min_score` or above, best score first, on boards
whose reader has `detail`, within `enrich.per_board` and `enrich.per_run`,
one batch per board. `plugins.ats.read_board_details` reads them and keeps
the per-board state; a read that gave no text is read again after
`enrich.retry_days` (finding 18). Filled roles are upserted, which drops
their vectors, and embed and score follow; `cascade.step` passes over both
when nothing was filled.

The pull cascade is pull, embed, score, enrich, embed, score, extract, score,
summarise. Saving an agent and changing a company's affinity queue the
`enrich` job (enrich, embed, score) behind their rescore; a vote does not.
The stage has a switch, `stage:enrich`, over `enrich.enabled`.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:53:29 +02:00
1 parent 3adafd8950
commit 77b45a4b0f
10 files changed
+956 -219

No files matched your search

+44 -5
View File
@@ -1,6 +1,6 @@
"""The tasks roledesk offers, and the order a pull runs them in.
pull → embed → score → extract → score → summarise
pull → embed → score → enrich → embed → score → extract → score → summarise
newsdesk's cascade convention: a cascade is a tuple of task names, and the
final job of one task starts the next (`step` is the callback it carries).
@@ -11,7 +11,10 @@ Every stage is always in the chain, and a stage that is switched off finds
nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull does not
satisfy, is an extract stage with no batches whose final job moves straight
on. The free stages go first: embed and score cost no tokens, and the score
they produce is what bounds the two that do. Extract reads only the best
they produce is what bounds the rest. Enrich reads the best roles without
text on their own pages (`app/jobs/enrich.py`), and embed and score follow for
the roles it filled; when it filled none, `step` passes over both. Extract
reads only the best
roles of each enabled agent, as deep and as low as that agent's own `limits`
say; the second score then applies the gates to the columns it filled.
Summarise runs last, on the best roles by the same agent's summarise limits.
@@ -23,6 +26,11 @@ having either way.
`pull` is a job as well as a cascade, because a schedule enqueues a job kind
and a cascade is started by one: the `pull` job's only work is to ask for the
chain. Its default schedule, `every 6h`, is declared here beside it.
`enrich` is a job and a cascade the same way, `enrich → embed → score`, queued
behind the `score` that follows a change to the ranking other than a pull's
(`enrich_after`): an agent saved, a company's affinity changed. A vote moves
one agent's taste a little and queues neither.
"""
from __future__ import annotations
@@ -32,16 +40,21 @@ from dataclasses import replace
from functools import partial
from typing import Any
from app.jobs import embed, extract, pull, score, summarise # noqa: F401 -- registers the jobs
from app.jobs import embed, enrich, extract, pull, score, summarise # noqa: F401 -- registers the jobs
from desk.tools import agent
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
STEP = "app.jobs.cascade:step"
CASCADES: dict[str, tuple[str, ...]] = {
"pull": ("pull", "embed", "score", "extract", "score", "summarise"),
"pull": ("pull", "embed", "score", "enrich", "embed", "score", "extract", "score",
"summarise"),
"enrich": ("enrich", "embed", "score"),
}
#: What follows `enrich` for the roles it filled, passed over when it filled none.
REFILL = ("embed", "score")
def _once(config: Any, store: Any) -> list[dict[str, Any]]:
"""One batch with nothing in its payload: a task that is a single job."""
@@ -91,6 +104,7 @@ STAGES: dict[str, Callable[[str], Task]] = {
"score": lambda cascade: Task(kind="score", finder=_once,
label="Scoring roles for every agent"),
"summarise": _summarise,
"enrich": lambda cascade: enrich.enrich_task(),
}
@@ -115,6 +129,10 @@ def step(config: Any, store: Any, tally: Tally, payload: dict[str, Any], *,
chain goes on, as in newsdesk's cascade."""
name, done = payload.get("cascade", ""), payload.get("task", "")
at = int(payload.get("at", -1))
tasks = CASCADES.get(name, ())
if done == "enrich" and not enrich.filled(tally) \
and tasks[at + 1:at + 1 + len(REFILL)] == REFILL:
at += len(REFILL)
following = follows(name, at)
if following:
context.task(task(following, cascade=name, at=at + 1))
@@ -169,6 +187,26 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
return {"started": " > ".join(CASCADES["pull"]), "run": run}
@job("enrich", "Reading the best roles on their own pages", solo=True, resource="http")
def run_enrich(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
"""Ask for the enrich cascade, `enrich → embed → score`. Its own work is
nil, as the `pull` job's. Under `NULL` there is no queue to take it."""
if not context.task(chain("enrich")):
raise RuntimeError("no queue to start the enrich cascade on")
return {"started": " > ".join(CASCADES["enrich"])}
def enrich_after(queue: Any, score_job: Any) -> Any:
"""Queue the `enrich` job behind `score_job`, the rescore that follows a
change to the ranking, however that score ends. One per score job
(`enrich:after:<id>`), so ten changes answered by one score are one
enrich. None without a score job, or when the queue refuses."""
if score_job is None:
return None
return queue.enqueue("enrich", depends_on=[score_job.id], after="done",
dedup_key=f"enrich:after:{score_job.id}")
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
#: each, on its own. `pull` is not a task here, since it is a job: a name that
#: is both would be listed twice by `/api/jobs/kinds`.
@@ -182,4 +220,5 @@ for _name in ("extract", "embed", "score", "summarise"):
# setting: a cadence edited or switched off under Runs survives
# every restart (see pipeline/schedule.py).
declare("pull", "pull", "every 6h",
label="Pull every board, then embed, score, extract, rescore and summarise")
label="Pull every board, then embed, score, read the best roles' pages, "
"extract, rescore and summarise")
+168
View File
@@ -0,0 +1,168 @@
"""`enrich`: the best roles without text, read on their own pages.
Some hosts list a role without its text and state it on the role's own page
only (`text="detail"`: Workday, SuccessFactors, Oracle, Phenom and the rest
with a `detail`). The pull reads listings only; this stage reads those pages
after scoring, so the calls go where the ranking is:
1. `enrich_waiting` hands over the open roles without text that an enabled
agent with a prompt ranks ungated among its best `enrich.top` and at
`enrich.min_score` or above, best first by the highest such score. A role
below that bar for every agent gets no call.
2. They are taken in that order, those whose read failed last, on boards whose reader has `detail`, at most
`enrich.per_board` from one board and `enrich.per_run` in all, leaving
out the roles already read, unless their read failed or gave no text
`enrich.retry_days` ago (`plugins.ats.detail_due`).
3. One batch per board, on the `http` pool, reads them
(`plugins.ats.read_board_details`, which keeps `ats:detailed:<target>`,
`ats:detailfailed:<target>` and `ats:detailempty:<target>`) and stores
what each page filled. `Store.upsert_role` drops the vector of a role
whose text improved, so the next `embed` encodes those roles only.
In the pull cascade `embed` and `score` follow; `cascade.step` passes over
both when no role was filled. The same stage follows a rescore after an agent
is saved or a company's affinity changes (`cascade.enrich_after`), never after
a vote. Whether it runs is `stages.stage_on(config, store, "enrich")`.
"""
from __future__ import annotations
import json
import logging
from collections import defaultdict
from datetime import datetime
from typing import Any
import httpx
from app.config import Config
from app.jobs import stages
from app.storage import Store
from pipeline import NULL, Batch, RunContext, Tally, Task, job
log = logging.getLogger(__name__)
KIND = "enrich_batch"
SETTLE = "app.jobs.enrich:settle"
def _ats() -> Any:
"""`plugins.ats`, imported when needed, the way `pull.run_batch` does."""
from plugins import ats
return ats
def _row(found: dict[str, Any], target: str) -> dict[str, Any]:
"""A stored role back as the row its reader listed: the stored `raw`
with the host's own id, the title, the link and the date."""
try:
raw = json.loads(found.get("raw") or "{}")
except ValueError:
raw = {}
raw = raw if isinstance(raw, dict) else {}
try:
posted: datetime | None = datetime.fromisoformat(str(found.get("published_at") or ""))
except ValueError:
posted = None
return raw | {"source_id": str(found["source_id"]).removeprefix(f"{target}:"),
"title": found["title"], "url": found["url"], "description": "",
"posted": posted}
def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
"""`(target, uids)` per board, best board first, within the budgets: the
roles `enrich_waiting` ranks, on fetchable boards whose reader has
`detail` and that are not failing, and due for a read."""
settings = config.enrich
ats = _ats()
boards = {sub.id: sub.target for sub in store.subscriptions(source="ats", fetchable=True)}
by_board: dict[str, list[dict[str, Any]]] = defaultdict(list)
for found in store.enrich_waiting(settings.top, settings.min_score):
target = boards.get(found["subscription_id"])
if target is None:
continue
module = ats.readers.HOSTS.get(target.partition(":")[0])
if module is None or not hasattr(module, "detail") \
or f"ats:failing:{target}" in store.state:
continue
by_board[target].append(found)
due = {target: set(ats.detail_due(store.state, target, [f["source_id"] for f in found],
retry_days=settings.retry_days))
for target, found in by_board.items()}
# Best first; roles whose read failed after every other, so a page that
# keeps failing cannot take the budget of the roles never tried.
failed = {target: ats._runner.id_set(store.state, ats.detail_keys(target)[1])
for target in by_board}
ranked = sorted((f for found in by_board.values() for f in found),
key=lambda f: (f["source_id"] in failed[boards[f["subscription_id"]]],
-float(f["score"] or 0), f["uid"]))
picked: dict[str, list[str]] = {}
total = 0
for found in ranked:
target = boards[found["subscription_id"]]
if found["source_id"] not in due[target]:
continue
if total >= settings.per_run:
break
uids = picked.setdefault(target, [])
if len(uids) >= settings.per_board:
continue
uids.append(found["uid"])
total += 1
return list(picked.items())
def batches(config: Config, store: Store) -> list[Batch]:
"""The finder: one batch per board, nothing while the stage is off."""
if not stages.stage_on(config, store, "enrich"):
return []
return [Batch({"target": target, "uids": uids},
label=f"Reading {len(uids)} roles on {target}")
for target, uids in plan(config, store)]
@job(KIND, "Reading roles on their own pages", resource="http")
def run_batch(config: Config, store: Store, *, target: str, uids: list[str],
context: RunContext = NULL) -> dict[str, Any]:
"""Read one board's roles in `uids` on their own pages, in that order,
and store what each page filled. `filled` is how many roles changed,
`texts` how many of them now have text."""
ats = _ats()
subs = [s for s in store.subscriptions(source="ats", fetchable=True) if s.target == target]
if not subs or not uids:
return {"target": target, "filled": 0, "texts": 0}
sub = subs[0]
found = {f["uid"]: f for f in store.enrich_waiting(0, 0.0, uids)}
rows = [_row(found[uid], target) for uid in uids if uid in found]
context.check()
with httpx.Client() as client:
filled = ats.read_board_details(store.state, target, rows, client=client,
cap=len(rows), retry_days=config.enrich.retry_days)
label = sub.company or sub.label
texts = 0
for row in filled:
item = ats.item_of(row, target, label)
item.source = "ats"
store.upsert_role(item, sub)
texts += bool(row.get("description"))
log.info("enrich: %s: %d read, %d filled, %d with text", target, len(rows), len(filled),
texts)
return {"target": target, "read": len(rows), "filled": len(filled), "texts": texts}
def filled(tally: Tally) -> int:
"""How many roles the batches behind `tally` filled."""
return sum(int(result.get("filled") or 0) for result in tally.results)
def settle(config: Config, store: Store, tally: Tally, *,
context: RunContext = NULL) -> dict[str, Any]:
"""The run's sum: roles filled, and how many of them now have text."""
return {"filled": filled(tally),
"texts": sum(int(result.get("texts") or 0) for result in tally.results)}
def enrich_task() -> Task:
return Task(kind=KIND, finder=batches, name="enrich", settle=SETTLE,
label="Reading the best roles on their own pages")
+16 -7
View File
@@ -1,4 +1,5 @@
"""Whether the two model stages of a pull, `extract` and `summarise`, run.
"""Whether the two model stages of a pull, `extract` and `summarise`, and
the `enrich` stage run.
`extract.enabled` and `summarise.enabled` in config.yaml are read at startup
and say nothing on the page, so the switch lives in the database as well: the
@@ -8,9 +9,13 @@ default for a fresh database and a deployment that never uses the switch
behaves as it did. `stage_on` is the one place that decides, and both tasks'
finders ask it (`extract.extract_task`, `summarise.summarise_task`).
`describe` is what `GET /api/stages` and `app stages` show: for each stage
whether it is on, where that comes from, the model it uses, and who would
answer.
`enrich` (reading the best roles on their own pages, `app/jobs/enrich.py`)
asks no model and is on by default (`enrich.enabled`); its switch is
`stage:enrich`, set the same way through `PUT /api/stages/enrich`.
`describe` is what `GET /api/stages` and `app stages` show: for each model
stage whether it is on, where that comes from, the model it uses, and who
would answer.
"""
from __future__ import annotations
@@ -25,6 +30,9 @@ from desk.tools import agent
NAMES = ("extract", "summarise")
#: Every stage with a switch: the model stages and `enrich`.
SWITCHES = (*NAMES, "enrich")
NO_MODEL = ("no model is reachable: set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN, "
"see docs/llm-backends.md")
ANSWERS = {
@@ -39,7 +47,7 @@ class Stage(BaseModel):
`backend` is `agent.backend`'s answer, "" for nobody, and `answer` says
it in words."""
name: Literal["extract", "summarise"]
name: Literal["extract", "summarise", "enrich"]
on: bool
source: Literal["stored", "config"]
model: str
@@ -65,7 +73,7 @@ def stage_on(config: Config, store: Store, name: str) -> bool:
def set_stage(store: Store, name: str, on: bool) -> None:
"""Store the switch. A `KeyError` for a name that is not a stage."""
if name not in NAMES:
if name not in SWITCHES:
raise KeyError(name)
store.state[_key(name)] = "on" if on else "off"
@@ -84,7 +92,8 @@ def describe_one(config: Config, store: Store, name: str) -> Stage:
backend, said = answer(config)
return Stage(name=name, on=stage_on(config, store, name),
source="config" if stored(store, name) is None else "stored",
model=getattr(config, name).model, backend=backend, answer=said)
model=getattr(getattr(config, name), "model", ""), backend=backend,
answer=said)
def describe(config: Config, store: Store) -> list[Stage]: