Read role pages in an enrich stage after scoring, best roles first
The pull reads listings only. `enrich` (app/jobs/enrich.py) takes the open roles without text that an enabled agent ranks ungated in its best `enrich.top` at `enrich.min_score` or above, best score first, on boards whose reader has `detail`, within `enrich.per_board` and `enrich.per_run`, one batch per board. `plugins.ats.read_board_details` reads them and keeps the per-board state; a read that gave no text is read again after `enrich.retry_days` (finding 18). Filled roles are upserted, which drops their vectors, and embed and score follow; `cascade.step` passes over both when nothing was filled. The pull cascade is pull, embed, score, enrich, embed, score, extract, score, summarise. Saving an agent and changing a company's affinity queue the `enrich` job (enrich, embed, score) behind their rescore; a vote does not. The stage has a switch, `stage:enrich`, over `enrich.enabled`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3adafd8950
commit
77b45a4b0f
10 files changed
+956
-219
No files matched your search
+44
-5
@@ -1,6 +1,6 @@
|
||||
"""The tasks roledesk offers, and the order a pull runs them in.
|
||||
|
||||
pull → embed → score → extract → score → summarise
|
||||
pull → embed → score → enrich → embed → score → extract → score → summarise
|
||||
|
||||
newsdesk's cascade convention: a cascade is a tuple of task names, and the
|
||||
final job of one task starts the next (`step` is the callback it carries).
|
||||
@@ -11,7 +11,10 @@ Every stage is always in the chain, and a stage that is switched off finds
|
||||
nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull does not
|
||||
satisfy, is an extract stage with no batches whose final job moves straight
|
||||
on. The free stages go first: embed and score cost no tokens, and the score
|
||||
they produce is what bounds the two that do. Extract reads only the best
|
||||
they produce is what bounds the rest. Enrich reads the best roles without
|
||||
text on their own pages (`app/jobs/enrich.py`), and embed and score follow for
|
||||
the roles it filled; when it filled none, `step` passes over both. Extract
|
||||
reads only the best
|
||||
roles of each enabled agent, as deep and as low as that agent's own `limits`
|
||||
say; the second score then applies the gates to the columns it filled.
|
||||
Summarise runs last, on the best roles by the same agent's summarise limits.
|
||||
@@ -23,6 +26,11 @@ having either way.
|
||||
`pull` is a job as well as a cascade, because a schedule enqueues a job kind
|
||||
and a cascade is started by one: the `pull` job's only work is to ask for the
|
||||
chain. Its default schedule, `every 6h`, is declared here beside it.
|
||||
|
||||
`enrich` is a job and a cascade the same way, `enrich → embed → score`, queued
|
||||
behind the `score` that follows a change to the ranking other than a pull's
|
||||
(`enrich_after`): an agent saved, a company's affinity changed. A vote moves
|
||||
one agent's taste a little and queues neither.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -32,16 +40,21 @@ from dataclasses import replace
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
from app.jobs import embed, extract, pull, score, summarise # noqa: F401 -- registers the jobs
|
||||
from app.jobs import embed, enrich, extract, pull, score, summarise # noqa: F401 -- registers the jobs
|
||||
from desk.tools import agent
|
||||
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
|
||||
|
||||
STEP = "app.jobs.cascade:step"
|
||||
|
||||
CASCADES: dict[str, tuple[str, ...]] = {
|
||||
"pull": ("pull", "embed", "score", "extract", "score", "summarise"),
|
||||
"pull": ("pull", "embed", "score", "enrich", "embed", "score", "extract", "score",
|
||||
"summarise"),
|
||||
"enrich": ("enrich", "embed", "score"),
|
||||
}
|
||||
|
||||
#: What follows `enrich` for the roles it filled, passed over when it filled none.
|
||||
REFILL = ("embed", "score")
|
||||
|
||||
|
||||
def _once(config: Any, store: Any) -> list[dict[str, Any]]:
|
||||
"""One batch with nothing in its payload: a task that is a single job."""
|
||||
@@ -91,6 +104,7 @@ STAGES: dict[str, Callable[[str], Task]] = {
|
||||
"score": lambda cascade: Task(kind="score", finder=_once,
|
||||
label="Scoring roles for every agent"),
|
||||
"summarise": _summarise,
|
||||
"enrich": lambda cascade: enrich.enrich_task(),
|
||||
}
|
||||
|
||||
|
||||
@@ -115,6 +129,10 @@ def step(config: Any, store: Any, tally: Tally, payload: dict[str, Any], *,
|
||||
chain goes on, as in newsdesk's cascade."""
|
||||
name, done = payload.get("cascade", ""), payload.get("task", "")
|
||||
at = int(payload.get("at", -1))
|
||||
tasks = CASCADES.get(name, ())
|
||||
if done == "enrich" and not enrich.filled(tally) \
|
||||
and tasks[at + 1:at + 1 + len(REFILL)] == REFILL:
|
||||
at += len(REFILL)
|
||||
following = follows(name, at)
|
||||
if following:
|
||||
context.task(task(following, cascade=name, at=at + 1))
|
||||
@@ -169,6 +187,26 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
|
||||
return {"started": " > ".join(CASCADES["pull"]), "run": run}
|
||||
|
||||
|
||||
@job("enrich", "Reading the best roles on their own pages", solo=True, resource="http")
|
||||
def run_enrich(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""Ask for the enrich cascade, `enrich → embed → score`. Its own work is
|
||||
nil, as the `pull` job's. Under `NULL` there is no queue to take it."""
|
||||
if not context.task(chain("enrich")):
|
||||
raise RuntimeError("no queue to start the enrich cascade on")
|
||||
return {"started": " > ".join(CASCADES["enrich"])}
|
||||
|
||||
|
||||
def enrich_after(queue: Any, score_job: Any) -> Any:
|
||||
"""Queue the `enrich` job behind `score_job`, the rescore that follows a
|
||||
change to the ranking, however that score ends. One per score job
|
||||
(`enrich:after:<id>`), so ten changes answered by one score are one
|
||||
enrich. None without a score job, or when the queue refuses."""
|
||||
if score_job is None:
|
||||
return None
|
||||
return queue.enqueue("enrich", depends_on=[score_job.id], after="done",
|
||||
dedup_key=f"enrich:after:{score_job.id}")
|
||||
|
||||
|
||||
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
|
||||
#: each, on its own. `pull` is not a task here, since it is a job: a name that
|
||||
#: is both would be listed twice by `/api/jobs/kinds`.
|
||||
@@ -182,4 +220,5 @@ for _name in ("extract", "embed", "score", "summarise"):
|
||||
# setting: a cadence edited or switched off under Runs survives
|
||||
# every restart (see pipeline/schedule.py).
|
||||
declare("pull", "pull", "every 6h",
|
||||
label="Pull every board, then embed, score, extract, rescore and summarise")
|
||||
label="Pull every board, then embed, score, read the best roles' pages, "
|
||||
"extract, rescore and summarise")
|
||||
@@ -0,0 +1,168 @@
|
||||
"""`enrich`: the best roles without text, read on their own pages.
|
||||
|
||||
Some hosts list a role without its text and state it on the role's own page
|
||||
only (`text="detail"`: Workday, SuccessFactors, Oracle, Phenom and the rest
|
||||
with a `detail`). The pull reads listings only; this stage reads those pages
|
||||
after scoring, so the calls go where the ranking is:
|
||||
|
||||
1. `enrich_waiting` hands over the open roles without text that an enabled
|
||||
agent with a prompt ranks ungated among its best `enrich.top` and at
|
||||
`enrich.min_score` or above, best first by the highest such score. A role
|
||||
below that bar for every agent gets no call.
|
||||
2. They are taken in that order, those whose read failed last, on boards whose reader has `detail`, at most
|
||||
`enrich.per_board` from one board and `enrich.per_run` in all, leaving
|
||||
out the roles already read, unless their read failed or gave no text
|
||||
`enrich.retry_days` ago (`plugins.ats.detail_due`).
|
||||
3. One batch per board, on the `http` pool, reads them
|
||||
(`plugins.ats.read_board_details`, which keeps `ats:detailed:<target>`,
|
||||
`ats:detailfailed:<target>` and `ats:detailempty:<target>`) and stores
|
||||
what each page filled. `Store.upsert_role` drops the vector of a role
|
||||
whose text improved, so the next `embed` encodes those roles only.
|
||||
|
||||
In the pull cascade `embed` and `score` follow; `cascade.step` passes over
|
||||
both when no role was filled. The same stage follows a rescore after an agent
|
||||
is saved or a company's affinity changes (`cascade.enrich_after`), never after
|
||||
a vote. Whether it runs is `stages.stage_on(config, store, "enrich")`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import Config
|
||||
from app.jobs import stages
|
||||
from app.storage import Store
|
||||
from pipeline import NULL, Batch, RunContext, Tally, Task, job
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
KIND = "enrich_batch"
|
||||
SETTLE = "app.jobs.enrich:settle"
|
||||
|
||||
|
||||
def _ats() -> Any:
|
||||
"""`plugins.ats`, imported when needed, the way `pull.run_batch` does."""
|
||||
from plugins import ats
|
||||
|
||||
return ats
|
||||
|
||||
|
||||
def _row(found: dict[str, Any], target: str) -> dict[str, Any]:
|
||||
"""A stored role back as the row its reader listed: the stored `raw`
|
||||
with the host's own id, the title, the link and the date."""
|
||||
try:
|
||||
raw = json.loads(found.get("raw") or "{}")
|
||||
except ValueError:
|
||||
raw = {}
|
||||
raw = raw if isinstance(raw, dict) else {}
|
||||
try:
|
||||
posted: datetime | None = datetime.fromisoformat(str(found.get("published_at") or ""))
|
||||
except ValueError:
|
||||
posted = None
|
||||
return raw | {"source_id": str(found["source_id"]).removeprefix(f"{target}:"),
|
||||
"title": found["title"], "url": found["url"], "description": "",
|
||||
"posted": posted}
|
||||
|
||||
|
||||
def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
|
||||
"""`(target, uids)` per board, best board first, within the budgets: the
|
||||
roles `enrich_waiting` ranks, on fetchable boards whose reader has
|
||||
`detail` and that are not failing, and due for a read."""
|
||||
settings = config.enrich
|
||||
ats = _ats()
|
||||
boards = {sub.id: sub.target for sub in store.subscriptions(source="ats", fetchable=True)}
|
||||
by_board: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for found in store.enrich_waiting(settings.top, settings.min_score):
|
||||
target = boards.get(found["subscription_id"])
|
||||
if target is None:
|
||||
continue
|
||||
module = ats.readers.HOSTS.get(target.partition(":")[0])
|
||||
if module is None or not hasattr(module, "detail") \
|
||||
or f"ats:failing:{target}" in store.state:
|
||||
continue
|
||||
by_board[target].append(found)
|
||||
due = {target: set(ats.detail_due(store.state, target, [f["source_id"] for f in found],
|
||||
retry_days=settings.retry_days))
|
||||
for target, found in by_board.items()}
|
||||
# Best first; roles whose read failed after every other, so a page that
|
||||
# keeps failing cannot take the budget of the roles never tried.
|
||||
failed = {target: ats._runner.id_set(store.state, ats.detail_keys(target)[1])
|
||||
for target in by_board}
|
||||
ranked = sorted((f for found in by_board.values() for f in found),
|
||||
key=lambda f: (f["source_id"] in failed[boards[f["subscription_id"]]],
|
||||
-float(f["score"] or 0), f["uid"]))
|
||||
picked: dict[str, list[str]] = {}
|
||||
total = 0
|
||||
for found in ranked:
|
||||
target = boards[found["subscription_id"]]
|
||||
if found["source_id"] not in due[target]:
|
||||
continue
|
||||
if total >= settings.per_run:
|
||||
break
|
||||
uids = picked.setdefault(target, [])
|
||||
if len(uids) >= settings.per_board:
|
||||
continue
|
||||
uids.append(found["uid"])
|
||||
total += 1
|
||||
return list(picked.items())
|
||||
|
||||
|
||||
def batches(config: Config, store: Store) -> list[Batch]:
|
||||
"""The finder: one batch per board, nothing while the stage is off."""
|
||||
if not stages.stage_on(config, store, "enrich"):
|
||||
return []
|
||||
return [Batch({"target": target, "uids": uids},
|
||||
label=f"Reading {len(uids)} roles on {target}")
|
||||
for target, uids in plan(config, store)]
|
||||
|
||||
|
||||
@job(KIND, "Reading roles on their own pages", resource="http")
|
||||
def run_batch(config: Config, store: Store, *, target: str, uids: list[str],
|
||||
context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""Read one board's roles in `uids` on their own pages, in that order,
|
||||
and store what each page filled. `filled` is how many roles changed,
|
||||
`texts` how many of them now have text."""
|
||||
ats = _ats()
|
||||
subs = [s for s in store.subscriptions(source="ats", fetchable=True) if s.target == target]
|
||||
if not subs or not uids:
|
||||
return {"target": target, "filled": 0, "texts": 0}
|
||||
sub = subs[0]
|
||||
found = {f["uid"]: f for f in store.enrich_waiting(0, 0.0, uids)}
|
||||
rows = [_row(found[uid], target) for uid in uids if uid in found]
|
||||
context.check()
|
||||
with httpx.Client() as client:
|
||||
filled = ats.read_board_details(store.state, target, rows, client=client,
|
||||
cap=len(rows), retry_days=config.enrich.retry_days)
|
||||
label = sub.company or sub.label
|
||||
texts = 0
|
||||
for row in filled:
|
||||
item = ats.item_of(row, target, label)
|
||||
item.source = "ats"
|
||||
store.upsert_role(item, sub)
|
||||
texts += bool(row.get("description"))
|
||||
log.info("enrich: %s: %d read, %d filled, %d with text", target, len(rows), len(filled),
|
||||
texts)
|
||||
return {"target": target, "read": len(rows), "filled": len(filled), "texts": texts}
|
||||
|
||||
|
||||
def filled(tally: Tally) -> int:
|
||||
"""How many roles the batches behind `tally` filled."""
|
||||
return sum(int(result.get("filled") or 0) for result in tally.results)
|
||||
|
||||
|
||||
def settle(config: Config, store: Store, tally: Tally, *,
|
||||
context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""The run's sum: roles filled, and how many of them now have text."""
|
||||
return {"filled": filled(tally),
|
||||
"texts": sum(int(result.get("texts") or 0) for result in tally.results)}
|
||||
|
||||
|
||||
def enrich_task() -> Task:
|
||||
return Task(kind=KIND, finder=batches, name="enrich", settle=SETTLE,
|
||||
label="Reading the best roles on their own pages")
|
||||
+16
-7
@@ -1,4 +1,5 @@
|
||||
"""Whether the two model stages of a pull, `extract` and `summarise`, run.
|
||||
"""Whether the two model stages of a pull, `extract` and `summarise`, and
|
||||
the `enrich` stage run.
|
||||
|
||||
`extract.enabled` and `summarise.enabled` in config.yaml are read at startup
|
||||
and say nothing on the page, so the switch lives in the database as well: the
|
||||
@@ -8,9 +9,13 @@ default for a fresh database and a deployment that never uses the switch
|
||||
behaves as it did. `stage_on` is the one place that decides, and both tasks'
|
||||
finders ask it (`extract.extract_task`, `summarise.summarise_task`).
|
||||
|
||||
`describe` is what `GET /api/stages` and `app stages` show: for each stage
|
||||
whether it is on, where that comes from, the model it uses, and who would
|
||||
answer.
|
||||
`enrich` (reading the best roles on their own pages, `app/jobs/enrich.py`)
|
||||
asks no model and is on by default (`enrich.enabled`); its switch is
|
||||
`stage:enrich`, set the same way through `PUT /api/stages/enrich`.
|
||||
|
||||
`describe` is what `GET /api/stages` and `app stages` show: for each model
|
||||
stage whether it is on, where that comes from, the model it uses, and who
|
||||
would answer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -25,6 +30,9 @@ from desk.tools import agent
|
||||
|
||||
NAMES = ("extract", "summarise")
|
||||
|
||||
#: Every stage with a switch: the model stages and `enrich`.
|
||||
SWITCHES = (*NAMES, "enrich")
|
||||
|
||||
NO_MODEL = ("no model is reachable: set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN, "
|
||||
"see docs/llm-backends.md")
|
||||
ANSWERS = {
|
||||
@@ -39,7 +47,7 @@ class Stage(BaseModel):
|
||||
`backend` is `agent.backend`'s answer, "" for nobody, and `answer` says
|
||||
it in words."""
|
||||
|
||||
name: Literal["extract", "summarise"]
|
||||
name: Literal["extract", "summarise", "enrich"]
|
||||
on: bool
|
||||
source: Literal["stored", "config"]
|
||||
model: str
|
||||
@@ -65,7 +73,7 @@ def stage_on(config: Config, store: Store, name: str) -> bool:
|
||||
|
||||
def set_stage(store: Store, name: str, on: bool) -> None:
|
||||
"""Store the switch. A `KeyError` for a name that is not a stage."""
|
||||
if name not in NAMES:
|
||||
if name not in SWITCHES:
|
||||
raise KeyError(name)
|
||||
store.state[_key(name)] = "on" if on else "off"
|
||||
|
||||
@@ -84,7 +92,8 @@ def describe_one(config: Config, store: Store, name: str) -> Stage:
|
||||
backend, said = answer(config)
|
||||
return Stage(name=name, on=stage_on(config, store, name),
|
||||
source="config" if stored(store, name) is None else "stored",
|
||||
model=getattr(config, name).model, backend=backend, answer=said)
|
||||
model=getattr(getattr(config, name), "model", ""), backend=backend,
|
||||
answer=said)
|
||||
|
||||
|
||||
def describe(config: Config, store: Store) -> list[Stage]:
|
||||
|
||||
Reference in new issue
Block a user