Merge branch 'wp-g-enrich'

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:53:57 +02:00
commit 38a80d5b55
10 files changed
+956 -219

No files matched your search

+20
View File
@@ -66,6 +66,25 @@ class SummariseConfig(ModelTaskConfig):
batch_size: int = Field(10, ge=1)
class EnrichConfig(Section):
"""Reading roles on their own pages after scoring, for the hosts whose
list carries no text. See app/jobs/enrich.py. A role is read when it has
no text and an enabled agent with a prompt ranks it ungated among its
best `top` and at `min_score` or above: the extract bar (`Limits`,
500 and 0.55) with the score a notch lower, since a role ranked on its
title alone is the one whose score its text moves most."""
enabled: bool = True
top: int = Field(500, ge=0)
min_score: float = Field(0.5, ge=0, le=1)
#: Reads per board and per run. Per board is the cap the pull had; the
#: boards are read in parallel on the `http` pool.
per_board: int = Field(200, ge=0)
per_run: int = Field(1000, ge=0)
#: Days before a page that gave no text is read again.
retry_days: int = Field(7, ge=1)
class Calibration(Section):
"""Where one embedding model's cosines sit, as `(lo, hi)` for fit and for
taste: `score.scale` maps `lo` to 0 and `hi` to 1 and clips outside, so a
@@ -181,6 +200,7 @@ class Config(Section):
extract: ExtractConfig = Field(default_factory=ExtractConfig)
summarise: SummariseConfig = Field(default_factory=SummariseConfig)
embed: EmbedConfig = Field(default_factory=EmbedConfig)
enrich: EnrichConfig = Field(default_factory=EnrichConfig)
#: Starter agents, imported once into an empty table.
agents: list[AgentSeed] = Field(default_factory=list)
#: Starter searches, imported once into an empty set of `search`
+44 -5
View File
@@ -1,6 +1,6 @@
"""The tasks roledesk offers, and the order a pull runs them in.
pull → embed → score → extract → score → summarise
pull → embed → score → enrich → embed → score → extract → score → summarise
newsdesk's cascade convention: a cascade is a tuple of task names, and the
final job of one task starts the next (`step` is the callback it carries).
@@ -11,7 +11,10 @@ Every stage is always in the chain, and a stage that is switched off finds
nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull does not
satisfy, is an extract stage with no batches whose final job moves straight
on. The free stages go first: embed and score cost no tokens, and the score
they produce is what bounds the two that do. Extract reads only the best
they produce is what bounds the rest. Enrich reads the best roles without
text on their own pages (`app/jobs/enrich.py`), and embed and score follow for
the roles it filled; when it filled none, `step` passes over both. Extract
reads only the best
roles of each enabled agent, as deep and as low as that agent's own `limits`
say; the second score then applies the gates to the columns it filled.
Summarise runs last, on the best roles by the same agent's summarise limits.
@@ -23,6 +26,11 @@ having either way.
`pull` is a job as well as a cascade, because a schedule enqueues a job kind
and a cascade is started by one: the `pull` job's only work is to ask for the
chain. Its default schedule, `every 6h`, is declared here beside it.
`enrich` is a job and a cascade the same way, `enrich → embed → score`, queued
behind the `score` that follows a change to the ranking other than a pull's
(`enrich_after`): an agent saved, a company's affinity changed. A vote moves
one agent's taste a little and queues neither.
"""
from __future__ import annotations
@@ -32,16 +40,21 @@ from dataclasses import replace
from functools import partial
from typing import Any
from app.jobs import embed, extract, pull, score, summarise # noqa: F401 -- registers the jobs
from app.jobs import embed, enrich, extract, pull, score, summarise # noqa: F401 -- registers the jobs
from desk.tools import agent
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
STEP = "app.jobs.cascade:step"
CASCADES: dict[str, tuple[str, ...]] = {
"pull": ("pull", "embed", "score", "extract", "score", "summarise"),
"pull": ("pull", "embed", "score", "enrich", "embed", "score", "extract", "score",
"summarise"),
"enrich": ("enrich", "embed", "score"),
}
#: What follows `enrich` for the roles it filled, passed over when it filled none.
REFILL = ("embed", "score")
def _once(config: Any, store: Any) -> list[dict[str, Any]]:
"""One batch with nothing in its payload: a task that is a single job."""
@@ -91,6 +104,7 @@ STAGES: dict[str, Callable[[str], Task]] = {
"score": lambda cascade: Task(kind="score", finder=_once,
label="Scoring roles for every agent"),
"summarise": _summarise,
"enrich": lambda cascade: enrich.enrich_task(),
}
@@ -115,6 +129,10 @@ def step(config: Any, store: Any, tally: Tally, payload: dict[str, Any], *,
chain goes on, as in newsdesk's cascade."""
name, done = payload.get("cascade", ""), payload.get("task", "")
at = int(payload.get("at", -1))
tasks = CASCADES.get(name, ())
if done == "enrich" and not enrich.filled(tally) \
and tasks[at + 1:at + 1 + len(REFILL)] == REFILL:
at += len(REFILL)
following = follows(name, at)
if following:
context.task(task(following, cascade=name, at=at + 1))
@@ -169,6 +187,26 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
return {"started": " > ".join(CASCADES["pull"]), "run": run}
@job("enrich", "Reading the best roles on their own pages", solo=True, resource="http")
def run_enrich(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
"""Ask for the enrich cascade, `enrich → embed → score`. Its own work is
nil, as the `pull` job's. Under `NULL` there is no queue to take it."""
if not context.task(chain("enrich")):
raise RuntimeError("no queue to start the enrich cascade on")
return {"started": " > ".join(CASCADES["enrich"])}
def enrich_after(queue: Any, score_job: Any) -> Any:
"""Queue the `enrich` job behind `score_job`, the rescore that follows a
change to the ranking, however that score ends. One per score job
(`enrich:after:<id>`), so ten changes answered by one score are one
enrich. None without a score job, or when the queue refuses."""
if score_job is None:
return None
return queue.enqueue("enrich", depends_on=[score_job.id], after="done",
dedup_key=f"enrich:after:{score_job.id}")
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
#: each, on its own. `pull` is not a task here, since it is a job: a name that
#: is both would be listed twice by `/api/jobs/kinds`.
@@ -182,4 +220,5 @@ for _name in ("extract", "embed", "score", "summarise"):
# setting: a cadence edited or switched off under Runs survives
# every restart (see pipeline/schedule.py).
declare("pull", "pull", "every 6h",
label="Pull every board, then embed, score, extract, rescore and summarise")
label="Pull every board, then embed, score, read the best roles' pages, "
"extract, rescore and summarise")
+168
View File
@@ -0,0 +1,168 @@
"""`enrich`: the best roles without text, read on their own pages.
Some hosts list a role without its text and state it on the role's own page
only (`text="detail"`: Workday, SuccessFactors, Oracle, Phenom and the rest
with a `detail`). The pull reads listings only; this stage reads those pages
after scoring, so the calls go where the ranking is:
1. `enrich_waiting` hands over the open roles without text that an enabled
agent with a prompt ranks ungated among its best `enrich.top` and at
`enrich.min_score` or above, best first by the highest such score. A role
below that bar for every agent gets no call.
2. They are taken in that order, those whose read failed last, on boards whose reader has `detail`, at most
`enrich.per_board` from one board and `enrich.per_run` in all, leaving
out the roles already read, unless their read failed or gave no text
`enrich.retry_days` ago (`plugins.ats.detail_due`).
3. One batch per board, on the `http` pool, reads them
(`plugins.ats.read_board_details`, which keeps `ats:detailed:<target>`,
`ats:detailfailed:<target>` and `ats:detailempty:<target>`) and stores
what each page filled. `Store.upsert_role` drops the vector of a role
whose text improved, so the next `embed` encodes those roles only.
In the pull cascade `embed` and `score` follow; `cascade.step` passes over
both when no role was filled. The same stage follows a rescore after an agent
is saved or a company's affinity changes (`cascade.enrich_after`), never after
a vote. Whether it runs is `stages.stage_on(config, store, "enrich")`.
"""
from __future__ import annotations
import json
import logging
from collections import defaultdict
from datetime import datetime
from typing import Any
import httpx
from app.config import Config
from app.jobs import stages
from app.storage import Store
from pipeline import NULL, Batch, RunContext, Tally, Task, job
log = logging.getLogger(__name__)
KIND = "enrich_batch"
SETTLE = "app.jobs.enrich:settle"
def _ats() -> Any:
"""`plugins.ats`, imported when needed, the way `pull.run_batch` does."""
from plugins import ats
return ats
def _row(found: dict[str, Any], target: str) -> dict[str, Any]:
"""A stored role back as the row its reader listed: the stored `raw`
with the host's own id, the title, the link and the date."""
try:
raw = json.loads(found.get("raw") or "{}")
except ValueError:
raw = {}
raw = raw if isinstance(raw, dict) else {}
try:
posted: datetime | None = datetime.fromisoformat(str(found.get("published_at") or ""))
except ValueError:
posted = None
return raw | {"source_id": str(found["source_id"]).removeprefix(f"{target}:"),
"title": found["title"], "url": found["url"], "description": "",
"posted": posted}
def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
"""`(target, uids)` per board, best board first, within the budgets: the
roles `enrich_waiting` ranks, on fetchable boards whose reader has
`detail` and that are not failing, and due for a read."""
settings = config.enrich
ats = _ats()
boards = {sub.id: sub.target for sub in store.subscriptions(source="ats", fetchable=True)}
by_board: dict[str, list[dict[str, Any]]] = defaultdict(list)
for found in store.enrich_waiting(settings.top, settings.min_score):
target = boards.get(found["subscription_id"])
if target is None:
continue
module = ats.readers.HOSTS.get(target.partition(":")[0])
if module is None or not hasattr(module, "detail") \
or f"ats:failing:{target}" in store.state:
continue
by_board[target].append(found)
due = {target: set(ats.detail_due(store.state, target, [f["source_id"] for f in found],
retry_days=settings.retry_days))
for target, found in by_board.items()}
# Best first; roles whose read failed after every other, so a page that
# keeps failing cannot take the budget of the roles never tried.
failed = {target: ats._runner.id_set(store.state, ats.detail_keys(target)[1])
for target in by_board}
ranked = sorted((f for found in by_board.values() for f in found),
key=lambda f: (f["source_id"] in failed[boards[f["subscription_id"]]],
-float(f["score"] or 0), f["uid"]))
picked: dict[str, list[str]] = {}
total = 0
for found in ranked:
target = boards[found["subscription_id"]]
if found["source_id"] not in due[target]:
continue
if total >= settings.per_run:
break
uids = picked.setdefault(target, [])
if len(uids) >= settings.per_board:
continue
uids.append(found["uid"])
total += 1
return list(picked.items())
def batches(config: Config, store: Store) -> list[Batch]:
"""The finder: one batch per board, nothing while the stage is off."""
if not stages.stage_on(config, store, "enrich"):
return []
return [Batch({"target": target, "uids": uids},
label=f"Reading {len(uids)} roles on {target}")
for target, uids in plan(config, store)]
@job(KIND, "Reading roles on their own pages", resource="http")
def run_batch(config: Config, store: Store, *, target: str, uids: list[str],
context: RunContext = NULL) -> dict[str, Any]:
"""Read one board's roles in `uids` on their own pages, in that order,
and store what each page filled. `filled` is how many roles changed,
`texts` how many of them now have text."""
ats = _ats()
subs = [s for s in store.subscriptions(source="ats", fetchable=True) if s.target == target]
if not subs or not uids:
return {"target": target, "filled": 0, "texts": 0}
sub = subs[0]
found = {f["uid"]: f for f in store.enrich_waiting(0, 0.0, uids)}
rows = [_row(found[uid], target) for uid in uids if uid in found]
context.check()
with httpx.Client() as client:
filled = ats.read_board_details(store.state, target, rows, client=client,
cap=len(rows), retry_days=config.enrich.retry_days)
label = sub.company or sub.label
texts = 0
for row in filled:
item = ats.item_of(row, target, label)
item.source = "ats"
store.upsert_role(item, sub)
texts += bool(row.get("description"))
log.info("enrich: %s: %d read, %d filled, %d with text", target, len(rows), len(filled),
texts)
return {"target": target, "read": len(rows), "filled": len(filled), "texts": texts}
def filled(tally: Tally) -> int:
"""How many roles the batches behind `tally` filled."""
return sum(int(result.get("filled") or 0) for result in tally.results)
def settle(config: Config, store: Store, tally: Tally, *,
context: RunContext = NULL) -> dict[str, Any]:
"""The run's sum: roles filled, and how many of them now have text."""
return {"filled": filled(tally),
"texts": sum(int(result.get("texts") or 0) for result in tally.results)}
def enrich_task() -> Task:
return Task(kind=KIND, finder=batches, name="enrich", settle=SETTLE,
label="Reading the best roles on their own pages")
+16 -7
View File
@@ -1,4 +1,5 @@
"""Whether the two model stages of a pull, `extract` and `summarise`, run.
"""Whether the two model stages of a pull, `extract` and `summarise`, and
the `enrich` stage run.
`extract.enabled` and `summarise.enabled` in config.yaml are read at startup
and say nothing on the page, so the switch lives in the database as well: the
@@ -8,9 +9,13 @@ default for a fresh database and a deployment that never uses the switch
behaves as it did. `stage_on` is the one place that decides, and both tasks'
finders ask it (`extract.extract_task`, `summarise.summarise_task`).
`describe` is what `GET /api/stages` and `app stages` show: for each stage
whether it is on, where that comes from, the model it uses, and who would
answer.
`enrich` (reading the best roles on their own pages, `app/jobs/enrich.py`)
asks no model and is on by default (`enrich.enabled`); its switch is
`stage:enrich`, set the same way through `PUT /api/stages/enrich`.
`describe` is what `GET /api/stages` and `app stages` show: for each model
stage whether it is on, where that comes from, the model it uses, and who
would answer.
"""
from __future__ import annotations
@@ -25,6 +30,9 @@ from desk.tools import agent
NAMES = ("extract", "summarise")
#: Every stage with a switch: the model stages and `enrich`.
SWITCHES = (*NAMES, "enrich")
NO_MODEL = ("no model is reachable: set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN, "
"see docs/llm-backends.md")
ANSWERS = {
@@ -39,7 +47,7 @@ class Stage(BaseModel):
`backend` is `agent.backend`'s answer, "" for nobody, and `answer` says
it in words."""
name: Literal["extract", "summarise"]
name: Literal["extract", "summarise", "enrich"]
on: bool
source: Literal["stored", "config"]
model: str
@@ -65,7 +73,7 @@ def stage_on(config: Config, store: Store, name: str) -> bool:
def set_stage(store: Store, name: str, on: bool) -> None:
"""Store the switch. A `KeyError` for a name that is not a stage."""
if name not in NAMES:
if name not in SWITCHES:
raise KeyError(name)
store.state[_key(name)] = "on" if on else "off"
@@ -84,7 +92,8 @@ def describe_one(config: Config, store: Store, name: str) -> Stage:
backend, said = answer(config)
return Stage(name=name, on=stage_on(config, store, name),
source="config" if stored(store, name) is None else "stored",
model=getattr(config, name).model, backend=backend, answer=said)
model=getattr(getattr(config, name), "model", ""), backend=backend,
answer=said)
def describe(config: Config, store: Store) -> list[Stage]:
+40
View File
@@ -1503,6 +1503,46 @@ class Store:
return None
return self.role(uid)
# --------------------------------------------------------------- enrich
def enrich_waiting(self, top: int, floor: float,
uids: Sequence[str] | None = None) -> list[dict[str, Any]]:
"""Open `ats` roles without text that an enabled agent with a prompt
ranks ungated among its best `top` and at `floor` or above, best
first by the highest such score, as `{uid, source_id,
subscription_id, title, url, published_at, raw, score}`. The ranking
is `_top_scope`'s, with the bar given rather than the agent's own.
Given `uids`, those of them still without text, whatever their rank,
in no order (`score` is None), since a batch finishes what it was
dealt."""
fields = ("r.uid, r.source_id, r.subscription_id, r.title, r.original_url AS url, "
"r.published_at, r.raw")
waiting = ("r.closed_at IS NULL AND r.dup_of IS NULL AND r.source = 'ats' "
"AND r.subscription_id IS NOT NULL AND r.description = '' "
"AND r.article_text = ''")
if uids is not None:
rows = self._rows(
f"SELECT {fields}, NULL AS score FROM roles r WHERE {waiting} "
"AND r.uid IN (SELECT value FROM json_each(?))", (json.dumps(list(uids)),))
else:
rows = self._rows(
f"""SELECT {fields}, MAX(ranked.score) AS score FROM (
SELECT s.uid, s.score, ROW_NUMBER() OVER (
PARTITION BY s.agent_id
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
FROM scores s
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
AND EXISTS (SELECT 1 FROM agent_prompts p
WHERE p.agent_id = a.id
AND TRIM(p.text, ' ' || char(9, 10, 13)) <> '')
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
AND top.dup_of IS NULL
WHERE s.gated_reason = '') ranked
JOIN roles r ON r.uid = ranked.uid
WHERE ranked.n <= ? AND ranked.score >= ? AND {waiting}
GROUP BY r.uid ORDER BY score DESC, r.uid""", (top, floor))
return [dict(row) for row in rows]
# -------------------------------------------------------------- vectors
def roles_to_embed(self, model: str, limit: int) -> list[Role]:
+9 -6
View File
@@ -681,6 +681,7 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep,
except KeyError:
raise HTTPException(404, f"no such company: {key}") from None
job = _score_after_change(queue)
jobs.cascade.enrich_after(queue, job)
return AffinityOut(**company.model_dump(), score_job=job.id if job else None)
@@ -965,17 +966,19 @@ class AgentBody(BaseModel):
@router.post("/agents")
def save_agent(body: AgentBody, store: StoreDep) -> Agent:
"""Scores are not recomputed here: the page asks `POST /api/score` after
a save and waits on the job it answers with."""
def save_agent(body: AgentBody, store: StoreDep, queue: QueueDep) -> Agent:
"""Queues the rescore, which the page's `POST /api/score` after a save
joins, and the enrich run behind it (`cascade.enrich_after`)."""
try:
return store.save_agent(body.name, body.intent, body.gates, body.weights,
body.enabled, agent_id=body.id, limits=body.limits,
prompts=body.prompts)
saved = store.save_agent(body.name, body.intent, body.gates, body.weights,
body.enabled, agent_id=body.id, limits=body.limits,
prompts=body.prompts)
except ValueError as exc:
raise HTTPException(409, str(exc)) from exc
except KeyError:
raise HTTPException(404, f"no such agent: {body.id}") from None
jobs.cascade.enrich_after(queue, _score_after_change(queue))
return saved
@router.delete("/agents/{agent_id}", status_code=204)