Files
jobboard/app/jobs/enrich.py
T
WolfandClaude Opus 5.5 77b45a4b0f Read role pages in an enrich stage after scoring, best roles first
The pull reads listings only. `enrich` (app/jobs/enrich.py) takes the open
roles without text that an enabled agent ranks ungated in its best
`enrich.top` at `enrich.min_score` or above, best score first, on boards
whose reader has `detail`, within `enrich.per_board` and `enrich.per_run`,
one batch per board. `plugins.ats.read_board_details` reads them and keeps
the per-board state; a read that gave no text is read again after
`enrich.retry_days` (finding 18). Filled roles are upserted, which drops
their vectors, and embed and score follow; `cascade.step` passes over both
when nothing was filled.

The pull cascade is pull, embed, score, enrich, embed, score, extract, score,
summarise. Saving an agent and changing a company's affinity queue the
`enrich` job (enrich, embed, score) behind their rescore; a vote does not.
The stage has a switch, `stage:enrich`, over `enrich.enabled`.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 09:53:29 +02:00

169 lines
7.2 KiB
Python

"""`enrich`: the best roles without text, read on their own pages.
Some hosts list a role without its text and state it on the role's own page
only (`text="detail"`: Workday, SuccessFactors, Oracle, Phenom and the rest
with a `detail`). The pull reads listings only; this stage reads those pages
after scoring, so the calls go where the ranking is:
1. `enrich_waiting` hands over the open roles without text that an enabled
agent with a prompt ranks ungated among its best `enrich.top` and at
`enrich.min_score` or above, best first by the highest such score. A role
below that bar for every agent gets no call.
2. They are taken in that order, those whose read failed last, on boards whose reader has `detail`, at most
`enrich.per_board` from one board and `enrich.per_run` in all, leaving
out the roles already read, unless their read failed or gave no text
`enrich.retry_days` ago (`plugins.ats.detail_due`).
3. One batch per board, on the `http` pool, reads them
(`plugins.ats.read_board_details`, which keeps `ats:detailed:<target>`,
`ats:detailfailed:<target>` and `ats:detailempty:<target>`) and stores
what each page filled. `Store.upsert_role` drops the vector of a role
whose text improved, so the next `embed` encodes those roles only.
In the pull cascade `embed` and `score` follow; `cascade.step` passes over
both when no role was filled. The same stage follows a rescore after an agent
is saved or a company's affinity changes (`cascade.enrich_after`), never after
a vote. Whether it runs is `stages.stage_on(config, store, "enrich")`.
"""
from __future__ import annotations
import json
import logging
from collections import defaultdict
from datetime import datetime
from typing import Any
import httpx
from app.config import Config
from app.jobs import stages
from app.storage import Store
from pipeline import NULL, Batch, RunContext, Tally, Task, job
log = logging.getLogger(__name__)
KIND = "enrich_batch"
SETTLE = "app.jobs.enrich:settle"
def _ats() -> Any:
"""`plugins.ats`, imported when needed, the way `pull.run_batch` does."""
from plugins import ats
return ats
def _row(found: dict[str, Any], target: str) -> dict[str, Any]:
"""A stored role back as the row its reader listed: the stored `raw`
with the host's own id, the title, the link and the date."""
try:
raw = json.loads(found.get("raw") or "{}")
except ValueError:
raw = {}
raw = raw if isinstance(raw, dict) else {}
try:
posted: datetime | None = datetime.fromisoformat(str(found.get("published_at") or ""))
except ValueError:
posted = None
return raw | {"source_id": str(found["source_id"]).removeprefix(f"{target}:"),
"title": found["title"], "url": found["url"], "description": "",
"posted": posted}
def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
"""`(target, uids)` per board, best board first, within the budgets: the
roles `enrich_waiting` ranks, on fetchable boards whose reader has
`detail` and that are not failing, and due for a read."""
settings = config.enrich
ats = _ats()
boards = {sub.id: sub.target for sub in store.subscriptions(source="ats", fetchable=True)}
by_board: dict[str, list[dict[str, Any]]] = defaultdict(list)
for found in store.enrich_waiting(settings.top, settings.min_score):
target = boards.get(found["subscription_id"])
if target is None:
continue
module = ats.readers.HOSTS.get(target.partition(":")[0])
if module is None or not hasattr(module, "detail") \
or f"ats:failing:{target}" in store.state:
continue
by_board[target].append(found)
due = {target: set(ats.detail_due(store.state, target, [f["source_id"] for f in found],
retry_days=settings.retry_days))
for target, found in by_board.items()}
# Best first; roles whose read failed after every other, so a page that
# keeps failing cannot take the budget of the roles never tried.
failed = {target: ats._runner.id_set(store.state, ats.detail_keys(target)[1])
for target in by_board}
ranked = sorted((f for found in by_board.values() for f in found),
key=lambda f: (f["source_id"] in failed[boards[f["subscription_id"]]],
-float(f["score"] or 0), f["uid"]))
picked: dict[str, list[str]] = {}
total = 0
for found in ranked:
target = boards[found["subscription_id"]]
if found["source_id"] not in due[target]:
continue
if total >= settings.per_run:
break
uids = picked.setdefault(target, [])
if len(uids) >= settings.per_board:
continue
uids.append(found["uid"])
total += 1
return list(picked.items())
def batches(config: Config, store: Store) -> list[Batch]:
"""The finder: one batch per board, nothing while the stage is off."""
if not stages.stage_on(config, store, "enrich"):
return []
return [Batch({"target": target, "uids": uids},
label=f"Reading {len(uids)} roles on {target}")
for target, uids in plan(config, store)]
@job(KIND, "Reading roles on their own pages", resource="http")
def run_batch(config: Config, store: Store, *, target: str, uids: list[str],
context: RunContext = NULL) -> dict[str, Any]:
"""Read one board's roles in `uids` on their own pages, in that order,
and store what each page filled. `filled` is how many roles changed,
`texts` how many of them now have text."""
ats = _ats()
subs = [s for s in store.subscriptions(source="ats", fetchable=True) if s.target == target]
if not subs or not uids:
return {"target": target, "filled": 0, "texts": 0}
sub = subs[0]
found = {f["uid"]: f for f in store.enrich_waiting(0, 0.0, uids)}
rows = [_row(found[uid], target) for uid in uids if uid in found]
context.check()
with httpx.Client() as client:
filled = ats.read_board_details(store.state, target, rows, client=client,
cap=len(rows), retry_days=config.enrich.retry_days)
label = sub.company or sub.label
texts = 0
for row in filled:
item = ats.item_of(row, target, label)
item.source = "ats"
store.upsert_role(item, sub)
texts += bool(row.get("description"))
log.info("enrich: %s: %d read, %d filled, %d with text", target, len(rows), len(filled),
texts)
return {"target": target, "read": len(rows), "filled": len(filled), "texts": texts}
def filled(tally: Tally) -> int:
"""How many roles the batches behind `tally` filled."""
return sum(int(result.get("filled") or 0) for result in tally.results)
def settle(config: Config, store: Store, tally: Tally, *,
context: RunContext = NULL) -> dict[str, Any]:
"""The run's sum: roles filled, and how many of them now have text."""
return {"filled": filled(tally),
"texts": sum(int(result.get("texts") or 0) for result in tally.results)}
def enrich_task() -> Task:
return Task(kind=KIND, finder=batches, name="enrich", settle=SETTLE,
label="Reading the best roles on their own pages")