Read a search role's own page for its text in enrich and on the Fetch button when its reader has detail.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet 5.5 committed 2026-10-03 12:29:19 +02:00
1 parent 07e3bb4756
commit b148647ac3
5 files changed
+346 -40

No files matched your search

+51 -31
View File
@@ -2,18 +2,22 @@
Some hosts list a role without its text and state it on the role's own page
only (`text="detail"`: Workday, SuccessFactors, Oracle, Phenom and the rest
with a `detail`). The pull reads listings only; this stage reads those pages
after scoring, so the calls go where the ranking is:
with a `detail`, and a search provider such as Jobbank). The pull reads
listings only; this stage reads those pages after scoring, so the calls go
where the ranking is. A saved search whose reader has `detail` is treated as
a board under its target `provider:name`: the same bar, budgets and state
keys, and its failing record is `search:failing:<target>`.
1. `enrich_waiting` hands over the open roles without text that an enabled
agent with a prompt ranks ungated among its best `enrich.top` and at
`enrich.min_score` or above, best first by the highest such score. A role
below that bar for every agent gets no call.
2. They are taken in that order, those whose read failed last, on boards whose reader has `detail`, at most
`enrich.per_board` from one board and `enrich.per_run` in all, leaving
out the roles already read, unless their read failed or gave no text
`enrich.retry_days` ago (`plugins.ats.detail_due`).
3. One batch per board, on the `http` pool, reads them
2. They are taken in that order, those whose read failed last, on boards and
searches whose reader has `detail`, at most `enrich.per_board` from one
board and `enrich.per_run` in all, leaving out the roles already read,
unless their read failed or gave no text `enrich.retry_days` ago
(`plugins.ats.detail_due`).
3. One batch per board or search, on the `http` pool, reads them
(`plugins.ats.read_board_details`, which keeps `ats:detailed:<target>`,
`ats:detailfailed:<target>` and `ats:detailempty:<target>`) and stores
what each page filled. `Store.upsert_role` drops the vector of a role
@@ -37,6 +41,7 @@ import httpx
from app.config import Config
from app.jobs import stages
from app.models import Subscription
from app.storage import Store
from pipeline import NULL, Batch, RunContext, Tally, Task, job
@@ -53,7 +58,14 @@ def _ats() -> Any:
return ats
def _row(found: dict[str, Any], target: str) -> dict[str, Any]:
def _native(found: dict[str, Any], sub: Subscription) -> str:
"""The host's own id of a stored role: a board's roles are stored as
`host:slug:id`, a search's as `provider:id`."""
prefix = sub.target if sub.source == "ats" else sub.target.partition(":")[0]
return str(found["source_id"]).removeprefix(f"{prefix}:")
def _row(found: dict[str, Any], sub: Subscription) -> dict[str, Any]:
"""A stored role back as the row its reader listed: the stored `raw`
with the host's own id, the title, the link and the date."""
try:
@@ -65,29 +77,36 @@ def _row(found: dict[str, Any], target: str) -> dict[str, Any]:
posted: datetime | None = datetime.fromisoformat(str(found.get("published_at") or ""))
except ValueError:
posted = None
return raw | {"source_id": str(found["source_id"]).removeprefix(f"{target}:"),
return raw | {"source_id": _native(found, sub),
"title": found["title"], "url": found["url"], "description": "",
"posted": posted}
def _readable(store: Store) -> dict[int, Subscription]:
"""The fetchable boards and searches whose reader has `detail`, by
subscription id."""
ats = _ats()
return {sub.id: sub for source in ("ats", "search")
for sub in store.subscriptions(source=source, fetchable=True)
if hasattr(ats.reader_of(sub.target), "detail")}
def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
"""`(target, uids)` per board, best board first, within the budgets: the
roles `enrich_waiting` ranks, on fetchable boards whose reader has
`detail` and that are not failing, and due for a read."""
"""`(target, uids)` per board or search, best first, within the budgets:
the roles `enrich_waiting` ranks, on fetchable boards and searches whose
reader has `detail` and that are not failing, and due for a read."""
settings = config.enrich
ats = _ats()
boards = {sub.id: sub.target for sub in store.subscriptions(source="ats", fetchable=True)}
subs = _readable(store)
by_board: dict[str, list[dict[str, Any]]] = defaultdict(list)
for found in store.enrich_waiting(settings.top, settings.min_score):
target = boards.get(found["subscription_id"])
if target is None:
sub = subs.get(found["subscription_id"])
if sub is None or f"{sub.source}:failing:{sub.target}" in store.state:
continue
module = ats.readers.HOSTS.get(target.partition(":")[0])
if module is None or not hasattr(module, "detail") \
or f"ats:failing:{target}" in store.state:
continue
by_board[target].append(found)
due = {target: set(ats.detail_due(store.state, target, [f["source_id"] for f in found],
# The id the board's per-role state keeps: `host:slug:id`.
found["compound"] = f"{sub.target}:{_native(found, sub)}"
by_board[sub.target].append(found)
due = {target: set(ats.detail_due(store.state, target, [f["compound"] for f in found],
retry_days=settings.retry_days))
for target, found in by_board.items()}
# Best first; roles whose read failed after every other, so a page that
@@ -95,13 +114,13 @@ def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
failed = {target: ats._runner.id_set(store.state, ats.detail_keys(target)[1])
for target in by_board}
ranked = sorted((f for found in by_board.values() for f in found),
key=lambda f: (f["source_id"] in failed[boards[f["subscription_id"]]],
key=lambda f: (f["compound"] in failed[subs[f["subscription_id"]].target],
-float(f["score"] or 0), f["uid"]))
picked: dict[str, list[str]] = {}
total = 0
for found in ranked:
target = boards[found["subscription_id"]]
if found["source_id"] not in due[target]:
target = subs[found["subscription_id"]].target
if found["compound"] not in due[target]:
continue
if total >= settings.per_run:
break
@@ -114,7 +133,8 @@ def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
def batches(config: Config, store: Store) -> list[Batch]:
"""The finder: one batch per board, nothing while the stage is off."""
"""The finder: one batch per board or search, nothing while the stage is
off."""
if not stages.stage_on(config, store, "enrich"):
return []
return [Batch({"target": target, "uids": uids},
@@ -125,16 +145,16 @@ def batches(config: Config, store: Store) -> list[Batch]:
@job(KIND, "Reading roles on their own pages", resource="http")
def run_batch(config: Config, store: Store, *, target: str, uids: list[str],
context: RunContext = NULL) -> dict[str, Any]:
"""Read one board's roles in `uids` on their own pages, in that order,
and store what each page filled. `filled` is how many roles changed,
`texts` how many of them now have text."""
"""Read one board's or search's roles in `uids` on their own pages, in
that order, and store what each page filled. `filled` is how many roles
changed, `texts` how many of them now have text."""
ats = _ats()
subs = [s for s in store.subscriptions(source="ats", fetchable=True) if s.target == target]
subs = [s for s in _readable(store).values() if s.target == target]
if not subs or not uids:
return {"target": target, "filled": 0, "texts": 0}
sub = subs[0]
found = {f["uid"]: f for f in store.enrich_waiting(0, 0.0, uids)}
rows = [_row(found[uid], target) for uid in uids if uid in found]
rows = [_row(found[uid], sub) for uid in uids if uid in found]
context.check()
with httpx.Client() as client:
filled = ats.read_board_details(store.state, target, rows, client=client,
@@ -143,7 +163,7 @@ def run_batch(config: Config, store: Store, *, target: str, uids: list[str],
texts = 0
for row in filled:
item = ats.item_of(row, target, label)
item.source = "ats"
item.source = sub.source
store.upsert_role(item, sub)
texts += bool(row.get("description"))
log.info("enrich: %s: %d read, %d filled, %d with text", target, len(rows), len(filled),