Read role pages in an enrich stage after scoring, best roles first

The pull reads listings only. `enrich` (app/jobs/enrich.py) takes the open
roles without text that an enabled agent ranks ungated in its best
`enrich.top` at `enrich.min_score` or above, best score first, on boards
whose reader has `detail`, within `enrich.per_board` and `enrich.per_run`,
one batch per board. `plugins.ats.read_board_details` reads them and keeps
the per-board state; a read that gave no text is read again after
`enrich.retry_days` (finding 18). Filled roles are upserted, which drops
their vectors, and embed and score follow; `cascade.step` passes over both
when nothing was filled.

The pull cascade is pull, embed, score, enrich, embed, score, extract, score,
summarise. Saving an agent and changing a company's affinity queue the
`enrich` job (enrich, embed, score) behind their rescore; a vote does not.
The stage has a switch, `stage:enrich`, over `enrich.enabled`.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:53:29 +02:00
1 parent 3adafd8950
commit 77b45a4b0f
10 files changed
+956 -219

No files matched your search

+144 -29
View File
@@ -20,6 +20,10 @@ recorded as failing under `ats:failing:<target>`, and skipped. A board's
subscription `params` (`{"countries": ["DK"], "remote": true}`) are its
filter, narrowed to what its reader declares (`_runner.ask_for`).
A pull reads listings only. A host that states a role's text on the role's
own page (`text="detail"`) is read there by `app/jobs/enrich.py` after
scoring, best roles first, through `read_board_details`.
The limits below are read when a board is read, so a test that replaces one
on this module reaches the runner.
"""
@@ -31,7 +35,7 @@ import logging
import re
import sys
from collections.abc import Callable, Iterator
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
@@ -363,41 +367,152 @@ class ATSSource(Source):
# Matches `roles.source_id`, which `Store.mark_closed` compares.
return f"{host}:{slug}:{source_id}"
# A host with `detail` states a role's text on the role's own page,
# which the `enrich` stage reads after scoring (`read_board_details`),
# never the pull. Its rows go out without a description, so the text
# a role's page gave stays, and a role read on its own page goes out
# without the listing's place and remote as well, so what its page
# said stays too (see `_runner.read_details`).
rows = listing.rows
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
detail_sets: tuple[set[str], set[str]] | None = None
if hasattr(module, "detail"):
detailed, failed = (_runner.id_set(self.state, key) for key in detail_keys)
listed = {compound(i) for i in listing.ids} if listing.complete else None
rows, detailed, failed = _runner.read_details(
module, host, slug, get, rows, detailed, failed, listed=listed,
limits=_limits())
detail_sets = (detailed, failed)
detailed, failed = (_runner.id_set(self.state, key)
for key in detail_keys(target)[:2])
rows = [row | {"description": ""}
| (_KEPT_FROM_DETAIL if compound(row["source_id"]) in detailed else {})
for row in rows]
if listing.complete:
listed = {compound(i) for i in listing.ids}
detail_sets = (detailed & listed, failed & listed)
for row in rows:
description = row.get("description") or ""
yield Item(
source_id=compound(row["source_id"]),
location=row["url"],
title=row["title"],
excerpt=description[:300],
body=description,
published_at=row.get("posted"),
subscription=label,
kind="role",
raw=_runner.raw_of(row) | {
"company": label,
"function": classify(row.get("department") or "", row["title"]),
"board": host,
"slug": slug,
},
)
yield item_of(row, target, label)
# Stamped only once every row went out, so a board that blew up half
# way through is neither marked ok nor has a partial id list treated
# as the whole of it, and a pull stopped half way reads the rest of
# its details again.
# as the whole of it.
_runner.stamp(self.state, "ats", target, run_marker, listing,
[compound(i) for i in listing.ids])
if detail_sets is not None:
for key, ids in zip(detail_keys, detail_sets, strict=True):
self.state[key] = json.dumps(sorted(ids))
# A complete listing drops the ids it no longer lists.
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
if key in self.state:
self.state[key] = json.dumps(sorted(ids))
# ------------------------------------------------------ the per-role reads
#: What a pull withholds for a role already read on its own page: the
#: listing's place and remote, which would replace what the page stated.
_KEPT_FROM_DETAIL: dict[str, Any] = {"location": "", "places": [], "remote": None}
#: Days before a role whose own page gave no text is read again: an Oracle
#: requisition not yet published, an empty SuccessFactors block. A 404 or a
#: 410 counts the same, since the listing closes a role that is gone.
EMPTY_RETRY_DAYS = 7
def detail_keys(target: str) -> tuple[str, str, str]:
"""The state keys of one board's per-role reads: the ids read, the ids
whose read failed, and `{id: date}` of the reads that gave no text."""
return (f"ats:detailed:{target}", f"ats:detailfailed:{target}",
f"ats:detailempty:{target}")
def _dates(state: Any, key: str) -> dict[str, str]:
try:
value = json.loads(state.get(key) or "{}")
except (TypeError, ValueError):
return {}
return {str(k): str(v) for k, v in value.items()} if isinstance(value, dict) else {}
def detail_due(state: Any, target: str, ids: list[str], *, now: datetime | None = None,
retry_days: int = EMPTY_RETRY_DAYS) -> list[str]:
"""Of `ids` (compound, `host:slug:id`, roles without text), those to read
on their own page, in the order given: never read, failed, or read with
no text at least `retry_days` ago. A role read before its date was kept
counts as read that long ago."""
detailed = _runner.id_set(state, detail_keys(target)[0])
empty = _dates(state, detail_keys(target)[2])
cutoff = ((now or datetime.now(UTC)) - timedelta(days=retry_days)).isoformat(
timespec="seconds")
return [i for i in ids if i not in detailed or empty.get(i, "") <= cutoff]
def read_board_details(state: Any, target: str, rows: list[Row], *, client: httpx.Client,
cap: int | None = None, now: datetime | None = None,
retry_days: int = EMPTY_RETRY_DAYS) -> list[Row]:
"""Read `rows` of one board on their own pages, in the order given with
the roles whose read failed last, at most `cap` (`DETAIL_CAP`), and keep the board's per-role state
(`detail_keys`). The rows are the stored roles without text, each with
its host's own `source_id`. Returns the rows whose page filled a field,
each merged with what it filled.
A read that gave no text is kept with its date and read again once it is
`retry_days` old (`detail_due`). The sets are read again just before they
are written and changed by what this read did, so a pull that dropped ids
meanwhile keeps its drop."""
host, _, slug = target.partition(":")
module = readers.HOSTS[host]
keys = detail_keys(target)
now = now or datetime.now(UTC)
def compound(row: Row) -> str:
return f"{host}:{slug}:{row['source_id']}"
due = set(detail_due(state, target, [compound(r) for r in rows], now=now,
retry_days=retry_days))
detailed, failed = (_runner.id_set(state, key) for key in keys[:2])
# Roles whose read failed go after the rest, so they cannot starve them.
wanted = sorted((row for row in rows if compound(row) in due),
key=lambda row: compound(row) in failed)
wanted = wanted[:DETAIL_CAP if cap is None else cap]
if not wanted:
return []
again = {compound(row) for row in wanted} & detailed
limits = _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=len(wanted))
out, done, failing = _runner.read_details(
module, host, slug, _getter(client, host, slug), wanted, detailed - again, failed,
limits=limits)
read = done - (detailed - again)
filled: list[Row] = []
empty = _dates(state, keys[2])
for before, after in zip(wanted, out, strict=True):
key = compound(before)
if key in read:
if after.get("description"):
empty.pop(key, None)
else:
empty[key] = now.isoformat(timespec="seconds")
if after != before | {"description": ""}:
filled.append(after)
detailed_now = (_runner.id_set(state, keys[0]) - again) | read
failed_now = (_runner.id_set(state, keys[1]) | (failing - failed)) - read
state[keys[0]] = json.dumps(sorted(detailed_now))
state[keys[1]] = json.dumps(sorted(failed_now))
state[keys[2]] = json.dumps({k: v for k, v in sorted(empty.items()) if k in detailed_now})
return filled
def item_of(row: Row, target: str, label: str) -> Item:
"""One row of board `target` as the `Item` the store takes: its id
compound, `host:slug:id`, as `roles.source_id` has it."""
host, _, slug = target.partition(":")
description = row.get("description") or ""
return Item(
source_id=f"{host}:{slug}:{row['source_id']}",
location=row["url"],
title=row["title"],
excerpt=description[:300],
body=description,
published_at=row.get("posted"),
subscription=label,
kind="role",
raw=_runner.raw_of(row) | {
"company": label,
"function": classify(row.get("department") or "", row["title"]),
"board": host,
"slug": slug,
},
)