Read a search role's own page for its text in enrich and on the Fetch button when its reader has detail.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet 5.5 committed 2026-10-03 12:29:19 +02:00
1 parent 07e3bb4756
commit b148647ac3
5 files changed
+346 -40

No files matched your search

+51 -31
View File
@@ -2,18 +2,22 @@
Some hosts list a role without its text and state it on the role's own page
only (`text="detail"`: Workday, SuccessFactors, Oracle, Phenom and the rest
with a `detail`). The pull reads listings only; this stage reads those pages
after scoring, so the calls go where the ranking is:
with a `detail`, and a search provider such as Jobbank). The pull reads
listings only; this stage reads those pages after scoring, so the calls go
where the ranking is. A saved search whose reader has `detail` is treated as
a board under its target `provider:name`: the same bar, budgets and state
keys, and its failing record is `search:failing:<target>`.
1. `enrich_waiting` hands over the open roles without text that an enabled
agent with a prompt ranks ungated among its best `enrich.top` and at
`enrich.min_score` or above, best first by the highest such score. A role
below that bar for every agent gets no call.
2. They are taken in that order, those whose read failed last, on boards whose reader has `detail`, at most
`enrich.per_board` from one board and `enrich.per_run` in all, leaving
out the roles already read, unless their read failed or gave no text
`enrich.retry_days` ago (`plugins.ats.detail_due`).
3. One batch per board, on the `http` pool, reads them
2. They are taken in that order, those whose read failed last, on boards and
searches whose reader has `detail`, at most `enrich.per_board` from one
board and `enrich.per_run` in all, leaving out the roles already read,
unless their read failed or gave no text `enrich.retry_days` ago
(`plugins.ats.detail_due`).
3. One batch per board or search, on the `http` pool, reads them
(`plugins.ats.read_board_details`, which keeps `ats:detailed:<target>`,
`ats:detailfailed:<target>` and `ats:detailempty:<target>`) and stores
what each page filled. `Store.upsert_role` drops the vector of a role
@@ -37,6 +41,7 @@ import httpx
from app.config import Config
from app.jobs import stages
from app.models import Subscription
from app.storage import Store
from pipeline import NULL, Batch, RunContext, Tally, Task, job
@@ -53,7 +58,14 @@ def _ats() -> Any:
return ats
def _row(found: dict[str, Any], target: str) -> dict[str, Any]:
def _native(found: dict[str, Any], sub: Subscription) -> str:
"""The host's own id of a stored role: a board's roles are stored as
`host:slug:id`, a search's as `provider:id`."""
prefix = sub.target if sub.source == "ats" else sub.target.partition(":")[0]
return str(found["source_id"]).removeprefix(f"{prefix}:")
def _row(found: dict[str, Any], sub: Subscription) -> dict[str, Any]:
"""A stored role back as the row its reader listed: the stored `raw`
with the host's own id, the title, the link and the date."""
try:
@@ -65,29 +77,36 @@ def _row(found: dict[str, Any], target: str) -> dict[str, Any]:
posted: datetime | None = datetime.fromisoformat(str(found.get("published_at") or ""))
except ValueError:
posted = None
return raw | {"source_id": str(found["source_id"]).removeprefix(f"{target}:"),
return raw | {"source_id": _native(found, sub),
"title": found["title"], "url": found["url"], "description": "",
"posted": posted}
def _readable(store: Store) -> dict[int, Subscription]:
"""The fetchable boards and searches whose reader has `detail`, by
subscription id."""
ats = _ats()
return {sub.id: sub for source in ("ats", "search")
for sub in store.subscriptions(source=source, fetchable=True)
if hasattr(ats.reader_of(sub.target), "detail")}
def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
"""`(target, uids)` per board, best board first, within the budgets: the
roles `enrich_waiting` ranks, on fetchable boards whose reader has
`detail` and that are not failing, and due for a read."""
"""`(target, uids)` per board or search, best first, within the budgets:
the roles `enrich_waiting` ranks, on fetchable boards and searches whose
reader has `detail` and that are not failing, and due for a read."""
settings = config.enrich
ats = _ats()
boards = {sub.id: sub.target for sub in store.subscriptions(source="ats", fetchable=True)}
subs = _readable(store)
by_board: dict[str, list[dict[str, Any]]] = defaultdict(list)
for found in store.enrich_waiting(settings.top, settings.min_score):
target = boards.get(found["subscription_id"])
if target is None:
sub = subs.get(found["subscription_id"])
if sub is None or f"{sub.source}:failing:{sub.target}" in store.state:
continue
module = ats.readers.HOSTS.get(target.partition(":")[0])
if module is None or not hasattr(module, "detail") \
or f"ats:failing:{target}" in store.state:
continue
by_board[target].append(found)
due = {target: set(ats.detail_due(store.state, target, [f["source_id"] for f in found],
# The id the board's per-role state keeps: `host:slug:id`.
found["compound"] = f"{sub.target}:{_native(found, sub)}"
by_board[sub.target].append(found)
due = {target: set(ats.detail_due(store.state, target, [f["compound"] for f in found],
retry_days=settings.retry_days))
for target, found in by_board.items()}
# Best first; roles whose read failed after every other, so a page that
@@ -95,13 +114,13 @@ def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
failed = {target: ats._runner.id_set(store.state, ats.detail_keys(target)[1])
for target in by_board}
ranked = sorted((f for found in by_board.values() for f in found),
key=lambda f: (f["source_id"] in failed[boards[f["subscription_id"]]],
key=lambda f: (f["compound"] in failed[subs[f["subscription_id"]].target],
-float(f["score"] or 0), f["uid"]))
picked: dict[str, list[str]] = {}
total = 0
for found in ranked:
target = boards[found["subscription_id"]]
if found["source_id"] not in due[target]:
target = subs[found["subscription_id"]].target
if found["compound"] not in due[target]:
continue
if total >= settings.per_run:
break
@@ -114,7 +133,8 @@ def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
def batches(config: Config, store: Store) -> list[Batch]:
"""The finder: one batch per board, nothing while the stage is off."""
"""The finder: one batch per board or search, nothing while the stage is
off."""
if not stages.stage_on(config, store, "enrich"):
return []
return [Batch({"target": target, "uids": uids},
@@ -125,16 +145,16 @@ def batches(config: Config, store: Store) -> list[Batch]:
@job(KIND, "Reading roles on their own pages", resource="http")
def run_batch(config: Config, store: Store, *, target: str, uids: list[str],
context: RunContext = NULL) -> dict[str, Any]:
"""Read one board's roles in `uids` on their own pages, in that order,
and store what each page filled. `filled` is how many roles changed,
`texts` how many of them now have text."""
"""Read one board's or search's roles in `uids` on their own pages, in
that order, and store what each page filled. `filled` is how many roles
changed, `texts` how many of them now have text."""
ats = _ats()
subs = [s for s in store.subscriptions(source="ats", fetchable=True) if s.target == target]
subs = [s for s in _readable(store).values() if s.target == target]
if not subs or not uids:
return {"target": target, "filled": 0, "texts": 0}
sub = subs[0]
found = {f["uid"]: f for f in store.enrich_waiting(0, 0.0, uids)}
rows = [_row(found[uid], target) for uid in uids if uid in found]
rows = [_row(found[uid], sub) for uid in uids if uid in found]
context.check()
with httpx.Client() as client:
filled = ats.read_board_details(store.state, target, rows, client=client,
@@ -143,7 +163,7 @@ def run_batch(config: Config, store: Store, *, target: str, uids: list[str],
texts = 0
for row in filled:
item = ats.item_of(row, target, label)
item.source = "ats"
item.source = sub.source
store.upsert_role(item, sub)
texts += bool(row.get("description"))
log.info("enrich: %s: %d read, %d filled, %d with text", target, len(rows), len(filled),
+51 -2
View File
@@ -5,6 +5,11 @@ link, keep the text as `article_text`. It is on demand, never in the pull,
because a fetch of a Jobindex page counts as a click in Jobindex's statistics
and one click for a posting somebody opened is fair.
A role of a search whose reader has `detail` (Jobbank) is read through that
`detail`, which reads the ad's own page and states its text, rather than
scraping the link; a failure of that read is an `Unreadable` with the
category its HTTP status or exception names.
A Jobindex role links to Jobindex's own `vis-job` page, which carries a short
ad and a "Se jobbet" button to the posting itself: the employer's site, The
Hub or Jobindex's `jobannonce`. That button's link is found, kept as
@@ -19,9 +24,14 @@ message the route answers with.
from __future__ import annotations
from datetime import datetime
from typing import Any
import httpx
from app.models import Role
from app.storage import Store
from desk.tools import scrape
from desk.tools import failures, scrape
from desk.tools.checks import Page
from plugins.readers import jobindex
@@ -113,6 +123,43 @@ def _jobindex_text(store: Store, role: Role) -> str:
raise Unreadable(failure or own.failure)
def _detail_target(role: Role) -> str:
"""The search target of a role whose reader has `detail`, else ''."""
from plugins import ats
target = str(role.raw.get("search") or "")
if role.source == "search" and hasattr(ats.reader_of(target), "detail"):
return target
return ""
def _detail_text(role: Role, target: str) -> str:
"""The text of a search role, read by its reader's `detail` on the ad's
own page as the pull's enrich stage reads it. Raises `Unreadable` when
the read fails or the page states no text."""
from plugins import ats
provider = target.partition(":")[0]
try:
posted: datetime | None = datetime.fromisoformat(role.published_at)
except ValueError:
posted = None
row: dict[str, Any] = role.raw | {
"source_id": role.source_id.removeprefix(f"{provider}:"), "title": role.title,
"url": role.original_url, "description": "", "posted": posted}
try:
with httpx.Client() as client:
found = ats.read_detail(client, target, row)
except httpx.HTTPStatusError as exc:
raise Unreadable(failures.of_status(exc.response.status_code)) from exc
except Exception as exc:
category = failures.of_exception(exc)
raise Unreadable("" if category == failures.UNKNOWN else category) from exc
if not (text := str(found.get("description") or "").strip()):
raise Unreadable("empty")
return text
def fetch_posting(store: Store, uid: str) -> Role:
"""Fetch the posting page once and keep it as plain text, for a role
whose description is a teaser. Synchronous: one request the person asked
@@ -124,7 +171,9 @@ def fetch_posting(store: Store, uid: str) -> Role:
raise AlreadyFetched("The posting text is already fetched.")
if not role.original_url:
raise NoLink("This role has no link to fetch the posting from.")
if _is_jobindex(role):
if target := _detail_target(role):
text = _detail_text(role, target)
elif _is_jobindex(role):
text = _jobindex_text(store, role)
else:
got = scrape.scrape(role.original_url)