Read a search role's own page for its text in enrich and on the Fetch button when its reader has detail.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet 5.5 committed 2026-10-03 12:29:19 +02:00
1 parent 07e3bb4756
commit b148647ac3
5 files changed
+346 -40

No files matched your search

+46 -7
View File
@@ -38,7 +38,9 @@ that read once it is complete, rather than by a second request.
A pull reads listings only. A host that states a role's text on the role's
own page (`text="detail"`) is read there by `app/jobs/enrich.py` after
scoring, best roles first, through `read_board_details`.
scoring, best roles first, through `read_board_details`. A search provider
whose reader has `detail` (Jobbank) is read the same way, under its search
target `provider:name`; `read_detail` is the one read the Fetch button makes.
The limits below are read when a board is read, so a test that replaces one
on this module reaches the runner.
@@ -53,6 +55,7 @@ import sys
from collections.abc import Callable, Iterator
from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import ModuleType
from typing import Any
import httpx
@@ -114,11 +117,18 @@ def _limits(stale_days: int | None = None) -> _runner.Limits:
stale_days=stale_days)
def reader_of(target: str) -> ModuleType | None:
"""The reader of a board (`host:slug`) or of a search (`provider:name`),
None for a name that is neither. The two never share a name."""
host = target.partition(":")[0]
return readers.HOSTS.get(host) or readers.PROVIDERS.get(host)
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
"""The hosts a request or a redirect may land on for one board: what its
host module's `allowed` says. Anything else is refused, at the first hop
and at every redirect."""
module = readers.HOSTS.get(host)
"""The hosts a request or a redirect may land on for one board or search:
what its module's `allowed` says. Anything else is refused, at the first
hop and at every redirect."""
module = reader_of(host)
return module.allowed(slug) if module is not None else lambda h: False
@@ -540,7 +550,9 @@ def read_board_details(state: Any, target: str, rows: list[Row], *, client: http
are written and changed by what this read did, so a pull that dropped ids
meanwhile keeps its drop."""
host, _, slug = target.partition(":")
module = readers.HOSTS[host]
module = reader_of(target)
if module is None or not hasattr(module, "detail"):
raise ValueError(f"{target!r} has no reader that reads a role on its own page")
keys = detail_keys(target)
now = now or datetime.now(UTC)
@@ -582,12 +594,39 @@ def read_board_details(state: Any, target: str, rows: list[Row], *, client: http
return filled
def read_detail(client: httpx.Client, target: str, row: Row) -> dict[str, Any]:
"""One role of board or search `target` read on its own page: what the
reader's `detail` returns, for the Fetch button. No state is kept and no
field is filtered. Raises what `detail` raises, and ValueError for a
target whose reader has none."""
host, _, slug = target.partition(":")
module = reader_of(target)
if module is None or not hasattr(module, "detail"):
raise ValueError(f"{target!r} has no reader that reads a role on its own page")
return module.detail(_getter(client, host, slug), slug, row)
def item_of(row: Row, target: str, label: str) -> Item:
"""One row of board `target` as the `Item` the store takes: its id
compound, `host:slug:id`, as `roles.source_id` has it. A row without a
date is dated now, as `Item` itself would date it."""
date is dated now, as `Item` itself would date it. A search's row is
stored as `plugins/search.py` stores it: `provider:id`, the employer from
`company` and `label` the search's own."""
host, _, slug = target.partition(":")
description = row.get("description") or ""
if host not in readers.HOSTS:
return Item(
source_id=f"{host}:{row['source_id']}",
location=row["url"],
title=row["title"],
excerpt=description[:300],
body=description,
published_at=row.get("posted") or datetime.now(UTC),
subscription=label,
kind="role",
raw=_runner.raw_of(row) | {"company": row.get("company") or "", "provider": host,
"search": target},
)
return Item(
source_id=f"{host}:{slug}:{row['source_id']}",
location=row["url"],