Merge text for search readers with detail (wp-src-searchdetail)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
04f4c09a21
5 files changed
+346
-40
No files matched your search
+46
-7
@@ -38,7 +38,9 @@ that read once it is complete, rather than by a second request.
|
||||
|
||||
A pull reads listings only. A host that states a role's text on the role's
|
||||
own page (`text="detail"`) is read there by `app/jobs/enrich.py` after
|
||||
scoring, best roles first, through `read_board_details`.
|
||||
scoring, best roles first, through `read_board_details`. A search provider
|
||||
whose reader has `detail` (Jobbank) is read the same way, under its search
|
||||
target `provider:name`; `read_detail` is the one read the Fetch button makes.
|
||||
|
||||
The limits below are read when a board is read, so a test that replaces one
|
||||
on this module reaches the runner.
|
||||
@@ -53,6 +55,7 @@ import sys
|
||||
from collections.abc import Callable, Iterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
@@ -114,11 +117,18 @@ def _limits(stale_days: int | None = None) -> _runner.Limits:
|
||||
stale_days=stale_days)
|
||||
|
||||
|
||||
def reader_of(target: str) -> ModuleType | None:
|
||||
"""The reader of a board (`host:slug`) or of a search (`provider:name`),
|
||||
None for a name that is neither. The two never share a name."""
|
||||
host = target.partition(":")[0]
|
||||
return readers.HOSTS.get(host) or readers.PROVIDERS.get(host)
|
||||
|
||||
|
||||
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
||||
"""The hosts a request or a redirect may land on for one board: what its
|
||||
host module's `allowed` says. Anything else is refused, at the first hop
|
||||
and at every redirect."""
|
||||
module = readers.HOSTS.get(host)
|
||||
"""The hosts a request or a redirect may land on for one board or search:
|
||||
what its module's `allowed` says. Anything else is refused, at the first
|
||||
hop and at every redirect."""
|
||||
module = reader_of(host)
|
||||
return module.allowed(slug) if module is not None else lambda h: False
|
||||
|
||||
|
||||
@@ -540,7 +550,9 @@ def read_board_details(state: Any, target: str, rows: list[Row], *, client: http
|
||||
are written and changed by what this read did, so a pull that dropped ids
|
||||
meanwhile keeps its drop."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = readers.HOSTS[host]
|
||||
module = reader_of(target)
|
||||
if module is None or not hasattr(module, "detail"):
|
||||
raise ValueError(f"{target!r} has no reader that reads a role on its own page")
|
||||
keys = detail_keys(target)
|
||||
now = now or datetime.now(UTC)
|
||||
|
||||
@@ -582,12 +594,39 @@ def read_board_details(state: Any, target: str, rows: list[Row], *, client: http
|
||||
return filled
|
||||
|
||||
|
||||
def read_detail(client: httpx.Client, target: str, row: Row) -> dict[str, Any]:
|
||||
"""One role of board or search `target` read on its own page: what the
|
||||
reader's `detail` returns, for the Fetch button. No state is kept and no
|
||||
field is filtered. Raises what `detail` raises, and ValueError for a
|
||||
target whose reader has none."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = reader_of(target)
|
||||
if module is None or not hasattr(module, "detail"):
|
||||
raise ValueError(f"{target!r} has no reader that reads a role on its own page")
|
||||
return module.detail(_getter(client, host, slug), slug, row)
|
||||
|
||||
|
||||
def item_of(row: Row, target: str, label: str) -> Item:
|
||||
"""One row of board `target` as the `Item` the store takes: its id
|
||||
compound, `host:slug:id`, as `roles.source_id` has it. A row without a
|
||||
date is dated now, as `Item` itself would date it."""
|
||||
date is dated now, as `Item` itself would date it. A search's row is
|
||||
stored as `plugins/search.py` stores it: `provider:id`, the employer from
|
||||
`company` and `label` the search's own."""
|
||||
host, _, slug = target.partition(":")
|
||||
description = row.get("description") or ""
|
||||
if host not in readers.HOSTS:
|
||||
return Item(
|
||||
source_id=f"{host}:{row['source_id']}",
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=description[:300],
|
||||
body=description,
|
||||
published_at=row.get("posted") or datetime.now(UTC),
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw=_runner.raw_of(row) | {"company": row.get("company") or "", "provider": host,
|
||||
"search": target},
|
||||
)
|
||||
return Item(
|
||||
source_id=f"{host}:{slug}:{row['source_id']}",
|
||||
location=row["url"],
|
||||
|
||||
Reference in new issue
Block a user