Read a search role's own page for its text in enrich and on the Fetch button when its reader has detail.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet 5.5 committed 2026-10-03 12:29:19 +02:00
1 parent 07e3bb4756
commit b148647ac3
5 files changed
+346 -40

No files matched your search

+51 -2
View File
@@ -5,6 +5,11 @@ link, keep the text as `article_text`. It is on demand, never in the pull,
because a fetch of a Jobindex page counts as a click in Jobindex's statistics
and one click for a posting somebody opened is fair.
A role of a search whose reader has `detail` (Jobbank) is read through that
`detail`, which reads the ad's own page and states its text, rather than
scraping the link; a failure of that read is an `Unreadable` with the
category its HTTP status or exception names.
A Jobindex role links to Jobindex's own `vis-job` page, which carries a short
ad and a "Se jobbet" button to the posting itself: the employer's site, The
Hub or Jobindex's `jobannonce`. That button's link is found, kept as
@@ -19,9 +24,14 @@ message the route answers with.
from __future__ import annotations
from datetime import datetime
from typing import Any
import httpx
from app.models import Role
from app.storage import Store
from desk.tools import scrape
from desk.tools import failures, scrape
from desk.tools.checks import Page
from plugins.readers import jobindex
@@ -113,6 +123,43 @@ def _jobindex_text(store: Store, role: Role) -> str:
raise Unreadable(failure or own.failure)
def _detail_target(role: Role) -> str:
"""The search target of a role whose reader has `detail`, else ''."""
from plugins import ats
target = str(role.raw.get("search") or "")
if role.source == "search" and hasattr(ats.reader_of(target), "detail"):
return target
return ""
def _detail_text(role: Role, target: str) -> str:
"""The text of a search role, read by its reader's `detail` on the ad's
own page as the pull's enrich stage reads it. Raises `Unreadable` when
the read fails or the page states no text."""
from plugins import ats
provider = target.partition(":")[0]
try:
posted: datetime | None = datetime.fromisoformat(role.published_at)
except ValueError:
posted = None
row: dict[str, Any] = role.raw | {
"source_id": role.source_id.removeprefix(f"{provider}:"), "title": role.title,
"url": role.original_url, "description": "", "posted": posted}
try:
with httpx.Client() as client:
found = ats.read_detail(client, target, row)
except httpx.HTTPStatusError as exc:
raise Unreadable(failures.of_status(exc.response.status_code)) from exc
except Exception as exc:
category = failures.of_exception(exc)
raise Unreadable("" if category == failures.UNKNOWN else category) from exc
if not (text := str(found.get("description") or "").strip()):
raise Unreadable("empty")
return text
def fetch_posting(store: Store, uid: str) -> Role:
"""Fetch the posting page once and keep it as plain text, for a role
whose description is a teaser. Synchronous: one request the person asked
@@ -124,7 +171,9 @@ def fetch_posting(store: Store, uid: str) -> Role:
raise AlreadyFetched("The posting text is already fetched.")
if not role.original_url:
raise NoLink("This role has no link to fetch the posting from.")
if _is_jobindex(role):
if target := _detail_target(role):
text = _detail_text(role, target)
elif _is_jobindex(role):
text = _jobindex_text(store, role)
else:
got = scrape.scrape(role.original_url)