Read a search role's own page for its text in enrich and on the Fetch button when its reader has detail.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
07e3bb4756
commit
b148647ac3
5 files changed
+346
-40
No files matched your search
+51
-2
@@ -5,6 +5,11 @@ link, keep the text as `article_text`. It is on demand, never in the pull,
|
||||
because a fetch of a Jobindex page counts as a click in Jobindex's statistics
|
||||
and one click for a posting somebody opened is fair.
|
||||
|
||||
A role of a search whose reader has `detail` (Jobbank) is read through that
|
||||
`detail`, which reads the ad's own page and states its text, rather than
|
||||
scraping the link; a failure of that read is an `Unreadable` with the
|
||||
category its HTTP status or exception names.
|
||||
|
||||
A Jobindex role links to Jobindex's own `vis-job` page, which carries a short
|
||||
ad and a "Se jobbet" button to the posting itself: the employer's site, The
|
||||
Hub or Jobindex's `jobannonce`. That button's link is found, kept as
|
||||
@@ -19,9 +24,14 @@ message the route answers with.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.models import Role
|
||||
from app.storage import Store
|
||||
from desk.tools import scrape
|
||||
from desk.tools import failures, scrape
|
||||
from desk.tools.checks import Page
|
||||
from plugins.readers import jobindex
|
||||
|
||||
@@ -113,6 +123,43 @@ def _jobindex_text(store: Store, role: Role) -> str:
|
||||
raise Unreadable(failure or own.failure)
|
||||
|
||||
|
||||
def _detail_target(role: Role) -> str:
|
||||
"""The search target of a role whose reader has `detail`, else ''."""
|
||||
from plugins import ats
|
||||
|
||||
target = str(role.raw.get("search") or "")
|
||||
if role.source == "search" and hasattr(ats.reader_of(target), "detail"):
|
||||
return target
|
||||
return ""
|
||||
|
||||
|
||||
def _detail_text(role: Role, target: str) -> str:
|
||||
"""The text of a search role, read by its reader's `detail` on the ad's
|
||||
own page as the pull's enrich stage reads it. Raises `Unreadable` when
|
||||
the read fails or the page states no text."""
|
||||
from plugins import ats
|
||||
|
||||
provider = target.partition(":")[0]
|
||||
try:
|
||||
posted: datetime | None = datetime.fromisoformat(role.published_at)
|
||||
except ValueError:
|
||||
posted = None
|
||||
row: dict[str, Any] = role.raw | {
|
||||
"source_id": role.source_id.removeprefix(f"{provider}:"), "title": role.title,
|
||||
"url": role.original_url, "description": "", "posted": posted}
|
||||
try:
|
||||
with httpx.Client() as client:
|
||||
found = ats.read_detail(client, target, row)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
raise Unreadable(failures.of_status(exc.response.status_code)) from exc
|
||||
except Exception as exc:
|
||||
category = failures.of_exception(exc)
|
||||
raise Unreadable("" if category == failures.UNKNOWN else category) from exc
|
||||
if not (text := str(found.get("description") or "").strip()):
|
||||
raise Unreadable("empty")
|
||||
return text
|
||||
|
||||
|
||||
def fetch_posting(store: Store, uid: str) -> Role:
|
||||
"""Fetch the posting page once and keep it as plain text, for a role
|
||||
whose description is a teaser. Synchronous: one request the person asked
|
||||
@@ -124,7 +171,9 @@ def fetch_posting(store: Store, uid: str) -> Role:
|
||||
raise AlreadyFetched("The posting text is already fetched.")
|
||||
if not role.original_url:
|
||||
raise NoLink("This role has no link to fetch the posting from.")
|
||||
if _is_jobindex(role):
|
||||
if target := _detail_target(role):
|
||||
text = _detail_text(role, target)
|
||||
elif _is_jobindex(role):
|
||||
text = _jobindex_text(store, role)
|
||||
else:
|
||||
got = scrape.scrape(role.original_url)
|
||||
|
||||
Reference in new issue
Block a user