186 lines
6.8 KiB
Python
186 lines
6.8 KiB
Python
"""A role's posting page, fetched once as plain text on demand.
|
|
|
|
`fetch_posting` is what `POST /api/roles/{uid}/text` does: scrape the role's
|
|
link, keep the text as `article_text`. It is on demand, never in the pull,
|
|
because a fetch of a Jobindex page counts as a click in Jobindex's statistics
|
|
and one click for a posting somebody opened is fair.
|
|
|
|
A role of a search whose reader has `detail` (Jobbank) is read through that
|
|
`detail`, which reads the ad's own page and states its text, rather than
|
|
scraping the link; a failure of that read is an `Unreadable` with the
|
|
category its HTTP status or exception names.
|
|
|
|
A Jobindex role links to Jobindex's own `vis-job` page, which carries a short
|
|
ad and a "Se jobbet" button to the posting itself: the employer's site, The
|
|
Hub or Jobindex's `jobannonce`. That button's link is found, kept as
|
|
`posting_url` and scraped for the text; the `vis-job` page's own text is the
|
|
fallback when there is no link or it gives nothing. A stored `posting_url` is
|
|
followed directly the next time, so the `vis-job` page is not fetched twice
|
|
for the same text.
|
|
|
|
Everything that stops the fetch is a `PostingError` with the HTTP status and
|
|
message the route answers with.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from app.models import Role
|
|
from app.storage import Store
|
|
from desk.tools import failures, scrape
|
|
from desk.tools.checks import Page
|
|
from plugins.readers import jobindex
|
|
|
|
#: What the read page toasts when a posting page gave nothing readable, by
|
|
#: `desk.tools.failures` category.
|
|
UNREADABLE = {
|
|
"timeout": "The posting page took too long to answer. Try again in a minute.",
|
|
"unreachable": "The posting page could not be reached.",
|
|
"rate_limited": "The posting site is refusing requests for now. Try again later.",
|
|
"server_error": "The posting site answered with an error. Try again later.",
|
|
"not_found": "The posting page is gone, so the role has probably closed.",
|
|
"denied": "The posting site refused to show the page.",
|
|
"consent_wall": "The posting page showed a consent banner instead of the posting.",
|
|
"js_required": "The posting page only renders in a browser. Open it with the link instead.",
|
|
"empty": "The posting page had no readable text on it.",
|
|
}
|
|
NOTHING_READABLE = "Nothing readable came back from the posting page."
|
|
|
|
|
|
class PostingError(Exception):
|
|
"""A posting text that was not fetched. `status` is the HTTP status the
|
|
route answers with, `str(error)` the message."""
|
|
|
|
status = 422
|
|
|
|
|
|
class UnknownRole(PostingError):
|
|
status = 404
|
|
|
|
|
|
class AlreadyFetched(PostingError):
|
|
status = 409
|
|
|
|
|
|
class NoLink(PostingError):
|
|
"""The role has no link to fetch the posting from."""
|
|
|
|
|
|
class Unreadable(PostingError):
|
|
"""The page answered with nothing readable; `failure` is its
|
|
`desk.tools.failures` category, '' when none was named."""
|
|
|
|
def __init__(self, failure: str = "") -> None:
|
|
super().__init__(UNREADABLE.get(failure, NOTHING_READABLE))
|
|
self.failure = failure
|
|
|
|
|
|
def _plain(got: scrape.Scraped) -> str:
|
|
return got.content.plain_text.strip() if got.content is not None else ""
|
|
|
|
|
|
def _keeping_html(url: str) -> tuple[scrape.Scraped, str]:
|
|
"""`scrape.scrape` over one request, and the HTML that request answered
|
|
with, '' when none did."""
|
|
pages: list[str] = []
|
|
|
|
def fetch(target: str) -> Page | None:
|
|
page = scrape.HTTP.fetch(target)
|
|
if page is not None:
|
|
pages.append(page.html)
|
|
return page
|
|
|
|
got = scrape.scrape(url, {"http": scrape.Getter("http", fetch)})
|
|
return got, pages[0] if pages else ""
|
|
|
|
|
|
def _is_jobindex(role: Role) -> bool:
|
|
return role.raw.get("provider") == "jobindex"
|
|
|
|
|
|
def _jobindex_text(store: Store, role: Role) -> str:
|
|
"""The text of a Jobindex role: the page its "Se jobbet" button leads to,
|
|
else the `vis-job` page's own. Raises `Unreadable` when neither reads."""
|
|
own: scrape.Scraped | None = None
|
|
link, failure = role.posting_url, ""
|
|
if not link:
|
|
own, html = _keeping_html(role.original_url)
|
|
link = jobindex.posting_link(html)
|
|
if link:
|
|
store.set_posting_url(role.uid, link)
|
|
if link and link != role.original_url:
|
|
got = scrape.scrape(link)
|
|
if text := _plain(got):
|
|
return text
|
|
failure = got.failure
|
|
own = own or scrape.scrape(role.original_url)
|
|
if text := _plain(own):
|
|
return text
|
|
raise Unreadable(failure or own.failure)
|
|
|
|
|
|
def _detail_target(role: Role) -> str:
|
|
"""The search target of a role whose reader has `detail`, else ''."""
|
|
from plugins import ats
|
|
|
|
target = str(role.raw.get("search") or "")
|
|
if role.source == "search" and hasattr(ats.reader_of(target), "detail"):
|
|
return target
|
|
return ""
|
|
|
|
|
|
def _detail_text(role: Role, target: str) -> str:
|
|
"""The text of a search role, read by its reader's `detail` on the ad's
|
|
own page as the pull's fetch text stage reads it. Raises `Unreadable` when
|
|
the read fails or the page states no text."""
|
|
from plugins import ats
|
|
|
|
provider = target.partition(":")[0]
|
|
try:
|
|
posted: datetime | None = datetime.fromisoformat(role.published_at)
|
|
except ValueError:
|
|
posted = None
|
|
row: dict[str, Any] = role.raw | {
|
|
"source_id": role.source_id.removeprefix(f"{provider}:"), "title": role.title,
|
|
"url": role.original_url, "description": "", "posted": posted}
|
|
try:
|
|
with httpx.Client() as client:
|
|
found = ats.read_detail(client, target, row)
|
|
except httpx.HTTPStatusError as exc:
|
|
raise Unreadable(failures.of_status(exc.response.status_code)) from exc
|
|
except Exception as exc:
|
|
category = failures.of_exception(exc)
|
|
raise Unreadable("" if category == failures.UNKNOWN else category) from exc
|
|
if not (text := str(found.get("description") or "").strip()):
|
|
raise Unreadable("empty")
|
|
return text
|
|
|
|
|
|
def fetch_posting(store: Store, uid: str) -> Role:
|
|
"""Fetch the posting page once and keep it as plain text, for a role
|
|
whose description is a teaser. Synchronous: one request the person asked
|
|
for. Raises `UnknownRole`, `AlreadyFetched`, `NoLink` or `Unreadable`."""
|
|
role = store.role(uid)
|
|
if role is None:
|
|
raise UnknownRole(f"no such role: {uid}")
|
|
if role.article_text:
|
|
raise AlreadyFetched("The posting text is already fetched.")
|
|
if not role.original_url:
|
|
raise NoLink("This role has no link to fetch the posting from.")
|
|
if target := _detail_target(role):
|
|
text = _detail_text(role, target)
|
|
elif _is_jobindex(role):
|
|
text = _jobindex_text(store, role)
|
|
else:
|
|
got = scrape.scrape(role.original_url)
|
|
if not (text := _plain(got)):
|
|
raise Unreadable(got.failure)
|
|
updated = store.set_article_text(uid, text)
|
|
if updated is None:
|
|
raise UnknownRole(f"no such role: {uid}")
|
|
return updated
|