"""A role's posting page, fetched once as plain text on demand. `fetch_posting` is what `POST /api/roles/{uid}/text` does: scrape the role's link, keep the text as `article_text`. It is on demand, never in the pull, because a fetch of a Jobindex page counts as a click in Jobindex's statistics and one click for a posting somebody opened is fair. A role of a search whose reader has `detail` (Jobbank) is read through that `detail`, which reads the ad's own page and states its text, rather than scraping the link; a failure of that read is an `Unreadable` with the category its HTTP status or exception names. A Jobindex role links to Jobindex's own `vis-job` page, which carries a short ad and a "Se jobbet" button to the posting itself: the employer's site, The Hub or Jobindex's `jobannonce`. That button's link is found, kept as `posting_url` and scraped for the text; the `vis-job` page's own text is the fallback when there is no link or it gives nothing. A stored `posting_url` is followed directly the next time, so the `vis-job` page is not fetched twice for the same text. Everything that stops the fetch is a `PostingError` with the HTTP status and message the route answers with. """ from __future__ import annotations from datetime import datetime from typing import Any import httpx from app.models import Role from app.storage import Store from desk.tools import failures, scrape from desk.tools.checks import Page from plugins.readers import jobindex #: What the read page toasts when a posting page gave nothing readable, by #: `desk.tools.failures` category. UNREADABLE = { "timeout": "The posting page took too long to answer. Try again in a minute.", "unreachable": "The posting page could not be reached.", "rate_limited": "The posting site is refusing requests for now. Try again later.", "server_error": "The posting site answered with an error. Try again later.", "not_found": "The posting page is gone, so the role has probably closed.", "denied": "The posting site refused to show the page.", "consent_wall": "The posting page showed a consent banner instead of the posting.", "js_required": "The posting page only renders in a browser. Open it with the link instead.", "empty": "The posting page had no readable text on it.", } NOTHING_READABLE = "Nothing readable came back from the posting page." class PostingError(Exception): """A posting text that was not fetched. `status` is the HTTP status the route answers with, `str(error)` the message.""" status = 422 class UnknownRole(PostingError): status = 404 class AlreadyFetched(PostingError): status = 409 class NoLink(PostingError): """The role has no link to fetch the posting from.""" class Unreadable(PostingError): """The page answered with nothing readable; `failure` is its `desk.tools.failures` category, '' when none was named.""" def __init__(self, failure: str = "") -> None: super().__init__(UNREADABLE.get(failure, NOTHING_READABLE)) self.failure = failure def _plain(got: scrape.Scraped) -> str: return got.content.plain_text.strip() if got.content is not None else "" def _keeping_html(url: str) -> tuple[scrape.Scraped, str]: """`scrape.scrape` over one request, and the HTML that request answered with, '' when none did.""" pages: list[str] = [] def fetch(target: str) -> Page | None: page = scrape.HTTP.fetch(target) if page is not None: pages.append(page.html) return page got = scrape.scrape(url, {"http": scrape.Getter("http", fetch)}) return got, pages[0] if pages else "" def _is_jobindex(role: Role) -> bool: return role.raw.get("provider") == "jobindex" def _jobindex_text(store: Store, role: Role) -> str: """The text of a Jobindex role: the page its "Se jobbet" button leads to, else the `vis-job` page's own. Raises `Unreadable` when neither reads.""" own: scrape.Scraped | None = None link, failure = role.posting_url, "" if not link: own, html = _keeping_html(role.original_url) link = jobindex.posting_link(html) if link: store.set_posting_url(role.uid, link) if link and link != role.original_url: got = scrape.scrape(link) if text := _plain(got): return text failure = got.failure own = own or scrape.scrape(role.original_url) if text := _plain(own): return text raise Unreadable(failure or own.failure) def _detail_target(role: Role) -> str: """The search target of a role whose reader has `detail`, else ''.""" from plugins import ats target = str(role.raw.get("search") or "") if role.source == "search" and hasattr(ats.reader_of(target), "detail"): return target return "" def _detail_text(role: Role, target: str) -> str: """The text of a search role, read by its reader's `detail` on the ad's own page as the pull's fetch text stage reads it. Raises `Unreadable` when the read fails or the page states no text.""" from plugins import ats provider = target.partition(":")[0] try: posted: datetime | None = datetime.fromisoformat(role.published_at) except ValueError: posted = None row: dict[str, Any] = role.raw | { "source_id": role.source_id.removeprefix(f"{provider}:"), "title": role.title, "url": role.original_url, "description": "", "posted": posted} try: with httpx.Client() as client: found = ats.read_detail(client, target, row) except httpx.HTTPStatusError as exc: raise Unreadable(failures.of_status(exc.response.status_code)) from exc except Exception as exc: category = failures.of_exception(exc) raise Unreadable("" if category == failures.UNKNOWN else category) from exc if not (text := str(found.get("description") or "").strip()): raise Unreadable("empty") return text def fetch_posting(store: Store, uid: str) -> Role: """Fetch the posting page once and keep it as plain text, for a role whose description is a teaser. Synchronous: one request the person asked for. Raises `UnknownRole`, `AlreadyFetched`, `NoLink` or `Unreadable`.""" role = store.role(uid) if role is None: raise UnknownRole(f"no such role: {uid}") if role.article_text: raise AlreadyFetched("The posting text is already fetched.") if not role.original_url: raise NoLink("This role has no link to fetch the posting from.") if target := _detail_target(role): text = _detail_text(role, target) elif _is_jobindex(role): text = _jobindex_text(store, role) else: got = scrape.scrape(role.original_url) if not (text := _plain(got)): raise Unreadable(got.failure) updated = store.set_article_text(uid, text) if updated is None: raise UnknownRole(f"no such role: {uid}") return updated