"""A role's posting page, fetched once as plain text on demand. `fetch_posting` is what `POST /api/roles/{uid}/text` does: scrape the role's link, keep the text as `article_text`. It is on demand, never in the pull, because a fetch of a Jobindex page counts as a click in Jobindex's statistics and one click for a posting somebody opened is fair. A Jobindex role links to Jobindex's own `vis-job` page, which carries a short ad and a "Se jobbet" button to the posting itself: the employer's site, The Hub or Jobindex's `jobannonce`. That button's link is found, kept as `posting_url` and scraped for the text; the `vis-job` page's own text is the fallback when there is no link or it gives nothing. A stored `posting_url` is followed directly the next time, so the `vis-job` page is not fetched twice for the same text. Everything that stops the fetch is a `PostingError` with the HTTP status and message the route answers with. """ from __future__ import annotations from app.models import Role from app.storage import Store from desk.tools import scrape from desk.tools.checks import Page from plugins.readers import jobindex #: What the read page toasts when a posting page gave nothing readable, by #: `desk.tools.failures` category. UNREADABLE = { "timeout": "The posting page took too long to answer. Try again in a minute.", "unreachable": "The posting page could not be reached.", "rate_limited": "The posting site is refusing requests for now. Try again later.", "server_error": "The posting site answered with an error. Try again later.", "not_found": "The posting page is gone, so the role has probably closed.", "denied": "The posting site refused to show the page.", "consent_wall": "The posting page showed a consent banner instead of the posting.", "js_required": "The posting page only renders in a browser. Open it with the link instead.", "empty": "The posting page had no readable text on it.", } NOTHING_READABLE = "Nothing readable came back from the posting page." class PostingError(Exception): """A posting text that was not fetched. `status` is the HTTP status the route answers with, `str(error)` the message.""" status = 422 class UnknownRole(PostingError): status = 404 class AlreadyFetched(PostingError): status = 409 class NoLink(PostingError): """The role has no link to fetch the posting from.""" class Unreadable(PostingError): """The page answered with nothing readable; `failure` is its `desk.tools.failures` category, '' when none was named.""" def __init__(self, failure: str = "") -> None: super().__init__(UNREADABLE.get(failure, NOTHING_READABLE)) self.failure = failure def _plain(got: scrape.Scraped) -> str: return got.content.plain_text.strip() if got.content is not None else "" def _keeping_html(url: str) -> tuple[scrape.Scraped, str]: """`scrape.scrape` over one request, and the HTML that request answered with, '' when none did.""" pages: list[str] = [] def fetch(target: str) -> Page | None: page = scrape.HTTP.fetch(target) if page is not None: pages.append(page.html) return page got = scrape.scrape(url, {"http": scrape.Getter("http", fetch)}) return got, pages[0] if pages else "" def _is_jobindex(role: Role) -> bool: return role.raw.get("provider") == "jobindex" def _jobindex_text(store: Store, role: Role) -> str: """The text of a Jobindex role: the page its "Se jobbet" button leads to, else the `vis-job` page's own. Raises `Unreadable` when neither reads.""" own: scrape.Scraped | None = None link, failure = role.posting_url, "" if not link: own, html = _keeping_html(role.original_url) link = jobindex.posting_link(html) if link: store.set_posting_url(role.uid, link) if link and link != role.original_url: got = scrape.scrape(link) if text := _plain(got): return text failure = got.failure own = own or scrape.scrape(role.original_url) if text := _plain(own): return text raise Unreadable(failure or own.failure) def fetch_posting(store: Store, uid: str) -> Role: """Fetch the posting page once and keep it as plain text, for a role whose description is a teaser. Synchronous: one request the person asked for. Raises `UnknownRole`, `AlreadyFetched`, `NoLink` or `Unreadable`.""" role = store.role(uid) if role is None: raise UnknownRole(f"no such role: {uid}") if role.article_text: raise AlreadyFetched("The posting text is already fetched.") if not role.original_url: raise NoLink("This role has no link to fetch the posting from.") if _is_jobindex(role): text = _jobindex_text(store, role) else: got = scrape.scrape(role.original_url) if not (text := _plain(got)): raise Unreadable(got.failure) updated = store.set_article_text(uid, text) if updated is None: raise UnknownRole(f"no such role: {uid}") return updated