"""Get an article out of a URL: fetch, check, convert, and fetch another way when the check says so. Three parts, each replaceable on its own: - a **getter** fetches a page: `HTTP`, `BROWSER` (Playwright, with the `browser` extra), or one a plugin registered for its own URLs, such as a post through its network's API - `desk.tools.checks` **checks** the page and answers with a `Verdict`: the failure category, what matched, and which getters to try next - `desk.tools.article` **converts** HTML that passed into the structured article, preserving code blocks and tables got = scrape(url) # standard: HTTP, check, convert got = scrape(url, getters=ADVANCED) # a failed check may try another getter got = scrape(url, getters=advanced(cache, plugin_getters)) # and a consent the browser accepted is sent by # HTTP next time, and a plugin's URLs go to it **A getter that claims a URL goes first.** A plugin that knows how to read its own network's addresses registers a getter with `claims=True`: when `handles` matches, it runs before `http`. What it brings back is converted like any page. A status it answers with ends the attempt, because the network's own API is the authority on its posts; no answer at all, or a page that fails the check, and the ordinary path runs as if it had not been asked. Standard and advanced run the same loop. The only difference is which getters are available to a verdict: with `http` alone a bot check ends as `js_required` on the row, with `browser` available the verdict's `next` sends the same URL there, and a page that passes is converted exactly as an HTTP one would have been. A getter named by a verdict and not available is skipped, so the checks can say where a page should go before that getter exists. Nothing but a verdict moves to another getter. A page that passed the check and converted to nothing is `empty` and ends there, because an extractor that finds no article in real HTML finds none in the same HTML from a browser. """ from __future__ import annotations import copy import logging from collections.abc import Callable, Iterable, Mapping, MutableMapping from configparser import ConfigParser from dataclasses import dataclass from desk.tools import article, browser, failures from desk.tools.article import ArticleContent from desk.tools.checks import Page, Verdict, check log = logging.getLogger(__name__) @dataclass(frozen=True) class Getter: """One way of fetching a page. `fetch` returns None for a request that never completed and raises for anything it can name. `handles` is for a getter that only knows some hosts, such as a platform's API; one that does not handle a URL is skipped without an attempt.""" name: str fetch: Callable[[str], Page | None] handles: Callable[[str], bool] = lambda url: True #: Run before `http` for every URL `handles` matches, rather than only #: when a check names it. For a getter that owns a kind of address. claims: bool = False @dataclass(frozen=True) class Attempt: getter: str failure: str = "" #: The signature or rule that failed the check, empty on success. matched: str = "" status: int = 0 @dataclass(frozen=True) class Scraped: """The article, or the failure of the last getter tried, and every try.""" content: ArticleContent | None failure: str getter: str attempts: tuple[Attempt, ...] #: The server's own Retry-After, in seconds, when one was given. retry_after: float | None = None def _http(url: str, cookie: str = "") -> Page | None: # Looked up per call, so a test that stubs the downloader stubs this. import trafilatura if cookie: answer = trafilatura.fetch_response(url, decode=True, with_headers=True, config=_with_cookie(cookie)) else: answer = trafilatura.fetch_response(url, decode=True, with_headers=True) if answer is None: return None html = answer.html or "" data = getattr(answer, "data", None) if isinstance(data, bytes) and data[:4] == ZSTD_MAGIC: html = _unzstd(data) return Page(status=answer.status, html=html, url=getattr(answer, "url", "") or url, headers=answer.headers or {}) #: The first bytes of a zstd frame. ZSTD_MAGIC = b"\x28\xb5\x2f\xfd" def _unzstd(data: bytes) -> str: """A zstd body as the HTML trafilatura would have decoded. Python 3.14 has `compression.zstd`, so urllib3 offers zstd, and Cloudflare hosts such as opendatascience.com answer with it. Streaming the body, urllib3 hands trafilatura the compressed bytes; trafilatura decompresses zstd only with the `zstandard` package, which refuses a frame that does not state its size, and Cloudflare's frames do not. Undecoded, the page is binary that passes the check and converts to `empty`. """ from compression import zstd from trafilatura.utils import decode_file return decode_file(zstd.decompress(data)) def _with_cookie(cookie: str) -> ConfigParser: """trafilatura's own config with a `Cookie` header in it. Its downloader takes no headers, only a config. A config that is not the default replaces the default headers, so the user agent is set again to the one it would have sent. `%` is escaped for `ConfigParser`'s interpolation; consent strings are full of it. """ from trafilatura.downloads import USER_AGENT from trafilatura.settings import DEFAULT_CONFIG config = copy.deepcopy(DEFAULT_CONFIG) config.set("DEFAULT", "USER_AGENTS", USER_AGENT) config.set("DEFAULT", "COOKIE", cookie.replace("%", "%%")) return config def _browser(url: str, cache: MutableMapping[str, str] | None = None) -> Page | None: got = browser.render(url) if cache is not None and got.consent: browser.remember(cache, got.url, got.cookies) return Page(status=got.status, html=got.html, url=got.url, headers=got.headers) #: One plain request through trafilatura's downloader: its headers, its size #: cap, and the status and headers handed back rather than thrown away. HTTP = Getter("http", _http) #: Headless Chromium. Accepts a consent dialog it recognises; see #: `desk.tools.browser`. BROWSER = Getter("browser", _browser) STANDARD: Mapping[str, Getter] = {"http": HTTP} #: The getters this module has itself. `browser` only with the `browser` #: extra installed; a verdict naming a getter that is not here skips it. The #: rest come from plugins, through `advanced`. ADVANCED: Mapping[str, Getter] = { "http": HTTP, **({"browser": BROWSER} if browser.installed() else {})} def advanced(cache: MutableMapping[str, str], registered: Iterable[Getter] = ()) -> Mapping[str, Getter]: """`ADVANCED` and the getters plugins `registered`, sharing one cookie jar. The browser keeps the cookies of a consent it accepted in `cache`, per host, and HTTP sends them. So a gate costs one render per host until its cookies expire, and every other article from that host is a plain request. A registered getter cannot replace `http` or `browser`. """ getters = {getter.name: getter for getter in registered} getters.update(ADVANCED) getters["http"] = Getter("http", lambda url: _http(url, browser.cookie_header(cache, url))) if "browser" in getters: getters["browser"] = Getter("browser", lambda url: _browser(url, cache)) return getters def scrape(url: str, getters: Mapping[str, Getter] = STANDARD, first: str = "http") -> Scraped: """A claiming getter, then `first`, then whatever each failed check names.""" attempts: list[Attempt] = [] hint: float | None = None for claimant in getters.values(): if not (claimant.claims and claimant.handles(url)): continue content, verdict, page = _one(claimant, url) attempts.append(Attempt(claimant.name, "" if content else verdict.category, "" if content else verdict.name, page.status if page else 0)) if content is not None: return Scraped(content, "", claimant.name, tuple(attempts), hint) if page is not None and page.status >= 400: return Scraped(None, verdict.category, claimant.name, tuple(attempts), failures.hint(page.headers)) log.debug("%s: %s via %s (%s); the ordinary path runs", url, verdict.category, claimant.name, verdict.name) plan = [first] while plan: getter = getters.get(plan.pop(0)) if getter is None or not getter.handles(url) or any( a.getter == getter.name for a in attempts): continue content, verdict, page = _one(getter, url) if content is not None: attempts.append(Attempt(getter.name, status=page.status if page else 0)) return Scraped(content, "", getter.name, tuple(attempts), hint) attempts.append(Attempt(getter.name, verdict.category, verdict.name, page.status if page else 0)) if page is not None and page.status >= 400: hint = hint or failures.hint(page.headers) log.debug("%s: %s via %s (%s)", url, verdict.category, getter.name, verdict.name) plan = list(verdict.next) # The row gets the last answer a page gave. A fallback that brought back # nothing (a browser that would not start, an API that did not answer) # says nothing about the page, and must not replace what the check found. answered = [a for a in attempts if a.status] or attempts last = answered[-1] if answered else Attempt("", "unknown") return Scraped(None, last.failure, last.getter, tuple(attempts), hint) def _one(getter: Getter, url: str) -> tuple[ArticleContent | None, Verdict, Page | None]: try: page = getter.fetch(url) except Exception as exc: return None, Verdict(failures.of_exception(exc), type(exc).__name__), None if verdict := check(page): return None, verdict, page assert page is not None # check() refuses a missing page content = article.extract_article(page.html, url) if content is None or not content.plain_text.strip(): return None, Verdict("empty", "no_article"), page return content, Verdict("", ""), page