The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
241 lines
10 KiB
Python
241 lines
10 KiB
Python
"""Get an article out of a URL: fetch, check, convert, and fetch another way
|
|
when the check says so.
|
|
|
|
Three parts, each replaceable on its own:
|
|
|
|
- a **getter** fetches a page: `HTTP`, `BROWSER` (Playwright, with the
|
|
`browser` extra), or one a plugin registered for its own URLs, such as a
|
|
post through its network's API
|
|
- `desk.tools.checks` **checks** the page and answers with a `Verdict`: the
|
|
failure category, what matched, and which getters to try next
|
|
- `desk.tools.article` **converts** HTML that passed into the structured
|
|
article, preserving code blocks and tables
|
|
|
|
got = scrape(url) # standard: HTTP, check, convert
|
|
got = scrape(url, getters=ADVANCED) # a failed check may try another getter
|
|
got = scrape(url, getters=advanced(cache, plugin_getters))
|
|
# and a consent the browser accepted is sent by
|
|
# HTTP next time, and a plugin's URLs go to it
|
|
|
|
**A getter that claims a URL goes first.** A plugin that knows how to read its
|
|
own network's addresses registers a getter with `claims=True`: when `handles`
|
|
matches, it runs before `http`. What it brings back is converted like any
|
|
page. A status it answers with ends the attempt, because the network's own API
|
|
is the authority on its posts; no answer at all, or a page that fails the
|
|
check, and the ordinary path runs as if it had not been asked.
|
|
|
|
Standard and advanced run the same loop. The only difference is which getters
|
|
are available to a verdict: with `http` alone a bot check ends as
|
|
`js_required` on the row, with `browser` available the verdict's `next` sends
|
|
the same URL there, and a page that passes is converted exactly as an HTTP one
|
|
would have been. A getter named by a verdict and not available is skipped, so
|
|
the checks can say where a page should go before that getter exists.
|
|
|
|
Nothing but a verdict moves to another getter. A page that passed the check
|
|
and converted to nothing is `empty` and ends there, because an extractor that
|
|
finds no article in real HTML finds none in the same HTML from a browser.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import logging
|
|
from collections.abc import Callable, Iterable, Mapping, MutableMapping
|
|
from configparser import ConfigParser
|
|
from dataclasses import dataclass
|
|
|
|
from desk.tools import article, browser, failures
|
|
from desk.tools.article import ArticleContent
|
|
from desk.tools.checks import Page, Verdict, check
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Getter:
|
|
"""One way of fetching a page. `fetch` returns None for a request that
|
|
never completed and raises for anything it can name. `handles` is for a
|
|
getter that only knows some hosts, such as a platform's API; one that does
|
|
not handle a URL is skipped without an attempt."""
|
|
|
|
name: str
|
|
fetch: Callable[[str], Page | None]
|
|
handles: Callable[[str], bool] = lambda url: True
|
|
#: Run before `http` for every URL `handles` matches, rather than only
|
|
#: when a check names it. For a getter that owns a kind of address.
|
|
claims: bool = False
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Attempt:
|
|
getter: str
|
|
failure: str = ""
|
|
#: The signature or rule that failed the check, empty on success.
|
|
matched: str = ""
|
|
status: int = 0
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Scraped:
|
|
"""The article, or the failure of the last getter tried, and every try."""
|
|
|
|
content: ArticleContent | None
|
|
failure: str
|
|
getter: str
|
|
attempts: tuple[Attempt, ...]
|
|
#: The server's own Retry-After, in seconds, when one was given.
|
|
retry_after: float | None = None
|
|
|
|
|
|
def _http(url: str, cookie: str = "") -> Page | None:
|
|
# Looked up per call, so a test that stubs the downloader stubs this.
|
|
import trafilatura
|
|
|
|
if cookie:
|
|
answer = trafilatura.fetch_response(url, decode=True, with_headers=True,
|
|
config=_with_cookie(cookie))
|
|
else:
|
|
answer = trafilatura.fetch_response(url, decode=True, with_headers=True)
|
|
if answer is None:
|
|
return None
|
|
html = answer.html or ""
|
|
data = getattr(answer, "data", None)
|
|
if isinstance(data, bytes) and data[:4] == ZSTD_MAGIC:
|
|
html = _unzstd(data)
|
|
return Page(status=answer.status, html=html,
|
|
url=getattr(answer, "url", "") or url,
|
|
headers=answer.headers or {})
|
|
|
|
|
|
#: The first bytes of a zstd frame.
|
|
ZSTD_MAGIC = b"\x28\xb5\x2f\xfd"
|
|
|
|
|
|
def _unzstd(data: bytes) -> str:
|
|
"""A zstd body as the HTML trafilatura would have decoded.
|
|
|
|
Python 3.14 has `compression.zstd`, so urllib3 offers zstd, and Cloudflare
|
|
hosts such as opendatascience.com answer with it. Streaming the body,
|
|
urllib3 hands trafilatura the compressed bytes; trafilatura decompresses
|
|
zstd only with the `zstandard` package, which refuses a frame that does not
|
|
state its size, and Cloudflare's frames do not. Undecoded, the page is
|
|
binary that passes the check and converts to `empty`.
|
|
"""
|
|
from compression import zstd
|
|
|
|
from trafilatura.utils import decode_file
|
|
|
|
return decode_file(zstd.decompress(data))
|
|
|
|
|
|
def _with_cookie(cookie: str) -> ConfigParser:
|
|
"""trafilatura's own config with a `Cookie` header in it.
|
|
|
|
Its downloader takes no headers, only a config. A config that is not the
|
|
default replaces the default headers, so the user agent is set again to
|
|
the one it would have sent. `%` is escaped for `ConfigParser`'s
|
|
interpolation; consent strings are full of it.
|
|
"""
|
|
from trafilatura.downloads import USER_AGENT
|
|
from trafilatura.settings import DEFAULT_CONFIG
|
|
|
|
config = copy.deepcopy(DEFAULT_CONFIG)
|
|
config.set("DEFAULT", "USER_AGENTS", USER_AGENT)
|
|
config.set("DEFAULT", "COOKIE", cookie.replace("%", "%%"))
|
|
return config
|
|
|
|
|
|
def _browser(url: str, cache: MutableMapping[str, str] | None = None) -> Page | None:
|
|
got = browser.render(url)
|
|
if cache is not None and got.consent:
|
|
browser.remember(cache, got.url, got.cookies)
|
|
return Page(status=got.status, html=got.html, url=got.url, headers=got.headers)
|
|
|
|
|
|
#: One plain request through trafilatura's downloader: its headers, its size
|
|
#: cap, and the status and headers handed back rather than thrown away.
|
|
HTTP = Getter("http", _http)
|
|
#: Headless Chromium. Accepts a consent dialog it recognises; see
|
|
#: `desk.tools.browser`.
|
|
BROWSER = Getter("browser", _browser)
|
|
STANDARD: Mapping[str, Getter] = {"http": HTTP}
|
|
#: The getters this module has itself. `browser` only with the `browser`
|
|
#: extra installed; a verdict naming a getter that is not here skips it. The
|
|
#: rest come from plugins, through `advanced`.
|
|
ADVANCED: Mapping[str, Getter] = {
|
|
"http": HTTP, **({"browser": BROWSER} if browser.installed() else {})}
|
|
|
|
|
|
def advanced(cache: MutableMapping[str, str],
|
|
registered: Iterable[Getter] = ()) -> Mapping[str, Getter]:
|
|
"""`ADVANCED` and the getters plugins `registered`, sharing one cookie jar.
|
|
|
|
The browser keeps the cookies of a consent it accepted in `cache`, per
|
|
host, and HTTP sends them. So a gate costs one render per host until its
|
|
cookies expire, and every other article from that host is a plain request.
|
|
A registered getter cannot replace `http` or `browser`.
|
|
"""
|
|
getters = {getter.name: getter for getter in registered}
|
|
getters.update(ADVANCED)
|
|
getters["http"] = Getter("http", lambda url: _http(url, browser.cookie_header(cache, url)))
|
|
if "browser" in getters:
|
|
getters["browser"] = Getter("browser", lambda url: _browser(url, cache))
|
|
return getters
|
|
|
|
|
|
def scrape(url: str, getters: Mapping[str, Getter] = STANDARD,
|
|
first: str = "http") -> Scraped:
|
|
"""A claiming getter, then `first`, then whatever each failed check names."""
|
|
attempts: list[Attempt] = []
|
|
hint: float | None = None
|
|
for claimant in getters.values():
|
|
if not (claimant.claims and claimant.handles(url)):
|
|
continue
|
|
content, verdict, page = _one(claimant, url)
|
|
attempts.append(Attempt(claimant.name, "" if content else verdict.category,
|
|
"" if content else verdict.name,
|
|
page.status if page else 0))
|
|
if content is not None:
|
|
return Scraped(content, "", claimant.name, tuple(attempts), hint)
|
|
if page is not None and page.status >= 400:
|
|
return Scraped(None, verdict.category, claimant.name, tuple(attempts),
|
|
failures.hint(page.headers))
|
|
log.debug("%s: %s via %s (%s); the ordinary path runs", url,
|
|
verdict.category, claimant.name, verdict.name)
|
|
plan = [first]
|
|
while plan:
|
|
getter = getters.get(plan.pop(0))
|
|
if getter is None or not getter.handles(url) or any(
|
|
a.getter == getter.name for a in attempts):
|
|
continue
|
|
content, verdict, page = _one(getter, url)
|
|
if content is not None:
|
|
attempts.append(Attempt(getter.name, status=page.status if page else 0))
|
|
return Scraped(content, "", getter.name, tuple(attempts), hint)
|
|
attempts.append(Attempt(getter.name, verdict.category, verdict.name,
|
|
page.status if page else 0))
|
|
if page is not None and page.status >= 400:
|
|
hint = hint or failures.hint(page.headers)
|
|
log.debug("%s: %s via %s (%s)", url, verdict.category, getter.name, verdict.name)
|
|
plan = list(verdict.next)
|
|
# The row gets the last answer a page gave. A fallback that brought back
|
|
# nothing (a browser that would not start, an API that did not answer)
|
|
# says nothing about the page, and must not replace what the check found.
|
|
answered = [a for a in attempts if a.status] or attempts
|
|
last = answered[-1] if answered else Attempt("", "unknown")
|
|
return Scraped(None, last.failure, last.getter, tuple(attempts), hint)
|
|
|
|
|
|
def _one(getter: Getter, url: str) -> tuple[ArticleContent | None, Verdict, Page | None]:
|
|
try:
|
|
page = getter.fetch(url)
|
|
except Exception as exc:
|
|
return None, Verdict(failures.of_exception(exc), type(exc).__name__), None
|
|
if verdict := check(page):
|
|
return None, verdict, page
|
|
assert page is not None # check() refuses a missing page
|
|
content = article.extract_article(page.html, url)
|
|
if content is None or not content.plain_text.strip():
|
|
return None, Verdict("empty", "no_article"), page
|
|
return content, Verdict("", ""), page
|