Files
jobboard/desk/tools/scrape.py
T
WolfandClaude Opus 5.5 248ea1636d Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:17 +02:00

241 lines
10 KiB
Python

"""Get an article out of a URL: fetch, check, convert, and fetch another way
when the check says so.
Three parts, each replaceable on its own:
- a **getter** fetches a page: `HTTP`, `BROWSER` (Playwright, with the
`browser` extra), or one a plugin registered for its own URLs, such as a
post through its network's API
- `desk.tools.checks` **checks** the page and answers with a `Verdict`: the
failure category, what matched, and which getters to try next
- `desk.tools.article` **converts** HTML that passed into the structured
article, preserving code blocks and tables
got = scrape(url) # standard: HTTP, check, convert
got = scrape(url, getters=ADVANCED) # a failed check may try another getter
got = scrape(url, getters=advanced(cache, plugin_getters))
# and a consent the browser accepted is sent by
# HTTP next time, and a plugin's URLs go to it
**A getter that claims a URL goes first.** A plugin that knows how to read its
own network's addresses registers a getter with `claims=True`: when `handles`
matches, it runs before `http`. What it brings back is converted like any
page. A status it answers with ends the attempt, because the network's own API
is the authority on its posts; no answer at all, or a page that fails the
check, and the ordinary path runs as if it had not been asked.
Standard and advanced run the same loop. The only difference is which getters
are available to a verdict: with `http` alone a bot check ends as
`js_required` on the row, with `browser` available the verdict's `next` sends
the same URL there, and a page that passes is converted exactly as an HTTP one
would have been. A getter named by a verdict and not available is skipped, so
the checks can say where a page should go before that getter exists.
Nothing but a verdict moves to another getter. A page that passed the check
and converted to nothing is `empty` and ends there, because an extractor that
finds no article in real HTML finds none in the same HTML from a browser.
"""
from __future__ import annotations
import copy
import logging
from collections.abc import Callable, Iterable, Mapping, MutableMapping
from configparser import ConfigParser
from dataclasses import dataclass
from desk.tools import article, browser, failures
from desk.tools.article import ArticleContent
from desk.tools.checks import Page, Verdict, check
log = logging.getLogger(__name__)
@dataclass(frozen=True)
class Getter:
"""One way of fetching a page. `fetch` returns None for a request that
never completed and raises for anything it can name. `handles` is for a
getter that only knows some hosts, such as a platform's API; one that does
not handle a URL is skipped without an attempt."""
name: str
fetch: Callable[[str], Page | None]
handles: Callable[[str], bool] = lambda url: True
#: Run before `http` for every URL `handles` matches, rather than only
#: when a check names it. For a getter that owns a kind of address.
claims: bool = False
@dataclass(frozen=True)
class Attempt:
getter: str
failure: str = ""
#: The signature or rule that failed the check, empty on success.
matched: str = ""
status: int = 0
@dataclass(frozen=True)
class Scraped:
"""The article, or the failure of the last getter tried, and every try."""
content: ArticleContent | None
failure: str
getter: str
attempts: tuple[Attempt, ...]
#: The server's own Retry-After, in seconds, when one was given.
retry_after: float | None = None
def _http(url: str, cookie: str = "") -> Page | None:
# Looked up per call, so a test that stubs the downloader stubs this.
import trafilatura
if cookie:
answer = trafilatura.fetch_response(url, decode=True, with_headers=True,
config=_with_cookie(cookie))
else:
answer = trafilatura.fetch_response(url, decode=True, with_headers=True)
if answer is None:
return None
html = answer.html or ""
data = getattr(answer, "data", None)
if isinstance(data, bytes) and data[:4] == ZSTD_MAGIC:
html = _unzstd(data)
return Page(status=answer.status, html=html,
url=getattr(answer, "url", "") or url,
headers=answer.headers or {})
#: The first bytes of a zstd frame.
ZSTD_MAGIC = b"\x28\xb5\x2f\xfd"
def _unzstd(data: bytes) -> str:
"""A zstd body as the HTML trafilatura would have decoded.
Python 3.14 has `compression.zstd`, so urllib3 offers zstd, and Cloudflare
hosts such as opendatascience.com answer with it. Streaming the body,
urllib3 hands trafilatura the compressed bytes; trafilatura decompresses
zstd only with the `zstandard` package, which refuses a frame that does not
state its size, and Cloudflare's frames do not. Undecoded, the page is
binary that passes the check and converts to `empty`.
"""
from compression import zstd
from trafilatura.utils import decode_file
return decode_file(zstd.decompress(data))
def _with_cookie(cookie: str) -> ConfigParser:
"""trafilatura's own config with a `Cookie` header in it.
Its downloader takes no headers, only a config. A config that is not the
default replaces the default headers, so the user agent is set again to
the one it would have sent. `%` is escaped for `ConfigParser`'s
interpolation; consent strings are full of it.
"""
from trafilatura.downloads import USER_AGENT
from trafilatura.settings import DEFAULT_CONFIG
config = copy.deepcopy(DEFAULT_CONFIG)
config.set("DEFAULT", "USER_AGENTS", USER_AGENT)
config.set("DEFAULT", "COOKIE", cookie.replace("%", "%%"))
return config
def _browser(url: str, cache: MutableMapping[str, str] | None = None) -> Page | None:
got = browser.render(url)
if cache is not None and got.consent:
browser.remember(cache, got.url, got.cookies)
return Page(status=got.status, html=got.html, url=got.url, headers=got.headers)
#: One plain request through trafilatura's downloader: its headers, its size
#: cap, and the status and headers handed back rather than thrown away.
HTTP = Getter("http", _http)
#: Headless Chromium. Accepts a consent dialog it recognises; see
#: `desk.tools.browser`.
BROWSER = Getter("browser", _browser)
STANDARD: Mapping[str, Getter] = {"http": HTTP}
#: The getters this module has itself. `browser` only with the `browser`
#: extra installed; a verdict naming a getter that is not here skips it. The
#: rest come from plugins, through `advanced`.
ADVANCED: Mapping[str, Getter] = {
"http": HTTP, **({"browser": BROWSER} if browser.installed() else {})}
def advanced(cache: MutableMapping[str, str],
registered: Iterable[Getter] = ()) -> Mapping[str, Getter]:
"""`ADVANCED` and the getters plugins `registered`, sharing one cookie jar.
The browser keeps the cookies of a consent it accepted in `cache`, per
host, and HTTP sends them. So a gate costs one render per host until its
cookies expire, and every other article from that host is a plain request.
A registered getter cannot replace `http` or `browser`.
"""
getters = {getter.name: getter for getter in registered}
getters.update(ADVANCED)
getters["http"] = Getter("http", lambda url: _http(url, browser.cookie_header(cache, url)))
if "browser" in getters:
getters["browser"] = Getter("browser", lambda url: _browser(url, cache))
return getters
def scrape(url: str, getters: Mapping[str, Getter] = STANDARD,
first: str = "http") -> Scraped:
"""A claiming getter, then `first`, then whatever each failed check names."""
attempts: list[Attempt] = []
hint: float | None = None
for claimant in getters.values():
if not (claimant.claims and claimant.handles(url)):
continue
content, verdict, page = _one(claimant, url)
attempts.append(Attempt(claimant.name, "" if content else verdict.category,
"" if content else verdict.name,
page.status if page else 0))
if content is not None:
return Scraped(content, "", claimant.name, tuple(attempts), hint)
if page is not None and page.status >= 400:
return Scraped(None, verdict.category, claimant.name, tuple(attempts),
failures.hint(page.headers))
log.debug("%s: %s via %s (%s); the ordinary path runs", url,
verdict.category, claimant.name, verdict.name)
plan = [first]
while plan:
getter = getters.get(plan.pop(0))
if getter is None or not getter.handles(url) or any(
a.getter == getter.name for a in attempts):
continue
content, verdict, page = _one(getter, url)
if content is not None:
attempts.append(Attempt(getter.name, status=page.status if page else 0))
return Scraped(content, "", getter.name, tuple(attempts), hint)
attempts.append(Attempt(getter.name, verdict.category, verdict.name,
page.status if page else 0))
if page is not None and page.status >= 400:
hint = hint or failures.hint(page.headers)
log.debug("%s: %s via %s (%s)", url, verdict.category, getter.name, verdict.name)
plan = list(verdict.next)
# The row gets the last answer a page gave. A fallback that brought back
# nothing (a browser that would not start, an API that did not answer)
# says nothing about the page, and must not replace what the check found.
answered = [a for a in attempts if a.status] or attempts
last = answered[-1] if answered else Attempt("", "unknown")
return Scraped(None, last.failure, last.getter, tuple(attempts), hint)
def _one(getter: Getter, url: str) -> tuple[ArticleContent | None, Verdict, Page | None]:
try:
page = getter.fetch(url)
except Exception as exc:
return None, Verdict(failures.of_exception(exc), type(exc).__name__), None
if verdict := check(page):
return None, verdict, page
assert page is not None # check() refuses a missing page
content = article.extract_article(page.html, url)
if content is None or not content.plain_text.strip():
return None, Verdict("empty", "no_article"), page
return content, Verdict("", ""), page