Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d666d0937
commit
248ea1636d
36 files changed
+8393
No files matched your search
@@ -0,0 +1,240 @@
|
||||
"""Get an article out of a URL: fetch, check, convert, and fetch another way
|
||||
when the check says so.
|
||||
|
||||
Three parts, each replaceable on its own:
|
||||
|
||||
- a **getter** fetches a page: `HTTP`, `BROWSER` (Playwright, with the
|
||||
`browser` extra), or one a plugin registered for its own URLs, such as a
|
||||
post through its network's API
|
||||
- `desk.tools.checks` **checks** the page and answers with a `Verdict`: the
|
||||
failure category, what matched, and which getters to try next
|
||||
- `desk.tools.article` **converts** HTML that passed into the structured
|
||||
article, preserving code blocks and tables
|
||||
|
||||
got = scrape(url) # standard: HTTP, check, convert
|
||||
got = scrape(url, getters=ADVANCED) # a failed check may try another getter
|
||||
got = scrape(url, getters=advanced(cache, plugin_getters))
|
||||
# and a consent the browser accepted is sent by
|
||||
# HTTP next time, and a plugin's URLs go to it
|
||||
|
||||
**A getter that claims a URL goes first.** A plugin that knows how to read its
|
||||
own network's addresses registers a getter with `claims=True`: when `handles`
|
||||
matches, it runs before `http`. What it brings back is converted like any
|
||||
page. A status it answers with ends the attempt, because the network's own API
|
||||
is the authority on its posts; no answer at all, or a page that fails the
|
||||
check, and the ordinary path runs as if it had not been asked.
|
||||
|
||||
Standard and advanced run the same loop. The only difference is which getters
|
||||
are available to a verdict: with `http` alone a bot check ends as
|
||||
`js_required` on the row, with `browser` available the verdict's `next` sends
|
||||
the same URL there, and a page that passes is converted exactly as an HTTP one
|
||||
would have been. A getter named by a verdict and not available is skipped, so
|
||||
the checks can say where a page should go before that getter exists.
|
||||
|
||||
Nothing but a verdict moves to another getter. A page that passed the check
|
||||
and converted to nothing is `empty` and ends there, because an extractor that
|
||||
finds no article in real HTML finds none in the same HTML from a browser.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import logging
|
||||
from collections.abc import Callable, Iterable, Mapping, MutableMapping
|
||||
from configparser import ConfigParser
|
||||
from dataclasses import dataclass
|
||||
|
||||
from desk.tools import article, browser, failures
|
||||
from desk.tools.article import ArticleContent
|
||||
from desk.tools.checks import Page, Verdict, check
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Getter:
|
||||
"""One way of fetching a page. `fetch` returns None for a request that
|
||||
never completed and raises for anything it can name. `handles` is for a
|
||||
getter that only knows some hosts, such as a platform's API; one that does
|
||||
not handle a URL is skipped without an attempt."""
|
||||
|
||||
name: str
|
||||
fetch: Callable[[str], Page | None]
|
||||
handles: Callable[[str], bool] = lambda url: True
|
||||
#: Run before `http` for every URL `handles` matches, rather than only
|
||||
#: when a check names it. For a getter that owns a kind of address.
|
||||
claims: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Attempt:
|
||||
getter: str
|
||||
failure: str = ""
|
||||
#: The signature or rule that failed the check, empty on success.
|
||||
matched: str = ""
|
||||
status: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Scraped:
|
||||
"""The article, or the failure of the last getter tried, and every try."""
|
||||
|
||||
content: ArticleContent | None
|
||||
failure: str
|
||||
getter: str
|
||||
attempts: tuple[Attempt, ...]
|
||||
#: The server's own Retry-After, in seconds, when one was given.
|
||||
retry_after: float | None = None
|
||||
|
||||
|
||||
def _http(url: str, cookie: str = "") -> Page | None:
|
||||
# Looked up per call, so a test that stubs the downloader stubs this.
|
||||
import trafilatura
|
||||
|
||||
if cookie:
|
||||
answer = trafilatura.fetch_response(url, decode=True, with_headers=True,
|
||||
config=_with_cookie(cookie))
|
||||
else:
|
||||
answer = trafilatura.fetch_response(url, decode=True, with_headers=True)
|
||||
if answer is None:
|
||||
return None
|
||||
html = answer.html or ""
|
||||
data = getattr(answer, "data", None)
|
||||
if isinstance(data, bytes) and data[:4] == ZSTD_MAGIC:
|
||||
html = _unzstd(data)
|
||||
return Page(status=answer.status, html=html,
|
||||
url=getattr(answer, "url", "") or url,
|
||||
headers=answer.headers or {})
|
||||
|
||||
|
||||
#: The first bytes of a zstd frame.
|
||||
ZSTD_MAGIC = b"\x28\xb5\x2f\xfd"
|
||||
|
||||
|
||||
def _unzstd(data: bytes) -> str:
|
||||
"""A zstd body as the HTML trafilatura would have decoded.
|
||||
|
||||
Python 3.14 has `compression.zstd`, so urllib3 offers zstd, and Cloudflare
|
||||
hosts such as opendatascience.com answer with it. Streaming the body,
|
||||
urllib3 hands trafilatura the compressed bytes; trafilatura decompresses
|
||||
zstd only with the `zstandard` package, which refuses a frame that does not
|
||||
state its size, and Cloudflare's frames do not. Undecoded, the page is
|
||||
binary that passes the check and converts to `empty`.
|
||||
"""
|
||||
from compression import zstd
|
||||
|
||||
from trafilatura.utils import decode_file
|
||||
|
||||
return decode_file(zstd.decompress(data))
|
||||
|
||||
|
||||
def _with_cookie(cookie: str) -> ConfigParser:
|
||||
"""trafilatura's own config with a `Cookie` header in it.
|
||||
|
||||
Its downloader takes no headers, only a config. A config that is not the
|
||||
default replaces the default headers, so the user agent is set again to
|
||||
the one it would have sent. `%` is escaped for `ConfigParser`'s
|
||||
interpolation; consent strings are full of it.
|
||||
"""
|
||||
from trafilatura.downloads import USER_AGENT
|
||||
from trafilatura.settings import DEFAULT_CONFIG
|
||||
|
||||
config = copy.deepcopy(DEFAULT_CONFIG)
|
||||
config.set("DEFAULT", "USER_AGENTS", USER_AGENT)
|
||||
config.set("DEFAULT", "COOKIE", cookie.replace("%", "%%"))
|
||||
return config
|
||||
|
||||
|
||||
def _browser(url: str, cache: MutableMapping[str, str] | None = None) -> Page | None:
|
||||
got = browser.render(url)
|
||||
if cache is not None and got.consent:
|
||||
browser.remember(cache, got.url, got.cookies)
|
||||
return Page(status=got.status, html=got.html, url=got.url, headers=got.headers)
|
||||
|
||||
|
||||
#: One plain request through trafilatura's downloader: its headers, its size
|
||||
#: cap, and the status and headers handed back rather than thrown away.
|
||||
HTTP = Getter("http", _http)
|
||||
#: Headless Chromium. Accepts a consent dialog it recognises; see
|
||||
#: `desk.tools.browser`.
|
||||
BROWSER = Getter("browser", _browser)
|
||||
STANDARD: Mapping[str, Getter] = {"http": HTTP}
|
||||
#: The getters this module has itself. `browser` only with the `browser`
|
||||
#: extra installed; a verdict naming a getter that is not here skips it. The
|
||||
#: rest come from plugins, through `advanced`.
|
||||
ADVANCED: Mapping[str, Getter] = {
|
||||
"http": HTTP, **({"browser": BROWSER} if browser.installed() else {})}
|
||||
|
||||
|
||||
def advanced(cache: MutableMapping[str, str],
|
||||
registered: Iterable[Getter] = ()) -> Mapping[str, Getter]:
|
||||
"""`ADVANCED` and the getters plugins `registered`, sharing one cookie jar.
|
||||
|
||||
The browser keeps the cookies of a consent it accepted in `cache`, per
|
||||
host, and HTTP sends them. So a gate costs one render per host until its
|
||||
cookies expire, and every other article from that host is a plain request.
|
||||
A registered getter cannot replace `http` or `browser`.
|
||||
"""
|
||||
getters = {getter.name: getter for getter in registered}
|
||||
getters.update(ADVANCED)
|
||||
getters["http"] = Getter("http", lambda url: _http(url, browser.cookie_header(cache, url)))
|
||||
if "browser" in getters:
|
||||
getters["browser"] = Getter("browser", lambda url: _browser(url, cache))
|
||||
return getters
|
||||
|
||||
|
||||
def scrape(url: str, getters: Mapping[str, Getter] = STANDARD,
|
||||
first: str = "http") -> Scraped:
|
||||
"""A claiming getter, then `first`, then whatever each failed check names."""
|
||||
attempts: list[Attempt] = []
|
||||
hint: float | None = None
|
||||
for claimant in getters.values():
|
||||
if not (claimant.claims and claimant.handles(url)):
|
||||
continue
|
||||
content, verdict, page = _one(claimant, url)
|
||||
attempts.append(Attempt(claimant.name, "" if content else verdict.category,
|
||||
"" if content else verdict.name,
|
||||
page.status if page else 0))
|
||||
if content is not None:
|
||||
return Scraped(content, "", claimant.name, tuple(attempts), hint)
|
||||
if page is not None and page.status >= 400:
|
||||
return Scraped(None, verdict.category, claimant.name, tuple(attempts),
|
||||
failures.hint(page.headers))
|
||||
log.debug("%s: %s via %s (%s); the ordinary path runs", url,
|
||||
verdict.category, claimant.name, verdict.name)
|
||||
plan = [first]
|
||||
while plan:
|
||||
getter = getters.get(plan.pop(0))
|
||||
if getter is None or not getter.handles(url) or any(
|
||||
a.getter == getter.name for a in attempts):
|
||||
continue
|
||||
content, verdict, page = _one(getter, url)
|
||||
if content is not None:
|
||||
attempts.append(Attempt(getter.name, status=page.status if page else 0))
|
||||
return Scraped(content, "", getter.name, tuple(attempts), hint)
|
||||
attempts.append(Attempt(getter.name, verdict.category, verdict.name,
|
||||
page.status if page else 0))
|
||||
if page is not None and page.status >= 400:
|
||||
hint = hint or failures.hint(page.headers)
|
||||
log.debug("%s: %s via %s (%s)", url, verdict.category, getter.name, verdict.name)
|
||||
plan = list(verdict.next)
|
||||
# The row gets the last answer a page gave. A fallback that brought back
|
||||
# nothing (a browser that would not start, an API that did not answer)
|
||||
# says nothing about the page, and must not replace what the check found.
|
||||
answered = [a for a in attempts if a.status] or attempts
|
||||
last = answered[-1] if answered else Attempt("", "unknown")
|
||||
return Scraped(None, last.failure, last.getter, tuple(attempts), hint)
|
||||
|
||||
|
||||
def _one(getter: Getter, url: str) -> tuple[ArticleContent | None, Verdict, Page | None]:
|
||||
try:
|
||||
page = getter.fetch(url)
|
||||
except Exception as exc:
|
||||
return None, Verdict(failures.of_exception(exc), type(exc).__name__), None
|
||||
if verdict := check(page):
|
||||
return None, verdict, page
|
||||
assert page is not None # check() refuses a missing page
|
||||
content = article.extract_article(page.html, url)
|
||||
if content is None or not content.plain_text.strip():
|
||||
return None, Verdict("empty", "no_article"), page
|
||||
return content, Verdict("", ""), page
|
||||
Reference in new issue
Block a user