The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
255 lines
9.9 KiB
Python
255 lines
9.9 KiB
Python
"""Render a page in headless Chromium, accepting a tracking-consent gate on the way.
|
|
|
|
The getter a check sends a page to when plain HTTP brought back a placeholder:
|
|
an app shell whose article is drawn by a script, or a consent gate standing in
|
|
front of every article on a host. Golem is the second kind. Every article URL
|
|
redirects to `/sonstiges/zustimmung/auswahl.html`, where Sourcepoint's CMP
|
|
draws its dialog in an iframe; accepting it sets the cookies and redirects
|
|
back to the article.
|
|
|
|
from desk.tools import browser
|
|
|
|
got = browser.render("https://www.golem.de/news/...")
|
|
got.status, got.html, got.url, got.consent, got.cookies
|
|
|
|
browser.remember(cache, got.url, got.cookies)
|
|
browser.cookie_header(cache, url) # "golem_consent20=cmp|250101; ..."
|
|
|
|
**Consent is accepted once and then reused.** Chromium costs a few seconds
|
|
and a few hundred megabytes per page. The cookies it earns open the gate for
|
|
a plain request too, so the caller stores them per host with `remember` and
|
|
sends them with `cookie_header` on the next HTTP fetch. The browser runs again
|
|
only when HTTP meets the gate again, which is when the cookies expired or the
|
|
outlet changed what it accepts. Golem's last three months.
|
|
|
|
**A CMP is one entry in `CONSENTS`**: the selector of the iframe it draws in
|
|
and a pattern for the accept button's name. A page with no such frame is
|
|
rendered without clicking anything, after waiting `gate_wait` seconds for one
|
|
to appear. Only accept buttons are listed. A bot check that wants a human is
|
|
not something this clicks; it stays `js_required` or `denied`.
|
|
|
|
Needs the `browser` extra and a Chromium build: `playwright install chromium`.
|
|
Imported only inside `render`, so the rest of `desk.tools` does not pay for it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import importlib.util
|
|
import json
|
|
import re
|
|
import threading
|
|
import time
|
|
from collections.abc import Mapping, MutableMapping, Sequence
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
from urllib.parse import urlsplit
|
|
|
|
from desk.tools.framing import host_of
|
|
|
|
#: Sits alongside `embed:` and `body:` in the same table.
|
|
CACHE_PREFIX = "cookies:"
|
|
|
|
#: Seconds for navigation and for the click. A gate that has not redirected
|
|
#: back by then is not going to.
|
|
TIMEOUT = 30.0
|
|
|
|
#: Seconds to wait for a consent frame on a page that may not have one. The
|
|
#: CMP injects its iframe from a script; on golem.de it is there within a
|
|
#: second.
|
|
GATE_WAIT = 5.0
|
|
|
|
#: Seconds to let a script-drawn page settle after `domcontentloaded`. Not
|
|
#: `networkidle` as a condition: an ad-carrying page never goes idle, so this is
|
|
#: a ceiling.
|
|
SETTLE = 3.0
|
|
|
|
#: One Chromium at a time in this process. Body batches run side by side, and
|
|
#: the first articles of a gated host would otherwise each start a browser
|
|
#: before the first one has earned the cookies.
|
|
_ONE_AT_A_TIME = threading.Lock()
|
|
|
|
|
|
def installed() -> bool:
|
|
"""Whether the `browser` extra is. Chromium itself is checked by launching."""
|
|
return importlib.util.find_spec("playwright") is not None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Consent:
|
|
"""How to accept one CMP's dialog."""
|
|
|
|
name: str
|
|
#: CSS selector of the iframe the dialog is drawn in.
|
|
frame: str
|
|
#: Case-insensitive regex, matched against the accept button's
|
|
#: accessible name. Anchored, so "Accept" does not match "Accept only
|
|
#: necessary" and a reject button is never the one clicked.
|
|
accept: str
|
|
#: A page this was found on.
|
|
seen: str = ""
|
|
|
|
def __post_init__(self) -> None:
|
|
re.compile(self.accept)
|
|
|
|
|
|
CONSENTS: tuple[Consent, ...] = (
|
|
Consent("sourcepoint", "iframe[id^='sp_message_iframe']",
|
|
r"^(Zustimmen und weiter|Alle akzeptieren|Akzeptieren|Zustimmen"
|
|
r"|Accept all|Accept|I agree|Agree)$",
|
|
seen="https://www.golem.de/sonstiges/zustimmung/auswahl.html"),
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Cookie:
|
|
name: str
|
|
value: str
|
|
domain: str
|
|
path: str = "/"
|
|
#: Unix seconds; -1 for a session cookie, which is kept until the gate
|
|
#: comes back rather than until a browser that no longer exists closes.
|
|
expires: float = -1
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Rendered:
|
|
"""What Chromium ended up showing. `url` is where it ended up."""
|
|
|
|
status: int
|
|
html: str
|
|
url: str
|
|
headers: Mapping[str, str] = field(default_factory=dict)
|
|
#: Every cookie the host of `url` would be sent.
|
|
cookies: tuple[Cookie, ...] = ()
|
|
#: The `Consent.name` that was accepted, or "" when no gate was met.
|
|
consent: str = ""
|
|
|
|
|
|
def render(url: str, *, consents: Sequence[Consent] = CONSENTS,
|
|
timeout: float = TIMEOUT, gate_wait: float = GATE_WAIT,
|
|
settle: float = SETTLE) -> Rendered:
|
|
"""Load `url`, accept a consent dialog if one appears, return the page.
|
|
|
|
Raises what Playwright raises: its `TimeoutError` names itself a timeout
|
|
to `failures.of_exception`, and a missing Chromium build is `unknown`.
|
|
One browser per call. A pull renders a page only where HTTP failed, so a
|
|
browser kept alive between calls would sit idle in a worker.
|
|
"""
|
|
from playwright.sync_api import TimeoutError as PlaywrightTimeout
|
|
from playwright.sync_api import sync_playwright
|
|
|
|
ms = int(timeout * 1000)
|
|
with _ONE_AT_A_TIME, sync_playwright() as playwright:
|
|
# /dev/shm is 64 MB in a container, which Chromium outgrows and
|
|
# crashes on; this has it use /tmp instead.
|
|
chromium = playwright.chromium.launch(
|
|
headless=True, args=["--disable-dev-shm-usage"])
|
|
try:
|
|
context = chromium.new_context()
|
|
page = context.new_page()
|
|
# The last document response of the main frame, so the status is
|
|
# the article's after a gate's redirect rather than the gate's.
|
|
documents: list[Any] = []
|
|
page.on("response", lambda response: documents.append(response)
|
|
if response.request.is_navigation_request()
|
|
and response.frame == page.main_frame else None)
|
|
# `domcontentloaded`, not `load`: an ad-carrying page such as a
|
|
# golem.de article may not fire `load` within any useful timeout.
|
|
page.goto(url, wait_until="domcontentloaded", timeout=ms)
|
|
accepted = _accept(page, consents, ms, gate_wait, PlaywrightTimeout)
|
|
with contextlib.suppress(PlaywrightTimeout):
|
|
page.wait_for_load_state("networkidle", timeout=int(settle * 1000))
|
|
last = documents[-1] if documents else None
|
|
final = page.url
|
|
return Rendered(
|
|
status=last.status if last else 0,
|
|
html=page.content(),
|
|
url=final,
|
|
headers=dict(last.headers) if last else {},
|
|
cookies=_for_host(_hostname(final), context.cookies()),
|
|
consent=accepted)
|
|
finally:
|
|
chromium.close()
|
|
|
|
|
|
def _accept(page: Any, consents: Sequence[Consent], ms: int, gate_wait: float,
|
|
timeout_error: type[Exception]) -> str:
|
|
"""Click the first consent dialog's accept button. The CMP's name, or ""."""
|
|
deadline = time.monotonic() + gate_wait
|
|
for consent in consents:
|
|
left = max(0, int((deadline - time.monotonic()) * 1000))
|
|
frame = page.locator(consent.frame).first
|
|
try:
|
|
frame.wait_for(state="attached", timeout=max(left, 1))
|
|
except timeout_error:
|
|
continue
|
|
button = page.frame_locator(consent.frame).first.get_by_role(
|
|
"button", name=re.compile(consent.accept, re.IGNORECASE))
|
|
button.first.click(timeout=ms)
|
|
# Gone either way it is dismissed: an overlay removes its frame, a
|
|
# gate page is navigated away from.
|
|
frame.wait_for(state="detached", timeout=ms)
|
|
page.wait_for_load_state("domcontentloaded", timeout=ms)
|
|
return consent.name
|
|
return ""
|
|
|
|
|
|
def _for_host(host: str, cookies: Sequence[Mapping[str, Any]]) -> tuple[Cookie, ...]:
|
|
return tuple(
|
|
Cookie(str(c["name"]), str(c["value"]), str(c["domain"]),
|
|
str(c.get("path", "/")), float(c.get("expires", -1)))
|
|
for c in cookies if _matches(host, str(c.get("domain", ""))))
|
|
|
|
|
|
def _hostname(url: str) -> str:
|
|
"""The host without a port, which is what a cookie's domain is matched to."""
|
|
try:
|
|
return (urlsplit(url).hostname or "").lower()
|
|
except ValueError:
|
|
return ""
|
|
|
|
|
|
def _matches(host: str, domain: str) -> bool:
|
|
domain = domain.lstrip(".").lower()
|
|
return bool(host) and (host == domain or host.endswith("." + domain))
|
|
|
|
|
|
# ------------------------------------------------------------- the cookie jar
|
|
|
|
def remember(cache: MutableMapping[str, str], url: str,
|
|
cookies: Sequence[Cookie]) -> None:
|
|
"""Keep the cookies a render earned, for the next plain fetch from this host.
|
|
|
|
Replaces whatever the host had: a render that met the gate again earned
|
|
the current set, and the old one is what stopped working.
|
|
"""
|
|
kept = [c for c in cookies if _matches(_hostname(url), c.domain)]
|
|
if kept:
|
|
cache[CACHE_PREFIX + host_of(url)] = json.dumps(
|
|
[[c.name, c.value, c.domain, c.path, c.expires] for c in kept])
|
|
|
|
|
|
def cookie_header(cache: Mapping[str, str], url: str, *,
|
|
now: float | None = None) -> str:
|
|
"""The `Cookie` header for a plain request to `url`, or "" for none.
|
|
|
|
Expired cookies are left out rather than deleted. A host whose cookies
|
|
have all expired sends none, meets the gate, and is rendered again.
|
|
"""
|
|
stored = cache.get(CACHE_PREFIX + host_of(url), "")
|
|
if not stored:
|
|
return ""
|
|
now = time.time() if now is None else now
|
|
try:
|
|
rows = json.loads(stored)
|
|
except ValueError:
|
|
return ""
|
|
return "; ".join(f"{name}={value}" for name, value, _domain, _path, expires in rows
|
|
if expires < 0 or expires > now)
|
|
|
|
|
|
def forget(cache: MutableMapping[str, str], url: str) -> None:
|
|
"""Drop a host's cookies, so its next fetch goes without them."""
|
|
cache.pop(CACHE_PREFIX + host_of(url), None)
|