"""Render a page in headless Chromium, accepting a tracking-consent gate on the way. The getter a check sends a page to when plain HTTP brought back a placeholder: an app shell whose article is drawn by a script, or a consent gate standing in front of every article on a host. Golem is the second kind. Every article URL redirects to `/sonstiges/zustimmung/auswahl.html`, where Sourcepoint's CMP draws its dialog in an iframe; accepting it sets the cookies and redirects back to the article. from desk.tools import browser got = browser.render("https://www.golem.de/news/...") got.status, got.html, got.url, got.consent, got.cookies browser.remember(cache, got.url, got.cookies) browser.cookie_header(cache, url) # "golem_consent20=cmp|250101; ..." **Consent is accepted once and then reused.** Chromium costs a few seconds and a few hundred megabytes per page. The cookies it earns open the gate for a plain request too, so the caller stores them per host with `remember` and sends them with `cookie_header` on the next HTTP fetch. The browser runs again only when HTTP meets the gate again, which is when the cookies expired or the outlet changed what it accepts. Golem's last three months. **A CMP is one entry in `CONSENTS`**: the selector of the iframe it draws in and a pattern for the accept button's name. A page with no such frame is rendered without clicking anything, after waiting `gate_wait` seconds for one to appear. Only accept buttons are listed. A bot check that wants a human is not something this clicks; it stays `js_required` or `denied`. Needs the `browser` extra and a Chromium build: `playwright install chromium`. Imported only inside `render`, so the rest of `desk.tools` does not pay for it. """ from __future__ import annotations import contextlib import importlib.util import json import re import threading import time from collections.abc import Mapping, MutableMapping, Sequence from dataclasses import dataclass, field from typing import Any from urllib.parse import urlsplit from desk.tools.framing import host_of #: Sits alongside `embed:` and `body:` in the same table. CACHE_PREFIX = "cookies:" #: Seconds for navigation and for the click. A gate that has not redirected #: back by then is not going to. TIMEOUT = 30.0 #: Seconds to wait for a consent frame on a page that may not have one. The #: CMP injects its iframe from a script; on golem.de it is there within a #: second. GATE_WAIT = 5.0 #: Seconds to let a script-drawn page settle after `domcontentloaded`. Not #: `networkidle` as a condition: an ad-carrying page never goes idle, so this is #: a ceiling. SETTLE = 3.0 #: One Chromium at a time in this process. Body batches run side by side, and #: the first articles of a gated host would otherwise each start a browser #: before the first one has earned the cookies. _ONE_AT_A_TIME = threading.Lock() def installed() -> bool: """Whether the `browser` extra is. Chromium itself is checked by launching.""" return importlib.util.find_spec("playwright") is not None @dataclass(frozen=True) class Consent: """How to accept one CMP's dialog.""" name: str #: CSS selector of the iframe the dialog is drawn in. frame: str #: Case-insensitive regex, matched against the accept button's #: accessible name. Anchored, so "Accept" does not match "Accept only #: necessary" and a reject button is never the one clicked. accept: str #: A page this was found on. seen: str = "" def __post_init__(self) -> None: re.compile(self.accept) CONSENTS: tuple[Consent, ...] = ( Consent("sourcepoint", "iframe[id^='sp_message_iframe']", r"^(Zustimmen und weiter|Alle akzeptieren|Akzeptieren|Zustimmen" r"|Accept all|Accept|I agree|Agree)$", seen="https://www.golem.de/sonstiges/zustimmung/auswahl.html"), ) @dataclass(frozen=True) class Cookie: name: str value: str domain: str path: str = "/" #: Unix seconds; -1 for a session cookie, which is kept until the gate #: comes back rather than until a browser that no longer exists closes. expires: float = -1 @dataclass(frozen=True) class Rendered: """What Chromium ended up showing. `url` is where it ended up.""" status: int html: str url: str headers: Mapping[str, str] = field(default_factory=dict) #: Every cookie the host of `url` would be sent. cookies: tuple[Cookie, ...] = () #: The `Consent.name` that was accepted, or "" when no gate was met. consent: str = "" def render(url: str, *, consents: Sequence[Consent] = CONSENTS, timeout: float = TIMEOUT, gate_wait: float = GATE_WAIT, settle: float = SETTLE) -> Rendered: """Load `url`, accept a consent dialog if one appears, return the page. Raises what Playwright raises: its `TimeoutError` names itself a timeout to `failures.of_exception`, and a missing Chromium build is `unknown`. One browser per call. A pull renders a page only where HTTP failed, so a browser kept alive between calls would sit idle in a worker. """ from playwright.sync_api import TimeoutError as PlaywrightTimeout from playwright.sync_api import sync_playwright ms = int(timeout * 1000) with _ONE_AT_A_TIME, sync_playwright() as playwright: # /dev/shm is 64 MB in a container, which Chromium outgrows and # crashes on; this has it use /tmp instead. chromium = playwright.chromium.launch( headless=True, args=["--disable-dev-shm-usage"]) try: context = chromium.new_context() page = context.new_page() # The last document response of the main frame, so the status is # the article's after a gate's redirect rather than the gate's. documents: list[Any] = [] page.on("response", lambda response: documents.append(response) if response.request.is_navigation_request() and response.frame == page.main_frame else None) # `domcontentloaded`, not `load`: an ad-carrying page such as a # golem.de article may not fire `load` within any useful timeout. page.goto(url, wait_until="domcontentloaded", timeout=ms) accepted = _accept(page, consents, ms, gate_wait, PlaywrightTimeout) with contextlib.suppress(PlaywrightTimeout): page.wait_for_load_state("networkidle", timeout=int(settle * 1000)) last = documents[-1] if documents else None final = page.url return Rendered( status=last.status if last else 0, html=page.content(), url=final, headers=dict(last.headers) if last else {}, cookies=_for_host(_hostname(final), context.cookies()), consent=accepted) finally: chromium.close() def _accept(page: Any, consents: Sequence[Consent], ms: int, gate_wait: float, timeout_error: type[Exception]) -> str: """Click the first consent dialog's accept button. The CMP's name, or "".""" deadline = time.monotonic() + gate_wait for consent in consents: left = max(0, int((deadline - time.monotonic()) * 1000)) frame = page.locator(consent.frame).first try: frame.wait_for(state="attached", timeout=max(left, 1)) except timeout_error: continue button = page.frame_locator(consent.frame).first.get_by_role( "button", name=re.compile(consent.accept, re.IGNORECASE)) button.first.click(timeout=ms) # Gone either way it is dismissed: an overlay removes its frame, a # gate page is navigated away from. frame.wait_for(state="detached", timeout=ms) page.wait_for_load_state("domcontentloaded", timeout=ms) return consent.name return "" def _for_host(host: str, cookies: Sequence[Mapping[str, Any]]) -> tuple[Cookie, ...]: return tuple( Cookie(str(c["name"]), str(c["value"]), str(c["domain"]), str(c.get("path", "/")), float(c.get("expires", -1))) for c in cookies if _matches(host, str(c.get("domain", "")))) def _hostname(url: str) -> str: """The host without a port, which is what a cookie's domain is matched to.""" try: return (urlsplit(url).hostname or "").lower() except ValueError: return "" def _matches(host: str, domain: str) -> bool: domain = domain.lstrip(".").lower() return bool(host) and (host == domain or host.endswith("." + domain)) # ------------------------------------------------------------- the cookie jar def remember(cache: MutableMapping[str, str], url: str, cookies: Sequence[Cookie]) -> None: """Keep the cookies a render earned, for the next plain fetch from this host. Replaces whatever the host had: a render that met the gate again earned the current set, and the old one is what stopped working. """ kept = [c for c in cookies if _matches(_hostname(url), c.domain)] if kept: cache[CACHE_PREFIX + host_of(url)] = json.dumps( [[c.name, c.value, c.domain, c.path, c.expires] for c in kept]) def cookie_header(cache: Mapping[str, str], url: str, *, now: float | None = None) -> str: """The `Cookie` header for a plain request to `url`, or "" for none. Expired cookies are left out rather than deleted. A host whose cookies have all expired sends none, meets the gate, and is rendered again. """ stored = cache.get(CACHE_PREFIX + host_of(url), "") if not stored: return "" now = time.time() if now is None else now try: rows = json.loads(stored) except ValueError: return "" return "; ".join(f"{name}={value}" for name, value, _domain, _path, expires in rows if expires < 0 or expires > now) def forget(cache: MutableMapping[str, str], url: str) -> None: """Drop a host's cookies, so its next fetch goes without them.""" cache.pop(CACHE_PREFIX + host_of(url), None)