Files
jobboard/desk/tools/browser.py
T
WolfandClaude Opus 5.5 248ea1636d Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:17 +02:00

255 lines
9.9 KiB
Python

"""Render a page in headless Chromium, accepting a tracking-consent gate on the way.
The getter a check sends a page to when plain HTTP brought back a placeholder:
an app shell whose article is drawn by a script, or a consent gate standing in
front of every article on a host. Golem is the second kind. Every article URL
redirects to `/sonstiges/zustimmung/auswahl.html`, where Sourcepoint's CMP
draws its dialog in an iframe; accepting it sets the cookies and redirects
back to the article.
from desk.tools import browser
got = browser.render("https://www.golem.de/news/...")
got.status, got.html, got.url, got.consent, got.cookies
browser.remember(cache, got.url, got.cookies)
browser.cookie_header(cache, url) # "golem_consent20=cmp|250101; ..."
**Consent is accepted once and then reused.** Chromium costs a few seconds
and a few hundred megabytes per page. The cookies it earns open the gate for
a plain request too, so the caller stores them per host with `remember` and
sends them with `cookie_header` on the next HTTP fetch. The browser runs again
only when HTTP meets the gate again, which is when the cookies expired or the
outlet changed what it accepts. Golem's last three months.
**A CMP is one entry in `CONSENTS`**: the selector of the iframe it draws in
and a pattern for the accept button's name. A page with no such frame is
rendered without clicking anything, after waiting `gate_wait` seconds for one
to appear. Only accept buttons are listed. A bot check that wants a human is
not something this clicks; it stays `js_required` or `denied`.
Needs the `browser` extra and a Chromium build: `playwright install chromium`.
Imported only inside `render`, so the rest of `desk.tools` does not pay for it.
"""
from __future__ import annotations
import contextlib
import importlib.util
import json
import re
import threading
import time
from collections.abc import Mapping, MutableMapping, Sequence
from dataclasses import dataclass, field
from typing import Any
from urllib.parse import urlsplit
from desk.tools.framing import host_of
#: Sits alongside `embed:` and `body:` in the same table.
CACHE_PREFIX = "cookies:"
#: Seconds for navigation and for the click. A gate that has not redirected
#: back by then is not going to.
TIMEOUT = 30.0
#: Seconds to wait for a consent frame on a page that may not have one. The
#: CMP injects its iframe from a script; on golem.de it is there within a
#: second.
GATE_WAIT = 5.0
#: Seconds to let a script-drawn page settle after `domcontentloaded`. Not
#: `networkidle` as a condition: an ad-carrying page never goes idle, so this is
#: a ceiling.
SETTLE = 3.0
#: One Chromium at a time in this process. Body batches run side by side, and
#: the first articles of a gated host would otherwise each start a browser
#: before the first one has earned the cookies.
_ONE_AT_A_TIME = threading.Lock()
def installed() -> bool:
"""Whether the `browser` extra is. Chromium itself is checked by launching."""
return importlib.util.find_spec("playwright") is not None
@dataclass(frozen=True)
class Consent:
"""How to accept one CMP's dialog."""
name: str
#: CSS selector of the iframe the dialog is drawn in.
frame: str
#: Case-insensitive regex, matched against the accept button's
#: accessible name. Anchored, so "Accept" does not match "Accept only
#: necessary" and a reject button is never the one clicked.
accept: str
#: A page this was found on.
seen: str = ""
def __post_init__(self) -> None:
re.compile(self.accept)
CONSENTS: tuple[Consent, ...] = (
Consent("sourcepoint", "iframe[id^='sp_message_iframe']",
r"^(Zustimmen und weiter|Alle akzeptieren|Akzeptieren|Zustimmen"
r"|Accept all|Accept|I agree|Agree)$",
seen="https://www.golem.de/sonstiges/zustimmung/auswahl.html"),
)
@dataclass(frozen=True)
class Cookie:
name: str
value: str
domain: str
path: str = "/"
#: Unix seconds; -1 for a session cookie, which is kept until the gate
#: comes back rather than until a browser that no longer exists closes.
expires: float = -1
@dataclass(frozen=True)
class Rendered:
"""What Chromium ended up showing. `url` is where it ended up."""
status: int
html: str
url: str
headers: Mapping[str, str] = field(default_factory=dict)
#: Every cookie the host of `url` would be sent.
cookies: tuple[Cookie, ...] = ()
#: The `Consent.name` that was accepted, or "" when no gate was met.
consent: str = ""
def render(url: str, *, consents: Sequence[Consent] = CONSENTS,
timeout: float = TIMEOUT, gate_wait: float = GATE_WAIT,
settle: float = SETTLE) -> Rendered:
"""Load `url`, accept a consent dialog if one appears, return the page.
Raises what Playwright raises: its `TimeoutError` names itself a timeout
to `failures.of_exception`, and a missing Chromium build is `unknown`.
One browser per call. A pull renders a page only where HTTP failed, so a
browser kept alive between calls would sit idle in a worker.
"""
from playwright.sync_api import TimeoutError as PlaywrightTimeout
from playwright.sync_api import sync_playwright
ms = int(timeout * 1000)
with _ONE_AT_A_TIME, sync_playwright() as playwright:
# /dev/shm is 64 MB in a container, which Chromium outgrows and
# crashes on; this has it use /tmp instead.
chromium = playwright.chromium.launch(
headless=True, args=["--disable-dev-shm-usage"])
try:
context = chromium.new_context()
page = context.new_page()
# The last document response of the main frame, so the status is
# the article's after a gate's redirect rather than the gate's.
documents: list[Any] = []
page.on("response", lambda response: documents.append(response)
if response.request.is_navigation_request()
and response.frame == page.main_frame else None)
# `domcontentloaded`, not `load`: an ad-carrying page such as a
# golem.de article may not fire `load` within any useful timeout.
page.goto(url, wait_until="domcontentloaded", timeout=ms)
accepted = _accept(page, consents, ms, gate_wait, PlaywrightTimeout)
with contextlib.suppress(PlaywrightTimeout):
page.wait_for_load_state("networkidle", timeout=int(settle * 1000))
last = documents[-1] if documents else None
final = page.url
return Rendered(
status=last.status if last else 0,
html=page.content(),
url=final,
headers=dict(last.headers) if last else {},
cookies=_for_host(_hostname(final), context.cookies()),
consent=accepted)
finally:
chromium.close()
def _accept(page: Any, consents: Sequence[Consent], ms: int, gate_wait: float,
timeout_error: type[Exception]) -> str:
"""Click the first consent dialog's accept button. The CMP's name, or ""."""
deadline = time.monotonic() + gate_wait
for consent in consents:
left = max(0, int((deadline - time.monotonic()) * 1000))
frame = page.locator(consent.frame).first
try:
frame.wait_for(state="attached", timeout=max(left, 1))
except timeout_error:
continue
button = page.frame_locator(consent.frame).first.get_by_role(
"button", name=re.compile(consent.accept, re.IGNORECASE))
button.first.click(timeout=ms)
# Gone either way it is dismissed: an overlay removes its frame, a
# gate page is navigated away from.
frame.wait_for(state="detached", timeout=ms)
page.wait_for_load_state("domcontentloaded", timeout=ms)
return consent.name
return ""
def _for_host(host: str, cookies: Sequence[Mapping[str, Any]]) -> tuple[Cookie, ...]:
return tuple(
Cookie(str(c["name"]), str(c["value"]), str(c["domain"]),
str(c.get("path", "/")), float(c.get("expires", -1)))
for c in cookies if _matches(host, str(c.get("domain", ""))))
def _hostname(url: str) -> str:
"""The host without a port, which is what a cookie's domain is matched to."""
try:
return (urlsplit(url).hostname or "").lower()
except ValueError:
return ""
def _matches(host: str, domain: str) -> bool:
domain = domain.lstrip(".").lower()
return bool(host) and (host == domain or host.endswith("." + domain))
# ------------------------------------------------------------- the cookie jar
def remember(cache: MutableMapping[str, str], url: str,
cookies: Sequence[Cookie]) -> None:
"""Keep the cookies a render earned, for the next plain fetch from this host.
Replaces whatever the host had: a render that met the gate again earned
the current set, and the old one is what stopped working.
"""
kept = [c for c in cookies if _matches(_hostname(url), c.domain)]
if kept:
cache[CACHE_PREFIX + host_of(url)] = json.dumps(
[[c.name, c.value, c.domain, c.path, c.expires] for c in kept])
def cookie_header(cache: Mapping[str, str], url: str, *,
now: float | None = None) -> str:
"""The `Cookie` header for a plain request to `url`, or "" for none.
Expired cookies are left out rather than deleted. A host whose cookies
have all expired sends none, meets the gate, and is rendered again.
"""
stored = cache.get(CACHE_PREFIX + host_of(url), "")
if not stored:
return ""
now = time.time() if now is None else now
try:
rows = json.loads(stored)
except ValueError:
return ""
return "; ".join(f"{name}={value}" for name, value, _domain, _path, expires in rows
if expires < 0 or expires > now)
def forget(cache: MutableMapping[str, str], url: str) -> None:
"""Drop a host's cookies, so its next fetch goes without them."""
cache.pop(CACHE_PREFIX + host_of(url), None)