Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d666d0937
commit
248ea1636d
36 files changed
+8393
No files matched your search
@@ -0,0 +1,254 @@
|
||||
"""Render a page in headless Chromium, accepting a tracking-consent gate on the way.
|
||||
|
||||
The getter a check sends a page to when plain HTTP brought back a placeholder:
|
||||
an app shell whose article is drawn by a script, or a consent gate standing in
|
||||
front of every article on a host. Golem is the second kind. Every article URL
|
||||
redirects to `/sonstiges/zustimmung/auswahl.html`, where Sourcepoint's CMP
|
||||
draws its dialog in an iframe; accepting it sets the cookies and redirects
|
||||
back to the article.
|
||||
|
||||
from desk.tools import browser
|
||||
|
||||
got = browser.render("https://www.golem.de/news/...")
|
||||
got.status, got.html, got.url, got.consent, got.cookies
|
||||
|
||||
browser.remember(cache, got.url, got.cookies)
|
||||
browser.cookie_header(cache, url) # "golem_consent20=cmp|250101; ..."
|
||||
|
||||
**Consent is accepted once and then reused.** Chromium costs a few seconds
|
||||
and a few hundred megabytes per page. The cookies it earns open the gate for
|
||||
a plain request too, so the caller stores them per host with `remember` and
|
||||
sends them with `cookie_header` on the next HTTP fetch. The browser runs again
|
||||
only when HTTP meets the gate again, which is when the cookies expired or the
|
||||
outlet changed what it accepts. Golem's last three months.
|
||||
|
||||
**A CMP is one entry in `CONSENTS`**: the selector of the iframe it draws in
|
||||
and a pattern for the accept button's name. A page with no such frame is
|
||||
rendered without clicking anything, after waiting `gate_wait` seconds for one
|
||||
to appear. Only accept buttons are listed. A bot check that wants a human is
|
||||
not something this clicks; it stays `js_required` or `denied`.
|
||||
|
||||
Needs the `browser` extra and a Chromium build: `playwright install chromium`.
|
||||
Imported only inside `render`, so the rest of `desk.tools` does not pay for it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from desk.tools.framing import host_of
|
||||
|
||||
#: Sits alongside `embed:` and `body:` in the same table.
|
||||
CACHE_PREFIX = "cookies:"
|
||||
|
||||
#: Seconds for navigation and for the click. A gate that has not redirected
|
||||
#: back by then is not going to.
|
||||
TIMEOUT = 30.0
|
||||
|
||||
#: Seconds to wait for a consent frame on a page that may not have one. The
|
||||
#: CMP injects its iframe from a script; on golem.de it is there within a
|
||||
#: second.
|
||||
GATE_WAIT = 5.0
|
||||
|
||||
#: Seconds to let a script-drawn page settle after `domcontentloaded`. Not
|
||||
#: `networkidle` as a condition: an ad-carrying page never goes idle, so this is
|
||||
#: a ceiling.
|
||||
SETTLE = 3.0
|
||||
|
||||
#: One Chromium at a time in this process. Body batches run side by side, and
|
||||
#: the first articles of a gated host would otherwise each start a browser
|
||||
#: before the first one has earned the cookies.
|
||||
_ONE_AT_A_TIME = threading.Lock()
|
||||
|
||||
|
||||
def installed() -> bool:
|
||||
"""Whether the `browser` extra is. Chromium itself is checked by launching."""
|
||||
return importlib.util.find_spec("playwright") is not None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Consent:
|
||||
"""How to accept one CMP's dialog."""
|
||||
|
||||
name: str
|
||||
#: CSS selector of the iframe the dialog is drawn in.
|
||||
frame: str
|
||||
#: Case-insensitive regex, matched against the accept button's
|
||||
#: accessible name. Anchored, so "Accept" does not match "Accept only
|
||||
#: necessary" and a reject button is never the one clicked.
|
||||
accept: str
|
||||
#: A page this was found on.
|
||||
seen: str = ""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
re.compile(self.accept)
|
||||
|
||||
|
||||
CONSENTS: tuple[Consent, ...] = (
|
||||
Consent("sourcepoint", "iframe[id^='sp_message_iframe']",
|
||||
r"^(Zustimmen und weiter|Alle akzeptieren|Akzeptieren|Zustimmen"
|
||||
r"|Accept all|Accept|I agree|Agree)$",
|
||||
seen="https://www.golem.de/sonstiges/zustimmung/auswahl.html"),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Cookie:
|
||||
name: str
|
||||
value: str
|
||||
domain: str
|
||||
path: str = "/"
|
||||
#: Unix seconds; -1 for a session cookie, which is kept until the gate
|
||||
#: comes back rather than until a browser that no longer exists closes.
|
||||
expires: float = -1
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Rendered:
|
||||
"""What Chromium ended up showing. `url` is where it ended up."""
|
||||
|
||||
status: int
|
||||
html: str
|
||||
url: str
|
||||
headers: Mapping[str, str] = field(default_factory=dict)
|
||||
#: Every cookie the host of `url` would be sent.
|
||||
cookies: tuple[Cookie, ...] = ()
|
||||
#: The `Consent.name` that was accepted, or "" when no gate was met.
|
||||
consent: str = ""
|
||||
|
||||
|
||||
def render(url: str, *, consents: Sequence[Consent] = CONSENTS,
|
||||
timeout: float = TIMEOUT, gate_wait: float = GATE_WAIT,
|
||||
settle: float = SETTLE) -> Rendered:
|
||||
"""Load `url`, accept a consent dialog if one appears, return the page.
|
||||
|
||||
Raises what Playwright raises: its `TimeoutError` names itself a timeout
|
||||
to `failures.of_exception`, and a missing Chromium build is `unknown`.
|
||||
One browser per call. A pull renders a page only where HTTP failed, so a
|
||||
browser kept alive between calls would sit idle in a worker.
|
||||
"""
|
||||
from playwright.sync_api import TimeoutError as PlaywrightTimeout
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
ms = int(timeout * 1000)
|
||||
with _ONE_AT_A_TIME, sync_playwright() as playwright:
|
||||
# /dev/shm is 64 MB in a container, which Chromium outgrows and
|
||||
# crashes on; this has it use /tmp instead.
|
||||
chromium = playwright.chromium.launch(
|
||||
headless=True, args=["--disable-dev-shm-usage"])
|
||||
try:
|
||||
context = chromium.new_context()
|
||||
page = context.new_page()
|
||||
# The last document response of the main frame, so the status is
|
||||
# the article's after a gate's redirect rather than the gate's.
|
||||
documents: list[Any] = []
|
||||
page.on("response", lambda response: documents.append(response)
|
||||
if response.request.is_navigation_request()
|
||||
and response.frame == page.main_frame else None)
|
||||
# `domcontentloaded`, not `load`: an ad-carrying page such as a
|
||||
# golem.de article may not fire `load` within any useful timeout.
|
||||
page.goto(url, wait_until="domcontentloaded", timeout=ms)
|
||||
accepted = _accept(page, consents, ms, gate_wait, PlaywrightTimeout)
|
||||
with contextlib.suppress(PlaywrightTimeout):
|
||||
page.wait_for_load_state("networkidle", timeout=int(settle * 1000))
|
||||
last = documents[-1] if documents else None
|
||||
final = page.url
|
||||
return Rendered(
|
||||
status=last.status if last else 0,
|
||||
html=page.content(),
|
||||
url=final,
|
||||
headers=dict(last.headers) if last else {},
|
||||
cookies=_for_host(_hostname(final), context.cookies()),
|
||||
consent=accepted)
|
||||
finally:
|
||||
chromium.close()
|
||||
|
||||
|
||||
def _accept(page: Any, consents: Sequence[Consent], ms: int, gate_wait: float,
|
||||
timeout_error: type[Exception]) -> str:
|
||||
"""Click the first consent dialog's accept button. The CMP's name, or ""."""
|
||||
deadline = time.monotonic() + gate_wait
|
||||
for consent in consents:
|
||||
left = max(0, int((deadline - time.monotonic()) * 1000))
|
||||
frame = page.locator(consent.frame).first
|
||||
try:
|
||||
frame.wait_for(state="attached", timeout=max(left, 1))
|
||||
except timeout_error:
|
||||
continue
|
||||
button = page.frame_locator(consent.frame).first.get_by_role(
|
||||
"button", name=re.compile(consent.accept, re.IGNORECASE))
|
||||
button.first.click(timeout=ms)
|
||||
# Gone either way it is dismissed: an overlay removes its frame, a
|
||||
# gate page is navigated away from.
|
||||
frame.wait_for(state="detached", timeout=ms)
|
||||
page.wait_for_load_state("domcontentloaded", timeout=ms)
|
||||
return consent.name
|
||||
return ""
|
||||
|
||||
|
||||
def _for_host(host: str, cookies: Sequence[Mapping[str, Any]]) -> tuple[Cookie, ...]:
|
||||
return tuple(
|
||||
Cookie(str(c["name"]), str(c["value"]), str(c["domain"]),
|
||||
str(c.get("path", "/")), float(c.get("expires", -1)))
|
||||
for c in cookies if _matches(host, str(c.get("domain", ""))))
|
||||
|
||||
|
||||
def _hostname(url: str) -> str:
|
||||
"""The host without a port, which is what a cookie's domain is matched to."""
|
||||
try:
|
||||
return (urlsplit(url).hostname or "").lower()
|
||||
except ValueError:
|
||||
return ""
|
||||
|
||||
|
||||
def _matches(host: str, domain: str) -> bool:
|
||||
domain = domain.lstrip(".").lower()
|
||||
return bool(host) and (host == domain or host.endswith("." + domain))
|
||||
|
||||
|
||||
# ------------------------------------------------------------- the cookie jar
|
||||
|
||||
def remember(cache: MutableMapping[str, str], url: str,
|
||||
cookies: Sequence[Cookie]) -> None:
|
||||
"""Keep the cookies a render earned, for the next plain fetch from this host.
|
||||
|
||||
Replaces whatever the host had: a render that met the gate again earned
|
||||
the current set, and the old one is what stopped working.
|
||||
"""
|
||||
kept = [c for c in cookies if _matches(_hostname(url), c.domain)]
|
||||
if kept:
|
||||
cache[CACHE_PREFIX + host_of(url)] = json.dumps(
|
||||
[[c.name, c.value, c.domain, c.path, c.expires] for c in kept])
|
||||
|
||||
|
||||
def cookie_header(cache: Mapping[str, str], url: str, *,
|
||||
now: float | None = None) -> str:
|
||||
"""The `Cookie` header for a plain request to `url`, or "" for none.
|
||||
|
||||
Expired cookies are left out rather than deleted. A host whose cookies
|
||||
have all expired sends none, meets the gate, and is rendered again.
|
||||
"""
|
||||
stored = cache.get(CACHE_PREFIX + host_of(url), "")
|
||||
if not stored:
|
||||
return ""
|
||||
now = time.time() if now is None else now
|
||||
try:
|
||||
rows = json.loads(stored)
|
||||
except ValueError:
|
||||
return ""
|
||||
return "; ".join(f"{name}={value}" for name, value, _domain, _path, expires in rows
|
||||
if expires < 0 or expires > now)
|
||||
|
||||
|
||||
def forget(cache: MutableMapping[str, str], url: str) -> None:
|
||||
"""Drop a host's cookies, so its next fetch goes without them."""
|
||||
cache.pop(CACHE_PREFIX + host_of(url), None)
|
||||
Reference in new issue
Block a user