Files
WolfandClaude Opus 5.5 248ea1636d Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:17 +02:00

133 lines
5.3 KiB
Python

"""Will this page load inside an iframe?
Publishers say no with one of two headers:
X-Frame-Options: DENY | SAMEORIGIN
Content-Security-Policy: ...; frame-ancestors <list>
Both are enforced by the browser, silently: a blocked frame is a blank box with
no event you can catch from the parent page. Cross-origin means you cannot read
the frame's document to see what happened either. So the check has to happen
server-side, before the frame is ever created — which is what this module is.
Both headers are set per host, not per article, so one HTTP request per host is
enough and the answer is cached in the `state` table. In practice this resolves
to "no" for most newspapers: at the time of writing the Guardian, BBC, Ars
Technica and Al Jazeera all send SAMEORIGIN, and the NYT sends DENY.
"""
from __future__ import annotations
from collections.abc import MutableMapping
import logging
import re
from typing import Any
from urllib.parse import urlsplit
log = logging.getLogger(__name__)
#: Kept as `embed:` — the rows are already in every database.
CACHE_PREFIX = "embed:"
UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/131.0 Safari/537.36")
_FRAME_ANCESTORS = re.compile(r"frame-ancestors\s+([^;]*)", re.IGNORECASE)
def host_of(url: str) -> str:
try:
return urlsplit(url).netloc.lower()
except ValueError:
return ""
def _verdict(headers: dict[str, str]) -> tuple[bool, str]:
"""Read the two headers. Returns (embeddable, human reason)."""
lower = {k.lower(): v for k, v in headers.items()}
xfo = (lower.get("x-frame-options") or "").strip().lower()
if xfo.startswith("deny"):
return False, "sends X-Frame-Options: DENY"
if xfo.startswith("sameorigin"):
return False, "sends X-Frame-Options: SAMEORIGIN"
match = _FRAME_ANCESTORS.search(lower.get("content-security-policy", ""))
if match:
sources = match.group(1).split()
# 'self' and 'none' both exclude us; a wildcard or an explicit list
# might not, but we're never on anyone's list, so only * lets us in.
if "*" not in sources:
return False, f"CSP frame-ancestors {' '.join(sources) or 'none'}"
return True, "no framing restriction"
def cached(cache: MutableMapping[str, str], url: str) -> dict[str, Any] | None:
"""The answer already known for this host, or None if nobody has asked.
The difference between this and `check` is a network request, and it is
the whole reason both exist: a page being rendered must not wait on an
outlet to find out whether it may be framed, but it does need to know the
difference between "no" and "not asked yet". None is the second one, and
the caller shows it as pending rather than as a refusal.
"""
host = host_of(url)
if not host or not url.lower().startswith(("http://", "https://")):
return {"embeddable": False, "reason": "not a web page", "host": host,
"certain": True}
stored = cache.get(CACHE_PREFIX + host, "")
if not stored:
return None
ok, _, reason = stored.partition(":")
return {"embeddable": ok == "1", "reason": reason, "host": host,
"certain": True}
def check(cache: MutableMapping[str, str], url: str, refresh: bool = False) -> dict[str, Any]:
"""Ask once per host, then remember.
`cache` is any mapping of strings — the app's state table, or a dict in a
test — because a tool has no store of its own to remember things in.
An unreachable host is reported as embeddable with `certain: False` — better
to let the browser try and possibly succeed than to refuse on our own
guess. Only a header we actually read counts as a no.
"""
host = host_of(url)
if not host or not url.lower().startswith(("http://", "https://")):
return {"embeddable": False, "reason": "not a web page", "host": host,
"certain": True}
key = CACHE_PREFIX + host
if not refresh:
cached = cache.get(key, "")
if cached:
ok, _, reason = cached.partition(":")
return {"embeddable": ok == "1", "reason": reason, "host": host,
"certain": True}
try:
import httpx
except ImportError:
return {"embeddable": True, "reason": "httpx not installed, not checked",
"host": host, "certain": False}
try:
with httpx.Client(follow_redirects=True, timeout=10.0,
headers={"User-Agent": UA}) as client:
resp = client.get(url)
# A 403 still carries the headers — the NYT refuses the request *and*
# refuses framing — so don't gate on the status code.
embeddable, reason = _verdict(dict(resp.headers))
# Redirects can land on a different host; cache the answer under both.
final = host_of(str(resp.url))
except Exception as exc:
log.debug("embed check failed for %s: %s", host, exc)
return {"embeddable": True, "reason": f"could not check ({type(exc).__name__})",
"host": host, "certain": False}
payload = f"{'1' if embeddable else '0'}:{reason}"
cache[key] = payload
if final and final != host:
cache[CACHE_PREFIX + final] = payload
return {"embeddable": embeddable, "reason": reason, "host": host, "certain": True}