The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
133 lines
5.3 KiB
Python
133 lines
5.3 KiB
Python
"""Will this page load inside an iframe?
|
|
|
|
Publishers say no with one of two headers:
|
|
|
|
X-Frame-Options: DENY | SAMEORIGIN
|
|
Content-Security-Policy: ...; frame-ancestors <list>
|
|
|
|
Both are enforced by the browser, silently: a blocked frame is a blank box with
|
|
no event you can catch from the parent page. Cross-origin means you cannot read
|
|
the frame's document to see what happened either. So the check has to happen
|
|
server-side, before the frame is ever created — which is what this module is.
|
|
|
|
Both headers are set per host, not per article, so one HTTP request per host is
|
|
enough and the answer is cached in the `state` table. In practice this resolves
|
|
to "no" for most newspapers: at the time of writing the Guardian, BBC, Ars
|
|
Technica and Al Jazeera all send SAMEORIGIN, and the NYT sends DENY.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import MutableMapping
|
|
import logging
|
|
import re
|
|
from typing import Any
|
|
from urllib.parse import urlsplit
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: Kept as `embed:` — the rows are already in every database.
|
|
CACHE_PREFIX = "embed:"
|
|
UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
|
"(KHTML, like Gecko) Chrome/131.0 Safari/537.36")
|
|
_FRAME_ANCESTORS = re.compile(r"frame-ancestors\s+([^;]*)", re.IGNORECASE)
|
|
|
|
|
|
def host_of(url: str) -> str:
|
|
try:
|
|
return urlsplit(url).netloc.lower()
|
|
except ValueError:
|
|
return ""
|
|
|
|
|
|
def _verdict(headers: dict[str, str]) -> tuple[bool, str]:
|
|
"""Read the two headers. Returns (embeddable, human reason)."""
|
|
lower = {k.lower(): v for k, v in headers.items()}
|
|
|
|
xfo = (lower.get("x-frame-options") or "").strip().lower()
|
|
if xfo.startswith("deny"):
|
|
return False, "sends X-Frame-Options: DENY"
|
|
if xfo.startswith("sameorigin"):
|
|
return False, "sends X-Frame-Options: SAMEORIGIN"
|
|
|
|
match = _FRAME_ANCESTORS.search(lower.get("content-security-policy", ""))
|
|
if match:
|
|
sources = match.group(1).split()
|
|
# 'self' and 'none' both exclude us; a wildcard or an explicit list
|
|
# might not, but we're never on anyone's list, so only * lets us in.
|
|
if "*" not in sources:
|
|
return False, f"CSP frame-ancestors {' '.join(sources) or 'none'}"
|
|
|
|
return True, "no framing restriction"
|
|
|
|
|
|
def cached(cache: MutableMapping[str, str], url: str) -> dict[str, Any] | None:
|
|
"""The answer already known for this host, or None if nobody has asked.
|
|
|
|
The difference between this and `check` is a network request, and it is
|
|
the whole reason both exist: a page being rendered must not wait on an
|
|
outlet to find out whether it may be framed, but it does need to know the
|
|
difference between "no" and "not asked yet". None is the second one, and
|
|
the caller shows it as pending rather than as a refusal.
|
|
"""
|
|
host = host_of(url)
|
|
if not host or not url.lower().startswith(("http://", "https://")):
|
|
return {"embeddable": False, "reason": "not a web page", "host": host,
|
|
"certain": True}
|
|
stored = cache.get(CACHE_PREFIX + host, "")
|
|
if not stored:
|
|
return None
|
|
ok, _, reason = stored.partition(":")
|
|
return {"embeddable": ok == "1", "reason": reason, "host": host,
|
|
"certain": True}
|
|
|
|
|
|
def check(cache: MutableMapping[str, str], url: str, refresh: bool = False) -> dict[str, Any]:
|
|
"""Ask once per host, then remember.
|
|
|
|
`cache` is any mapping of strings — the app's state table, or a dict in a
|
|
test — because a tool has no store of its own to remember things in.
|
|
|
|
An unreachable host is reported as embeddable with `certain: False` — better
|
|
to let the browser try and possibly succeed than to refuse on our own
|
|
guess. Only a header we actually read counts as a no.
|
|
"""
|
|
host = host_of(url)
|
|
if not host or not url.lower().startswith(("http://", "https://")):
|
|
return {"embeddable": False, "reason": "not a web page", "host": host,
|
|
"certain": True}
|
|
|
|
key = CACHE_PREFIX + host
|
|
if not refresh:
|
|
cached = cache.get(key, "")
|
|
if cached:
|
|
ok, _, reason = cached.partition(":")
|
|
return {"embeddable": ok == "1", "reason": reason, "host": host,
|
|
"certain": True}
|
|
|
|
try:
|
|
import httpx
|
|
except ImportError:
|
|
return {"embeddable": True, "reason": "httpx not installed, not checked",
|
|
"host": host, "certain": False}
|
|
|
|
try:
|
|
with httpx.Client(follow_redirects=True, timeout=10.0,
|
|
headers={"User-Agent": UA}) as client:
|
|
resp = client.get(url)
|
|
# A 403 still carries the headers — the NYT refuses the request *and*
|
|
# refuses framing — so don't gate on the status code.
|
|
embeddable, reason = _verdict(dict(resp.headers))
|
|
# Redirects can land on a different host; cache the answer under both.
|
|
final = host_of(str(resp.url))
|
|
except Exception as exc:
|
|
log.debug("embed check failed for %s: %s", host, exc)
|
|
return {"embeddable": True, "reason": f"could not check ({type(exc).__name__})",
|
|
"host": host, "certain": False}
|
|
|
|
payload = f"{'1' if embeddable else '0'}:{reason}"
|
|
cache[key] = payload
|
|
if final and final != host:
|
|
cache[CACHE_PREFIX + final] = payload
|
|
return {"embeddable": embeddable, "reason": reason, "host": host, "certain": True}
|