Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d666d0937
commit
248ea1636d
36 files changed
+8393
No files matched your search
@@ -0,0 +1,132 @@
|
||||
"""Will this page load inside an iframe?
|
||||
|
||||
Publishers say no with one of two headers:
|
||||
|
||||
X-Frame-Options: DENY | SAMEORIGIN
|
||||
Content-Security-Policy: ...; frame-ancestors <list>
|
||||
|
||||
Both are enforced by the browser, silently: a blocked frame is a blank box with
|
||||
no event you can catch from the parent page. Cross-origin means you cannot read
|
||||
the frame's document to see what happened either. So the check has to happen
|
||||
server-side, before the frame is ever created — which is what this module is.
|
||||
|
||||
Both headers are set per host, not per article, so one HTTP request per host is
|
||||
enough and the answer is cached in the `state` table. In practice this resolves
|
||||
to "no" for most newspapers: at the time of writing the Guardian, BBC, Ars
|
||||
Technica and Al Jazeera all send SAMEORIGIN, and the NYT sends DENY.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import MutableMapping
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
#: Kept as `embed:` — the rows are already in every database.
|
||||
CACHE_PREFIX = "embed:"
|
||||
UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0 Safari/537.36")
|
||||
_FRAME_ANCESTORS = re.compile(r"frame-ancestors\s+([^;]*)", re.IGNORECASE)
|
||||
|
||||
|
||||
def host_of(url: str) -> str:
|
||||
try:
|
||||
return urlsplit(url).netloc.lower()
|
||||
except ValueError:
|
||||
return ""
|
||||
|
||||
|
||||
def _verdict(headers: dict[str, str]) -> tuple[bool, str]:
|
||||
"""Read the two headers. Returns (embeddable, human reason)."""
|
||||
lower = {k.lower(): v for k, v in headers.items()}
|
||||
|
||||
xfo = (lower.get("x-frame-options") or "").strip().lower()
|
||||
if xfo.startswith("deny"):
|
||||
return False, "sends X-Frame-Options: DENY"
|
||||
if xfo.startswith("sameorigin"):
|
||||
return False, "sends X-Frame-Options: SAMEORIGIN"
|
||||
|
||||
match = _FRAME_ANCESTORS.search(lower.get("content-security-policy", ""))
|
||||
if match:
|
||||
sources = match.group(1).split()
|
||||
# 'self' and 'none' both exclude us; a wildcard or an explicit list
|
||||
# might not, but we're never on anyone's list, so only * lets us in.
|
||||
if "*" not in sources:
|
||||
return False, f"CSP frame-ancestors {' '.join(sources) or 'none'}"
|
||||
|
||||
return True, "no framing restriction"
|
||||
|
||||
|
||||
def cached(cache: MutableMapping[str, str], url: str) -> dict[str, Any] | None:
|
||||
"""The answer already known for this host, or None if nobody has asked.
|
||||
|
||||
The difference between this and `check` is a network request, and it is
|
||||
the whole reason both exist: a page being rendered must not wait on an
|
||||
outlet to find out whether it may be framed, but it does need to know the
|
||||
difference between "no" and "not asked yet". None is the second one, and
|
||||
the caller shows it as pending rather than as a refusal.
|
||||
"""
|
||||
host = host_of(url)
|
||||
if not host or not url.lower().startswith(("http://", "https://")):
|
||||
return {"embeddable": False, "reason": "not a web page", "host": host,
|
||||
"certain": True}
|
||||
stored = cache.get(CACHE_PREFIX + host, "")
|
||||
if not stored:
|
||||
return None
|
||||
ok, _, reason = stored.partition(":")
|
||||
return {"embeddable": ok == "1", "reason": reason, "host": host,
|
||||
"certain": True}
|
||||
|
||||
|
||||
def check(cache: MutableMapping[str, str], url: str, refresh: bool = False) -> dict[str, Any]:
|
||||
"""Ask once per host, then remember.
|
||||
|
||||
`cache` is any mapping of strings — the app's state table, or a dict in a
|
||||
test — because a tool has no store of its own to remember things in.
|
||||
|
||||
An unreachable host is reported as embeddable with `certain: False` — better
|
||||
to let the browser try and possibly succeed than to refuse on our own
|
||||
guess. Only a header we actually read counts as a no.
|
||||
"""
|
||||
host = host_of(url)
|
||||
if not host or not url.lower().startswith(("http://", "https://")):
|
||||
return {"embeddable": False, "reason": "not a web page", "host": host,
|
||||
"certain": True}
|
||||
|
||||
key = CACHE_PREFIX + host
|
||||
if not refresh:
|
||||
cached = cache.get(key, "")
|
||||
if cached:
|
||||
ok, _, reason = cached.partition(":")
|
||||
return {"embeddable": ok == "1", "reason": reason, "host": host,
|
||||
"certain": True}
|
||||
|
||||
try:
|
||||
import httpx
|
||||
except ImportError:
|
||||
return {"embeddable": True, "reason": "httpx not installed, not checked",
|
||||
"host": host, "certain": False}
|
||||
|
||||
try:
|
||||
with httpx.Client(follow_redirects=True, timeout=10.0,
|
||||
headers={"User-Agent": UA}) as client:
|
||||
resp = client.get(url)
|
||||
# A 403 still carries the headers — the NYT refuses the request *and*
|
||||
# refuses framing — so don't gate on the status code.
|
||||
embeddable, reason = _verdict(dict(resp.headers))
|
||||
# Redirects can land on a different host; cache the answer under both.
|
||||
final = host_of(str(resp.url))
|
||||
except Exception as exc:
|
||||
log.debug("embed check failed for %s: %s", host, exc)
|
||||
return {"embeddable": True, "reason": f"could not check ({type(exc).__name__})",
|
||||
"host": host, "certain": False}
|
||||
|
||||
payload = f"{'1' if embeddable else '0'}:{reason}"
|
||||
cache[key] = payload
|
||||
if final and final != host:
|
||||
cache[CACHE_PREFIX + final] = payload
|
||||
return {"embeddable": embeddable, "reason": reason, "host": host, "certain": True}
|
||||
Reference in new issue
Block a user