The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
237 lines
11 KiB
Python
237 lines
11 KiB
Python
"""RSS and Atom feed fetching, as a base class for a publisher's plugin.
|
|
|
|
Not itself a source: see `RssSource`, which is what a masthead subclasses.
|
|
`check_feed` is the same fetching asked whether one address still answers.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from collections.abc import Iterator
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
import feedparser
|
|
import httpx
|
|
|
|
from desk.sources.base import (
|
|
USER_AGENT, CheckResult, Item, Source, entry_time, http_failure, outlet_name,
|
|
strip_html,
|
|
)
|
|
from desk.tools.http import session
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: Seconds one feed gets, per operation. Which number it is matters far less
|
|
#: than there being one at all — see `_read`.
|
|
FETCH_TIMEOUT = 30
|
|
|
|
|
|
class FeedsFailed(Exception):
|
|
"""Some of a pull's feeds could not be fetched, and the rest were.
|
|
|
|
Raised at the end of `fetch` rather than where it happened, and that is the
|
|
whole of the design: everything the live feeds gave has already been
|
|
yielded and stored by the time this lands, so one outlet returning 403
|
|
costs you that outlet and not the other nineteen. The runner turns it into
|
|
`notices.source_failed`, which is what puts a feed that has been dead all
|
|
week on screen — the one failure a reader cannot spot by eye, because a
|
|
dead source and a quiet news day look identical.
|
|
|
|
Worded through `http_failure`, so every line in it reads `HTTP 403` or
|
|
`unreachable: ReadTimeout`. That is also what has the notice file it as the
|
|
outlet's business rather than as a defect of ours; see
|
|
`desk/notices.py`.
|
|
"""
|
|
|
|
|
|
def _categories(entry) -> list[str]:
|
|
"""Whatever the outlet put in <category>.
|
|
|
|
The Guardian, Ars Technica, the NYT and Al Jazeera all populate these;
|
|
the BBC and Hacker News don't. They're inconsistent enough between outlets
|
|
to be useless as a filter — the NYT files people ('Putin, Vladimir V'),
|
|
the Guardian files desks ('Television & radio') — so they're carried as a
|
|
hint for the tagger rather than shown to you.
|
|
"""
|
|
seen, out = set(), []
|
|
for tag in entry.get("tags") or []:
|
|
term = strip_html(tag.get("term", ""))
|
|
if term and term.lower() not in seen:
|
|
seen.add(term.lower())
|
|
out.append(term)
|
|
return out
|
|
|
|
|
|
def check_feed(url: str, label: str = "", kind: str = "rss") -> CheckResult:
|
|
"""Fetch one feed and say what is in it. Shared with the podcast source,
|
|
whose subscription is a feed too."""
|
|
try:
|
|
r = httpx.get(url, headers={"User-Agent": USER_AGENT}, timeout=20,
|
|
follow_redirects=True)
|
|
except httpx.HTTPError as exc:
|
|
return CheckResult(kind, url, False, f"unreachable: {type(exc).__name__}", label)
|
|
if r.status_code != 200:
|
|
return CheckResult(kind, url, False, f"HTTP {r.status_code}", label)
|
|
|
|
parsed = feedparser.parse(r.content)
|
|
n = len(parsed.entries)
|
|
if n == 0:
|
|
reason = parsed.get("bozo_exception")
|
|
return CheckResult(kind, url, False, f"0 entries ({reason or 'not a feed?'})", label)
|
|
return CheckResult(kind, url, True, f"{n} entries, {_age(parsed.entries)}", label,
|
|
outlet=_proposed_outlet(parsed, url, label))
|
|
|
|
|
|
def _read(http: httpx.Client, url: str) -> Any:
|
|
"""One feed, fetched here and parsed from the bytes it answered with.
|
|
|
|
Deliberately not `feedparser.parse(url)`, which is the same call with the
|
|
fetching left to somebody else and which goes through urllib with **no
|
|
timeout**. A
|
|
host that accepts the connection and then says nothing holds the thread for
|
|
ever, and the thread it holds is a worker's: the pull is `solo`, so nothing
|
|
else pulls behind it; cancellation is cooperative, so Kill cannot reach a
|
|
thread parked in a socket read; and a restart releases the lease only to
|
|
re-queue the job and stall it on the same feed. One bad feed stopped the
|
|
application from pulling at all until somebody edited the subscription.
|
|
feedparser offers no way to pass urllib a deadline, so the fetch comes
|
|
here, where the client has one.
|
|
|
|
Two things fall out of doing it here, both of which `check_feed` has had
|
|
all along. The request goes out as `USER_AGENT`, which `base.py` promises
|
|
every plugin sends and which feedparser was quietly replacing with its own.
|
|
And the status is read, so a 403, 410 or 429 — each of which arrives with
|
|
an empty body that parses perfectly well as a feed with nothing in it — is
|
|
a failure of that feed rather than a shrug about a slow news day.
|
|
"""
|
|
response = http.get(url, headers={"User-Agent": USER_AGENT})
|
|
response.raise_for_status()
|
|
# The bytes rather than `.text`, so the document's own XML declaration
|
|
# decides its encoding instead of a guess made from a header the outlet
|
|
# may never have set.
|
|
return feedparser.parse(response.content)
|
|
|
|
|
|
def _proposed_outlet(parsed: Any, url: str, label: str) -> str:
|
|
"""What this feed calls itself — but only when you didn't say.
|
|
|
|
A label you wrote is the better name and the check must never talk you out
|
|
of it: "Guardian Tech" splits into an outlet and a section that a feed's
|
|
own `<title>` ("Technology | The Guardian") cannot improve on. A label is
|
|
only ever *this* when you pasted a URL and named nothing, and then the
|
|
alternative is filing your reading under `https://…`, which the feed's
|
|
title beats outright.
|
|
"""
|
|
if label and label != url:
|
|
return ""
|
|
return outlet_name(strip_html(parsed.feed.get("title", "")))
|
|
|
|
|
|
def _age(entries) -> str:
|
|
stamps = [entry_time(e) for e in entries
|
|
if e.get("published_parsed") or e.get("updated_parsed")]
|
|
if not stamps:
|
|
return "no dates"
|
|
hours = (datetime.now(UTC) - max(stamps)).total_seconds() / 3600
|
|
return f"newest {hours:.0f}h ago" if hours < 48 else f"newest {hours / 24:.0f}d ago"
|
|
|
|
|
|
class RssSource(Source):
|
|
"""Feed fetching, for a publisher's plugin to inherit. Not a plugin.
|
|
|
|
`shared` is what says so, and retiring the generic "Any RSS feed" source
|
|
it used to be is §9 of the rework: an outlet is a plugin, so a masthead
|
|
has a file that declares its own feeds, marks and policies rather than a
|
|
row somebody pasted a URL into. Everything below is unchanged — this is
|
|
the same fetching the Guardian, the BBC, Ars Technica, Golem and Hacker
|
|
News have always run on, reached now by subclassing rather than by
|
|
configuring.
|
|
|
|
A subclass states its own `name`; the empty one here is deliberate, so
|
|
that forgetting it is a plugin the registry skips loudly rather than a
|
|
second class claiming the archive of the first.
|
|
"""
|
|
|
|
name = ""
|
|
shared = True
|
|
#: Inherited by a subclass that states none of its own.
|
|
description = "An RSS or Atom feed."
|
|
subscription_field = "feeds"
|
|
|
|
def fetch(self) -> Iterator[Item]:
|
|
feeds = self.options.get("feeds", [])
|
|
failed: list[str] = []
|
|
# One client for the whole pull, so twenty feeds off one host are one
|
|
# connection rather than twenty. Closed on the way out however this
|
|
# generator ends.
|
|
with session(None, timeout=FETCH_TIMEOUT) as http:
|
|
for entry in feeds:
|
|
# The label is the whole of what an item says about where it came
|
|
# from. Which outlet and which section of it that label stands for
|
|
# is the subscription's business, settled once when you added it —
|
|
# see `CheckResult.outlet`, which is how this plugin has its say.
|
|
if isinstance(entry, dict):
|
|
url = entry["url"]
|
|
label = entry.get("label") or url
|
|
else:
|
|
url = label = entry
|
|
try:
|
|
parsed = _read(http, url)
|
|
except (httpx.HTTPError, httpx.InvalidURL) as exc:
|
|
# Everything this feed's own end can do to us: a status, a
|
|
# host that never answers, an address that was mistyped
|
|
# when it was pasted. Noted and carried past, because the
|
|
# next feed in the list has nothing to do with any of it.
|
|
#
|
|
# Nothing broader is caught. What else could land here is a
|
|
# wrong assumption of ours about the shape of a response,
|
|
# and notices.py is explicit that those have to reach you
|
|
# as bugs rather than be folded in with the weather.
|
|
log.warning("could not fetch %s: %s", url, http_failure(exc))
|
|
failed.append(f"{label} ({http_failure(exc)})")
|
|
continue
|
|
if parsed.bozo and not parsed.entries:
|
|
# It answered, and what came back is not a feed. Empty
|
|
# rather than failed: the server did its part, and there is
|
|
# nothing here for anyone to go and fix.
|
|
log.warning("no entries from %s (%s)", url, parsed.get("bozo_exception"))
|
|
continue
|
|
|
|
yield from self._items(parsed, url, label)
|
|
|
|
if failed:
|
|
raise FeedsFailed(
|
|
f"{len(failed)} of {len(feeds)} feeds failed: " + "; ".join(failed))
|
|
|
|
def _items(self, parsed: Any, url: str, label: str) -> Iterator[Item]:
|
|
"""One feed's entries, as rows. Split out of `fetch` only so that the
|
|
loop over your subscriptions stays readable beside what it catches."""
|
|
for e in parsed.entries[: self.options.get("max_items", 60)]:
|
|
yield self.item_from_entry(e, url, label, parsed.feed.get("title", ""))
|
|
|
|
def item_from_entry(self, entry: Any, url: str, label: str, author: str = "") -> Item:
|
|
"""Map one parsed entry; outlets may refine fields without owning HTTP."""
|
|
link = entry.get("link") or entry.get("id") or ""
|
|
body = ""
|
|
if entry.get("content"):
|
|
body = strip_html(entry.content[0].get("value", ""))
|
|
body = body or strip_html(entry.get("summary", ""))
|
|
raw: dict[str, Any] = {"feed": url}
|
|
if cats := _categories(entry):
|
|
raw["categories"] = cats[:12]
|
|
return Item(
|
|
source_id=entry.get("id") or link,
|
|
location=link,
|
|
title=strip_html(entry.get("title", "")),
|
|
excerpt=body,
|
|
author=entry.get("author", "") or author,
|
|
published_at=entry_time(entry),
|
|
subscription=label,
|
|
kind="article",
|
|
raw=raw,
|
|
)
|
|
|
|
def check(self) -> list[CheckResult]:
|
|
return [check_feed(url, label) for url, label in self.targets()]
|