Files
jobboard/desk/sources/rss.py
T
WolfandClaude Opus 5.5 248ea1636d Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:17 +02:00

237 lines
11 KiB
Python

"""RSS and Atom feed fetching, as a base class for a publisher's plugin.
Not itself a source: see `RssSource`, which is what a masthead subclasses.
`check_feed` is the same fetching asked whether one address still answers.
"""
from __future__ import annotations
import logging
from collections.abc import Iterator
from datetime import UTC, datetime
from typing import Any
import feedparser
import httpx
from desk.sources.base import (
USER_AGENT, CheckResult, Item, Source, entry_time, http_failure, outlet_name,
strip_html,
)
from desk.tools.http import session
log = logging.getLogger(__name__)
#: Seconds one feed gets, per operation. Which number it is matters far less
#: than there being one at all — see `_read`.
FETCH_TIMEOUT = 30
class FeedsFailed(Exception):
"""Some of a pull's feeds could not be fetched, and the rest were.
Raised at the end of `fetch` rather than where it happened, and that is the
whole of the design: everything the live feeds gave has already been
yielded and stored by the time this lands, so one outlet returning 403
costs you that outlet and not the other nineteen. The runner turns it into
`notices.source_failed`, which is what puts a feed that has been dead all
week on screen — the one failure a reader cannot spot by eye, because a
dead source and a quiet news day look identical.
Worded through `http_failure`, so every line in it reads `HTTP 403` or
`unreachable: ReadTimeout`. That is also what has the notice file it as the
outlet's business rather than as a defect of ours; see
`desk/notices.py`.
"""
def _categories(entry) -> list[str]:
"""Whatever the outlet put in <category>.
The Guardian, Ars Technica, the NYT and Al Jazeera all populate these;
the BBC and Hacker News don't. They're inconsistent enough between outlets
to be useless as a filter — the NYT files people ('Putin, Vladimir V'),
the Guardian files desks ('Television & radio') — so they're carried as a
hint for the tagger rather than shown to you.
"""
seen, out = set(), []
for tag in entry.get("tags") or []:
term = strip_html(tag.get("term", ""))
if term and term.lower() not in seen:
seen.add(term.lower())
out.append(term)
return out
def check_feed(url: str, label: str = "", kind: str = "rss") -> CheckResult:
"""Fetch one feed and say what is in it. Shared with the podcast source,
whose subscription is a feed too."""
try:
r = httpx.get(url, headers={"User-Agent": USER_AGENT}, timeout=20,
follow_redirects=True)
except httpx.HTTPError as exc:
return CheckResult(kind, url, False, f"unreachable: {type(exc).__name__}", label)
if r.status_code != 200:
return CheckResult(kind, url, False, f"HTTP {r.status_code}", label)
parsed = feedparser.parse(r.content)
n = len(parsed.entries)
if n == 0:
reason = parsed.get("bozo_exception")
return CheckResult(kind, url, False, f"0 entries ({reason or 'not a feed?'})", label)
return CheckResult(kind, url, True, f"{n} entries, {_age(parsed.entries)}", label,
outlet=_proposed_outlet(parsed, url, label))
def _read(http: httpx.Client, url: str) -> Any:
"""One feed, fetched here and parsed from the bytes it answered with.
Deliberately not `feedparser.parse(url)`, which is the same call with the
fetching left to somebody else and which goes through urllib with **no
timeout**. A
host that accepts the connection and then says nothing holds the thread for
ever, and the thread it holds is a worker's: the pull is `solo`, so nothing
else pulls behind it; cancellation is cooperative, so Kill cannot reach a
thread parked in a socket read; and a restart releases the lease only to
re-queue the job and stall it on the same feed. One bad feed stopped the
application from pulling at all until somebody edited the subscription.
feedparser offers no way to pass urllib a deadline, so the fetch comes
here, where the client has one.
Two things fall out of doing it here, both of which `check_feed` has had
all along. The request goes out as `USER_AGENT`, which `base.py` promises
every plugin sends and which feedparser was quietly replacing with its own.
And the status is read, so a 403, 410 or 429 — each of which arrives with
an empty body that parses perfectly well as a feed with nothing in it — is
a failure of that feed rather than a shrug about a slow news day.
"""
response = http.get(url, headers={"User-Agent": USER_AGENT})
response.raise_for_status()
# The bytes rather than `.text`, so the document's own XML declaration
# decides its encoding instead of a guess made from a header the outlet
# may never have set.
return feedparser.parse(response.content)
def _proposed_outlet(parsed: Any, url: str, label: str) -> str:
"""What this feed calls itself — but only when you didn't say.
A label you wrote is the better name and the check must never talk you out
of it: "Guardian Tech" splits into an outlet and a section that a feed's
own `<title>` ("Technology | The Guardian") cannot improve on. A label is
only ever *this* when you pasted a URL and named nothing, and then the
alternative is filing your reading under `https://…`, which the feed's
title beats outright.
"""
if label and label != url:
return ""
return outlet_name(strip_html(parsed.feed.get("title", "")))
def _age(entries) -> str:
stamps = [entry_time(e) for e in entries
if e.get("published_parsed") or e.get("updated_parsed")]
if not stamps:
return "no dates"
hours = (datetime.now(UTC) - max(stamps)).total_seconds() / 3600
return f"newest {hours:.0f}h ago" if hours < 48 else f"newest {hours / 24:.0f}d ago"
class RssSource(Source):
"""Feed fetching, for a publisher's plugin to inherit. Not a plugin.
`shared` is what says so, and retiring the generic "Any RSS feed" source
it used to be is §9 of the rework: an outlet is a plugin, so a masthead
has a file that declares its own feeds, marks and policies rather than a
row somebody pasted a URL into. Everything below is unchanged — this is
the same fetching the Guardian, the BBC, Ars Technica, Golem and Hacker
News have always run on, reached now by subclassing rather than by
configuring.
A subclass states its own `name`; the empty one here is deliberate, so
that forgetting it is a plugin the registry skips loudly rather than a
second class claiming the archive of the first.
"""
name = ""
shared = True
#: Inherited by a subclass that states none of its own.
description = "An RSS or Atom feed."
subscription_field = "feeds"
def fetch(self) -> Iterator[Item]:
feeds = self.options.get("feeds", [])
failed: list[str] = []
# One client for the whole pull, so twenty feeds off one host are one
# connection rather than twenty. Closed on the way out however this
# generator ends.
with session(None, timeout=FETCH_TIMEOUT) as http:
for entry in feeds:
# The label is the whole of what an item says about where it came
# from. Which outlet and which section of it that label stands for
# is the subscription's business, settled once when you added it —
# see `CheckResult.outlet`, which is how this plugin has its say.
if isinstance(entry, dict):
url = entry["url"]
label = entry.get("label") or url
else:
url = label = entry
try:
parsed = _read(http, url)
except (httpx.HTTPError, httpx.InvalidURL) as exc:
# Everything this feed's own end can do to us: a status, a
# host that never answers, an address that was mistyped
# when it was pasted. Noted and carried past, because the
# next feed in the list has nothing to do with any of it.
#
# Nothing broader is caught. What else could land here is a
# wrong assumption of ours about the shape of a response,
# and notices.py is explicit that those have to reach you
# as bugs rather than be folded in with the weather.
log.warning("could not fetch %s: %s", url, http_failure(exc))
failed.append(f"{label} ({http_failure(exc)})")
continue
if parsed.bozo and not parsed.entries:
# It answered, and what came back is not a feed. Empty
# rather than failed: the server did its part, and there is
# nothing here for anyone to go and fix.
log.warning("no entries from %s (%s)", url, parsed.get("bozo_exception"))
continue
yield from self._items(parsed, url, label)
if failed:
raise FeedsFailed(
f"{len(failed)} of {len(feeds)} feeds failed: " + "; ".join(failed))
def _items(self, parsed: Any, url: str, label: str) -> Iterator[Item]:
"""One feed's entries, as rows. Split out of `fetch` only so that the
loop over your subscriptions stays readable beside what it catches."""
for e in parsed.entries[: self.options.get("max_items", 60)]:
yield self.item_from_entry(e, url, label, parsed.feed.get("title", ""))
def item_from_entry(self, entry: Any, url: str, label: str, author: str = "") -> Item:
"""Map one parsed entry; outlets may refine fields without owning HTTP."""
link = entry.get("link") or entry.get("id") or ""
body = ""
if entry.get("content"):
body = strip_html(entry.content[0].get("value", ""))
body = body or strip_html(entry.get("summary", ""))
raw: dict[str, Any] = {"feed": url}
if cats := _categories(entry):
raw["categories"] = cats[:12]
return Item(
source_id=entry.get("id") or link,
location=link,
title=strip_html(entry.get("title", "")),
excerpt=body,
author=entry.get("author", "") or author,
published_at=entry_time(entry),
subscription=label,
kind="article",
raw=raw,
)
def check(self) -> list[CheckResult]:
return [check_feed(url, label) for url, label in self.targets()]