"""RSS and Atom feed fetching, as a base class for a publisher's plugin. Not itself a source: see `RssSource`, which is what a masthead subclasses. `check_feed` is the same fetching asked whether one address still answers. """ from __future__ import annotations import logging from collections.abc import Iterator from datetime import UTC, datetime from typing import Any import feedparser import httpx from desk.sources.base import ( USER_AGENT, CheckResult, Item, Source, entry_time, http_failure, outlet_name, strip_html, ) from desk.tools.http import session log = logging.getLogger(__name__) #: Seconds one feed gets, per operation. Which number it is matters far less #: than there being one at all — see `_read`. FETCH_TIMEOUT = 30 class FeedsFailed(Exception): """Some of a pull's feeds could not be fetched, and the rest were. Raised at the end of `fetch` rather than where it happened, and that is the whole of the design: everything the live feeds gave has already been yielded and stored by the time this lands, so one outlet returning 403 costs you that outlet and not the other nineteen. The runner turns it into `notices.source_failed`, which is what puts a feed that has been dead all week on screen — the one failure a reader cannot spot by eye, because a dead source and a quiet news day look identical. Worded through `http_failure`, so every line in it reads `HTTP 403` or `unreachable: ReadTimeout`. That is also what has the notice file it as the outlet's business rather than as a defect of ours; see `desk/notices.py`. """ def _categories(entry) -> list[str]: """Whatever the outlet put in . The Guardian, Ars Technica, the NYT and Al Jazeera all populate these; the BBC and Hacker News don't. They're inconsistent enough between outlets to be useless as a filter — the NYT files people ('Putin, Vladimir V'), the Guardian files desks ('Television & radio') — so they're carried as a hint for the tagger rather than shown to you. """ seen, out = set(), [] for tag in entry.get("tags") or []: term = strip_html(tag.get("term", "")) if term and term.lower() not in seen: seen.add(term.lower()) out.append(term) return out def check_feed(url: str, label: str = "", kind: str = "rss") -> CheckResult: """Fetch one feed and say what is in it. Shared with the podcast source, whose subscription is a feed too.""" try: r = httpx.get(url, headers={"User-Agent": USER_AGENT}, timeout=20, follow_redirects=True) except httpx.HTTPError as exc: return CheckResult(kind, url, False, f"unreachable: {type(exc).__name__}", label) if r.status_code != 200: return CheckResult(kind, url, False, f"HTTP {r.status_code}", label) parsed = feedparser.parse(r.content) n = len(parsed.entries) if n == 0: reason = parsed.get("bozo_exception") return CheckResult(kind, url, False, f"0 entries ({reason or 'not a feed?'})", label) return CheckResult(kind, url, True, f"{n} entries, {_age(parsed.entries)}", label, outlet=_proposed_outlet(parsed, url, label)) def _read(http: httpx.Client, url: str) -> Any: """One feed, fetched here and parsed from the bytes it answered with. Deliberately not `feedparser.parse(url)`, which is the same call with the fetching left to somebody else and which goes through urllib with **no timeout**. A host that accepts the connection and then says nothing holds the thread for ever, and the thread it holds is a worker's: the pull is `solo`, so nothing else pulls behind it; cancellation is cooperative, so Kill cannot reach a thread parked in a socket read; and a restart releases the lease only to re-queue the job and stall it on the same feed. One bad feed stopped the application from pulling at all until somebody edited the subscription. feedparser offers no way to pass urllib a deadline, so the fetch comes here, where the client has one. Two things fall out of doing it here, both of which `check_feed` has had all along. The request goes out as `USER_AGENT`, which `base.py` promises every plugin sends and which feedparser was quietly replacing with its own. And the status is read, so a 403, 410 or 429 — each of which arrives with an empty body that parses perfectly well as a feed with nothing in it — is a failure of that feed rather than a shrug about a slow news day. """ response = http.get(url, headers={"User-Agent": USER_AGENT}) response.raise_for_status() # The bytes rather than `.text`, so the document's own XML declaration # decides its encoding instead of a guess made from a header the outlet # may never have set. return feedparser.parse(response.content) def _proposed_outlet(parsed: Any, url: str, label: str) -> str: """What this feed calls itself — but only when you didn't say. A label you wrote is the better name and the check must never talk you out of it: "Guardian Tech" splits into an outlet and a section that a feed's own `` ("Technology | The Guardian") cannot improve on. A label is only ever *this* when you pasted a URL and named nothing, and then the alternative is filing your reading under `https://…`, which the feed's title beats outright. """ if label and label != url: return "" return outlet_name(strip_html(parsed.feed.get("title", ""))) def _age(entries) -> str: stamps = [entry_time(e) for e in entries if e.get("published_parsed") or e.get("updated_parsed")] if not stamps: return "no dates" hours = (datetime.now(UTC) - max(stamps)).total_seconds() / 3600 return f"newest {hours:.0f}h ago" if hours < 48 else f"newest {hours / 24:.0f}d ago" class RssSource(Source): """Feed fetching, for a publisher's plugin to inherit. Not a plugin. `shared` is what says so, and retiring the generic "Any RSS feed" source it used to be is §9 of the rework: an outlet is a plugin, so a masthead has a file that declares its own feeds, marks and policies rather than a row somebody pasted a URL into. Everything below is unchanged — this is the same fetching the Guardian, the BBC, Ars Technica, Golem and Hacker News have always run on, reached now by subclassing rather than by configuring. A subclass states its own `name`; the empty one here is deliberate, so that forgetting it is a plugin the registry skips loudly rather than a second class claiming the archive of the first. """ name = "" shared = True #: Inherited by a subclass that states none of its own. description = "An RSS or Atom feed." subscription_field = "feeds" def fetch(self) -> Iterator[Item]: feeds = self.options.get("feeds", []) failed: list[str] = [] # One client for the whole pull, so twenty feeds off one host are one # connection rather than twenty. Closed on the way out however this # generator ends. with session(None, timeout=FETCH_TIMEOUT) as http: for entry in feeds: # The label is the whole of what an item says about where it came # from. Which outlet and which section of it that label stands for # is the subscription's business, settled once when you added it — # see `CheckResult.outlet`, which is how this plugin has its say. if isinstance(entry, dict): url = entry["url"] label = entry.get("label") or url else: url = label = entry try: parsed = _read(http, url) except (httpx.HTTPError, httpx.InvalidURL) as exc: # Everything this feed's own end can do to us: a status, a # host that never answers, an address that was mistyped # when it was pasted. Noted and carried past, because the # next feed in the list has nothing to do with any of it. # # Nothing broader is caught. What else could land here is a # wrong assumption of ours about the shape of a response, # and notices.py is explicit that those have to reach you # as bugs rather than be folded in with the weather. log.warning("could not fetch %s: %s", url, http_failure(exc)) failed.append(f"{label} ({http_failure(exc)})") continue if parsed.bozo and not parsed.entries: # It answered, and what came back is not a feed. Empty # rather than failed: the server did its part, and there is # nothing here for anyone to go and fix. log.warning("no entries from %s (%s)", url, parsed.get("bozo_exception")) continue yield from self._items(parsed, url, label) if failed: raise FeedsFailed( f"{len(failed)} of {len(feeds)} feeds failed: " + "; ".join(failed)) def _items(self, parsed: Any, url: str, label: str) -> Iterator[Item]: """One feed's entries, as rows. Split out of `fetch` only so that the loop over your subscriptions stays readable beside what it catches.""" for e in parsed.entries[: self.options.get("max_items", 60)]: yield self.item_from_entry(e, url, label, parsed.feed.get("title", "")) def item_from_entry(self, entry: Any, url: str, label: str, author: str = "") -> Item: """Map one parsed entry; outlets may refine fields without owning HTTP.""" link = entry.get("link") or entry.get("id") or "" body = "" if entry.get("content"): body = strip_html(entry.content[0].get("value", "")) body = body or strip_html(entry.get("summary", "")) raw: dict[str, Any] = {"feed": url} if cats := _categories(entry): raw["categories"] = cats[:12] return Item( source_id=entry.get("id") or link, location=link, title=strip_html(entry.get("title", "")), excerpt=body, author=entry.get("author", "") or author, published_at=entry_time(entry), subscription=label, kind="article", raw=raw, ) def check(self) -> list[CheckResult]: return [check_feed(url, label) for url, label in self.targets()]