Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d666d0937
commit
248ea1636d
36 files changed
+8393
No files matched your search
@@ -0,0 +1,236 @@
|
||||
"""RSS and Atom feed fetching, as a base class for a publisher's plugin.
|
||||
|
||||
Not itself a source: see `RssSource`, which is what a masthead subclasses.
|
||||
`check_feed` is the same fetching asked whether one address still answers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
import feedparser
|
||||
import httpx
|
||||
|
||||
from desk.sources.base import (
|
||||
USER_AGENT, CheckResult, Item, Source, entry_time, http_failure, outlet_name,
|
||||
strip_html,
|
||||
)
|
||||
from desk.tools.http import session
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
#: Seconds one feed gets, per operation. Which number it is matters far less
|
||||
#: than there being one at all — see `_read`.
|
||||
FETCH_TIMEOUT = 30
|
||||
|
||||
|
||||
class FeedsFailed(Exception):
|
||||
"""Some of a pull's feeds could not be fetched, and the rest were.
|
||||
|
||||
Raised at the end of `fetch` rather than where it happened, and that is the
|
||||
whole of the design: everything the live feeds gave has already been
|
||||
yielded and stored by the time this lands, so one outlet returning 403
|
||||
costs you that outlet and not the other nineteen. The runner turns it into
|
||||
`notices.source_failed`, which is what puts a feed that has been dead all
|
||||
week on screen — the one failure a reader cannot spot by eye, because a
|
||||
dead source and a quiet news day look identical.
|
||||
|
||||
Worded through `http_failure`, so every line in it reads `HTTP 403` or
|
||||
`unreachable: ReadTimeout`. That is also what has the notice file it as the
|
||||
outlet's business rather than as a defect of ours; see
|
||||
`desk/notices.py`.
|
||||
"""
|
||||
|
||||
|
||||
def _categories(entry) -> list[str]:
|
||||
"""Whatever the outlet put in <category>.
|
||||
|
||||
The Guardian, Ars Technica, the NYT and Al Jazeera all populate these;
|
||||
the BBC and Hacker News don't. They're inconsistent enough between outlets
|
||||
to be useless as a filter — the NYT files people ('Putin, Vladimir V'),
|
||||
the Guardian files desks ('Television & radio') — so they're carried as a
|
||||
hint for the tagger rather than shown to you.
|
||||
"""
|
||||
seen, out = set(), []
|
||||
for tag in entry.get("tags") or []:
|
||||
term = strip_html(tag.get("term", ""))
|
||||
if term and term.lower() not in seen:
|
||||
seen.add(term.lower())
|
||||
out.append(term)
|
||||
return out
|
||||
|
||||
|
||||
def check_feed(url: str, label: str = "", kind: str = "rss") -> CheckResult:
|
||||
"""Fetch one feed and say what is in it. Shared with the podcast source,
|
||||
whose subscription is a feed too."""
|
||||
try:
|
||||
r = httpx.get(url, headers={"User-Agent": USER_AGENT}, timeout=20,
|
||||
follow_redirects=True)
|
||||
except httpx.HTTPError as exc:
|
||||
return CheckResult(kind, url, False, f"unreachable: {type(exc).__name__}", label)
|
||||
if r.status_code != 200:
|
||||
return CheckResult(kind, url, False, f"HTTP {r.status_code}", label)
|
||||
|
||||
parsed = feedparser.parse(r.content)
|
||||
n = len(parsed.entries)
|
||||
if n == 0:
|
||||
reason = parsed.get("bozo_exception")
|
||||
return CheckResult(kind, url, False, f"0 entries ({reason or 'not a feed?'})", label)
|
||||
return CheckResult(kind, url, True, f"{n} entries, {_age(parsed.entries)}", label,
|
||||
outlet=_proposed_outlet(parsed, url, label))
|
||||
|
||||
|
||||
def _read(http: httpx.Client, url: str) -> Any:
|
||||
"""One feed, fetched here and parsed from the bytes it answered with.
|
||||
|
||||
Deliberately not `feedparser.parse(url)`, which is the same call with the
|
||||
fetching left to somebody else and which goes through urllib with **no
|
||||
timeout**. A
|
||||
host that accepts the connection and then says nothing holds the thread for
|
||||
ever, and the thread it holds is a worker's: the pull is `solo`, so nothing
|
||||
else pulls behind it; cancellation is cooperative, so Kill cannot reach a
|
||||
thread parked in a socket read; and a restart releases the lease only to
|
||||
re-queue the job and stall it on the same feed. One bad feed stopped the
|
||||
application from pulling at all until somebody edited the subscription.
|
||||
feedparser offers no way to pass urllib a deadline, so the fetch comes
|
||||
here, where the client has one.
|
||||
|
||||
Two things fall out of doing it here, both of which `check_feed` has had
|
||||
all along. The request goes out as `USER_AGENT`, which `base.py` promises
|
||||
every plugin sends and which feedparser was quietly replacing with its own.
|
||||
And the status is read, so a 403, 410 or 429 — each of which arrives with
|
||||
an empty body that parses perfectly well as a feed with nothing in it — is
|
||||
a failure of that feed rather than a shrug about a slow news day.
|
||||
"""
|
||||
response = http.get(url, headers={"User-Agent": USER_AGENT})
|
||||
response.raise_for_status()
|
||||
# The bytes rather than `.text`, so the document's own XML declaration
|
||||
# decides its encoding instead of a guess made from a header the outlet
|
||||
# may never have set.
|
||||
return feedparser.parse(response.content)
|
||||
|
||||
|
||||
def _proposed_outlet(parsed: Any, url: str, label: str) -> str:
|
||||
"""What this feed calls itself — but only when you didn't say.
|
||||
|
||||
A label you wrote is the better name and the check must never talk you out
|
||||
of it: "Guardian Tech" splits into an outlet and a section that a feed's
|
||||
own `<title>` ("Technology | The Guardian") cannot improve on. A label is
|
||||
only ever *this* when you pasted a URL and named nothing, and then the
|
||||
alternative is filing your reading under `https://…`, which the feed's
|
||||
title beats outright.
|
||||
"""
|
||||
if label and label != url:
|
||||
return ""
|
||||
return outlet_name(strip_html(parsed.feed.get("title", "")))
|
||||
|
||||
|
||||
def _age(entries) -> str:
|
||||
stamps = [entry_time(e) for e in entries
|
||||
if e.get("published_parsed") or e.get("updated_parsed")]
|
||||
if not stamps:
|
||||
return "no dates"
|
||||
hours = (datetime.now(UTC) - max(stamps)).total_seconds() / 3600
|
||||
return f"newest {hours:.0f}h ago" if hours < 48 else f"newest {hours / 24:.0f}d ago"
|
||||
|
||||
|
||||
class RssSource(Source):
|
||||
"""Feed fetching, for a publisher's plugin to inherit. Not a plugin.
|
||||
|
||||
`shared` is what says so, and retiring the generic "Any RSS feed" source
|
||||
it used to be is §9 of the rework: an outlet is a plugin, so a masthead
|
||||
has a file that declares its own feeds, marks and policies rather than a
|
||||
row somebody pasted a URL into. Everything below is unchanged — this is
|
||||
the same fetching the Guardian, the BBC, Ars Technica, Golem and Hacker
|
||||
News have always run on, reached now by subclassing rather than by
|
||||
configuring.
|
||||
|
||||
A subclass states its own `name`; the empty one here is deliberate, so
|
||||
that forgetting it is a plugin the registry skips loudly rather than a
|
||||
second class claiming the archive of the first.
|
||||
"""
|
||||
|
||||
name = ""
|
||||
shared = True
|
||||
#: Inherited by a subclass that states none of its own.
|
||||
description = "An RSS or Atom feed."
|
||||
subscription_field = "feeds"
|
||||
|
||||
def fetch(self) -> Iterator[Item]:
|
||||
feeds = self.options.get("feeds", [])
|
||||
failed: list[str] = []
|
||||
# One client for the whole pull, so twenty feeds off one host are one
|
||||
# connection rather than twenty. Closed on the way out however this
|
||||
# generator ends.
|
||||
with session(None, timeout=FETCH_TIMEOUT) as http:
|
||||
for entry in feeds:
|
||||
# The label is the whole of what an item says about where it came
|
||||
# from. Which outlet and which section of it that label stands for
|
||||
# is the subscription's business, settled once when you added it —
|
||||
# see `CheckResult.outlet`, which is how this plugin has its say.
|
||||
if isinstance(entry, dict):
|
||||
url = entry["url"]
|
||||
label = entry.get("label") or url
|
||||
else:
|
||||
url = label = entry
|
||||
try:
|
||||
parsed = _read(http, url)
|
||||
except (httpx.HTTPError, httpx.InvalidURL) as exc:
|
||||
# Everything this feed's own end can do to us: a status, a
|
||||
# host that never answers, an address that was mistyped
|
||||
# when it was pasted. Noted and carried past, because the
|
||||
# next feed in the list has nothing to do with any of it.
|
||||
#
|
||||
# Nothing broader is caught. What else could land here is a
|
||||
# wrong assumption of ours about the shape of a response,
|
||||
# and notices.py is explicit that those have to reach you
|
||||
# as bugs rather than be folded in with the weather.
|
||||
log.warning("could not fetch %s: %s", url, http_failure(exc))
|
||||
failed.append(f"{label} ({http_failure(exc)})")
|
||||
continue
|
||||
if parsed.bozo and not parsed.entries:
|
||||
# It answered, and what came back is not a feed. Empty
|
||||
# rather than failed: the server did its part, and there is
|
||||
# nothing here for anyone to go and fix.
|
||||
log.warning("no entries from %s (%s)", url, parsed.get("bozo_exception"))
|
||||
continue
|
||||
|
||||
yield from self._items(parsed, url, label)
|
||||
|
||||
if failed:
|
||||
raise FeedsFailed(
|
||||
f"{len(failed)} of {len(feeds)} feeds failed: " + "; ".join(failed))
|
||||
|
||||
def _items(self, parsed: Any, url: str, label: str) -> Iterator[Item]:
|
||||
"""One feed's entries, as rows. Split out of `fetch` only so that the
|
||||
loop over your subscriptions stays readable beside what it catches."""
|
||||
for e in parsed.entries[: self.options.get("max_items", 60)]:
|
||||
yield self.item_from_entry(e, url, label, parsed.feed.get("title", ""))
|
||||
|
||||
def item_from_entry(self, entry: Any, url: str, label: str, author: str = "") -> Item:
|
||||
"""Map one parsed entry; outlets may refine fields without owning HTTP."""
|
||||
link = entry.get("link") or entry.get("id") or ""
|
||||
body = ""
|
||||
if entry.get("content"):
|
||||
body = strip_html(entry.content[0].get("value", ""))
|
||||
body = body or strip_html(entry.get("summary", ""))
|
||||
raw: dict[str, Any] = {"feed": url}
|
||||
if cats := _categories(entry):
|
||||
raw["categories"] = cats[:12]
|
||||
return Item(
|
||||
source_id=entry.get("id") or link,
|
||||
location=link,
|
||||
title=strip_html(entry.get("title", "")),
|
||||
excerpt=body,
|
||||
author=entry.get("author", "") or author,
|
||||
published_at=entry_time(entry),
|
||||
subscription=label,
|
||||
kind="article",
|
||||
raw=raw,
|
||||
)
|
||||
|
||||
def check(self) -> list[CheckResult]:
|
||||
return [check_feed(url, label) for url, label in self.targets()]
|
||||
Reference in new issue
Block a user