The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
326 lines
14 KiB
Python
326 lines
14 KiB
Python
"""The plugin contract.
|
|
|
|
Every source lives in its own file. Drop a .py file into `desk/sources/`
|
|
or into any directory listed under `plugin_paths:` in config.yaml, define a
|
|
subclass of `Source`, and it will be discovered automatically.
|
|
|
|
Minimum viable plugin:
|
|
|
|
from desk.sources import Source, Item
|
|
|
|
class MySource(Source):
|
|
name = "mysource"
|
|
description = "One line shown in the Sources tab."
|
|
|
|
def fetch(self):
|
|
yield Item(
|
|
source_id="unique-per-source",
|
|
location="https://example.com/story",
|
|
title="Headline",
|
|
excerpt="Summary or body",
|
|
author="someone",
|
|
published_at=datetime.now(timezone.utc), # aware; stored as UTC
|
|
)
|
|
|
|
`self.options` is whatever you put under `sources.mysource:` in the config —
|
|
by the time a pull runs, its subscription list has been replaced with the
|
|
fetch-enabled rows from the Sources tab (see newsdesk/jobs/subscriptions.py).
|
|
|
|
Two optional methods round out the contract, both with do-nothing defaults:
|
|
`discover` proposes accounts to follow, and `check` says whether each
|
|
subscription still resolves. Neither is required for a source to be useful.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from abc import ABC, abstractmethod
|
|
from collections.abc import Iterator, MutableMapping
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime
|
|
from typing import Any, ClassVar
|
|
|
|
from desk import __version__
|
|
from desk.models import Item
|
|
from desk.tools.scrape import Getter
|
|
|
|
#: Sent by every plugin that talks HTTP, so an outlet reading its logs can see
|
|
#: who this is. (`desk/tools/framing.py` deliberately sends a browser string
|
|
#: instead — it is asking what a browser would be told, so it has to ask as
|
|
#: one.)
|
|
USER_AGENT = f"desk/{__version__} (personal aggregator)"
|
|
|
|
_HTML_TAGS = re.compile(r"<[^>]+>")
|
|
_WHITESPACE = re.compile(r"\s+")
|
|
#: Tidying up after the line above — see `strip_html`. Deliberately only the
|
|
#: punctuation that always closes: an apostrophe or a dash can legitimately
|
|
#: have a space in front of it.
|
|
_SPACE_BEFORE_PUNCT = re.compile(r" +([.,;:!?%)\]}])")
|
|
_SPACE_AFTER_OPENING = re.compile(r"([(\[{]) +")
|
|
|
|
|
|
def parse_timestamp(value: str) -> datetime:
|
|
"""An ISO-8601 stamp from a social API, as an aware datetime, or now.
|
|
|
|
Every one of these APIs writes UTC as a trailing `Z`, which
|
|
`datetime.fromisoformat` did not accept before 3.11 and which is cheaper to
|
|
translate than to special-case. A missing or malformed stamp is not worth
|
|
dropping a post over: `now` sorts it to the top of the queue, where you
|
|
will see it and can judge it yourself.
|
|
|
|
A stamp with no offset at all is read as UTC, and that is a decision this
|
|
helper is entitled to make where `Item` is not. Here we know whose stamp it
|
|
is — an API that documents UTC and then omits the `Z` on some of its rows —
|
|
whereas the model is handed a bare datetime by anything at all, and a wrong
|
|
guess there is an item filed hours out with nothing on the row to show it.
|
|
A plugin whose API sends local time should not be reaching for this.
|
|
"""
|
|
try:
|
|
parsed = datetime.fromisoformat((value or "").replace("Z", "+00:00"))
|
|
except (ValueError, AttributeError):
|
|
return datetime.now(UTC)
|
|
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
|
|
|
|
|
def entry_time(entry: Any) -> datetime:
|
|
"""When a feedparser entry was published, or now if it doesn't say.
|
|
|
|
feedparser hands back a 9-tuple in UTC under `published_parsed`, falling
|
|
back to `updated_parsed`. Three modules used to do this slice on their
|
|
own; the tuple is sliced to six here, once.
|
|
"""
|
|
for key in ("published_parsed", "updated_parsed"):
|
|
parsed = entry.get(key)
|
|
if parsed:
|
|
return datetime(*parsed[:6], tzinfo=UTC) # type: ignore[misc] # 9-tuple, sliced to 6
|
|
return datetime.now(UTC)
|
|
|
|
|
|
def http_failure(exc: Exception) -> str:
|
|
"""One line for a failed request, for a check result: the status when the
|
|
server answered, the exception's name when it didn't."""
|
|
response = getattr(exc, "response", None)
|
|
if response is not None:
|
|
return f"HTTP {response.status_code}"
|
|
return f"unreachable: {type(exc).__name__}"
|
|
|
|
|
|
def strip_html(html: str) -> str:
|
|
"""Tags out, whitespace collapsed. Feeds and Mastodon both hand us HTML.
|
|
|
|
A tag becomes a *space* rather than nothing, because `</p><p>` has to stop
|
|
the last word of one paragraph running into the first word of the next.
|
|
That is right for block tags and wrong for inline ones — `a <b>link</b>.`
|
|
would come out as "a link ." — so the space is taken back off where it
|
|
landed against punctuation. It reads better, and the ranker and the tagger
|
|
are both looking at these strings.
|
|
"""
|
|
text = _WHITESPACE.sub(" ", _HTML_TAGS.sub(" ", html or "")).strip()
|
|
return _SPACE_BEFORE_PUNCT.sub(r"\1", _SPACE_AFTER_OPENING.sub(r"\1", text))
|
|
|
|
|
|
#: Outlets whose name is more than one word, so "Al Jazeera Sport" doesn't
|
|
#: become outlet "Al". Anything not listed splits on the first space, which is
|
|
#: right for "Guardian Tech" and "BBC World". Set `outlet:`/`section:` on the
|
|
#: feed in config.yaml when you want neither guess.
|
|
MULTIWORD_OUTLETS = (
|
|
"Al Jazeera", "Hacker News", "Ars Technica", "Washington Post",
|
|
"Financial Times", "New York Times", "Wall Street Journal",
|
|
"Der Spiegel", "Le Monde", "Foreign Policy", "MIT Technology Review",
|
|
)
|
|
|
|
|
|
def split_label(label: str) -> tuple[str, str]:
|
|
"""'Guardian Tech' -> ('Guardian', 'Tech'). Fallback when config is silent.
|
|
|
|
A reading of a feed's *label*, which is why it is part of the source
|
|
contract rather than of any store: the RSS plugin applies it to a feed it
|
|
was given with no outlet, and an app applies it to a subscription seeded
|
|
the same way. One rule, and the plugin side owns it.
|
|
"""
|
|
label = (label or "").strip()
|
|
if not label:
|
|
return "", ""
|
|
for outlet in MULTIWORD_OUTLETS:
|
|
if label == outlet:
|
|
return outlet, ""
|
|
if label.startswith(outlet + " "):
|
|
return outlet, label[len(outlet):].strip()
|
|
head, _, rest = label.partition(" ")
|
|
return head, rest.strip()
|
|
|
|
|
|
#: What sits between a site's name and its tagline, for `outlet_name`. Spelled
|
|
#: with escapes because a literal en dash next to a hyphen inside a character
|
|
#: class is unreadable, and ruff is right to say so.
|
|
_TAGLINE = re.compile("\\s+[-\u2013\u2014|:\u00b7]\\s+")
|
|
|
|
|
|
def outlet_name(raw: str) -> str:
|
|
"""A name a site gave for itself, cut back to something a sidebar can hold.
|
|
|
|
Both places that ask a server what it is called get the SEO title back,
|
|
tagline and all: WordPress keeps ODSC's as "Open Data Science - Your News
|
|
Source for AI, Machine Learning & more", and a feed's `<title>` is often
|
|
"Ars Technica - All content". Everything after the first separator is the
|
|
tagline, and the name is the part in front of it.
|
|
|
|
A reading of what a *server* said, which is why it is part of the source
|
|
contract next to `split_label` — that one reads a label you wrote, this
|
|
one reads a string you never saw.
|
|
"""
|
|
return _TAGLINE.split((raw or "").strip(), 1)[0].strip() or (raw or "").strip()
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class Suggestion:
|
|
"""An account you don't follow yet, that the graph thinks you should."""
|
|
|
|
handle: str
|
|
display_name: str = ""
|
|
reason: str = ""
|
|
score: float = 0.0
|
|
url: str = ""
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class CheckResult:
|
|
"""Whether one subscription still resolves, and what the server said it is.
|
|
|
|
`target` is the feed URL or the handle — what the subscription row holds —
|
|
and `label` is what *you* call it, so the Jobs view can name a failing row.
|
|
|
|
`outlet` and `section` are the other half: what the server calls itself.
|
|
They are how a plugin that learns a publisher's real name from the network
|
|
hands it over — WordPress reads it off `/wp-json`, a feed off its own
|
|
`<title>` — and they are written onto the subscription, once, by
|
|
`name_grouping`. Empty from every plugin that has nothing to add, which is
|
|
most of them: an outlet is normally known from the label before anything
|
|
is fetched.
|
|
|
|
Deliberately *not* `label`. That field already carries the subscription's
|
|
own name in every plugin and in the Jobs view's failing-row line, and a
|
|
publisher's name is a different fact about a different thing.
|
|
"""
|
|
|
|
kind: str
|
|
target: str
|
|
ok: bool
|
|
detail: str
|
|
label: str = ""
|
|
outlet: str = ""
|
|
section: str = ""
|
|
|
|
|
|
class Source(ABC):
|
|
"""Base class for every ingest plugin."""
|
|
|
|
#: config key under `sources:` in config.yaml. Required, must be unique.
|
|
name: str = ""
|
|
#: True for a class that exists to be subclassed rather than to be run:
|
|
#: `RssSource` is the fetching every publisher's plugin inherits, not a
|
|
#: plugin of its own. The registry passes over one, so there is no
|
|
#: "Any RSS feed" row to paste a masthead into — see `desk.sources.rss`.
|
|
#:
|
|
#: Read off the class's own `__dict__` and never inherited, or declaring
|
|
#: it once would take every plugin that subclasses that class with it.
|
|
shared: ClassVar[bool] = False
|
|
#: shown in the Sources tab, under Plugins
|
|
description: str = ""
|
|
#: Which key under `sources.<name>:` holds this plugin's subscriptions, and
|
|
#: therefore whether one is a URL or a handle. Declared here rather than
|
|
#: inferred, because everything downstream needs the answer: what
|
|
#: `subscriptions.apply` puts back into the config for a pull, what the Add
|
|
#: box in the Sources tab asks for, and what the UI validates.
|
|
#:
|
|
#: This used to be `name == "rss"` spelled out at each of those sites,
|
|
#: which meant the second URL-shaped plugin silently had its feed list
|
|
#: replaced by an empty account list and fetched nothing.
|
|
subscription_field: ClassVar[str] = "accounts" # "accounts" or "feeds"
|
|
#: The outlet every subscription of this plugin sits under, when the plugin
|
|
#: is a network rather than a publisher — a Bluesky handle is a channel of
|
|
#: Bluesky, not a masthead of its own. Written onto a subscription when it
|
|
#: is added, and only over a name nobody chose; `check` overrides it for a
|
|
#: plugin that can ask the network what a subscription is really called.
|
|
#: Empty means "read it off the label", which is right for a feed.
|
|
default_outlet: str = ""
|
|
#: The addresses this plugin knows how to read better than a web page, and
|
|
#: how: a `Getter` whose `handles` matches them and whose `claims` puts it
|
|
#: ahead of the plain request. The body fetch collects these from every
|
|
#: active plugin and hands them to `desk.tools.scrape`, so a Hacker News
|
|
#: link to a Bluesky post is read through Bluesky's API while the Bluesky
|
|
#: plugin is on, and as a web page while it is off. A getter only gets the
|
|
#: page: which link an item follows is decided when the plugin makes it.
|
|
getters: ClassVar[tuple[Getter, ...]] = ()
|
|
|
|
def __init__(self, options: dict[str, Any] | None = None) -> None:
|
|
self.options = options or {}
|
|
#: What this plugin remembers between pulls — a spend counter, which
|
|
#: episodes are already done — as a plain mapping of strings. A pull
|
|
#: hands in one backed by the database; driven directly (a test, the
|
|
#: dry-run `main` in a plugin file) it is this dict and lives as long
|
|
#: as the instance. It is the only thing a plugin is given by the
|
|
#: application, which is what keeps a plugin one file.
|
|
self.state: MutableMapping[str, str] = {}
|
|
|
|
@property
|
|
def enabled(self) -> bool:
|
|
return bool(self.options.get("enabled", True))
|
|
|
|
@abstractmethod
|
|
def fetch(self) -> Iterator[Item]:
|
|
"""Yield everything new. Runner handles dedup, so over-fetching is fine."""
|
|
raise NotImplementedError
|
|
|
|
def discover(self) -> list[Suggestion]:
|
|
"""Optional: propose accounts to follow. Return [] if unsupported."""
|
|
return []
|
|
|
|
def check(self) -> list[CheckResult]:
|
|
"""Optional: does each subscription still resolve, and what is it called?
|
|
|
|
One result per feed or account, and never a paid request — the `check`
|
|
job runs this for every source at once. Return [] if there is nothing
|
|
to ask.
|
|
|
|
This is also the naming channel. A plugin that can only learn a
|
|
publisher's real name by asking the network says so here, in the
|
|
result's `outlet`/`section`, and the app writes it onto the
|
|
subscription. It is asked of one target at a time when you add a
|
|
subscription — build the source with that one entry in its options —
|
|
so a URL you paste comes back named.
|
|
"""
|
|
return []
|
|
|
|
def params_for(self, target: str) -> dict[str, Any]:
|
|
"""What this subscription asks for, when it is a query and not an address.
|
|
|
|
A feed URL and a handle need nothing; a search takes keywords, and
|
|
maybe a location or a floor. Those live on the subscription, not in the
|
|
plugin's own settings, because they are what distinguishes one of your
|
|
saved searches from the next — and so that adding one is the same act
|
|
as adding a feed, with the same switches on it afterwards.
|
|
"""
|
|
for entry in self.options.get(self.subscription_field) or []:
|
|
if isinstance(entry, dict):
|
|
this = str(entry.get("url") or entry.get("handle") or "")
|
|
if this == target:
|
|
got = entry.get("params")
|
|
return dict(got) if isinstance(got, dict) else {}
|
|
return {}
|
|
|
|
def targets(self) -> list[tuple[str, str]]:
|
|
"""The subscriptions this instance was configured with, as
|
|
`(target, label)` pairs — a URL or a handle, and what it is called.
|
|
A bare string in the list is its own label."""
|
|
out = []
|
|
for entry in self.options.get(self.subscription_field) or []:
|
|
if isinstance(entry, dict):
|
|
target = str(entry.get("url") or entry.get("handle") or "")
|
|
label = str(entry.get("label") or target)
|
|
else:
|
|
target = label = str(entry)
|
|
if target:
|
|
out.append((target, label))
|
|
return out
|