"""The plugin contract.
Every source lives in its own file. Drop a .py file into `desk/sources/`
or into any directory listed under `plugin_paths:` in config.yaml, define a
subclass of `Source`, and it will be discovered automatically.
Minimum viable plugin:
from desk.sources import Source, Item
class MySource(Source):
name = "mysource"
description = "One line shown in the Sources tab."
def fetch(self):
yield Item(
source_id="unique-per-source",
location="https://example.com/story",
title="Headline",
excerpt="Summary or body",
author="someone",
published_at=datetime.now(timezone.utc), # aware; stored as UTC
)
`self.options` is whatever you put under `sources.mysource:` in the config —
by the time a pull runs, its subscription list has been replaced with the
fetch-enabled rows from the Sources tab (see newsdesk/jobs/subscriptions.py).
Two optional methods round out the contract, both with do-nothing defaults:
`discover` proposes accounts to follow, and `check` says whether each
subscription still resolves. Neither is required for a source to be useful.
"""
from __future__ import annotations
import re
from abc import ABC, abstractmethod
from collections.abc import Iterator, MutableMapping
from dataclasses import dataclass
from datetime import UTC, datetime
from typing import Any, ClassVar
from desk import __version__
from desk.models import Item
from desk.tools.scrape import Getter
#: Sent by every plugin that talks HTTP, so an outlet reading its logs can see
#: who this is. (`desk/tools/framing.py` deliberately sends a browser string
#: instead — it is asking what a browser would be told, so it has to ask as
#: one.)
USER_AGENT = f"desk/{__version__} (personal aggregator)"
_HTML_TAGS = re.compile(r"<[^>]+>")
_WHITESPACE = re.compile(r"\s+")
#: Tidying up after the line above — see `strip_html`. Deliberately only the
#: punctuation that always closes: an apostrophe or a dash can legitimately
#: have a space in front of it.
_SPACE_BEFORE_PUNCT = re.compile(r" +([.,;:!?%)\]}])")
_SPACE_AFTER_OPENING = re.compile(r"([(\[{]) +")
def parse_timestamp(value: str) -> datetime:
"""An ISO-8601 stamp from a social API, as an aware datetime, or now.
Every one of these APIs writes UTC as a trailing `Z`, which
`datetime.fromisoformat` did not accept before 3.11 and which is cheaper to
translate than to special-case. A missing or malformed stamp is not worth
dropping a post over: `now` sorts it to the top of the queue, where you
will see it and can judge it yourself.
A stamp with no offset at all is read as UTC, and that is a decision this
helper is entitled to make where `Item` is not. Here we know whose stamp it
is — an API that documents UTC and then omits the `Z` on some of its rows —
whereas the model is handed a bare datetime by anything at all, and a wrong
guess there is an item filed hours out with nothing on the row to show it.
A plugin whose API sends local time should not be reaching for this.
"""
try:
parsed = datetime.fromisoformat((value or "").replace("Z", "+00:00"))
except (ValueError, AttributeError):
return datetime.now(UTC)
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
def entry_time(entry: Any) -> datetime:
"""When a feedparser entry was published, or now if it doesn't say.
feedparser hands back a 9-tuple in UTC under `published_parsed`, falling
back to `updated_parsed`. Three modules used to do this slice on their
own; the tuple is sliced to six here, once.
"""
for key in ("published_parsed", "updated_parsed"):
parsed = entry.get(key)
if parsed:
return datetime(*parsed[:6], tzinfo=UTC) # type: ignore[misc] # 9-tuple, sliced to 6
return datetime.now(UTC)
def http_failure(exc: Exception) -> str:
"""One line for a failed request, for a check result: the status when the
server answered, the exception's name when it didn't."""
response = getattr(exc, "response", None)
if response is not None:
return f"HTTP {response.status_code}"
return f"unreachable: {type(exc).__name__}"
def strip_html(html: str) -> str:
"""Tags out, whitespace collapsed. Feeds and Mastodon both hand us HTML.
A tag becomes a *space* rather than nothing, because `
` has to stop
the last word of one paragraph running into the first word of the next.
That is right for block tags and wrong for inline ones — `a link.`
would come out as "a link ." — so the space is taken back off where it
landed against punctuation. It reads better, and the ranker and the tagger
are both looking at these strings.
"""
text = _WHITESPACE.sub(" ", _HTML_TAGS.sub(" ", html or "")).strip()
return _SPACE_BEFORE_PUNCT.sub(r"\1", _SPACE_AFTER_OPENING.sub(r"\1", text))
#: Outlets whose name is more than one word, so "Al Jazeera Sport" doesn't
#: become outlet "Al". Anything not listed splits on the first space, which is
#: right for "Guardian Tech" and "BBC World". Set `outlet:`/`section:` on the
#: feed in config.yaml when you want neither guess.
MULTIWORD_OUTLETS = (
"Al Jazeera", "Hacker News", "Ars Technica", "Washington Post",
"Financial Times", "New York Times", "Wall Street Journal",
"Der Spiegel", "Le Monde", "Foreign Policy", "MIT Technology Review",
)
def split_label(label: str) -> tuple[str, str]:
"""'Guardian Tech' -> ('Guardian', 'Tech'). Fallback when config is silent.
A reading of a feed's *label*, which is why it is part of the source
contract rather than of any store: the RSS plugin applies it to a feed it
was given with no outlet, and an app applies it to a subscription seeded
the same way. One rule, and the plugin side owns it.
"""
label = (label or "").strip()
if not label:
return "", ""
for outlet in MULTIWORD_OUTLETS:
if label == outlet:
return outlet, ""
if label.startswith(outlet + " "):
return outlet, label[len(outlet):].strip()
head, _, rest = label.partition(" ")
return head, rest.strip()
#: What sits between a site's name and its tagline, for `outlet_name`. Spelled
#: with escapes because a literal en dash next to a hyphen inside a character
#: class is unreadable, and ruff is right to say so.
_TAGLINE = re.compile("\\s+[-\u2013\u2014|:\u00b7]\\s+")
def outlet_name(raw: str) -> str:
"""A name a site gave for itself, cut back to something a sidebar can hold.
Both places that ask a server what it is called get the SEO title back,
tagline and all: WordPress keeps ODSC's as "Open Data Science - Your News
Source for AI, Machine Learning & more", and a feed's `
` is often
"Ars Technica - All content". Everything after the first separator is the
tagline, and the name is the part in front of it.
A reading of what a *server* said, which is why it is part of the source
contract next to `split_label` — that one reads a label you wrote, this
one reads a string you never saw.
"""
return _TAGLINE.split((raw or "").strip(), 1)[0].strip() or (raw or "").strip()
@dataclass(slots=True)
class Suggestion:
"""An account you don't follow yet, that the graph thinks you should."""
handle: str
display_name: str = ""
reason: str = ""
score: float = 0.0
url: str = ""
@dataclass(slots=True)
class CheckResult:
"""Whether one subscription still resolves, and what the server said it is.
`target` is the feed URL or the handle — what the subscription row holds —
and `label` is what *you* call it, so the Jobs view can name a failing row.
`outlet` and `section` are the other half: what the server calls itself.
They are how a plugin that learns a publisher's real name from the network
hands it over — WordPress reads it off `/wp-json`, a feed off its own
`` — and they are written onto the subscription, once, by
`name_grouping`. Empty from every plugin that has nothing to add, which is
most of them: an outlet is normally known from the label before anything
is fetched.
Deliberately *not* `label`. That field already carries the subscription's
own name in every plugin and in the Jobs view's failing-row line, and a
publisher's name is a different fact about a different thing.
"""
kind: str
target: str
ok: bool
detail: str
label: str = ""
outlet: str = ""
section: str = ""
class Source(ABC):
"""Base class for every ingest plugin."""
#: config key under `sources:` in config.yaml. Required, must be unique.
name: str = ""
#: True for a class that exists to be subclassed rather than to be run:
#: `RssSource` is the fetching every publisher's plugin inherits, not a
#: plugin of its own. The registry passes over one, so there is no
#: "Any RSS feed" row to paste a masthead into — see `desk.sources.rss`.
#:
#: Read off the class's own `__dict__` and never inherited, or declaring
#: it once would take every plugin that subclasses that class with it.
shared: ClassVar[bool] = False
#: shown in the Sources tab, under Plugins
description: str = ""
#: Which key under `sources.:` holds this plugin's subscriptions, and
#: therefore whether one is a URL or a handle. Declared here rather than
#: inferred, because everything downstream needs the answer: what
#: `subscriptions.apply` puts back into the config for a pull, what the Add
#: box in the Sources tab asks for, and what the UI validates.
#:
#: This used to be `name == "rss"` spelled out at each of those sites,
#: which meant the second URL-shaped plugin silently had its feed list
#: replaced by an empty account list and fetched nothing.
subscription_field: ClassVar[str] = "accounts" # "accounts" or "feeds"
#: The outlet every subscription of this plugin sits under, when the plugin
#: is a network rather than a publisher — a Bluesky handle is a channel of
#: Bluesky, not a masthead of its own. Written onto a subscription when it
#: is added, and only over a name nobody chose; `check` overrides it for a
#: plugin that can ask the network what a subscription is really called.
#: Empty means "read it off the label", which is right for a feed.
default_outlet: str = ""
#: The addresses this plugin knows how to read better than a web page, and
#: how: a `Getter` whose `handles` matches them and whose `claims` puts it
#: ahead of the plain request. The body fetch collects these from every
#: active plugin and hands them to `desk.tools.scrape`, so a Hacker News
#: link to a Bluesky post is read through Bluesky's API while the Bluesky
#: plugin is on, and as a web page while it is off. A getter only gets the
#: page: which link an item follows is decided when the plugin makes it.
getters: ClassVar[tuple[Getter, ...]] = ()
def __init__(self, options: dict[str, Any] | None = None) -> None:
self.options = options or {}
#: What this plugin remembers between pulls — a spend counter, which
#: episodes are already done — as a plain mapping of strings. A pull
#: hands in one backed by the database; driven directly (a test, the
#: dry-run `main` in a plugin file) it is this dict and lives as long
#: as the instance. It is the only thing a plugin is given by the
#: application, which is what keeps a plugin one file.
self.state: MutableMapping[str, str] = {}
@property
def enabled(self) -> bool:
return bool(self.options.get("enabled", True))
@abstractmethod
def fetch(self) -> Iterator[Item]:
"""Yield everything new. Runner handles dedup, so over-fetching is fine."""
raise NotImplementedError
def discover(self) -> list[Suggestion]:
"""Optional: propose accounts to follow. Return [] if unsupported."""
return []
def check(self) -> list[CheckResult]:
"""Optional: does each subscription still resolve, and what is it called?
One result per feed or account, and never a paid request — the `check`
job runs this for every source at once. Return [] if there is nothing
to ask.
This is also the naming channel. A plugin that can only learn a
publisher's real name by asking the network says so here, in the
result's `outlet`/`section`, and the app writes it onto the
subscription. It is asked of one target at a time when you add a
subscription — build the source with that one entry in its options —
so a URL you paste comes back named.
"""
return []
def params_for(self, target: str) -> dict[str, Any]:
"""What this subscription asks for, when it is a query and not an address.
A feed URL and a handle need nothing; a search takes keywords, and
maybe a location or a floor. Those live on the subscription, not in the
plugin's own settings, because they are what distinguishes one of your
saved searches from the next — and so that adding one is the same act
as adding a feed, with the same switches on it afterwards.
"""
for entry in self.options.get(self.subscription_field) or []:
if isinstance(entry, dict):
this = str(entry.get("url") or entry.get("handle") or "")
if this == target:
got = entry.get("params")
return dict(got) if isinstance(got, dict) else {}
return {}
def targets(self) -> list[tuple[str, str]]:
"""The subscriptions this instance was configured with, as
`(target, label)` pairs — a URL or a handle, and what it is called.
A bare string in the list is its own label."""
out = []
for entry in self.options.get(self.subscription_field) or []:
if isinstance(entry, dict):
target = str(entry.get("url") or entry.get("handle") or "")
label = str(entry.get("label") or target)
else:
target = label = str(entry)
if target:
out.append((target, label))
return out