"""The plugin contract. Every source lives in its own file. Drop a .py file into `desk/sources/` or into any directory listed under `plugin_paths:` in config.yaml, define a subclass of `Source`, and it will be discovered automatically. Minimum viable plugin: from desk.sources import Source, Item class MySource(Source): name = "mysource" description = "One line shown in the Sources tab." def fetch(self): yield Item( source_id="unique-per-source", location="https://example.com/story", title="Headline", excerpt="Summary or body", author="someone", published_at=datetime.now(timezone.utc), # aware; stored as UTC ) `self.options` is whatever you put under `sources.mysource:` in the config — by the time a pull runs, its subscription list has been replaced with the fetch-enabled rows from the Sources tab (see newsdesk/jobs/subscriptions.py). Two optional methods round out the contract, both with do-nothing defaults: `discover` proposes accounts to follow, and `check` says whether each subscription still resolves. Neither is required for a source to be useful. """ from __future__ import annotations import re from abc import ABC, abstractmethod from collections.abc import Iterator, MutableMapping from dataclasses import dataclass from datetime import UTC, datetime from typing import Any, ClassVar from desk import __version__ from desk.models import Item from desk.tools.scrape import Getter #: Sent by every plugin that talks HTTP, so an outlet reading its logs can see #: who this is. (`desk/tools/framing.py` deliberately sends a browser string #: instead — it is asking what a browser would be told, so it has to ask as #: one.) USER_AGENT = f"desk/{__version__} (personal aggregator)" _HTML_TAGS = re.compile(r"<[^>]+>") _WHITESPACE = re.compile(r"\s+") #: Tidying up after the line above — see `strip_html`. Deliberately only the #: punctuation that always closes: an apostrophe or a dash can legitimately #: have a space in front of it. _SPACE_BEFORE_PUNCT = re.compile(r" +([.,;:!?%)\]}])") _SPACE_AFTER_OPENING = re.compile(r"([(\[{]) +") def parse_timestamp(value: str) -> datetime: """An ISO-8601 stamp from a social API, as an aware datetime, or now. Every one of these APIs writes UTC as a trailing `Z`, which `datetime.fromisoformat` did not accept before 3.11 and which is cheaper to translate than to special-case. A missing or malformed stamp is not worth dropping a post over: `now` sorts it to the top of the queue, where you will see it and can judge it yourself. A stamp with no offset at all is read as UTC, and that is a decision this helper is entitled to make where `Item` is not. Here we know whose stamp it is — an API that documents UTC and then omits the `Z` on some of its rows — whereas the model is handed a bare datetime by anything at all, and a wrong guess there is an item filed hours out with nothing on the row to show it. A plugin whose API sends local time should not be reaching for this. """ try: parsed = datetime.fromisoformat((value or "").replace("Z", "+00:00")) except (ValueError, AttributeError): return datetime.now(UTC) return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC) def entry_time(entry: Any) -> datetime: """When a feedparser entry was published, or now if it doesn't say. feedparser hands back a 9-tuple in UTC under `published_parsed`, falling back to `updated_parsed`. Three modules used to do this slice on their own; the tuple is sliced to six here, once. """ for key in ("published_parsed", "updated_parsed"): parsed = entry.get(key) if parsed: return datetime(*parsed[:6], tzinfo=UTC) # type: ignore[misc] # 9-tuple, sliced to 6 return datetime.now(UTC) def http_failure(exc: Exception) -> str: """One line for a failed request, for a check result: the status when the server answered, the exception's name when it didn't.""" response = getattr(exc, "response", None) if response is not None: return f"HTTP {response.status_code}" return f"unreachable: {type(exc).__name__}" def strip_html(html: str) -> str: """Tags out, whitespace collapsed. Feeds and Mastodon both hand us HTML. A tag becomes a *space* rather than nothing, because `

` has to stop the last word of one paragraph running into the first word of the next. That is right for block tags and wrong for inline ones — `a link.` would come out as "a link ." — so the space is taken back off where it landed against punctuation. It reads better, and the ranker and the tagger are both looking at these strings. """ text = _WHITESPACE.sub(" ", _HTML_TAGS.sub(" ", html or "")).strip() return _SPACE_BEFORE_PUNCT.sub(r"\1", _SPACE_AFTER_OPENING.sub(r"\1", text)) #: Outlets whose name is more than one word, so "Al Jazeera Sport" doesn't #: become outlet "Al". Anything not listed splits on the first space, which is #: right for "Guardian Tech" and "BBC World". Set `outlet:`/`section:` on the #: feed in config.yaml when you want neither guess. MULTIWORD_OUTLETS = ( "Al Jazeera", "Hacker News", "Ars Technica", "Washington Post", "Financial Times", "New York Times", "Wall Street Journal", "Der Spiegel", "Le Monde", "Foreign Policy", "MIT Technology Review", ) def split_label(label: str) -> tuple[str, str]: """'Guardian Tech' -> ('Guardian', 'Tech'). Fallback when config is silent. A reading of a feed's *label*, which is why it is part of the source contract rather than of any store: the RSS plugin applies it to a feed it was given with no outlet, and an app applies it to a subscription seeded the same way. One rule, and the plugin side owns it. """ label = (label or "").strip() if not label: return "", "" for outlet in MULTIWORD_OUTLETS: if label == outlet: return outlet, "" if label.startswith(outlet + " "): return outlet, label[len(outlet):].strip() head, _, rest = label.partition(" ") return head, rest.strip() #: What sits between a site's name and its tagline, for `outlet_name`. Spelled #: with escapes because a literal en dash next to a hyphen inside a character #: class is unreadable, and ruff is right to say so. _TAGLINE = re.compile("\\s+[-\u2013\u2014|:\u00b7]\\s+") def outlet_name(raw: str) -> str: """A name a site gave for itself, cut back to something a sidebar can hold. Both places that ask a server what it is called get the SEO title back, tagline and all: WordPress keeps ODSC's as "Open Data Science - Your News Source for AI, Machine Learning & more", and a feed's `` is often "Ars Technica - All content". Everything after the first separator is the tagline, and the name is the part in front of it. A reading of what a *server* said, which is why it is part of the source contract next to `split_label` — that one reads a label you wrote, this one reads a string you never saw. """ return _TAGLINE.split((raw or "").strip(), 1)[0].strip() or (raw or "").strip() @dataclass(slots=True) class Suggestion: """An account you don't follow yet, that the graph thinks you should.""" handle: str display_name: str = "" reason: str = "" score: float = 0.0 url: str = "" @dataclass(slots=True) class CheckResult: """Whether one subscription still resolves, and what the server said it is. `target` is the feed URL or the handle — what the subscription row holds — and `label` is what *you* call it, so the Jobs view can name a failing row. `outlet` and `section` are the other half: what the server calls itself. They are how a plugin that learns a publisher's real name from the network hands it over — WordPress reads it off `/wp-json`, a feed off its own `<title>` — and they are written onto the subscription, once, by `name_grouping`. Empty from every plugin that has nothing to add, which is most of them: an outlet is normally known from the label before anything is fetched. Deliberately *not* `label`. That field already carries the subscription's own name in every plugin and in the Jobs view's failing-row line, and a publisher's name is a different fact about a different thing. """ kind: str target: str ok: bool detail: str label: str = "" outlet: str = "" section: str = "" class Source(ABC): """Base class for every ingest plugin.""" #: config key under `sources:` in config.yaml. Required, must be unique. name: str = "" #: True for a class that exists to be subclassed rather than to be run: #: `RssSource` is the fetching every publisher's plugin inherits, not a #: plugin of its own. The registry passes over one, so there is no #: "Any RSS feed" row to paste a masthead into — see `desk.sources.rss`. #: #: Read off the class's own `__dict__` and never inherited, or declaring #: it once would take every plugin that subclasses that class with it. shared: ClassVar[bool] = False #: shown in the Sources tab, under Plugins description: str = "" #: Which key under `sources.<name>:` holds this plugin's subscriptions, and #: therefore whether one is a URL or a handle. Declared here rather than #: inferred, because everything downstream needs the answer: what #: `subscriptions.apply` puts back into the config for a pull, what the Add #: box in the Sources tab asks for, and what the UI validates. #: #: This used to be `name == "rss"` spelled out at each of those sites, #: which meant the second URL-shaped plugin silently had its feed list #: replaced by an empty account list and fetched nothing. subscription_field: ClassVar[str] = "accounts" # "accounts" or "feeds" #: The outlet every subscription of this plugin sits under, when the plugin #: is a network rather than a publisher — a Bluesky handle is a channel of #: Bluesky, not a masthead of its own. Written onto a subscription when it #: is added, and only over a name nobody chose; `check` overrides it for a #: plugin that can ask the network what a subscription is really called. #: Empty means "read it off the label", which is right for a feed. default_outlet: str = "" #: The addresses this plugin knows how to read better than a web page, and #: how: a `Getter` whose `handles` matches them and whose `claims` puts it #: ahead of the plain request. The body fetch collects these from every #: active plugin and hands them to `desk.tools.scrape`, so a Hacker News #: link to a Bluesky post is read through Bluesky's API while the Bluesky #: plugin is on, and as a web page while it is off. A getter only gets the #: page: which link an item follows is decided when the plugin makes it. getters: ClassVar[tuple[Getter, ...]] = () def __init__(self, options: dict[str, Any] | None = None) -> None: self.options = options or {} #: What this plugin remembers between pulls — a spend counter, which #: episodes are already done — as a plain mapping of strings. A pull #: hands in one backed by the database; driven directly (a test, the #: dry-run `main` in a plugin file) it is this dict and lives as long #: as the instance. It is the only thing a plugin is given by the #: application, which is what keeps a plugin one file. self.state: MutableMapping[str, str] = {} @property def enabled(self) -> bool: return bool(self.options.get("enabled", True)) @abstractmethod def fetch(self) -> Iterator[Item]: """Yield everything new. Runner handles dedup, so over-fetching is fine.""" raise NotImplementedError def discover(self) -> list[Suggestion]: """Optional: propose accounts to follow. Return [] if unsupported.""" return [] def check(self) -> list[CheckResult]: """Optional: does each subscription still resolve, and what is it called? One result per feed or account, and never a paid request — the `check` job runs this for every source at once. Return [] if there is nothing to ask. This is also the naming channel. A plugin that can only learn a publisher's real name by asking the network says so here, in the result's `outlet`/`section`, and the app writes it onto the subscription. It is asked of one target at a time when you add a subscription — build the source with that one entry in its options — so a URL you paste comes back named. """ return [] def params_for(self, target: str) -> dict[str, Any]: """What this subscription asks for, when it is a query and not an address. A feed URL and a handle need nothing; a search takes keywords, and maybe a location or a floor. Those live on the subscription, not in the plugin's own settings, because they are what distinguishes one of your saved searches from the next — and so that adding one is the same act as adding a feed, with the same switches on it afterwards. """ for entry in self.options.get(self.subscription_field) or []: if isinstance(entry, dict): this = str(entry.get("url") or entry.get("handle") or "") if this == target: got = entry.get("params") return dict(got) if isinstance(got, dict) else {} return {} def targets(self) -> list[tuple[str, str]]: """The subscriptions this instance was configured with, as `(target, label)` pairs — a URL or a handle, and what it is called. A bare string in the list is its own label.""" out = [] for entry in self.options.get(self.subscription_field) or []: if isinstance(entry, dict): target = str(entry.get("url") or entry.get("handle") or "") label = str(entry.get("label") or target) else: target = label = str(entry) if target: out.append((target, label)) return out