Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d666d0937
commit
248ea1636d
36 files changed
+8393
No files matched your search
@@ -0,0 +1,325 @@
|
||||
"""The plugin contract.
|
||||
|
||||
Every source lives in its own file. Drop a .py file into `desk/sources/`
|
||||
or into any directory listed under `plugin_paths:` in config.yaml, define a
|
||||
subclass of `Source`, and it will be discovered automatically.
|
||||
|
||||
Minimum viable plugin:
|
||||
|
||||
from desk.sources import Source, Item
|
||||
|
||||
class MySource(Source):
|
||||
name = "mysource"
|
||||
description = "One line shown in the Sources tab."
|
||||
|
||||
def fetch(self):
|
||||
yield Item(
|
||||
source_id="unique-per-source",
|
||||
location="https://example.com/story",
|
||||
title="Headline",
|
||||
excerpt="Summary or body",
|
||||
author="someone",
|
||||
published_at=datetime.now(timezone.utc), # aware; stored as UTC
|
||||
)
|
||||
|
||||
`self.options` is whatever you put under `sources.mysource:` in the config —
|
||||
by the time a pull runs, its subscription list has been replaced with the
|
||||
fetch-enabled rows from the Sources tab (see newsdesk/jobs/subscriptions.py).
|
||||
|
||||
Two optional methods round out the contract, both with do-nothing defaults:
|
||||
`discover` proposes accounts to follow, and `check` says whether each
|
||||
subscription still resolves. Neither is required for a source to be useful.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterator, MutableMapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from desk import __version__
|
||||
from desk.models import Item
|
||||
from desk.tools.scrape import Getter
|
||||
|
||||
#: Sent by every plugin that talks HTTP, so an outlet reading its logs can see
|
||||
#: who this is. (`desk/tools/framing.py` deliberately sends a browser string
|
||||
#: instead — it is asking what a browser would be told, so it has to ask as
|
||||
#: one.)
|
||||
USER_AGENT = f"desk/{__version__} (personal aggregator)"
|
||||
|
||||
_HTML_TAGS = re.compile(r"<[^>]+>")
|
||||
_WHITESPACE = re.compile(r"\s+")
|
||||
#: Tidying up after the line above — see `strip_html`. Deliberately only the
|
||||
#: punctuation that always closes: an apostrophe or a dash can legitimately
|
||||
#: have a space in front of it.
|
||||
_SPACE_BEFORE_PUNCT = re.compile(r" +([.,;:!?%)\]}])")
|
||||
_SPACE_AFTER_OPENING = re.compile(r"([(\[{]) +")
|
||||
|
||||
|
||||
def parse_timestamp(value: str) -> datetime:
|
||||
"""An ISO-8601 stamp from a social API, as an aware datetime, or now.
|
||||
|
||||
Every one of these APIs writes UTC as a trailing `Z`, which
|
||||
`datetime.fromisoformat` did not accept before 3.11 and which is cheaper to
|
||||
translate than to special-case. A missing or malformed stamp is not worth
|
||||
dropping a post over: `now` sorts it to the top of the queue, where you
|
||||
will see it and can judge it yourself.
|
||||
|
||||
A stamp with no offset at all is read as UTC, and that is a decision this
|
||||
helper is entitled to make where `Item` is not. Here we know whose stamp it
|
||||
is — an API that documents UTC and then omits the `Z` on some of its rows —
|
||||
whereas the model is handed a bare datetime by anything at all, and a wrong
|
||||
guess there is an item filed hours out with nothing on the row to show it.
|
||||
A plugin whose API sends local time should not be reaching for this.
|
||||
"""
|
||||
try:
|
||||
parsed = datetime.fromisoformat((value or "").replace("Z", "+00:00"))
|
||||
except (ValueError, AttributeError):
|
||||
return datetime.now(UTC)
|
||||
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
def entry_time(entry: Any) -> datetime:
|
||||
"""When a feedparser entry was published, or now if it doesn't say.
|
||||
|
||||
feedparser hands back a 9-tuple in UTC under `published_parsed`, falling
|
||||
back to `updated_parsed`. Three modules used to do this slice on their
|
||||
own; the tuple is sliced to six here, once.
|
||||
"""
|
||||
for key in ("published_parsed", "updated_parsed"):
|
||||
parsed = entry.get(key)
|
||||
if parsed:
|
||||
return datetime(*parsed[:6], tzinfo=UTC) # type: ignore[misc] # 9-tuple, sliced to 6
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def http_failure(exc: Exception) -> str:
|
||||
"""One line for a failed request, for a check result: the status when the
|
||||
server answered, the exception's name when it didn't."""
|
||||
response = getattr(exc, "response", None)
|
||||
if response is not None:
|
||||
return f"HTTP {response.status_code}"
|
||||
return f"unreachable: {type(exc).__name__}"
|
||||
|
||||
|
||||
def strip_html(html: str) -> str:
|
||||
"""Tags out, whitespace collapsed. Feeds and Mastodon both hand us HTML.
|
||||
|
||||
A tag becomes a *space* rather than nothing, because `</p><p>` has to stop
|
||||
the last word of one paragraph running into the first word of the next.
|
||||
That is right for block tags and wrong for inline ones — `a <b>link</b>.`
|
||||
would come out as "a link ." — so the space is taken back off where it
|
||||
landed against punctuation. It reads better, and the ranker and the tagger
|
||||
are both looking at these strings.
|
||||
"""
|
||||
text = _WHITESPACE.sub(" ", _HTML_TAGS.sub(" ", html or "")).strip()
|
||||
return _SPACE_BEFORE_PUNCT.sub(r"\1", _SPACE_AFTER_OPENING.sub(r"\1", text))
|
||||
|
||||
|
||||
#: Outlets whose name is more than one word, so "Al Jazeera Sport" doesn't
|
||||
#: become outlet "Al". Anything not listed splits on the first space, which is
|
||||
#: right for "Guardian Tech" and "BBC World". Set `outlet:`/`section:` on the
|
||||
#: feed in config.yaml when you want neither guess.
|
||||
MULTIWORD_OUTLETS = (
|
||||
"Al Jazeera", "Hacker News", "Ars Technica", "Washington Post",
|
||||
"Financial Times", "New York Times", "Wall Street Journal",
|
||||
"Der Spiegel", "Le Monde", "Foreign Policy", "MIT Technology Review",
|
||||
)
|
||||
|
||||
|
||||
def split_label(label: str) -> tuple[str, str]:
|
||||
"""'Guardian Tech' -> ('Guardian', 'Tech'). Fallback when config is silent.
|
||||
|
||||
A reading of a feed's *label*, which is why it is part of the source
|
||||
contract rather than of any store: the RSS plugin applies it to a feed it
|
||||
was given with no outlet, and an app applies it to a subscription seeded
|
||||
the same way. One rule, and the plugin side owns it.
|
||||
"""
|
||||
label = (label or "").strip()
|
||||
if not label:
|
||||
return "", ""
|
||||
for outlet in MULTIWORD_OUTLETS:
|
||||
if label == outlet:
|
||||
return outlet, ""
|
||||
if label.startswith(outlet + " "):
|
||||
return outlet, label[len(outlet):].strip()
|
||||
head, _, rest = label.partition(" ")
|
||||
return head, rest.strip()
|
||||
|
||||
|
||||
#: What sits between a site's name and its tagline, for `outlet_name`. Spelled
|
||||
#: with escapes because a literal en dash next to a hyphen inside a character
|
||||
#: class is unreadable, and ruff is right to say so.
|
||||
_TAGLINE = re.compile("\\s+[-\u2013\u2014|:\u00b7]\\s+")
|
||||
|
||||
|
||||
def outlet_name(raw: str) -> str:
|
||||
"""A name a site gave for itself, cut back to something a sidebar can hold.
|
||||
|
||||
Both places that ask a server what it is called get the SEO title back,
|
||||
tagline and all: WordPress keeps ODSC's as "Open Data Science - Your News
|
||||
Source for AI, Machine Learning & more", and a feed's `<title>` is often
|
||||
"Ars Technica - All content". Everything after the first separator is the
|
||||
tagline, and the name is the part in front of it.
|
||||
|
||||
A reading of what a *server* said, which is why it is part of the source
|
||||
contract next to `split_label` — that one reads a label you wrote, this
|
||||
one reads a string you never saw.
|
||||
"""
|
||||
return _TAGLINE.split((raw or "").strip(), 1)[0].strip() or (raw or "").strip()
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Suggestion:
|
||||
"""An account you don't follow yet, that the graph thinks you should."""
|
||||
|
||||
handle: str
|
||||
display_name: str = ""
|
||||
reason: str = ""
|
||||
score: float = 0.0
|
||||
url: str = ""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class CheckResult:
|
||||
"""Whether one subscription still resolves, and what the server said it is.
|
||||
|
||||
`target` is the feed URL or the handle — what the subscription row holds —
|
||||
and `label` is what *you* call it, so the Jobs view can name a failing row.
|
||||
|
||||
`outlet` and `section` are the other half: what the server calls itself.
|
||||
They are how a plugin that learns a publisher's real name from the network
|
||||
hands it over — WordPress reads it off `/wp-json`, a feed off its own
|
||||
`<title>` — and they are written onto the subscription, once, by
|
||||
`name_grouping`. Empty from every plugin that has nothing to add, which is
|
||||
most of them: an outlet is normally known from the label before anything
|
||||
is fetched.
|
||||
|
||||
Deliberately *not* `label`. That field already carries the subscription's
|
||||
own name in every plugin and in the Jobs view's failing-row line, and a
|
||||
publisher's name is a different fact about a different thing.
|
||||
"""
|
||||
|
||||
kind: str
|
||||
target: str
|
||||
ok: bool
|
||||
detail: str
|
||||
label: str = ""
|
||||
outlet: str = ""
|
||||
section: str = ""
|
||||
|
||||
|
||||
class Source(ABC):
|
||||
"""Base class for every ingest plugin."""
|
||||
|
||||
#: config key under `sources:` in config.yaml. Required, must be unique.
|
||||
name: str = ""
|
||||
#: True for a class that exists to be subclassed rather than to be run:
|
||||
#: `RssSource` is the fetching every publisher's plugin inherits, not a
|
||||
#: plugin of its own. The registry passes over one, so there is no
|
||||
#: "Any RSS feed" row to paste a masthead into — see `desk.sources.rss`.
|
||||
#:
|
||||
#: Read off the class's own `__dict__` and never inherited, or declaring
|
||||
#: it once would take every plugin that subclasses that class with it.
|
||||
shared: ClassVar[bool] = False
|
||||
#: shown in the Sources tab, under Plugins
|
||||
description: str = ""
|
||||
#: Which key under `sources.<name>:` holds this plugin's subscriptions, and
|
||||
#: therefore whether one is a URL or a handle. Declared here rather than
|
||||
#: inferred, because everything downstream needs the answer: what
|
||||
#: `subscriptions.apply` puts back into the config for a pull, what the Add
|
||||
#: box in the Sources tab asks for, and what the UI validates.
|
||||
#:
|
||||
#: This used to be `name == "rss"` spelled out at each of those sites,
|
||||
#: which meant the second URL-shaped plugin silently had its feed list
|
||||
#: replaced by an empty account list and fetched nothing.
|
||||
subscription_field: ClassVar[str] = "accounts" # "accounts" or "feeds"
|
||||
#: The outlet every subscription of this plugin sits under, when the plugin
|
||||
#: is a network rather than a publisher — a Bluesky handle is a channel of
|
||||
#: Bluesky, not a masthead of its own. Written onto a subscription when it
|
||||
#: is added, and only over a name nobody chose; `check` overrides it for a
|
||||
#: plugin that can ask the network what a subscription is really called.
|
||||
#: Empty means "read it off the label", which is right for a feed.
|
||||
default_outlet: str = ""
|
||||
#: The addresses this plugin knows how to read better than a web page, and
|
||||
#: how: a `Getter` whose `handles` matches them and whose `claims` puts it
|
||||
#: ahead of the plain request. The body fetch collects these from every
|
||||
#: active plugin and hands them to `desk.tools.scrape`, so a Hacker News
|
||||
#: link to a Bluesky post is read through Bluesky's API while the Bluesky
|
||||
#: plugin is on, and as a web page while it is off. A getter only gets the
|
||||
#: page: which link an item follows is decided when the plugin makes it.
|
||||
getters: ClassVar[tuple[Getter, ...]] = ()
|
||||
|
||||
def __init__(self, options: dict[str, Any] | None = None) -> None:
|
||||
self.options = options or {}
|
||||
#: What this plugin remembers between pulls — a spend counter, which
|
||||
#: episodes are already done — as a plain mapping of strings. A pull
|
||||
#: hands in one backed by the database; driven directly (a test, the
|
||||
#: dry-run `main` in a plugin file) it is this dict and lives as long
|
||||
#: as the instance. It is the only thing a plugin is given by the
|
||||
#: application, which is what keeps a plugin one file.
|
||||
self.state: MutableMapping[str, str] = {}
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return bool(self.options.get("enabled", True))
|
||||
|
||||
@abstractmethod
|
||||
def fetch(self) -> Iterator[Item]:
|
||||
"""Yield everything new. Runner handles dedup, so over-fetching is fine."""
|
||||
raise NotImplementedError
|
||||
|
||||
def discover(self) -> list[Suggestion]:
|
||||
"""Optional: propose accounts to follow. Return [] if unsupported."""
|
||||
return []
|
||||
|
||||
def check(self) -> list[CheckResult]:
|
||||
"""Optional: does each subscription still resolve, and what is it called?
|
||||
|
||||
One result per feed or account, and never a paid request — the `check`
|
||||
job runs this for every source at once. Return [] if there is nothing
|
||||
to ask.
|
||||
|
||||
This is also the naming channel. A plugin that can only learn a
|
||||
publisher's real name by asking the network says so here, in the
|
||||
result's `outlet`/`section`, and the app writes it onto the
|
||||
subscription. It is asked of one target at a time when you add a
|
||||
subscription — build the source with that one entry in its options —
|
||||
so a URL you paste comes back named.
|
||||
"""
|
||||
return []
|
||||
|
||||
def params_for(self, target: str) -> dict[str, Any]:
|
||||
"""What this subscription asks for, when it is a query and not an address.
|
||||
|
||||
A feed URL and a handle need nothing; a search takes keywords, and
|
||||
maybe a location or a floor. Those live on the subscription, not in the
|
||||
plugin's own settings, because they are what distinguishes one of your
|
||||
saved searches from the next — and so that adding one is the same act
|
||||
as adding a feed, with the same switches on it afterwards.
|
||||
"""
|
||||
for entry in self.options.get(self.subscription_field) or []:
|
||||
if isinstance(entry, dict):
|
||||
this = str(entry.get("url") or entry.get("handle") or "")
|
||||
if this == target:
|
||||
got = entry.get("params")
|
||||
return dict(got) if isinstance(got, dict) else {}
|
||||
return {}
|
||||
|
||||
def targets(self) -> list[tuple[str, str]]:
|
||||
"""The subscriptions this instance was configured with, as
|
||||
`(target, label)` pairs — a URL or a handle, and what it is called.
|
||||
A bare string in the list is its own label."""
|
||||
out = []
|
||||
for entry in self.options.get(self.subscription_field) or []:
|
||||
if isinstance(entry, dict):
|
||||
target = str(entry.get("url") or entry.get("handle") or "")
|
||||
label = str(entry.get("label") or target)
|
||||
else:
|
||||
target = label = str(entry)
|
||||
if target:
|
||||
out.append((target, label))
|
||||
return out
|
||||
Reference in new issue
Block a user