Bring in desk from news

The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-23 12:29:17 +02:00
1 parent 9d666d0937
commit 248ea1636d
36 files changed
+8393

No files matched your search

+175
View File
@@ -0,0 +1,175 @@
"""The plugins: where items come from. One door, used from both sides.
from desk.sources import Source, Item # writing one
from desk.sources import build, available # running them
A plugin is a file with a `Source` subclass in it — `base.py` is the contract
— dropped into this package or into a directory under `plugin_paths:` in
config.yaml. It yields `Item`s and touches nothing else: no store, no queue,
no config beyond its own `options`. What it needs to remember between pulls
goes in `self.state`, a mapping the runner backs with the database. That is
what makes a new source one file, and what lets everything above treat every
source the same way.
base.py `Source`, `Item`, `Suggestion`, `CheckResult`, and the
helpers every plugin ends up wanting
spec.py `PluginSpec`: what a plugin *declares* — its profiles, the
shared tools that acquire its content, what the shared
summariser and tagger should do with it, how it is laid out
registry.py finds the plugins on disk and builds the configured ones
bluesky.py, mastodon.py, x.py, podcast.py the built-in ones
rss.py feed fetching, which a publisher's plugin subclasses
Declaring a `spec` is optional in the contract and every plugin installed here
declares one: `spec_of` adapts a plugin that declares none, and such a plugin
runs exactly as it always has, but nothing about it is a claim its author made.
`PodcastSource` and `RssSource` are exported next to `Source` because that is
how a publisher's plugin is written: subclass the class that already does the
fetching, give it a name and a `spec`, and add nothing else.
`plugins/lastweekin_ai.py` and `plugins/guardian.py` are the two models.
`RssSource` is `shared` and is therefore not a plugin itself — there is no
generic "Any RSS feed" source to paste a masthead into.
"""
from __future__ import annotations
import logging
from typing import Any, Protocol
from desk.models import Item, ItemLink
from desk.sources.base import (
MULTIWORD_OUTLETS, USER_AGENT, CheckResult, Getter, Source, Suggestion,
entry_time, http_failure, outlet_name, parse_timestamp, split_label, strip_html,
)
from desk.sources.podcast import PodcastSource, segment_profile
from desk.sources.registry import available
from desk.sources.rss import RssSource, check_feed
from desk.sources.spec import (
API_VERSION, ARTICLE_READER, FEED_READER, LINK_READER, POST_READER,
RECORDING_READER, RSS, Discovery,
ExternalLink, MarkRule, OriginalView, PluginSpec, Presentation, Profile,
QueueLayout, RetryPolicy, Section, Step, SummaryPolicy, TaggingPolicy,
Target, TargetSpec, Tasks, View, Vocabulary, Wall, problems, spec_of,
)
__all__ = [
"API_VERSION",
"ARTICLE_READER",
"FEED_READER",
"LINK_READER",
"MULTIWORD_OUTLETS",
"POST_READER",
"RECORDING_READER",
"RSS",
"USER_AGENT",
"CheckResult",
"Discovery",
"ExternalLink",
"Getter",
"Item",
"ItemLink",
"MarkRule",
"OriginalView",
"PluginSpec",
"PodcastSource",
"Presentation",
"Profile",
"QueueLayout",
"RetryPolicy",
"RssSource",
"Section",
"Source",
"SourcesConfig",
"Step",
"Suggestion",
"SummaryPolicy",
"TaggingPolicy",
"Target",
"TargetSpec",
"Tasks",
"View",
"Vocabulary",
"Wall",
"available",
"build",
"check_feed",
"entry_time",
"faults",
"http_failure",
"outlet_name",
"parse_timestamp",
"problems",
"segment_profile",
"spec_of",
"split_label",
"strip_html",
]
log = logging.getLogger(__name__)
class SourcesConfig(Protocol):
"""What `build` needs from an app's config: where plugins are found, and
the section each configured one gets. Anything with these two attributes
will do; newsdesk's `Config` is one."""
@property
def plugin_paths(self) -> list[str]: ...
@property
def sources(self) -> dict[str, Any]: ...
def build(config: SourcesConfig,
vocabulary: Vocabulary = Vocabulary()) -> list[Source]:
"""Instantiate every source that is both installed and configured.
`vocabulary` is what this application supports, and an empty one — the
default — restricts nothing, which is every caller today. When an app
passes its own, a plugin whose declared spec names a tool, kind or
component the app does not have is left out with its diagnostics in the
log, and the others are built. One outlet's mistake is not an evening
without news.
A plugin that declares no spec is never left out on these grounds: there
is nothing in an adapted spec that anyone wrote. See `spec.spec_of`.
"""
classes = available(config.plugin_paths)
instances = []
for name, options in config.sources.items():
cls = classes.get(name)
if cls is None:
log.warning("config has `sources.%s` but no plugin defines it (have: %s)",
name, ", ".join(sorted(classes)) or "none")
continue
if broken := faults(cls, vocabulary):
log.error("source %s is not loadable here and is disabled:\n %s",
name, "\n ".join(broken))
continue
try:
instance = cls(options or {})
except Exception:
log.exception("source %s could not be constructed; continuing", name)
continue
if not instance.enabled:
log.info("source %s is disabled in config", name)
continue
instances.append(instance)
return instances
def faults(cls: type[Source], vocabulary: Vocabulary) -> list[str]:
"""Why this application cannot run this plugin, or nothing.
A spec that will not even build is caught here rather than at import,
because a plugin is free to construct its spec in `__init__`; either way
the plugin is disabled and the reason is the message.
Exported because a plugin left out of a pull is also a row in a list of
installed plugins, saying why it is unavailable — and asking that
question twice, in two places, is how the two answers start to differ.
"""
try:
return problems(spec_of(cls), vocabulary)
except Exception as exc:
return [f"its spec could not be read: {exc}"]
+325
View File
@@ -0,0 +1,325 @@
"""The plugin contract.
Every source lives in its own file. Drop a .py file into `desk/sources/`
or into any directory listed under `plugin_paths:` in config.yaml, define a
subclass of `Source`, and it will be discovered automatically.
Minimum viable plugin:
from desk.sources import Source, Item
class MySource(Source):
name = "mysource"
description = "One line shown in the Sources tab."
def fetch(self):
yield Item(
source_id="unique-per-source",
location="https://example.com/story",
title="Headline",
excerpt="Summary or body",
author="someone",
published_at=datetime.now(timezone.utc), # aware; stored as UTC
)
`self.options` is whatever you put under `sources.mysource:` in the config —
by the time a pull runs, its subscription list has been replaced with the
fetch-enabled rows from the Sources tab (see newsdesk/jobs/subscriptions.py).
Two optional methods round out the contract, both with do-nothing defaults:
`discover` proposes accounts to follow, and `check` says whether each
subscription still resolves. Neither is required for a source to be useful.
"""
from __future__ import annotations
import re
from abc import ABC, abstractmethod
from collections.abc import Iterator, MutableMapping
from dataclasses import dataclass
from datetime import UTC, datetime
from typing import Any, ClassVar
from desk import __version__
from desk.models import Item
from desk.tools.scrape import Getter
#: Sent by every plugin that talks HTTP, so an outlet reading its logs can see
#: who this is. (`desk/tools/framing.py` deliberately sends a browser string
#: instead — it is asking what a browser would be told, so it has to ask as
#: one.)
USER_AGENT = f"desk/{__version__} (personal aggregator)"
_HTML_TAGS = re.compile(r"<[^>]+>")
_WHITESPACE = re.compile(r"\s+")
#: Tidying up after the line above — see `strip_html`. Deliberately only the
#: punctuation that always closes: an apostrophe or a dash can legitimately
#: have a space in front of it.
_SPACE_BEFORE_PUNCT = re.compile(r" +([.,;:!?%)\]}])")
_SPACE_AFTER_OPENING = re.compile(r"([(\[{]) +")
def parse_timestamp(value: str) -> datetime:
"""An ISO-8601 stamp from a social API, as an aware datetime, or now.
Every one of these APIs writes UTC as a trailing `Z`, which
`datetime.fromisoformat` did not accept before 3.11 and which is cheaper to
translate than to special-case. A missing or malformed stamp is not worth
dropping a post over: `now` sorts it to the top of the queue, where you
will see it and can judge it yourself.
A stamp with no offset at all is read as UTC, and that is a decision this
helper is entitled to make where `Item` is not. Here we know whose stamp it
is — an API that documents UTC and then omits the `Z` on some of its rows —
whereas the model is handed a bare datetime by anything at all, and a wrong
guess there is an item filed hours out with nothing on the row to show it.
A plugin whose API sends local time should not be reaching for this.
"""
try:
parsed = datetime.fromisoformat((value or "").replace("Z", "+00:00"))
except (ValueError, AttributeError):
return datetime.now(UTC)
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
def entry_time(entry: Any) -> datetime:
"""When a feedparser entry was published, or now if it doesn't say.
feedparser hands back a 9-tuple in UTC under `published_parsed`, falling
back to `updated_parsed`. Three modules used to do this slice on their
own; the tuple is sliced to six here, once.
"""
for key in ("published_parsed", "updated_parsed"):
parsed = entry.get(key)
if parsed:
return datetime(*parsed[:6], tzinfo=UTC) # type: ignore[misc] # 9-tuple, sliced to 6
return datetime.now(UTC)
def http_failure(exc: Exception) -> str:
"""One line for a failed request, for a check result: the status when the
server answered, the exception's name when it didn't."""
response = getattr(exc, "response", None)
if response is not None:
return f"HTTP {response.status_code}"
return f"unreachable: {type(exc).__name__}"
def strip_html(html: str) -> str:
"""Tags out, whitespace collapsed. Feeds and Mastodon both hand us HTML.
A tag becomes a *space* rather than nothing, because `</p><p>` has to stop
the last word of one paragraph running into the first word of the next.
That is right for block tags and wrong for inline ones — `a <b>link</b>.`
would come out as "a link ." — so the space is taken back off where it
landed against punctuation. It reads better, and the ranker and the tagger
are both looking at these strings.
"""
text = _WHITESPACE.sub(" ", _HTML_TAGS.sub(" ", html or "")).strip()
return _SPACE_BEFORE_PUNCT.sub(r"\1", _SPACE_AFTER_OPENING.sub(r"\1", text))
#: Outlets whose name is more than one word, so "Al Jazeera Sport" doesn't
#: become outlet "Al". Anything not listed splits on the first space, which is
#: right for "Guardian Tech" and "BBC World". Set `outlet:`/`section:` on the
#: feed in config.yaml when you want neither guess.
MULTIWORD_OUTLETS = (
"Al Jazeera", "Hacker News", "Ars Technica", "Washington Post",
"Financial Times", "New York Times", "Wall Street Journal",
"Der Spiegel", "Le Monde", "Foreign Policy", "MIT Technology Review",
)
def split_label(label: str) -> tuple[str, str]:
"""'Guardian Tech' -> ('Guardian', 'Tech'). Fallback when config is silent.
A reading of a feed's *label*, which is why it is part of the source
contract rather than of any store: the RSS plugin applies it to a feed it
was given with no outlet, and an app applies it to a subscription seeded
the same way. One rule, and the plugin side owns it.
"""
label = (label or "").strip()
if not label:
return "", ""
for outlet in MULTIWORD_OUTLETS:
if label == outlet:
return outlet, ""
if label.startswith(outlet + " "):
return outlet, label[len(outlet):].strip()
head, _, rest = label.partition(" ")
return head, rest.strip()
#: What sits between a site's name and its tagline, for `outlet_name`. Spelled
#: with escapes because a literal en dash next to a hyphen inside a character
#: class is unreadable, and ruff is right to say so.
_TAGLINE = re.compile("\\s+[-\u2013\u2014|:\u00b7]\\s+")
def outlet_name(raw: str) -> str:
"""A name a site gave for itself, cut back to something a sidebar can hold.
Both places that ask a server what it is called get the SEO title back,
tagline and all: WordPress keeps ODSC's as "Open Data Science - Your News
Source for AI, Machine Learning & more", and a feed's `<title>` is often
"Ars Technica - All content". Everything after the first separator is the
tagline, and the name is the part in front of it.
A reading of what a *server* said, which is why it is part of the source
contract next to `split_label` — that one reads a label you wrote, this
one reads a string you never saw.
"""
return _TAGLINE.split((raw or "").strip(), 1)[0].strip() or (raw or "").strip()
@dataclass(slots=True)
class Suggestion:
"""An account you don't follow yet, that the graph thinks you should."""
handle: str
display_name: str = ""
reason: str = ""
score: float = 0.0
url: str = ""
@dataclass(slots=True)
class CheckResult:
"""Whether one subscription still resolves, and what the server said it is.
`target` is the feed URL or the handle — what the subscription row holds —
and `label` is what *you* call it, so the Jobs view can name a failing row.
`outlet` and `section` are the other half: what the server calls itself.
They are how a plugin that learns a publisher's real name from the network
hands it over — WordPress reads it off `/wp-json`, a feed off its own
`<title>` — and they are written onto the subscription, once, by
`name_grouping`. Empty from every plugin that has nothing to add, which is
most of them: an outlet is normally known from the label before anything
is fetched.
Deliberately *not* `label`. That field already carries the subscription's
own name in every plugin and in the Jobs view's failing-row line, and a
publisher's name is a different fact about a different thing.
"""
kind: str
target: str
ok: bool
detail: str
label: str = ""
outlet: str = ""
section: str = ""
class Source(ABC):
"""Base class for every ingest plugin."""
#: config key under `sources:` in config.yaml. Required, must be unique.
name: str = ""
#: True for a class that exists to be subclassed rather than to be run:
#: `RssSource` is the fetching every publisher's plugin inherits, not a
#: plugin of its own. The registry passes over one, so there is no
#: "Any RSS feed" row to paste a masthead into — see `desk.sources.rss`.
#:
#: Read off the class's own `__dict__` and never inherited, or declaring
#: it once would take every plugin that subclasses that class with it.
shared: ClassVar[bool] = False
#: shown in the Sources tab, under Plugins
description: str = ""
#: Which key under `sources.<name>:` holds this plugin's subscriptions, and
#: therefore whether one is a URL or a handle. Declared here rather than
#: inferred, because everything downstream needs the answer: what
#: `subscriptions.apply` puts back into the config for a pull, what the Add
#: box in the Sources tab asks for, and what the UI validates.
#:
#: This used to be `name == "rss"` spelled out at each of those sites,
#: which meant the second URL-shaped plugin silently had its feed list
#: replaced by an empty account list and fetched nothing.
subscription_field: ClassVar[str] = "accounts" # "accounts" or "feeds"
#: The outlet every subscription of this plugin sits under, when the plugin
#: is a network rather than a publisher — a Bluesky handle is a channel of
#: Bluesky, not a masthead of its own. Written onto a subscription when it
#: is added, and only over a name nobody chose; `check` overrides it for a
#: plugin that can ask the network what a subscription is really called.
#: Empty means "read it off the label", which is right for a feed.
default_outlet: str = ""
#: The addresses this plugin knows how to read better than a web page, and
#: how: a `Getter` whose `handles` matches them and whose `claims` puts it
#: ahead of the plain request. The body fetch collects these from every
#: active plugin and hands them to `desk.tools.scrape`, so a Hacker News
#: link to a Bluesky post is read through Bluesky's API while the Bluesky
#: plugin is on, and as a web page while it is off. A getter only gets the
#: page: which link an item follows is decided when the plugin makes it.
getters: ClassVar[tuple[Getter, ...]] = ()
def __init__(self, options: dict[str, Any] | None = None) -> None:
self.options = options or {}
#: What this plugin remembers between pulls — a spend counter, which
#: episodes are already done — as a plain mapping of strings. A pull
#: hands in one backed by the database; driven directly (a test, the
#: dry-run `main` in a plugin file) it is this dict and lives as long
#: as the instance. It is the only thing a plugin is given by the
#: application, which is what keeps a plugin one file.
self.state: MutableMapping[str, str] = {}
@property
def enabled(self) -> bool:
return bool(self.options.get("enabled", True))
@abstractmethod
def fetch(self) -> Iterator[Item]:
"""Yield everything new. Runner handles dedup, so over-fetching is fine."""
raise NotImplementedError
def discover(self) -> list[Suggestion]:
"""Optional: propose accounts to follow. Return [] if unsupported."""
return []
def check(self) -> list[CheckResult]:
"""Optional: does each subscription still resolve, and what is it called?
One result per feed or account, and never a paid request — the `check`
job runs this for every source at once. Return [] if there is nothing
to ask.
This is also the naming channel. A plugin that can only learn a
publisher's real name by asking the network says so here, in the
result's `outlet`/`section`, and the app writes it onto the
subscription. It is asked of one target at a time when you add a
subscription — build the source with that one entry in its options —
so a URL you paste comes back named.
"""
return []
def params_for(self, target: str) -> dict[str, Any]:
"""What this subscription asks for, when it is a query and not an address.
A feed URL and a handle need nothing; a search takes keywords, and
maybe a location or a floor. Those live on the subscription, not in the
plugin's own settings, because they are what distinguishes one of your
saved searches from the next — and so that adding one is the same act
as adding a feed, with the same switches on it afterwards.
"""
for entry in self.options.get(self.subscription_field) or []:
if isinstance(entry, dict):
this = str(entry.get("url") or entry.get("handle") or "")
if this == target:
got = entry.get("params")
return dict(got) if isinstance(got, dict) else {}
return {}
def targets(self) -> list[tuple[str, str]]:
"""The subscriptions this instance was configured with, as
`(target, label)` pairs — a URL or a handle, and what it is called.
A bare string in the list is its own label."""
out = []
for entry in self.options.get(self.subscription_field) or []:
if isinstance(entry, dict):
target = str(entry.get("url") or entry.get("handle") or "")
label = str(entry.get("label") or target)
else:
target = label = str(entry)
if target:
out.append((target, label))
return out
+242
View File
@@ -0,0 +1,242 @@
"""Bluesky via the public AppView. No auth, no key, no bill.
Public app.bsky.* endpoints can be called directly against
https://public.api.bsky.app, which is what this plugin does. Endpoints used:
app.bsky.feed.getAuthorFeed - posts by one account
app.bsky.graph.getFollows - who an account follows (for discovery)
app.bsky.graph.getSuggestedFollowsByActor - accounts similar to a given one
"""
from __future__ import annotations
import logging
from collections import defaultdict
from collections.abc import Iterator
from typing import Any
import httpx
from desk.sources.base import (
USER_AGENT, CheckResult, Getter, Item, Source, Suggestion, http_failure,
parse_timestamp,
)
from desk.sources.spec import (
API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy,
TaggingPolicy, TargetSpec, Tasks,
)
from desk.tools import posts
log = logging.getLogger(__name__)
API = "https://public.api.bsky.app/xrpc"
class BlueskySource(Source):
name = "bluesky"
description = "Posts from Bluesky accounts (free public API)."
default_outlet = "Bluesky"
#: A `bsky.app/profile/.../post/...` link from any outlet is read through
#: the same public API, not as the web app's script shell.
getters = (Getter("bluesky_api", posts.bluesky, posts.is_bluesky, claims=True),)
#: A network declares no accounts, because which ones you follow are
#: yours. `mode="editable"` is that sentence in the contract, and it is
#: what puts an Add box under this plugin and nothing but Fetch and Show
#: under a publisher's — see §8 of docs/plugin-rework.md.
#:
#: Both policies are declared as these items already behave rather than
#: as anyone would design them fresh, which is step 11's rule that
#: silence is not a decision. Posts are tagged today and have been for as
#: long as there has been a tagger; a post is already the length an
#: abstract would be, so summarising is off and says so.
spec = PluginSpec(
api_version=API_VERSION, id=name, version="1", label="Bluesky",
targets=TargetSpec(mode="editable", kind="account", defaults=[],
validator="bluesky_handle"),
profiles={"default": Profile(
kind="post",
# The fetching a link post has always had: `links_only` keeps the
# ones that point somewhere, and where they point is extracted
# into the field the reader shows under the post itself. A post
# linking to nothing carries its own permalink as `location` and
# as `raw.post_url`, and the body fetch skips it.
acquisition=[Step(id="article", tool="extract_article",
inputs={"url": "inputs.original_url"},
outputs={"content": "inputs.article_text"})],
tasks=Tasks(
summary=SummaryPolicy(enabled=False),
tagging=TaggingPolicy(
enabled=True, input_fields=["inputs.feed_text"],
context_fields=["author"]),
),
presentation=POST_READER,
)},
)
def __init__(self, options: dict[str, Any] | None = None) -> None:
super().__init__(options)
self.client = httpx.Client(
timeout=20,
headers={"User-Agent": USER_AGENT},
)
def _get(self, method: str, **params) -> dict[str, Any]:
r = self.client.get(f"{API}/{method}", params=params)
r.raise_for_status()
return r.json()
# ------------------------------------------------------------- ingest
def fetch(self) -> Iterator[Item]:
limit = int(self.options.get("posts_per_account", 40))
skip_replies = self.options.get("skip_replies", True)
links_only = self.options.get("links_only", False)
for handle, label in self.targets():
try:
data = self._get(
"app.bsky.feed.getAuthorFeed",
actor=handle,
limit=min(limit, 100),
filter="posts_no_replies" if skip_replies else "posts_with_replies",
)
except httpx.HTTPError as exc:
log.warning("bluesky: %s failed (%s)", handle, exc)
continue
for entry in data.get("feed", []):
post = entry.get("post") or {}
record = post.get("record") or {}
author = post.get("author") or {}
text = record.get("text", "")
link = self._external_link(post)
if links_only and not link:
continue
rkey = (post.get("uri") or "").rsplit("/", 1)[-1]
yield Item(
source_id=post.get("uri", ""),
location=link
or f"https://bsky.app/profile/{author.get('handle', handle)}/post/{rkey}",
title="", # base class derives one from the text
excerpt=text,
author=author.get("displayName") or author.get("handle", handle),
published_at=parse_timestamp(record.get("createdAt", "")),
subscription=label,
kind="post",
raw={
"likes": post.get("likeCount", 0),
"reposts": post.get("repostCount", 0),
"post_url": f"https://bsky.app/profile/"
f"{author.get('handle', handle)}/post/{rkey}",
},
)
@staticmethod
def _external_link(post: dict[str, Any]) -> str:
"""Prefer the article a post links to over the post itself."""
embed = post.get("embed") or {}
external = embed.get("external") or {}
if external.get("uri"):
return external["uri"]
for facet in (post.get("record") or {}).get("facets", []) or []:
for feature in facet.get("features", []):
if feature.get("$type", "").endswith("#link") and feature.get("uri"):
return feature["uri"]
return ""
# --------------------------------------------------------------- check
def check(self) -> list[CheckResult]:
return [self._check_one(handle) for handle, _ in self.targets()]
def _check_one(self, handle: str) -> CheckResult:
try:
data = self._get("app.bsky.actor.getProfile", actor=handle)
except httpx.HTTPError as exc:
return CheckResult("bluesky", handle, False, http_failure(exc))
return CheckResult(
"bluesky", handle, True,
f"{data.get('postsCount', '?')} posts, "
f"{data.get('followersCount', '?')} followers",
)
# ----------------------------------------------------------- discovery
def discover(self) -> list[Suggestion]:
settings = self.options.get("discover") or {}
if not settings.get("enabled", True):
return []
seeds = [target for target, _ in self.targets()][: settings.get("max_seeds", 25)]
if not seeds:
return []
following = {s.lower() for s in seeds}
# Fractional, because signal 2 is weighted above signal 1 — so not a
# Counter, whose values typeshed (correctly) says are ints.
counts: defaultdict[str, float] = defaultdict(float)
meta: dict[str, dict[str, str]] = {}
# Signal 1: accounts that several of your accounts follow.
for seed in seeds:
for account in self._paged_follows(seed):
handle = account.get("handle", "").lower()
if not handle or handle in following:
continue
counts[handle] += 1
meta.setdefault(handle, {
"display_name": account.get("displayName", ""),
"description": (account.get("description") or "")[:160],
})
# Signal 2: Bluesky's own "similar to this account" endpoint.
for seed in seeds[:10]:
try:
data = self._get("app.bsky.graph.getSuggestedFollowsByActor", actor=seed)
except httpx.HTTPError:
continue
for account in data.get("suggestions", []):
handle = account.get("handle", "").lower()
if not handle or handle in following:
continue
counts[handle] += 1.5 # weight it slightly above a raw co-follow
meta.setdefault(handle, {
"display_name": account.get("displayName", ""),
"description": (account.get("description") or "")[:160],
})
threshold = settings.get("min_overlap", 3)
out = []
ranked = sorted(counts.items(), key=lambda kv: -kv[1])[:200]
for handle, score in ranked:
if score < threshold:
break
info = meta.get(handle, {})
out.append(Suggestion(
handle=handle,
display_name=info.get("display_name", ""),
reason=f"followed by {int(score)} of the accounts you follow",
score=float(score),
url=f"https://bsky.app/profile/{handle}",
))
return out
def _paged_follows(self, actor: str, cap: int = 400) -> list[dict[str, Any]]:
out: list[dict[str, Any]] = []
cursor: str | None = None
while len(out) < cap:
params = {"actor": actor, "limit": 100}
if cursor:
params["cursor"] = cursor
try:
data = self._get("app.bsky.graph.getFollows", **params)
except httpx.HTTPError as exc:
log.warning("bluesky: follows for %s failed (%s)", actor, exc)
break
out.extend(data.get("follows", []))
cursor = data.get("cursor")
if not cursor:
break
return out
+196
View File
@@ -0,0 +1,196 @@
"""Mastodon via each instance's public REST API.
Uses /api/v1/accounts/lookup, /api/v1/accounts/{id}/statuses and
/api/v1/accounts/{id}/following. Public statuses are readable without a token
on Mastodon 2.7.0 and later, so no app registration is needed for public
accounts. Instances that run in whitelist mode will 401; those get skipped.
Accounts are written as user@instance.tld.
"""
from __future__ import annotations
import logging
import time
from collections import Counter
from collections.abc import Iterator
from typing import Any
import httpx
from desk.sources.base import (
USER_AGENT, CheckResult, Getter, Item, Source, Suggestion, http_failure,
parse_timestamp, strip_html,
)
from desk.sources.spec import (
API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy,
TaggingPolicy, TargetSpec, Tasks,
)
from desk.tools import posts
log = logging.getLogger(__name__)
class MastodonSource(Source):
name = "mastodon"
description = "Public posts from Mastodon accounts (no token needed)."
default_outlet = "Mastodon"
#: A post link on any instance (`/@user/123`) is read through that
#: instance's API. The shape is not proof of Mastodon; a host that does not
#: answer the API in JSON gets the ordinary path.
getters = (Getter("mastodon_api", posts.mastodon, posts.is_mastodon, claims=True),)
#: The same declaration Bluesky makes, for the same reasons — read that
#: one. What differs is the handle: `user@instance.tld` names the server
#: as well as the person, so the check is its own.
spec = PluginSpec(
api_version=API_VERSION, id=name, version="1", label="Mastodon",
targets=TargetSpec(mode="editable", kind="account", defaults=[],
validator="mastodon_handle"),
profiles={"default": Profile(
kind="post",
acquisition=[Step(id="article", tool="extract_article",
inputs={"url": "inputs.original_url"},
outputs={"content": "inputs.article_text"})],
tasks=Tasks(
summary=SummaryPolicy(enabled=False),
tagging=TaggingPolicy(
enabled=True, input_fields=["inputs.feed_text"],
context_fields=["author"]),
),
presentation=POST_READER,
)},
)
def __init__(self, options: dict[str, Any] | None = None) -> None:
super().__init__(options)
self.client = httpx.Client(
timeout=20,
follow_redirects=True,
headers={"User-Agent": USER_AGENT},
)
self._delay = float(self.options.get("request_delay", 0.3))
@staticmethod
def _split(acct: str) -> tuple[str, str]:
acct = acct.lstrip("@")
user, _, host = acct.partition("@")
return user, host
def _get(self, host: str, path: str, **params) -> Any:
time.sleep(self._delay)
r = self.client.get(f"https://{host}/api/v1/{path}", params=params)
r.raise_for_status()
return r.json()
def _lookup(self, acct: str) -> dict[str, Any] | None:
user, host = self._split(acct)
if not host:
log.warning("mastodon: %s is missing an instance (use user@instance)", acct)
return None
try:
return self._get(host, "accounts/lookup", acct=user)
except httpx.HTTPError as exc:
log.warning("mastodon: lookup %s failed (%s)", acct, exc)
return None
# ------------------------------------------------------------- ingest
def fetch(self) -> Iterator[Item]:
limit = int(self.options.get("posts_per_account", 40))
for acct, label in self.targets():
account = self._lookup(acct)
if not account:
continue
_, host = self._split(acct)
try:
statuses = self._get(
host,
f"accounts/{account['id']}/statuses",
limit=min(limit, 40),
exclude_replies=str(self.options.get("skip_replies", True)).lower(),
exclude_reblogs=str(self.options.get("skip_boosts", True)).lower(),
)
except httpx.HTTPError as exc:
log.warning("mastodon: statuses for %s failed (%s)", acct, exc)
continue
for status in statuses:
card = status.get("card") or {}
text = strip_html(status.get("content", ""))
yield Item(
source_id=status.get("uri") or status.get("url", ""),
location=card.get("url") or status.get("url", ""),
title=card.get("title", ""),
excerpt=text,
author=(status.get("account") or {}).get("display_name") or acct,
published_at=parse_timestamp(status.get("created_at", "")),
subscription=label,
kind="post",
raw={
"boosts": status.get("reblogs_count", 0),
"favourites": status.get("favourites_count", 0),
"post_url": status.get("url", ""),
},
)
# --------------------------------------------------------------- check
def check(self) -> list[CheckResult]:
return [self._check_one(acct) for acct, _ in self.targets()]
def _check_one(self, acct: str) -> CheckResult:
user, host = self._split(acct)
if not host:
return CheckResult("mastodon", acct, False,
"missing instance (use user@instance)")
try:
data = self._get(host, "accounts/lookup", acct=user)
except httpx.HTTPError as exc:
return CheckResult("mastodon", acct, False, http_failure(exc))
return CheckResult("mastodon", acct, True,
f"{data.get('statuses_count', '?')} posts")
# ----------------------------------------------------------- discovery
def discover(self) -> list[Suggestion]:
settings = self.options.get("discover") or {}
if not settings.get("enabled", True):
return []
seeds = [target for target, _ in self.targets()]
following = {s.lstrip("@").lower() for s in seeds}
counts: Counter[str] = Counter()
meta: dict[str, str] = {}
for acct in seeds:
account = self._lookup(acct)
if not account:
continue
_, host = self._split(acct)
try:
# Many accounts hide this list; a 401/403 just means skip.
follows = self._get(host, f"accounts/{account['id']}/following", limit=80)
except httpx.HTTPError:
continue
for other in follows:
handle = other.get("acct", "")
if "@" not in handle:
handle = f"{handle}@{host}"
if handle.lower() in following:
continue
counts[handle] += 1
meta.setdefault(handle, other.get("display_name", ""))
threshold = settings.get("min_overlap", 2)
return [
Suggestion(
handle=handle,
display_name=meta.get(handle, ""),
reason=f"followed by {n} of the accounts you follow",
score=float(n),
url=f"https://{handle.split('@')[-1]}/@{handle.split('@')[0]}",
)
for handle, n in counts.most_common(200)
if n >= threshold
]
+546
View File
@@ -0,0 +1,546 @@
"""A news podcast, read as one item per story rather than one per episode.
The general half. `desk/tools/video/` below knows how to cut a video into
chapters; this knows the things that are true of a *show*: it has a feed, it
publishes on a cycle, its episodes are somewhere on YouTube, and its stories
belong in your queue next to the RSS copies of those same articles.
`plugins/lastweekin_ai.py` above knows one show.
That three-layer split is not decoration. A show that publishes a timestamped
outline works here with nothing show-specific involved — Dwarkesh and Latent
Space both do. What varies between shows is smaller than it looks, and is
declared as two methods rather than guessed at:
video_key(entry) where this show puts its video. The default reads
the feed body, which is the common case and is
wrong for several of the biggest shows.
episode_number(title) how this show writes an episode number.
Where each piece comes from, for one episode:
the feed title, date, the audio enclosure, the video link,
and the show notes as HTML — one fetch, everything
the notes chapter marks, sometimes with the article links attached,
which is what makes a segment a story rather than a topic
YouTube the transcript, with timings
video.cut() one Segment per chapter
this file one Item per Segment
**The subscription is just a feed.** `feeds:` is the same config key `rss`
uses, so a show imports into the `subscriptions` table like any other
subscription and gets add/remove/fetch/show, the Sources tab and the sidebar
for free — none of which had to know podcasts exist.
Config, for any show that is not one somebody has written a plugin for:
sources:
podcast:
feeds:
- url: https://api.substack.com/feed/podcast/69345.rss
label: Dwarkesh
episodes_per_feed: 2
Two things this deliberately does not do. It does not summarise: `text` starts
as the opening of the transcript and is replaced when a summary lands. What the
hosts made of a story is the one thing here that no feed already gives you, and
so the only part of this worth a model — which makes it a separate job, whose
failure costs you the abstracts and not the pull. See
newsdesk/jobs/summarise.py.
And it does not store the episode as an item, or as anything else: an episode
is a date and a video id, both of which live on every segment it produced, so
"every segment of episode 255, in order" is already a query and needs no
entity behind it.
**A show with no outline gets nothing stored, on purpose.** That claim above
used to hold only by luck: with no marks anywhere, `cut` hands back the whole
episode as one segment — right for a link you added by hand, wrong here — and
this stored it, so a two-hour recording arrived as a single eleven-thousand-word
item whose standfirst was the first two sentences of the intro and which then
queued itself for a summariser expecting one story. Now `_one_episode` stops
before the transcript is even fetched and says so in the log. Only shows that
publish a timestamped outline work here, which was always true and is now also
enforced.
"""
from __future__ import annotations
import logging
import re
from collections.abc import Iterator
from datetime import UTC, datetime, timedelta
from typing import Any
import feedparser
import httpx
from desk.sources.base import USER_AGENT, CheckResult, Item, Source, entry_time
from desk.sources.rss import check_feed
from desk.sources.spec import (
API_VERSION, RECORDING_READER, RSS, PluginSpec, Profile, RetryPolicy, Step,
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks,
)
from desk.tools.http import session
from desk.tools.video import Segment, Video, cut, marks, media, opening, youtube
log = logging.getLogger(__name__)
#: "LWiAI Podcast #255 - Gemini 3.7, ..." -> 255. Every show numbers its
#: episodes somewhere; the title is where they all agree to put it.
_NUMBER = re.compile(r"(?:#|episode\s+|ep\.?\s*)(\d{1,5})\b", re.IGNORECASE)
def segment_profile(instructions: str = "", tags: str = "",
target_words: int = 70) -> Profile:
"""What a show that publishes a timestamped outline produces, declared.
A reusable typed preset, so that a plugin for one show is its feed, its
numbering and its editorial line — and not a second copy of this
machinery's description. `plugins/lastweekin_ai.py` is the model.
The two steps are what `_one_episode` below does, in the contract's words:
retrieve the recording's captions *once*, then cut them at the marks the
show published. Nothing executes them yet — the runner is a later step of
`docs/plugin-rework.md` — and they are written now because they are the
part a future reader has to be able to check this file against.
`instructions` is the one thing a show is expected to fill in: what its
own segments want said about them. What is general about summarising
speech — write what they made of it, keep the disagreement, do not quote a
mangled caption back — belongs to the shared task and is written once in
`newsdesk/jobs/summarise.py`, not per show.
"""
return Profile(
# One profile, because one thing is stored: the stories, not the
# episodes they came out of. An episode is a date and a video id, both
# of which are on every segment it produced.
kind="segment",
acquisition=[
Step(id="captions", tool="retrieve_captions",
inputs={"media": "inputs.media"},
outputs={"transcript": "inputs.transcript"},
# Auto-captions on a two-hour upload can take hours to
# appear, so absence is a wait and not a failure. Today that
# wait is "the next pull, for ever", because the episode is
# simply not marked done; a bound is what this declares.
retry=RetryPolicy(max_attempts=4, delay_seconds=3600,
backoff=2.0, on=["not_ready"])),
Step(id="segments", tool="cut_segments", needs=["captions"],
# The marks come from the show's own notes, which arrive in
# the feed entry — HTML, so each story keeps the article link
# beside its timestamp.
inputs={"transcript": "inputs.transcript",
"marks": "inputs.feed_text"},
outputs={"transcript": "inputs.transcript",
"links": "inputs.links"}),
],
tasks=Tasks(
summary=SummaryPolicy(
enabled=True, input_fields=["inputs.transcript"],
require_complete=True,
# No floor: a two-sentence housekeeping segment gets one flat
# sentence saying so, which is worth having and is what the
# shared prompt already asks for.
min_words=0, target_words=target_words,
instructions=instructions),
tagging=TaggingPolicy(
# The transcript, not the abstract: tagging a segment must not
# quietly wait on a summariser that failed. Naming it decides
# *which* text and not how much of it — the tagger reads the
# opening few hundred characters of whatever a profile names,
# because it is choosing from a closed list of labels and a
# two-hour transcript would be thirty times the tokens for
# the same answer. See `newsdesk/jobs/tagging.py`.
enabled=True, input_fields=["inputs.transcript"],
context_fields=["title", "inputs.links"], instructions=tags),
),
# The player first, then what the hosts made of it; the articles
# they were reading stay above it in every view. See the preset.
presentation=RECORDING_READER,
)
class PodcastSource(Source):
name = "podcast"
description = "A show's feed, cut into one item per story discussed."
subscription_field = "feeds"
default_outlet = "Podcast"
#: The generic source declares the same profile it hands its subclasses,
#: with no editorial line of its own: it is any show, and it cannot know
#: what one of them wants said. Its targets are editable because they are
#: whatever you put under `podcast:` in config.yaml — a show with a plugin
#: of its own declares a catalogue instead.
spec = PluginSpec(
api_version=API_VERSION, id=name, version="1", label="Podcast",
discovery=RSS(),
targets=TargetSpec(mode="editable", kind="rss_feed", defaults=[]),
profiles={"default": segment_profile()},
)
# -------------------------------------------------------- what a show is
#
# The two things that differ between shows, as methods rather than as
# config knobs — because neither has a value you could write down. Finding
# the video means knowing where that show puts it, and numbering an episode
# means knowing how that show writes a number. A subclass that knows one
# show answers both; everything else in this file is the same for all of
# them.
def video_key(self, entry: Any) -> str:
"""The YouTube id for this episode, or "" if there isn't one to find.
**The one place a show's publishing habits really bite.** The default
reads the feed body, which works when the show embeds or links its own
video there — Last Week in AI, Dwarkesh, Latent Space all do.
It is not general, and it fails in a way worth naming: The Vergecast,
Hard Fork, TWiT, All-In and Lex Fridman all publish full video to
YouTube and none of them mentions it in the feed, so all five come back
empty here. Fixing that means matching a feed entry against a channel's
uploads — which title, which date, whose channel — and that is a
per-show judgement, not a parser. This is where it goes when someone
wants one of those shows: override this, return the id, and every other
line in this file already works.
"""
try:
return youtube.video_id(_body(entry))
except ValueError:
return ""
def episode_number(self, title: str) -> int | None:
"""Which episode this is, from its title, or None if it doesn't say.
Every show numbers its episodes and the title is where they all agree
to put it — but not in one format. `#255` and `Episode 42` and `Ep. 7`
are the common family and are what the default reads; `TWiT 1100` and
`SN 1094` are not, and an interview show may not number at all.
Nothing downstream depends on the answer: it is stored on the item for
display and for "every segment of episode 255, in order". So the honest
default is a regex that catches most of them and returns None rather
than guessing when it doesn't, and a subclass overrides it when its
show writes numbers its own way.
"""
found = _NUMBER.search(title or "")
return int(found.group(1)) if found else None
# ------------------------------------------------------------------ fetch
def fetch(self) -> Iterator[Item]:
limit = int(self.options.get("episodes_per_feed", 2))
with session(None, timeout=40) as http:
for entry in self.options.get("feeds", []):
url = entry["url"] if isinstance(entry, dict) else entry
label = _label(entry, url)
try:
yield from self._one_feed(http, url, label, limit)
except youtube.Blocked:
# The one failure that is not this show's. YouTube refused
# the machine, so every other show here would be refused
# too, and each attempt is one more request from an address
# already taken for a bot. Out of `fetch` altogether: the
# runner keeps what was yielded and turns this into a
# notice, which is what the log line below never was.
raise
except Exception:
# One dead show must not cost the others their episodes.
log.exception("podcast feed %s failed", url)
def _one_feed(self, http: httpx.Client, url: str, label: str,
limit: int) -> Iterator[Item]:
response = http.get(url, headers={"User-Agent": USER_AGENT})
response.raise_for_status()
# The bytes, not `.text`. A feed declares its own encoding in its XML
# declaration and plenty of them are not UTF-8; handing the parser a
# string that httpx has already decoded by its own guess throws that
# declaration away, and what comes back is an episode title with a
# mojibake apostrophe in it. `check_feed` parses the bytes too.
feed = feedparser.parse(response.content)
if not feed.entries:
log.warning("no episodes in %s", url)
return
for entry in feed.entries[:limit]:
episode = self._episode(entry, label)
if episode is None:
continue
if self._already_done(episode.key):
log.debug("%s: already ingested", episode.title)
continue
try:
yield from self._one_episode(http, episode, entry, label)
except youtube.Blocked:
raise # not this episode's failure — see `fetch`
except Exception:
log.exception("episode %s failed; continuing the feed", episode.key)
def _one_episode(self, http: httpx.Client | None, episode: Video, entry: Any,
label: str) -> Iterator[Item]:
# The show's own notes first: same outline as the video description,
# but the links survived. Then the description, then the comments.
chapters = marks.best(episode.notes)
if not chapters:
described = youtube.details(episode.key, client=http)
chapters = marks.best(described["description"])
episode.duration = episode.duration or described["duration"]
if not chapters and self.options.get("read_comments", True):
chapters = marks.in_comments(youtube.comments(episode.key, client=http))
if not chapters:
# Nothing published an outline: not the notes, not the description,
# not a listener in the comments. `cut` would hand back the whole
# episode as one segment — correct of it, and what a hand-added
# link wants — but this source's contract is one item per *story*,
# and quietly substituting one item per episode is a different
# product wearing its name.
#
# It is also unusable downstream in a way that is easy to miss. The
# item's `text` would be `opening(transcript)`, the first two
# sentences of a two-hour recording, which is what the ranker and
# the tag filters would then judge it on; and eleven thousand words
# would go to a summariser whose prompt promises it a single story
# with the article the hosts were reading. Better to store nothing and say
# why than to store one wall and three wrong things about it.
#
# Marked done regardless. Reaching here means the episode was
# fetched and read successfully; there was simply nothing to cut
# along. A show that never publishes an outline would otherwise
# re-download a full transcript on every pull, for ever, to throw
# it away again — which is the waste `_already_done` exists to stop.
log.warning(
"%s: no timestamped outline in the notes, the description or "
"the comments, so there are no stories to cut it into — "
"storing nothing for this episode", episode.title)
self._mark_done(episode.key)
return
try:
cues = youtube.transcript(episode.key, client=http)
except youtube.Blocked:
# A `TranscriptUnavailable` too, and the one that must not be
# treated as one here. Waiting is the answer to captions that are
# still being made; to a refusal it is the same nothing on every
# pull, at INFO, with the episode never marked done and nobody
# told. Still not marked done, so the pull after the address is
# clean reads it — but it leaves as a failure, and reaches the
# screen. `details` above raises the same thing the same way.
raise
except youtube.TranscriptUnavailable as exc:
# Not done, so no marker is written and the next pull tries again.
# Auto-captions on a two-hour upload can take hours to appear, and
# an episode without them is worth waiting for rather than
# ingesting as twenty titles with nothing under them.
log.info("%s: %s", episode.title, exc)
return
segments = cut(episode, cues, chapters)
log.info("%s: %d segments", episode.title, len(segments))
for segment in segments:
yield _item(segment, entry, label,
number=self.episode_number(episode.title))
self._mark_done(episode.key)
def check(self) -> list[CheckResult]:
"""A show is a feed, so the feed check is the right one."""
return [check_feed(url, label) for url, label in self.targets()]
def _episode(self, entry: Any, show: str) -> Video | None:
"""A feed entry as a `Video`. None when there is no video to read.
A show whose video this cannot find is not a failure — see
`video_key`, which is the part a subclass replaces. Saying so once,
quietly, beats failing per episode.
"""
key = self.video_key(entry)
if not key:
# Deliberately not "audio-only shows need local transcription",
# which is what this used to say and which was wrong for the five
# biggest shows it fires on: they all post video, they just do not
# say so in the feed. Naming the actual condition is what stops the
# next person going and building a transcriber they did not need.
log.info("no video id in the feed entry for %r — the show may still "
"publish one; see PodcastSource.video_key",
(entry.get("title") or "")[:60])
return None
return Video(
ref=media.youtube(key),
title=(entry.get("title") or "").strip(),
url=entry.get("link") or "",
author=entry.get("author") or show,
published_at=entry_time(entry),
notes=_body(entry),
)
# ------------------------------------------------------------ bookkeeping
def _already_done(self, key: str) -> bool:
"""Whether this episode has been through here successfully.
In the `state` table rather than in a table of its own, because that is
genuinely all the state an episode has: it is done or it is not. An
episode is not a thing you read, so it needs no row of its own, and one
key in a table that already exists needs no migration.
The key is written only on success, so an episode whose captions
YouTube had not finished generating is left unmarked and comes round
again on the next pull.
Without this, a weekly pull would re-fetch and re-cut every transcript
in the window on every run and then discard the lot at the dedup check
— correct, and several wasted seconds per pull.
"""
return bool(self.state.get(self._done_key(key)))
def _mark_done(self, key: str) -> None:
self.state[self._done_key(key)] = datetime.now(UTC).isoformat()
def _done_key(self, key: str) -> str:
"""Namespaced by the plugin, not hard-coded to `podcast`.
Two shows can publish the same video id — a guest episode cross-posted,
a network re-run — and with one shared prefix the second would find the
first's marker and skip an episode it had never read. One subclass per
show means one namespace per show for free.
"""
return f"{self.name}:done:{key}"
# ------------------------------------------------------------------ internals
def _label(entry: Any, url: str) -> str:
"""What one configured show is called. Which outlet that stands for is the
subscription's business — see `Item.subscription`."""
if not isinstance(entry, dict):
return url
return entry.get("label") or url
def _item(segment: Segment, entry: Any, label: str,
number: int | None = None) -> Item:
"""One segment, as the row newsdesk stores.
Three of the field choices are not obvious:
`body` is the transcript, which is both where it belongs and how this
tells `fetch_bodies` not to go and scrape a YouTube page for one. It
is not the `excerpt`, because the excerpt is what the queue row renders,
what the ranker's document is built from and what search indexes, and
sixteen hundred words of speech would swamp all three.
`excerpt` is a placeholder until a summary is written over it. It is the
opening of the transcript rather than empty, because an item with no
excerpt is an empty row in the queue, and the ranker and the tagger both
read it. It used to be accompanied by a `summarised: False` flag, which
was this source telling the summariser that the row was waiting for it.
Nothing needs telling now: the profile declares that these items are
summarised, and an empty `summary` column is the whole of "not yet".
`published_at` is the episode's stamp plus the segment's offset, so twenty
items published in the same instant still sort in the order they were said.
`links` is the reason a segment is a story rather than a topic: the
articles the outline attached to this stretch, typed and roled, so the
reading pane and the summariser read them from the same field an outlet's
links live in rather than from this plugin's own corner of `raw`.
"""
episode = segment.video
return Item(
source_id=segment.external_id,
location=segment.url(),
title=segment.title,
excerpt=opening(segment.transcript),
body=segment.transcript,
author=episode.author,
published_at=episode.published_at + timedelta(seconds=segment.start),
subscription=label,
links=segment.reading(),
# Not `article`, which is what every one of these used to be stored as
# — twenty-three transcripts filed as prose. What the pipeline made,
# not what carried it: this arrived over RSS and left as segments.
kind="segment",
raw={
# The chapter this segment sits under. Per *item* — it changes
# down the episode — so it is here rather than on the
# subscription's `section`, which names a branch of an outlet and
# is the same for everything that subscription hands over.
"heading": segment.heading,
# Where it plays and what it linked to: the record every stored
# segment carries, defined once on `Segment`.
**segment.record(),
"show": label,
"episode": number,
"episode_title": episode.title,
"episode_url": episode.url,
"episode_published": episode.published_at.isoformat(),
"audio_url": _enclosure(entry),
},
)
def _body(entry: Any) -> str:
"""The richest form of the notes the feed offers."""
for content in entry.get("content") or []:
if value := content.get("value"):
return value
return entry.get("summary") or ""
def _enclosure(entry: Any) -> str:
return next(
(e.get("href", "") for e in entry.get("enclosures") or []
if (e.get("type") or "").startswith("audio")),
"",
)
# ----------------------------------------------------------------- dry run
def main(argv: list[str] | None = None) -> int:
"""`python -m desk.sources.podcast <feed url>` — what a pull would store.
Runs the plugin with no store behind it and prints the items instead of
writing them. This is the thing to run when a show changes the shape of
its notes: everything else in the pipeline is downstream of these fields.
"""
import argparse
import sys
parser = argparse.ArgumentParser(prog="python -m desk.sources.podcast")
parser.add_argument("feed", help="the show's RSS feed")
parser.add_argument("--episodes", type=int, default=1)
parser.add_argument("-v", "--verbose", action="store_true")
args = parser.parse_args(argv)
logging.basicConfig(level=logging.INFO if args.verbose else logging.WARNING,
format="%(levelname)-7s %(message)s")
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
source = PodcastSource({
"feeds": [args.feed], "episodes_per_feed": args.episodes,
})
items = list(source.fetch())
if not items:
print("no items — run with -v to see why", file=sys.stderr)
return 1
print(f"{len(items)} items from {args.feed}\n")
heading = None
for item in items:
if item.raw["heading"] != heading:
heading = item.raw["heading"]
print(f" {heading or '(unfiled)'}")
stamp = marks.hhmmss(item.raw["start"])
print(f" {stamp:>8} {item.title[:80]}")
print(f" {item.location}")
for link in item.links:
print(f" → {link.url}")
print(f" {len(item.body.split()):,} words · {item.excerpt[:90]}")
print()
return 0
if __name__ == "__main__":
raise SystemExit(main())
+103
View File
@@ -0,0 +1,103 @@
"""Finds Source subclasses on disk. `desk.sources.build` instantiates them."""
from __future__ import annotations
import importlib.util
import inspect
import logging
import pkgutil
import sys
import hashlib
import threading
from functools import lru_cache
from pathlib import Path
from desk.sources.base import Source
log = logging.getLogger(__name__)
_load_lock = threading.RLock()
def _classes_in_module(module) -> list[type[Source]]:
found = []
for _, obj in inspect.getmembers(module, inspect.isclass):
if not issubclass(obj, Source) or obj is Source:
continue
if inspect.isabstract(obj):
continue
# A class written to be subclassed is not a plugin. `RssSource` is
# the fetching a publisher's plugin inherits; it has no feeds, no
# archive and nothing to switch on.
if obj.__dict__.get("shared", False):
continue
if not getattr(obj, "name", ""):
log.warning("%s.%s has no `name`, skipping", module.__name__, obj.__name__)
continue
# Only take classes defined in this module, not imported into it.
if obj.__module__ != module.__name__:
continue
found.append(obj)
return found
def _load_file(path: Path) -> list[type[Source]]:
path = path.resolve()
try:
stamp = path.stat()
# A concurrent status request must not execute the same plugin twice.
with _load_lock:
return _load_version(path, stamp.st_mtime_ns, stamp.st_size)
except OSError:
log.exception("cannot read plugin %s", path)
return []
@lru_cache(maxsize=128)
def _load_version(path: Path, modified: int, size: int) -> list[type[Source]]:
"""Reuse unchanged plugins while allowing edits to take effect on next use."""
identity = hashlib.sha256(str(path).encode()).hexdigest()[:16]
mod_name = f"newsdesk_plugin_{path.stem}_{identity}"
spec = importlib.util.spec_from_file_location(mod_name, path)
if spec is None or spec.loader is None:
return []
module = importlib.util.module_from_spec(spec)
sys.modules[mod_name] = module
try:
spec.loader.exec_module(module)
except Exception:
sys.modules.pop(mod_name, None)
log.exception("failed to load plugin %s", path)
return []
return _classes_in_module(module)
def available(plugin_paths: list[str] | None = None) -> dict[str, type[Source]]:
"""Map of name -> Source class. Later definitions win, so a user plugin
named `rss` overrides the built-in one, with a warning naming both classes."""
classes: dict[str, type[Source]] = {}
import desk.sources as builtin_pkg
for info in pkgutil.iter_modules(builtin_pkg.__path__):
if info.name == "base":
continue
module = importlib.import_module(f"desk.sources.{info.name}")
for cls in _classes_in_module(module):
classes[cls.name] = cls
for raw in plugin_paths or []:
directory = Path(raw).expanduser()
if not directory.is_dir():
log.warning("plugin_path %s is not a directory", directory)
continue
for path in sorted(directory.glob("*.py")):
if path.name.startswith("_"):
continue
for cls in _load_file(path):
if (previous := classes.get(cls.name)) and previous is not cls:
log.warning("plugin name %s: %s.%s overrides %s.%s",
cls.name, cls.__module__, cls.__name__,
previous.__module__, previous.__name__)
classes[cls.name] = cls
return classes
+236
View File
@@ -0,0 +1,236 @@
"""RSS and Atom feed fetching, as a base class for a publisher's plugin.
Not itself a source: see `RssSource`, which is what a masthead subclasses.
`check_feed` is the same fetching asked whether one address still answers.
"""
from __future__ import annotations
import logging
from collections.abc import Iterator
from datetime import UTC, datetime
from typing import Any
import feedparser
import httpx
from desk.sources.base import (
USER_AGENT, CheckResult, Item, Source, entry_time, http_failure, outlet_name,
strip_html,
)
from desk.tools.http import session
log = logging.getLogger(__name__)
#: Seconds one feed gets, per operation. Which number it is matters far less
#: than there being one at all — see `_read`.
FETCH_TIMEOUT = 30
class FeedsFailed(Exception):
"""Some of a pull's feeds could not be fetched, and the rest were.
Raised at the end of `fetch` rather than where it happened, and that is the
whole of the design: everything the live feeds gave has already been
yielded and stored by the time this lands, so one outlet returning 403
costs you that outlet and not the other nineteen. The runner turns it into
`notices.source_failed`, which is what puts a feed that has been dead all
week on screen — the one failure a reader cannot spot by eye, because a
dead source and a quiet news day look identical.
Worded through `http_failure`, so every line in it reads `HTTP 403` or
`unreachable: ReadTimeout`. That is also what has the notice file it as the
outlet's business rather than as a defect of ours; see
`desk/notices.py`.
"""
def _categories(entry) -> list[str]:
"""Whatever the outlet put in <category>.
The Guardian, Ars Technica, the NYT and Al Jazeera all populate these;
the BBC and Hacker News don't. They're inconsistent enough between outlets
to be useless as a filter — the NYT files people ('Putin, Vladimir V'),
the Guardian files desks ('Television & radio') — so they're carried as a
hint for the tagger rather than shown to you.
"""
seen, out = set(), []
for tag in entry.get("tags") or []:
term = strip_html(tag.get("term", ""))
if term and term.lower() not in seen:
seen.add(term.lower())
out.append(term)
return out
def check_feed(url: str, label: str = "", kind: str = "rss") -> CheckResult:
"""Fetch one feed and say what is in it. Shared with the podcast source,
whose subscription is a feed too."""
try:
r = httpx.get(url, headers={"User-Agent": USER_AGENT}, timeout=20,
follow_redirects=True)
except httpx.HTTPError as exc:
return CheckResult(kind, url, False, f"unreachable: {type(exc).__name__}", label)
if r.status_code != 200:
return CheckResult(kind, url, False, f"HTTP {r.status_code}", label)
parsed = feedparser.parse(r.content)
n = len(parsed.entries)
if n == 0:
reason = parsed.get("bozo_exception")
return CheckResult(kind, url, False, f"0 entries ({reason or 'not a feed?'})", label)
return CheckResult(kind, url, True, f"{n} entries, {_age(parsed.entries)}", label,
outlet=_proposed_outlet(parsed, url, label))
def _read(http: httpx.Client, url: str) -> Any:
"""One feed, fetched here and parsed from the bytes it answered with.
Deliberately not `feedparser.parse(url)`, which is the same call with the
fetching left to somebody else and which goes through urllib with **no
timeout**. A
host that accepts the connection and then says nothing holds the thread for
ever, and the thread it holds is a worker's: the pull is `solo`, so nothing
else pulls behind it; cancellation is cooperative, so Kill cannot reach a
thread parked in a socket read; and a restart releases the lease only to
re-queue the job and stall it on the same feed. One bad feed stopped the
application from pulling at all until somebody edited the subscription.
feedparser offers no way to pass urllib a deadline, so the fetch comes
here, where the client has one.
Two things fall out of doing it here, both of which `check_feed` has had
all along. The request goes out as `USER_AGENT`, which `base.py` promises
every plugin sends and which feedparser was quietly replacing with its own.
And the status is read, so a 403, 410 or 429 — each of which arrives with
an empty body that parses perfectly well as a feed with nothing in it — is
a failure of that feed rather than a shrug about a slow news day.
"""
response = http.get(url, headers={"User-Agent": USER_AGENT})
response.raise_for_status()
# The bytes rather than `.text`, so the document's own XML declaration
# decides its encoding instead of a guess made from a header the outlet
# may never have set.
return feedparser.parse(response.content)
def _proposed_outlet(parsed: Any, url: str, label: str) -> str:
"""What this feed calls itself — but only when you didn't say.
A label you wrote is the better name and the check must never talk you out
of it: "Guardian Tech" splits into an outlet and a section that a feed's
own `<title>` ("Technology | The Guardian") cannot improve on. A label is
only ever *this* when you pasted a URL and named nothing, and then the
alternative is filing your reading under `https://…`, which the feed's
title beats outright.
"""
if label and label != url:
return ""
return outlet_name(strip_html(parsed.feed.get("title", "")))
def _age(entries) -> str:
stamps = [entry_time(e) for e in entries
if e.get("published_parsed") or e.get("updated_parsed")]
if not stamps:
return "no dates"
hours = (datetime.now(UTC) - max(stamps)).total_seconds() / 3600
return f"newest {hours:.0f}h ago" if hours < 48 else f"newest {hours / 24:.0f}d ago"
class RssSource(Source):
"""Feed fetching, for a publisher's plugin to inherit. Not a plugin.
`shared` is what says so, and retiring the generic "Any RSS feed" source
it used to be is §9 of the rework: an outlet is a plugin, so a masthead
has a file that declares its own feeds, marks and policies rather than a
row somebody pasted a URL into. Everything below is unchanged — this is
the same fetching the Guardian, the BBC, Ars Technica, Golem and Hacker
News have always run on, reached now by subclassing rather than by
configuring.
A subclass states its own `name`; the empty one here is deliberate, so
that forgetting it is a plugin the registry skips loudly rather than a
second class claiming the archive of the first.
"""
name = ""
shared = True
#: Inherited by a subclass that states none of its own.
description = "An RSS or Atom feed."
subscription_field = "feeds"
def fetch(self) -> Iterator[Item]:
feeds = self.options.get("feeds", [])
failed: list[str] = []
# One client for the whole pull, so twenty feeds off one host are one
# connection rather than twenty. Closed on the way out however this
# generator ends.
with session(None, timeout=FETCH_TIMEOUT) as http:
for entry in feeds:
# The label is the whole of what an item says about where it came
# from. Which outlet and which section of it that label stands for
# is the subscription's business, settled once when you added it —
# see `CheckResult.outlet`, which is how this plugin has its say.
if isinstance(entry, dict):
url = entry["url"]
label = entry.get("label") or url
else:
url = label = entry
try:
parsed = _read(http, url)
except (httpx.HTTPError, httpx.InvalidURL) as exc:
# Everything this feed's own end can do to us: a status, a
# host that never answers, an address that was mistyped
# when it was pasted. Noted and carried past, because the
# next feed in the list has nothing to do with any of it.
#
# Nothing broader is caught. What else could land here is a
# wrong assumption of ours about the shape of a response,
# and notices.py is explicit that those have to reach you
# as bugs rather than be folded in with the weather.
log.warning("could not fetch %s: %s", url, http_failure(exc))
failed.append(f"{label} ({http_failure(exc)})")
continue
if parsed.bozo and not parsed.entries:
# It answered, and what came back is not a feed. Empty
# rather than failed: the server did its part, and there is
# nothing here for anyone to go and fix.
log.warning("no entries from %s (%s)", url, parsed.get("bozo_exception"))
continue
yield from self._items(parsed, url, label)
if failed:
raise FeedsFailed(
f"{len(failed)} of {len(feeds)} feeds failed: " + "; ".join(failed))
def _items(self, parsed: Any, url: str, label: str) -> Iterator[Item]:
"""One feed's entries, as rows. Split out of `fetch` only so that the
loop over your subscriptions stays readable beside what it catches."""
for e in parsed.entries[: self.options.get("max_items", 60)]:
yield self.item_from_entry(e, url, label, parsed.feed.get("title", ""))
def item_from_entry(self, entry: Any, url: str, label: str, author: str = "") -> Item:
"""Map one parsed entry; outlets may refine fields without owning HTTP."""
link = entry.get("link") or entry.get("id") or ""
body = ""
if entry.get("content"):
body = strip_html(entry.content[0].get("value", ""))
body = body or strip_html(entry.get("summary", ""))
raw: dict[str, Any] = {"feed": url}
if cats := _categories(entry):
raw["categories"] = cats[:12]
return Item(
source_id=entry.get("id") or link,
location=link,
title=strip_html(entry.get("title", "")),
excerpt=body,
author=entry.get("author", "") or author,
published_at=entry_time(entry),
subscription=label,
kind="article",
raw=raw,
)
def check(self) -> list[CheckResult]:
return [check_feed(url, label) for url, label in self.targets()]
+110
View File
@@ -0,0 +1,110 @@
# `desk.sources` — the plugins
## What it is
**"Source" and "plugin" are the same thing.** One `.py` file with a `Source` subclass in it. It answers one question: *where do items come from* — fetch, prepare, yield. Nothing else: no store, no queue, no config beyond its own options.
| | |
|---|---|
| Contract | `base.py` — `Source`, `Item`, `Suggestion`, `CheckResult` |
| Discovery | `registry.available()` — every module in `desk/sources/`, plus every `*.py` in the dirs under `plugin_paths:` in config.yaml (leading `_` skipped) |
| Built-ins | `bluesky`, `mastodon`, `x`, `podcast`. `RssSource` is here too and is **not** a plugin: it is `shared`, which the registry passes over, and a publisher subclasses it |
| Packages | `httpx` fetches, `feedparser` parses what a feed answered with. No plugin framework, no entry points |
| Door | `from desk.sources import Source, Item, build, available, PodcastSource` |
| Override | later definition wins — a user plugin named `bluesky` replaces the built-in, with a warning naming both |
## The three steps
1. **Fetch** — `fetch()` is a generator; hit the network, yield as you go. Put a timeout on every request you make — see Limits.
2. **Prepare** — normalise into `Item`. Over-fetching is fine: the runner dedups on `uid = sha1(source + source_id)`.
3. **Output** — `Item`s. The runner fills `source`, applies the default outlet, and stores them.
`Item(source_id, location, title, excerpt="", body="", author="", published_at=now, subscription="", kind="article", marks=[], raw={}, links=[])`. `published_at` must be an aware datetime; it is stored as UTC. A non-empty `body` means "don't scrape this page" — that's how a podcast segment keeps its transcript. A post whose `location` equals `raw["post_url"]` links nothing and is not fetched either. The source's own labels go in `raw` under `categories`, which the tagger reads as a weak hint. `ItemLink`, exported through the same door, carries an absolute HTTP(S) URL, title and `reading`/`discussion`/`source` role separately from the item's primary destination.
**A plugin may register how to read its own URLs.** `getters = (Getter("bluesky_api", posts.bluesky, posts.is_bluesky, claims=True),)` on the class says that a `bsky.app` post link, from whichever outlet, is read through that function rather than as a web page, while this plugin is active. It gets the page only; which link an item points to is decided in `fetch`. See `docs/architecture.md` §1.1.
**A plugin declares what should be done with its items, and the app does it.** `spec.py` is that declaration — `PluginSpec`, its profiles, and a `SummaryPolicy` and `TaggingPolicy` per profile. Both shared jobs now run off one: a profile says whether its items are summarised and tagged, which stored text each task reads in order of preference, how short is too short for an abstract and how long one should be, and what this outlet wants said or emphasised about its own items. Those instructions travel with each item in a mixed batch and never apply to another outlet's. See `newsdesk/jobs/summarise.py` and `newsdesk/jobs/tagging.py`.
**An outlet is a plugin.** There is no generic "Any RSS feed" source to paste a masthead into — step 19 retired it, because a feed belongs to a publisher and a publisher has extraction, marks, retries and a summary floor of its own. `RssSource` is the fetching those plugins share: subclass it, give it a `name` and a `spec`, declare the feeds. `plugins/ars_technica.py` is the smallest one, a catalogue of a single feed. A feed nobody has written a plugin for keeps its archive and its Show switch and is simply not fetched; the v28 migration names each one in the log.
**Declaring nothing is not declaring no.** A plugin with no `spec` gets an adapted one with both tasks off, and the two jobs read that silence differently on purpose. Nothing undeclared has ever been summarised, so the summariser takes it as no and loses nothing. Tagging has run over every row in the archive since long before this contract, so it takes it as *today's behaviour* — the acquired text, then the feed's own, then an abstract as a last resort — and only a profile that says `enabled=False` out loud is left untagged. A converted outlet must therefore say `enabled=True` where its items were already tagged; Guardian and Hacker News say so. Which field a tagging policy names decides *which* text and not how much of it: the tagger reads the opening few hundred characters whatever is named, because it is choosing from a closed list of ~35 labels and a whole transcript would be thirty times the tokens for the same answer.
**A profile also declares how its items are read.** The drawer is fixed: Summary, Text, Original and a new-tab action, in that order, for every plugin. The content is the plugin's: which slots it fills, what to call them (a segment's text is its Transcript, its original is Watch), and an ordered list of sections per view, each `prose`, `links` or `media` bound to a named field. It cannot add a fourth view, emit markup, or overrule an outlet that refuses framing. `header` holds links shown above every view: the host of the page the item is, the discussion thread of an aggregator's link, the articles a podcast segment discusses. `QueueLayout.destination` adds that host to the queue row too, and is for a plugin whose outlet is not the publisher. Name a preset rather than restating one: `ARTICLE_READER` for a publisher's own article, `LINK_READER` for a link to someone else's page, `FEED_READER` for a walled outlet, `POST_READER` for a social post, `RECORDING_READER` for a recording's segment. The table in `docs/architecture.md` §1.4 says which flow each is. A view may declare what to show `instead` when its own field is empty, with its own label: `ARTICLE_READER`'s Summary control falls through to the feed's own text and calls itself RSS feed, which is what the queue row beside it has always done. The rename is half of it: a publisher's teaser under a control marked Summary reads as our abstract of the piece, and as a poor one.
**A mark and a wall are different facts, and only one of them changes the flow.** `framing` asks whether the browser will render a page in a frame, and the page behind a paywalled item renders perfectly — it is the paywall. `OriginalView.blocked_by` names the marks that refuse the embed for the items carrying one (`paywalled` by default); the layout does not change, because the piece beside it in the same feed is an ordinary article, and the control greys out with the mark's own words (`Mark.framed`). A `Wall` on a profile is the other thing: there is no article behind any URL this outlet publishes, no abstract will ever be written from one, and the destination frames the gate — so the flow differs rather than the ordinary one standing there with two permanently dead controls. `FEED_READER` is that flow, the feed's own text and the way out, and a walled profile declaring an Original view of its destination is refused where it is written. Its reason rides on `ReaderModel.note` as the one line in the bar, since there is no greyed button left to hover. A plugin that learns to get past its wall deletes the declaration and is an ordinary article profile again. The server resolves the declaration against one item and serves the answer; the browser renders it and decides nothing. See `newsdesk/web/reader.py`.
A show subclasses `PodcastSource`, which finds the episode, retrieves its captions once and cuts them at the marks the show published. `plugins/lastweekin_ai.py` is the model: a catalog of one feed, `segment_profile()` — the machinery's own declaration of what it produces, exported through this door for a show to reuse — and nothing of its own beyond how this show numbers an episode and what it wants said about its segments. Each segment carries the articles its outline linked as `links` with the `reading` role, so the reading pane and the summariser read them from the same field an outlet's links live in.
A show whose site already publishes the cut needs none of that. `plugins/ai_daily_brief.py` subclasses `Source` directly: the site lists its editions in `/agent.json`, each as JSON nuggets and a minute-marked transcript, so the plugin makes one segment per nugget from two requests, with no feed, no captions and no summariser.
RSS outlets subclass `RssSource`. Its `item_from_entry(entry, url, label, author)` hook refines one parsed entry while shared code owns HTTP, feed parsing, limits and failure reporting. `plugins/hacker_news.py` uses it to preserve publisher destinations and HN discussion links while keeping points/comment metadata out of preview text. Guardian needs no override. Both declare fixed catalogs; unknown custom feeds remain supported through generic RSS.
**There is no `outlet` and no `section`.** Who publishes a thing is a fact about the subscription it came from, held once on the subscription row and joined when something needs it — so `subscription` is the only attribution an item carries. They used to be fields, copied onto every item, and a rename left four hundred rows disagreeing with the feed they came from until somebody walked the table.
## What a plugin declares about failing
`spec.py` is the other half of the contract: not where items come from but what should be done with them. `Step.retry` is the part of it that runs today.
```python
Step(id="article", tool="extract_article",
inputs={"url": "inputs.original_url"},
outputs={"content": "inputs.article_text"},
retry=RetryPolicy(max_attempts=2, delay_seconds=300, backoff=1.0,
on=["timeout", "unreachable", "server_error", "rate_limited"],
manual=True))
```
**Per step and per plugin, which is the whole reason it is declared.** A transcript that was not ready an hour ago and a host that timed out are different waits, and retrying the whole outlet to get one of them is how a source ends up fetched four times an evening. `max_attempts` counts the first attempt, so the default of 1 is "no retry" and is what every plugin that declares nothing gets — behaviour unchanged from before any of this existed.
**`on` names failure categories from `desk.tools.failures`**, and a name that is not one of them is refused where the plugin is written rather than becoming a policy that silently matches nothing. Name the ones that could answer differently next time. A consent wall is a category and belongs in nobody's list: it is the same answer however many times it is asked for. Where the server named its own wait — a `Retry-After` — that is honoured in place of `delay_seconds`, but it buys no extra attempt: the bound is the plugin's and the wait is the world's.
**`manual` is whether the reader offers a person the retry** the policy would not take on its own. The app still checks that the offer means anything: the plugin must be active, and the item must actually have been tried and come back empty. A target's Fetch switch is *not* checked for a manual retry — Fetch governs the evening's automatic acquisition, not a repair somebody asked for.
## Minimum example
```python
# plugins/mysource.py (or desk/sources/mysource.py)
from desk.sources import Source, Item
class MySource(Source):
name = "mysource" # the key under `sources:` in config.yaml
description = "One line shown in the Sources tab."
subscription_field = "accounts" # or "feeds" for URL-shaped subscriptions
def fetch(self):
for target, label in self.targets(): # subscriptions from config/UI
yield Item(source_id="abc123", location="https://example.com/x",
title="Headline", excerpt="Summary", subscription=target)
```
```yaml
# config.yaml
plugin_paths: ["plugins"]
sources:
mysource:
enabled: true
accounts: ["someone"]
```
Optional, both defaulting to nothing: `discover() -> [Suggestion]` (accounts worth following) and `check() -> [CheckResult]` (does each subscription still resolve — free requests only, it runs for every source at once).
## What a plugin is given
- `self.options` — its own config block. By pull time the subscription list has been replaced with the fetch-enabled rows from the Sources tab.
- `self.state` — a `MutableMapping[str, str]` backed by the database, for what it must remember between pulls (a spend counter, which episodes are done). Driven directly it's a plain dict.
**`self.state` is one namespace shared by every plugin**, not one per source. Nothing prefixes your keys for you, so prefix them yourself: `x.py` writes `x_spend_2026_09`, and `PodcastSource` derives its done-marker key from the subclass name. A bare key like `last_seen` is a collision waiting for the second plugin that wants one, and the symptom is one source silently reading another's bookkeeping.
Helpers worth knowing: `strip_html`, `parse_timestamp`, `entry_time`, `split_label`, `outlet_name`, `MULTIWORD_OUTLETS`, `http_failure`, `USER_AGENT`, `check_feed`.
Two of those read names, and which one you want depends on who wrote the string. `split_label` reads a label *you* wrote and cuts it into outlet and section — "Guardian Tech" into the two — with `MULTIWORD_OUTLETS` as the list of names that contain a space themselves, so "Hacker News Best" does not come out as an outlet called Hacker. `outlet_name` reads a name a *server* gave for itself and cuts the tagline off it, because both a feed's `<title>` and a site's own title tend to arrive as the SEO version.
## Limits
- **One file, no state but `self.state`.** No store, no queue, no cross-source access. That's the deal that keeps a new source one file.
- **A failed target and an empty one are different answers.** Yielding nothing means there was nothing new; raising means something is wrong and you are owed a notice about it — `desk/notices.py` decides from the exception whether it reads as the world misbehaving or as a bug of ours. A source with several subscriptions gets both at once by yielding everything that worked and raising once at the end for what didn't, which is what stops one dead feed costing you the other nineteen. `rss.FeedsFailed` is that exception.
- **`fetch` is synchronous and blocking.** Runs in the worker's thread. Async, rate-limiting and *timeouts* are yours. A slow source holds the pull; a source that fetches without a deadline holds it for ever, because cancellation is cooperative and Kill cannot reach a thread parked in a socket read. The case to watch for is a library that does its own fetching: `feedparser.parse(url)` and `requests` with no `timeout=` both wait indefinitely by default, so `rss.py` fetches the bytes itself and hands the parser a document.
- **No sandbox, no versioning.** A plugin file is imported and trusted; a plugin that raises on import is logged and skipped, one that raises inside `fetch` fails that pull's source.
- **`name` must be unique and set** — no `name`, not discovered.
- **Discovery is flat**: `*.py` in a directory, not packages, not installed distributions.
- **A source can't create jobs.** It yields items; what happens next (body, vectors, tags) is the app's ingest job.
+1012
View File
File diff suppressed because it is too large. Load diff
+203
View File
@@ -0,0 +1,203 @@
"""X (Twitter) via the official API v2. Off by default, because it costs money.
X moved to pay-per-usage billing. As published on the official pricing page,
post reads are $0.005 per resource and user reads are $0.010 per resource,
with pay-per-usage plans capped at 3 million post reads per billing cycle.
Reads of your *own* account's data ("Owned Reads") are $0.001 per resource.
So a config with 30 accounts x 20 posts x 4 polls a day is roughly
30*20*4*30 = 72,000 reads/month = about $360/month. Check the estimate this
plugin prints before you enable it.
This plugin therefore:
- is disabled unless you set `enabled: true`
- refuses to run without `budget_usd_per_month`
- tracks spend in the `state` table and stops when the budget is gone
- caches user IDs so it never pays for a lookup twice
Source for rates: https://docs.x.com/x-api/getting-started/pricing
"""
from __future__ import annotations
from urllib.parse import urlsplit
import logging
import os
from collections.abc import Iterator
from datetime import UTC, datetime
from typing import Any
import httpx
from desk.sources.base import USER_AGENT, Item, Source, Suggestion, parse_timestamp
from desk.sources.spec import (
API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy,
TaggingPolicy, TargetSpec, Tasks,
)
log = logging.getLogger(__name__)
API = "https://api.x.com/2"
POST_READ_USD = 0.005
USER_READ_USD = 0.010
def _is_internal(url: str) -> bool:
"""Match the host boundary so linux.com remains an external article."""
try:
host = (urlsplit(url).hostname or "").lower().rstrip(".")
except ValueError:
return False
return any(host == domain or host.endswith("." + domain)
for domain in ("x.com", "twitter.com"))
class XSource(Source):
name = "x"
default_outlet = "X"
description = "X/Twitter timelines (paid API — off by default)."
#: Declared like the other two networks. The budget, the token and the
#: refusal to run without either stay in the code above: they are what
#: this plugin *does*, and none of them is a thing the shared jobs or the
#: reader need to be told about.
spec = PluginSpec(
api_version=API_VERSION, id=name, version="1", label="X",
targets=TargetSpec(mode="editable", kind="account", defaults=[],
validator="x_handle"),
profiles={"default": Profile(
kind="post",
acquisition=[Step(id="article", tool="extract_article",
inputs={"url": "inputs.original_url"},
outputs={"content": "inputs.article_text"})],
tasks=Tasks(
summary=SummaryPolicy(enabled=False),
tagging=TaggingPolicy(
enabled=True, input_fields=["inputs.feed_text"],
context_fields=["author"]),
),
presentation=POST_READER,
)},
)
def __init__(self, options: dict[str, Any] | None = None) -> None:
super().__init__(options)
token_env = self.options.get("bearer_token_env", "X_BEARER_TOKEN")
self.token = os.environ.get(token_env, "")
self.client = httpx.Client(
timeout=25,
headers={
"Authorization": f"Bearer {self.token}",
"User-Agent": USER_AGENT,
},
)
@property
def enabled(self) -> bool:
return bool(self.options.get("enabled", False))
def estimate_monthly_usd(self, polls_per_day: int = 4) -> float:
n = len(self.options.get("accounts", []))
per_poll = n * int(self.options.get("posts_per_account", 10))
return per_poll * polls_per_day * 30 * POST_READ_USD
# ------------------------------------------------------------ budget
def _spent(self) -> float:
month = datetime.now(UTC).strftime("%Y-%m")
return float(self.state.get(f"x_spend_{month}", "0") or 0)
def _charge(self, usd: float) -> None:
month = datetime.now(UTC).strftime("%Y-%m")
self.state[f"x_spend_{month}"] = str(self._spent() + usd)
def _user_id(self, handle: str) -> str | None:
cache_key = f"x_userid_{handle.lower()}"
cached = self.state.get(cache_key)
if cached:
return cached
r = self.client.get(f"{API}/users/by/username/{handle}")
if r.status_code != 200:
log.warning("x: lookup %s returned %s", handle, r.status_code)
return None
self._charge(USER_READ_USD)
uid = (r.json().get("data") or {}).get("id")
if uid:
self.state[cache_key] = uid
return uid
# ------------------------------------------------------------- ingest
def fetch(self) -> Iterator[Item]:
if not self.token:
log.error("x: no bearer token in $%s",
self.options.get("bearer_token_env", "X_BEARER_TOKEN"))
return
budget = self.options.get("budget_usd_per_month")
if budget is None:
log.error("x: refusing to run without `budget_usd_per_month` in config")
return
per_account = int(self.options.get("posts_per_account", 10))
exclude = ",".join(self.options.get("exclude", ["retweets", "replies"]))
for handle, label in self.targets():
spent = self._spent()
if spent >= float(budget):
log.warning("x: monthly budget of $%.2f exhausted ($%.2f spent), stopping",
float(budget), spent)
return
uid = self._user_id(handle)
if not uid:
continue
r = self.client.get(
f"{API}/users/{uid}/tweets",
params={
"max_results": max(5, min(per_account, 100)),
"exclude": exclude,
"tweet.fields": "created_at,entities,public_metrics",
},
)
if r.status_code != 200:
log.warning("x: timeline %s returned %s: %s",
handle, r.status_code, r.text[:200])
continue
payload = r.json()
posts = payload.get("data", []) or []
self._charge(len(posts) * POST_READ_USD)
for post in posts:
urls = ((post.get("entities") or {}).get("urls") or [])
external = next(
(u.get("expanded_url") for u in urls
if u.get("expanded_url")
and not _is_internal(u["expanded_url"])),
"",
)
yield Item(
source_id=post["id"],
location=external or f"https://x.com/{handle}/status/{post['id']}",
title="",
excerpt=post.get("text", ""),
author=handle,
published_at=parse_timestamp(post.get("created_at", "")),
subscription=label,
kind="post",
raw={
"metrics": post.get("public_metrics", {}),
"post_url": f"https://x.com/{handle}/status/{post['id']}",
},
)
def discover(self) -> list[Suggestion]:
# The follow-graph endpoints are billed per user read, which makes
# graph-walking on X expensive enough that it is not worth doing
# automatically. Use Bluesky/Mastodon discovery instead.
return []
# `check` is left at the default too: a user lookup is a billed read, and
# a check job must never cost money.