Drops news's own sources (Bluesky, Mastodon, X, podcast) and the tools only they use (video, posts, walls), with the PodcastSource and segment_profile exports. scrape, article, checks, browser and framing stay: base.py imports scrape's Getter at module level and scrape imports the other four, so dropping them would mean editing the plugin contract. desk/README.md records the source of truth, the prune and how to refresh. All 371 tests pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
175 lines
6.1 KiB
Python
175 lines
6.1 KiB
Python
"""The plugins: where items come from. One door, used from both sides.
|
|
|
|
from desk.sources import Source, Item # writing one
|
|
from desk.sources import build, available # running them
|
|
|
|
A plugin is a file with a `Source` subclass in it — `base.py` is the contract
|
|
— dropped into this package or into a directory under `plugin_paths:` in
|
|
config.yaml. It yields `Item`s and touches nothing else: no store, no queue,
|
|
no config beyond its own `options`. What it needs to remember between pulls
|
|
goes in `self.state`, a mapping the runner backs with the database. That is
|
|
what makes a new source one file, and what lets everything above treat every
|
|
source the same way.
|
|
|
|
base.py `Source`, `Item`, `Suggestion`, `CheckResult`, and the
|
|
helpers every plugin ends up wanting
|
|
spec.py `PluginSpec`: what a plugin *declares* — its profiles, the
|
|
shared tools that acquire its content, what the shared
|
|
summariser and tagger should do with it, how it is laid out
|
|
registry.py finds the plugins on disk and builds the configured ones
|
|
rss.py feed fetching, which a publisher's plugin subclasses
|
|
|
|
Declaring a `spec` is optional in the contract and every plugin installed here
|
|
declares one: `spec_of` adapts a plugin that declares none, and such a plugin
|
|
runs exactly as it always has, but nothing about it is a claim its author made.
|
|
|
|
`RssSource` is exported next to `Source` because that is how a publisher's
|
|
plugin is written: subclass the class that already does the fetching, give it
|
|
a name and a `spec`, and add nothing else. `RssSource` is `shared` and is
|
|
therefore not a plugin itself — there is no generic "Any RSS feed" source to
|
|
paste a masthead into.
|
|
|
|
roledesk's copy has no built-in sources: news's Bluesky, Mastodon, X and
|
|
podcast plugins are pruned (see `desk/README.md`), and roledesk's one plugin
|
|
is `plugins/ats.py`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from typing import Any, Protocol
|
|
from desk.models import Item, ItemLink
|
|
from desk.sources.base import (
|
|
MULTIWORD_OUTLETS, USER_AGENT, CheckResult, Getter, Source, Suggestion,
|
|
entry_time, http_failure, outlet_name, parse_timestamp, split_label, strip_html,
|
|
)
|
|
from desk.sources.registry import available
|
|
from desk.sources.rss import RssSource, check_feed
|
|
from desk.sources.spec import (
|
|
API_VERSION, ARTICLE_READER, FEED_READER, LINK_READER, POST_READER,
|
|
RECORDING_READER, RSS, Discovery,
|
|
ExternalLink, MarkRule, OriginalView, PluginSpec, Presentation, Profile,
|
|
QueueLayout, RetryPolicy, Section, Step, SummaryPolicy, TaggingPolicy,
|
|
Target, TargetSpec, Tasks, View, Vocabulary, Wall, problems, spec_of,
|
|
)
|
|
|
|
__all__ = [
|
|
"API_VERSION",
|
|
"ARTICLE_READER",
|
|
"FEED_READER",
|
|
"LINK_READER",
|
|
"MULTIWORD_OUTLETS",
|
|
"POST_READER",
|
|
"RECORDING_READER",
|
|
"RSS",
|
|
"USER_AGENT",
|
|
"CheckResult",
|
|
"Discovery",
|
|
"ExternalLink",
|
|
"Getter",
|
|
"Item",
|
|
"ItemLink",
|
|
"MarkRule",
|
|
"OriginalView",
|
|
"PluginSpec",
|
|
"Presentation",
|
|
"Profile",
|
|
"QueueLayout",
|
|
"RetryPolicy",
|
|
"RssSource",
|
|
"Section",
|
|
"Source",
|
|
"SourcesConfig",
|
|
"Step",
|
|
"Suggestion",
|
|
"SummaryPolicy",
|
|
"TaggingPolicy",
|
|
"Target",
|
|
"TargetSpec",
|
|
"Tasks",
|
|
"View",
|
|
"Vocabulary",
|
|
"Wall",
|
|
"available",
|
|
"build",
|
|
"check_feed",
|
|
"entry_time",
|
|
"faults",
|
|
"http_failure",
|
|
"outlet_name",
|
|
"parse_timestamp",
|
|
"problems",
|
|
"spec_of",
|
|
"split_label",
|
|
"strip_html",
|
|
]
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class SourcesConfig(Protocol):
|
|
"""What `build` needs from an app's config: where plugins are found, and
|
|
the section each configured one gets. Anything with these two attributes
|
|
will do; newsdesk's `Config` is one."""
|
|
|
|
@property
|
|
def plugin_paths(self) -> list[str]: ...
|
|
|
|
@property
|
|
def sources(self) -> dict[str, Any]: ...
|
|
|
|
|
|
def build(config: SourcesConfig,
|
|
vocabulary: Vocabulary = Vocabulary()) -> list[Source]:
|
|
"""Instantiate every source that is both installed and configured.
|
|
|
|
`vocabulary` is what this application supports, and an empty one — the
|
|
default — restricts nothing, which is every caller today. When an app
|
|
passes its own, a plugin whose declared spec names a tool, kind or
|
|
component the app does not have is left out with its diagnostics in the
|
|
log, and the others are built. One outlet's mistake is not an evening
|
|
without news.
|
|
|
|
A plugin that declares no spec is never left out on these grounds: there
|
|
is nothing in an adapted spec that anyone wrote. See `spec.spec_of`.
|
|
"""
|
|
classes = available(config.plugin_paths)
|
|
instances = []
|
|
for name, options in config.sources.items():
|
|
cls = classes.get(name)
|
|
if cls is None:
|
|
log.warning("config has `sources.%s` but no plugin defines it (have: %s)",
|
|
name, ", ".join(sorted(classes)) or "none")
|
|
continue
|
|
if broken := faults(cls, vocabulary):
|
|
log.error("source %s is not loadable here and is disabled:\n %s",
|
|
name, "\n ".join(broken))
|
|
continue
|
|
try:
|
|
instance = cls(options or {})
|
|
except Exception:
|
|
log.exception("source %s could not be constructed; continuing", name)
|
|
continue
|
|
if not instance.enabled:
|
|
log.info("source %s is disabled in config", name)
|
|
continue
|
|
instances.append(instance)
|
|
return instances
|
|
|
|
|
|
def faults(cls: type[Source], vocabulary: Vocabulary) -> list[str]:
|
|
"""Why this application cannot run this plugin, or nothing.
|
|
|
|
A spec that will not even build is caught here rather than at import,
|
|
because a plugin is free to construct its spec in `__init__`; either way
|
|
the plugin is disabled and the reason is the message.
|
|
|
|
Exported because a plugin left out of a pull is also a row in a list of
|
|
installed plugins, saying why it is unavailable — and asking that
|
|
question twice, in two places, is how the two answers start to differ.
|
|
"""
|
|
try:
|
|
return problems(spec_of(cls), vocabulary)
|
|
except Exception as exc:
|
|
return [f"its spec could not be read: {exc}"]
|