"""The plugins: where items come from. One door, used from both sides. from desk.sources import Source, Item # writing one from desk.sources import build, available # running them A plugin is a file with a `Source` subclass in it — `base.py` is the contract — dropped into this package or into a directory under `plugin_paths:` in config.yaml. It yields `Item`s and touches nothing else: no store, no queue, no config beyond its own `options`. What it needs to remember between pulls goes in `self.state`, a mapping the runner backs with the database. That is what makes a new source one file, and what lets everything above treat every source the same way. base.py `Source`, `Item`, `Suggestion`, `CheckResult`, and the helpers every plugin ends up wanting spec.py `PluginSpec`: what a plugin *declares* — its profiles, the shared tools that acquire its content, what the shared summariser and tagger should do with it, how it is laid out registry.py finds the plugins on disk and builds the configured ones bluesky.py, mastodon.py, x.py, podcast.py the built-in ones rss.py feed fetching, which a publisher's plugin subclasses Declaring a `spec` is optional in the contract and every plugin installed here declares one: `spec_of` adapts a plugin that declares none, and such a plugin runs exactly as it always has, but nothing about it is a claim its author made. `PodcastSource` and `RssSource` are exported next to `Source` because that is how a publisher's plugin is written: subclass the class that already does the fetching, give it a name and a `spec`, and add nothing else. `plugins/lastweekin_ai.py` and `plugins/guardian.py` are the two models. `RssSource` is `shared` and is therefore not a plugin itself — there is no generic "Any RSS feed" source to paste a masthead into. """ from __future__ import annotations import logging from typing import Any, Protocol from desk.models import Item, ItemLink from desk.sources.base import ( MULTIWORD_OUTLETS, USER_AGENT, CheckResult, Getter, Source, Suggestion, entry_time, http_failure, outlet_name, parse_timestamp, split_label, strip_html, ) from desk.sources.podcast import PodcastSource, segment_profile from desk.sources.registry import available from desk.sources.rss import RssSource, check_feed from desk.sources.spec import ( API_VERSION, ARTICLE_READER, FEED_READER, LINK_READER, POST_READER, RECORDING_READER, RSS, Discovery, ExternalLink, MarkRule, OriginalView, PluginSpec, Presentation, Profile, QueueLayout, RetryPolicy, Section, Step, SummaryPolicy, TaggingPolicy, Target, TargetSpec, Tasks, View, Vocabulary, Wall, problems, spec_of, ) __all__ = [ "API_VERSION", "ARTICLE_READER", "FEED_READER", "LINK_READER", "MULTIWORD_OUTLETS", "POST_READER", "RECORDING_READER", "RSS", "USER_AGENT", "CheckResult", "Discovery", "ExternalLink", "Getter", "Item", "ItemLink", "MarkRule", "OriginalView", "PluginSpec", "PodcastSource", "Presentation", "Profile", "QueueLayout", "RetryPolicy", "RssSource", "Section", "Source", "SourcesConfig", "Step", "Suggestion", "SummaryPolicy", "TaggingPolicy", "Target", "TargetSpec", "Tasks", "View", "Vocabulary", "Wall", "available", "build", "check_feed", "entry_time", "faults", "http_failure", "outlet_name", "parse_timestamp", "problems", "segment_profile", "spec_of", "split_label", "strip_html", ] log = logging.getLogger(__name__) class SourcesConfig(Protocol): """What `build` needs from an app's config: where plugins are found, and the section each configured one gets. Anything with these two attributes will do; newsdesk's `Config` is one.""" @property def plugin_paths(self) -> list[str]: ... @property def sources(self) -> dict[str, Any]: ... def build(config: SourcesConfig, vocabulary: Vocabulary = Vocabulary()) -> list[Source]: """Instantiate every source that is both installed and configured. `vocabulary` is what this application supports, and an empty one — the default — restricts nothing, which is every caller today. When an app passes its own, a plugin whose declared spec names a tool, kind or component the app does not have is left out with its diagnostics in the log, and the others are built. One outlet's mistake is not an evening without news. A plugin that declares no spec is never left out on these grounds: there is nothing in an adapted spec that anyone wrote. See `spec.spec_of`. """ classes = available(config.plugin_paths) instances = [] for name, options in config.sources.items(): cls = classes.get(name) if cls is None: log.warning("config has `sources.%s` but no plugin defines it (have: %s)", name, ", ".join(sorted(classes)) or "none") continue if broken := faults(cls, vocabulary): log.error("source %s is not loadable here and is disabled:\n %s", name, "\n ".join(broken)) continue try: instance = cls(options or {}) except Exception: log.exception("source %s could not be constructed; continuing", name) continue if not instance.enabled: log.info("source %s is disabled in config", name) continue instances.append(instance) return instances def faults(cls: type[Source], vocabulary: Vocabulary) -> list[str]: """Why this application cannot run this plugin, or nothing. A spec that will not even build is caught here rather than at import, because a plugin is free to construct its spec in `__init__`; either way the plugin is disabled and the reason is the message. Exported because a plugin left out of a pull is also a row in a list of installed plugins, saying why it is unavailable — and asking that question twice, in two places, is how the two answers start to differ. """ try: return problems(spec_of(cls), vocabulary) except Exception as exc: return [f"its spec could not be read: {exc}"]