Files
jobboard/desk/sources/__init__.py
T
WolfandClaude Opus 5.5 248ea1636d Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:17 +02:00

176 lines
6.3 KiB
Python

"""The plugins: where items come from. One door, used from both sides.
from desk.sources import Source, Item # writing one
from desk.sources import build, available # running them
A plugin is a file with a `Source` subclass in it — `base.py` is the contract
— dropped into this package or into a directory under `plugin_paths:` in
config.yaml. It yields `Item`s and touches nothing else: no store, no queue,
no config beyond its own `options`. What it needs to remember between pulls
goes in `self.state`, a mapping the runner backs with the database. That is
what makes a new source one file, and what lets everything above treat every
source the same way.
base.py `Source`, `Item`, `Suggestion`, `CheckResult`, and the
helpers every plugin ends up wanting
spec.py `PluginSpec`: what a plugin *declares* — its profiles, the
shared tools that acquire its content, what the shared
summariser and tagger should do with it, how it is laid out
registry.py finds the plugins on disk and builds the configured ones
bluesky.py, mastodon.py, x.py, podcast.py the built-in ones
rss.py feed fetching, which a publisher's plugin subclasses
Declaring a `spec` is optional in the contract and every plugin installed here
declares one: `spec_of` adapts a plugin that declares none, and such a plugin
runs exactly as it always has, but nothing about it is a claim its author made.
`PodcastSource` and `RssSource` are exported next to `Source` because that is
how a publisher's plugin is written: subclass the class that already does the
fetching, give it a name and a `spec`, and add nothing else.
`plugins/lastweekin_ai.py` and `plugins/guardian.py` are the two models.
`RssSource` is `shared` and is therefore not a plugin itself — there is no
generic "Any RSS feed" source to paste a masthead into.
"""
from __future__ import annotations
import logging
from typing import Any, Protocol
from desk.models import Item, ItemLink
from desk.sources.base import (
MULTIWORD_OUTLETS, USER_AGENT, CheckResult, Getter, Source, Suggestion,
entry_time, http_failure, outlet_name, parse_timestamp, split_label, strip_html,
)
from desk.sources.podcast import PodcastSource, segment_profile
from desk.sources.registry import available
from desk.sources.rss import RssSource, check_feed
from desk.sources.spec import (
API_VERSION, ARTICLE_READER, FEED_READER, LINK_READER, POST_READER,
RECORDING_READER, RSS, Discovery,
ExternalLink, MarkRule, OriginalView, PluginSpec, Presentation, Profile,
QueueLayout, RetryPolicy, Section, Step, SummaryPolicy, TaggingPolicy,
Target, TargetSpec, Tasks, View, Vocabulary, Wall, problems, spec_of,
)
__all__ = [
"API_VERSION",
"ARTICLE_READER",
"FEED_READER",
"LINK_READER",
"MULTIWORD_OUTLETS",
"POST_READER",
"RECORDING_READER",
"RSS",
"USER_AGENT",
"CheckResult",
"Discovery",
"ExternalLink",
"Getter",
"Item",
"ItemLink",
"MarkRule",
"OriginalView",
"PluginSpec",
"PodcastSource",
"Presentation",
"Profile",
"QueueLayout",
"RetryPolicy",
"RssSource",
"Section",
"Source",
"SourcesConfig",
"Step",
"Suggestion",
"SummaryPolicy",
"TaggingPolicy",
"Target",
"TargetSpec",
"Tasks",
"View",
"Vocabulary",
"Wall",
"available",
"build",
"check_feed",
"entry_time",
"faults",
"http_failure",
"outlet_name",
"parse_timestamp",
"problems",
"segment_profile",
"spec_of",
"split_label",
"strip_html",
]
log = logging.getLogger(__name__)
class SourcesConfig(Protocol):
"""What `build` needs from an app's config: where plugins are found, and
the section each configured one gets. Anything with these two attributes
will do; newsdesk's `Config` is one."""
@property
def plugin_paths(self) -> list[str]: ...
@property
def sources(self) -> dict[str, Any]: ...
def build(config: SourcesConfig,
vocabulary: Vocabulary = Vocabulary()) -> list[Source]:
"""Instantiate every source that is both installed and configured.
`vocabulary` is what this application supports, and an empty one — the
default — restricts nothing, which is every caller today. When an app
passes its own, a plugin whose declared spec names a tool, kind or
component the app does not have is left out with its diagnostics in the
log, and the others are built. One outlet's mistake is not an evening
without news.
A plugin that declares no spec is never left out on these grounds: there
is nothing in an adapted spec that anyone wrote. See `spec.spec_of`.
"""
classes = available(config.plugin_paths)
instances = []
for name, options in config.sources.items():
cls = classes.get(name)
if cls is None:
log.warning("config has `sources.%s` but no plugin defines it (have: %s)",
name, ", ".join(sorted(classes)) or "none")
continue
if broken := faults(cls, vocabulary):
log.error("source %s is not loadable here and is disabled:\n %s",
name, "\n ".join(broken))
continue
try:
instance = cls(options or {})
except Exception:
log.exception("source %s could not be constructed; continuing", name)
continue
if not instance.enabled:
log.info("source %s is disabled in config", name)
continue
instances.append(instance)
return instances
def faults(cls: type[Source], vocabulary: Vocabulary) -> list[str]:
"""Why this application cannot run this plugin, or nothing.
A spec that will not even build is caught here rather than at import,
because a plugin is free to construct its spec in `__init__`; either way
the plugin is disabled and the reason is the message.
Exported because a plugin left out of a pull is also a row in a list of
installed plugins, saying why it is unavailable — and asking that
question twice, in two places, is how the two answers start to differ.
"""
try:
return problems(spec_of(cls), vocabulary)
except Exception as exc:
return [f"its spec could not be read: {exc}"]