Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d666d0937
commit
248ea1636d
36 files changed
+8393
No files matched your search
@@ -0,0 +1,175 @@
|
||||
"""The plugins: where items come from. One door, used from both sides.
|
||||
|
||||
from desk.sources import Source, Item # writing one
|
||||
from desk.sources import build, available # running them
|
||||
|
||||
A plugin is a file with a `Source` subclass in it — `base.py` is the contract
|
||||
— dropped into this package or into a directory under `plugin_paths:` in
|
||||
config.yaml. It yields `Item`s and touches nothing else: no store, no queue,
|
||||
no config beyond its own `options`. What it needs to remember between pulls
|
||||
goes in `self.state`, a mapping the runner backs with the database. That is
|
||||
what makes a new source one file, and what lets everything above treat every
|
||||
source the same way.
|
||||
|
||||
base.py `Source`, `Item`, `Suggestion`, `CheckResult`, and the
|
||||
helpers every plugin ends up wanting
|
||||
spec.py `PluginSpec`: what a plugin *declares* — its profiles, the
|
||||
shared tools that acquire its content, what the shared
|
||||
summariser and tagger should do with it, how it is laid out
|
||||
registry.py finds the plugins on disk and builds the configured ones
|
||||
bluesky.py, mastodon.py, x.py, podcast.py the built-in ones
|
||||
rss.py feed fetching, which a publisher's plugin subclasses
|
||||
|
||||
Declaring a `spec` is optional in the contract and every plugin installed here
|
||||
declares one: `spec_of` adapts a plugin that declares none, and such a plugin
|
||||
runs exactly as it always has, but nothing about it is a claim its author made.
|
||||
|
||||
`PodcastSource` and `RssSource` are exported next to `Source` because that is
|
||||
how a publisher's plugin is written: subclass the class that already does the
|
||||
fetching, give it a name and a `spec`, and add nothing else.
|
||||
`plugins/lastweekin_ai.py` and `plugins/guardian.py` are the two models.
|
||||
`RssSource` is `shared` and is therefore not a plugin itself — there is no
|
||||
generic "Any RSS feed" source to paste a masthead into.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Protocol
|
||||
from desk.models import Item, ItemLink
|
||||
from desk.sources.base import (
|
||||
MULTIWORD_OUTLETS, USER_AGENT, CheckResult, Getter, Source, Suggestion,
|
||||
entry_time, http_failure, outlet_name, parse_timestamp, split_label, strip_html,
|
||||
)
|
||||
from desk.sources.podcast import PodcastSource, segment_profile
|
||||
from desk.sources.registry import available
|
||||
from desk.sources.rss import RssSource, check_feed
|
||||
from desk.sources.spec import (
|
||||
API_VERSION, ARTICLE_READER, FEED_READER, LINK_READER, POST_READER,
|
||||
RECORDING_READER, RSS, Discovery,
|
||||
ExternalLink, MarkRule, OriginalView, PluginSpec, Presentation, Profile,
|
||||
QueueLayout, RetryPolicy, Section, Step, SummaryPolicy, TaggingPolicy,
|
||||
Target, TargetSpec, Tasks, View, Vocabulary, Wall, problems, spec_of,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"API_VERSION",
|
||||
"ARTICLE_READER",
|
||||
"FEED_READER",
|
||||
"LINK_READER",
|
||||
"MULTIWORD_OUTLETS",
|
||||
"POST_READER",
|
||||
"RECORDING_READER",
|
||||
"RSS",
|
||||
"USER_AGENT",
|
||||
"CheckResult",
|
||||
"Discovery",
|
||||
"ExternalLink",
|
||||
"Getter",
|
||||
"Item",
|
||||
"ItemLink",
|
||||
"MarkRule",
|
||||
"OriginalView",
|
||||
"PluginSpec",
|
||||
"PodcastSource",
|
||||
"Presentation",
|
||||
"Profile",
|
||||
"QueueLayout",
|
||||
"RetryPolicy",
|
||||
"RssSource",
|
||||
"Section",
|
||||
"Source",
|
||||
"SourcesConfig",
|
||||
"Step",
|
||||
"Suggestion",
|
||||
"SummaryPolicy",
|
||||
"TaggingPolicy",
|
||||
"Target",
|
||||
"TargetSpec",
|
||||
"Tasks",
|
||||
"View",
|
||||
"Vocabulary",
|
||||
"Wall",
|
||||
"available",
|
||||
"build",
|
||||
"check_feed",
|
||||
"entry_time",
|
||||
"faults",
|
||||
"http_failure",
|
||||
"outlet_name",
|
||||
"parse_timestamp",
|
||||
"problems",
|
||||
"segment_profile",
|
||||
"spec_of",
|
||||
"split_label",
|
||||
"strip_html",
|
||||
]
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SourcesConfig(Protocol):
|
||||
"""What `build` needs from an app's config: where plugins are found, and
|
||||
the section each configured one gets. Anything with these two attributes
|
||||
will do; newsdesk's `Config` is one."""
|
||||
|
||||
@property
|
||||
def plugin_paths(self) -> list[str]: ...
|
||||
|
||||
@property
|
||||
def sources(self) -> dict[str, Any]: ...
|
||||
|
||||
|
||||
def build(config: SourcesConfig,
|
||||
vocabulary: Vocabulary = Vocabulary()) -> list[Source]:
|
||||
"""Instantiate every source that is both installed and configured.
|
||||
|
||||
`vocabulary` is what this application supports, and an empty one — the
|
||||
default — restricts nothing, which is every caller today. When an app
|
||||
passes its own, a plugin whose declared spec names a tool, kind or
|
||||
component the app does not have is left out with its diagnostics in the
|
||||
log, and the others are built. One outlet's mistake is not an evening
|
||||
without news.
|
||||
|
||||
A plugin that declares no spec is never left out on these grounds: there
|
||||
is nothing in an adapted spec that anyone wrote. See `spec.spec_of`.
|
||||
"""
|
||||
classes = available(config.plugin_paths)
|
||||
instances = []
|
||||
for name, options in config.sources.items():
|
||||
cls = classes.get(name)
|
||||
if cls is None:
|
||||
log.warning("config has `sources.%s` but no plugin defines it (have: %s)",
|
||||
name, ", ".join(sorted(classes)) or "none")
|
||||
continue
|
||||
if broken := faults(cls, vocabulary):
|
||||
log.error("source %s is not loadable here and is disabled:\n %s",
|
||||
name, "\n ".join(broken))
|
||||
continue
|
||||
try:
|
||||
instance = cls(options or {})
|
||||
except Exception:
|
||||
log.exception("source %s could not be constructed; continuing", name)
|
||||
continue
|
||||
if not instance.enabled:
|
||||
log.info("source %s is disabled in config", name)
|
||||
continue
|
||||
instances.append(instance)
|
||||
return instances
|
||||
|
||||
|
||||
def faults(cls: type[Source], vocabulary: Vocabulary) -> list[str]:
|
||||
"""Why this application cannot run this plugin, or nothing.
|
||||
|
||||
A spec that will not even build is caught here rather than at import,
|
||||
because a plugin is free to construct its spec in `__init__`; either way
|
||||
the plugin is disabled and the reason is the message.
|
||||
|
||||
Exported because a plugin left out of a pull is also a row in a list of
|
||||
installed plugins, saying why it is unavailable — and asking that
|
||||
question twice, in two places, is how the two answers start to differ.
|
||||
"""
|
||||
try:
|
||||
return problems(spec_of(cls), vocabulary)
|
||||
except Exception as exc:
|
||||
return [f"its spec could not be read: {exc}"]
|
||||
Reference in new issue
Block a user