Bring in desk from news

The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-23 12:29:17 +02:00
1 parent 9d666d0937
commit 248ea1636d
36 files changed
+8393

No files matched your search

+196
View File
@@ -0,0 +1,196 @@
"""Mastodon via each instance's public REST API.
Uses /api/v1/accounts/lookup, /api/v1/accounts/{id}/statuses and
/api/v1/accounts/{id}/following. Public statuses are readable without a token
on Mastodon 2.7.0 and later, so no app registration is needed for public
accounts. Instances that run in whitelist mode will 401; those get skipped.
Accounts are written as user@instance.tld.
"""
from __future__ import annotations
import logging
import time
from collections import Counter
from collections.abc import Iterator
from typing import Any
import httpx
from desk.sources.base import (
USER_AGENT, CheckResult, Getter, Item, Source, Suggestion, http_failure,
parse_timestamp, strip_html,
)
from desk.sources.spec import (
API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy,
TaggingPolicy, TargetSpec, Tasks,
)
from desk.tools import posts
log = logging.getLogger(__name__)
class MastodonSource(Source):
name = "mastodon"
description = "Public posts from Mastodon accounts (no token needed)."
default_outlet = "Mastodon"
#: A post link on any instance (`/@user/123`) is read through that
#: instance's API. The shape is not proof of Mastodon; a host that does not
#: answer the API in JSON gets the ordinary path.
getters = (Getter("mastodon_api", posts.mastodon, posts.is_mastodon, claims=True),)
#: The same declaration Bluesky makes, for the same reasons — read that
#: one. What differs is the handle: `user@instance.tld` names the server
#: as well as the person, so the check is its own.
spec = PluginSpec(
api_version=API_VERSION, id=name, version="1", label="Mastodon",
targets=TargetSpec(mode="editable", kind="account", defaults=[],
validator="mastodon_handle"),
profiles={"default": Profile(
kind="post",
acquisition=[Step(id="article", tool="extract_article",
inputs={"url": "inputs.original_url"},
outputs={"content": "inputs.article_text"})],
tasks=Tasks(
summary=SummaryPolicy(enabled=False),
tagging=TaggingPolicy(
enabled=True, input_fields=["inputs.feed_text"],
context_fields=["author"]),
),
presentation=POST_READER,
)},
)
def __init__(self, options: dict[str, Any] | None = None) -> None:
super().__init__(options)
self.client = httpx.Client(
timeout=20,
follow_redirects=True,
headers={"User-Agent": USER_AGENT},
)
self._delay = float(self.options.get("request_delay", 0.3))
@staticmethod
def _split(acct: str) -> tuple[str, str]:
acct = acct.lstrip("@")
user, _, host = acct.partition("@")
return user, host
def _get(self, host: str, path: str, **params) -> Any:
time.sleep(self._delay)
r = self.client.get(f"https://{host}/api/v1/{path}", params=params)
r.raise_for_status()
return r.json()
def _lookup(self, acct: str) -> dict[str, Any] | None:
user, host = self._split(acct)
if not host:
log.warning("mastodon: %s is missing an instance (use user@instance)", acct)
return None
try:
return self._get(host, "accounts/lookup", acct=user)
except httpx.HTTPError as exc:
log.warning("mastodon: lookup %s failed (%s)", acct, exc)
return None
# ------------------------------------------------------------- ingest
def fetch(self) -> Iterator[Item]:
limit = int(self.options.get("posts_per_account", 40))
for acct, label in self.targets():
account = self._lookup(acct)
if not account:
continue
_, host = self._split(acct)
try:
statuses = self._get(
host,
f"accounts/{account['id']}/statuses",
limit=min(limit, 40),
exclude_replies=str(self.options.get("skip_replies", True)).lower(),
exclude_reblogs=str(self.options.get("skip_boosts", True)).lower(),
)
except httpx.HTTPError as exc:
log.warning("mastodon: statuses for %s failed (%s)", acct, exc)
continue
for status in statuses:
card = status.get("card") or {}
text = strip_html(status.get("content", ""))
yield Item(
source_id=status.get("uri") or status.get("url", ""),
location=card.get("url") or status.get("url", ""),
title=card.get("title", ""),
excerpt=text,
author=(status.get("account") or {}).get("display_name") or acct,
published_at=parse_timestamp(status.get("created_at", "")),
subscription=label,
kind="post",
raw={
"boosts": status.get("reblogs_count", 0),
"favourites": status.get("favourites_count", 0),
"post_url": status.get("url", ""),
},
)
# --------------------------------------------------------------- check
def check(self) -> list[CheckResult]:
return [self._check_one(acct) for acct, _ in self.targets()]
def _check_one(self, acct: str) -> CheckResult:
user, host = self._split(acct)
if not host:
return CheckResult("mastodon", acct, False,
"missing instance (use user@instance)")
try:
data = self._get(host, "accounts/lookup", acct=user)
except httpx.HTTPError as exc:
return CheckResult("mastodon", acct, False, http_failure(exc))
return CheckResult("mastodon", acct, True,
f"{data.get('statuses_count', '?')} posts")
# ----------------------------------------------------------- discovery
def discover(self) -> list[Suggestion]:
settings = self.options.get("discover") or {}
if not settings.get("enabled", True):
return []
seeds = [target for target, _ in self.targets()]
following = {s.lstrip("@").lower() for s in seeds}
counts: Counter[str] = Counter()
meta: dict[str, str] = {}
for acct in seeds:
account = self._lookup(acct)
if not account:
continue
_, host = self._split(acct)
try:
# Many accounts hide this list; a 401/403 just means skip.
follows = self._get(host, f"accounts/{account['id']}/following", limit=80)
except httpx.HTTPError:
continue
for other in follows:
handle = other.get("acct", "")
if "@" not in handle:
handle = f"{handle}@{host}"
if handle.lower() in following:
continue
counts[handle] += 1
meta.setdefault(handle, other.get("display_name", ""))
threshold = settings.get("min_overlap", 2)
return [
Suggestion(
handle=handle,
display_name=meta.get(handle, ""),
reason=f"followed by {n} of the accounts you follow",
score=float(n),
url=f"https://{handle.split('@')[-1]}/@{handle.split('@')[0]}",
)
for handle, n in counts.most_common(200)
if n >= threshold
]