The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
197 lines
7.5 KiB
Python
197 lines
7.5 KiB
Python
"""Mastodon via each instance's public REST API.
|
|
|
|
Uses /api/v1/accounts/lookup, /api/v1/accounts/{id}/statuses and
|
|
/api/v1/accounts/{id}/following. Public statuses are readable without a token
|
|
on Mastodon 2.7.0 and later, so no app registration is needed for public
|
|
accounts. Instances that run in whitelist mode will 401; those get skipped.
|
|
|
|
Accounts are written as user@instance.tld.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import time
|
|
from collections import Counter
|
|
from collections.abc import Iterator
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from desk.sources.base import (
|
|
USER_AGENT, CheckResult, Getter, Item, Source, Suggestion, http_failure,
|
|
parse_timestamp, strip_html,
|
|
)
|
|
from desk.sources.spec import (
|
|
API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy,
|
|
TaggingPolicy, TargetSpec, Tasks,
|
|
)
|
|
from desk.tools import posts
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class MastodonSource(Source):
|
|
name = "mastodon"
|
|
description = "Public posts from Mastodon accounts (no token needed)."
|
|
default_outlet = "Mastodon"
|
|
#: A post link on any instance (`/@user/123`) is read through that
|
|
#: instance's API. The shape is not proof of Mastodon; a host that does not
|
|
#: answer the API in JSON gets the ordinary path.
|
|
getters = (Getter("mastodon_api", posts.mastodon, posts.is_mastodon, claims=True),)
|
|
|
|
#: The same declaration Bluesky makes, for the same reasons — read that
|
|
#: one. What differs is the handle: `user@instance.tld` names the server
|
|
#: as well as the person, so the check is its own.
|
|
spec = PluginSpec(
|
|
api_version=API_VERSION, id=name, version="1", label="Mastodon",
|
|
targets=TargetSpec(mode="editable", kind="account", defaults=[],
|
|
validator="mastodon_handle"),
|
|
profiles={"default": Profile(
|
|
kind="post",
|
|
acquisition=[Step(id="article", tool="extract_article",
|
|
inputs={"url": "inputs.original_url"},
|
|
outputs={"content": "inputs.article_text"})],
|
|
tasks=Tasks(
|
|
summary=SummaryPolicy(enabled=False),
|
|
tagging=TaggingPolicy(
|
|
enabled=True, input_fields=["inputs.feed_text"],
|
|
context_fields=["author"]),
|
|
),
|
|
presentation=POST_READER,
|
|
)},
|
|
)
|
|
|
|
def __init__(self, options: dict[str, Any] | None = None) -> None:
|
|
super().__init__(options)
|
|
self.client = httpx.Client(
|
|
timeout=20,
|
|
follow_redirects=True,
|
|
headers={"User-Agent": USER_AGENT},
|
|
)
|
|
self._delay = float(self.options.get("request_delay", 0.3))
|
|
|
|
@staticmethod
|
|
def _split(acct: str) -> tuple[str, str]:
|
|
acct = acct.lstrip("@")
|
|
user, _, host = acct.partition("@")
|
|
return user, host
|
|
|
|
def _get(self, host: str, path: str, **params) -> Any:
|
|
time.sleep(self._delay)
|
|
r = self.client.get(f"https://{host}/api/v1/{path}", params=params)
|
|
r.raise_for_status()
|
|
return r.json()
|
|
|
|
def _lookup(self, acct: str) -> dict[str, Any] | None:
|
|
user, host = self._split(acct)
|
|
if not host:
|
|
log.warning("mastodon: %s is missing an instance (use user@instance)", acct)
|
|
return None
|
|
try:
|
|
return self._get(host, "accounts/lookup", acct=user)
|
|
except httpx.HTTPError as exc:
|
|
log.warning("mastodon: lookup %s failed (%s)", acct, exc)
|
|
return None
|
|
|
|
# ------------------------------------------------------------- ingest
|
|
|
|
def fetch(self) -> Iterator[Item]:
|
|
limit = int(self.options.get("posts_per_account", 40))
|
|
for acct, label in self.targets():
|
|
account = self._lookup(acct)
|
|
if not account:
|
|
continue
|
|
_, host = self._split(acct)
|
|
try:
|
|
statuses = self._get(
|
|
host,
|
|
f"accounts/{account['id']}/statuses",
|
|
limit=min(limit, 40),
|
|
exclude_replies=str(self.options.get("skip_replies", True)).lower(),
|
|
exclude_reblogs=str(self.options.get("skip_boosts", True)).lower(),
|
|
)
|
|
except httpx.HTTPError as exc:
|
|
log.warning("mastodon: statuses for %s failed (%s)", acct, exc)
|
|
continue
|
|
|
|
for status in statuses:
|
|
card = status.get("card") or {}
|
|
text = strip_html(status.get("content", ""))
|
|
yield Item(
|
|
source_id=status.get("uri") or status.get("url", ""),
|
|
location=card.get("url") or status.get("url", ""),
|
|
title=card.get("title", ""),
|
|
excerpt=text,
|
|
author=(status.get("account") or {}).get("display_name") or acct,
|
|
published_at=parse_timestamp(status.get("created_at", "")),
|
|
subscription=label,
|
|
kind="post",
|
|
raw={
|
|
"boosts": status.get("reblogs_count", 0),
|
|
"favourites": status.get("favourites_count", 0),
|
|
"post_url": status.get("url", ""),
|
|
},
|
|
)
|
|
|
|
# --------------------------------------------------------------- check
|
|
|
|
def check(self) -> list[CheckResult]:
|
|
return [self._check_one(acct) for acct, _ in self.targets()]
|
|
|
|
def _check_one(self, acct: str) -> CheckResult:
|
|
user, host = self._split(acct)
|
|
if not host:
|
|
return CheckResult("mastodon", acct, False,
|
|
"missing instance (use user@instance)")
|
|
try:
|
|
data = self._get(host, "accounts/lookup", acct=user)
|
|
except httpx.HTTPError as exc:
|
|
return CheckResult("mastodon", acct, False, http_failure(exc))
|
|
return CheckResult("mastodon", acct, True,
|
|
f"{data.get('statuses_count', '?')} posts")
|
|
|
|
# ----------------------------------------------------------- discovery
|
|
|
|
def discover(self) -> list[Suggestion]:
|
|
settings = self.options.get("discover") or {}
|
|
if not settings.get("enabled", True):
|
|
return []
|
|
|
|
seeds = [target for target, _ in self.targets()]
|
|
following = {s.lstrip("@").lower() for s in seeds}
|
|
counts: Counter[str] = Counter()
|
|
meta: dict[str, str] = {}
|
|
|
|
for acct in seeds:
|
|
account = self._lookup(acct)
|
|
if not account:
|
|
continue
|
|
_, host = self._split(acct)
|
|
try:
|
|
# Many accounts hide this list; a 401/403 just means skip.
|
|
follows = self._get(host, f"accounts/{account['id']}/following", limit=80)
|
|
except httpx.HTTPError:
|
|
continue
|
|
for other in follows:
|
|
handle = other.get("acct", "")
|
|
if "@" not in handle:
|
|
handle = f"{handle}@{host}"
|
|
if handle.lower() in following:
|
|
continue
|
|
counts[handle] += 1
|
|
meta.setdefault(handle, other.get("display_name", ""))
|
|
|
|
threshold = settings.get("min_overlap", 2)
|
|
return [
|
|
Suggestion(
|
|
handle=handle,
|
|
display_name=meta.get(handle, ""),
|
|
reason=f"followed by {n} of the accounts you follow",
|
|
score=float(n),
|
|
url=f"https://{handle.split('@')[-1]}/@{handle.split('@')[0]}",
|
|
)
|
|
for handle, n in counts.most_common(200)
|
|
if n >= threshold
|
|
]
|