"""Mastodon via each instance's public REST API. Uses /api/v1/accounts/lookup, /api/v1/accounts/{id}/statuses and /api/v1/accounts/{id}/following. Public statuses are readable without a token on Mastodon 2.7.0 and later, so no app registration is needed for public accounts. Instances that run in whitelist mode will 401; those get skipped. Accounts are written as user@instance.tld. """ from __future__ import annotations import logging import time from collections import Counter from collections.abc import Iterator from typing import Any import httpx from desk.sources.base import ( USER_AGENT, CheckResult, Getter, Item, Source, Suggestion, http_failure, parse_timestamp, strip_html, ) from desk.sources.spec import ( API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, ) from desk.tools import posts log = logging.getLogger(__name__) class MastodonSource(Source): name = "mastodon" description = "Public posts from Mastodon accounts (no token needed)." default_outlet = "Mastodon" #: A post link on any instance (`/@user/123`) is read through that #: instance's API. The shape is not proof of Mastodon; a host that does not #: answer the API in JSON gets the ordinary path. getters = (Getter("mastodon_api", posts.mastodon, posts.is_mastodon, claims=True),) #: The same declaration Bluesky makes, for the same reasons — read that #: one. What differs is the handle: `user@instance.tld` names the server #: as well as the person, so the check is its own. spec = PluginSpec( api_version=API_VERSION, id=name, version="1", label="Mastodon", targets=TargetSpec(mode="editable", kind="account", defaults=[], validator="mastodon_handle"), profiles={"default": Profile( kind="post", acquisition=[Step(id="article", tool="extract_article", inputs={"url": "inputs.original_url"}, outputs={"content": "inputs.article_text"})], tasks=Tasks( summary=SummaryPolicy(enabled=False), tagging=TaggingPolicy( enabled=True, input_fields=["inputs.feed_text"], context_fields=["author"]), ), presentation=POST_READER, )}, ) def __init__(self, options: dict[str, Any] | None = None) -> None: super().__init__(options) self.client = httpx.Client( timeout=20, follow_redirects=True, headers={"User-Agent": USER_AGENT}, ) self._delay = float(self.options.get("request_delay", 0.3)) @staticmethod def _split(acct: str) -> tuple[str, str]: acct = acct.lstrip("@") user, _, host = acct.partition("@") return user, host def _get(self, host: str, path: str, **params) -> Any: time.sleep(self._delay) r = self.client.get(f"https://{host}/api/v1/{path}", params=params) r.raise_for_status() return r.json() def _lookup(self, acct: str) -> dict[str, Any] | None: user, host = self._split(acct) if not host: log.warning("mastodon: %s is missing an instance (use user@instance)", acct) return None try: return self._get(host, "accounts/lookup", acct=user) except httpx.HTTPError as exc: log.warning("mastodon: lookup %s failed (%s)", acct, exc) return None # ------------------------------------------------------------- ingest def fetch(self) -> Iterator[Item]: limit = int(self.options.get("posts_per_account", 40)) for acct, label in self.targets(): account = self._lookup(acct) if not account: continue _, host = self._split(acct) try: statuses = self._get( host, f"accounts/{account['id']}/statuses", limit=min(limit, 40), exclude_replies=str(self.options.get("skip_replies", True)).lower(), exclude_reblogs=str(self.options.get("skip_boosts", True)).lower(), ) except httpx.HTTPError as exc: log.warning("mastodon: statuses for %s failed (%s)", acct, exc) continue for status in statuses: card = status.get("card") or {} text = strip_html(status.get("content", "")) yield Item( source_id=status.get("uri") or status.get("url", ""), location=card.get("url") or status.get("url", ""), title=card.get("title", ""), excerpt=text, author=(status.get("account") or {}).get("display_name") or acct, published_at=parse_timestamp(status.get("created_at", "")), subscription=label, kind="post", raw={ "boosts": status.get("reblogs_count", 0), "favourites": status.get("favourites_count", 0), "post_url": status.get("url", ""), }, ) # --------------------------------------------------------------- check def check(self) -> list[CheckResult]: return [self._check_one(acct) for acct, _ in self.targets()] def _check_one(self, acct: str) -> CheckResult: user, host = self._split(acct) if not host: return CheckResult("mastodon", acct, False, "missing instance (use user@instance)") try: data = self._get(host, "accounts/lookup", acct=user) except httpx.HTTPError as exc: return CheckResult("mastodon", acct, False, http_failure(exc)) return CheckResult("mastodon", acct, True, f"{data.get('statuses_count', '?')} posts") # ----------------------------------------------------------- discovery def discover(self) -> list[Suggestion]: settings = self.options.get("discover") or {} if not settings.get("enabled", True): return [] seeds = [target for target, _ in self.targets()] following = {s.lstrip("@").lower() for s in seeds} counts: Counter[str] = Counter() meta: dict[str, str] = {} for acct in seeds: account = self._lookup(acct) if not account: continue _, host = self._split(acct) try: # Many accounts hide this list; a 401/403 just means skip. follows = self._get(host, f"accounts/{account['id']}/following", limit=80) except httpx.HTTPError: continue for other in follows: handle = other.get("acct", "") if "@" not in handle: handle = f"{handle}@{host}" if handle.lower() in following: continue counts[handle] += 1 meta.setdefault(handle, other.get("display_name", "")) threshold = settings.get("min_overlap", 2) return [ Suggestion( handle=handle, display_name=meta.get(handle, ""), reason=f"followed by {n} of the accounts you follow", score=float(n), url=f"https://{handle.split('@')[-1]}/@{handle.split('@')[0]}", ) for handle, n in counts.most_common(200) if n >= threshold ]