Files
jobboard/desk/sources/mastodon.py
T
WolfandClaude Opus 5.5 248ea1636d Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:17 +02:00

197 lines
7.5 KiB
Python

"""Mastodon via each instance's public REST API.
Uses /api/v1/accounts/lookup, /api/v1/accounts/{id}/statuses and
/api/v1/accounts/{id}/following. Public statuses are readable without a token
on Mastodon 2.7.0 and later, so no app registration is needed for public
accounts. Instances that run in whitelist mode will 401; those get skipped.
Accounts are written as user@instance.tld.
"""
from __future__ import annotations
import logging
import time
from collections import Counter
from collections.abc import Iterator
from typing import Any
import httpx
from desk.sources.base import (
USER_AGENT, CheckResult, Getter, Item, Source, Suggestion, http_failure,
parse_timestamp, strip_html,
)
from desk.sources.spec import (
API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy,
TaggingPolicy, TargetSpec, Tasks,
)
from desk.tools import posts
log = logging.getLogger(__name__)
class MastodonSource(Source):
name = "mastodon"
description = "Public posts from Mastodon accounts (no token needed)."
default_outlet = "Mastodon"
#: A post link on any instance (`/@user/123`) is read through that
#: instance's API. The shape is not proof of Mastodon; a host that does not
#: answer the API in JSON gets the ordinary path.
getters = (Getter("mastodon_api", posts.mastodon, posts.is_mastodon, claims=True),)
#: The same declaration Bluesky makes, for the same reasons — read that
#: one. What differs is the handle: `user@instance.tld` names the server
#: as well as the person, so the check is its own.
spec = PluginSpec(
api_version=API_VERSION, id=name, version="1", label="Mastodon",
targets=TargetSpec(mode="editable", kind="account", defaults=[],
validator="mastodon_handle"),
profiles={"default": Profile(
kind="post",
acquisition=[Step(id="article", tool="extract_article",
inputs={"url": "inputs.original_url"},
outputs={"content": "inputs.article_text"})],
tasks=Tasks(
summary=SummaryPolicy(enabled=False),
tagging=TaggingPolicy(
enabled=True, input_fields=["inputs.feed_text"],
context_fields=["author"]),
),
presentation=POST_READER,
)},
)
def __init__(self, options: dict[str, Any] | None = None) -> None:
super().__init__(options)
self.client = httpx.Client(
timeout=20,
follow_redirects=True,
headers={"User-Agent": USER_AGENT},
)
self._delay = float(self.options.get("request_delay", 0.3))
@staticmethod
def _split(acct: str) -> tuple[str, str]:
acct = acct.lstrip("@")
user, _, host = acct.partition("@")
return user, host
def _get(self, host: str, path: str, **params) -> Any:
time.sleep(self._delay)
r = self.client.get(f"https://{host}/api/v1/{path}", params=params)
r.raise_for_status()
return r.json()
def _lookup(self, acct: str) -> dict[str, Any] | None:
user, host = self._split(acct)
if not host:
log.warning("mastodon: %s is missing an instance (use user@instance)", acct)
return None
try:
return self._get(host, "accounts/lookup", acct=user)
except httpx.HTTPError as exc:
log.warning("mastodon: lookup %s failed (%s)", acct, exc)
return None
# ------------------------------------------------------------- ingest
def fetch(self) -> Iterator[Item]:
limit = int(self.options.get("posts_per_account", 40))
for acct, label in self.targets():
account = self._lookup(acct)
if not account:
continue
_, host = self._split(acct)
try:
statuses = self._get(
host,
f"accounts/{account['id']}/statuses",
limit=min(limit, 40),
exclude_replies=str(self.options.get("skip_replies", True)).lower(),
exclude_reblogs=str(self.options.get("skip_boosts", True)).lower(),
)
except httpx.HTTPError as exc:
log.warning("mastodon: statuses for %s failed (%s)", acct, exc)
continue
for status in statuses:
card = status.get("card") or {}
text = strip_html(status.get("content", ""))
yield Item(
source_id=status.get("uri") or status.get("url", ""),
location=card.get("url") or status.get("url", ""),
title=card.get("title", ""),
excerpt=text,
author=(status.get("account") or {}).get("display_name") or acct,
published_at=parse_timestamp(status.get("created_at", "")),
subscription=label,
kind="post",
raw={
"boosts": status.get("reblogs_count", 0),
"favourites": status.get("favourites_count", 0),
"post_url": status.get("url", ""),
},
)
# --------------------------------------------------------------- check
def check(self) -> list[CheckResult]:
return [self._check_one(acct) for acct, _ in self.targets()]
def _check_one(self, acct: str) -> CheckResult:
user, host = self._split(acct)
if not host:
return CheckResult("mastodon", acct, False,
"missing instance (use user@instance)")
try:
data = self._get(host, "accounts/lookup", acct=user)
except httpx.HTTPError as exc:
return CheckResult("mastodon", acct, False, http_failure(exc))
return CheckResult("mastodon", acct, True,
f"{data.get('statuses_count', '?')} posts")
# ----------------------------------------------------------- discovery
def discover(self) -> list[Suggestion]:
settings = self.options.get("discover") or {}
if not settings.get("enabled", True):
return []
seeds = [target for target, _ in self.targets()]
following = {s.lstrip("@").lower() for s in seeds}
counts: Counter[str] = Counter()
meta: dict[str, str] = {}
for acct in seeds:
account = self._lookup(acct)
if not account:
continue
_, host = self._split(acct)
try:
# Many accounts hide this list; a 401/403 just means skip.
follows = self._get(host, f"accounts/{account['id']}/following", limit=80)
except httpx.HTTPError:
continue
for other in follows:
handle = other.get("acct", "")
if "@" not in handle:
handle = f"{handle}@{host}"
if handle.lower() in following:
continue
counts[handle] += 1
meta.setdefault(handle, other.get("display_name", ""))
threshold = settings.get("min_overlap", 2)
return [
Suggestion(
handle=handle,
display_name=meta.get(handle, ""),
reason=f"followed by {n} of the accounts you follow",
score=float(n),
url=f"https://{handle.split('@')[-1]}/@{handle.split('@')[0]}",
)
for handle, n in counts.most_common(200)
if n >= threshold
]