Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d666d0937
commit
248ea1636d
36 files changed
+8393
No files matched your search
@@ -0,0 +1,196 @@
|
||||
"""Mastodon via each instance's public REST API.
|
||||
|
||||
Uses /api/v1/accounts/lookup, /api/v1/accounts/{id}/statuses and
|
||||
/api/v1/accounts/{id}/following. Public statuses are readable without a token
|
||||
on Mastodon 2.7.0 and later, so no app registration is needed for public
|
||||
accounts. Instances that run in whitelist mode will 401; those get skipped.
|
||||
|
||||
Accounts are written as user@instance.tld.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from collections import Counter
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from desk.sources.base import (
|
||||
USER_AGENT, CheckResult, Getter, Item, Source, Suggestion, http_failure,
|
||||
parse_timestamp, strip_html,
|
||||
)
|
||||
from desk.sources.spec import (
|
||||
API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy,
|
||||
TaggingPolicy, TargetSpec, Tasks,
|
||||
)
|
||||
from desk.tools import posts
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MastodonSource(Source):
|
||||
name = "mastodon"
|
||||
description = "Public posts from Mastodon accounts (no token needed)."
|
||||
default_outlet = "Mastodon"
|
||||
#: A post link on any instance (`/@user/123`) is read through that
|
||||
#: instance's API. The shape is not proof of Mastodon; a host that does not
|
||||
#: answer the API in JSON gets the ordinary path.
|
||||
getters = (Getter("mastodon_api", posts.mastodon, posts.is_mastodon, claims=True),)
|
||||
|
||||
#: The same declaration Bluesky makes, for the same reasons — read that
|
||||
#: one. What differs is the handle: `user@instance.tld` names the server
|
||||
#: as well as the person, so the check is its own.
|
||||
spec = PluginSpec(
|
||||
api_version=API_VERSION, id=name, version="1", label="Mastodon",
|
||||
targets=TargetSpec(mode="editable", kind="account", defaults=[],
|
||||
validator="mastodon_handle"),
|
||||
profiles={"default": Profile(
|
||||
kind="post",
|
||||
acquisition=[Step(id="article", tool="extract_article",
|
||||
inputs={"url": "inputs.original_url"},
|
||||
outputs={"content": "inputs.article_text"})],
|
||||
tasks=Tasks(
|
||||
summary=SummaryPolicy(enabled=False),
|
||||
tagging=TaggingPolicy(
|
||||
enabled=True, input_fields=["inputs.feed_text"],
|
||||
context_fields=["author"]),
|
||||
),
|
||||
presentation=POST_READER,
|
||||
)},
|
||||
)
|
||||
|
||||
def __init__(self, options: dict[str, Any] | None = None) -> None:
|
||||
super().__init__(options)
|
||||
self.client = httpx.Client(
|
||||
timeout=20,
|
||||
follow_redirects=True,
|
||||
headers={"User-Agent": USER_AGENT},
|
||||
)
|
||||
self._delay = float(self.options.get("request_delay", 0.3))
|
||||
|
||||
@staticmethod
|
||||
def _split(acct: str) -> tuple[str, str]:
|
||||
acct = acct.lstrip("@")
|
||||
user, _, host = acct.partition("@")
|
||||
return user, host
|
||||
|
||||
def _get(self, host: str, path: str, **params) -> Any:
|
||||
time.sleep(self._delay)
|
||||
r = self.client.get(f"https://{host}/api/v1/{path}", params=params)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def _lookup(self, acct: str) -> dict[str, Any] | None:
|
||||
user, host = self._split(acct)
|
||||
if not host:
|
||||
log.warning("mastodon: %s is missing an instance (use user@instance)", acct)
|
||||
return None
|
||||
try:
|
||||
return self._get(host, "accounts/lookup", acct=user)
|
||||
except httpx.HTTPError as exc:
|
||||
log.warning("mastodon: lookup %s failed (%s)", acct, exc)
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------- ingest
|
||||
|
||||
def fetch(self) -> Iterator[Item]:
|
||||
limit = int(self.options.get("posts_per_account", 40))
|
||||
for acct, label in self.targets():
|
||||
account = self._lookup(acct)
|
||||
if not account:
|
||||
continue
|
||||
_, host = self._split(acct)
|
||||
try:
|
||||
statuses = self._get(
|
||||
host,
|
||||
f"accounts/{account['id']}/statuses",
|
||||
limit=min(limit, 40),
|
||||
exclude_replies=str(self.options.get("skip_replies", True)).lower(),
|
||||
exclude_reblogs=str(self.options.get("skip_boosts", True)).lower(),
|
||||
)
|
||||
except httpx.HTTPError as exc:
|
||||
log.warning("mastodon: statuses for %s failed (%s)", acct, exc)
|
||||
continue
|
||||
|
||||
for status in statuses:
|
||||
card = status.get("card") or {}
|
||||
text = strip_html(status.get("content", ""))
|
||||
yield Item(
|
||||
source_id=status.get("uri") or status.get("url", ""),
|
||||
location=card.get("url") or status.get("url", ""),
|
||||
title=card.get("title", ""),
|
||||
excerpt=text,
|
||||
author=(status.get("account") or {}).get("display_name") or acct,
|
||||
published_at=parse_timestamp(status.get("created_at", "")),
|
||||
subscription=label,
|
||||
kind="post",
|
||||
raw={
|
||||
"boosts": status.get("reblogs_count", 0),
|
||||
"favourites": status.get("favourites_count", 0),
|
||||
"post_url": status.get("url", ""),
|
||||
},
|
||||
)
|
||||
|
||||
# --------------------------------------------------------------- check
|
||||
|
||||
def check(self) -> list[CheckResult]:
|
||||
return [self._check_one(acct) for acct, _ in self.targets()]
|
||||
|
||||
def _check_one(self, acct: str) -> CheckResult:
|
||||
user, host = self._split(acct)
|
||||
if not host:
|
||||
return CheckResult("mastodon", acct, False,
|
||||
"missing instance (use user@instance)")
|
||||
try:
|
||||
data = self._get(host, "accounts/lookup", acct=user)
|
||||
except httpx.HTTPError as exc:
|
||||
return CheckResult("mastodon", acct, False, http_failure(exc))
|
||||
return CheckResult("mastodon", acct, True,
|
||||
f"{data.get('statuses_count', '?')} posts")
|
||||
|
||||
# ----------------------------------------------------------- discovery
|
||||
|
||||
def discover(self) -> list[Suggestion]:
|
||||
settings = self.options.get("discover") or {}
|
||||
if not settings.get("enabled", True):
|
||||
return []
|
||||
|
||||
seeds = [target for target, _ in self.targets()]
|
||||
following = {s.lstrip("@").lower() for s in seeds}
|
||||
counts: Counter[str] = Counter()
|
||||
meta: dict[str, str] = {}
|
||||
|
||||
for acct in seeds:
|
||||
account = self._lookup(acct)
|
||||
if not account:
|
||||
continue
|
||||
_, host = self._split(acct)
|
||||
try:
|
||||
# Many accounts hide this list; a 401/403 just means skip.
|
||||
follows = self._get(host, f"accounts/{account['id']}/following", limit=80)
|
||||
except httpx.HTTPError:
|
||||
continue
|
||||
for other in follows:
|
||||
handle = other.get("acct", "")
|
||||
if "@" not in handle:
|
||||
handle = f"{handle}@{host}"
|
||||
if handle.lower() in following:
|
||||
continue
|
||||
counts[handle] += 1
|
||||
meta.setdefault(handle, other.get("display_name", ""))
|
||||
|
||||
threshold = settings.get("min_overlap", 2)
|
||||
return [
|
||||
Suggestion(
|
||||
handle=handle,
|
||||
display_name=meta.get(handle, ""),
|
||||
reason=f"followed by {n} of the accounts you follow",
|
||||
score=float(n),
|
||||
url=f"https://{handle.split('@')[-1]}/@{handle.split('@')[0]}",
|
||||
)
|
||||
for handle, n in counts.most_common(200)
|
||||
if n >= threshold
|
||||
]
|
||||
Reference in new issue
Block a user