Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d666d0937
commit
248ea1636d
36 files changed
+8393
No files matched your search
@@ -0,0 +1,242 @@
|
||||
"""Bluesky via the public AppView. No auth, no key, no bill.
|
||||
|
||||
Public app.bsky.* endpoints can be called directly against
|
||||
https://public.api.bsky.app, which is what this plugin does. Endpoints used:
|
||||
|
||||
app.bsky.feed.getAuthorFeed - posts by one account
|
||||
app.bsky.graph.getFollows - who an account follows (for discovery)
|
||||
app.bsky.graph.getSuggestedFollowsByActor - accounts similar to a given one
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from desk.sources.base import (
|
||||
USER_AGENT, CheckResult, Getter, Item, Source, Suggestion, http_failure,
|
||||
parse_timestamp,
|
||||
)
|
||||
from desk.sources.spec import (
|
||||
API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy,
|
||||
TaggingPolicy, TargetSpec, Tasks,
|
||||
)
|
||||
from desk.tools import posts
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
API = "https://public.api.bsky.app/xrpc"
|
||||
|
||||
|
||||
class BlueskySource(Source):
|
||||
name = "bluesky"
|
||||
description = "Posts from Bluesky accounts (free public API)."
|
||||
default_outlet = "Bluesky"
|
||||
#: A `bsky.app/profile/.../post/...` link from any outlet is read through
|
||||
#: the same public API, not as the web app's script shell.
|
||||
getters = (Getter("bluesky_api", posts.bluesky, posts.is_bluesky, claims=True),)
|
||||
|
||||
#: A network declares no accounts, because which ones you follow are
|
||||
#: yours. `mode="editable"` is that sentence in the contract, and it is
|
||||
#: what puts an Add box under this plugin and nothing but Fetch and Show
|
||||
#: under a publisher's — see §8 of docs/plugin-rework.md.
|
||||
#:
|
||||
#: Both policies are declared as these items already behave rather than
|
||||
#: as anyone would design them fresh, which is step 11's rule that
|
||||
#: silence is not a decision. Posts are tagged today and have been for as
|
||||
#: long as there has been a tagger; a post is already the length an
|
||||
#: abstract would be, so summarising is off and says so.
|
||||
spec = PluginSpec(
|
||||
api_version=API_VERSION, id=name, version="1", label="Bluesky",
|
||||
targets=TargetSpec(mode="editable", kind="account", defaults=[],
|
||||
validator="bluesky_handle"),
|
||||
profiles={"default": Profile(
|
||||
kind="post",
|
||||
# The fetching a link post has always had: `links_only` keeps the
|
||||
# ones that point somewhere, and where they point is extracted
|
||||
# into the field the reader shows under the post itself. A post
|
||||
# linking to nothing carries its own permalink as `location` and
|
||||
# as `raw.post_url`, and the body fetch skips it.
|
||||
acquisition=[Step(id="article", tool="extract_article",
|
||||
inputs={"url": "inputs.original_url"},
|
||||
outputs={"content": "inputs.article_text"})],
|
||||
tasks=Tasks(
|
||||
summary=SummaryPolicy(enabled=False),
|
||||
tagging=TaggingPolicy(
|
||||
enabled=True, input_fields=["inputs.feed_text"],
|
||||
context_fields=["author"]),
|
||||
),
|
||||
presentation=POST_READER,
|
||||
)},
|
||||
)
|
||||
|
||||
def __init__(self, options: dict[str, Any] | None = None) -> None:
|
||||
super().__init__(options)
|
||||
self.client = httpx.Client(
|
||||
timeout=20,
|
||||
headers={"User-Agent": USER_AGENT},
|
||||
)
|
||||
|
||||
def _get(self, method: str, **params) -> dict[str, Any]:
|
||||
r = self.client.get(f"{API}/{method}", params=params)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
# ------------------------------------------------------------- ingest
|
||||
|
||||
def fetch(self) -> Iterator[Item]:
|
||||
limit = int(self.options.get("posts_per_account", 40))
|
||||
skip_replies = self.options.get("skip_replies", True)
|
||||
links_only = self.options.get("links_only", False)
|
||||
|
||||
for handle, label in self.targets():
|
||||
try:
|
||||
data = self._get(
|
||||
"app.bsky.feed.getAuthorFeed",
|
||||
actor=handle,
|
||||
limit=min(limit, 100),
|
||||
filter="posts_no_replies" if skip_replies else "posts_with_replies",
|
||||
)
|
||||
except httpx.HTTPError as exc:
|
||||
log.warning("bluesky: %s failed (%s)", handle, exc)
|
||||
continue
|
||||
|
||||
for entry in data.get("feed", []):
|
||||
post = entry.get("post") or {}
|
||||
record = post.get("record") or {}
|
||||
author = post.get("author") or {}
|
||||
text = record.get("text", "")
|
||||
link = self._external_link(post)
|
||||
|
||||
if links_only and not link:
|
||||
continue
|
||||
|
||||
rkey = (post.get("uri") or "").rsplit("/", 1)[-1]
|
||||
yield Item(
|
||||
source_id=post.get("uri", ""),
|
||||
location=link
|
||||
or f"https://bsky.app/profile/{author.get('handle', handle)}/post/{rkey}",
|
||||
title="", # base class derives one from the text
|
||||
excerpt=text,
|
||||
author=author.get("displayName") or author.get("handle", handle),
|
||||
published_at=parse_timestamp(record.get("createdAt", "")),
|
||||
subscription=label,
|
||||
kind="post",
|
||||
raw={
|
||||
"likes": post.get("likeCount", 0),
|
||||
"reposts": post.get("repostCount", 0),
|
||||
"post_url": f"https://bsky.app/profile/"
|
||||
f"{author.get('handle', handle)}/post/{rkey}",
|
||||
},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _external_link(post: dict[str, Any]) -> str:
|
||||
"""Prefer the article a post links to over the post itself."""
|
||||
embed = post.get("embed") or {}
|
||||
external = embed.get("external") or {}
|
||||
if external.get("uri"):
|
||||
return external["uri"]
|
||||
for facet in (post.get("record") or {}).get("facets", []) or []:
|
||||
for feature in facet.get("features", []):
|
||||
if feature.get("$type", "").endswith("#link") and feature.get("uri"):
|
||||
return feature["uri"]
|
||||
return ""
|
||||
|
||||
# --------------------------------------------------------------- check
|
||||
|
||||
def check(self) -> list[CheckResult]:
|
||||
return [self._check_one(handle) for handle, _ in self.targets()]
|
||||
|
||||
def _check_one(self, handle: str) -> CheckResult:
|
||||
try:
|
||||
data = self._get("app.bsky.actor.getProfile", actor=handle)
|
||||
except httpx.HTTPError as exc:
|
||||
return CheckResult("bluesky", handle, False, http_failure(exc))
|
||||
return CheckResult(
|
||||
"bluesky", handle, True,
|
||||
f"{data.get('postsCount', '?')} posts, "
|
||||
f"{data.get('followersCount', '?')} followers",
|
||||
)
|
||||
|
||||
# ----------------------------------------------------------- discovery
|
||||
|
||||
def discover(self) -> list[Suggestion]:
|
||||
settings = self.options.get("discover") or {}
|
||||
if not settings.get("enabled", True):
|
||||
return []
|
||||
|
||||
seeds = [target for target, _ in self.targets()][: settings.get("max_seeds", 25)]
|
||||
if not seeds:
|
||||
return []
|
||||
|
||||
following = {s.lower() for s in seeds}
|
||||
# Fractional, because signal 2 is weighted above signal 1 — so not a
|
||||
# Counter, whose values typeshed (correctly) says are ints.
|
||||
counts: defaultdict[str, float] = defaultdict(float)
|
||||
meta: dict[str, dict[str, str]] = {}
|
||||
|
||||
# Signal 1: accounts that several of your accounts follow.
|
||||
for seed in seeds:
|
||||
for account in self._paged_follows(seed):
|
||||
handle = account.get("handle", "").lower()
|
||||
if not handle or handle in following:
|
||||
continue
|
||||
counts[handle] += 1
|
||||
meta.setdefault(handle, {
|
||||
"display_name": account.get("displayName", ""),
|
||||
"description": (account.get("description") or "")[:160],
|
||||
})
|
||||
|
||||
# Signal 2: Bluesky's own "similar to this account" endpoint.
|
||||
for seed in seeds[:10]:
|
||||
try:
|
||||
data = self._get("app.bsky.graph.getSuggestedFollowsByActor", actor=seed)
|
||||
except httpx.HTTPError:
|
||||
continue
|
||||
for account in data.get("suggestions", []):
|
||||
handle = account.get("handle", "").lower()
|
||||
if not handle or handle in following:
|
||||
continue
|
||||
counts[handle] += 1.5 # weight it slightly above a raw co-follow
|
||||
meta.setdefault(handle, {
|
||||
"display_name": account.get("displayName", ""),
|
||||
"description": (account.get("description") or "")[:160],
|
||||
})
|
||||
|
||||
threshold = settings.get("min_overlap", 3)
|
||||
out = []
|
||||
ranked = sorted(counts.items(), key=lambda kv: -kv[1])[:200]
|
||||
for handle, score in ranked:
|
||||
if score < threshold:
|
||||
break
|
||||
info = meta.get(handle, {})
|
||||
out.append(Suggestion(
|
||||
handle=handle,
|
||||
display_name=info.get("display_name", ""),
|
||||
reason=f"followed by {int(score)} of the accounts you follow",
|
||||
score=float(score),
|
||||
url=f"https://bsky.app/profile/{handle}",
|
||||
))
|
||||
return out
|
||||
|
||||
def _paged_follows(self, actor: str, cap: int = 400) -> list[dict[str, Any]]:
|
||||
out: list[dict[str, Any]] = []
|
||||
cursor: str | None = None
|
||||
while len(out) < cap:
|
||||
params = {"actor": actor, "limit": 100}
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
try:
|
||||
data = self._get("app.bsky.graph.getFollows", **params)
|
||||
except httpx.HTTPError as exc:
|
||||
log.warning("bluesky: follows for %s failed (%s)", actor, exc)
|
||||
break
|
||||
out.extend(data.get("follows", []))
|
||||
cursor = data.get("cursor")
|
||||
if not cursor:
|
||||
break
|
||||
return out
|
||||
Reference in new issue
Block a user