Files
jobboard/desk/tools/posts.py
T
WolfandClaude Opus 5.5 248ea1636d Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:17 +02:00

194 lines
8.2 KiB
Python

"""A Mastodon or Bluesky post read through its network's public API.
Both web apps serve a script shell to a plain request, so `checks` names these
getters before `browser`: the API answers in a fraction of a second, needs no
Chromium and no login, and returns the post rather than a rendering of it.
Each getter builds a small HTML page from the answer, so `scrape` checks and
converts it exactly as it would a publisher's page.
from desk.tools import posts
posts.is_mastodon("https://hachyderm.io/@simontatham/117201594980991062")
page = posts.mastodon(url) # checks.Page
page = posts.bluesky("https://bsky.app/profile/bsky.app/post/3mv3shqdfuc2e")
The Mastodon and Bluesky plugins register them with `scrape` as the
`mastodon_api` and `bluesky_api` getters for their URLs; see `Source.getters`.
**A thread is read as far as its author continued it.** A post a link points
to is often the first of several by the same person; the replies by anyone
else are not part of it.
X is not here: its API bills every read and needs a token, and x.com shows a
browser a login wall.
GET https://{host}/api/v1/statuses/{id} Mastodon, any instance
GET https://{host}/api/v1/statuses/{id}/context
GET https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread
"""
from __future__ import annotations
import html
import re
from typing import Any
from urllib.parse import urlsplit
import httpx
from desk.tools.checks import Page
TIMEOUT = 20.0
#: Says what is asking, the way `desk.sources.base.USER_AGENT` does for the
#: plugins; a tool sits below `sources` and cannot import it.
USER_AGENT = "newsdesk (+https://github.com/; reading a linked post)"
BLUESKY_API = "https://public.api.bsky.app/xrpc"
#: `/@user/123` is what an instance links; `/users/user/statuses/123` is the
#: ActivityPub id some links carry instead.
_MASTODON_PATH = re.compile(r"^/(?:@[^/]+|users/[^/]+/statuses)/(\d+)/?$")
_BLUESKY_PATH = re.compile(r"^/profile/([^/]+)/post/([a-z0-9]+)/?$")
#: How deep a Bluesky thread is asked for. A self-thread longer than this is
#: cut, which is rare and costs the tail rather than the post.
THREAD_DEPTH = 20
def _get(url: str, params: dict[str, Any] | None = None) -> httpx.Response:
# Module-level, so a test replaces this one function.
return httpx.get(url, params=params, timeout=TIMEOUT,
headers={"User-Agent": USER_AGENT}, follow_redirects=True)
# ---------------------------------------------------------------- Mastodon
def _mastodon_id(url: str) -> tuple[str, str] | None:
parts = urlsplit(url)
found = _MASTODON_PATH.match(parts.path)
return (parts.netloc, found.group(1)) if parts.scheme == "https" and found else None
def is_mastodon(url: str) -> bool:
"""Whether `url` has a Mastodon post's shape. Any host: an instance is
recognised by the check that found its web app's shell, not by name."""
return _mastodon_id(url) is not None
def mastodon(url: str) -> Page | None:
"""The post at `url`, and its author's continuation, from the instance's API."""
host, status_id = _mastodon_id(url) or ("", "")
answer = _get(f"https://{host}/api/v1/statuses/{status_id}")
if not _json(answer):
# Not an instance: `/@user/123` is a Mastodon address's shape, and
# another site can have it. Nothing to say about the page, so the
# ordinary path runs.
return None
if answer.status_code != 200:
return Page(answer.status_code, "", url, dict(answer.headers))
status = answer.json()
author = status.get("account") or {}
thread = [status]
context = _get(f"https://{host}/api/v1/statuses/{status_id}/context")
if context.status_code == 200 and _json(context):
# Descendants come in thread order; the author's continuation is the
# chain of their replies, each to the one before.
for reply in context.json().get("descendants") or []:
if ((reply.get("account") or {}).get("id") == author.get("id")
and reply.get("in_reply_to_id") == thread[-1].get("id")):
thread.append(reply)
name = author.get("display_name") or author.get("acct") or host
card = status.get("card") or {}
return Page(200, _page(
title=f"{name}: {_first_words(_text(status.get('content', '')))}",
byline=f"{name} (@{author.get('acct', '')})",
published=status.get("created_at", ""),
paragraphs=[s.get("content", "") for s in thread],
link=(card.get("url", ""), card.get("title", "")),
alts=[m.get("description") or "" for s in thread
for m in s.get("media_attachments") or []],
escaped=False), url)
# ---------------------------------------------------------------- Bluesky
def _bluesky_uri(url: str) -> str:
parts = urlsplit(url)
found = _BLUESKY_PATH.match(parts.path)
if parts.netloc not in {"bsky.app", "www.bsky.app"} or not found:
return ""
return f"at://{found.group(1)}/app.bsky.feed.post/{found.group(2)}"
def is_bluesky(url: str) -> bool:
return bool(_bluesky_uri(url))
def bluesky(url: str) -> Page | None:
"""The post at `url`, and its author's continuation, from Bluesky's AppView."""
answer = _get(f"{BLUESKY_API}/app.bsky.feed.getPostThread",
{"uri": _bluesky_uri(url), "depth": THREAD_DEPTH, "parentHeight": 0})
if answer.status_code != 200:
# A post that is gone is a 400 `NotFound` here, not a 404.
gone = answer.status_code == 400 and "NotFound" in answer.text
return Page(404 if gone else answer.status_code, "", url, dict(answer.headers))
thread = answer.json().get("thread") or {}
post = thread.get("post") or {}
author = post.get("author") or {}
posts = [post]
node = thread
while True:
# The author's own reply, if there is one, is the next part of the thread.
nxt = next((r for r in node.get("replies") or []
if ((r.get("post") or {}).get("author") or {}).get("did") == author.get("did")),
None)
if nxt is None:
break
posts.append(nxt["post"])
node = nxt
record = post.get("record") or {}
external = ((post.get("embed") or {}).get("external") or {})
name = author.get("displayName") or author.get("handle", "")
return Page(200, _page(
title=f"{name}: {_first_words(record.get('text', ''))}",
byline=f"{name} (@{author.get('handle', '')})",
published=record.get("createdAt", ""),
paragraphs=[(p.get("record") or {}).get("text", "") for p in posts],
link=(external.get("uri", ""), external.get("title", "")),
alts=[image.get("alt") or "" for p in posts
for image in (p.get("embed") or {}).get("images") or []],
escaped=True), url)
# ---------------------------------------------------------------- the page
def _page(*, title: str, byline: str, published: str, paragraphs: list[str],
link: tuple[str, str], alts: list[str], escaped: bool) -> str:
"""An article page for the converter. Mastodon's content is HTML already;
Bluesky's is text, escaped here and split into paragraphs on blank lines."""
if escaped:
paragraphs = ["".join(f"<p>{html.escape(block).replace(chr(10), '<br>')}</p>"
for block in re.split(r"\n\s*\n", text) if block.strip())
for text in paragraphs]
body = "\n".join(paragraphs)
url, label = link
if url:
body += f'\n<p><a href="{html.escape(url)}">{html.escape(label or url)}</a></p>'
body += "".join(f"\n<p>[Image: {html.escape(alt)}]</p>" for alt in alts if alt.strip())
return (f"<!DOCTYPE html><html><head><title>{html.escape(title)}</title></head><body>"
f"<article><header><p>{html.escape(byline)}</p>"
f"<time datetime=\"{html.escape(published)}\">{html.escape(published)}</time></header>"
f"\n{body}\n</article></body></html>")
def _json(answer: httpx.Response) -> bool:
return "json" in answer.headers.get("content-type", "")
def _text(markup: str) -> str:
return html.unescape(re.sub(r"<[^>]+>", " ", markup))
def _first_words(text: str, count: int = 8) -> str:
words = text.split()
return " ".join(words[:count]) + ("…" if len(words) > count else "")