Bring in desk from news

The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-23 12:29:17 +02:00
1 parent 9d666d0937
commit 248ea1636d
36 files changed
+8393

No files matched your search

+193
View File
@@ -0,0 +1,193 @@
"""A Mastodon or Bluesky post read through its network's public API.
Both web apps serve a script shell to a plain request, so `checks` names these
getters before `browser`: the API answers in a fraction of a second, needs no
Chromium and no login, and returns the post rather than a rendering of it.
Each getter builds a small HTML page from the answer, so `scrape` checks and
converts it exactly as it would a publisher's page.
from desk.tools import posts
posts.is_mastodon("https://hachyderm.io/@simontatham/117201594980991062")
page = posts.mastodon(url) # checks.Page
page = posts.bluesky("https://bsky.app/profile/bsky.app/post/3mv3shqdfuc2e")
The Mastodon and Bluesky plugins register them with `scrape` as the
`mastodon_api` and `bluesky_api` getters for their URLs; see `Source.getters`.
**A thread is read as far as its author continued it.** A post a link points
to is often the first of several by the same person; the replies by anyone
else are not part of it.
X is not here: its API bills every read and needs a token, and x.com shows a
browser a login wall.
GET https://{host}/api/v1/statuses/{id} Mastodon, any instance
GET https://{host}/api/v1/statuses/{id}/context
GET https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread
"""
from __future__ import annotations
import html
import re
from typing import Any
from urllib.parse import urlsplit
import httpx
from desk.tools.checks import Page
TIMEOUT = 20.0
#: Says what is asking, the way `desk.sources.base.USER_AGENT` does for the
#: plugins; a tool sits below `sources` and cannot import it.
USER_AGENT = "newsdesk (+https://github.com/; reading a linked post)"
BLUESKY_API = "https://public.api.bsky.app/xrpc"
#: `/@user/123` is what an instance links; `/users/user/statuses/123` is the
#: ActivityPub id some links carry instead.
_MASTODON_PATH = re.compile(r"^/(?:@[^/]+|users/[^/]+/statuses)/(\d+)/?$")
_BLUESKY_PATH = re.compile(r"^/profile/([^/]+)/post/([a-z0-9]+)/?$")
#: How deep a Bluesky thread is asked for. A self-thread longer than this is
#: cut, which is rare and costs the tail rather than the post.
THREAD_DEPTH = 20
def _get(url: str, params: dict[str, Any] | None = None) -> httpx.Response:
# Module-level, so a test replaces this one function.
return httpx.get(url, params=params, timeout=TIMEOUT,
headers={"User-Agent": USER_AGENT}, follow_redirects=True)
# ---------------------------------------------------------------- Mastodon
def _mastodon_id(url: str) -> tuple[str, str] | None:
parts = urlsplit(url)
found = _MASTODON_PATH.match(parts.path)
return (parts.netloc, found.group(1)) if parts.scheme == "https" and found else None
def is_mastodon(url: str) -> bool:
"""Whether `url` has a Mastodon post's shape. Any host: an instance is
recognised by the check that found its web app's shell, not by name."""
return _mastodon_id(url) is not None
def mastodon(url: str) -> Page | None:
"""The post at `url`, and its author's continuation, from the instance's API."""
host, status_id = _mastodon_id(url) or ("", "")
answer = _get(f"https://{host}/api/v1/statuses/{status_id}")
if not _json(answer):
# Not an instance: `/@user/123` is a Mastodon address's shape, and
# another site can have it. Nothing to say about the page, so the
# ordinary path runs.
return None
if answer.status_code != 200:
return Page(answer.status_code, "", url, dict(answer.headers))
status = answer.json()
author = status.get("account") or {}
thread = [status]
context = _get(f"https://{host}/api/v1/statuses/{status_id}/context")
if context.status_code == 200 and _json(context):
# Descendants come in thread order; the author's continuation is the
# chain of their replies, each to the one before.
for reply in context.json().get("descendants") or []:
if ((reply.get("account") or {}).get("id") == author.get("id")
and reply.get("in_reply_to_id") == thread[-1].get("id")):
thread.append(reply)
name = author.get("display_name") or author.get("acct") or host
card = status.get("card") or {}
return Page(200, _page(
title=f"{name}: {_first_words(_text(status.get('content', '')))}",
byline=f"{name} (@{author.get('acct', '')})",
published=status.get("created_at", ""),
paragraphs=[s.get("content", "") for s in thread],
link=(card.get("url", ""), card.get("title", "")),
alts=[m.get("description") or "" for s in thread
for m in s.get("media_attachments") or []],
escaped=False), url)
# ---------------------------------------------------------------- Bluesky
def _bluesky_uri(url: str) -> str:
parts = urlsplit(url)
found = _BLUESKY_PATH.match(parts.path)
if parts.netloc not in {"bsky.app", "www.bsky.app"} or not found:
return ""
return f"at://{found.group(1)}/app.bsky.feed.post/{found.group(2)}"
def is_bluesky(url: str) -> bool:
return bool(_bluesky_uri(url))
def bluesky(url: str) -> Page | None:
"""The post at `url`, and its author's continuation, from Bluesky's AppView."""
answer = _get(f"{BLUESKY_API}/app.bsky.feed.getPostThread",
{"uri": _bluesky_uri(url), "depth": THREAD_DEPTH, "parentHeight": 0})
if answer.status_code != 200:
# A post that is gone is a 400 `NotFound` here, not a 404.
gone = answer.status_code == 400 and "NotFound" in answer.text
return Page(404 if gone else answer.status_code, "", url, dict(answer.headers))
thread = answer.json().get("thread") or {}
post = thread.get("post") or {}
author = post.get("author") or {}
posts = [post]
node = thread
while True:
# The author's own reply, if there is one, is the next part of the thread.
nxt = next((r for r in node.get("replies") or []
if ((r.get("post") or {}).get("author") or {}).get("did") == author.get("did")),
None)
if nxt is None:
break
posts.append(nxt["post"])
node = nxt
record = post.get("record") or {}
external = ((post.get("embed") or {}).get("external") or {})
name = author.get("displayName") or author.get("handle", "")
return Page(200, _page(
title=f"{name}: {_first_words(record.get('text', ''))}",
byline=f"{name} (@{author.get('handle', '')})",
published=record.get("createdAt", ""),
paragraphs=[(p.get("record") or {}).get("text", "") for p in posts],
link=(external.get("uri", ""), external.get("title", "")),
alts=[image.get("alt") or "" for p in posts
for image in (p.get("embed") or {}).get("images") or []],
escaped=True), url)
# ---------------------------------------------------------------- the page
def _page(*, title: str, byline: str, published: str, paragraphs: list[str],
link: tuple[str, str], alts: list[str], escaped: bool) -> str:
"""An article page for the converter. Mastodon's content is HTML already;
Bluesky's is text, escaped here and split into paragraphs on blank lines."""
if escaped:
paragraphs = ["".join(f"<p>{html.escape(block).replace(chr(10), '<br>')}</p>"
for block in re.split(r"\n\s*\n", text) if block.strip())
for text in paragraphs]
body = "\n".join(paragraphs)
url, label = link
if url:
body += f'\n<p><a href="{html.escape(url)}">{html.escape(label or url)}</a></p>'
body += "".join(f"\n<p>[Image: {html.escape(alt)}]</p>" for alt in alts if alt.strip())
return (f"<!DOCTYPE html><html><head><title>{html.escape(title)}</title></head><body>"
f"<article><header><p>{html.escape(byline)}</p>"
f"<time datetime=\"{html.escape(published)}\">{html.escape(published)}</time></header>"
f"\n{body}\n</article></body></html>")
def _json(answer: httpx.Response) -> bool:
return "json" in answer.headers.get("content-type", "")
def _text(markup: str) -> str:
return html.unescape(re.sub(r"<[^>]+>", " ", markup))
def _first_words(text: str, count: int = 8) -> str:
words = text.split()
return " ".join(words[:count]) + ("…" if len(words) > count else "")