Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d666d0937
commit
248ea1636d
36 files changed
+8393
No files matched your search
@@ -0,0 +1,193 @@
|
||||
"""A Mastodon or Bluesky post read through its network's public API.
|
||||
|
||||
Both web apps serve a script shell to a plain request, so `checks` names these
|
||||
getters before `browser`: the API answers in a fraction of a second, needs no
|
||||
Chromium and no login, and returns the post rather than a rendering of it.
|
||||
Each getter builds a small HTML page from the answer, so `scrape` checks and
|
||||
converts it exactly as it would a publisher's page.
|
||||
|
||||
from desk.tools import posts
|
||||
|
||||
posts.is_mastodon("https://hachyderm.io/@simontatham/117201594980991062")
|
||||
page = posts.mastodon(url) # checks.Page
|
||||
page = posts.bluesky("https://bsky.app/profile/bsky.app/post/3mv3shqdfuc2e")
|
||||
|
||||
The Mastodon and Bluesky plugins register them with `scrape` as the
|
||||
`mastodon_api` and `bluesky_api` getters for their URLs; see `Source.getters`.
|
||||
|
||||
**A thread is read as far as its author continued it.** A post a link points
|
||||
to is often the first of several by the same person; the replies by anyone
|
||||
else are not part of it.
|
||||
|
||||
X is not here: its API bills every read and needs a token, and x.com shows a
|
||||
browser a login wall.
|
||||
|
||||
GET https://{host}/api/v1/statuses/{id} Mastodon, any instance
|
||||
GET https://{host}/api/v1/statuses/{id}/context
|
||||
GET https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
|
||||
from desk.tools.checks import Page
|
||||
|
||||
TIMEOUT = 20.0
|
||||
#: Says what is asking, the way `desk.sources.base.USER_AGENT` does for the
|
||||
#: plugins; a tool sits below `sources` and cannot import it.
|
||||
USER_AGENT = "newsdesk (+https://github.com/; reading a linked post)"
|
||||
BLUESKY_API = "https://public.api.bsky.app/xrpc"
|
||||
|
||||
#: `/@user/123` is what an instance links; `/users/user/statuses/123` is the
|
||||
#: ActivityPub id some links carry instead.
|
||||
_MASTODON_PATH = re.compile(r"^/(?:@[^/]+|users/[^/]+/statuses)/(\d+)/?$")
|
||||
_BLUESKY_PATH = re.compile(r"^/profile/([^/]+)/post/([a-z0-9]+)/?$")
|
||||
|
||||
#: How deep a Bluesky thread is asked for. A self-thread longer than this is
|
||||
#: cut, which is rare and costs the tail rather than the post.
|
||||
THREAD_DEPTH = 20
|
||||
|
||||
|
||||
def _get(url: str, params: dict[str, Any] | None = None) -> httpx.Response:
|
||||
# Module-level, so a test replaces this one function.
|
||||
return httpx.get(url, params=params, timeout=TIMEOUT,
|
||||
headers={"User-Agent": USER_AGENT}, follow_redirects=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- Mastodon
|
||||
|
||||
def _mastodon_id(url: str) -> tuple[str, str] | None:
|
||||
parts = urlsplit(url)
|
||||
found = _MASTODON_PATH.match(parts.path)
|
||||
return (parts.netloc, found.group(1)) if parts.scheme == "https" and found else None
|
||||
|
||||
|
||||
def is_mastodon(url: str) -> bool:
|
||||
"""Whether `url` has a Mastodon post's shape. Any host: an instance is
|
||||
recognised by the check that found its web app's shell, not by name."""
|
||||
return _mastodon_id(url) is not None
|
||||
|
||||
|
||||
def mastodon(url: str) -> Page | None:
|
||||
"""The post at `url`, and its author's continuation, from the instance's API."""
|
||||
host, status_id = _mastodon_id(url) or ("", "")
|
||||
answer = _get(f"https://{host}/api/v1/statuses/{status_id}")
|
||||
if not _json(answer):
|
||||
# Not an instance: `/@user/123` is a Mastodon address's shape, and
|
||||
# another site can have it. Nothing to say about the page, so the
|
||||
# ordinary path runs.
|
||||
return None
|
||||
if answer.status_code != 200:
|
||||
return Page(answer.status_code, "", url, dict(answer.headers))
|
||||
status = answer.json()
|
||||
author = status.get("account") or {}
|
||||
thread = [status]
|
||||
context = _get(f"https://{host}/api/v1/statuses/{status_id}/context")
|
||||
if context.status_code == 200 and _json(context):
|
||||
# Descendants come in thread order; the author's continuation is the
|
||||
# chain of their replies, each to the one before.
|
||||
for reply in context.json().get("descendants") or []:
|
||||
if ((reply.get("account") or {}).get("id") == author.get("id")
|
||||
and reply.get("in_reply_to_id") == thread[-1].get("id")):
|
||||
thread.append(reply)
|
||||
name = author.get("display_name") or author.get("acct") or host
|
||||
card = status.get("card") or {}
|
||||
return Page(200, _page(
|
||||
title=f"{name}: {_first_words(_text(status.get('content', '')))}",
|
||||
byline=f"{name} (@{author.get('acct', '')})",
|
||||
published=status.get("created_at", ""),
|
||||
paragraphs=[s.get("content", "") for s in thread],
|
||||
link=(card.get("url", ""), card.get("title", "")),
|
||||
alts=[m.get("description") or "" for s in thread
|
||||
for m in s.get("media_attachments") or []],
|
||||
escaped=False), url)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- Bluesky
|
||||
|
||||
def _bluesky_uri(url: str) -> str:
|
||||
parts = urlsplit(url)
|
||||
found = _BLUESKY_PATH.match(parts.path)
|
||||
if parts.netloc not in {"bsky.app", "www.bsky.app"} or not found:
|
||||
return ""
|
||||
return f"at://{found.group(1)}/app.bsky.feed.post/{found.group(2)}"
|
||||
|
||||
|
||||
def is_bluesky(url: str) -> bool:
|
||||
return bool(_bluesky_uri(url))
|
||||
|
||||
|
||||
def bluesky(url: str) -> Page | None:
|
||||
"""The post at `url`, and its author's continuation, from Bluesky's AppView."""
|
||||
answer = _get(f"{BLUESKY_API}/app.bsky.feed.getPostThread",
|
||||
{"uri": _bluesky_uri(url), "depth": THREAD_DEPTH, "parentHeight": 0})
|
||||
if answer.status_code != 200:
|
||||
# A post that is gone is a 400 `NotFound` here, not a 404.
|
||||
gone = answer.status_code == 400 and "NotFound" in answer.text
|
||||
return Page(404 if gone else answer.status_code, "", url, dict(answer.headers))
|
||||
thread = answer.json().get("thread") or {}
|
||||
post = thread.get("post") or {}
|
||||
author = post.get("author") or {}
|
||||
posts = [post]
|
||||
node = thread
|
||||
while True:
|
||||
# The author's own reply, if there is one, is the next part of the thread.
|
||||
nxt = next((r for r in node.get("replies") or []
|
||||
if ((r.get("post") or {}).get("author") or {}).get("did") == author.get("did")),
|
||||
None)
|
||||
if nxt is None:
|
||||
break
|
||||
posts.append(nxt["post"])
|
||||
node = nxt
|
||||
record = post.get("record") or {}
|
||||
external = ((post.get("embed") or {}).get("external") or {})
|
||||
name = author.get("displayName") or author.get("handle", "")
|
||||
return Page(200, _page(
|
||||
title=f"{name}: {_first_words(record.get('text', ''))}",
|
||||
byline=f"{name} (@{author.get('handle', '')})",
|
||||
published=record.get("createdAt", ""),
|
||||
paragraphs=[(p.get("record") or {}).get("text", "") for p in posts],
|
||||
link=(external.get("uri", ""), external.get("title", "")),
|
||||
alts=[image.get("alt") or "" for p in posts
|
||||
for image in (p.get("embed") or {}).get("images") or []],
|
||||
escaped=True), url)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the page
|
||||
|
||||
def _page(*, title: str, byline: str, published: str, paragraphs: list[str],
|
||||
link: tuple[str, str], alts: list[str], escaped: bool) -> str:
|
||||
"""An article page for the converter. Mastodon's content is HTML already;
|
||||
Bluesky's is text, escaped here and split into paragraphs on blank lines."""
|
||||
if escaped:
|
||||
paragraphs = ["".join(f"<p>{html.escape(block).replace(chr(10), '<br>')}</p>"
|
||||
for block in re.split(r"\n\s*\n", text) if block.strip())
|
||||
for text in paragraphs]
|
||||
body = "\n".join(paragraphs)
|
||||
url, label = link
|
||||
if url:
|
||||
body += f'\n<p><a href="{html.escape(url)}">{html.escape(label or url)}</a></p>'
|
||||
body += "".join(f"\n<p>[Image: {html.escape(alt)}]</p>" for alt in alts if alt.strip())
|
||||
return (f"<!DOCTYPE html><html><head><title>{html.escape(title)}</title></head><body>"
|
||||
f"<article><header><p>{html.escape(byline)}</p>"
|
||||
f"<time datetime=\"{html.escape(published)}\">{html.escape(published)}</time></header>"
|
||||
f"\n{body}\n</article></body></html>")
|
||||
|
||||
|
||||
def _json(answer: httpx.Response) -> bool:
|
||||
return "json" in answer.headers.get("content-type", "")
|
||||
|
||||
|
||||
def _text(markup: str) -> str:
|
||||
return html.unescape(re.sub(r"<[^>]+>", " ", markup))
|
||||
|
||||
|
||||
def _first_words(text: str, count: int = 8) -> str:
|
||||
words = text.split()
|
||||
return " ".join(words[:count]) + ("…" if len(words) > count else "")
|
||||
Reference in new issue
Block a user