"""A Mastodon or Bluesky post read through its network's public API. Both web apps serve a script shell to a plain request, so `checks` names these getters before `browser`: the API answers in a fraction of a second, needs no Chromium and no login, and returns the post rather than a rendering of it. Each getter builds a small HTML page from the answer, so `scrape` checks and converts it exactly as it would a publisher's page. from desk.tools import posts posts.is_mastodon("https://hachyderm.io/@simontatham/117201594980991062") page = posts.mastodon(url) # checks.Page page = posts.bluesky("https://bsky.app/profile/bsky.app/post/3mv3shqdfuc2e") The Mastodon and Bluesky plugins register them with `scrape` as the `mastodon_api` and `bluesky_api` getters for their URLs; see `Source.getters`. **A thread is read as far as its author continued it.** A post a link points to is often the first of several by the same person; the replies by anyone else are not part of it. X is not here: its API bills every read and needs a token, and x.com shows a browser a login wall. GET https://{host}/api/v1/statuses/{id} Mastodon, any instance GET https://{host}/api/v1/statuses/{id}/context GET https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread """ from __future__ import annotations import html import re from typing import Any from urllib.parse import urlsplit import httpx from desk.tools.checks import Page TIMEOUT = 20.0 #: Says what is asking, the way `desk.sources.base.USER_AGENT` does for the #: plugins; a tool sits below `sources` and cannot import it. USER_AGENT = "newsdesk (+https://github.com/; reading a linked post)" BLUESKY_API = "https://public.api.bsky.app/xrpc" #: `/@user/123` is what an instance links; `/users/user/statuses/123` is the #: ActivityPub id some links carry instead. _MASTODON_PATH = re.compile(r"^/(?:@[^/]+|users/[^/]+/statuses)/(\d+)/?$") _BLUESKY_PATH = re.compile(r"^/profile/([^/]+)/post/([a-z0-9]+)/?$") #: How deep a Bluesky thread is asked for. A self-thread longer than this is #: cut, which is rare and costs the tail rather than the post. THREAD_DEPTH = 20 def _get(url: str, params: dict[str, Any] | None = None) -> httpx.Response: # Module-level, so a test replaces this one function. return httpx.get(url, params=params, timeout=TIMEOUT, headers={"User-Agent": USER_AGENT}, follow_redirects=True) # ---------------------------------------------------------------- Mastodon def _mastodon_id(url: str) -> tuple[str, str] | None: parts = urlsplit(url) found = _MASTODON_PATH.match(parts.path) return (parts.netloc, found.group(1)) if parts.scheme == "https" and found else None def is_mastodon(url: str) -> bool: """Whether `url` has a Mastodon post's shape. Any host: an instance is recognised by the check that found its web app's shell, not by name.""" return _mastodon_id(url) is not None def mastodon(url: str) -> Page | None: """The post at `url`, and its author's continuation, from the instance's API.""" host, status_id = _mastodon_id(url) or ("", "") answer = _get(f"https://{host}/api/v1/statuses/{status_id}") if not _json(answer): # Not an instance: `/@user/123` is a Mastodon address's shape, and # another site can have it. Nothing to say about the page, so the # ordinary path runs. return None if answer.status_code != 200: return Page(answer.status_code, "", url, dict(answer.headers)) status = answer.json() author = status.get("account") or {} thread = [status] context = _get(f"https://{host}/api/v1/statuses/{status_id}/context") if context.status_code == 200 and _json(context): # Descendants come in thread order; the author's continuation is the # chain of their replies, each to the one before. for reply in context.json().get("descendants") or []: if ((reply.get("account") or {}).get("id") == author.get("id") and reply.get("in_reply_to_id") == thread[-1].get("id")): thread.append(reply) name = author.get("display_name") or author.get("acct") or host card = status.get("card") or {} return Page(200, _page( title=f"{name}: {_first_words(_text(status.get('content', '')))}", byline=f"{name} (@{author.get('acct', '')})", published=status.get("created_at", ""), paragraphs=[s.get("content", "") for s in thread], link=(card.get("url", ""), card.get("title", "")), alts=[m.get("description") or "" for s in thread for m in s.get("media_attachments") or []], escaped=False), url) # ---------------------------------------------------------------- Bluesky def _bluesky_uri(url: str) -> str: parts = urlsplit(url) found = _BLUESKY_PATH.match(parts.path) if parts.netloc not in {"bsky.app", "www.bsky.app"} or not found: return "" return f"at://{found.group(1)}/app.bsky.feed.post/{found.group(2)}" def is_bluesky(url: str) -> bool: return bool(_bluesky_uri(url)) def bluesky(url: str) -> Page | None: """The post at `url`, and its author's continuation, from Bluesky's AppView.""" answer = _get(f"{BLUESKY_API}/app.bsky.feed.getPostThread", {"uri": _bluesky_uri(url), "depth": THREAD_DEPTH, "parentHeight": 0}) if answer.status_code != 200: # A post that is gone is a 400 `NotFound` here, not a 404. gone = answer.status_code == 400 and "NotFound" in answer.text return Page(404 if gone else answer.status_code, "", url, dict(answer.headers)) thread = answer.json().get("thread") or {} post = thread.get("post") or {} author = post.get("author") or {} posts = [post] node = thread while True: # The author's own reply, if there is one, is the next part of the thread. nxt = next((r for r in node.get("replies") or [] if ((r.get("post") or {}).get("author") or {}).get("did") == author.get("did")), None) if nxt is None: break posts.append(nxt["post"]) node = nxt record = post.get("record") or {} external = ((post.get("embed") or {}).get("external") or {}) name = author.get("displayName") or author.get("handle", "") return Page(200, _page( title=f"{name}: {_first_words(record.get('text', ''))}", byline=f"{name} (@{author.get('handle', '')})", published=record.get("createdAt", ""), paragraphs=[(p.get("record") or {}).get("text", "") for p in posts], link=(external.get("uri", ""), external.get("title", "")), alts=[image.get("alt") or "" for p in posts for image in (p.get("embed") or {}).get("images") or []], escaped=True), url) # ---------------------------------------------------------------- the page def _page(*, title: str, byline: str, published: str, paragraphs: list[str], link: tuple[str, str], alts: list[str], escaped: bool) -> str: """An article page for the converter. Mastodon's content is HTML already; Bluesky's is text, escaped here and split into paragraphs on blank lines.""" if escaped: paragraphs = ["".join(f"

{html.escape(block).replace(chr(10), '
')}

" for block in re.split(r"\n\s*\n", text) if block.strip()) for text in paragraphs] body = "\n".join(paragraphs) url, label = link if url: body += f'\n

{html.escape(label or url)}

' body += "".join(f"\n

[Image: {html.escape(alt)}]

" for alt in alts if alt.strip()) return (f"{html.escape(title)}" f"

{html.escape(byline)}

" f"
" f"\n{body}\n
") def _json(answer: httpx.Response) -> bool: return "json" in answer.headers.get("content-type", "") def _text(markup: str) -> str: return html.unescape(re.sub(r"<[^>]+>", " ", markup)) def _first_words(text: str, count: int = 8) -> str: words = text.split() return " ".join(words[:count]) + ("…" if len(words) > count else "")