diff --git a/desk/README.md b/desk/README.md new file mode 100644 index 0000000..af8fc03 --- /dev/null +++ b/desk/README.md @@ -0,0 +1,23 @@ +# desk + +The plugin contract (`desk.sources`), the model task (`desk.modeltask`), the settings sections (`desk.config`), the shared models and notices, and the tools roledesk uses: `agent`, `embedding`, `http`, `failures`, `marks`. + +## Source of truth + +`../news/desk`. This folder is a downstream copy. A fix lands in news first and is copied here, otherwise the next refresh reverts it. The commit that brought each copy in says which news commit it came from (`from news/desk at `). + +## What was pruned, and why + +news's own sources and the tools only they use: `sources/bluesky.py`, `mastodon.py`, `x.py`, `podcast.py`, and `tools/video/`, `posts.py`, `walls.py`. roledesk's one plugin is `plugins/ats.py`, which imports none of them. `sources/__init__.py` drops the `PodcastSource` and `segment_profile` exports with them, and its and `tools/__init__.py`'s docstrings say what is left. + +Kept although nothing in roledesk calls them: `tools/scrape.py`, because `sources/base.py` imports its `Getter` at module level, and `tools/article.py`, `checks.py`, `browser.py` and `framing.py`, because `scrape.py` imports them. Pruning them would mean editing the contract itself. `tools/credentials.py` stays because `agent.py` imports it. `sources/sources.md` and `tools/tools.md` are news's and still describe the full set. + +## Refreshing + +```bash +git -C ../news archive --format=tar HEAD desk | tar -x -C . +rm -rf desk/sources/bluesky.py desk/sources/mastodon.py desk/sources/x.py desk/sources/podcast.py desk/tools/video desk/tools/posts.py desk/tools/walls.py +git diff desk/sources/__init__.py desk/tools/__init__.py +``` + +The archive overwrites the two `__init__.py` files. If news did not change them, `git checkout -- desk/sources/__init__.py desk/tools/__init__.py` restores the pruned versions; if it did, merge its change into them by hand. Then run `uv run pytest` and commit with `from news/desk at ` in the body. diff --git a/desk/sources/__init__.py b/desk/sources/__init__.py index ff78e37..efcac2c 100644 --- a/desk/sources/__init__.py +++ b/desk/sources/__init__.py @@ -17,19 +17,21 @@ source the same way. shared tools that acquire its content, what the shared summariser and tagger should do with it, how it is laid out registry.py finds the plugins on disk and builds the configured ones - bluesky.py, mastodon.py, x.py, podcast.py the built-in ones rss.py feed fetching, which a publisher's plugin subclasses Declaring a `spec` is optional in the contract and every plugin installed here declares one: `spec_of` adapts a plugin that declares none, and such a plugin runs exactly as it always has, but nothing about it is a claim its author made. -`PodcastSource` and `RssSource` are exported next to `Source` because that is -how a publisher's plugin is written: subclass the class that already does the -fetching, give it a name and a `spec`, and add nothing else. -`plugins/lastweekin_ai.py` and `plugins/guardian.py` are the two models. -`RssSource` is `shared` and is therefore not a plugin itself — there is no -generic "Any RSS feed" source to paste a masthead into. +`RssSource` is exported next to `Source` because that is how a publisher's +plugin is written: subclass the class that already does the fetching, give it +a name and a `spec`, and add nothing else. `RssSource` is `shared` and is +therefore not a plugin itself — there is no generic "Any RSS feed" source to +paste a masthead into. + +roledesk's copy has no built-in sources: news's Bluesky, Mastodon, X and +podcast plugins are pruned (see `desk/README.md`), and roledesk's one plugin +is `plugins/ats.py`. """ from __future__ import annotations @@ -41,7 +43,6 @@ from desk.sources.base import ( MULTIWORD_OUTLETS, USER_AGENT, CheckResult, Getter, Source, Suggestion, entry_time, http_failure, outlet_name, parse_timestamp, split_label, strip_html, ) -from desk.sources.podcast import PodcastSource, segment_profile from desk.sources.registry import available from desk.sources.rss import RssSource, check_feed from desk.sources.spec import ( @@ -71,7 +72,6 @@ __all__ = [ "MarkRule", "OriginalView", "PluginSpec", - "PodcastSource", "Presentation", "Profile", "QueueLayout", @@ -99,7 +99,6 @@ __all__ = [ "outlet_name", "parse_timestamp", "problems", - "segment_profile", "spec_of", "split_label", "strip_html", diff --git a/desk/sources/bluesky.py b/desk/sources/bluesky.py deleted file mode 100644 index d5542cc..0000000 --- a/desk/sources/bluesky.py +++ /dev/null @@ -1,242 +0,0 @@ -"""Bluesky via the public AppView. No auth, no key, no bill. - -Public app.bsky.* endpoints can be called directly against -https://public.api.bsky.app, which is what this plugin does. Endpoints used: - - app.bsky.feed.getAuthorFeed - posts by one account - app.bsky.graph.getFollows - who an account follows (for discovery) - app.bsky.graph.getSuggestedFollowsByActor - accounts similar to a given one -""" - -from __future__ import annotations - -import logging -from collections import defaultdict -from collections.abc import Iterator -from typing import Any - -import httpx - -from desk.sources.base import ( - USER_AGENT, CheckResult, Getter, Item, Source, Suggestion, http_failure, - parse_timestamp, -) -from desk.sources.spec import ( - API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy, - TaggingPolicy, TargetSpec, Tasks, -) -from desk.tools import posts - -log = logging.getLogger(__name__) -API = "https://public.api.bsky.app/xrpc" - - -class BlueskySource(Source): - name = "bluesky" - description = "Posts from Bluesky accounts (free public API)." - default_outlet = "Bluesky" - #: A `bsky.app/profile/.../post/...` link from any outlet is read through - #: the same public API, not as the web app's script shell. - getters = (Getter("bluesky_api", posts.bluesky, posts.is_bluesky, claims=True),) - - #: A network declares no accounts, because which ones you follow are - #: yours. `mode="editable"` is that sentence in the contract, and it is - #: what puts an Add box under this plugin and nothing but Fetch and Show - #: under a publisher's — see §8 of docs/plugin-rework.md. - #: - #: Both policies are declared as these items already behave rather than - #: as anyone would design them fresh, which is step 11's rule that - #: silence is not a decision. Posts are tagged today and have been for as - #: long as there has been a tagger; a post is already the length an - #: abstract would be, so summarising is off and says so. - spec = PluginSpec( - api_version=API_VERSION, id=name, version="1", label="Bluesky", - targets=TargetSpec(mode="editable", kind="account", defaults=[], - validator="bluesky_handle"), - profiles={"default": Profile( - kind="post", - # The fetching a link post has always had: `links_only` keeps the - # ones that point somewhere, and where they point is extracted - # into the field the reader shows under the post itself. A post - # linking to nothing carries its own permalink as `location` and - # as `raw.post_url`, and the body fetch skips it. - acquisition=[Step(id="article", tool="extract_article", - inputs={"url": "inputs.original_url"}, - outputs={"content": "inputs.article_text"})], - tasks=Tasks( - summary=SummaryPolicy(enabled=False), - tagging=TaggingPolicy( - enabled=True, input_fields=["inputs.feed_text"], - context_fields=["author"]), - ), - presentation=POST_READER, - )}, - ) - - def __init__(self, options: dict[str, Any] | None = None) -> None: - super().__init__(options) - self.client = httpx.Client( - timeout=20, - headers={"User-Agent": USER_AGENT}, - ) - - def _get(self, method: str, **params) -> dict[str, Any]: - r = self.client.get(f"{API}/{method}", params=params) - r.raise_for_status() - return r.json() - - # ------------------------------------------------------------- ingest - - def fetch(self) -> Iterator[Item]: - limit = int(self.options.get("posts_per_account", 40)) - skip_replies = self.options.get("skip_replies", True) - links_only = self.options.get("links_only", False) - - for handle, label in self.targets(): - try: - data = self._get( - "app.bsky.feed.getAuthorFeed", - actor=handle, - limit=min(limit, 100), - filter="posts_no_replies" if skip_replies else "posts_with_replies", - ) - except httpx.HTTPError as exc: - log.warning("bluesky: %s failed (%s)", handle, exc) - continue - - for entry in data.get("feed", []): - post = entry.get("post") or {} - record = post.get("record") or {} - author = post.get("author") or {} - text = record.get("text", "") - link = self._external_link(post) - - if links_only and not link: - continue - - rkey = (post.get("uri") or "").rsplit("/", 1)[-1] - yield Item( - source_id=post.get("uri", ""), - location=link - or f"https://bsky.app/profile/{author.get('handle', handle)}/post/{rkey}", - title="", # base class derives one from the text - excerpt=text, - author=author.get("displayName") or author.get("handle", handle), - published_at=parse_timestamp(record.get("createdAt", "")), - subscription=label, - kind="post", - raw={ - "likes": post.get("likeCount", 0), - "reposts": post.get("repostCount", 0), - "post_url": f"https://bsky.app/profile/" - f"{author.get('handle', handle)}/post/{rkey}", - }, - ) - - @staticmethod - def _external_link(post: dict[str, Any]) -> str: - """Prefer the article a post links to over the post itself.""" - embed = post.get("embed") or {} - external = embed.get("external") or {} - if external.get("uri"): - return external["uri"] - for facet in (post.get("record") or {}).get("facets", []) or []: - for feature in facet.get("features", []): - if feature.get("$type", "").endswith("#link") and feature.get("uri"): - return feature["uri"] - return "" - - # --------------------------------------------------------------- check - - def check(self) -> list[CheckResult]: - return [self._check_one(handle) for handle, _ in self.targets()] - - def _check_one(self, handle: str) -> CheckResult: - try: - data = self._get("app.bsky.actor.getProfile", actor=handle) - except httpx.HTTPError as exc: - return CheckResult("bluesky", handle, False, http_failure(exc)) - return CheckResult( - "bluesky", handle, True, - f"{data.get('postsCount', '?')} posts, " - f"{data.get('followersCount', '?')} followers", - ) - - # ----------------------------------------------------------- discovery - - def discover(self) -> list[Suggestion]: - settings = self.options.get("discover") or {} - if not settings.get("enabled", True): - return [] - - seeds = [target for target, _ in self.targets()][: settings.get("max_seeds", 25)] - if not seeds: - return [] - - following = {s.lower() for s in seeds} - # Fractional, because signal 2 is weighted above signal 1 — so not a - # Counter, whose values typeshed (correctly) says are ints. - counts: defaultdict[str, float] = defaultdict(float) - meta: dict[str, dict[str, str]] = {} - - # Signal 1: accounts that several of your accounts follow. - for seed in seeds: - for account in self._paged_follows(seed): - handle = account.get("handle", "").lower() - if not handle or handle in following: - continue - counts[handle] += 1 - meta.setdefault(handle, { - "display_name": account.get("displayName", ""), - "description": (account.get("description") or "")[:160], - }) - - # Signal 2: Bluesky's own "similar to this account" endpoint. - for seed in seeds[:10]: - try: - data = self._get("app.bsky.graph.getSuggestedFollowsByActor", actor=seed) - except httpx.HTTPError: - continue - for account in data.get("suggestions", []): - handle = account.get("handle", "").lower() - if not handle or handle in following: - continue - counts[handle] += 1.5 # weight it slightly above a raw co-follow - meta.setdefault(handle, { - "display_name": account.get("displayName", ""), - "description": (account.get("description") or "")[:160], - }) - - threshold = settings.get("min_overlap", 3) - out = [] - ranked = sorted(counts.items(), key=lambda kv: -kv[1])[:200] - for handle, score in ranked: - if score < threshold: - break - info = meta.get(handle, {}) - out.append(Suggestion( - handle=handle, - display_name=info.get("display_name", ""), - reason=f"followed by {int(score)} of the accounts you follow", - score=float(score), - url=f"https://bsky.app/profile/{handle}", - )) - return out - - def _paged_follows(self, actor: str, cap: int = 400) -> list[dict[str, Any]]: - out: list[dict[str, Any]] = [] - cursor: str | None = None - while len(out) < cap: - params = {"actor": actor, "limit": 100} - if cursor: - params["cursor"] = cursor - try: - data = self._get("app.bsky.graph.getFollows", **params) - except httpx.HTTPError as exc: - log.warning("bluesky: follows for %s failed (%s)", actor, exc) - break - out.extend(data.get("follows", [])) - cursor = data.get("cursor") - if not cursor: - break - return out diff --git a/desk/sources/mastodon.py b/desk/sources/mastodon.py deleted file mode 100644 index 6a157ef..0000000 --- a/desk/sources/mastodon.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Mastodon via each instance's public REST API. - -Uses /api/v1/accounts/lookup, /api/v1/accounts/{id}/statuses and -/api/v1/accounts/{id}/following. Public statuses are readable without a token -on Mastodon 2.7.0 and later, so no app registration is needed for public -accounts. Instances that run in whitelist mode will 401; those get skipped. - -Accounts are written as user@instance.tld. -""" - -from __future__ import annotations - -import logging -import time -from collections import Counter -from collections.abc import Iterator -from typing import Any - -import httpx - -from desk.sources.base import ( - USER_AGENT, CheckResult, Getter, Item, Source, Suggestion, http_failure, - parse_timestamp, strip_html, -) -from desk.sources.spec import ( - API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy, - TaggingPolicy, TargetSpec, Tasks, -) -from desk.tools import posts - -log = logging.getLogger(__name__) - - -class MastodonSource(Source): - name = "mastodon" - description = "Public posts from Mastodon accounts (no token needed)." - default_outlet = "Mastodon" - #: A post link on any instance (`/@user/123`) is read through that - #: instance's API. The shape is not proof of Mastodon; a host that does not - #: answer the API in JSON gets the ordinary path. - getters = (Getter("mastodon_api", posts.mastodon, posts.is_mastodon, claims=True),) - - #: The same declaration Bluesky makes, for the same reasons — read that - #: one. What differs is the handle: `user@instance.tld` names the server - #: as well as the person, so the check is its own. - spec = PluginSpec( - api_version=API_VERSION, id=name, version="1", label="Mastodon", - targets=TargetSpec(mode="editable", kind="account", defaults=[], - validator="mastodon_handle"), - profiles={"default": Profile( - kind="post", - acquisition=[Step(id="article", tool="extract_article", - inputs={"url": "inputs.original_url"}, - outputs={"content": "inputs.article_text"})], - tasks=Tasks( - summary=SummaryPolicy(enabled=False), - tagging=TaggingPolicy( - enabled=True, input_fields=["inputs.feed_text"], - context_fields=["author"]), - ), - presentation=POST_READER, - )}, - ) - - def __init__(self, options: dict[str, Any] | None = None) -> None: - super().__init__(options) - self.client = httpx.Client( - timeout=20, - follow_redirects=True, - headers={"User-Agent": USER_AGENT}, - ) - self._delay = float(self.options.get("request_delay", 0.3)) - - @staticmethod - def _split(acct: str) -> tuple[str, str]: - acct = acct.lstrip("@") - user, _, host = acct.partition("@") - return user, host - - def _get(self, host: str, path: str, **params) -> Any: - time.sleep(self._delay) - r = self.client.get(f"https://{host}/api/v1/{path}", params=params) - r.raise_for_status() - return r.json() - - def _lookup(self, acct: str) -> dict[str, Any] | None: - user, host = self._split(acct) - if not host: - log.warning("mastodon: %s is missing an instance (use user@instance)", acct) - return None - try: - return self._get(host, "accounts/lookup", acct=user) - except httpx.HTTPError as exc: - log.warning("mastodon: lookup %s failed (%s)", acct, exc) - return None - - # ------------------------------------------------------------- ingest - - def fetch(self) -> Iterator[Item]: - limit = int(self.options.get("posts_per_account", 40)) - for acct, label in self.targets(): - account = self._lookup(acct) - if not account: - continue - _, host = self._split(acct) - try: - statuses = self._get( - host, - f"accounts/{account['id']}/statuses", - limit=min(limit, 40), - exclude_replies=str(self.options.get("skip_replies", True)).lower(), - exclude_reblogs=str(self.options.get("skip_boosts", True)).lower(), - ) - except httpx.HTTPError as exc: - log.warning("mastodon: statuses for %s failed (%s)", acct, exc) - continue - - for status in statuses: - card = status.get("card") or {} - text = strip_html(status.get("content", "")) - yield Item( - source_id=status.get("uri") or status.get("url", ""), - location=card.get("url") or status.get("url", ""), - title=card.get("title", ""), - excerpt=text, - author=(status.get("account") or {}).get("display_name") or acct, - published_at=parse_timestamp(status.get("created_at", "")), - subscription=label, - kind="post", - raw={ - "boosts": status.get("reblogs_count", 0), - "favourites": status.get("favourites_count", 0), - "post_url": status.get("url", ""), - }, - ) - - # --------------------------------------------------------------- check - - def check(self) -> list[CheckResult]: - return [self._check_one(acct) for acct, _ in self.targets()] - - def _check_one(self, acct: str) -> CheckResult: - user, host = self._split(acct) - if not host: - return CheckResult("mastodon", acct, False, - "missing instance (use user@instance)") - try: - data = self._get(host, "accounts/lookup", acct=user) - except httpx.HTTPError as exc: - return CheckResult("mastodon", acct, False, http_failure(exc)) - return CheckResult("mastodon", acct, True, - f"{data.get('statuses_count', '?')} posts") - - # ----------------------------------------------------------- discovery - - def discover(self) -> list[Suggestion]: - settings = self.options.get("discover") or {} - if not settings.get("enabled", True): - return [] - - seeds = [target for target, _ in self.targets()] - following = {s.lstrip("@").lower() for s in seeds} - counts: Counter[str] = Counter() - meta: dict[str, str] = {} - - for acct in seeds: - account = self._lookup(acct) - if not account: - continue - _, host = self._split(acct) - try: - # Many accounts hide this list; a 401/403 just means skip. - follows = self._get(host, f"accounts/{account['id']}/following", limit=80) - except httpx.HTTPError: - continue - for other in follows: - handle = other.get("acct", "") - if "@" not in handle: - handle = f"{handle}@{host}" - if handle.lower() in following: - continue - counts[handle] += 1 - meta.setdefault(handle, other.get("display_name", "")) - - threshold = settings.get("min_overlap", 2) - return [ - Suggestion( - handle=handle, - display_name=meta.get(handle, ""), - reason=f"followed by {n} of the accounts you follow", - score=float(n), - url=f"https://{handle.split('@')[-1]}/@{handle.split('@')[0]}", - ) - for handle, n in counts.most_common(200) - if n >= threshold - ] diff --git a/desk/sources/podcast.py b/desk/sources/podcast.py deleted file mode 100644 index 247675d..0000000 --- a/desk/sources/podcast.py +++ /dev/null @@ -1,546 +0,0 @@ -"""A news podcast, read as one item per story rather than one per episode. - -The general half. `desk/tools/video/` below knows how to cut a video into -chapters; this knows the things that are true of a *show*: it has a feed, it -publishes on a cycle, its episodes are somewhere on YouTube, and its stories -belong in your queue next to the RSS copies of those same articles. -`plugins/lastweekin_ai.py` above knows one show. - -That three-layer split is not decoration. A show that publishes a timestamped -outline works here with nothing show-specific involved — Dwarkesh and Latent -Space both do. What varies between shows is smaller than it looks, and is -declared as two methods rather than guessed at: - - video_key(entry) where this show puts its video. The default reads - the feed body, which is the common case and is - wrong for several of the biggest shows. - episode_number(title) how this show writes an episode number. - -Where each piece comes from, for one episode: - - the feed title, date, the audio enclosure, the video link, - and the show notes as HTML — one fetch, everything - the notes chapter marks, sometimes with the article links attached, - which is what makes a segment a story rather than a topic - YouTube the transcript, with timings - video.cut() one Segment per chapter - this file one Item per Segment - -**The subscription is just a feed.** `feeds:` is the same config key `rss` -uses, so a show imports into the `subscriptions` table like any other -subscription and gets add/remove/fetch/show, the Sources tab and the sidebar -for free — none of which had to know podcasts exist. - -Config, for any show that is not one somebody has written a plugin for: - - sources: - podcast: - feeds: - - url: https://api.substack.com/feed/podcast/69345.rss - label: Dwarkesh - episodes_per_feed: 2 - -Two things this deliberately does not do. It does not summarise: `text` starts -as the opening of the transcript and is replaced when a summary lands. What the -hosts made of a story is the one thing here that no feed already gives you, and -so the only part of this worth a model — which makes it a separate job, whose -failure costs you the abstracts and not the pull. See -newsdesk/jobs/summarise.py. - -And it does not store the episode as an item, or as anything else: an episode -is a date and a video id, both of which live on every segment it produced, so -"every segment of episode 255, in order" is already a query and needs no -entity behind it. - -**A show with no outline gets nothing stored, on purpose.** That claim above -used to hold only by luck: with no marks anywhere, `cut` hands back the whole -episode as one segment — right for a link you added by hand, wrong here — and -this stored it, so a two-hour recording arrived as a single eleven-thousand-word -item whose standfirst was the first two sentences of the intro and which then -queued itself for a summariser expecting one story. Now `_one_episode` stops -before the transcript is even fetched and says so in the log. Only shows that -publish a timestamped outline work here, which was always true and is now also -enforced. -""" - -from __future__ import annotations - -import logging -import re -from collections.abc import Iterator -from datetime import UTC, datetime, timedelta -from typing import Any - -import feedparser -import httpx - -from desk.sources.base import USER_AGENT, CheckResult, Item, Source, entry_time -from desk.sources.rss import check_feed -from desk.sources.spec import ( - API_VERSION, RECORDING_READER, RSS, PluginSpec, Profile, RetryPolicy, Step, - SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, -) -from desk.tools.http import session -from desk.tools.video import Segment, Video, cut, marks, media, opening, youtube - -log = logging.getLogger(__name__) - -#: "LWiAI Podcast #255 - Gemini 3.7, ..." -> 255. Every show numbers its -#: episodes somewhere; the title is where they all agree to put it. -_NUMBER = re.compile(r"(?:#|episode\s+|ep\.?\s*)(\d{1,5})\b", re.IGNORECASE) - - -def segment_profile(instructions: str = "", tags: str = "", - target_words: int = 70) -> Profile: - """What a show that publishes a timestamped outline produces, declared. - - A reusable typed preset, so that a plugin for one show is its feed, its - numbering and its editorial line — and not a second copy of this - machinery's description. `plugins/lastweekin_ai.py` is the model. - - The two steps are what `_one_episode` below does, in the contract's words: - retrieve the recording's captions *once*, then cut them at the marks the - show published. Nothing executes them yet — the runner is a later step of - `docs/plugin-rework.md` — and they are written now because they are the - part a future reader has to be able to check this file against. - - `instructions` is the one thing a show is expected to fill in: what its - own segments want said about them. What is general about summarising - speech — write what they made of it, keep the disagreement, do not quote a - mangled caption back — belongs to the shared task and is written once in - `newsdesk/jobs/summarise.py`, not per show. - """ - return Profile( - # One profile, because one thing is stored: the stories, not the - # episodes they came out of. An episode is a date and a video id, both - # of which are on every segment it produced. - kind="segment", - acquisition=[ - Step(id="captions", tool="retrieve_captions", - inputs={"media": "inputs.media"}, - outputs={"transcript": "inputs.transcript"}, - # Auto-captions on a two-hour upload can take hours to - # appear, so absence is a wait and not a failure. Today that - # wait is "the next pull, for ever", because the episode is - # simply not marked done; a bound is what this declares. - retry=RetryPolicy(max_attempts=4, delay_seconds=3600, - backoff=2.0, on=["not_ready"])), - Step(id="segments", tool="cut_segments", needs=["captions"], - # The marks come from the show's own notes, which arrive in - # the feed entry — HTML, so each story keeps the article link - # beside its timestamp. - inputs={"transcript": "inputs.transcript", - "marks": "inputs.feed_text"}, - outputs={"transcript": "inputs.transcript", - "links": "inputs.links"}), - ], - tasks=Tasks( - summary=SummaryPolicy( - enabled=True, input_fields=["inputs.transcript"], - require_complete=True, - # No floor: a two-sentence housekeeping segment gets one flat - # sentence saying so, which is worth having and is what the - # shared prompt already asks for. - min_words=0, target_words=target_words, - instructions=instructions), - tagging=TaggingPolicy( - # The transcript, not the abstract: tagging a segment must not - # quietly wait on a summariser that failed. Naming it decides - # *which* text and not how much of it — the tagger reads the - # opening few hundred characters of whatever a profile names, - # because it is choosing from a closed list of labels and a - # two-hour transcript would be thirty times the tokens for - # the same answer. See `newsdesk/jobs/tagging.py`. - enabled=True, input_fields=["inputs.transcript"], - context_fields=["title", "inputs.links"], instructions=tags), - ), - # The player first, then what the hosts made of it; the articles - # they were reading stay above it in every view. See the preset. - presentation=RECORDING_READER, - ) - - -class PodcastSource(Source): - name = "podcast" - description = "A show's feed, cut into one item per story discussed." - subscription_field = "feeds" - default_outlet = "Podcast" - - #: The generic source declares the same profile it hands its subclasses, - #: with no editorial line of its own: it is any show, and it cannot know - #: what one of them wants said. Its targets are editable because they are - #: whatever you put under `podcast:` in config.yaml — a show with a plugin - #: of its own declares a catalogue instead. - spec = PluginSpec( - api_version=API_VERSION, id=name, version="1", label="Podcast", - discovery=RSS(), - targets=TargetSpec(mode="editable", kind="rss_feed", defaults=[]), - profiles={"default": segment_profile()}, - ) - - # -------------------------------------------------------- what a show is - # - # The two things that differ between shows, as methods rather than as - # config knobs — because neither has a value you could write down. Finding - # the video means knowing where that show puts it, and numbering an episode - # means knowing how that show writes a number. A subclass that knows one - # show answers both; everything else in this file is the same for all of - # them. - - def video_key(self, entry: Any) -> str: - """The YouTube id for this episode, or "" if there isn't one to find. - - **The one place a show's publishing habits really bite.** The default - reads the feed body, which works when the show embeds or links its own - video there — Last Week in AI, Dwarkesh, Latent Space all do. - - It is not general, and it fails in a way worth naming: The Vergecast, - Hard Fork, TWiT, All-In and Lex Fridman all publish full video to - YouTube and none of them mentions it in the feed, so all five come back - empty here. Fixing that means matching a feed entry against a channel's - uploads — which title, which date, whose channel — and that is a - per-show judgement, not a parser. This is where it goes when someone - wants one of those shows: override this, return the id, and every other - line in this file already works. - """ - try: - return youtube.video_id(_body(entry)) - except ValueError: - return "" - - def episode_number(self, title: str) -> int | None: - """Which episode this is, from its title, or None if it doesn't say. - - Every show numbers its episodes and the title is where they all agree - to put it — but not in one format. `#255` and `Episode 42` and `Ep. 7` - are the common family and are what the default reads; `TWiT 1100` and - `SN 1094` are not, and an interview show may not number at all. - - Nothing downstream depends on the answer: it is stored on the item for - display and for "every segment of episode 255, in order". So the honest - default is a regex that catches most of them and returns None rather - than guessing when it doesn't, and a subclass overrides it when its - show writes numbers its own way. - """ - found = _NUMBER.search(title or "") - return int(found.group(1)) if found else None - - # ------------------------------------------------------------------ fetch - - def fetch(self) -> Iterator[Item]: - limit = int(self.options.get("episodes_per_feed", 2)) - with session(None, timeout=40) as http: - for entry in self.options.get("feeds", []): - url = entry["url"] if isinstance(entry, dict) else entry - label = _label(entry, url) - try: - yield from self._one_feed(http, url, label, limit) - except youtube.Blocked: - # The one failure that is not this show's. YouTube refused - # the machine, so every other show here would be refused - # too, and each attempt is one more request from an address - # already taken for a bot. Out of `fetch` altogether: the - # runner keeps what was yielded and turns this into a - # notice, which is what the log line below never was. - raise - except Exception: - # One dead show must not cost the others their episodes. - log.exception("podcast feed %s failed", url) - - def _one_feed(self, http: httpx.Client, url: str, label: str, - limit: int) -> Iterator[Item]: - response = http.get(url, headers={"User-Agent": USER_AGENT}) - response.raise_for_status() - # The bytes, not `.text`. A feed declares its own encoding in its XML - # declaration and plenty of them are not UTF-8; handing the parser a - # string that httpx has already decoded by its own guess throws that - # declaration away, and what comes back is an episode title with a - # mojibake apostrophe in it. `check_feed` parses the bytes too. - feed = feedparser.parse(response.content) - if not feed.entries: - log.warning("no episodes in %s", url) - return - - for entry in feed.entries[:limit]: - episode = self._episode(entry, label) - if episode is None: - continue - if self._already_done(episode.key): - log.debug("%s: already ingested", episode.title) - continue - try: - yield from self._one_episode(http, episode, entry, label) - except youtube.Blocked: - raise # not this episode's failure — see `fetch` - except Exception: - log.exception("episode %s failed; continuing the feed", episode.key) - - def _one_episode(self, http: httpx.Client | None, episode: Video, entry: Any, - label: str) -> Iterator[Item]: - # The show's own notes first: same outline as the video description, - # but the links survived. Then the description, then the comments. - chapters = marks.best(episode.notes) - if not chapters: - described = youtube.details(episode.key, client=http) - chapters = marks.best(described["description"]) - episode.duration = episode.duration or described["duration"] - if not chapters and self.options.get("read_comments", True): - chapters = marks.in_comments(youtube.comments(episode.key, client=http)) - - if not chapters: - # Nothing published an outline: not the notes, not the description, - # not a listener in the comments. `cut` would hand back the whole - # episode as one segment — correct of it, and what a hand-added - # link wants — but this source's contract is one item per *story*, - # and quietly substituting one item per episode is a different - # product wearing its name. - # - # It is also unusable downstream in a way that is easy to miss. The - # item's `text` would be `opening(transcript)`, the first two - # sentences of a two-hour recording, which is what the ranker and - # the tag filters would then judge it on; and eleven thousand words - # would go to a summariser whose prompt promises it a single story - # with the article the hosts were reading. Better to store nothing and say - # why than to store one wall and three wrong things about it. - # - # Marked done regardless. Reaching here means the episode was - # fetched and read successfully; there was simply nothing to cut - # along. A show that never publishes an outline would otherwise - # re-download a full transcript on every pull, for ever, to throw - # it away again — which is the waste `_already_done` exists to stop. - log.warning( - "%s: no timestamped outline in the notes, the description or " - "the comments, so there are no stories to cut it into — " - "storing nothing for this episode", episode.title) - self._mark_done(episode.key) - return - - try: - cues = youtube.transcript(episode.key, client=http) - except youtube.Blocked: - # A `TranscriptUnavailable` too, and the one that must not be - # treated as one here. Waiting is the answer to captions that are - # still being made; to a refusal it is the same nothing on every - # pull, at INFO, with the episode never marked done and nobody - # told. Still not marked done, so the pull after the address is - # clean reads it — but it leaves as a failure, and reaches the - # screen. `details` above raises the same thing the same way. - raise - except youtube.TranscriptUnavailable as exc: - # Not done, so no marker is written and the next pull tries again. - # Auto-captions on a two-hour upload can take hours to appear, and - # an episode without them is worth waiting for rather than - # ingesting as twenty titles with nothing under them. - log.info("%s: %s", episode.title, exc) - return - - segments = cut(episode, cues, chapters) - log.info("%s: %d segments", episode.title, len(segments)) - for segment in segments: - yield _item(segment, entry, label, - number=self.episode_number(episode.title)) - self._mark_done(episode.key) - - def check(self) -> list[CheckResult]: - """A show is a feed, so the feed check is the right one.""" - return [check_feed(url, label) for url, label in self.targets()] - - def _episode(self, entry: Any, show: str) -> Video | None: - """A feed entry as a `Video`. None when there is no video to read. - - A show whose video this cannot find is not a failure — see - `video_key`, which is the part a subclass replaces. Saying so once, - quietly, beats failing per episode. - """ - key = self.video_key(entry) - if not key: - # Deliberately not "audio-only shows need local transcription", - # which is what this used to say and which was wrong for the five - # biggest shows it fires on: they all post video, they just do not - # say so in the feed. Naming the actual condition is what stops the - # next person going and building a transcriber they did not need. - log.info("no video id in the feed entry for %r — the show may still " - "publish one; see PodcastSource.video_key", - (entry.get("title") or "")[:60]) - return None - - return Video( - ref=media.youtube(key), - title=(entry.get("title") or "").strip(), - url=entry.get("link") or "", - author=entry.get("author") or show, - published_at=entry_time(entry), - notes=_body(entry), - ) - - # ------------------------------------------------------------ bookkeeping - - def _already_done(self, key: str) -> bool: - """Whether this episode has been through here successfully. - - In the `state` table rather than in a table of its own, because that is - genuinely all the state an episode has: it is done or it is not. An - episode is not a thing you read, so it needs no row of its own, and one - key in a table that already exists needs no migration. - - The key is written only on success, so an episode whose captions - YouTube had not finished generating is left unmarked and comes round - again on the next pull. - - Without this, a weekly pull would re-fetch and re-cut every transcript - in the window on every run and then discard the lot at the dedup check - — correct, and several wasted seconds per pull. - """ - return bool(self.state.get(self._done_key(key))) - - def _mark_done(self, key: str) -> None: - self.state[self._done_key(key)] = datetime.now(UTC).isoformat() - - def _done_key(self, key: str) -> str: - """Namespaced by the plugin, not hard-coded to `podcast`. - - Two shows can publish the same video id — a guest episode cross-posted, - a network re-run — and with one shared prefix the second would find the - first's marker and skip an episode it had never read. One subclass per - show means one namespace per show for free. - """ - return f"{self.name}:done:{key}" - - -# ------------------------------------------------------------------ internals - -def _label(entry: Any, url: str) -> str: - """What one configured show is called. Which outlet that stands for is the - subscription's business — see `Item.subscription`.""" - if not isinstance(entry, dict): - return url - return entry.get("label") or url - - -def _item(segment: Segment, entry: Any, label: str, - number: int | None = None) -> Item: - """One segment, as the row newsdesk stores. - - Three of the field choices are not obvious: - - `body` is the transcript, which is both where it belongs and how this - tells `fetch_bodies` not to go and scrape a YouTube page for one. It - is not the `excerpt`, because the excerpt is what the queue row renders, - what the ranker's document is built from and what search indexes, and - sixteen hundred words of speech would swamp all three. - - `excerpt` is a placeholder until a summary is written over it. It is the - opening of the transcript rather than empty, because an item with no - excerpt is an empty row in the queue, and the ranker and the tagger both - read it. It used to be accompanied by a `summarised: False` flag, which - was this source telling the summariser that the row was waiting for it. - Nothing needs telling now: the profile declares that these items are - summarised, and an empty `summary` column is the whole of "not yet". - - `published_at` is the episode's stamp plus the segment's offset, so twenty - items published in the same instant still sort in the order they were said. - - `links` is the reason a segment is a story rather than a topic: the - articles the outline attached to this stretch, typed and roled, so the - reading pane and the summariser read them from the same field an outlet's - links live in rather than from this plugin's own corner of `raw`. - """ - episode = segment.video - return Item( - source_id=segment.external_id, - location=segment.url(), - title=segment.title, - excerpt=opening(segment.transcript), - body=segment.transcript, - author=episode.author, - published_at=episode.published_at + timedelta(seconds=segment.start), - subscription=label, - links=segment.reading(), - # Not `article`, which is what every one of these used to be stored as - # — twenty-three transcripts filed as prose. What the pipeline made, - # not what carried it: this arrived over RSS and left as segments. - kind="segment", - raw={ - # The chapter this segment sits under. Per *item* — it changes - # down the episode — so it is here rather than on the - # subscription's `section`, which names a branch of an outlet and - # is the same for everything that subscription hands over. - "heading": segment.heading, - # Where it plays and what it linked to: the record every stored - # segment carries, defined once on `Segment`. - **segment.record(), - "show": label, - "episode": number, - "episode_title": episode.title, - "episode_url": episode.url, - "episode_published": episode.published_at.isoformat(), - "audio_url": _enclosure(entry), - }, - ) - - -def _body(entry: Any) -> str: - """The richest form of the notes the feed offers.""" - for content in entry.get("content") or []: - if value := content.get("value"): - return value - return entry.get("summary") or "" - - -def _enclosure(entry: Any) -> str: - return next( - (e.get("href", "") for e in entry.get("enclosures") or [] - if (e.get("type") or "").startswith("audio")), - "", - ) - - -# ----------------------------------------------------------------- dry run - -def main(argv: list[str] | None = None) -> int: - """`python -m desk.sources.podcast ` — what a pull would store. - - Runs the plugin with no store behind it and prints the items instead of - writing them. This is the thing to run when a show changes the shape of - its notes: everything else in the pipeline is downstream of these fields. - """ - import argparse - import sys - - parser = argparse.ArgumentParser(prog="python -m desk.sources.podcast") - parser.add_argument("feed", help="the show's RSS feed") - parser.add_argument("--episodes", type=int, default=1) - parser.add_argument("-v", "--verbose", action="store_true") - args = parser.parse_args(argv) - - logging.basicConfig(level=logging.INFO if args.verbose else logging.WARNING, - format="%(levelname)-7s %(message)s") - if hasattr(sys.stdout, "reconfigure"): - sys.stdout.reconfigure(encoding="utf-8", errors="replace") - - source = PodcastSource({ - "feeds": [args.feed], "episodes_per_feed": args.episodes, - }) - items = list(source.fetch()) - if not items: - print("no items — run with -v to see why", file=sys.stderr) - return 1 - - print(f"{len(items)} items from {args.feed}\n") - heading = None - for item in items: - if item.raw["heading"] != heading: - heading = item.raw["heading"] - print(f" {heading or '(unfiled)'}") - stamp = marks.hhmmss(item.raw["start"]) - print(f" {stamp:>8} {item.title[:80]}") - print(f" {item.location}") - for link in item.links: - print(f" → {link.url}") - print(f" {len(item.body.split()):,} words · {item.excerpt[:90]}") - print() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/desk/sources/x.py b/desk/sources/x.py deleted file mode 100644 index 9353586..0000000 --- a/desk/sources/x.py +++ /dev/null @@ -1,203 +0,0 @@ -"""X (Twitter) via the official API v2. Off by default, because it costs money. - -X moved to pay-per-usage billing. As published on the official pricing page, -post reads are $0.005 per resource and user reads are $0.010 per resource, -with pay-per-usage plans capped at 3 million post reads per billing cycle. -Reads of your *own* account's data ("Owned Reads") are $0.001 per resource. - -So a config with 30 accounts x 20 posts x 4 polls a day is roughly -30*20*4*30 = 72,000 reads/month = about $360/month. Check the estimate this -plugin prints before you enable it. - -This plugin therefore: - - is disabled unless you set `enabled: true` - - refuses to run without `budget_usd_per_month` - - tracks spend in the `state` table and stops when the budget is gone - - caches user IDs so it never pays for a lookup twice - -Source for rates: https://docs.x.com/x-api/getting-started/pricing -""" - -from __future__ import annotations - -from urllib.parse import urlsplit - -import logging -import os -from collections.abc import Iterator -from datetime import UTC, datetime -from typing import Any - -import httpx - -from desk.sources.base import USER_AGENT, Item, Source, Suggestion, parse_timestamp -from desk.sources.spec import ( - API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy, - TaggingPolicy, TargetSpec, Tasks, -) - -log = logging.getLogger(__name__) -API = "https://api.x.com/2" - -POST_READ_USD = 0.005 -USER_READ_USD = 0.010 - - -def _is_internal(url: str) -> bool: - """Match the host boundary so linux.com remains an external article.""" - try: - host = (urlsplit(url).hostname or "").lower().rstrip(".") - except ValueError: - return False - return any(host == domain or host.endswith("." + domain) - for domain in ("x.com", "twitter.com")) - - -class XSource(Source): - name = "x" - default_outlet = "X" - description = "X/Twitter timelines (paid API — off by default)." - - #: Declared like the other two networks. The budget, the token and the - #: refusal to run without either stay in the code above: they are what - #: this plugin *does*, and none of them is a thing the shared jobs or the - #: reader need to be told about. - spec = PluginSpec( - api_version=API_VERSION, id=name, version="1", label="X", - targets=TargetSpec(mode="editable", kind="account", defaults=[], - validator="x_handle"), - profiles={"default": Profile( - kind="post", - acquisition=[Step(id="article", tool="extract_article", - inputs={"url": "inputs.original_url"}, - outputs={"content": "inputs.article_text"})], - tasks=Tasks( - summary=SummaryPolicy(enabled=False), - tagging=TaggingPolicy( - enabled=True, input_fields=["inputs.feed_text"], - context_fields=["author"]), - ), - presentation=POST_READER, - )}, - ) - - def __init__(self, options: dict[str, Any] | None = None) -> None: - super().__init__(options) - token_env = self.options.get("bearer_token_env", "X_BEARER_TOKEN") - self.token = os.environ.get(token_env, "") - self.client = httpx.Client( - timeout=25, - headers={ - "Authorization": f"Bearer {self.token}", - "User-Agent": USER_AGENT, - }, - ) - - @property - def enabled(self) -> bool: - return bool(self.options.get("enabled", False)) - - def estimate_monthly_usd(self, polls_per_day: int = 4) -> float: - n = len(self.options.get("accounts", [])) - per_poll = n * int(self.options.get("posts_per_account", 10)) - return per_poll * polls_per_day * 30 * POST_READ_USD - - # ------------------------------------------------------------ budget - - def _spent(self) -> float: - month = datetime.now(UTC).strftime("%Y-%m") - return float(self.state.get(f"x_spend_{month}", "0") or 0) - - def _charge(self, usd: float) -> None: - month = datetime.now(UTC).strftime("%Y-%m") - self.state[f"x_spend_{month}"] = str(self._spent() + usd) - - def _user_id(self, handle: str) -> str | None: - cache_key = f"x_userid_{handle.lower()}" - cached = self.state.get(cache_key) - if cached: - return cached - r = self.client.get(f"{API}/users/by/username/{handle}") - if r.status_code != 200: - log.warning("x: lookup %s returned %s", handle, r.status_code) - return None - self._charge(USER_READ_USD) - uid = (r.json().get("data") or {}).get("id") - if uid: - self.state[cache_key] = uid - return uid - - # ------------------------------------------------------------- ingest - - def fetch(self) -> Iterator[Item]: - if not self.token: - log.error("x: no bearer token in $%s", - self.options.get("bearer_token_env", "X_BEARER_TOKEN")) - return - budget = self.options.get("budget_usd_per_month") - if budget is None: - log.error("x: refusing to run without `budget_usd_per_month` in config") - return - - per_account = int(self.options.get("posts_per_account", 10)) - exclude = ",".join(self.options.get("exclude", ["retweets", "replies"])) - - for handle, label in self.targets(): - spent = self._spent() - if spent >= float(budget): - log.warning("x: monthly budget of $%.2f exhausted ($%.2f spent), stopping", - float(budget), spent) - return - - uid = self._user_id(handle) - if not uid: - continue - - r = self.client.get( - f"{API}/users/{uid}/tweets", - params={ - "max_results": max(5, min(per_account, 100)), - "exclude": exclude, - "tweet.fields": "created_at,entities,public_metrics", - }, - ) - if r.status_code != 200: - log.warning("x: timeline %s returned %s: %s", - handle, r.status_code, r.text[:200]) - continue - - payload = r.json() - posts = payload.get("data", []) or [] - self._charge(len(posts) * POST_READ_USD) - - for post in posts: - urls = ((post.get("entities") or {}).get("urls") or []) - external = next( - (u.get("expanded_url") for u in urls - if u.get("expanded_url") - and not _is_internal(u["expanded_url"])), - "", - ) - yield Item( - source_id=post["id"], - location=external or f"https://x.com/{handle}/status/{post['id']}", - title="", - excerpt=post.get("text", ""), - author=handle, - published_at=parse_timestamp(post.get("created_at", "")), - subscription=label, - kind="post", - raw={ - "metrics": post.get("public_metrics", {}), - "post_url": f"https://x.com/{handle}/status/{post['id']}", - }, - ) - - def discover(self) -> list[Suggestion]: - # The follow-graph endpoints are billed per user read, which makes - # graph-walking on X expensive enough that it is not worth doing - # automatically. Use Bluesky/Mastodon discovery instead. - return [] - - # `check` is left at the default too: a user lookup is a billed read, and - # a check job must never cost money. diff --git a/desk/tools/__init__.py b/desk/tools/__init__.py index 2a5daa9..866b7a0 100644 --- a/desk/tools/__init__.py +++ b/desk/tools/__init__.py @@ -7,14 +7,20 @@ if it would make sense in a different application, it is a tool. agent.py who answers when something needs a model: the Claude API with a key, the `claude` CLI on a subscription, or nobody + credentials.py the token `agent` reads from the app's database embedding.py text in, vector out - framing.py will this page load inside an iframe? http.py one shared HTTP client per call chain + failures.py what went wrong, in one word a retry policy can use marks.py what kind of thing is this, if not an article? - video/ a recording, read as text with timings on it - walls.py was that the article, or the page shown instead of it? + scrape.py an article out of a URL: fetch, check, convert. + Kept because `desk.sources.base` imports its `Getter`; + `article.py`, `checks.py`, `browser.py` and `framing.py` + stay only because `scrape.py` imports them -Nothing is imported here on purpose. `video` pulls in httpx and the caption -parsers, `embedding` a sentence-transformer, `agent` an SDK; each is paid for -by the caller that wants it and by nobody else. +roledesk's copy prunes news's `video/`, `posts.py` and `walls.py`, which only +news's own sources use (see `desk/README.md`). + +Nothing is imported here on purpose. `embedding` pulls in a +sentence-transformer, `agent` an SDK; each is paid for by the caller that +wants it and by nobody else. """ diff --git a/desk/tools/posts.py b/desk/tools/posts.py deleted file mode 100644 index 45e71a6..0000000 --- a/desk/tools/posts.py +++ /dev/null @@ -1,193 +0,0 @@ -"""A Mastodon or Bluesky post read through its network's public API. - -Both web apps serve a script shell to a plain request, so `checks` names these -getters before `browser`: the API answers in a fraction of a second, needs no -Chromium and no login, and returns the post rather than a rendering of it. -Each getter builds a small HTML page from the answer, so `scrape` checks and -converts it exactly as it would a publisher's page. - - from desk.tools import posts - - posts.is_mastodon("https://hachyderm.io/@simontatham/117201594980991062") - page = posts.mastodon(url) # checks.Page - page = posts.bluesky("https://bsky.app/profile/bsky.app/post/3mv3shqdfuc2e") - -The Mastodon and Bluesky plugins register them with `scrape` as the -`mastodon_api` and `bluesky_api` getters for their URLs; see `Source.getters`. - -**A thread is read as far as its author continued it.** A post a link points -to is often the first of several by the same person; the replies by anyone -else are not part of it. - -X is not here: its API bills every read and needs a token, and x.com shows a -browser a login wall. - - GET https://{host}/api/v1/statuses/{id} Mastodon, any instance - GET https://{host}/api/v1/statuses/{id}/context - GET https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread -""" - -from __future__ import annotations - -import html -import re -from typing import Any -from urllib.parse import urlsplit - -import httpx - -from desk.tools.checks import Page - -TIMEOUT = 20.0 -#: Says what is asking, the way `desk.sources.base.USER_AGENT` does for the -#: plugins; a tool sits below `sources` and cannot import it. -USER_AGENT = "newsdesk (+https://github.com/; reading a linked post)" -BLUESKY_API = "https://public.api.bsky.app/xrpc" - -#: `/@user/123` is what an instance links; `/users/user/statuses/123` is the -#: ActivityPub id some links carry instead. -_MASTODON_PATH = re.compile(r"^/(?:@[^/]+|users/[^/]+/statuses)/(\d+)/?$") -_BLUESKY_PATH = re.compile(r"^/profile/([^/]+)/post/([a-z0-9]+)/?$") - -#: How deep a Bluesky thread is asked for. A self-thread longer than this is -#: cut, which is rare and costs the tail rather than the post. -THREAD_DEPTH = 20 - - -def _get(url: str, params: dict[str, Any] | None = None) -> httpx.Response: - # Module-level, so a test replaces this one function. - return httpx.get(url, params=params, timeout=TIMEOUT, - headers={"User-Agent": USER_AGENT}, follow_redirects=True) - - -# ---------------------------------------------------------------- Mastodon - -def _mastodon_id(url: str) -> tuple[str, str] | None: - parts = urlsplit(url) - found = _MASTODON_PATH.match(parts.path) - return (parts.netloc, found.group(1)) if parts.scheme == "https" and found else None - - -def is_mastodon(url: str) -> bool: - """Whether `url` has a Mastodon post's shape. Any host: an instance is - recognised by the check that found its web app's shell, not by name.""" - return _mastodon_id(url) is not None - - -def mastodon(url: str) -> Page | None: - """The post at `url`, and its author's continuation, from the instance's API.""" - host, status_id = _mastodon_id(url) or ("", "") - answer = _get(f"https://{host}/api/v1/statuses/{status_id}") - if not _json(answer): - # Not an instance: `/@user/123` is a Mastodon address's shape, and - # another site can have it. Nothing to say about the page, so the - # ordinary path runs. - return None - if answer.status_code != 200: - return Page(answer.status_code, "", url, dict(answer.headers)) - status = answer.json() - author = status.get("account") or {} - thread = [status] - context = _get(f"https://{host}/api/v1/statuses/{status_id}/context") - if context.status_code == 200 and _json(context): - # Descendants come in thread order; the author's continuation is the - # chain of their replies, each to the one before. - for reply in context.json().get("descendants") or []: - if ((reply.get("account") or {}).get("id") == author.get("id") - and reply.get("in_reply_to_id") == thread[-1].get("id")): - thread.append(reply) - name = author.get("display_name") or author.get("acct") or host - card = status.get("card") or {} - return Page(200, _page( - title=f"{name}: {_first_words(_text(status.get('content', '')))}", - byline=f"{name} (@{author.get('acct', '')})", - published=status.get("created_at", ""), - paragraphs=[s.get("content", "") for s in thread], - link=(card.get("url", ""), card.get("title", "")), - alts=[m.get("description") or "" for s in thread - for m in s.get("media_attachments") or []], - escaped=False), url) - - -# ---------------------------------------------------------------- Bluesky - -def _bluesky_uri(url: str) -> str: - parts = urlsplit(url) - found = _BLUESKY_PATH.match(parts.path) - if parts.netloc not in {"bsky.app", "www.bsky.app"} or not found: - return "" - return f"at://{found.group(1)}/app.bsky.feed.post/{found.group(2)}" - - -def is_bluesky(url: str) -> bool: - return bool(_bluesky_uri(url)) - - -def bluesky(url: str) -> Page | None: - """The post at `url`, and its author's continuation, from Bluesky's AppView.""" - answer = _get(f"{BLUESKY_API}/app.bsky.feed.getPostThread", - {"uri": _bluesky_uri(url), "depth": THREAD_DEPTH, "parentHeight": 0}) - if answer.status_code != 200: - # A post that is gone is a 400 `NotFound` here, not a 404. - gone = answer.status_code == 400 and "NotFound" in answer.text - return Page(404 if gone else answer.status_code, "", url, dict(answer.headers)) - thread = answer.json().get("thread") or {} - post = thread.get("post") or {} - author = post.get("author") or {} - posts = [post] - node = thread - while True: - # The author's own reply, if there is one, is the next part of the thread. - nxt = next((r for r in node.get("replies") or [] - if ((r.get("post") or {}).get("author") or {}).get("did") == author.get("did")), - None) - if nxt is None: - break - posts.append(nxt["post"]) - node = nxt - record = post.get("record") or {} - external = ((post.get("embed") or {}).get("external") or {}) - name = author.get("displayName") or author.get("handle", "") - return Page(200, _page( - title=f"{name}: {_first_words(record.get('text', ''))}", - byline=f"{name} (@{author.get('handle', '')})", - published=record.get("createdAt", ""), - paragraphs=[(p.get("record") or {}).get("text", "") for p in posts], - link=(external.get("uri", ""), external.get("title", "")), - alts=[image.get("alt") or "" for p in posts - for image in (p.get("embed") or {}).get("images") or []], - escaped=True), url) - - -# ---------------------------------------------------------------- the page - -def _page(*, title: str, byline: str, published: str, paragraphs: list[str], - link: tuple[str, str], alts: list[str], escaped: bool) -> str: - """An article page for the converter. Mastodon's content is HTML already; - Bluesky's is text, escaped here and split into paragraphs on blank lines.""" - if escaped: - paragraphs = ["".join(f"

{html.escape(block).replace(chr(10), '
')}

" - for block in re.split(r"\n\s*\n", text) if block.strip()) - for text in paragraphs] - body = "\n".join(paragraphs) - url, label = link - if url: - body += f'\n

{html.escape(label or url)}

' - body += "".join(f"\n

[Image: {html.escape(alt)}]

" for alt in alts if alt.strip()) - return (f"{html.escape(title)}" - f"

{html.escape(byline)}

" - f"
" - f"\n{body}\n
") - - -def _json(answer: httpx.Response) -> bool: - return "json" in answer.headers.get("content-type", "") - - -def _text(markup: str) -> str: - return html.unescape(re.sub(r"<[^>]+>", " ", markup)) - - -def _first_words(text: str, count: int = 8) -> str: - words = text.split() - return " ".join(words[:count]) + ("…" if len(words) > count else "") diff --git a/desk/tools/video/__init__.py b/desk/tools/video/__init__.py deleted file mode 100644 index 9e3a977..0000000 --- a/desk/tools/video/__init__.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Video, read as text with timings on it. - -The generic half of watching things so you don't have to. Give it a video and -it gives back a list of `Segment`s — each one a stretch of the recording with a -title, the words spoken in it, and a link that plays it from that second. - - read() a video -> Video + Segment[] the whole thing, one call - youtube captions, metadata, comments where both inputs come from - marks an outline -> chapter marks description, notes, comments - media a MediaRef -> a playable link YouTube today, a file later - segment cues + marks -> Segment[] the cut, and transcript text - -**This package knows nothing about newsdesk.** No store, no queue, no config, -no `Item`. That is what makes it reusable, and it is the seam the design is -built on: `desk/sources/podcast.py` sits on top and adds the things that -*are* specific to a show — its feed, its weekly cadence, the article links in -its notes, and turning segments into rows. - -Two decisions worth knowing before reading further. - -**A video is a `MediaRef`, not a URL.** Today every ref is a YouTube id; the -reading pane's link is computed from it on the way out. Point one at a file on -disk in config and every link to it changes, with nothing rewritten and no -item's identity touched. See `media.py`. - -**Transcripts come from whoever can supply one.** `youtube-transcript-api` -first, and a built-in fetcher against YouTube's own player endpoint when the -library can't get the track. Transcribing audio locally (yt-dlp plus -faster-whisper) is the third backend and the one that makes local video work; -it is not written yet, and `read` is shaped so that adding it changes nothing -above. - -Try it: - - python -m desk.tools.video https://www.youtube.com/watch?v=GUYEofBebDc -""" - -from __future__ import annotations - -import logging - -import httpx - -from desk.tools.video import marks, media, youtube -from desk.tools.http import session -from desk.tools.video.marks import Chapter, Link -from desk.tools.video.media import MediaRef -from desk.tools.video.segment import Segment, Video, cut, opening, paragraphs -from desk.tools.video.youtube import Blocked, Cue, TranscriptUnavailable - -log = logging.getLogger(__name__) - -__all__ = [ - "Blocked", "Chapter", "Cue", "Link", "MediaRef", "Segment", - "TranscriptUnavailable", "Video", "cut", "describe", "marks", "media", - "opening", "paragraphs", "read", "youtube", -] - - -def read(url: str, *, comments: bool = True, - client: httpx.Client | None = None) -> tuple[Video, list[Segment]]: - """One video, cut into its chapters. The whole package in one call. - - `comments` is the only cost worth thinking about: it is an extra request to - the watch page and is only consulted when the description has no outline in - it, so for a show that publishes chapter marks it never happens. - """ - with session(client, timeout=40) as http: - video = describe(url, client=http) - try: - cues = youtube.transcript(video.key, client=http) - except TranscriptUnavailable as exc: - # Not fatal on its own. The outline still gives every chapter, its - # links and its timecode; only the quoted words are missing, and - # auto-captions usually appear within a few hours of an upload. - log.warning("%s", exc) - cues = [] - - chapters = marks.best(video.notes) - if not chapters and comments: - chapters = marks.in_comments(youtube.comments(video.key, client=http)) - - return video, cut(video, cues, chapters) - - -def describe(url: str, *, client: httpx.Client | None = None) -> Video: - """What a video says about itself: title, author, length, description. - - YouTube only, for now. A local file would come back with its stem as the - title and no notes, which is why `Video.notes` is allowed to be empty and - why `read` has a comments fallback at all. - """ - key = youtube.video_id(url) - with session(client) as http: - meta = youtube.details(key, client=http) - return Video( - ref=media.youtube(key), - title=meta["title"], - url=media.youtube(key).at(0), - author=meta["author"], - duration=meta["duration"], - notes=meta["description"], - ) diff --git a/desk/tools/video/__main__.py b/desk/tools/video/__main__.py deleted file mode 100644 index bc0cf23..0000000 --- a/desk/tools/video/__main__.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Cut a video up and print what came out. - - python -m desk.tools.video - python -m desk.tools.video --json > segments.json - -Videos only. A podcast episode — where the chapter marks and the article links -live on the show's own page rather than on YouTube — goes through the layer -above: - - python -m desk.sources.podcast -""" - -from __future__ import annotations - -import argparse -import dataclasses -import json -import logging -import sys - -import desk.tools.video as video_pkg -from desk.tools.video import marks -from desk.tools.video.segment import Segment, Video - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser( - prog="python -m desk.tools.video", - description="Cut a video into one item per chapter mark.", - ) - parser.add_argument("url", help="a YouTube link or bare video id") - parser.add_argument("--json", action="store_true", - help="dump everything, transcripts included, as JSON") - parser.add_argument("--chars", type=int, default=160, - help="how much transcript to preview per segment (default 160)") - parser.add_argument("--no-comments", action="store_true", - help="don't fall back to chapter marks typed in the comments") - parser.add_argument("-v", "--verbose", action="store_true") - args = parser.parse_args(argv) - - logging.basicConfig( - level=logging.INFO if args.verbose else logging.WARNING, - format="%(levelname)-7s %(message)s", - ) - # Windows still opens a console in the system codepage, and a video title - # with a "ñ" in it — or the arrows this prints — is an unhandled - # UnicodeEncodeError half way through the report rather than a bad glyph. - if hasattr(sys.stdout, "reconfigure"): - sys.stdout.reconfigure(encoding="utf-8", errors="replace") - - try: - video, segments = video_pkg.read(args.url, comments=not args.no_comments) - except Exception as exc: - print(f"error: {exc}", file=sys.stderr) - return 1 - - if args.json: - json.dump(dump(video, segments), sys.stdout, indent=2, default=str) - print() - else: - report(video, segments, args.chars) - return 0 - - -def report(video: Video, segments: list[Segment], chars: int) -> None: - """The human-readable view. Also used by the podcast module.""" - words = sum(len(s.transcript.split()) for s in segments) - length = video.duration or (segments[-1].end if segments else 0) - print(f"{video.title}") - print(f" {video.author or '—'} {video.ref.kind}:{video.ref.key}" - f" {marks.hhmmss(length)}") - print(f" {len(segments)} segments, {words:,} words of transcript") - print() - - heading = None - for segment in segments: - if segment.heading != heading: - heading = segment.heading - print(f" {heading or '(unfiled)'}") - preview = segment.transcript[:chars] - if len(segment.transcript) > chars: - preview += "…" - print(f" {segment.timecode:>8} {segment.title[:78]}") - print(f" {marks.hhmmss(segment.duration)} long · {segment.url()}") - for link in segment.links: - print(f" → {link.url}") - print(f" {preview or '(no transcript under this mark)'}") - print() - - -def dump(video: Video, segments: list[Segment]) -> dict[str, object]: - """The shape a stored item is built from, plus the fields a dry run wants - to eyeball. The stored half is `Segment.record()` and is defined there; - desk/sources/podcast.py turns the rest into a row.""" - return { - "video": { - "ref": dataclasses.asdict(video.ref), - "title": video.title, - "url": video.url, - "author": video.author, - "published_at": video.published_at, - "duration": video.duration, - }, - "segments": [ - { - **s.record(), - "external_id": s.external_id, - "title": s.title, - "heading": s.heading, - "timecode": s.timecode, - "url": s.url(), - "transcript": s.transcript, - } - for s in segments - ], - } - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/desk/tools/video/marks.py b/desk/tools/video/marks.py deleted file mode 100644 index ef32fb0..0000000 --- a/desk/tools/video/marks.py +++ /dev/null @@ -1,265 +0,0 @@ -"""Chapter marks: a video's own table of contents, wherever it was written. - -The important discovery about this kind of video is that **the segmentation -already exists and somebody else did it**. Last Week in AI ships every episode -with a timestamped outline — a rubric, then one line per story, each with the -time it starts and a link to the article being discussed: - - Tools & Apps - (00:03:32) Google announces Gemini 3.7 Flash ... - Ars Technica - (00:13:11) SpaceXAI Releases Grok 4.6 ... - MarkTechPost - -That is a better set of boundaries than any topic-segmentation model would -produce, for nothing, with the source links attached. So this module parses it -rather than inferring it. An LLM is still wanted downstream — to summarise what -was *said* in each stretch — but not to work out where one stretch ends. - -**Three places to find that outline**, in descending order of usefulness, all -parsed by the same function because they are the same convention written in -different markup: - -1. **The show's own page or feed body** — HTML, so the links are real - `` and each chapter arrives with the article it is about. Only a - show with a website has this, which is why the podcast source hands it in - rather than this module going to look. -2. **The video description** — plain text. Usually the same outline with the - links flattened away. -3. **The comments** — see `in_comments`. When a show publishes no marks at - all, a listener has often typed some out; YouTube itself reads those to - draw its chapter markers. - -`best` takes them in that order and returns the first that is real. Nothing -here is specific to one show beyond the shape "timestamp, then a title, -optionally under a heading", which is the near-universal convention. -""" - -from __future__ import annotations - -import logging -import re -from collections.abc import Iterable -from dataclasses import dataclass, field -from html.parser import HTMLParser - -log = logging.getLogger(__name__) - -#: `(00:03:32)`, `00:03:32`, `3:32` — at the start of the line, which is what -#: makes it a chapter mark rather than a time mentioned in a sentence. -TIMESTAMP = re.compile( - # The character class ends with hyphen, en dash and em dash on purpose: - # a show separates the time from the title with whichever of the three its - # editor's keyboard produced, and means the same thing by all of them. - r"^\(?\s*(\d{1,2}):([0-5]\d)(?::([0-5]\d))?\s*\)?[\s\-–—:]*" # noqa: RUF001 -) -_URL = re.compile(r"https?://[^\s<>\"')\]]+") -#: Tags that end a line of the outline. Everything else is inline and stays -#: on the line it was found in, which is what keeps `(00:03:32) ` -#: attached to the `` that follows it. -_BLOCKS = frozenset({ - "li", "p", "br", "div", "tr", "ul", "ol", "h1", "h2", "h3", "h4", "blockquote", -}) - - -@dataclass(frozen=True, slots=True) -class Link: - """One article the hosts were talking about.""" - - url: str - title: str = "" - - -@dataclass(slots=True) -class Chapter: - """One entry in the outline: where a story starts and what it is about.""" - - start: int # seconds into the episode - title: str - heading: str = "" # the rubric it sits under - links: list[Link] = field(default_factory=list) - - @property - def url(self) -> str: - """The article being discussed, if the notes linked one.""" - return self.links[0].url if self.links else "" - - -def parse(body: str) -> list[Chapter]: - """Every chapter mark in a set of show notes, in the order they play. - - Takes HTML or plain text and works out which it was given. Returns `[]` - for notes with no outline in them at all, which is a real answer — plenty - of shows don't write one, and the caller falls back to treating the episode - as a single undivided item rather than guessing at boundaries. - """ - lines = _html_lines(body) if "<" in body and ">" in body else _text_lines(body) - - chapters: list[Chapter] = [] - heading = "" - for text, links in lines: - match = TIMESTAMP.match(text) - if match is None: - # A heading is only a heading once the outline has started; - # before that every line is preamble — the blurb, the sponsors, - # the "Timestamps:" label itself. - if chapters and _is_heading(text, links): - heading = text.rstrip(":") - continue - title = text[match.end():].strip(" -–—|") # noqa: RUF001 — as above - # Prefer the anchor's own text: a line that links two follow-ups reads - # "First story | The Verge + Second story", and the first link's text - # is the story, where the whole line is the story plus its footnote. - if links and links[0].title: - title = links[0].title - chapters.append(Chapter( - start=_seconds(match), title=title, heading=heading, links=links, - )) - - if not chapters: - log.debug("no chapter marks found in %d chars of notes", len(body)) - return chapters - - -def best(*candidates: str, minimum: int = 3) -> list[Chapter]: - """The first candidate that yields a real outline. - - Chapter marks come from up to three places and they are not equally good, - so the order they are passed in is the order of preference — for a podcast - that is: the show's own notes as HTML (links intact), then the YouTube - description (same outline, links flattened away), then the comments. - - `minimum` is what stops a stray "at 3:20 he says" in a blurb becoming a - one-chapter outline that suppresses the good source behind it. Three marks - is the point where a line of text stops being a coincidence. - """ - for text in candidates: - found = parse(text) - if len(found) >= minimum: - return found - return [] - - -def in_comments(comments: Iterable[str], minimum: int = 3) -> list[Chapter]: - """The outline somebody typed into the comments, if anybody did. - - Read as the fallback for shows that publish no chapter marks of their own, - where it is common for a listener to write one out — YouTube itself treats - such a comment as good enough to draw chapter markers from. - - Each comment is parsed on its own rather than the lot being concatenated. - Joined together, an outline comment and thirty unrelated replies produce - one bogus outline interleaved with chatter; separately, the outline is the - comment with the most marks in it and the chatter scores zero. - """ - outlines = [parse(comment) for comment in comments] - outlines = [found for found in outlines if len(found) >= minimum] - if not outlines: - return [] - richest = max(outlines, key=len) - log.info("took %d chapter marks from a comment", len(richest)) - return richest - - -def hhmmss(seconds: float) -> str: - """3932 -> '1:05:32'. What a timecode looks like to a reader.""" - total = int(seconds) - hours, rest = divmod(total, 3600) - minutes, secs = divmod(rest, 60) - return f"{hours}:{minutes:02d}:{secs:02d}" if hours else f"{minutes}:{secs:02d}" - - -# ------------------------------------------------------------------ internals - -def _seconds(match: re.Match[str]) -> int: - a, b, c = match.group(1), match.group(2), match.group(3) - if c is None: # mm:ss - return int(a) * 60 + int(b) - return int(a) * 3600 + int(b) * 60 + int(c) - - -def _is_heading(text: str, links: list[Link]) -> bool: - """A rubric like 'Policy & Safety', not a sentence and not a story. - - Deliberately strict. Anything that gets through becomes the heading for - every chapter after it, so a false positive mislabels the rest of the - episode — where a false negative just leaves a few items unfiled. - """ - return ( - not links - and len(text) <= 60 - and len(text.split()) <= 6 - and not text.endswith((".", "!", "?")) - ) - - -def _text_lines(body: str) -> list[tuple[str, list[Link]]]: - """Plain-text notes — a YouTube description. Bare URLs become links.""" - lines = [] - for raw in body.splitlines(): - line = raw.strip() - if not line: - continue - links = [Link(url=u) for u in _URL.findall(line)] - lines.append((_URL.sub("", line).strip(), links)) - return lines - - -def _html_lines(body: str) -> list[tuple[str, list[Link]]]: - parser = _Outline() - parser.feed(body) - parser.close() - return parser.lines - - -class _Outline(HTMLParser): - """HTML to (line, links) pairs, without pulling in a parser library. - - A full HTML tree would be nicer and is not worth a dependency for this: - the only structure that matters is where the line breaks fall and which - anchors landed on which line. `convert_charrefs` (on by default) means - entities arrive already decoded, so `Tools & Apps` is `Tools & Apps` - by the time it reaches `handle_data`. - """ - - def __init__(self) -> None: - super().__init__() - self.lines: list[tuple[str, list[Link]]] = [] - self._text: list[str] = [] - self._links: list[Link] = [] - self._href = "" - self._anchor: list[str] = [] - - def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - if tag in _BLOCKS: - self._flush() - elif tag == "a": - self._href = dict(attrs).get("href") or "" - self._anchor = [] - - def handle_endtag(self, tag: str) -> None: - if tag in _BLOCKS: - self._flush() - elif tag == "a": - if self._href: - self._links.append( - Link(url=self._href, title=_squash("".join(self._anchor))) - ) - self._href, self._anchor = "", [] - - def handle_data(self, data: str) -> None: - self._text.append(data) - if self._href: - self._anchor.append(data) - - def _flush(self) -> None: - text = _squash("".join(self._text)) - if text or self._links: - self.lines.append((text, self._links)) - self._text, self._links = [], [] - - def close(self) -> None: - super().close() - self._flush() - - -def _squash(text: str) -> str: - return re.sub(r"\s+", " ", text or "").strip() diff --git a/desk/tools/video/media.py b/desk/tools/video/media.py deleted file mode 100644 index 61dd67d..0000000 --- a/desk/tools/video/media.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Where a video actually lives, and how to link into the middle of it. - -Every segment newsdesk stores has to answer one question: *press play at -1:08:27 — where?* Today the answer is a YouTube URL. It will not always be. -The same episode may sit on disk, ripped or recorded; a show may take its -uploads down; a video may only ever have existed as a local file. - -So nothing in this codebase stores a YouTube URL as a fact. It stores a -`MediaRef` — a kind and a key — and asks this module for a link when it needs -one. Swapping YouTube for a copy on disk is then a line of config rather than -an UPDATE across every item that mentioned it. - - video: - local_media: - GUYEofBebDc: D:/media/lwiai/255.mp4 - -`resolve` applies that map. Everything downstream keeps working because it was -never holding a URL in the first place. - -The one thing this deliberately does not do is *find* local copies. Matching -"is this file the same episode as that video" by filename or duration is a -guess, and a wrong guess points a citation at the wrong recording. The mapping -is written by a person, or it does not exist. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path, PurePath - -#: A local file, playable in the reading pane's