Files
jobboard/desk/tools/video/youtube.py
T
WolfandClaude Opus 5.5 248ea1636d Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:17 +02:00

601 lines
24 KiB
Python

"""Captions, with their timings, for one YouTube video.
YouTube has always published a machine-readable transcript for anything with
captions — including the auto-generated ones, which is most podcasts. It is the
cheapest transcript in existence: somebody else already paid for the ASR.
Getting at it is the fiddly part, and it is fiddly for a reason worth writing
down. The caption URLs embedded in a `watch` page are signed for a *browser*
session, and fetching one from anywhere else now returns `200 OK` with an empty
body — no error, no redirect, nothing to catch. So the obvious implementation
(scrape the watch page, follow `baseUrl`) fails silently, which is the worst
way for an ingest step to fail.
What works is asking the same endpoint the mobile app asks. `/youtubei/v1/player`
with a mobile client context hands back a player response whose caption URLs are
not tied to a browser session, and those fetch fine. That is the whole trick,
and it is why this module talks to an undocumented endpoint instead of scraping
a page: the scrape is the fragile one.
Two clients are tried in order, because they fail differently — `IOS` returns
JSON3 when asked and `ANDROID` returns SRV3 whatever you ask for, so both
formats are parsed. If Google breaks both, the fallbacks are `yt-dlp`, which
tracks this arms race full-time, and transcribing the audio yourself. Neither
is in here — `__init__.py` says where they would go. Until one of them is
written, a break in both clients raises `TranscriptUnavailable` rather than
storing an empty transcript: loudly, which is the whole point of the paragraph
above.
**A refusal is not an absence.** YouTube gates by address: from a VPN exit or
a datacentre range a client is answered `LOGIN_REQUIRED` ("Sign in to confirm
you're not a bot") for every video, captioned or not. Measured on 2026-09-21,
one machine, one video, seven clients, each asked bare and again with the
`visitorData` a watch page hands out:
VPN on `IOS`, `ANDROID`, `WEB`, `MWEB`, `TVHTML5` and `ANDROID_VR` all
get the bot check, and `WEB_EMBEDDED_PLAYER` an error. No tracks
from any of them, and `visitorData` changes none of it.
`youtube-transcript-api` raises `RequestBlocked`.
VPN off `IOS` and `ANDROID` return tracks that fetch. The rest are no
use even here: `WEB`, `MWEB` and `WEB_EMBEDDED_PLAYER` are
unplayable without a PO token, and `TVHTML5` and `ANDROID_VR`
are answered with the bot check from a clean address too.
So there is no anonymous client that gets past the gate and none to add: the
fix is on the network. That case raises `Blocked`, a `TranscriptUnavailable` a
caller can tell apart, because "wait for the captions" is the wrong answer to
it and would be given for ever.
"""
from __future__ import annotations
import html
import json
import logging
import re
import xml.etree.ElementTree as ET
from collections.abc import Iterator
from dataclasses import dataclass
from typing import Any
from urllib.parse import parse_qs, urlsplit
import httpx
from desk.tools.http import session
log = logging.getLogger(__name__)
INNERTUBE = "https://www.youtube.com/youtubei/v1/player?prettyPrint=false"
#: Only for the watch page, which is read for comments. A caption fetch
#: deliberately identifies as a mobile app instead — see CLIENTS.
BROWSER_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/124.0 Safari/537.36")
#: Tried in order. Each is (client context, the User-Agent that goes with it) —
#: they have to match, or the response comes back playable but captionless.
CLIENTS = (
(
{"clientName": "IOS", "clientVersion": "20.10.4",
"deviceModel": "iPhone16,2", "hl": "en"},
"com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X)",
),
(
{"clientName": "ANDROID", "clientVersion": "20.10.38",
"androidSdkVersion": 30, "hl": "en"},
"com.google.android.youtube/20.10.38 (Linux; U; Android 11) gzip",
),
)
_VIDEO_ID = re.compile(r"^[\w-]{11}$")
#: Every shape a YouTube link comes in, including the one Substack embeds
#: (`<div class="youtube-wrap" id="youtube2-VIDEOID">`).
_IN_TEXT = re.compile(
r"(?:youtu\.be/|youtube\.com/(?:watch\?(?:[^\"'\s]*&)?v=|embed/|live/|shorts/)"
r"|youtube2-)([\w-]{11})"
)
class TranscriptUnavailable(Exception):
"""No captions to be had — yet, or at all.
Raised rather than returning `[]` because the two cases an ingest has to
tell apart both look like an empty list: a video with no captions, and a
video whose captions are still being generated. The message says which,
and the caller decides whether to retry on the next pull.
"""
class Blocked(TranscriptUnavailable):
"""YouTube refused this machine, whatever the video has.
A subclass, so a caller that only wants "no captions, carry on" — a link
added by hand, a repair — is unchanged. A caller that waits for captions
has to catch this first: a refusal is not lifted by waiting, it answers
every video the same way, and retrying it quietly is how a show stores
nothing for a fortnight with nobody told. See `_refused`.
"""
@dataclass(frozen=True, slots=True)
class Cue:
"""One caption line: when it starts, how long it lasts, what was said."""
start: float
duration: float
text: str
@property
def end(self) -> float:
return self.start + self.duration
def video_id(value: str) -> str:
"""The 11-character id, from a bare id, any YouTube URL, or a blob of HTML.
Deliberately tolerant about its input: the id arrives from three different
places in this pipeline — typed by hand, in a feed entry's link, and buried
in the HTML of a Substack post — and each of them would otherwise need its
own extraction.
"""
value = (value or "").strip()
if _VIDEO_ID.match(value):
return value
if match := _IN_TEXT.search(value):
return match.group(1)
# A URL shape the pattern above doesn't cover, but with ?v= in the query.
found = parse_qs(urlsplit(value).query).get("v")
if found and _VIDEO_ID.match(found[0]):
return found[0]
raise ValueError(f"no YouTube video id in {value[:120]!r}")
def transcript(video: str, *, language: str = "en",
client: httpx.Client | None = None) -> list[Cue]:
"""Every caption cue for one video, in order.
`language` is matched as a prefix, so "en" accepts "en-US" — which matters,
because a channel that uploads a real transcript files it under a regional
code while the auto-generated one is plain "en".
Two backends, tried in that order. `youtube-transcript-api` first — it is
a dependency, and it is somebody's full-time job to keep up with YouTube's
changes. The built-in fetcher below when the library can't get the track,
so one broken path doesn't cost the transcript.
"""
cues = _via_library(video, language)
if cues is not None:
return cues
return _via_innertube(video, language, client)
#: Seconds one `requests` call inside `youtube-transcript-api` gets. Roughly
#: what `desk/tools/http.py` gives an httpx call, for the same reason.
LIBRARY_TIMEOUT = 30
def _timed_client() -> Any:
"""A `requests` session with a deadline on it, for a library that owns its HTTP.
`youtube-transcript-api` does its own fetching, on `requests`, and a
`requests` call with no `timeout=` waits on a socket read for ever. What
that costs is not a slow transcript. The work runs in a worker thread;
cancellation is cooperative, so a thread parked in a read is one the Kill
button cannot reach, and a restart re-queues the job to park on the same
video. The library takes a session of ours through `http_client`, which is
its own supported way of being told this; a `requests.Session` has no
timeout of its own to set, so the default goes on one method down.
**This is why there is no `socket.setdefaulttimeout` anywhere.** A
process-wide default is the tempting one-liner for library code that hides
its HTTP, and it is the wrong trade here: it would be inherited by every
socket the process opens, uvicorn's listener and each websocket included,
to buy a deadline for call sites that can each be told one directly. There
were two. The other was feedparser, and it is handled the same way — by
fetching where a timeout can be set; see `desk/sources/rss.py`.
Typed as `Any` because `requests` is only imported inside the guard below.
It is a `requests.Session`.
"""
import requests
class Timed(requests.Session):
def request(self, *args: Any, **kwargs: Any) -> Any:
kwargs.setdefault("timeout", LIBRARY_TIMEOUT)
return super().request(*args, **kwargs)
return Timed()
def _via_library(video: str, language: str) -> list[Cue] | None:
"""`youtube-transcript-api`. None means "not this way".
Every failure is None rather than an exception, including a video with no
captions at all: the fallback below raises the error that gets reported, so
there is one message for "no transcript" instead of two that depend on
which packages happen to be present.
"""
try:
from youtube_transcript_api import YouTubeTranscriptApi
client = _timed_client()
except ImportError:
return None
try:
tracks = YouTubeTranscriptApi(http_client=client).list(video)
wanted = [language, f"{language}-US", f"{language}-GB"]
try:
track = tracks.find_manually_created_transcript(wanted)
except Exception:
track = tracks.find_generated_transcript(wanted)
fetched = track.fetch()
except Exception as exc:
log.debug("youtube-transcript-api could not fetch %s: %s", video, exc)
return None
cues = [
Cue(start=c.start, duration=c.duration, text=clean)
for c in fetched
if (clean := _clean(c.text))
]
log.info("%s: %d cues via youtube-transcript-api", video, len(cues))
return cues or None
def _via_innertube(video: str, language: str,
client: httpx.Client | None) -> list[Cue]:
with session(client) as http:
player = _player(video, http)
track = _pick(_tracks(player), language)
if track is None:
raise TranscriptUnavailable(
f"{video}: no {language} captions. "
"Auto-captions can take a few hours to appear on a long upload."
)
# `&fmt=json3` is a request, not a promise: the ANDROID client answers
# it with SRV3 anyway. Both are parsed, so it doesn't matter which
# arrives — see `_cues`.
response = http.get(track["baseUrl"] + "&fmt=json3",
headers={"User-Agent": player.get("_caption_agent", BROWSER_UA)})
if _throttled(response):
raise Blocked(_refusal(video, [f"captions: HTTP {response.status_code}"]))
response.raise_for_status()
body = response.text
if not body.strip():
raise TranscriptUnavailable(
f"{video}: caption track fetched empty. YouTube does this when "
"the request doesn't look like the client that asked for it."
)
cues = _cues(body)
log.info("%s: %d cues, %.0f minutes", video, len(cues),
(cues[-1].end / 60) if cues else 0)
return cues
def details(video: str, *, client: httpx.Client | None = None) -> dict[str, Any]:
"""Title, author, length and description for a video.
The description is here because for a show that publishes no newsletter
it is the only place the chapter marks live — `marks.parse` reads it as
plain text. Everything else is the metadata an episode needs when YouTube
is the only source there is.
"""
with session(client) as http:
player = _player(video, http, require_captions=False)
video_details = player.get("videoDetails", {})
return {
"title": video_details.get("title", ""),
"author": video_details.get("author", ""),
"duration": float(video_details.get("lengthSeconds") or 0),
"description": video_details.get("shortDescription", ""),
}
def comments(video: str, *, limit: int = 100,
client: httpx.Client | None = None) -> list[str]:
"""Top-level comments, newest-and-most-liked first, as plain text.
Here for one reason: when a show does not publish an outline, somebody in
the comments usually has. A viewer typing out "0:00 intro / 3:32 Gemini /
13:11 Grok" is doing the same work the show notes would have done, and
YouTube itself treats such a comment as authoritative enough to draw
chapter markers from it. `marks.parse` reads these exactly as it reads a
description — see `marks.best`.
Returns `[]` rather than raising when comments are off, or when YouTube
changes the shape again. A missing fallback is not worth failing an ingest
that already has a transcript.
"""
try:
with session(client) as http:
page = http.get(f"https://www.youtube.com/watch?v={video}",
headers={"User-Agent": BROWSER_UA,
"Accept-Language": "en-US,en"}).text
context = _json_after(page, '"INNERTUBE_CONTEXT":')
initial = _json_after(page, "ytInitialData = ")
if not context or not initial:
return []
out: list[str] = []
# Several continuations live in the page (the related rail, the
# description); only one of them is the comment feed, and which
# index that is moves around. Trying them in order and keeping the
# first that yields comments is cheaper than pinning a path
# through a structure Google reshuffles.
for token in _continuations(initial)[:4]:
response = http.post(
"https://www.youtube.com/youtubei/v1/next?prettyPrint=false",
json={"context": context, "continuation": token},
headers={"Content-Type": "application/json",
"User-Agent": BROWSER_UA},
)
out = _comment_texts(response.json())
if out:
break
except Exception as exc:
log.debug("could not read comments for %s: %s", video, exc)
return []
log.info("%s: %d comments read", video, len(out))
return out[:limit]
def text_between(cues: list[Cue], start: float, end: float | None = None) -> str:
"""The words spoken in a span, as one paragraph.
A cue counts as inside the span when it *starts* inside it. Overlap would
be the other defensible rule, but it double-counts the sentence that
straddles a boundary — and these boundaries come from chapter marks a human
typed, so a sentence of drift at the seam is already there and doesn't need
doubling.
"""
return " ".join(
cue.text for cue in cues
if cue.start >= start and (end is None or cue.start < end)
).strip()
# ------------------------------------------------- internals: the watch page
def _json_after(page: str, marker: str) -> dict[str, Any] | None:
"""The JSON object that follows `marker` in a watch page.
Brace-matched rather than matched with a regex. The two blobs this reads —
`ytInitialData` and `INNERTUBE_CONTEXT` — are hundreds of kilobytes of
nested JSON containing every bracket character inside string values, and a
non-greedy regex stops at the first `}` inside a caption or a title. This
walks the string properly, which is about fifteen lines and always right.
"""
start = page.find(marker)
if start < 0:
return None
start = page.find("{", start + len(marker))
if start < 0:
return None
depth, in_string, escaped = 0, False, False
for index in range(start, len(page)):
char = page[index]
if in_string:
if escaped:
escaped = False
elif char == "\\":
escaped = True
elif char == '"':
in_string = False
continue
if char == '"':
in_string = True
elif char == "{":
depth += 1
elif char == "}":
depth -= 1
if depth == 0:
try:
return json.loads(page[start:index + 1])
except ValueError:
return None
return None
def _walk(node: Any, key: str) -> Iterator[Any]:
"""Every value stored under `key`, anywhere in a nested structure."""
if isinstance(node, dict):
if key in node:
yield node[key]
for value in node.values():
yield from _walk(value, key)
elif isinstance(node, list):
for value in node:
yield from _walk(value, key)
def _continuations(initial: dict[str, Any]) -> list[str]:
return [
token for renderer in _walk(initial, "continuationItemRenderer")
if (token := (renderer.get("continuationEndpoint", {})
.get("continuationCommand", {}) or {}).get("token"))
]
def _comment_texts(response: dict[str, Any]) -> list[str]:
"""Comment bodies out of a `next` response.
Comments arrive as `commentEntityPayload` entities in a flat batch beside
the renderers that position them, so reading them needs no understanding of
the layout at all — which is the part that keeps changing.
"""
out = []
for payload in _walk(response, "commentEntityPayload"):
text = (payload.get("properties", {}).get("content", {}) or {}).get("content", "")
if text:
out.append(text)
return out
# ------------------------------------------------------------------ internals
def _player(video: str, http: httpx.Client,
*, require_captions: bool = True) -> dict[str, Any]:
"""The player response, from whichever client answers with captions.
`require_captions` is what separates the two callers. A transcript wants
the first client that actually offers caption tracks; metadata wants the
first that answers at all, and would otherwise fail on a video that plays
fine and simply has no captions yet.
Every client's answer is kept for the error, not only the last one's: the
two fail differently, and which of them said what is the first thing
anyone asks when this breaks.
"""
said: list[str] = []
refused = False
for context, agent in CLIENTS:
name = context["clientName"]
payload = {"context": {"client": context}, "videoId": video,
"contentCheckOk": True, "racyCheckOk": True}
try:
response = http.post(
INNERTUBE, json=payload,
headers={"Content-Type": "application/json", "User-Agent": agent},
)
if _throttled(response):
refused = True
said.append(f"{name}: HTTP {response.status_code}")
continue
data = response.json()
except (httpx.HTTPError, ValueError) as exc:
said.append(f"{name}: {exc}")
continue
playability = data.get("playabilityStatus", {})
status = playability.get("status", "?")
if _tracks(data) or (not require_captions and status == "OK"):
# Caption URLs are bound to the client that obtained the player.
data["_caption_agent"] = agent
return data
refused = refused or _refused(playability)
reason = playability.get("reason") or ""
said.append(f"{name}: {status}" + (f" ({reason})" if reason else "")
+ ", no caption tracks")
log.debug("innertube %s", said[-1])
if refused:
raise Blocked(_refusal(video, said))
raise TranscriptUnavailable(
f"{video}: {'; '.join(said) or 'no client returned captions'}")
def _refused(playability: dict[str, Any]) -> bool:
"""Whether a player response is YouTube declining to serve this machine.
The status alone, not the reason beside it. The bot check reads "Sign in
to confirm you're not a bot" today, and a match on that sentence would go
back to reporting a refusal as captions not being ready on the day it is
reworded — silently, which is the failure this exists to end. The status
is the stable half, and it is sufficient: nothing asked anonymously is
ever let past `LOGIN_REQUIRED` by being asked again later.
That also takes in a private or age-gated video, where the refusal is
about the video rather than the address. It is still not a wait, and the
reason is quoted in the error for whoever reads it.
"""
return playability.get("status") == "LOGIN_REQUIRED"
def _throttled(response: httpx.Response) -> bool:
"""The same refusal, made in HTTP: a 403, a 429, or Google's `/sorry/`
interstitial, which arrives as a redirect to a captcha page."""
return response.status_code in (403, 429) or "/sorry/" in response.url.path
def _refusal(video: str, said: list[str]) -> str:
"""`Blocked`'s sentence. "YouTube refused" is what `desk/notices.py` reads
to tell this from a feed that failed, so the wording is load-bearing."""
return (f"{video}: YouTube refused this address and wants a sign-in — "
+ "; ".join(said))
def _tracks(player: dict[str, Any]) -> list[dict[str, Any]]:
return (player.get("captions", {})
.get("playerCaptionsTracklistRenderer", {})
.get("captionTracks", []) or [])
def _pick(tracks: list[dict[str, Any]], language: str) -> dict[str, Any] | None:
"""Best track for a language: a real transcript over an auto-generated one.
A channel that uploads its own captions has punctuation, speaker turns and
correctly spelled names; ASR has none of those and spells "Qwen" six ways.
Worth preferring, and never worth requiring — most shows only have ASR.
"""
wanted = [t for t in tracks
if (t.get("languageCode") or "").lower().startswith(language.lower())]
wanted.sort(key=lambda t: t.get("kind") == "asr")
return wanted[0] if wanted else None
def _cues(body: str) -> list[Cue]:
"""Parse whichever of YouTube's three caption formats turned up."""
body = body.strip()
if body.startswith("{"):
return _from_json3(json.loads(body))
root = ET.fromstring(body)
# SRV1 is `transcript > text[start,dur]`; SRV3 is `timedtext > body > p[t,d]`.
if root.tag == "transcript":
return _from_srv1(root)
return _from_srv3(root)
def _from_json3(data: dict[str, Any]) -> list[Cue]:
cues = []
for event in data.get("events", []):
text = "".join(seg.get("utf8", "") for seg in event.get("segs", []))
clean = _clean(text)
if not clean:
continue
cues.append(Cue(
start=event.get("tStartMs", 0) / 1000,
duration=event.get("dDurationMs", 0) / 1000,
text=clean,
))
return cues
def _from_srv1(root: ET.Element) -> list[Cue]:
cues = []
for node in root.iter("text"):
clean = _clean(node.text or "")
if not clean:
continue
cues.append(Cue(
start=float(node.get("start", 0)),
duration=float(node.get("dur", 0)),
text=clean,
))
return cues
def _from_srv3(root: ET.Element) -> list[Cue]:
cues = []
for node in root.iter("p"):
clean = _clean("".join(node.itertext()))
if not clean:
continue
cues.append(Cue(
start=int(node.get("t", 0)) / 1000,
duration=int(node.get("d", 0)) / 1000,
text=clean,
))
return cues
def _clean(text: str) -> str:
"""Entities out, whitespace collapsed, sound cues dropped.
Unescaped twice on purpose: SRV1 sends `&amp;#39;` — an entity for an
entity — because the caption text was escaped once when it was stored and
once again on the way into the XML.
"""
text = html.unescape(html.unescape(text or ""))
text = re.sub(r"\[(?:Music|Applause|Laughter)\]", " ", text)
return re.sub(r"\s+", " ", text).strip()