Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d666d0937
commit
248ea1636d
36 files changed
+8393
No files matched your search
@@ -0,0 +1,600 @@
|
||||
"""Captions, with their timings, for one YouTube video.
|
||||
|
||||
YouTube has always published a machine-readable transcript for anything with
|
||||
captions — including the auto-generated ones, which is most podcasts. It is the
|
||||
cheapest transcript in existence: somebody else already paid for the ASR.
|
||||
|
||||
Getting at it is the fiddly part, and it is fiddly for a reason worth writing
|
||||
down. The caption URLs embedded in a `watch` page are signed for a *browser*
|
||||
session, and fetching one from anywhere else now returns `200 OK` with an empty
|
||||
body — no error, no redirect, nothing to catch. So the obvious implementation
|
||||
(scrape the watch page, follow `baseUrl`) fails silently, which is the worst
|
||||
way for an ingest step to fail.
|
||||
|
||||
What works is asking the same endpoint the mobile app asks. `/youtubei/v1/player`
|
||||
with a mobile client context hands back a player response whose caption URLs are
|
||||
not tied to a browser session, and those fetch fine. That is the whole trick,
|
||||
and it is why this module talks to an undocumented endpoint instead of scraping
|
||||
a page: the scrape is the fragile one.
|
||||
|
||||
Two clients are tried in order, because they fail differently — `IOS` returns
|
||||
JSON3 when asked and `ANDROID` returns SRV3 whatever you ask for, so both
|
||||
formats are parsed. If Google breaks both, the fallbacks are `yt-dlp`, which
|
||||
tracks this arms race full-time, and transcribing the audio yourself. Neither
|
||||
is in here — `__init__.py` says where they would go. Until one of them is
|
||||
written, a break in both clients raises `TranscriptUnavailable` rather than
|
||||
storing an empty transcript: loudly, which is the whole point of the paragraph
|
||||
above.
|
||||
|
||||
**A refusal is not an absence.** YouTube gates by address: from a VPN exit or
|
||||
a datacentre range a client is answered `LOGIN_REQUIRED` ("Sign in to confirm
|
||||
you're not a bot") for every video, captioned or not. Measured on 2026-09-21,
|
||||
one machine, one video, seven clients, each asked bare and again with the
|
||||
`visitorData` a watch page hands out:
|
||||
|
||||
VPN on `IOS`, `ANDROID`, `WEB`, `MWEB`, `TVHTML5` and `ANDROID_VR` all
|
||||
get the bot check, and `WEB_EMBEDDED_PLAYER` an error. No tracks
|
||||
from any of them, and `visitorData` changes none of it.
|
||||
`youtube-transcript-api` raises `RequestBlocked`.
|
||||
VPN off `IOS` and `ANDROID` return tracks that fetch. The rest are no
|
||||
use even here: `WEB`, `MWEB` and `WEB_EMBEDDED_PLAYER` are
|
||||
unplayable without a PO token, and `TVHTML5` and `ANDROID_VR`
|
||||
are answered with the bot check from a clean address too.
|
||||
|
||||
So there is no anonymous client that gets past the gate and none to add: the
|
||||
fix is on the network. That case raises `Blocked`, a `TranscriptUnavailable` a
|
||||
caller can tell apart, because "wait for the captions" is the wrong answer to
|
||||
it and would be given for ever.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import httpx
|
||||
|
||||
from desk.tools.http import session
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
INNERTUBE = "https://www.youtube.com/youtubei/v1/player?prettyPrint=false"
|
||||
|
||||
#: Only for the watch page, which is read for comments. A caption fetch
|
||||
#: deliberately identifies as a mobile app instead — see CLIENTS.
|
||||
BROWSER_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/124.0 Safari/537.36")
|
||||
|
||||
#: Tried in order. Each is (client context, the User-Agent that goes with it) —
|
||||
#: they have to match, or the response comes back playable but captionless.
|
||||
CLIENTS = (
|
||||
(
|
||||
{"clientName": "IOS", "clientVersion": "20.10.4",
|
||||
"deviceModel": "iPhone16,2", "hl": "en"},
|
||||
"com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X)",
|
||||
),
|
||||
(
|
||||
{"clientName": "ANDROID", "clientVersion": "20.10.38",
|
||||
"androidSdkVersion": 30, "hl": "en"},
|
||||
"com.google.android.youtube/20.10.38 (Linux; U; Android 11) gzip",
|
||||
),
|
||||
)
|
||||
|
||||
_VIDEO_ID = re.compile(r"^[\w-]{11}$")
|
||||
#: Every shape a YouTube link comes in, including the one Substack embeds
|
||||
#: (`<div class="youtube-wrap" id="youtube2-VIDEOID">`).
|
||||
_IN_TEXT = re.compile(
|
||||
r"(?:youtu\.be/|youtube\.com/(?:watch\?(?:[^\"'\s]*&)?v=|embed/|live/|shorts/)"
|
||||
r"|youtube2-)([\w-]{11})"
|
||||
)
|
||||
|
||||
|
||||
class TranscriptUnavailable(Exception):
|
||||
"""No captions to be had — yet, or at all.
|
||||
|
||||
Raised rather than returning `[]` because the two cases an ingest has to
|
||||
tell apart both look like an empty list: a video with no captions, and a
|
||||
video whose captions are still being generated. The message says which,
|
||||
and the caller decides whether to retry on the next pull.
|
||||
"""
|
||||
|
||||
|
||||
class Blocked(TranscriptUnavailable):
|
||||
"""YouTube refused this machine, whatever the video has.
|
||||
|
||||
A subclass, so a caller that only wants "no captions, carry on" — a link
|
||||
added by hand, a repair — is unchanged. A caller that waits for captions
|
||||
has to catch this first: a refusal is not lifted by waiting, it answers
|
||||
every video the same way, and retrying it quietly is how a show stores
|
||||
nothing for a fortnight with nobody told. See `_refused`.
|
||||
"""
|
||||
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Cue:
|
||||
"""One caption line: when it starts, how long it lasts, what was said."""
|
||||
|
||||
start: float
|
||||
duration: float
|
||||
text: str
|
||||
|
||||
@property
|
||||
def end(self) -> float:
|
||||
return self.start + self.duration
|
||||
|
||||
|
||||
def video_id(value: str) -> str:
|
||||
"""The 11-character id, from a bare id, any YouTube URL, or a blob of HTML.
|
||||
|
||||
Deliberately tolerant about its input: the id arrives from three different
|
||||
places in this pipeline — typed by hand, in a feed entry's link, and buried
|
||||
in the HTML of a Substack post — and each of them would otherwise need its
|
||||
own extraction.
|
||||
"""
|
||||
value = (value or "").strip()
|
||||
if _VIDEO_ID.match(value):
|
||||
return value
|
||||
if match := _IN_TEXT.search(value):
|
||||
return match.group(1)
|
||||
# A URL shape the pattern above doesn't cover, but with ?v= in the query.
|
||||
found = parse_qs(urlsplit(value).query).get("v")
|
||||
if found and _VIDEO_ID.match(found[0]):
|
||||
return found[0]
|
||||
raise ValueError(f"no YouTube video id in {value[:120]!r}")
|
||||
|
||||
|
||||
def transcript(video: str, *, language: str = "en",
|
||||
client: httpx.Client | None = None) -> list[Cue]:
|
||||
"""Every caption cue for one video, in order.
|
||||
|
||||
`language` is matched as a prefix, so "en" accepts "en-US" — which matters,
|
||||
because a channel that uploads a real transcript files it under a regional
|
||||
code while the auto-generated one is plain "en".
|
||||
|
||||
Two backends, tried in that order. `youtube-transcript-api` first — it is
|
||||
a dependency, and it is somebody's full-time job to keep up with YouTube's
|
||||
changes. The built-in fetcher below when the library can't get the track,
|
||||
so one broken path doesn't cost the transcript.
|
||||
"""
|
||||
cues = _via_library(video, language)
|
||||
if cues is not None:
|
||||
return cues
|
||||
return _via_innertube(video, language, client)
|
||||
|
||||
|
||||
#: Seconds one `requests` call inside `youtube-transcript-api` gets. Roughly
|
||||
#: what `desk/tools/http.py` gives an httpx call, for the same reason.
|
||||
LIBRARY_TIMEOUT = 30
|
||||
|
||||
|
||||
def _timed_client() -> Any:
|
||||
"""A `requests` session with a deadline on it, for a library that owns its HTTP.
|
||||
|
||||
`youtube-transcript-api` does its own fetching, on `requests`, and a
|
||||
`requests` call with no `timeout=` waits on a socket read for ever. What
|
||||
that costs is not a slow transcript. The work runs in a worker thread;
|
||||
cancellation is cooperative, so a thread parked in a read is one the Kill
|
||||
button cannot reach, and a restart re-queues the job to park on the same
|
||||
video. The library takes a session of ours through `http_client`, which is
|
||||
its own supported way of being told this; a `requests.Session` has no
|
||||
timeout of its own to set, so the default goes on one method down.
|
||||
|
||||
**This is why there is no `socket.setdefaulttimeout` anywhere.** A
|
||||
process-wide default is the tempting one-liner for library code that hides
|
||||
its HTTP, and it is the wrong trade here: it would be inherited by every
|
||||
socket the process opens, uvicorn's listener and each websocket included,
|
||||
to buy a deadline for call sites that can each be told one directly. There
|
||||
were two. The other was feedparser, and it is handled the same way — by
|
||||
fetching where a timeout can be set; see `desk/sources/rss.py`.
|
||||
|
||||
Typed as `Any` because `requests` is only imported inside the guard below.
|
||||
It is a `requests.Session`.
|
||||
"""
|
||||
import requests
|
||||
|
||||
class Timed(requests.Session):
|
||||
def request(self, *args: Any, **kwargs: Any) -> Any:
|
||||
kwargs.setdefault("timeout", LIBRARY_TIMEOUT)
|
||||
return super().request(*args, **kwargs)
|
||||
|
||||
return Timed()
|
||||
|
||||
|
||||
def _via_library(video: str, language: str) -> list[Cue] | None:
|
||||
"""`youtube-transcript-api`. None means "not this way".
|
||||
|
||||
Every failure is None rather than an exception, including a video with no
|
||||
captions at all: the fallback below raises the error that gets reported, so
|
||||
there is one message for "no transcript" instead of two that depend on
|
||||
which packages happen to be present.
|
||||
"""
|
||||
try:
|
||||
from youtube_transcript_api import YouTubeTranscriptApi
|
||||
|
||||
client = _timed_client()
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
try:
|
||||
tracks = YouTubeTranscriptApi(http_client=client).list(video)
|
||||
wanted = [language, f"{language}-US", f"{language}-GB"]
|
||||
try:
|
||||
track = tracks.find_manually_created_transcript(wanted)
|
||||
except Exception:
|
||||
track = tracks.find_generated_transcript(wanted)
|
||||
fetched = track.fetch()
|
||||
except Exception as exc:
|
||||
log.debug("youtube-transcript-api could not fetch %s: %s", video, exc)
|
||||
return None
|
||||
|
||||
cues = [
|
||||
Cue(start=c.start, duration=c.duration, text=clean)
|
||||
for c in fetched
|
||||
if (clean := _clean(c.text))
|
||||
]
|
||||
log.info("%s: %d cues via youtube-transcript-api", video, len(cues))
|
||||
return cues or None
|
||||
|
||||
|
||||
def _via_innertube(video: str, language: str,
|
||||
client: httpx.Client | None) -> list[Cue]:
|
||||
with session(client) as http:
|
||||
player = _player(video, http)
|
||||
track = _pick(_tracks(player), language)
|
||||
if track is None:
|
||||
raise TranscriptUnavailable(
|
||||
f"{video}: no {language} captions. "
|
||||
"Auto-captions can take a few hours to appear on a long upload."
|
||||
)
|
||||
# `&fmt=json3` is a request, not a promise: the ANDROID client answers
|
||||
# it with SRV3 anyway. Both are parsed, so it doesn't matter which
|
||||
# arrives — see `_cues`.
|
||||
response = http.get(track["baseUrl"] + "&fmt=json3",
|
||||
headers={"User-Agent": player.get("_caption_agent", BROWSER_UA)})
|
||||
if _throttled(response):
|
||||
raise Blocked(_refusal(video, [f"captions: HTTP {response.status_code}"]))
|
||||
response.raise_for_status()
|
||||
body = response.text
|
||||
if not body.strip():
|
||||
raise TranscriptUnavailable(
|
||||
f"{video}: caption track fetched empty. YouTube does this when "
|
||||
"the request doesn't look like the client that asked for it."
|
||||
)
|
||||
cues = _cues(body)
|
||||
log.info("%s: %d cues, %.0f minutes", video, len(cues),
|
||||
(cues[-1].end / 60) if cues else 0)
|
||||
return cues
|
||||
|
||||
|
||||
def details(video: str, *, client: httpx.Client | None = None) -> dict[str, Any]:
|
||||
"""Title, author, length and description for a video.
|
||||
|
||||
The description is here because for a show that publishes no newsletter
|
||||
it is the only place the chapter marks live — `marks.parse` reads it as
|
||||
plain text. Everything else is the metadata an episode needs when YouTube
|
||||
is the only source there is.
|
||||
"""
|
||||
with session(client) as http:
|
||||
player = _player(video, http, require_captions=False)
|
||||
video_details = player.get("videoDetails", {})
|
||||
return {
|
||||
"title": video_details.get("title", ""),
|
||||
"author": video_details.get("author", ""),
|
||||
"duration": float(video_details.get("lengthSeconds") or 0),
|
||||
"description": video_details.get("shortDescription", ""),
|
||||
}
|
||||
|
||||
|
||||
def comments(video: str, *, limit: int = 100,
|
||||
client: httpx.Client | None = None) -> list[str]:
|
||||
"""Top-level comments, newest-and-most-liked first, as plain text.
|
||||
|
||||
Here for one reason: when a show does not publish an outline, somebody in
|
||||
the comments usually has. A viewer typing out "0:00 intro / 3:32 Gemini /
|
||||
13:11 Grok" is doing the same work the show notes would have done, and
|
||||
YouTube itself treats such a comment as authoritative enough to draw
|
||||
chapter markers from it. `marks.parse` reads these exactly as it reads a
|
||||
description — see `marks.best`.
|
||||
|
||||
Returns `[]` rather than raising when comments are off, or when YouTube
|
||||
changes the shape again. A missing fallback is not worth failing an ingest
|
||||
that already has a transcript.
|
||||
"""
|
||||
try:
|
||||
with session(client) as http:
|
||||
page = http.get(f"https://www.youtube.com/watch?v={video}",
|
||||
headers={"User-Agent": BROWSER_UA,
|
||||
"Accept-Language": "en-US,en"}).text
|
||||
context = _json_after(page, '"INNERTUBE_CONTEXT":')
|
||||
initial = _json_after(page, "ytInitialData = ")
|
||||
if not context or not initial:
|
||||
return []
|
||||
out: list[str] = []
|
||||
# Several continuations live in the page (the related rail, the
|
||||
# description); only one of them is the comment feed, and which
|
||||
# index that is moves around. Trying them in order and keeping the
|
||||
# first that yields comments is cheaper than pinning a path
|
||||
# through a structure Google reshuffles.
|
||||
for token in _continuations(initial)[:4]:
|
||||
response = http.post(
|
||||
"https://www.youtube.com/youtubei/v1/next?prettyPrint=false",
|
||||
json={"context": context, "continuation": token},
|
||||
headers={"Content-Type": "application/json",
|
||||
"User-Agent": BROWSER_UA},
|
||||
)
|
||||
out = _comment_texts(response.json())
|
||||
if out:
|
||||
break
|
||||
except Exception as exc:
|
||||
log.debug("could not read comments for %s: %s", video, exc)
|
||||
return []
|
||||
log.info("%s: %d comments read", video, len(out))
|
||||
return out[:limit]
|
||||
|
||||
|
||||
def text_between(cues: list[Cue], start: float, end: float | None = None) -> str:
|
||||
"""The words spoken in a span, as one paragraph.
|
||||
|
||||
A cue counts as inside the span when it *starts* inside it. Overlap would
|
||||
be the other defensible rule, but it double-counts the sentence that
|
||||
straddles a boundary — and these boundaries come from chapter marks a human
|
||||
typed, so a sentence of drift at the seam is already there and doesn't need
|
||||
doubling.
|
||||
"""
|
||||
return " ".join(
|
||||
cue.text for cue in cues
|
||||
if cue.start >= start and (end is None or cue.start < end)
|
||||
).strip()
|
||||
|
||||
|
||||
# ------------------------------------------------- internals: the watch page
|
||||
|
||||
def _json_after(page: str, marker: str) -> dict[str, Any] | None:
|
||||
"""The JSON object that follows `marker` in a watch page.
|
||||
|
||||
Brace-matched rather than matched with a regex. The two blobs this reads —
|
||||
`ytInitialData` and `INNERTUBE_CONTEXT` — are hundreds of kilobytes of
|
||||
nested JSON containing every bracket character inside string values, and a
|
||||
non-greedy regex stops at the first `}` inside a caption or a title. This
|
||||
walks the string properly, which is about fifteen lines and always right.
|
||||
"""
|
||||
start = page.find(marker)
|
||||
if start < 0:
|
||||
return None
|
||||
start = page.find("{", start + len(marker))
|
||||
if start < 0:
|
||||
return None
|
||||
|
||||
depth, in_string, escaped = 0, False, False
|
||||
for index in range(start, len(page)):
|
||||
char = page[index]
|
||||
if in_string:
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif char == '"':
|
||||
in_string = False
|
||||
continue
|
||||
if char == '"':
|
||||
in_string = True
|
||||
elif char == "{":
|
||||
depth += 1
|
||||
elif char == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
try:
|
||||
return json.loads(page[start:index + 1])
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _walk(node: Any, key: str) -> Iterator[Any]:
|
||||
"""Every value stored under `key`, anywhere in a nested structure."""
|
||||
if isinstance(node, dict):
|
||||
if key in node:
|
||||
yield node[key]
|
||||
for value in node.values():
|
||||
yield from _walk(value, key)
|
||||
elif isinstance(node, list):
|
||||
for value in node:
|
||||
yield from _walk(value, key)
|
||||
|
||||
|
||||
def _continuations(initial: dict[str, Any]) -> list[str]:
|
||||
return [
|
||||
token for renderer in _walk(initial, "continuationItemRenderer")
|
||||
if (token := (renderer.get("continuationEndpoint", {})
|
||||
.get("continuationCommand", {}) or {}).get("token"))
|
||||
]
|
||||
|
||||
|
||||
def _comment_texts(response: dict[str, Any]) -> list[str]:
|
||||
"""Comment bodies out of a `next` response.
|
||||
|
||||
Comments arrive as `commentEntityPayload` entities in a flat batch beside
|
||||
the renderers that position them, so reading them needs no understanding of
|
||||
the layout at all — which is the part that keeps changing.
|
||||
"""
|
||||
out = []
|
||||
for payload in _walk(response, "commentEntityPayload"):
|
||||
text = (payload.get("properties", {}).get("content", {}) or {}).get("content", "")
|
||||
if text:
|
||||
out.append(text)
|
||||
return out
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ internals
|
||||
|
||||
def _player(video: str, http: httpx.Client,
|
||||
*, require_captions: bool = True) -> dict[str, Any]:
|
||||
"""The player response, from whichever client answers with captions.
|
||||
|
||||
`require_captions` is what separates the two callers. A transcript wants
|
||||
the first client that actually offers caption tracks; metadata wants the
|
||||
first that answers at all, and would otherwise fail on a video that plays
|
||||
fine and simply has no captions yet.
|
||||
|
||||
Every client's answer is kept for the error, not only the last one's: the
|
||||
two fail differently, and which of them said what is the first thing
|
||||
anyone asks when this breaks.
|
||||
"""
|
||||
said: list[str] = []
|
||||
refused = False
|
||||
for context, agent in CLIENTS:
|
||||
name = context["clientName"]
|
||||
payload = {"context": {"client": context}, "videoId": video,
|
||||
"contentCheckOk": True, "racyCheckOk": True}
|
||||
try:
|
||||
response = http.post(
|
||||
INNERTUBE, json=payload,
|
||||
headers={"Content-Type": "application/json", "User-Agent": agent},
|
||||
)
|
||||
if _throttled(response):
|
||||
refused = True
|
||||
said.append(f"{name}: HTTP {response.status_code}")
|
||||
continue
|
||||
data = response.json()
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
said.append(f"{name}: {exc}")
|
||||
continue
|
||||
playability = data.get("playabilityStatus", {})
|
||||
status = playability.get("status", "?")
|
||||
if _tracks(data) or (not require_captions and status == "OK"):
|
||||
# Caption URLs are bound to the client that obtained the player.
|
||||
data["_caption_agent"] = agent
|
||||
return data
|
||||
refused = refused or _refused(playability)
|
||||
reason = playability.get("reason") or ""
|
||||
said.append(f"{name}: {status}" + (f" ({reason})" if reason else "")
|
||||
+ ", no caption tracks")
|
||||
log.debug("innertube %s", said[-1])
|
||||
if refused:
|
||||
raise Blocked(_refusal(video, said))
|
||||
raise TranscriptUnavailable(
|
||||
f"{video}: {'; '.join(said) or 'no client returned captions'}")
|
||||
|
||||
|
||||
def _refused(playability: dict[str, Any]) -> bool:
|
||||
"""Whether a player response is YouTube declining to serve this machine.
|
||||
|
||||
The status alone, not the reason beside it. The bot check reads "Sign in
|
||||
to confirm you're not a bot" today, and a match on that sentence would go
|
||||
back to reporting a refusal as captions not being ready on the day it is
|
||||
reworded — silently, which is the failure this exists to end. The status
|
||||
is the stable half, and it is sufficient: nothing asked anonymously is
|
||||
ever let past `LOGIN_REQUIRED` by being asked again later.
|
||||
|
||||
That also takes in a private or age-gated video, where the refusal is
|
||||
about the video rather than the address. It is still not a wait, and the
|
||||
reason is quoted in the error for whoever reads it.
|
||||
"""
|
||||
return playability.get("status") == "LOGIN_REQUIRED"
|
||||
|
||||
|
||||
def _throttled(response: httpx.Response) -> bool:
|
||||
"""The same refusal, made in HTTP: a 403, a 429, or Google's `/sorry/`
|
||||
interstitial, which arrives as a redirect to a captcha page."""
|
||||
return response.status_code in (403, 429) or "/sorry/" in response.url.path
|
||||
|
||||
|
||||
def _refusal(video: str, said: list[str]) -> str:
|
||||
"""`Blocked`'s sentence. "YouTube refused" is what `desk/notices.py` reads
|
||||
to tell this from a feed that failed, so the wording is load-bearing."""
|
||||
return (f"{video}: YouTube refused this address and wants a sign-in — "
|
||||
+ "; ".join(said))
|
||||
|
||||
|
||||
def _tracks(player: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
return (player.get("captions", {})
|
||||
.get("playerCaptionsTracklistRenderer", {})
|
||||
.get("captionTracks", []) or [])
|
||||
|
||||
|
||||
def _pick(tracks: list[dict[str, Any]], language: str) -> dict[str, Any] | None:
|
||||
"""Best track for a language: a real transcript over an auto-generated one.
|
||||
|
||||
A channel that uploads its own captions has punctuation, speaker turns and
|
||||
correctly spelled names; ASR has none of those and spells "Qwen" six ways.
|
||||
Worth preferring, and never worth requiring — most shows only have ASR.
|
||||
"""
|
||||
wanted = [t for t in tracks
|
||||
if (t.get("languageCode") or "").lower().startswith(language.lower())]
|
||||
wanted.sort(key=lambda t: t.get("kind") == "asr")
|
||||
return wanted[0] if wanted else None
|
||||
|
||||
|
||||
def _cues(body: str) -> list[Cue]:
|
||||
"""Parse whichever of YouTube's three caption formats turned up."""
|
||||
body = body.strip()
|
||||
if body.startswith("{"):
|
||||
return _from_json3(json.loads(body))
|
||||
root = ET.fromstring(body)
|
||||
# SRV1 is `transcript > text[start,dur]`; SRV3 is `timedtext > body > p[t,d]`.
|
||||
if root.tag == "transcript":
|
||||
return _from_srv1(root)
|
||||
return _from_srv3(root)
|
||||
|
||||
|
||||
def _from_json3(data: dict[str, Any]) -> list[Cue]:
|
||||
cues = []
|
||||
for event in data.get("events", []):
|
||||
text = "".join(seg.get("utf8", "") for seg in event.get("segs", []))
|
||||
clean = _clean(text)
|
||||
if not clean:
|
||||
continue
|
||||
cues.append(Cue(
|
||||
start=event.get("tStartMs", 0) / 1000,
|
||||
duration=event.get("dDurationMs", 0) / 1000,
|
||||
text=clean,
|
||||
))
|
||||
return cues
|
||||
|
||||
|
||||
def _from_srv1(root: ET.Element) -> list[Cue]:
|
||||
cues = []
|
||||
for node in root.iter("text"):
|
||||
clean = _clean(node.text or "")
|
||||
if not clean:
|
||||
continue
|
||||
cues.append(Cue(
|
||||
start=float(node.get("start", 0)),
|
||||
duration=float(node.get("dur", 0)),
|
||||
text=clean,
|
||||
))
|
||||
return cues
|
||||
|
||||
|
||||
def _from_srv3(root: ET.Element) -> list[Cue]:
|
||||
cues = []
|
||||
for node in root.iter("p"):
|
||||
clean = _clean("".join(node.itertext()))
|
||||
if not clean:
|
||||
continue
|
||||
cues.append(Cue(
|
||||
start=int(node.get("t", 0)) / 1000,
|
||||
duration=int(node.get("d", 0)) / 1000,
|
||||
text=clean,
|
||||
))
|
||||
return cues
|
||||
|
||||
|
||||
def _clean(text: str) -> str:
|
||||
"""Entities out, whitespace collapsed, sound cues dropped.
|
||||
|
||||
Unescaped twice on purpose: SRV1 sends `&#39;` — an entity for an
|
||||
entity — because the caption text was escaped once when it was stored and
|
||||
once again on the way into the XML.
|
||||
"""
|
||||
text = html.unescape(html.unescape(text or ""))
|
||||
text = re.sub(r"\[(?:Music|Applause|Laughter)\]", " ", text)
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
Reference in new issue
Block a user