The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
601 lines
24 KiB
Python
601 lines
24 KiB
Python
"""Captions, with their timings, for one YouTube video.
|
|
|
|
YouTube has always published a machine-readable transcript for anything with
|
|
captions — including the auto-generated ones, which is most podcasts. It is the
|
|
cheapest transcript in existence: somebody else already paid for the ASR.
|
|
|
|
Getting at it is the fiddly part, and it is fiddly for a reason worth writing
|
|
down. The caption URLs embedded in a `watch` page are signed for a *browser*
|
|
session, and fetching one from anywhere else now returns `200 OK` with an empty
|
|
body — no error, no redirect, nothing to catch. So the obvious implementation
|
|
(scrape the watch page, follow `baseUrl`) fails silently, which is the worst
|
|
way for an ingest step to fail.
|
|
|
|
What works is asking the same endpoint the mobile app asks. `/youtubei/v1/player`
|
|
with a mobile client context hands back a player response whose caption URLs are
|
|
not tied to a browser session, and those fetch fine. That is the whole trick,
|
|
and it is why this module talks to an undocumented endpoint instead of scraping
|
|
a page: the scrape is the fragile one.
|
|
|
|
Two clients are tried in order, because they fail differently — `IOS` returns
|
|
JSON3 when asked and `ANDROID` returns SRV3 whatever you ask for, so both
|
|
formats are parsed. If Google breaks both, the fallbacks are `yt-dlp`, which
|
|
tracks this arms race full-time, and transcribing the audio yourself. Neither
|
|
is in here — `__init__.py` says where they would go. Until one of them is
|
|
written, a break in both clients raises `TranscriptUnavailable` rather than
|
|
storing an empty transcript: loudly, which is the whole point of the paragraph
|
|
above.
|
|
|
|
**A refusal is not an absence.** YouTube gates by address: from a VPN exit or
|
|
a datacentre range a client is answered `LOGIN_REQUIRED` ("Sign in to confirm
|
|
you're not a bot") for every video, captioned or not. Measured on 2026-09-21,
|
|
one machine, one video, seven clients, each asked bare and again with the
|
|
`visitorData` a watch page hands out:
|
|
|
|
VPN on `IOS`, `ANDROID`, `WEB`, `MWEB`, `TVHTML5` and `ANDROID_VR` all
|
|
get the bot check, and `WEB_EMBEDDED_PLAYER` an error. No tracks
|
|
from any of them, and `visitorData` changes none of it.
|
|
`youtube-transcript-api` raises `RequestBlocked`.
|
|
VPN off `IOS` and `ANDROID` return tracks that fetch. The rest are no
|
|
use even here: `WEB`, `MWEB` and `WEB_EMBEDDED_PLAYER` are
|
|
unplayable without a PO token, and `TVHTML5` and `ANDROID_VR`
|
|
are answered with the bot check from a clean address too.
|
|
|
|
So there is no anonymous client that gets past the gate and none to add: the
|
|
fix is on the network. That case raises `Blocked`, a `TranscriptUnavailable` a
|
|
caller can tell apart, because "wait for the captions" is the wrong answer to
|
|
it and would be given for ever.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import html
|
|
import json
|
|
import logging
|
|
import re
|
|
import xml.etree.ElementTree as ET
|
|
from collections.abc import Iterator
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
from urllib.parse import parse_qs, urlsplit
|
|
|
|
import httpx
|
|
|
|
from desk.tools.http import session
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
INNERTUBE = "https://www.youtube.com/youtubei/v1/player?prettyPrint=false"
|
|
|
|
#: Only for the watch page, which is read for comments. A caption fetch
|
|
#: deliberately identifies as a mobile app instead — see CLIENTS.
|
|
BROWSER_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
|
"(KHTML, like Gecko) Chrome/124.0 Safari/537.36")
|
|
|
|
#: Tried in order. Each is (client context, the User-Agent that goes with it) —
|
|
#: they have to match, or the response comes back playable but captionless.
|
|
CLIENTS = (
|
|
(
|
|
{"clientName": "IOS", "clientVersion": "20.10.4",
|
|
"deviceModel": "iPhone16,2", "hl": "en"},
|
|
"com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X)",
|
|
),
|
|
(
|
|
{"clientName": "ANDROID", "clientVersion": "20.10.38",
|
|
"androidSdkVersion": 30, "hl": "en"},
|
|
"com.google.android.youtube/20.10.38 (Linux; U; Android 11) gzip",
|
|
),
|
|
)
|
|
|
|
_VIDEO_ID = re.compile(r"^[\w-]{11}$")
|
|
#: Every shape a YouTube link comes in, including the one Substack embeds
|
|
#: (`<div class="youtube-wrap" id="youtube2-VIDEOID">`).
|
|
_IN_TEXT = re.compile(
|
|
r"(?:youtu\.be/|youtube\.com/(?:watch\?(?:[^\"'\s]*&)?v=|embed/|live/|shorts/)"
|
|
r"|youtube2-)([\w-]{11})"
|
|
)
|
|
|
|
|
|
class TranscriptUnavailable(Exception):
|
|
"""No captions to be had — yet, or at all.
|
|
|
|
Raised rather than returning `[]` because the two cases an ingest has to
|
|
tell apart both look like an empty list: a video with no captions, and a
|
|
video whose captions are still being generated. The message says which,
|
|
and the caller decides whether to retry on the next pull.
|
|
"""
|
|
|
|
|
|
class Blocked(TranscriptUnavailable):
|
|
"""YouTube refused this machine, whatever the video has.
|
|
|
|
A subclass, so a caller that only wants "no captions, carry on" — a link
|
|
added by hand, a repair — is unchanged. A caller that waits for captions
|
|
has to catch this first: a refusal is not lifted by waiting, it answers
|
|
every video the same way, and retrying it quietly is how a show stores
|
|
nothing for a fortnight with nobody told. See `_refused`.
|
|
"""
|
|
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Cue:
|
|
"""One caption line: when it starts, how long it lasts, what was said."""
|
|
|
|
start: float
|
|
duration: float
|
|
text: str
|
|
|
|
@property
|
|
def end(self) -> float:
|
|
return self.start + self.duration
|
|
|
|
|
|
def video_id(value: str) -> str:
|
|
"""The 11-character id, from a bare id, any YouTube URL, or a blob of HTML.
|
|
|
|
Deliberately tolerant about its input: the id arrives from three different
|
|
places in this pipeline — typed by hand, in a feed entry's link, and buried
|
|
in the HTML of a Substack post — and each of them would otherwise need its
|
|
own extraction.
|
|
"""
|
|
value = (value or "").strip()
|
|
if _VIDEO_ID.match(value):
|
|
return value
|
|
if match := _IN_TEXT.search(value):
|
|
return match.group(1)
|
|
# A URL shape the pattern above doesn't cover, but with ?v= in the query.
|
|
found = parse_qs(urlsplit(value).query).get("v")
|
|
if found and _VIDEO_ID.match(found[0]):
|
|
return found[0]
|
|
raise ValueError(f"no YouTube video id in {value[:120]!r}")
|
|
|
|
|
|
def transcript(video: str, *, language: str = "en",
|
|
client: httpx.Client | None = None) -> list[Cue]:
|
|
"""Every caption cue for one video, in order.
|
|
|
|
`language` is matched as a prefix, so "en" accepts "en-US" — which matters,
|
|
because a channel that uploads a real transcript files it under a regional
|
|
code while the auto-generated one is plain "en".
|
|
|
|
Two backends, tried in that order. `youtube-transcript-api` first — it is
|
|
a dependency, and it is somebody's full-time job to keep up with YouTube's
|
|
changes. The built-in fetcher below when the library can't get the track,
|
|
so one broken path doesn't cost the transcript.
|
|
"""
|
|
cues = _via_library(video, language)
|
|
if cues is not None:
|
|
return cues
|
|
return _via_innertube(video, language, client)
|
|
|
|
|
|
#: Seconds one `requests` call inside `youtube-transcript-api` gets. Roughly
|
|
#: what `desk/tools/http.py` gives an httpx call, for the same reason.
|
|
LIBRARY_TIMEOUT = 30
|
|
|
|
|
|
def _timed_client() -> Any:
|
|
"""A `requests` session with a deadline on it, for a library that owns its HTTP.
|
|
|
|
`youtube-transcript-api` does its own fetching, on `requests`, and a
|
|
`requests` call with no `timeout=` waits on a socket read for ever. What
|
|
that costs is not a slow transcript. The work runs in a worker thread;
|
|
cancellation is cooperative, so a thread parked in a read is one the Kill
|
|
button cannot reach, and a restart re-queues the job to park on the same
|
|
video. The library takes a session of ours through `http_client`, which is
|
|
its own supported way of being told this; a `requests.Session` has no
|
|
timeout of its own to set, so the default goes on one method down.
|
|
|
|
**This is why there is no `socket.setdefaulttimeout` anywhere.** A
|
|
process-wide default is the tempting one-liner for library code that hides
|
|
its HTTP, and it is the wrong trade here: it would be inherited by every
|
|
socket the process opens, uvicorn's listener and each websocket included,
|
|
to buy a deadline for call sites that can each be told one directly. There
|
|
were two. The other was feedparser, and it is handled the same way — by
|
|
fetching where a timeout can be set; see `desk/sources/rss.py`.
|
|
|
|
Typed as `Any` because `requests` is only imported inside the guard below.
|
|
It is a `requests.Session`.
|
|
"""
|
|
import requests
|
|
|
|
class Timed(requests.Session):
|
|
def request(self, *args: Any, **kwargs: Any) -> Any:
|
|
kwargs.setdefault("timeout", LIBRARY_TIMEOUT)
|
|
return super().request(*args, **kwargs)
|
|
|
|
return Timed()
|
|
|
|
|
|
def _via_library(video: str, language: str) -> list[Cue] | None:
|
|
"""`youtube-transcript-api`. None means "not this way".
|
|
|
|
Every failure is None rather than an exception, including a video with no
|
|
captions at all: the fallback below raises the error that gets reported, so
|
|
there is one message for "no transcript" instead of two that depend on
|
|
which packages happen to be present.
|
|
"""
|
|
try:
|
|
from youtube_transcript_api import YouTubeTranscriptApi
|
|
|
|
client = _timed_client()
|
|
except ImportError:
|
|
return None
|
|
|
|
try:
|
|
tracks = YouTubeTranscriptApi(http_client=client).list(video)
|
|
wanted = [language, f"{language}-US", f"{language}-GB"]
|
|
try:
|
|
track = tracks.find_manually_created_transcript(wanted)
|
|
except Exception:
|
|
track = tracks.find_generated_transcript(wanted)
|
|
fetched = track.fetch()
|
|
except Exception as exc:
|
|
log.debug("youtube-transcript-api could not fetch %s: %s", video, exc)
|
|
return None
|
|
|
|
cues = [
|
|
Cue(start=c.start, duration=c.duration, text=clean)
|
|
for c in fetched
|
|
if (clean := _clean(c.text))
|
|
]
|
|
log.info("%s: %d cues via youtube-transcript-api", video, len(cues))
|
|
return cues or None
|
|
|
|
|
|
def _via_innertube(video: str, language: str,
|
|
client: httpx.Client | None) -> list[Cue]:
|
|
with session(client) as http:
|
|
player = _player(video, http)
|
|
track = _pick(_tracks(player), language)
|
|
if track is None:
|
|
raise TranscriptUnavailable(
|
|
f"{video}: no {language} captions. "
|
|
"Auto-captions can take a few hours to appear on a long upload."
|
|
)
|
|
# `&fmt=json3` is a request, not a promise: the ANDROID client answers
|
|
# it with SRV3 anyway. Both are parsed, so it doesn't matter which
|
|
# arrives — see `_cues`.
|
|
response = http.get(track["baseUrl"] + "&fmt=json3",
|
|
headers={"User-Agent": player.get("_caption_agent", BROWSER_UA)})
|
|
if _throttled(response):
|
|
raise Blocked(_refusal(video, [f"captions: HTTP {response.status_code}"]))
|
|
response.raise_for_status()
|
|
body = response.text
|
|
if not body.strip():
|
|
raise TranscriptUnavailable(
|
|
f"{video}: caption track fetched empty. YouTube does this when "
|
|
"the request doesn't look like the client that asked for it."
|
|
)
|
|
cues = _cues(body)
|
|
log.info("%s: %d cues, %.0f minutes", video, len(cues),
|
|
(cues[-1].end / 60) if cues else 0)
|
|
return cues
|
|
|
|
|
|
def details(video: str, *, client: httpx.Client | None = None) -> dict[str, Any]:
|
|
"""Title, author, length and description for a video.
|
|
|
|
The description is here because for a show that publishes no newsletter
|
|
it is the only place the chapter marks live — `marks.parse` reads it as
|
|
plain text. Everything else is the metadata an episode needs when YouTube
|
|
is the only source there is.
|
|
"""
|
|
with session(client) as http:
|
|
player = _player(video, http, require_captions=False)
|
|
video_details = player.get("videoDetails", {})
|
|
return {
|
|
"title": video_details.get("title", ""),
|
|
"author": video_details.get("author", ""),
|
|
"duration": float(video_details.get("lengthSeconds") or 0),
|
|
"description": video_details.get("shortDescription", ""),
|
|
}
|
|
|
|
|
|
def comments(video: str, *, limit: int = 100,
|
|
client: httpx.Client | None = None) -> list[str]:
|
|
"""Top-level comments, newest-and-most-liked first, as plain text.
|
|
|
|
Here for one reason: when a show does not publish an outline, somebody in
|
|
the comments usually has. A viewer typing out "0:00 intro / 3:32 Gemini /
|
|
13:11 Grok" is doing the same work the show notes would have done, and
|
|
YouTube itself treats such a comment as authoritative enough to draw
|
|
chapter markers from it. `marks.parse` reads these exactly as it reads a
|
|
description — see `marks.best`.
|
|
|
|
Returns `[]` rather than raising when comments are off, or when YouTube
|
|
changes the shape again. A missing fallback is not worth failing an ingest
|
|
that already has a transcript.
|
|
"""
|
|
try:
|
|
with session(client) as http:
|
|
page = http.get(f"https://www.youtube.com/watch?v={video}",
|
|
headers={"User-Agent": BROWSER_UA,
|
|
"Accept-Language": "en-US,en"}).text
|
|
context = _json_after(page, '"INNERTUBE_CONTEXT":')
|
|
initial = _json_after(page, "ytInitialData = ")
|
|
if not context or not initial:
|
|
return []
|
|
out: list[str] = []
|
|
# Several continuations live in the page (the related rail, the
|
|
# description); only one of them is the comment feed, and which
|
|
# index that is moves around. Trying them in order and keeping the
|
|
# first that yields comments is cheaper than pinning a path
|
|
# through a structure Google reshuffles.
|
|
for token in _continuations(initial)[:4]:
|
|
response = http.post(
|
|
"https://www.youtube.com/youtubei/v1/next?prettyPrint=false",
|
|
json={"context": context, "continuation": token},
|
|
headers={"Content-Type": "application/json",
|
|
"User-Agent": BROWSER_UA},
|
|
)
|
|
out = _comment_texts(response.json())
|
|
if out:
|
|
break
|
|
except Exception as exc:
|
|
log.debug("could not read comments for %s: %s", video, exc)
|
|
return []
|
|
log.info("%s: %d comments read", video, len(out))
|
|
return out[:limit]
|
|
|
|
|
|
def text_between(cues: list[Cue], start: float, end: float | None = None) -> str:
|
|
"""The words spoken in a span, as one paragraph.
|
|
|
|
A cue counts as inside the span when it *starts* inside it. Overlap would
|
|
be the other defensible rule, but it double-counts the sentence that
|
|
straddles a boundary — and these boundaries come from chapter marks a human
|
|
typed, so a sentence of drift at the seam is already there and doesn't need
|
|
doubling.
|
|
"""
|
|
return " ".join(
|
|
cue.text for cue in cues
|
|
if cue.start >= start and (end is None or cue.start < end)
|
|
).strip()
|
|
|
|
|
|
# ------------------------------------------------- internals: the watch page
|
|
|
|
def _json_after(page: str, marker: str) -> dict[str, Any] | None:
|
|
"""The JSON object that follows `marker` in a watch page.
|
|
|
|
Brace-matched rather than matched with a regex. The two blobs this reads —
|
|
`ytInitialData` and `INNERTUBE_CONTEXT` — are hundreds of kilobytes of
|
|
nested JSON containing every bracket character inside string values, and a
|
|
non-greedy regex stops at the first `}` inside a caption or a title. This
|
|
walks the string properly, which is about fifteen lines and always right.
|
|
"""
|
|
start = page.find(marker)
|
|
if start < 0:
|
|
return None
|
|
start = page.find("{", start + len(marker))
|
|
if start < 0:
|
|
return None
|
|
|
|
depth, in_string, escaped = 0, False, False
|
|
for index in range(start, len(page)):
|
|
char = page[index]
|
|
if in_string:
|
|
if escaped:
|
|
escaped = False
|
|
elif char == "\\":
|
|
escaped = True
|
|
elif char == '"':
|
|
in_string = False
|
|
continue
|
|
if char == '"':
|
|
in_string = True
|
|
elif char == "{":
|
|
depth += 1
|
|
elif char == "}":
|
|
depth -= 1
|
|
if depth == 0:
|
|
try:
|
|
return json.loads(page[start:index + 1])
|
|
except ValueError:
|
|
return None
|
|
return None
|
|
|
|
|
|
def _walk(node: Any, key: str) -> Iterator[Any]:
|
|
"""Every value stored under `key`, anywhere in a nested structure."""
|
|
if isinstance(node, dict):
|
|
if key in node:
|
|
yield node[key]
|
|
for value in node.values():
|
|
yield from _walk(value, key)
|
|
elif isinstance(node, list):
|
|
for value in node:
|
|
yield from _walk(value, key)
|
|
|
|
|
|
def _continuations(initial: dict[str, Any]) -> list[str]:
|
|
return [
|
|
token for renderer in _walk(initial, "continuationItemRenderer")
|
|
if (token := (renderer.get("continuationEndpoint", {})
|
|
.get("continuationCommand", {}) or {}).get("token"))
|
|
]
|
|
|
|
|
|
def _comment_texts(response: dict[str, Any]) -> list[str]:
|
|
"""Comment bodies out of a `next` response.
|
|
|
|
Comments arrive as `commentEntityPayload` entities in a flat batch beside
|
|
the renderers that position them, so reading them needs no understanding of
|
|
the layout at all — which is the part that keeps changing.
|
|
"""
|
|
out = []
|
|
for payload in _walk(response, "commentEntityPayload"):
|
|
text = (payload.get("properties", {}).get("content", {}) or {}).get("content", "")
|
|
if text:
|
|
out.append(text)
|
|
return out
|
|
|
|
|
|
# ------------------------------------------------------------------ internals
|
|
|
|
def _player(video: str, http: httpx.Client,
|
|
*, require_captions: bool = True) -> dict[str, Any]:
|
|
"""The player response, from whichever client answers with captions.
|
|
|
|
`require_captions` is what separates the two callers. A transcript wants
|
|
the first client that actually offers caption tracks; metadata wants the
|
|
first that answers at all, and would otherwise fail on a video that plays
|
|
fine and simply has no captions yet.
|
|
|
|
Every client's answer is kept for the error, not only the last one's: the
|
|
two fail differently, and which of them said what is the first thing
|
|
anyone asks when this breaks.
|
|
"""
|
|
said: list[str] = []
|
|
refused = False
|
|
for context, agent in CLIENTS:
|
|
name = context["clientName"]
|
|
payload = {"context": {"client": context}, "videoId": video,
|
|
"contentCheckOk": True, "racyCheckOk": True}
|
|
try:
|
|
response = http.post(
|
|
INNERTUBE, json=payload,
|
|
headers={"Content-Type": "application/json", "User-Agent": agent},
|
|
)
|
|
if _throttled(response):
|
|
refused = True
|
|
said.append(f"{name}: HTTP {response.status_code}")
|
|
continue
|
|
data = response.json()
|
|
except (httpx.HTTPError, ValueError) as exc:
|
|
said.append(f"{name}: {exc}")
|
|
continue
|
|
playability = data.get("playabilityStatus", {})
|
|
status = playability.get("status", "?")
|
|
if _tracks(data) or (not require_captions and status == "OK"):
|
|
# Caption URLs are bound to the client that obtained the player.
|
|
data["_caption_agent"] = agent
|
|
return data
|
|
refused = refused or _refused(playability)
|
|
reason = playability.get("reason") or ""
|
|
said.append(f"{name}: {status}" + (f" ({reason})" if reason else "")
|
|
+ ", no caption tracks")
|
|
log.debug("innertube %s", said[-1])
|
|
if refused:
|
|
raise Blocked(_refusal(video, said))
|
|
raise TranscriptUnavailable(
|
|
f"{video}: {'; '.join(said) or 'no client returned captions'}")
|
|
|
|
|
|
def _refused(playability: dict[str, Any]) -> bool:
|
|
"""Whether a player response is YouTube declining to serve this machine.
|
|
|
|
The status alone, not the reason beside it. The bot check reads "Sign in
|
|
to confirm you're not a bot" today, and a match on that sentence would go
|
|
back to reporting a refusal as captions not being ready on the day it is
|
|
reworded — silently, which is the failure this exists to end. The status
|
|
is the stable half, and it is sufficient: nothing asked anonymously is
|
|
ever let past `LOGIN_REQUIRED` by being asked again later.
|
|
|
|
That also takes in a private or age-gated video, where the refusal is
|
|
about the video rather than the address. It is still not a wait, and the
|
|
reason is quoted in the error for whoever reads it.
|
|
"""
|
|
return playability.get("status") == "LOGIN_REQUIRED"
|
|
|
|
|
|
def _throttled(response: httpx.Response) -> bool:
|
|
"""The same refusal, made in HTTP: a 403, a 429, or Google's `/sorry/`
|
|
interstitial, which arrives as a redirect to a captcha page."""
|
|
return response.status_code in (403, 429) or "/sorry/" in response.url.path
|
|
|
|
|
|
def _refusal(video: str, said: list[str]) -> str:
|
|
"""`Blocked`'s sentence. "YouTube refused" is what `desk/notices.py` reads
|
|
to tell this from a feed that failed, so the wording is load-bearing."""
|
|
return (f"{video}: YouTube refused this address and wants a sign-in — "
|
|
+ "; ".join(said))
|
|
|
|
|
|
def _tracks(player: dict[str, Any]) -> list[dict[str, Any]]:
|
|
return (player.get("captions", {})
|
|
.get("playerCaptionsTracklistRenderer", {})
|
|
.get("captionTracks", []) or [])
|
|
|
|
|
|
def _pick(tracks: list[dict[str, Any]], language: str) -> dict[str, Any] | None:
|
|
"""Best track for a language: a real transcript over an auto-generated one.
|
|
|
|
A channel that uploads its own captions has punctuation, speaker turns and
|
|
correctly spelled names; ASR has none of those and spells "Qwen" six ways.
|
|
Worth preferring, and never worth requiring — most shows only have ASR.
|
|
"""
|
|
wanted = [t for t in tracks
|
|
if (t.get("languageCode") or "").lower().startswith(language.lower())]
|
|
wanted.sort(key=lambda t: t.get("kind") == "asr")
|
|
return wanted[0] if wanted else None
|
|
|
|
|
|
def _cues(body: str) -> list[Cue]:
|
|
"""Parse whichever of YouTube's three caption formats turned up."""
|
|
body = body.strip()
|
|
if body.startswith("{"):
|
|
return _from_json3(json.loads(body))
|
|
root = ET.fromstring(body)
|
|
# SRV1 is `transcript > text[start,dur]`; SRV3 is `timedtext > body > p[t,d]`.
|
|
if root.tag == "transcript":
|
|
return _from_srv1(root)
|
|
return _from_srv3(root)
|
|
|
|
|
|
def _from_json3(data: dict[str, Any]) -> list[Cue]:
|
|
cues = []
|
|
for event in data.get("events", []):
|
|
text = "".join(seg.get("utf8", "") for seg in event.get("segs", []))
|
|
clean = _clean(text)
|
|
if not clean:
|
|
continue
|
|
cues.append(Cue(
|
|
start=event.get("tStartMs", 0) / 1000,
|
|
duration=event.get("dDurationMs", 0) / 1000,
|
|
text=clean,
|
|
))
|
|
return cues
|
|
|
|
|
|
def _from_srv1(root: ET.Element) -> list[Cue]:
|
|
cues = []
|
|
for node in root.iter("text"):
|
|
clean = _clean(node.text or "")
|
|
if not clean:
|
|
continue
|
|
cues.append(Cue(
|
|
start=float(node.get("start", 0)),
|
|
duration=float(node.get("dur", 0)),
|
|
text=clean,
|
|
))
|
|
return cues
|
|
|
|
|
|
def _from_srv3(root: ET.Element) -> list[Cue]:
|
|
cues = []
|
|
for node in root.iter("p"):
|
|
clean = _clean("".join(node.itertext()))
|
|
if not clean:
|
|
continue
|
|
cues.append(Cue(
|
|
start=int(node.get("t", 0)) / 1000,
|
|
duration=int(node.get("d", 0)) / 1000,
|
|
text=clean,
|
|
))
|
|
return cues
|
|
|
|
|
|
def _clean(text: str) -> str:
|
|
"""Entities out, whitespace collapsed, sound cues dropped.
|
|
|
|
Unescaped twice on purpose: SRV1 sends `&#39;` — an entity for an
|
|
entity — because the caption text was escaped once when it was stored and
|
|
once again on the way into the XML.
|
|
"""
|
|
text = html.unescape(html.unescape(text or ""))
|
|
text = re.sub(r"\[(?:Music|Applause|Laughter)\]", " ", text)
|
|
return re.sub(r"\s+", " ", text).strip()
|