"""Chapter marks: a video's own table of contents, wherever it was written. The important discovery about this kind of video is that **the segmentation already exists and somebody else did it**. Last Week in AI ships every episode with a timestamped outline — a rubric, then one line per story, each with the time it starts and a link to the article being discussed: Tools & Apps (00:03:32) Google announces Gemini 3.7 Flash ... - Ars Technica (00:13:11) SpaceXAI Releases Grok 4.6 ... - MarkTechPost That is a better set of boundaries than any topic-segmentation model would produce, for nothing, with the source links attached. So this module parses it rather than inferring it. An LLM is still wanted downstream — to summarise what was *said* in each stretch — but not to work out where one stretch ends. **Three places to find that outline**, in descending order of usefulness, all parsed by the same function because they are the same convention written in different markup: 1. **The show's own page or feed body** — HTML, so the links are real `` and each chapter arrives with the article it is about. Only a show with a website has this, which is why the podcast source hands it in rather than this module going to look. 2. **The video description** — plain text. Usually the same outline with the links flattened away. 3. **The comments** — see `in_comments`. When a show publishes no marks at all, a listener has often typed some out; YouTube itself reads those to draw its chapter markers. `best` takes them in that order and returns the first that is real. Nothing here is specific to one show beyond the shape "timestamp, then a title, optionally under a heading", which is the near-universal convention. """ from __future__ import annotations import logging import re from collections.abc import Iterable from dataclasses import dataclass, field from html.parser import HTMLParser log = logging.getLogger(__name__) #: `(00:03:32)`, `00:03:32`, `3:32` — at the start of the line, which is what #: makes it a chapter mark rather than a time mentioned in a sentence. TIMESTAMP = re.compile( # The character class ends with hyphen, en dash and em dash on purpose: # a show separates the time from the title with whichever of the three its # editor's keyboard produced, and means the same thing by all of them. r"^\(?\s*(\d{1,2}):([0-5]\d)(?::([0-5]\d))?\s*\)?[\s\-–—:]*" # noqa: RUF001 ) _URL = re.compile(r"https?://[^\s<>\"')\]]+") #: Tags that end a line of the outline. Everything else is inline and stays #: on the line it was found in, which is what keeps `(00:03:32) ` #: attached to the `` that follows it. _BLOCKS = frozenset({ "li", "p", "br", "div", "tr", "ul", "ol", "h1", "h2", "h3", "h4", "blockquote", }) @dataclass(frozen=True, slots=True) class Link: """One article the hosts were talking about.""" url: str title: str = "" @dataclass(slots=True) class Chapter: """One entry in the outline: where a story starts and what it is about.""" start: int # seconds into the episode title: str heading: str = "" # the rubric it sits under links: list[Link] = field(default_factory=list) @property def url(self) -> str: """The article being discussed, if the notes linked one.""" return self.links[0].url if self.links else "" def parse(body: str) -> list[Chapter]: """Every chapter mark in a set of show notes, in the order they play. Takes HTML or plain text and works out which it was given. Returns `[]` for notes with no outline in them at all, which is a real answer — plenty of shows don't write one, and the caller falls back to treating the episode as a single undivided item rather than guessing at boundaries. """ lines = _html_lines(body) if "<" in body and ">" in body else _text_lines(body) chapters: list[Chapter] = [] heading = "" for text, links in lines: match = TIMESTAMP.match(text) if match is None: # A heading is only a heading once the outline has started; # before that every line is preamble — the blurb, the sponsors, # the "Timestamps:" label itself. if chapters and _is_heading(text, links): heading = text.rstrip(":") continue title = text[match.end():].strip(" -–—|") # noqa: RUF001 — as above # Prefer the anchor's own text: a line that links two follow-ups reads # "First story | The Verge + Second story", and the first link's text # is the story, where the whole line is the story plus its footnote. if links and links[0].title: title = links[0].title chapters.append(Chapter( start=_seconds(match), title=title, heading=heading, links=links, )) if not chapters: log.debug("no chapter marks found in %d chars of notes", len(body)) return chapters def best(*candidates: str, minimum: int = 3) -> list[Chapter]: """The first candidate that yields a real outline. Chapter marks come from up to three places and they are not equally good, so the order they are passed in is the order of preference — for a podcast that is: the show's own notes as HTML (links intact), then the YouTube description (same outline, links flattened away), then the comments. `minimum` is what stops a stray "at 3:20 he says" in a blurb becoming a one-chapter outline that suppresses the good source behind it. Three marks is the point where a line of text stops being a coincidence. """ for text in candidates: found = parse(text) if len(found) >= minimum: return found return [] def in_comments(comments: Iterable[str], minimum: int = 3) -> list[Chapter]: """The outline somebody typed into the comments, if anybody did. Read as the fallback for shows that publish no chapter marks of their own, where it is common for a listener to write one out — YouTube itself treats such a comment as good enough to draw chapter markers from. Each comment is parsed on its own rather than the lot being concatenated. Joined together, an outline comment and thirty unrelated replies produce one bogus outline interleaved with chatter; separately, the outline is the comment with the most marks in it and the chatter scores zero. """ outlines = [parse(comment) for comment in comments] outlines = [found for found in outlines if len(found) >= minimum] if not outlines: return [] richest = max(outlines, key=len) log.info("took %d chapter marks from a comment", len(richest)) return richest def hhmmss(seconds: float) -> str: """3932 -> '1:05:32'. What a timecode looks like to a reader.""" total = int(seconds) hours, rest = divmod(total, 3600) minutes, secs = divmod(rest, 60) return f"{hours}:{minutes:02d}:{secs:02d}" if hours else f"{minutes}:{secs:02d}" # ------------------------------------------------------------------ internals def _seconds(match: re.Match[str]) -> int: a, b, c = match.group(1), match.group(2), match.group(3) if c is None: # mm:ss return int(a) * 60 + int(b) return int(a) * 3600 + int(b) * 60 + int(c) def _is_heading(text: str, links: list[Link]) -> bool: """A rubric like 'Policy & Safety', not a sentence and not a story. Deliberately strict. Anything that gets through becomes the heading for every chapter after it, so a false positive mislabels the rest of the episode — where a false negative just leaves a few items unfiled. """ return ( not links and len(text) <= 60 and len(text.split()) <= 6 and not text.endswith((".", "!", "?")) ) def _text_lines(body: str) -> list[tuple[str, list[Link]]]: """Plain-text notes — a YouTube description. Bare URLs become links.""" lines = [] for raw in body.splitlines(): line = raw.strip() if not line: continue links = [Link(url=u) for u in _URL.findall(line)] lines.append((_URL.sub("", line).strip(), links)) return lines def _html_lines(body: str) -> list[tuple[str, list[Link]]]: parser = _Outline() parser.feed(body) parser.close() return parser.lines class _Outline(HTMLParser): """HTML to (line, links) pairs, without pulling in a parser library. A full HTML tree would be nicer and is not worth a dependency for this: the only structure that matters is where the line breaks fall and which anchors landed on which line. `convert_charrefs` (on by default) means entities arrive already decoded, so `Tools & Apps` is `Tools & Apps` by the time it reaches `handle_data`. """ def __init__(self) -> None: super().__init__() self.lines: list[tuple[str, list[Link]]] = [] self._text: list[str] = [] self._links: list[Link] = [] self._href = "" self._anchor: list[str] = [] def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: if tag in _BLOCKS: self._flush() elif tag == "a": self._href = dict(attrs).get("href") or "" self._anchor = [] def handle_endtag(self, tag: str) -> None: if tag in _BLOCKS: self._flush() elif tag == "a": if self._href: self._links.append( Link(url=self._href, title=_squash("".join(self._anchor))) ) self._href, self._anchor = "", [] def handle_data(self, data: str) -> None: self._text.append(data) if self._href: self._anchor.append(data) def _flush(self) -> None: text = _squash("".join(self._text)) if text or self._links: self.lines.append((text, self._links)) self._text, self._links = [], [] def close(self) -> None: super().close() self._flush() def _squash(text: str) -> str: return re.sub(r"\s+", " ", text or "").strip()