The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
266 lines
10 KiB
Python
266 lines
10 KiB
Python
"""Chapter marks: a video's own table of contents, wherever it was written.
|
|
|
|
The important discovery about this kind of video is that **the segmentation
|
|
already exists and somebody else did it**. Last Week in AI ships every episode
|
|
with a timestamped outline — a rubric, then one line per story, each with the
|
|
time it starts and a link to the article being discussed:
|
|
|
|
Tools & Apps
|
|
(00:03:32) Google announces Gemini 3.7 Flash ... - Ars Technica
|
|
(00:13:11) SpaceXAI Releases Grok 4.6 ... - MarkTechPost
|
|
|
|
That is a better set of boundaries than any topic-segmentation model would
|
|
produce, for nothing, with the source links attached. So this module parses it
|
|
rather than inferring it. An LLM is still wanted downstream — to summarise what
|
|
was *said* in each stretch — but not to work out where one stretch ends.
|
|
|
|
**Three places to find that outline**, in descending order of usefulness, all
|
|
parsed by the same function because they are the same convention written in
|
|
different markup:
|
|
|
|
1. **The show's own page or feed body** — HTML, so the links are real
|
|
`<a href>` and each chapter arrives with the article it is about. Only a
|
|
show with a website has this, which is why the podcast source hands it in
|
|
rather than this module going to look.
|
|
2. **The video description** — plain text. Usually the same outline with the
|
|
links flattened away.
|
|
3. **The comments** — see `in_comments`. When a show publishes no marks at
|
|
all, a listener has often typed some out; YouTube itself reads those to
|
|
draw its chapter markers.
|
|
|
|
`best` takes them in that order and returns the first that is real. Nothing
|
|
here is specific to one show beyond the shape "timestamp, then a title,
|
|
optionally under a heading", which is the near-universal convention.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import re
|
|
from collections.abc import Iterable
|
|
from dataclasses import dataclass, field
|
|
from html.parser import HTMLParser
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: `(00:03:32)`, `00:03:32`, `3:32` — at the start of the line, which is what
|
|
#: makes it a chapter mark rather than a time mentioned in a sentence.
|
|
TIMESTAMP = re.compile(
|
|
# The character class ends with hyphen, en dash and em dash on purpose:
|
|
# a show separates the time from the title with whichever of the three its
|
|
# editor's keyboard produced, and means the same thing by all of them.
|
|
r"^\(?\s*(\d{1,2}):([0-5]\d)(?::([0-5]\d))?\s*\)?[\s\-–—:]*" # noqa: RUF001
|
|
)
|
|
_URL = re.compile(r"https?://[^\s<>\"')\]]+")
|
|
#: Tags that end a line of the outline. Everything else is inline and stays
|
|
#: on the line it was found in, which is what keeps `<span>(00:03:32) </span>`
|
|
#: attached to the `<a>` that follows it.
|
|
_BLOCKS = frozenset({
|
|
"li", "p", "br", "div", "tr", "ul", "ol", "h1", "h2", "h3", "h4", "blockquote",
|
|
})
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Link:
|
|
"""One article the hosts were talking about."""
|
|
|
|
url: str
|
|
title: str = ""
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class Chapter:
|
|
"""One entry in the outline: where a story starts and what it is about."""
|
|
|
|
start: int # seconds into the episode
|
|
title: str
|
|
heading: str = "" # the rubric it sits under
|
|
links: list[Link] = field(default_factory=list)
|
|
|
|
@property
|
|
def url(self) -> str:
|
|
"""The article being discussed, if the notes linked one."""
|
|
return self.links[0].url if self.links else ""
|
|
|
|
|
|
def parse(body: str) -> list[Chapter]:
|
|
"""Every chapter mark in a set of show notes, in the order they play.
|
|
|
|
Takes HTML or plain text and works out which it was given. Returns `[]`
|
|
for notes with no outline in them at all, which is a real answer — plenty
|
|
of shows don't write one, and the caller falls back to treating the episode
|
|
as a single undivided item rather than guessing at boundaries.
|
|
"""
|
|
lines = _html_lines(body) if "<" in body and ">" in body else _text_lines(body)
|
|
|
|
chapters: list[Chapter] = []
|
|
heading = ""
|
|
for text, links in lines:
|
|
match = TIMESTAMP.match(text)
|
|
if match is None:
|
|
# A heading is only a heading once the outline has started;
|
|
# before that every line is preamble — the blurb, the sponsors,
|
|
# the "Timestamps:" label itself.
|
|
if chapters and _is_heading(text, links):
|
|
heading = text.rstrip(":")
|
|
continue
|
|
title = text[match.end():].strip(" -–—|") # noqa: RUF001 — as above
|
|
# Prefer the anchor's own text: a line that links two follow-ups reads
|
|
# "First story | The Verge + Second story", and the first link's text
|
|
# is the story, where the whole line is the story plus its footnote.
|
|
if links and links[0].title:
|
|
title = links[0].title
|
|
chapters.append(Chapter(
|
|
start=_seconds(match), title=title, heading=heading, links=links,
|
|
))
|
|
|
|
if not chapters:
|
|
log.debug("no chapter marks found in %d chars of notes", len(body))
|
|
return chapters
|
|
|
|
|
|
def best(*candidates: str, minimum: int = 3) -> list[Chapter]:
|
|
"""The first candidate that yields a real outline.
|
|
|
|
Chapter marks come from up to three places and they are not equally good,
|
|
so the order they are passed in is the order of preference — for a podcast
|
|
that is: the show's own notes as HTML (links intact), then the YouTube
|
|
description (same outline, links flattened away), then the comments.
|
|
|
|
`minimum` is what stops a stray "at 3:20 he says" in a blurb becoming a
|
|
one-chapter outline that suppresses the good source behind it. Three marks
|
|
is the point where a line of text stops being a coincidence.
|
|
"""
|
|
for text in candidates:
|
|
found = parse(text)
|
|
if len(found) >= minimum:
|
|
return found
|
|
return []
|
|
|
|
|
|
def in_comments(comments: Iterable[str], minimum: int = 3) -> list[Chapter]:
|
|
"""The outline somebody typed into the comments, if anybody did.
|
|
|
|
Read as the fallback for shows that publish no chapter marks of their own,
|
|
where it is common for a listener to write one out — YouTube itself treats
|
|
such a comment as good enough to draw chapter markers from.
|
|
|
|
Each comment is parsed on its own rather than the lot being concatenated.
|
|
Joined together, an outline comment and thirty unrelated replies produce
|
|
one bogus outline interleaved with chatter; separately, the outline is the
|
|
comment with the most marks in it and the chatter scores zero.
|
|
"""
|
|
outlines = [parse(comment) for comment in comments]
|
|
outlines = [found for found in outlines if len(found) >= minimum]
|
|
if not outlines:
|
|
return []
|
|
richest = max(outlines, key=len)
|
|
log.info("took %d chapter marks from a comment", len(richest))
|
|
return richest
|
|
|
|
|
|
def hhmmss(seconds: float) -> str:
|
|
"""3932 -> '1:05:32'. What a timecode looks like to a reader."""
|
|
total = int(seconds)
|
|
hours, rest = divmod(total, 3600)
|
|
minutes, secs = divmod(rest, 60)
|
|
return f"{hours}:{minutes:02d}:{secs:02d}" if hours else f"{minutes}:{secs:02d}"
|
|
|
|
|
|
# ------------------------------------------------------------------ internals
|
|
|
|
def _seconds(match: re.Match[str]) -> int:
|
|
a, b, c = match.group(1), match.group(2), match.group(3)
|
|
if c is None: # mm:ss
|
|
return int(a) * 60 + int(b)
|
|
return int(a) * 3600 + int(b) * 60 + int(c)
|
|
|
|
|
|
def _is_heading(text: str, links: list[Link]) -> bool:
|
|
"""A rubric like 'Policy & Safety', not a sentence and not a story.
|
|
|
|
Deliberately strict. Anything that gets through becomes the heading for
|
|
every chapter after it, so a false positive mislabels the rest of the
|
|
episode — where a false negative just leaves a few items unfiled.
|
|
"""
|
|
return (
|
|
not links
|
|
and len(text) <= 60
|
|
and len(text.split()) <= 6
|
|
and not text.endswith((".", "!", "?"))
|
|
)
|
|
|
|
|
|
def _text_lines(body: str) -> list[tuple[str, list[Link]]]:
|
|
"""Plain-text notes — a YouTube description. Bare URLs become links."""
|
|
lines = []
|
|
for raw in body.splitlines():
|
|
line = raw.strip()
|
|
if not line:
|
|
continue
|
|
links = [Link(url=u) for u in _URL.findall(line)]
|
|
lines.append((_URL.sub("", line).strip(), links))
|
|
return lines
|
|
|
|
|
|
def _html_lines(body: str) -> list[tuple[str, list[Link]]]:
|
|
parser = _Outline()
|
|
parser.feed(body)
|
|
parser.close()
|
|
return parser.lines
|
|
|
|
|
|
class _Outline(HTMLParser):
|
|
"""HTML to (line, links) pairs, without pulling in a parser library.
|
|
|
|
A full HTML tree would be nicer and is not worth a dependency for this:
|
|
the only structure that matters is where the line breaks fall and which
|
|
anchors landed on which line. `convert_charrefs` (on by default) means
|
|
entities arrive already decoded, so `Tools & Apps` is `Tools & Apps`
|
|
by the time it reaches `handle_data`.
|
|
"""
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__()
|
|
self.lines: list[tuple[str, list[Link]]] = []
|
|
self._text: list[str] = []
|
|
self._links: list[Link] = []
|
|
self._href = ""
|
|
self._anchor: list[str] = []
|
|
|
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
if tag in _BLOCKS:
|
|
self._flush()
|
|
elif tag == "a":
|
|
self._href = dict(attrs).get("href") or ""
|
|
self._anchor = []
|
|
|
|
def handle_endtag(self, tag: str) -> None:
|
|
if tag in _BLOCKS:
|
|
self._flush()
|
|
elif tag == "a":
|
|
if self._href:
|
|
self._links.append(
|
|
Link(url=self._href, title=_squash("".join(self._anchor)))
|
|
)
|
|
self._href, self._anchor = "", []
|
|
|
|
def handle_data(self, data: str) -> None:
|
|
self._text.append(data)
|
|
if self._href:
|
|
self._anchor.append(data)
|
|
|
|
def _flush(self) -> None:
|
|
text = _squash("".join(self._text))
|
|
if text or self._links:
|
|
self.lines.append((text, self._links))
|
|
self._text, self._links = [], []
|
|
|
|
def close(self) -> None:
|
|
super().close()
|
|
self._flush()
|
|
|
|
|
|
def _squash(text: str) -> str:
|
|
return re.sub(r"\s+", " ", text or "").strip()
|