Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d666d0937
commit
248ea1636d
36 files changed
+8393
No files matched your search
@@ -0,0 +1,265 @@
|
||||
"""Chapter marks: a video's own table of contents, wherever it was written.
|
||||
|
||||
The important discovery about this kind of video is that **the segmentation
|
||||
already exists and somebody else did it**. Last Week in AI ships every episode
|
||||
with a timestamped outline — a rubric, then one line per story, each with the
|
||||
time it starts and a link to the article being discussed:
|
||||
|
||||
Tools & Apps
|
||||
(00:03:32) Google announces Gemini 3.7 Flash ... - Ars Technica
|
||||
(00:13:11) SpaceXAI Releases Grok 4.6 ... - MarkTechPost
|
||||
|
||||
That is a better set of boundaries than any topic-segmentation model would
|
||||
produce, for nothing, with the source links attached. So this module parses it
|
||||
rather than inferring it. An LLM is still wanted downstream — to summarise what
|
||||
was *said* in each stretch — but not to work out where one stretch ends.
|
||||
|
||||
**Three places to find that outline**, in descending order of usefulness, all
|
||||
parsed by the same function because they are the same convention written in
|
||||
different markup:
|
||||
|
||||
1. **The show's own page or feed body** — HTML, so the links are real
|
||||
`<a href>` and each chapter arrives with the article it is about. Only a
|
||||
show with a website has this, which is why the podcast source hands it in
|
||||
rather than this module going to look.
|
||||
2. **The video description** — plain text. Usually the same outline with the
|
||||
links flattened away.
|
||||
3. **The comments** — see `in_comments`. When a show publishes no marks at
|
||||
all, a listener has often typed some out; YouTube itself reads those to
|
||||
draw its chapter markers.
|
||||
|
||||
`best` takes them in that order and returns the first that is real. Nothing
|
||||
here is specific to one show beyond the shape "timestamp, then a title,
|
||||
optionally under a heading", which is the near-universal convention.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass, field
|
||||
from html.parser import HTMLParser
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
#: `(00:03:32)`, `00:03:32`, `3:32` — at the start of the line, which is what
|
||||
#: makes it a chapter mark rather than a time mentioned in a sentence.
|
||||
TIMESTAMP = re.compile(
|
||||
# The character class ends with hyphen, en dash and em dash on purpose:
|
||||
# a show separates the time from the title with whichever of the three its
|
||||
# editor's keyboard produced, and means the same thing by all of them.
|
||||
r"^\(?\s*(\d{1,2}):([0-5]\d)(?::([0-5]\d))?\s*\)?[\s\-–—:]*" # noqa: RUF001
|
||||
)
|
||||
_URL = re.compile(r"https?://[^\s<>\"')\]]+")
|
||||
#: Tags that end a line of the outline. Everything else is inline and stays
|
||||
#: on the line it was found in, which is what keeps `<span>(00:03:32) </span>`
|
||||
#: attached to the `<a>` that follows it.
|
||||
_BLOCKS = frozenset({
|
||||
"li", "p", "br", "div", "tr", "ul", "ol", "h1", "h2", "h3", "h4", "blockquote",
|
||||
})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Link:
|
||||
"""One article the hosts were talking about."""
|
||||
|
||||
url: str
|
||||
title: str = ""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Chapter:
|
||||
"""One entry in the outline: where a story starts and what it is about."""
|
||||
|
||||
start: int # seconds into the episode
|
||||
title: str
|
||||
heading: str = "" # the rubric it sits under
|
||||
links: list[Link] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
"""The article being discussed, if the notes linked one."""
|
||||
return self.links[0].url if self.links else ""
|
||||
|
||||
|
||||
def parse(body: str) -> list[Chapter]:
|
||||
"""Every chapter mark in a set of show notes, in the order they play.
|
||||
|
||||
Takes HTML or plain text and works out which it was given. Returns `[]`
|
||||
for notes with no outline in them at all, which is a real answer — plenty
|
||||
of shows don't write one, and the caller falls back to treating the episode
|
||||
as a single undivided item rather than guessing at boundaries.
|
||||
"""
|
||||
lines = _html_lines(body) if "<" in body and ">" in body else _text_lines(body)
|
||||
|
||||
chapters: list[Chapter] = []
|
||||
heading = ""
|
||||
for text, links in lines:
|
||||
match = TIMESTAMP.match(text)
|
||||
if match is None:
|
||||
# A heading is only a heading once the outline has started;
|
||||
# before that every line is preamble — the blurb, the sponsors,
|
||||
# the "Timestamps:" label itself.
|
||||
if chapters and _is_heading(text, links):
|
||||
heading = text.rstrip(":")
|
||||
continue
|
||||
title = text[match.end():].strip(" -–—|") # noqa: RUF001 — as above
|
||||
# Prefer the anchor's own text: a line that links two follow-ups reads
|
||||
# "First story | The Verge + Second story", and the first link's text
|
||||
# is the story, where the whole line is the story plus its footnote.
|
||||
if links and links[0].title:
|
||||
title = links[0].title
|
||||
chapters.append(Chapter(
|
||||
start=_seconds(match), title=title, heading=heading, links=links,
|
||||
))
|
||||
|
||||
if not chapters:
|
||||
log.debug("no chapter marks found in %d chars of notes", len(body))
|
||||
return chapters
|
||||
|
||||
|
||||
def best(*candidates: str, minimum: int = 3) -> list[Chapter]:
|
||||
"""The first candidate that yields a real outline.
|
||||
|
||||
Chapter marks come from up to three places and they are not equally good,
|
||||
so the order they are passed in is the order of preference — for a podcast
|
||||
that is: the show's own notes as HTML (links intact), then the YouTube
|
||||
description (same outline, links flattened away), then the comments.
|
||||
|
||||
`minimum` is what stops a stray "at 3:20 he says" in a blurb becoming a
|
||||
one-chapter outline that suppresses the good source behind it. Three marks
|
||||
is the point where a line of text stops being a coincidence.
|
||||
"""
|
||||
for text in candidates:
|
||||
found = parse(text)
|
||||
if len(found) >= minimum:
|
||||
return found
|
||||
return []
|
||||
|
||||
|
||||
def in_comments(comments: Iterable[str], minimum: int = 3) -> list[Chapter]:
|
||||
"""The outline somebody typed into the comments, if anybody did.
|
||||
|
||||
Read as the fallback for shows that publish no chapter marks of their own,
|
||||
where it is common for a listener to write one out — YouTube itself treats
|
||||
such a comment as good enough to draw chapter markers from.
|
||||
|
||||
Each comment is parsed on its own rather than the lot being concatenated.
|
||||
Joined together, an outline comment and thirty unrelated replies produce
|
||||
one bogus outline interleaved with chatter; separately, the outline is the
|
||||
comment with the most marks in it and the chatter scores zero.
|
||||
"""
|
||||
outlines = [parse(comment) for comment in comments]
|
||||
outlines = [found for found in outlines if len(found) >= minimum]
|
||||
if not outlines:
|
||||
return []
|
||||
richest = max(outlines, key=len)
|
||||
log.info("took %d chapter marks from a comment", len(richest))
|
||||
return richest
|
||||
|
||||
|
||||
def hhmmss(seconds: float) -> str:
|
||||
"""3932 -> '1:05:32'. What a timecode looks like to a reader."""
|
||||
total = int(seconds)
|
||||
hours, rest = divmod(total, 3600)
|
||||
minutes, secs = divmod(rest, 60)
|
||||
return f"{hours}:{minutes:02d}:{secs:02d}" if hours else f"{minutes}:{secs:02d}"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ internals
|
||||
|
||||
def _seconds(match: re.Match[str]) -> int:
|
||||
a, b, c = match.group(1), match.group(2), match.group(3)
|
||||
if c is None: # mm:ss
|
||||
return int(a) * 60 + int(b)
|
||||
return int(a) * 3600 + int(b) * 60 + int(c)
|
||||
|
||||
|
||||
def _is_heading(text: str, links: list[Link]) -> bool:
|
||||
"""A rubric like 'Policy & Safety', not a sentence and not a story.
|
||||
|
||||
Deliberately strict. Anything that gets through becomes the heading for
|
||||
every chapter after it, so a false positive mislabels the rest of the
|
||||
episode — where a false negative just leaves a few items unfiled.
|
||||
"""
|
||||
return (
|
||||
not links
|
||||
and len(text) <= 60
|
||||
and len(text.split()) <= 6
|
||||
and not text.endswith((".", "!", "?"))
|
||||
)
|
||||
|
||||
|
||||
def _text_lines(body: str) -> list[tuple[str, list[Link]]]:
|
||||
"""Plain-text notes — a YouTube description. Bare URLs become links."""
|
||||
lines = []
|
||||
for raw in body.splitlines():
|
||||
line = raw.strip()
|
||||
if not line:
|
||||
continue
|
||||
links = [Link(url=u) for u in _URL.findall(line)]
|
||||
lines.append((_URL.sub("", line).strip(), links))
|
||||
return lines
|
||||
|
||||
|
||||
def _html_lines(body: str) -> list[tuple[str, list[Link]]]:
|
||||
parser = _Outline()
|
||||
parser.feed(body)
|
||||
parser.close()
|
||||
return parser.lines
|
||||
|
||||
|
||||
class _Outline(HTMLParser):
|
||||
"""HTML to (line, links) pairs, without pulling in a parser library.
|
||||
|
||||
A full HTML tree would be nicer and is not worth a dependency for this:
|
||||
the only structure that matters is where the line breaks fall and which
|
||||
anchors landed on which line. `convert_charrefs` (on by default) means
|
||||
entities arrive already decoded, so `Tools & Apps` is `Tools & Apps`
|
||||
by the time it reaches `handle_data`.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.lines: list[tuple[str, list[Link]]] = []
|
||||
self._text: list[str] = []
|
||||
self._links: list[Link] = []
|
||||
self._href = ""
|
||||
self._anchor: list[str] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if tag in _BLOCKS:
|
||||
self._flush()
|
||||
elif tag == "a":
|
||||
self._href = dict(attrs).get("href") or ""
|
||||
self._anchor = []
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag in _BLOCKS:
|
||||
self._flush()
|
||||
elif tag == "a":
|
||||
if self._href:
|
||||
self._links.append(
|
||||
Link(url=self._href, title=_squash("".join(self._anchor)))
|
||||
)
|
||||
self._href, self._anchor = "", []
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
self._text.append(data)
|
||||
if self._href:
|
||||
self._anchor.append(data)
|
||||
|
||||
def _flush(self) -> None:
|
||||
text = _squash("".join(self._text))
|
||||
if text or self._links:
|
||||
self.lines.append((text, self._links))
|
||||
self._text, self._links = [], []
|
||||
|
||||
def close(self) -> None:
|
||||
super().close()
|
||||
self._flush()
|
||||
|
||||
|
||||
def _squash(text: str) -> str:
|
||||
return re.sub(r"\s+", " ", text or "").strip()
|
||||
Reference in new issue
Block a user