"""A video, and the pieces it breaks into. This is the generic half and it knows nothing about podcasts, feeds or newsdesk's tables. A `Video` is something with a transcript and a duration; a `Segment` is a span of one, with a title, a start, an end and the words spoken inside it. Cutting one into the other needs two inputs and no model: Video --+-- transcript -> timed cues -----+ | +-> cut() -> Segment[] +-- chapter marks (marks.best) ----+ What supplies those inputs is somebody else's problem, and deliberately so. YouTube supplies both today. A show's own web page supplies better marks, with the article links attached — that is what `desk/sources/podcast.py` does, by passing its own marks in. A local recording plus a Whisper transcript would supply both without touching this file. """ from __future__ import annotations import logging import re from dataclasses import dataclass, field from datetime import UTC, datetime from typing import Any from desk.models import ItemLink from desk.tools.video import marks as marks_mod from desk.tools.video import youtube from desk.tools.video.marks import Chapter, Link from desk.tools.video.media import MediaRef log = logging.getLogger(__name__) @dataclass(slots=True) class Video: """One video, before anything has been done to it.""" ref: MediaRef title: str = "" #: The page it was published on, when that is a different thing from the #: video itself — a Substack post, a show's episode page. Empty for a video #: that is only a video. url: str = "" author: str = "" published_at: datetime = field(default_factory=lambda: datetime.now(UTC)) duration: float = 0.0 #: Whatever text came with it and might carry chapter marks: a YouTube #: description, a feed body, show notes as HTML. notes: str = "" @property def key(self) -> str: """What identifies this video for the life of the archive. The media *ref's* key, not a URL, and never the resolved playback location — pointing an item at a local copy must not change its identity or every segment of that video re-ingests as a new item. """ return self.ref.key @dataclass(slots=True) class Segment: """One span of a video: the unit that becomes a newsdesk item.""" video: Video index: int start: int end: int title: str heading: str = "" # the rubric it was filed under links: list[Link] = field(default_factory=list) transcript: str = "" # verbatim, this span only @property def external_id(self) -> str: """Stable across re-ingests of the same video. Keyed on the video and the start time rather than the position in the list, so a show that inserts a chapter it forgot doesn't renumber — and therefore doesn't duplicate — everything after it. Re-cutting against corrected timestamps *does* produce new ids, which is the honest outcome: they are different spans of the recording. """ return f"{self.video.key}#t={self.start}" def url(self, overrides: dict[str, str] | None = None) -> str: """A link that plays this segment, wherever the video currently lives. Resolved on the way out rather than stored, which is the whole point of `media.MediaRef`: mapping a YouTube id to a file on disk changes every link in the archive and rewrites nothing. """ from desk.tools.video import media return media.resolve(self.video.ref, overrides).at(self.start) @property def duration(self) -> int: return max(0, self.end - self.start) @property def timecode(self) -> str: return marks_mod.hhmmss(self.start) def record(self) -> dict[str, Any]: """The segment as a JSON-able dict: what newsdesk stores in `items.raw`. One definition of the layout, so the two writers (a podcast segment, a video added by hand) and the readers (the reading pane, the summariser's query) agree on the keys. `media` is the reference and never a URL — see `media.py`; `duration` is denormalised so a queue row can show a length chip without arithmetic across two JSON extractions. The articles are not in here: they are a typed collection of their own now — see `reading`. `raw` is what no shared field has a name for, and a link the whole reader understands is not that. """ return { "media": {"kind": self.video.ref.kind, "key": self.video.ref.key}, "start": self.start, "end": self.end, "duration": self.duration, "index": self.index, } def reading(self) -> list[ItemLink]: """What the hosts were reading, as links the whole app understands. The outline's links with a role on them, which is the difference between "a URL this show happened to print" and "the article this stretch is about": the reader lists them under the player, the summariser names them in the prompt, and neither has to know a podcast produced them. A line of show notes is whatever an editor typed, so a relative href or a `mailto:` is an ordinary outcome rather than a fault — dropped with a note, because one malformed link in the outline must not cost the segment it sits on. """ out = [] for link in self.links: try: out.append(ItemLink(url=link.url, title=link.title, role="reading")) except ValueError: log.debug("%s: dropping unusable outline link %r", self.external_id, link.url[:120]) return out def cut(video: Video, cues: list[youtube.Cue], chapters: list[Chapter] | None = None) -> list[Segment]: """Cut a video into one segment per chapter mark. A chapter runs until the next one starts; the last one runs to the end of the recording. That is the only boundary rule, and it is exact, because the marks were written by the people who recorded it. With no chapter marks at all the video comes back as a single segment holding the whole transcript. That is deliberately not a failure: an unsegmented video is still worth having, and inventing boundaries where nobody published any is a job for a model and a later stage, not for a slicing function. """ chapters = chapters if chapters is not None else marks_mod.parse(video.notes) end_of_video = int(video.duration or (cues[-1].end if cues else 0)) if not chapters: return [Segment( video=video, index=0, start=0, end=end_of_video, title=video.title, transcript=youtube.text_between(cues, 0), )] ordered = sorted(chapters, key=lambda c: c.start) out = [] for index, chapter in enumerate(ordered): following = (ordered[index + 1].start if index + 1 < len(ordered) else end_of_video) # A show that mistypes a timestamp can put one mark after the next. # Clamping keeps the slice empty rather than reversed, and leaves the # neighbouring segments intact. end = max(chapter.start, following) out.append(Segment( video=video, index=index, start=chapter.start, end=end, title=chapter.title, heading=chapter.heading, links=chapter.links, transcript=youtube.text_between(cues, chapter.start, end or None), )) empty = sum(1 for s in out if not s.transcript) if empty: log.warning("%s: %d of %d segments have no transcript under them", video.key, empty, len(out)) return out # ------------------------------------------------------ transcript as text _SENTENCE_END = re.compile(r"(?<=[.!?])\s+") def opening(transcript: str, sentences: int = 2, cap: int = 400) -> str: """The first sentence or two, as a stand-in for a summary that isn't written yet. An empty `text` is an empty row in the queue.""" if not transcript: return "" parts = _SENTENCE_END.split(transcript.strip()) return " ".join(parts[:sentences])[:cap] def paragraphs(text: str, target: int = 700) -> str: """Break a captions stream into paragraphs a reading pane can render. Captions arrive as one unbroken run, and the reader splits a body on blank lines — so without this a forty-minute talk is one paragraph. Sentences are grouped until roughly `target` characters, then a break: arbitrary, but readable, and the alternative is a wall. Auto-generated captions often have no punctuation at all, so one "sentence" can be the whole talk. Those are broken on a word boundary instead — worse than a real sentence end, better than the wall. """ text = " ".join((text or "").split()) if not text: return "" out: list[str] = [] current: list[str] = [] length = 0 for sentence in _SENTENCE_END.split(text): for piece in _words(sentence, target * 2): current.append(piece) length += len(piece) + 1 if length >= target: out.append(" ".join(current)) current, length = [], 0 if current: out.append(" ".join(current)) return "\n\n".join(out) def _words(run: str, limit: int) -> list[str]: """Split an over-long run at spaces, keeping every piece under `limit`.""" if len(run) <= limit: return [run] pieces: list[str] = [] current: list[str] = [] length = 0 for word in run.split(" "): if current and length + len(word) + 1 > limit: pieces.append(" ".join(current)) current, length = [], 0 current.append(word) length += len(word) + 1 if current: pieces.append(" ".join(current)) return pieces