Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d666d0937
commit
248ea1636d
36 files changed
+8393
No files matched your search
@@ -0,0 +1,261 @@
|
||||
"""A video, and the pieces it breaks into.
|
||||
|
||||
This is the generic half and it knows nothing about podcasts, feeds or
|
||||
newsdesk's tables. A `Video` is something with a transcript and a duration; a
|
||||
`Segment` is a span of one, with a title, a start, an end and the words spoken
|
||||
inside it. Cutting one into the other needs two inputs and no model:
|
||||
|
||||
Video --+-- transcript -> timed cues -----+
|
||||
| +-> cut() -> Segment[]
|
||||
+-- chapter marks (marks.best) ----+
|
||||
|
||||
What supplies those inputs is somebody else's problem, and deliberately so.
|
||||
YouTube supplies both today. A show's own web page supplies better marks, with
|
||||
the article links attached — that is what `desk/sources/podcast.py` does,
|
||||
by passing its own marks in. A local recording plus a Whisper transcript would
|
||||
supply both without touching this file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from desk.models import ItemLink
|
||||
from desk.tools.video import marks as marks_mod
|
||||
from desk.tools.video import youtube
|
||||
from desk.tools.video.marks import Chapter, Link
|
||||
from desk.tools.video.media import MediaRef
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Video:
|
||||
"""One video, before anything has been done to it."""
|
||||
|
||||
ref: MediaRef
|
||||
title: str = ""
|
||||
#: The page it was published on, when that is a different thing from the
|
||||
#: video itself — a Substack post, a show's episode page. Empty for a video
|
||||
#: that is only a video.
|
||||
url: str = ""
|
||||
author: str = ""
|
||||
published_at: datetime = field(default_factory=lambda: datetime.now(UTC))
|
||||
duration: float = 0.0
|
||||
#: Whatever text came with it and might carry chapter marks: a YouTube
|
||||
#: description, a feed body, show notes as HTML.
|
||||
notes: str = ""
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
"""What identifies this video for the life of the archive.
|
||||
|
||||
The media *ref's* key, not a URL, and never the resolved playback
|
||||
location — pointing an item at a local copy must not change its
|
||||
identity or every segment of that video re-ingests as a new item.
|
||||
"""
|
||||
return self.ref.key
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Segment:
|
||||
"""One span of a video: the unit that becomes a newsdesk item."""
|
||||
|
||||
video: Video
|
||||
index: int
|
||||
start: int
|
||||
end: int
|
||||
title: str
|
||||
heading: str = "" # the rubric it was filed under
|
||||
links: list[Link] = field(default_factory=list)
|
||||
transcript: str = "" # verbatim, this span only
|
||||
|
||||
@property
|
||||
def external_id(self) -> str:
|
||||
"""Stable across re-ingests of the same video.
|
||||
|
||||
Keyed on the video and the start time rather than the position in the
|
||||
list, so a show that inserts a chapter it forgot doesn't renumber —
|
||||
and therefore doesn't duplicate — everything after it. Re-cutting
|
||||
against corrected timestamps *does* produce new ids, which is the
|
||||
honest outcome: they are different spans of the recording.
|
||||
"""
|
||||
return f"{self.video.key}#t={self.start}"
|
||||
|
||||
def url(self, overrides: dict[str, str] | None = None) -> str:
|
||||
"""A link that plays this segment, wherever the video currently lives.
|
||||
|
||||
Resolved on the way out rather than stored, which is the whole point of
|
||||
`media.MediaRef`: mapping a YouTube id to a file on disk changes every
|
||||
link in the archive and rewrites nothing.
|
||||
"""
|
||||
from desk.tools.video import media
|
||||
return media.resolve(self.video.ref, overrides).at(self.start)
|
||||
|
||||
@property
|
||||
def duration(self) -> int:
|
||||
return max(0, self.end - self.start)
|
||||
|
||||
@property
|
||||
def timecode(self) -> str:
|
||||
return marks_mod.hhmmss(self.start)
|
||||
|
||||
def record(self) -> dict[str, Any]:
|
||||
"""The segment as a JSON-able dict: what newsdesk stores in `items.raw`.
|
||||
|
||||
One definition of the layout, so the two writers (a podcast segment,
|
||||
a video added by hand) and the readers (the reading pane, the
|
||||
summariser's query) agree on the keys. `media` is the reference and
|
||||
never a URL — see `media.py`; `duration` is denormalised so a queue row
|
||||
can show a length chip without arithmetic across two JSON extractions.
|
||||
|
||||
The articles are not in here: they are a typed collection of their own
|
||||
now — see `reading`. `raw` is what no shared field has a name for, and
|
||||
a link the whole reader understands is not that.
|
||||
"""
|
||||
return {
|
||||
"media": {"kind": self.video.ref.kind, "key": self.video.ref.key},
|
||||
"start": self.start,
|
||||
"end": self.end,
|
||||
"duration": self.duration,
|
||||
"index": self.index,
|
||||
}
|
||||
|
||||
def reading(self) -> list[ItemLink]:
|
||||
"""What the hosts were reading, as links the whole app understands.
|
||||
|
||||
The outline's links with a role on them, which is the difference
|
||||
between "a URL this show happened to print" and "the article this
|
||||
stretch is about": the reader lists them under the player, the
|
||||
summariser names them in the prompt, and neither has to know a podcast
|
||||
produced them.
|
||||
|
||||
A line of show notes is whatever an editor typed, so a relative href
|
||||
or a `mailto:` is an ordinary outcome rather than a fault — dropped
|
||||
with a note, because one malformed link in the outline must not cost
|
||||
the segment it sits on.
|
||||
"""
|
||||
out = []
|
||||
for link in self.links:
|
||||
try:
|
||||
out.append(ItemLink(url=link.url, title=link.title, role="reading"))
|
||||
except ValueError:
|
||||
log.debug("%s: dropping unusable outline link %r",
|
||||
self.external_id, link.url[:120])
|
||||
return out
|
||||
|
||||
|
||||
def cut(video: Video, cues: list[youtube.Cue],
|
||||
chapters: list[Chapter] | None = None) -> list[Segment]:
|
||||
"""Cut a video into one segment per chapter mark.
|
||||
|
||||
A chapter runs until the next one starts; the last one runs to the end of
|
||||
the recording. That is the only boundary rule, and it is exact, because the
|
||||
marks were written by the people who recorded it.
|
||||
|
||||
With no chapter marks at all the video comes back as a single segment
|
||||
holding the whole transcript. That is deliberately not a failure: an
|
||||
unsegmented video is still worth having, and inventing boundaries where
|
||||
nobody published any is a job for a model and a later stage, not for a
|
||||
slicing function.
|
||||
"""
|
||||
chapters = chapters if chapters is not None else marks_mod.parse(video.notes)
|
||||
end_of_video = int(video.duration or (cues[-1].end if cues else 0))
|
||||
|
||||
if not chapters:
|
||||
return [Segment(
|
||||
video=video, index=0, start=0, end=end_of_video,
|
||||
title=video.title, transcript=youtube.text_between(cues, 0),
|
||||
)]
|
||||
|
||||
ordered = sorted(chapters, key=lambda c: c.start)
|
||||
out = []
|
||||
for index, chapter in enumerate(ordered):
|
||||
following = (ordered[index + 1].start if index + 1 < len(ordered)
|
||||
else end_of_video)
|
||||
# A show that mistypes a timestamp can put one mark after the next.
|
||||
# Clamping keeps the slice empty rather than reversed, and leaves the
|
||||
# neighbouring segments intact.
|
||||
end = max(chapter.start, following)
|
||||
out.append(Segment(
|
||||
video=video,
|
||||
index=index,
|
||||
start=chapter.start,
|
||||
end=end,
|
||||
title=chapter.title,
|
||||
heading=chapter.heading,
|
||||
links=chapter.links,
|
||||
transcript=youtube.text_between(cues, chapter.start, end or None),
|
||||
))
|
||||
|
||||
empty = sum(1 for s in out if not s.transcript)
|
||||
if empty:
|
||||
log.warning("%s: %d of %d segments have no transcript under them",
|
||||
video.key, empty, len(out))
|
||||
return out
|
||||
|
||||
|
||||
# ------------------------------------------------------ transcript as text
|
||||
|
||||
_SENTENCE_END = re.compile(r"(?<=[.!?])\s+")
|
||||
|
||||
|
||||
def opening(transcript: str, sentences: int = 2, cap: int = 400) -> str:
|
||||
"""The first sentence or two, as a stand-in for a summary that isn't
|
||||
written yet. An empty `text` is an empty row in the queue."""
|
||||
if not transcript:
|
||||
return ""
|
||||
parts = _SENTENCE_END.split(transcript.strip())
|
||||
return " ".join(parts[:sentences])[:cap]
|
||||
|
||||
|
||||
def paragraphs(text: str, target: int = 700) -> str:
|
||||
"""Break a captions stream into paragraphs a reading pane can render.
|
||||
|
||||
Captions arrive as one unbroken run, and the reader splits a body on blank
|
||||
lines — so without this a forty-minute talk is one paragraph. Sentences
|
||||
are grouped until roughly `target` characters, then a break: arbitrary,
|
||||
but readable, and the alternative is a wall.
|
||||
|
||||
Auto-generated captions often have no punctuation at all, so one
|
||||
"sentence" can be the whole talk. Those are broken on a word boundary
|
||||
instead — worse than a real sentence end, better than the wall.
|
||||
"""
|
||||
text = " ".join((text or "").split())
|
||||
if not text:
|
||||
return ""
|
||||
out: list[str] = []
|
||||
current: list[str] = []
|
||||
length = 0
|
||||
for sentence in _SENTENCE_END.split(text):
|
||||
for piece in _words(sentence, target * 2):
|
||||
current.append(piece)
|
||||
length += len(piece) + 1
|
||||
if length >= target:
|
||||
out.append(" ".join(current))
|
||||
current, length = [], 0
|
||||
if current:
|
||||
out.append(" ".join(current))
|
||||
return "\n\n".join(out)
|
||||
|
||||
|
||||
def _words(run: str, limit: int) -> list[str]:
|
||||
"""Split an over-long run at spaces, keeping every piece under `limit`."""
|
||||
if len(run) <= limit:
|
||||
return [run]
|
||||
pieces: list[str] = []
|
||||
current: list[str] = []
|
||||
length = 0
|
||||
for word in run.split(" "):
|
||||
if current and length + len(word) + 1 > limit:
|
||||
pieces.append(" ".join(current))
|
||||
current, length = [], 0
|
||||
current.append(word)
|
||||
length += len(word) + 1
|
||||
if current:
|
||||
pieces.append(" ".join(current))
|
||||
return pieces
|
||||
Reference in new issue
Block a user