Bring in desk from news

The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-23 12:29:17 +02:00
1 parent 9d666d0937
commit 248ea1636d
36 files changed
+8393

No files matched your search

+241
View File
@@ -0,0 +1,241 @@
"""The shapes newsdesk hands round: an item, a link, a result from the model.
One set of models, not two — there is no ORM here, so the row the table
returns and the object the API serves are the same declaration.
The queue's own shapes — `Job`, `Run`, `JobsSnapshot`, `Notice` and the
`Model` base they share — live in `pipeline/models.py`, because the pipeline
is a folder that gets copied into other projects and cannot depend on this
one. They are re-exported here so that `desk.models` stays the one place an
app's code looks: the API serves a `Job` next to an `Item`, and which folder
each was declared in is not something a route should have to know.
"""
from __future__ import annotations
import hashlib
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any, Literal
from urllib.parse import urlsplit
from pydantic import field_validator
from pipeline.models import (
TERMINAL, CascadeProgress, Job, JobsSnapshot, JobState, JsonDict, JsonIds,
Model, Notice, NoticeKind, NoticeLevel, Run, TaskProgress,
)
# ------------------------------------------------------------ model tasks
class ModelTaskResult(Model):
"""What one run of a task for the model did. Never an error: when it can't
run it says why. See desk/modeltask.py.
Subclasses add the counters that mean something for their task and say
which of them is `done`. Everything the runner adds up is here.
"""
skipped: bool = False
pending: int = 0
reason: str | None = None
batches: int = 0
#: Batches that asked the model and got nothing back. Kept apart from
#: `batches` because a run with 0 done and 0 failed had nothing to do,
#: while 0 done and 4 failed is a broken backend — and reporting both as
#: "0 done" is exactly the silence this is here to end.
failed: int = 0
#: What the model answered that couldn't be used: a slug not in the
#: vocabulary, a uid not in the batch, an abstract too short to be one.
rejected: list[str] = []
#: Rows this run stopped offering: handed to the model often enough,
#: without an answer, that asking again is only spending. Reported rather
#: than left to be inferred from a backlog that quietly got smaller — see
#: `ModelTask.missed`.
given_up: int = 0
#: Which of desk/tools/agent.py's backends answered — `api` or
#: `claude_code`, and empty when nothing did. Worth keeping: it is the
#: difference between a run that cost money and one that didn't.
backend: str = ""
@property
def done(self) -> int:
"""How many rows this run finished. The task's own counter."""
raise NotImplementedError
def absorb(self, other: ModelTaskResult) -> None:
"""Add one chunk's result into a running total.
Counts add, lists extend; `skipped`, `pending`, `reason` and
`backend` are about the run, not the chunk, and are left to the
runner. Written once here so a new task's counters count without it
saying how.
"""
for name in type(other).model_fields:
if name in ("skipped", "pending", "reason", "backend"):
continue
mine, theirs = getattr(self, name), getattr(other, name)
if isinstance(mine, list):
mine.extend(theirs)
elif isinstance(mine, int) and not isinstance(mine, bool):
setattr(self, name, mine + theirs)
class ItemLink(Model):
"""A source-supplied link with a shared meaning, separate from the destination."""
url: str
title: str = ""
role: Literal["reading", "discussion", "source"] = "source"
@field_validator("url")
@classmethod
def http_url(cls, value: str) -> str:
parsed = urlsplit(value)
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
raise ValueError("item links must be absolute HTTP(S) URLs")
return value
@dataclass(slots=True)
class Item:
"""One thing that showed up: an article, a post, a link."""
#: The source's own id for this thing. With `source`, it makes the `uid`,
#: so refetching the same thing recognises it rather than duplicating it.
source_id: str
#: Where the original lives. Any URI — `https://` for a feed entry,
#: `file:///…` for something added by hand.
location: str
title: str
#: The short version the source handed over: a feed summary, a post.
excerpt: str = ""
#: The whole thing, when the source already has it. Left empty by every
#: source that doesn't: `ingest.fetch_bodies` looks for an empty body
#: and fetches the page with trafilatura, so filling this in is also how a
#: source says "don't go and scrape this" — which is what a podcast
#: segment wants, its body being a transcript rather than a web page.
body: str = ""
author: str = ""
#: Always set and always UTC once `__post_init__` has run — a plugin may
#: omit it, and gets `now`; a plugin that supplies one must make it aware,
#: and it is converted. Declared as a plain `datetime` rather than
#: `datetime | None` because that is what every reader is handed; the
#: `None` only ever exists between the two.
#:
#: UTC and not merely aware, because this is the one field that is written
#: to storage as its own text and then *compared as text* — ordered,
#: `MAX`ed, ranged over. A stamp keeping the offset its API sent sorts by
#: the digits it happens to start with, so 10:00+02:00 comes out newer than
#: 09:00+00:00 when it is an hour older. Converting here rather than at the
#: `INSERT` is what makes every reader of the field agree with the column.
published_at: datetime = field(default_factory=lambda: datetime.now(UTC))
#: Which feed or handle you follow that produced it. The only attribution
#: an item carries: who publishes it — the outlet, and the section of it —
#: is a fact about *this subscription*, held once on the subscription row
#: and joined when something needs it. It used to be copied onto every
#: item, which meant a rename left four hundred rows disagreeing with the
#: feed they came from until a repair pass walked the table.
subscription: str = ""
source: str = "" # plugin name; filled in by the runner
#: What this source made, in the app's own words. A free string, and
#: deliberately: which shapes exist is the app's business, not the plugin
#: contract's. newsdesk reads `article`, `post` and `segment` and lays each
#: out its own way; a jobs board on this core will say `role` and `desk`
#: has no opinion about either list.
#:
#: What was *made*, not what carried it — an RSS feed of a podcast, once
#: the audio is fetched and cut, yields segments and should say so. This
#: used to read `article | video | audio`, which named the medium; the
#: medium is not the shape, and every row in the table said `article`
#: anyway because no plugin ever set it.
kind: str = "article"
#: Which of its plugin's declared profiles made this, by name. A plugin
#: that declares one uses `default` and never sets this; a show that
#: publishes both whole episodes and cut segments names the profile each
#: item belongs to, and that recorded name is what decides how the item is
#: processed and laid out for the rest of its life. Never worked out later
#: from what the item happens to carry — see `desk/sources/spec.py`.
profile: str = "default"
#: What this is, when it is not an ordinary article: `paywalled`,
#: `sponsored`. Usually left alone — `ingest` reads them off the title
#: using the tokens the subscription declares (see desk/tools/marks.py).
#: A plugin talking to an API that says so outright can set them directly.
marks: list[str] = field(default_factory=list)
#: The title as it was published. `marks.classify` writes it onto every
#: item it looks at, whether or not it cut anything, because a title with
#: a token already cut off cannot be reclassified later — the evidence
#: goes with it, and "we looked and there was nothing to cut" is a fact
#: worth having rather than an absence to re-derive.
#:
#: Empty therefore means *unknown*: a row stored before originals were
#: kept, or one no rule pass has seen. `Store` reads it that way — see the
#: unknown-original count `reclassify_marks` reports.
source_title: str = ""
#: Whatever else the source returned, kept as JSON on the row. One key is
#: read by name: `categories`, the source's own labels for this item, which
#: the tagger takes as a weak hint. Deliberately not merged into `tags` —
#: every outlet's scheme is its own ("Television & radio", "Putin, Vladimir
#: V"), and a filter built from the union of all of them is unusable. A
#: plugin that sets it caps its own list; a dozen is plenty.
raw: dict[str, Any] = field(default_factory=dict)
links: list[ItemLink] = field(default_factory=list)
def __post_init__(self) -> None:
# A plugin passing `published_at=None` explicitly bypasses the default
# factory. Unreachable as far as the annotation is concerned, and the
# annotation is what every *reader* should go by — but the plugin
# contract is a published interface and this used to be allowed, so a
# drop-in source under `plugin_paths` must not start raising.
if self.published_at is None:
self.published_at = datetime.now(UTC) # type: ignore[unreachable]
# A naive stamp is a source bug, and the only one of these three that
# cannot be repaired here: which hour it means is a fact about the API
# that sent it, and this end has nothing to read it off. It used to be
# taken as UTC, which is a guess wearing a fixed offset — wrong by up
# to a day for anything that sends local time, and wrong silently,
# since the row that comes back looks like every other row. A source
# that knows its API's convention says so where it parses; see
# `desk/sources/base.py`.
elif self.published_at.utcoffset() is None:
raise ValueError(
f"{self.source or 'a source'} gave a naive published_at "
f"({self.published_at.isoformat()}) for {self.source_id!r}. "
"A source must hand over an aware datetime: attach the offset "
"the API sent, or UTC when it documents UTC and says nothing."
)
else:
self.published_at = self.published_at.astimezone(UTC)
self.title = (self.title or "").strip()
self.excerpt = (self.excerpt or "").strip()
if not self.title and self.excerpt:
# Social posts have no title. Use the first line, capped.
first = self.excerpt.split("\n", 1)[0]
self.title = first[:200] + ("…" if len(first) > 200 else "")
@property
def uid(self) -> str:
return hashlib.sha1(
f"{self.source}\x00{self.source_id}".encode()
).hexdigest()
__all__ = [
"TERMINAL",
"CascadeProgress",
"Item",
"ItemLink",
"Job",
"JobState",
"JobsSnapshot",
"JsonDict",
"JsonIds",
"Model",
"ModelTaskResult",
"Notice",
"NoticeKind",
"NoticeLevel",
"Run",
"TaskProgress",
]