The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
242 lines
11 KiB
Python
242 lines
11 KiB
Python
"""The shapes newsdesk hands round: an item, a link, a result from the model.
|
|
|
|
One set of models, not two — there is no ORM here, so the row the table
|
|
returns and the object the API serves are the same declaration.
|
|
|
|
The queue's own shapes — `Job`, `Run`, `JobsSnapshot`, `Notice` and the
|
|
`Model` base they share — live in `pipeline/models.py`, because the pipeline
|
|
is a folder that gets copied into other projects and cannot depend on this
|
|
one. They are re-exported here so that `desk.models` stays the one place an
|
|
app's code looks: the API serves a `Job` next to an `Item`, and which folder
|
|
each was declared in is not something a route should have to know.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from dataclasses import dataclass, field
|
|
from datetime import UTC, datetime
|
|
from typing import Any, Literal
|
|
from urllib.parse import urlsplit
|
|
|
|
from pydantic import field_validator
|
|
|
|
from pipeline.models import (
|
|
TERMINAL, CascadeProgress, Job, JobsSnapshot, JobState, JsonDict, JsonIds,
|
|
Model, Notice, NoticeKind, NoticeLevel, Run, TaskProgress,
|
|
)
|
|
|
|
|
|
# ------------------------------------------------------------ model tasks
|
|
|
|
class ModelTaskResult(Model):
|
|
"""What one run of a task for the model did. Never an error: when it can't
|
|
run it says why. See desk/modeltask.py.
|
|
|
|
Subclasses add the counters that mean something for their task and say
|
|
which of them is `done`. Everything the runner adds up is here.
|
|
"""
|
|
|
|
skipped: bool = False
|
|
pending: int = 0
|
|
reason: str | None = None
|
|
batches: int = 0
|
|
#: Batches that asked the model and got nothing back. Kept apart from
|
|
#: `batches` because a run with 0 done and 0 failed had nothing to do,
|
|
#: while 0 done and 4 failed is a broken backend — and reporting both as
|
|
#: "0 done" is exactly the silence this is here to end.
|
|
failed: int = 0
|
|
#: What the model answered that couldn't be used: a slug not in the
|
|
#: vocabulary, a uid not in the batch, an abstract too short to be one.
|
|
rejected: list[str] = []
|
|
#: Rows this run stopped offering: handed to the model often enough,
|
|
#: without an answer, that asking again is only spending. Reported rather
|
|
#: than left to be inferred from a backlog that quietly got smaller — see
|
|
#: `ModelTask.missed`.
|
|
given_up: int = 0
|
|
#: Which of desk/tools/agent.py's backends answered — `api` or
|
|
#: `claude_code`, and empty when nothing did. Worth keeping: it is the
|
|
#: difference between a run that cost money and one that didn't.
|
|
backend: str = ""
|
|
|
|
@property
|
|
def done(self) -> int:
|
|
"""How many rows this run finished. The task's own counter."""
|
|
raise NotImplementedError
|
|
|
|
def absorb(self, other: ModelTaskResult) -> None:
|
|
"""Add one chunk's result into a running total.
|
|
|
|
Counts add, lists extend; `skipped`, `pending`, `reason` and
|
|
`backend` are about the run, not the chunk, and are left to the
|
|
runner. Written once here so a new task's counters count without it
|
|
saying how.
|
|
"""
|
|
for name in type(other).model_fields:
|
|
if name in ("skipped", "pending", "reason", "backend"):
|
|
continue
|
|
mine, theirs = getattr(self, name), getattr(other, name)
|
|
if isinstance(mine, list):
|
|
mine.extend(theirs)
|
|
elif isinstance(mine, int) and not isinstance(mine, bool):
|
|
setattr(self, name, mine + theirs)
|
|
|
|
|
|
class ItemLink(Model):
|
|
"""A source-supplied link with a shared meaning, separate from the destination."""
|
|
|
|
url: str
|
|
title: str = ""
|
|
role: Literal["reading", "discussion", "source"] = "source"
|
|
|
|
@field_validator("url")
|
|
@classmethod
|
|
def http_url(cls, value: str) -> str:
|
|
parsed = urlsplit(value)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
|
raise ValueError("item links must be absolute HTTP(S) URLs")
|
|
return value
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class Item:
|
|
"""One thing that showed up: an article, a post, a link."""
|
|
|
|
#: The source's own id for this thing. With `source`, it makes the `uid`,
|
|
#: so refetching the same thing recognises it rather than duplicating it.
|
|
source_id: str
|
|
#: Where the original lives. Any URI — `https://` for a feed entry,
|
|
#: `file:///…` for something added by hand.
|
|
location: str
|
|
title: str
|
|
#: The short version the source handed over: a feed summary, a post.
|
|
excerpt: str = ""
|
|
#: The whole thing, when the source already has it. Left empty by every
|
|
#: source that doesn't: `ingest.fetch_bodies` looks for an empty body
|
|
#: and fetches the page with trafilatura, so filling this in is also how a
|
|
#: source says "don't go and scrape this" — which is what a podcast
|
|
#: segment wants, its body being a transcript rather than a web page.
|
|
body: str = ""
|
|
author: str = ""
|
|
#: Always set and always UTC once `__post_init__` has run — a plugin may
|
|
#: omit it, and gets `now`; a plugin that supplies one must make it aware,
|
|
#: and it is converted. Declared as a plain `datetime` rather than
|
|
#: `datetime | None` because that is what every reader is handed; the
|
|
#: `None` only ever exists between the two.
|
|
#:
|
|
#: UTC and not merely aware, because this is the one field that is written
|
|
#: to storage as its own text and then *compared as text* — ordered,
|
|
#: `MAX`ed, ranged over. A stamp keeping the offset its API sent sorts by
|
|
#: the digits it happens to start with, so 10:00+02:00 comes out newer than
|
|
#: 09:00+00:00 when it is an hour older. Converting here rather than at the
|
|
#: `INSERT` is what makes every reader of the field agree with the column.
|
|
published_at: datetime = field(default_factory=lambda: datetime.now(UTC))
|
|
#: Which feed or handle you follow that produced it. The only attribution
|
|
#: an item carries: who publishes it — the outlet, and the section of it —
|
|
#: is a fact about *this subscription*, held once on the subscription row
|
|
#: and joined when something needs it. It used to be copied onto every
|
|
#: item, which meant a rename left four hundred rows disagreeing with the
|
|
#: feed they came from until a repair pass walked the table.
|
|
subscription: str = ""
|
|
source: str = "" # plugin name; filled in by the runner
|
|
#: What this source made, in the app's own words. A free string, and
|
|
#: deliberately: which shapes exist is the app's business, not the plugin
|
|
#: contract's. newsdesk reads `article`, `post` and `segment` and lays each
|
|
#: out its own way; a jobs board on this core will say `role` and `desk`
|
|
#: has no opinion about either list.
|
|
#:
|
|
#: What was *made*, not what carried it — an RSS feed of a podcast, once
|
|
#: the audio is fetched and cut, yields segments and should say so. This
|
|
#: used to read `article | video | audio`, which named the medium; the
|
|
#: medium is not the shape, and every row in the table said `article`
|
|
#: anyway because no plugin ever set it.
|
|
kind: str = "article"
|
|
#: Which of its plugin's declared profiles made this, by name. A plugin
|
|
#: that declares one uses `default` and never sets this; a show that
|
|
#: publishes both whole episodes and cut segments names the profile each
|
|
#: item belongs to, and that recorded name is what decides how the item is
|
|
#: processed and laid out for the rest of its life. Never worked out later
|
|
#: from what the item happens to carry — see `desk/sources/spec.py`.
|
|
profile: str = "default"
|
|
#: What this is, when it is not an ordinary article: `paywalled`,
|
|
#: `sponsored`. Usually left alone — `ingest` reads them off the title
|
|
#: using the tokens the subscription declares (see desk/tools/marks.py).
|
|
#: A plugin talking to an API that says so outright can set them directly.
|
|
marks: list[str] = field(default_factory=list)
|
|
#: The title as it was published. `marks.classify` writes it onto every
|
|
#: item it looks at, whether or not it cut anything, because a title with
|
|
#: a token already cut off cannot be reclassified later — the evidence
|
|
#: goes with it, and "we looked and there was nothing to cut" is a fact
|
|
#: worth having rather than an absence to re-derive.
|
|
#:
|
|
#: Empty therefore means *unknown*: a row stored before originals were
|
|
#: kept, or one no rule pass has seen. `Store` reads it that way — see the
|
|
#: unknown-original count `reclassify_marks` reports.
|
|
source_title: str = ""
|
|
#: Whatever else the source returned, kept as JSON on the row. One key is
|
|
#: read by name: `categories`, the source's own labels for this item, which
|
|
#: the tagger takes as a weak hint. Deliberately not merged into `tags` —
|
|
#: every outlet's scheme is its own ("Television & radio", "Putin, Vladimir
|
|
#: V"), and a filter built from the union of all of them is unusable. A
|
|
#: plugin that sets it caps its own list; a dozen is plenty.
|
|
raw: dict[str, Any] = field(default_factory=dict)
|
|
links: list[ItemLink] = field(default_factory=list)
|
|
|
|
def __post_init__(self) -> None:
|
|
# A plugin passing `published_at=None` explicitly bypasses the default
|
|
# factory. Unreachable as far as the annotation is concerned, and the
|
|
# annotation is what every *reader* should go by — but the plugin
|
|
# contract is a published interface and this used to be allowed, so a
|
|
# drop-in source under `plugin_paths` must not start raising.
|
|
if self.published_at is None:
|
|
self.published_at = datetime.now(UTC) # type: ignore[unreachable]
|
|
# A naive stamp is a source bug, and the only one of these three that
|
|
# cannot be repaired here: which hour it means is a fact about the API
|
|
# that sent it, and this end has nothing to read it off. It used to be
|
|
# taken as UTC, which is a guess wearing a fixed offset — wrong by up
|
|
# to a day for anything that sends local time, and wrong silently,
|
|
# since the row that comes back looks like every other row. A source
|
|
# that knows its API's convention says so where it parses; see
|
|
# `desk/sources/base.py`.
|
|
elif self.published_at.utcoffset() is None:
|
|
raise ValueError(
|
|
f"{self.source or 'a source'} gave a naive published_at "
|
|
f"({self.published_at.isoformat()}) for {self.source_id!r}. "
|
|
"A source must hand over an aware datetime: attach the offset "
|
|
"the API sent, or UTC when it documents UTC and says nothing."
|
|
)
|
|
else:
|
|
self.published_at = self.published_at.astimezone(UTC)
|
|
self.title = (self.title or "").strip()
|
|
self.excerpt = (self.excerpt or "").strip()
|
|
if not self.title and self.excerpt:
|
|
# Social posts have no title. Use the first line, capped.
|
|
first = self.excerpt.split("\n", 1)[0]
|
|
self.title = first[:200] + ("…" if len(first) > 200 else "")
|
|
|
|
@property
|
|
def uid(self) -> str:
|
|
return hashlib.sha1(
|
|
f"{self.source}\x00{self.source_id}".encode()
|
|
).hexdigest()
|
|
|
|
|
|
__all__ = [
|
|
"TERMINAL",
|
|
"CascadeProgress",
|
|
"Item",
|
|
"ItemLink",
|
|
"Job",
|
|
"JobState",
|
|
"JobsSnapshot",
|
|
"JsonDict",
|
|
"JsonIds",
|
|
"Model",
|
|
"ModelTaskResult",
|
|
"Notice",
|
|
"NoticeKind",
|
|
"NoticeLevel",
|
|
"Run",
|
|
"TaskProgress",
|
|
]
|