Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d666d0937
commit
248ea1636d
36 files changed
+8393
No files matched your search
+241
@@ -0,0 +1,241 @@
|
||||
"""The shapes newsdesk hands round: an item, a link, a result from the model.
|
||||
|
||||
One set of models, not two — there is no ORM here, so the row the table
|
||||
returns and the object the API serves are the same declaration.
|
||||
|
||||
The queue's own shapes — `Job`, `Run`, `JobsSnapshot`, `Notice` and the
|
||||
`Model` base they share — live in `pipeline/models.py`, because the pipeline
|
||||
is a folder that gets copied into other projects and cannot depend on this
|
||||
one. They are re-exported here so that `desk.models` stays the one place an
|
||||
app's code looks: the API serves a `Job` next to an `Item`, and which folder
|
||||
each was declared in is not something a route should have to know.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, Literal
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from pydantic import field_validator
|
||||
|
||||
from pipeline.models import (
|
||||
TERMINAL, CascadeProgress, Job, JobsSnapshot, JobState, JsonDict, JsonIds,
|
||||
Model, Notice, NoticeKind, NoticeLevel, Run, TaskProgress,
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------ model tasks
|
||||
|
||||
class ModelTaskResult(Model):
|
||||
"""What one run of a task for the model did. Never an error: when it can't
|
||||
run it says why. See desk/modeltask.py.
|
||||
|
||||
Subclasses add the counters that mean something for their task and say
|
||||
which of them is `done`. Everything the runner adds up is here.
|
||||
"""
|
||||
|
||||
skipped: bool = False
|
||||
pending: int = 0
|
||||
reason: str | None = None
|
||||
batches: int = 0
|
||||
#: Batches that asked the model and got nothing back. Kept apart from
|
||||
#: `batches` because a run with 0 done and 0 failed had nothing to do,
|
||||
#: while 0 done and 4 failed is a broken backend — and reporting both as
|
||||
#: "0 done" is exactly the silence this is here to end.
|
||||
failed: int = 0
|
||||
#: What the model answered that couldn't be used: a slug not in the
|
||||
#: vocabulary, a uid not in the batch, an abstract too short to be one.
|
||||
rejected: list[str] = []
|
||||
#: Rows this run stopped offering: handed to the model often enough,
|
||||
#: without an answer, that asking again is only spending. Reported rather
|
||||
#: than left to be inferred from a backlog that quietly got smaller — see
|
||||
#: `ModelTask.missed`.
|
||||
given_up: int = 0
|
||||
#: Which of desk/tools/agent.py's backends answered — `api` or
|
||||
#: `claude_code`, and empty when nothing did. Worth keeping: it is the
|
||||
#: difference between a run that cost money and one that didn't.
|
||||
backend: str = ""
|
||||
|
||||
@property
|
||||
def done(self) -> int:
|
||||
"""How many rows this run finished. The task's own counter."""
|
||||
raise NotImplementedError
|
||||
|
||||
def absorb(self, other: ModelTaskResult) -> None:
|
||||
"""Add one chunk's result into a running total.
|
||||
|
||||
Counts add, lists extend; `skipped`, `pending`, `reason` and
|
||||
`backend` are about the run, not the chunk, and are left to the
|
||||
runner. Written once here so a new task's counters count without it
|
||||
saying how.
|
||||
"""
|
||||
for name in type(other).model_fields:
|
||||
if name in ("skipped", "pending", "reason", "backend"):
|
||||
continue
|
||||
mine, theirs = getattr(self, name), getattr(other, name)
|
||||
if isinstance(mine, list):
|
||||
mine.extend(theirs)
|
||||
elif isinstance(mine, int) and not isinstance(mine, bool):
|
||||
setattr(self, name, mine + theirs)
|
||||
|
||||
|
||||
class ItemLink(Model):
|
||||
"""A source-supplied link with a shared meaning, separate from the destination."""
|
||||
|
||||
url: str
|
||||
title: str = ""
|
||||
role: Literal["reading", "discussion", "source"] = "source"
|
||||
|
||||
@field_validator("url")
|
||||
@classmethod
|
||||
def http_url(cls, value: str) -> str:
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
raise ValueError("item links must be absolute HTTP(S) URLs")
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Item:
|
||||
"""One thing that showed up: an article, a post, a link."""
|
||||
|
||||
#: The source's own id for this thing. With `source`, it makes the `uid`,
|
||||
#: so refetching the same thing recognises it rather than duplicating it.
|
||||
source_id: str
|
||||
#: Where the original lives. Any URI — `https://` for a feed entry,
|
||||
#: `file:///…` for something added by hand.
|
||||
location: str
|
||||
title: str
|
||||
#: The short version the source handed over: a feed summary, a post.
|
||||
excerpt: str = ""
|
||||
#: The whole thing, when the source already has it. Left empty by every
|
||||
#: source that doesn't: `ingest.fetch_bodies` looks for an empty body
|
||||
#: and fetches the page with trafilatura, so filling this in is also how a
|
||||
#: source says "don't go and scrape this" — which is what a podcast
|
||||
#: segment wants, its body being a transcript rather than a web page.
|
||||
body: str = ""
|
||||
author: str = ""
|
||||
#: Always set and always UTC once `__post_init__` has run — a plugin may
|
||||
#: omit it, and gets `now`; a plugin that supplies one must make it aware,
|
||||
#: and it is converted. Declared as a plain `datetime` rather than
|
||||
#: `datetime | None` because that is what every reader is handed; the
|
||||
#: `None` only ever exists between the two.
|
||||
#:
|
||||
#: UTC and not merely aware, because this is the one field that is written
|
||||
#: to storage as its own text and then *compared as text* — ordered,
|
||||
#: `MAX`ed, ranged over. A stamp keeping the offset its API sent sorts by
|
||||
#: the digits it happens to start with, so 10:00+02:00 comes out newer than
|
||||
#: 09:00+00:00 when it is an hour older. Converting here rather than at the
|
||||
#: `INSERT` is what makes every reader of the field agree with the column.
|
||||
published_at: datetime = field(default_factory=lambda: datetime.now(UTC))
|
||||
#: Which feed or handle you follow that produced it. The only attribution
|
||||
#: an item carries: who publishes it — the outlet, and the section of it —
|
||||
#: is a fact about *this subscription*, held once on the subscription row
|
||||
#: and joined when something needs it. It used to be copied onto every
|
||||
#: item, which meant a rename left four hundred rows disagreeing with the
|
||||
#: feed they came from until a repair pass walked the table.
|
||||
subscription: str = ""
|
||||
source: str = "" # plugin name; filled in by the runner
|
||||
#: What this source made, in the app's own words. A free string, and
|
||||
#: deliberately: which shapes exist is the app's business, not the plugin
|
||||
#: contract's. newsdesk reads `article`, `post` and `segment` and lays each
|
||||
#: out its own way; a jobs board on this core will say `role` and `desk`
|
||||
#: has no opinion about either list.
|
||||
#:
|
||||
#: What was *made*, not what carried it — an RSS feed of a podcast, once
|
||||
#: the audio is fetched and cut, yields segments and should say so. This
|
||||
#: used to read `article | video | audio`, which named the medium; the
|
||||
#: medium is not the shape, and every row in the table said `article`
|
||||
#: anyway because no plugin ever set it.
|
||||
kind: str = "article"
|
||||
#: Which of its plugin's declared profiles made this, by name. A plugin
|
||||
#: that declares one uses `default` and never sets this; a show that
|
||||
#: publishes both whole episodes and cut segments names the profile each
|
||||
#: item belongs to, and that recorded name is what decides how the item is
|
||||
#: processed and laid out for the rest of its life. Never worked out later
|
||||
#: from what the item happens to carry — see `desk/sources/spec.py`.
|
||||
profile: str = "default"
|
||||
#: What this is, when it is not an ordinary article: `paywalled`,
|
||||
#: `sponsored`. Usually left alone — `ingest` reads them off the title
|
||||
#: using the tokens the subscription declares (see desk/tools/marks.py).
|
||||
#: A plugin talking to an API that says so outright can set them directly.
|
||||
marks: list[str] = field(default_factory=list)
|
||||
#: The title as it was published. `marks.classify` writes it onto every
|
||||
#: item it looks at, whether or not it cut anything, because a title with
|
||||
#: a token already cut off cannot be reclassified later — the evidence
|
||||
#: goes with it, and "we looked and there was nothing to cut" is a fact
|
||||
#: worth having rather than an absence to re-derive.
|
||||
#:
|
||||
#: Empty therefore means *unknown*: a row stored before originals were
|
||||
#: kept, or one no rule pass has seen. `Store` reads it that way — see the
|
||||
#: unknown-original count `reclassify_marks` reports.
|
||||
source_title: str = ""
|
||||
#: Whatever else the source returned, kept as JSON on the row. One key is
|
||||
#: read by name: `categories`, the source's own labels for this item, which
|
||||
#: the tagger takes as a weak hint. Deliberately not merged into `tags` —
|
||||
#: every outlet's scheme is its own ("Television & radio", "Putin, Vladimir
|
||||
#: V"), and a filter built from the union of all of them is unusable. A
|
||||
#: plugin that sets it caps its own list; a dozen is plenty.
|
||||
raw: dict[str, Any] = field(default_factory=dict)
|
||||
links: list[ItemLink] = field(default_factory=list)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
# A plugin passing `published_at=None` explicitly bypasses the default
|
||||
# factory. Unreachable as far as the annotation is concerned, and the
|
||||
# annotation is what every *reader* should go by — but the plugin
|
||||
# contract is a published interface and this used to be allowed, so a
|
||||
# drop-in source under `plugin_paths` must not start raising.
|
||||
if self.published_at is None:
|
||||
self.published_at = datetime.now(UTC) # type: ignore[unreachable]
|
||||
# A naive stamp is a source bug, and the only one of these three that
|
||||
# cannot be repaired here: which hour it means is a fact about the API
|
||||
# that sent it, and this end has nothing to read it off. It used to be
|
||||
# taken as UTC, which is a guess wearing a fixed offset — wrong by up
|
||||
# to a day for anything that sends local time, and wrong silently,
|
||||
# since the row that comes back looks like every other row. A source
|
||||
# that knows its API's convention says so where it parses; see
|
||||
# `desk/sources/base.py`.
|
||||
elif self.published_at.utcoffset() is None:
|
||||
raise ValueError(
|
||||
f"{self.source or 'a source'} gave a naive published_at "
|
||||
f"({self.published_at.isoformat()}) for {self.source_id!r}. "
|
||||
"A source must hand over an aware datetime: attach the offset "
|
||||
"the API sent, or UTC when it documents UTC and says nothing."
|
||||
)
|
||||
else:
|
||||
self.published_at = self.published_at.astimezone(UTC)
|
||||
self.title = (self.title or "").strip()
|
||||
self.excerpt = (self.excerpt or "").strip()
|
||||
if not self.title and self.excerpt:
|
||||
# Social posts have no title. Use the first line, capped.
|
||||
first = self.excerpt.split("\n", 1)[0]
|
||||
self.title = first[:200] + ("…" if len(first) > 200 else "")
|
||||
|
||||
@property
|
||||
def uid(self) -> str:
|
||||
return hashlib.sha1(
|
||||
f"{self.source}\x00{self.source_id}".encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"TERMINAL",
|
||||
"CascadeProgress",
|
||||
"Item",
|
||||
"ItemLink",
|
||||
"Job",
|
||||
"JobState",
|
||||
"JobsSnapshot",
|
||||
"JsonDict",
|
||||
"JsonIds",
|
||||
"Model",
|
||||
"ModelTaskResult",
|
||||
"Notice",
|
||||
"NoticeKind",
|
||||
"NoticeLevel",
|
||||
"Run",
|
||||
"TaskProgress",
|
||||
]
|
||||
Reference in new issue
Block a user