"""The shapes newsdesk hands round: an item, a link, a result from the model. One set of models, not two — there is no ORM here, so the row the table returns and the object the API serves are the same declaration. The queue's own shapes — `Job`, `Run`, `JobsSnapshot`, `Notice` and the `Model` base they share — live in `pipeline/models.py`, because the pipeline is a folder that gets copied into other projects and cannot depend on this one. They are re-exported here so that `desk.models` stays the one place an app's code looks: the API serves a `Job` next to an `Item`, and which folder each was declared in is not something a route should have to know. """ from __future__ import annotations import hashlib from dataclasses import dataclass, field from datetime import UTC, datetime from typing import Any, Literal from urllib.parse import urlsplit from pydantic import field_validator from pipeline.models import ( TERMINAL, CascadeProgress, Job, JobsSnapshot, JobState, JsonDict, JsonIds, Model, Notice, NoticeKind, NoticeLevel, Run, TaskProgress, ) # ------------------------------------------------------------ model tasks class ModelTaskResult(Model): """What one run of a task for the model did. Never an error: when it can't run it says why. See desk/modeltask.py. Subclasses add the counters that mean something for their task and say which of them is `done`. Everything the runner adds up is here. """ skipped: bool = False pending: int = 0 reason: str | None = None batches: int = 0 #: Batches that asked the model and got nothing back. Kept apart from #: `batches` because a run with 0 done and 0 failed had nothing to do, #: while 0 done and 4 failed is a broken backend — and reporting both as #: "0 done" is exactly the silence this is here to end. failed: int = 0 #: What the model answered that couldn't be used: a slug not in the #: vocabulary, a uid not in the batch, an abstract too short to be one. rejected: list[str] = [] #: Rows this run stopped offering: handed to the model often enough, #: without an answer, that asking again is only spending. Reported rather #: than left to be inferred from a backlog that quietly got smaller — see #: `ModelTask.missed`. given_up: int = 0 #: Which of desk/tools/agent.py's backends answered — `api` or #: `claude_code`, and empty when nothing did. Worth keeping: it is the #: difference between a run that cost money and one that didn't. backend: str = "" @property def done(self) -> int: """How many rows this run finished. The task's own counter.""" raise NotImplementedError def absorb(self, other: ModelTaskResult) -> None: """Add one chunk's result into a running total. Counts add, lists extend; `skipped`, `pending`, `reason` and `backend` are about the run, not the chunk, and are left to the runner. Written once here so a new task's counters count without it saying how. """ for name in type(other).model_fields: if name in ("skipped", "pending", "reason", "backend"): continue mine, theirs = getattr(self, name), getattr(other, name) if isinstance(mine, list): mine.extend(theirs) elif isinstance(mine, int) and not isinstance(mine, bool): setattr(self, name, mine + theirs) class ItemLink(Model): """A source-supplied link with a shared meaning, separate from the destination.""" url: str title: str = "" role: Literal["reading", "discussion", "source"] = "source" @field_validator("url") @classmethod def http_url(cls, value: str) -> str: parsed = urlsplit(value) if parsed.scheme not in {"http", "https"} or not parsed.hostname: raise ValueError("item links must be absolute HTTP(S) URLs") return value @dataclass(slots=True) class Item: """One thing that showed up: an article, a post, a link.""" #: The source's own id for this thing. With `source`, it makes the `uid`, #: so refetching the same thing recognises it rather than duplicating it. source_id: str #: Where the original lives. Any URI — `https://` for a feed entry, #: `file:///…` for something added by hand. location: str title: str #: The short version the source handed over: a feed summary, a post. excerpt: str = "" #: The whole thing, when the source already has it. Left empty by every #: source that doesn't: `ingest.fetch_bodies` looks for an empty body #: and fetches the page with trafilatura, so filling this in is also how a #: source says "don't go and scrape this" — which is what a podcast #: segment wants, its body being a transcript rather than a web page. body: str = "" author: str = "" #: Always set and always UTC once `__post_init__` has run — a plugin may #: omit it, and gets `now`; a plugin that supplies one must make it aware, #: and it is converted. Declared as a plain `datetime` rather than #: `datetime | None` because that is what every reader is handed; the #: `None` only ever exists between the two. #: #: UTC and not merely aware, because this is the one field that is written #: to storage as its own text and then *compared as text* — ordered, #: `MAX`ed, ranged over. A stamp keeping the offset its API sent sorts by #: the digits it happens to start with, so 10:00+02:00 comes out newer than #: 09:00+00:00 when it is an hour older. Converting here rather than at the #: `INSERT` is what makes every reader of the field agree with the column. published_at: datetime = field(default_factory=lambda: datetime.now(UTC)) #: Which feed or handle you follow that produced it. The only attribution #: an item carries: who publishes it — the outlet, and the section of it — #: is a fact about *this subscription*, held once on the subscription row #: and joined when something needs it. It used to be copied onto every #: item, which meant a rename left four hundred rows disagreeing with the #: feed they came from until a repair pass walked the table. subscription: str = "" source: str = "" # plugin name; filled in by the runner #: What this source made, in the app's own words. A free string, and #: deliberately: which shapes exist is the app's business, not the plugin #: contract's. newsdesk reads `article`, `post` and `segment` and lays each #: out its own way; a jobs board on this core will say `role` and `desk` #: has no opinion about either list. #: #: What was *made*, not what carried it — an RSS feed of a podcast, once #: the audio is fetched and cut, yields segments and should say so. This #: used to read `article | video | audio`, which named the medium; the #: medium is not the shape, and every row in the table said `article` #: anyway because no plugin ever set it. kind: str = "article" #: Which of its plugin's declared profiles made this, by name. A plugin #: that declares one uses `default` and never sets this; a show that #: publishes both whole episodes and cut segments names the profile each #: item belongs to, and that recorded name is what decides how the item is #: processed and laid out for the rest of its life. Never worked out later #: from what the item happens to carry — see `desk/sources/spec.py`. profile: str = "default" #: What this is, when it is not an ordinary article: `paywalled`, #: `sponsored`. Usually left alone — `ingest` reads them off the title #: using the tokens the subscription declares (see desk/tools/marks.py). #: A plugin talking to an API that says so outright can set them directly. marks: list[str] = field(default_factory=list) #: The title as it was published. `marks.classify` writes it onto every #: item it looks at, whether or not it cut anything, because a title with #: a token already cut off cannot be reclassified later — the evidence #: goes with it, and "we looked and there was nothing to cut" is a fact #: worth having rather than an absence to re-derive. #: #: Empty therefore means *unknown*: a row stored before originals were #: kept, or one no rule pass has seen. `Store` reads it that way — see the #: unknown-original count `reclassify_marks` reports. source_title: str = "" #: Whatever else the source returned, kept as JSON on the row. One key is #: read by name: `categories`, the source's own labels for this item, which #: the tagger takes as a weak hint. Deliberately not merged into `tags` — #: every outlet's scheme is its own ("Television & radio", "Putin, Vladimir #: V"), and a filter built from the union of all of them is unusable. A #: plugin that sets it caps its own list; a dozen is plenty. raw: dict[str, Any] = field(default_factory=dict) links: list[ItemLink] = field(default_factory=list) def __post_init__(self) -> None: # A plugin passing `published_at=None` explicitly bypasses the default # factory. Unreachable as far as the annotation is concerned, and the # annotation is what every *reader* should go by — but the plugin # contract is a published interface and this used to be allowed, so a # drop-in source under `plugin_paths` must not start raising. if self.published_at is None: self.published_at = datetime.now(UTC) # type: ignore[unreachable] # A naive stamp is a source bug, and the only one of these three that # cannot be repaired here: which hour it means is a fact about the API # that sent it, and this end has nothing to read it off. It used to be # taken as UTC, which is a guess wearing a fixed offset — wrong by up # to a day for anything that sends local time, and wrong silently, # since the row that comes back looks like every other row. A source # that knows its API's convention says so where it parses; see # `desk/sources/base.py`. elif self.published_at.utcoffset() is None: raise ValueError( f"{self.source or 'a source'} gave a naive published_at " f"({self.published_at.isoformat()}) for {self.source_id!r}. " "A source must hand over an aware datetime: attach the offset " "the API sent, or UTC when it documents UTC and says nothing." ) else: self.published_at = self.published_at.astimezone(UTC) self.title = (self.title or "").strip() self.excerpt = (self.excerpt or "").strip() if not self.title and self.excerpt: # Social posts have no title. Use the first line, capped. first = self.excerpt.split("\n", 1)[0] self.title = first[:200] + ("…" if len(first) > 200 else "") @property def uid(self) -> str: return hashlib.sha1( f"{self.source}\x00{self.source_id}".encode() ).hexdigest() __all__ = [ "TERMINAL", "CascadeProgress", "Item", "ItemLink", "Job", "JobState", "JobsSnapshot", "JsonDict", "JsonIds", "Model", "ModelTaskResult", "Notice", "NoticeKind", "NoticeLevel", "Run", "TaskProgress", ]