Files
jobboard/desk/models.py
T
WolfandClaude Opus 5.5 248ea1636d Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:17 +02:00

242 lines
11 KiB
Python

"""The shapes newsdesk hands round: an item, a link, a result from the model.
One set of models, not two — there is no ORM here, so the row the table
returns and the object the API serves are the same declaration.
The queue's own shapes — `Job`, `Run`, `JobsSnapshot`, `Notice` and the
`Model` base they share — live in `pipeline/models.py`, because the pipeline
is a folder that gets copied into other projects and cannot depend on this
one. They are re-exported here so that `desk.models` stays the one place an
app's code looks: the API serves a `Job` next to an `Item`, and which folder
each was declared in is not something a route should have to know.
"""
from __future__ import annotations
import hashlib
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any, Literal
from urllib.parse import urlsplit
from pydantic import field_validator
from pipeline.models import (
TERMINAL, CascadeProgress, Job, JobsSnapshot, JobState, JsonDict, JsonIds,
Model, Notice, NoticeKind, NoticeLevel, Run, TaskProgress,
)
# ------------------------------------------------------------ model tasks
class ModelTaskResult(Model):
"""What one run of a task for the model did. Never an error: when it can't
run it says why. See desk/modeltask.py.
Subclasses add the counters that mean something for their task and say
which of them is `done`. Everything the runner adds up is here.
"""
skipped: bool = False
pending: int = 0
reason: str | None = None
batches: int = 0
#: Batches that asked the model and got nothing back. Kept apart from
#: `batches` because a run with 0 done and 0 failed had nothing to do,
#: while 0 done and 4 failed is a broken backend — and reporting both as
#: "0 done" is exactly the silence this is here to end.
failed: int = 0
#: What the model answered that couldn't be used: a slug not in the
#: vocabulary, a uid not in the batch, an abstract too short to be one.
rejected: list[str] = []
#: Rows this run stopped offering: handed to the model often enough,
#: without an answer, that asking again is only spending. Reported rather
#: than left to be inferred from a backlog that quietly got smaller — see
#: `ModelTask.missed`.
given_up: int = 0
#: Which of desk/tools/agent.py's backends answered — `api` or
#: `claude_code`, and empty when nothing did. Worth keeping: it is the
#: difference between a run that cost money and one that didn't.
backend: str = ""
@property
def done(self) -> int:
"""How many rows this run finished. The task's own counter."""
raise NotImplementedError
def absorb(self, other: ModelTaskResult) -> None:
"""Add one chunk's result into a running total.
Counts add, lists extend; `skipped`, `pending`, `reason` and
`backend` are about the run, not the chunk, and are left to the
runner. Written once here so a new task's counters count without it
saying how.
"""
for name in type(other).model_fields:
if name in ("skipped", "pending", "reason", "backend"):
continue
mine, theirs = getattr(self, name), getattr(other, name)
if isinstance(mine, list):
mine.extend(theirs)
elif isinstance(mine, int) and not isinstance(mine, bool):
setattr(self, name, mine + theirs)
class ItemLink(Model):
"""A source-supplied link with a shared meaning, separate from the destination."""
url: str
title: str = ""
role: Literal["reading", "discussion", "source"] = "source"
@field_validator("url")
@classmethod
def http_url(cls, value: str) -> str:
parsed = urlsplit(value)
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
raise ValueError("item links must be absolute HTTP(S) URLs")
return value
@dataclass(slots=True)
class Item:
"""One thing that showed up: an article, a post, a link."""
#: The source's own id for this thing. With `source`, it makes the `uid`,
#: so refetching the same thing recognises it rather than duplicating it.
source_id: str
#: Where the original lives. Any URI — `https://` for a feed entry,
#: `file:///…` for something added by hand.
location: str
title: str
#: The short version the source handed over: a feed summary, a post.
excerpt: str = ""
#: The whole thing, when the source already has it. Left empty by every
#: source that doesn't: `ingest.fetch_bodies` looks for an empty body
#: and fetches the page with trafilatura, so filling this in is also how a
#: source says "don't go and scrape this" — which is what a podcast
#: segment wants, its body being a transcript rather than a web page.
body: str = ""
author: str = ""
#: Always set and always UTC once `__post_init__` has run — a plugin may
#: omit it, and gets `now`; a plugin that supplies one must make it aware,
#: and it is converted. Declared as a plain `datetime` rather than
#: `datetime | None` because that is what every reader is handed; the
#: `None` only ever exists between the two.
#:
#: UTC and not merely aware, because this is the one field that is written
#: to storage as its own text and then *compared as text* — ordered,
#: `MAX`ed, ranged over. A stamp keeping the offset its API sent sorts by
#: the digits it happens to start with, so 10:00+02:00 comes out newer than
#: 09:00+00:00 when it is an hour older. Converting here rather than at the
#: `INSERT` is what makes every reader of the field agree with the column.
published_at: datetime = field(default_factory=lambda: datetime.now(UTC))
#: Which feed or handle you follow that produced it. The only attribution
#: an item carries: who publishes it — the outlet, and the section of it —
#: is a fact about *this subscription*, held once on the subscription row
#: and joined when something needs it. It used to be copied onto every
#: item, which meant a rename left four hundred rows disagreeing with the
#: feed they came from until a repair pass walked the table.
subscription: str = ""
source: str = "" # plugin name; filled in by the runner
#: What this source made, in the app's own words. A free string, and
#: deliberately: which shapes exist is the app's business, not the plugin
#: contract's. newsdesk reads `article`, `post` and `segment` and lays each
#: out its own way; a jobs board on this core will say `role` and `desk`
#: has no opinion about either list.
#:
#: What was *made*, not what carried it — an RSS feed of a podcast, once
#: the audio is fetched and cut, yields segments and should say so. This
#: used to read `article | video | audio`, which named the medium; the
#: medium is not the shape, and every row in the table said `article`
#: anyway because no plugin ever set it.
kind: str = "article"
#: Which of its plugin's declared profiles made this, by name. A plugin
#: that declares one uses `default` and never sets this; a show that
#: publishes both whole episodes and cut segments names the profile each
#: item belongs to, and that recorded name is what decides how the item is
#: processed and laid out for the rest of its life. Never worked out later
#: from what the item happens to carry — see `desk/sources/spec.py`.
profile: str = "default"
#: What this is, when it is not an ordinary article: `paywalled`,
#: `sponsored`. Usually left alone — `ingest` reads them off the title
#: using the tokens the subscription declares (see desk/tools/marks.py).
#: A plugin talking to an API that says so outright can set them directly.
marks: list[str] = field(default_factory=list)
#: The title as it was published. `marks.classify` writes it onto every
#: item it looks at, whether or not it cut anything, because a title with
#: a token already cut off cannot be reclassified later — the evidence
#: goes with it, and "we looked and there was nothing to cut" is a fact
#: worth having rather than an absence to re-derive.
#:
#: Empty therefore means *unknown*: a row stored before originals were
#: kept, or one no rule pass has seen. `Store` reads it that way — see the
#: unknown-original count `reclassify_marks` reports.
source_title: str = ""
#: Whatever else the source returned, kept as JSON on the row. One key is
#: read by name: `categories`, the source's own labels for this item, which
#: the tagger takes as a weak hint. Deliberately not merged into `tags` —
#: every outlet's scheme is its own ("Television & radio", "Putin, Vladimir
#: V"), and a filter built from the union of all of them is unusable. A
#: plugin that sets it caps its own list; a dozen is plenty.
raw: dict[str, Any] = field(default_factory=dict)
links: list[ItemLink] = field(default_factory=list)
def __post_init__(self) -> None:
# A plugin passing `published_at=None` explicitly bypasses the default
# factory. Unreachable as far as the annotation is concerned, and the
# annotation is what every *reader* should go by — but the plugin
# contract is a published interface and this used to be allowed, so a
# drop-in source under `plugin_paths` must not start raising.
if self.published_at is None:
self.published_at = datetime.now(UTC) # type: ignore[unreachable]
# A naive stamp is a source bug, and the only one of these three that
# cannot be repaired here: which hour it means is a fact about the API
# that sent it, and this end has nothing to read it off. It used to be
# taken as UTC, which is a guess wearing a fixed offset — wrong by up
# to a day for anything that sends local time, and wrong silently,
# since the row that comes back looks like every other row. A source
# that knows its API's convention says so where it parses; see
# `desk/sources/base.py`.
elif self.published_at.utcoffset() is None:
raise ValueError(
f"{self.source or 'a source'} gave a naive published_at "
f"({self.published_at.isoformat()}) for {self.source_id!r}. "
"A source must hand over an aware datetime: attach the offset "
"the API sent, or UTC when it documents UTC and says nothing."
)
else:
self.published_at = self.published_at.astimezone(UTC)
self.title = (self.title or "").strip()
self.excerpt = (self.excerpt or "").strip()
if not self.title and self.excerpt:
# Social posts have no title. Use the first line, capped.
first = self.excerpt.split("\n", 1)[0]
self.title = first[:200] + ("…" if len(first) > 200 else "")
@property
def uid(self) -> str:
return hashlib.sha1(
f"{self.source}\x00{self.source_id}".encode()
).hexdigest()
__all__ = [
"TERMINAL",
"CascadeProgress",
"Item",
"ItemLink",
"Job",
"JobState",
"JobsSnapshot",
"JsonDict",
"JsonIds",
"Model",
"ModelTaskResult",
"Notice",
"NoticeKind",
"NoticeLevel",
"Run",
"TaskProgress",
]