diff --git a/desk/__init__.py b/desk/__init__.py
new file mode 100644
index 0000000..4345376
--- /dev/null
+++ b/desk/__init__.py
@@ -0,0 +1,33 @@
+"""The shared core: what any desk-shaped app is built from.
+
+An app here is something that pulls things in from sources, runs work over
+them as jobs, asks a model for the parts a model is good at, and shows the
+result. newsdesk is the first one. A library of things added by hand and a
+board of job postings are the next two, and they share everything in this
+package and nothing outside it.
+
+ tools/ machinery with no opinion about the app: a model, an
+ encoder, an HTTP session, a video read as text
+ sources/ the plugin contract, the registry that finds plugins on
+ disk, and the built-in sources
+ models.py the shapes an app hands round — an item, a link — plus the
+ queue's own, re-exported from `pipeline.models`
+ modeltask.py a job that needs a model: chunk the batch, ask, apply the
+ answers, and give up on a row no model will settle rather
+ than paying for it every run
+ notices.py which failures newsdesk will interrupt you about, and the
+ sentence it uses. The queue's half of this is
+ `pipeline/notices.py`
+ config.py the settings sections an app composes into its own config
+
+The job queue used to be `desk/pipeline/`. It is `pipeline/` at the top of
+the repository now, one tree below this one, because it turned out to be the
+part worth copying into other projects and a folder cannot be copied if it
+imports the tree above it. `docs/pipeline-package.md` is why and how.
+
+Nothing in here imports from an app. `tests/test_architecture.py` holds it
+to that, and to one door per package: from outside, `desk.sources` is
+imported by that name and nothing behind it.
+"""
+
+__version__ = "0.1.0"
diff --git a/desk/config.py b/desk/config.py
new file mode 100644
index 0000000..ec5c511
--- /dev/null
+++ b/desk/config.py
@@ -0,0 +1,161 @@
+"""The settings sections an app composes into its own config.
+
+`Section` is the base: a block of YAML validated into a model, with an
+unrecognised key being an error rather than a silent no-op. `AgentConfig`
+is how a model is reached and `ModelTaskConfig` is how a task for one is
+paced — both read by the pipeline, so both live here rather than in any one
+app's config. An app declares its own `Config(Section)` with these as fields
+and whatever else it needs next to them.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal, Protocol, get_args, get_origin
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+#: The bare words YAML 1.1 resolves to each boolean, in the order a field gets
+#: asked about them. See `Section._spell_yaml_booleans_back`.
+_YAML_WORDS = {False: ("off", "no", "false"), True: ("on", "yes", "true")}
+
+
+def _literal_strings(annotation: Any) -> set[str]:
+ """Every string a `Literal` annotation accepts, through any wrapping."""
+ if get_origin(annotation) is Literal:
+ return {a for a in get_args(annotation) if isinstance(a, str)}
+ return {s for arg in get_args(annotation) for s in _literal_strings(arg)}
+
+
+class Section(BaseModel):
+ """A block of settings.
+
+ `extra="forbid"` is the whole point: an unrecognised key is a mistake — a
+ typo, or a setting that was renamed and never updated — and saying so is
+ more useful than ignoring it. `protected_namespaces=()` because pydantic
+ reserves the `model_` prefix for its own methods and two settings here are
+ honestly named `model` and `model_weight`.
+ """
+
+ model_config = ConfigDict(extra="forbid", protected_namespaces=())
+
+ @model_validator(mode="before")
+ @classmethod
+ def _spell_yaml_booleans_back(cls, data: Any) -> Any:
+ """Let a setting whose value is spelled `off` be written as `off`.
+
+ YAML 1.1 reads the bare words `off no false on yes true` as booleans,
+ so `backend: off` arrives here as `False` and fails against a `str`
+ field — with an error that helpfully lists `off` as one of the values
+ it will accept. The documented spelling being the one spelling that
+ doesn't work is not a thing to answer with a note in the example file;
+ `agent.backend: off` is what the example and the module that reads it
+ both tell you to write.
+
+ So a boolean landing on a field whose `Literal` lists a word for that
+ boolean is read as the word. Nothing else moves: `off` on a field that
+ has no `off` to offer is still an error, and a real `bool` field is
+ untouched, because `True` and `False` are what it asked for.
+ """
+ if not isinstance(data, dict):
+ return data
+ spelled = data
+ for name, value in data.items():
+ if not isinstance(value, bool):
+ continue
+ field = cls.model_fields.get(name)
+ if field is None:
+ continue
+ choices = _literal_strings(field.annotation)
+ word = next((w for w in _YAML_WORDS[value] if w in choices), None)
+ if word is not None:
+ # A copy, because this dict is the caller's — `load` hands the
+ # parsed YAML straight in, and reading a file shouldn't edit it.
+ spelled = dict(spelled) if spelled is data else spelled
+ spelled[name] = word
+ return spelled
+
+
+class AgentConfig(Section):
+ """Who answers when a job needs a model. See desk/tools/agent.py."""
+
+ #: `auto` prefers the API when there are credentials and otherwise drives
+ #: the `claude` CLI on your subscription through the Claude Agent SDK.
+ #: Naming one skips the detection; `off` means no model is reachable, so
+ #: every batch that needs one fails with a notice saying which switch did
+ #: it. A stage whose batches all fail still reports and the cascade
+ #: carries on — turning the model off stops the model, not the pull.
+ backend: Literal["auto", "api", "claude_code", "off"] = "auto"
+ #: Overrides the per-task model everywhere. Empty means each job picks its
+ #: own — tagging asks for Haiku, because it is Haiku work.
+ model: str = ""
+ #: Seconds for one call, either backend. The Claude Code path spawns a
+ #: process and is the slower of the two by a few seconds.
+ timeout: float = Field(300.0, gt=0)
+ max_tokens: int = Field(8192, ge=1)
+ #: Where the `claude` binary is, if it isn't on PATH.
+ cli_path: str = ""
+ credentials_database: str = Field("", exclude=True, repr=False)
+
+
+class ModelTaskConfig(Section):
+ """What every task for the model is governed by. See desk/modeltask.py."""
+
+ #: Off is off everywhere: the pull, the button and `run("tag")` alike all
+ #: find nothing to do and say so. Not a refusal — a task that is off has
+ #: an empty backlog, so a cascade carries straight on past it.
+ enabled: bool = True
+ #: Whether a *pull* runs it, and nothing else — the button never reads
+ #: this. `auto` runs it during a pull when `agent.backend` resolves to
+ #: something and skips it quietly when it does not, so a machine with no
+ #: model pulls in silence rather than filing a failure per batch about
+ #: being set up the way it was meant to be. `always` runs it either way,
+ #: and a missing backend fails each batch saying so. `never` keeps it out
+ #: of the pull altogether, leaving the button.
+ on_ingest: Literal["auto", "always", "never"] = "auto"
+ model: str = ""
+ #: Whether the model reasons before it answers. Per *task* rather than per
+ #: backend on purpose: both backends take the same shapes, so how hard to
+ #: think is a property of the work, not of how the model is reached.
+ #:
+ #: `off` by default because neither task here is thinking work and the cost
+ #: is not marginal — tagging twenty-five headlines measured 128s with
+ #: thinking and 17s without, for tags that came back the same. Turn it on
+ #: for a task whose answer is a judgement rather than a lookup.
+ #:
+ #: `effort` is deliberately not offered. The CLI has it and the Messages
+ #: API does not, and a setting that silently means nothing on one backend
+ #: is exactly the asymmetry this seam exists to avoid.
+ thinking: Literal["off", "adaptive"] = "off"
+ #: Rows per model call — how the task slices its own backlog, and the one
+ #: number here that is genuinely the task's. Twenty-five is what one call
+ #: and one schema hold; it is prompt economics, and no generic queue could
+ #: know it.
+ #:
+ #: How many run at once is not here and should not be: that is how many
+ #: workers there are, and two tasks that both want a model share them
+ #: rather than each helping itself. See `workers` on the app's config.
+ batch_size: int = Field(25, ge=1)
+
+
+class WorkersConfig(Section):
+ """How many workers of each kind the queue has.
+
+ A property of the machine and the subscription, not of any one task, which
+ is why it is here and not next to `batch_size`. `llm` is what one Claude
+ subscription answers without queueing behind itself; `http` is wider
+ because a fetch is mostly waiting; `cpu` is one because the ranker and the
+ embedder want the machine. A resource nobody names gets one worker.
+ """
+
+ llm: int = Field(4, ge=1)
+ http: int = Field(8, ge=1)
+ cpu: int = Field(1, ge=1)
+
+
+class HasAgent(Protocol):
+ """What the pipeline asks of an app's config: that it has an `agent`
+ section. `modeltask` reads nothing else from it — the rest is handed to
+ the task's own `settings` callable, which knows the app."""
+
+ @property
+ def agent(self) -> AgentConfig: ...
diff --git a/desk/models.py b/desk/models.py
new file mode 100644
index 0000000..19d1bd7
--- /dev/null
+++ b/desk/models.py
@@ -0,0 +1,241 @@
+"""The shapes newsdesk hands round: an item, a link, a result from the model.
+
+One set of models, not two — there is no ORM here, so the row the table
+returns and the object the API serves are the same declaration.
+
+The queue's own shapes — `Job`, `Run`, `JobsSnapshot`, `Notice` and the
+`Model` base they share — live in `pipeline/models.py`, because the pipeline
+is a folder that gets copied into other projects and cannot depend on this
+one. They are re-exported here so that `desk.models` stays the one place an
+app's code looks: the API serves a `Job` next to an `Item`, and which folder
+each was declared in is not something a route should have to know.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import Any, Literal
+from urllib.parse import urlsplit
+
+from pydantic import field_validator
+
+from pipeline.models import (
+ TERMINAL, CascadeProgress, Job, JobsSnapshot, JobState, JsonDict, JsonIds,
+ Model, Notice, NoticeKind, NoticeLevel, Run, TaskProgress,
+)
+
+
+# ------------------------------------------------------------ model tasks
+
+class ModelTaskResult(Model):
+ """What one run of a task for the model did. Never an error: when it can't
+ run it says why. See desk/modeltask.py.
+
+ Subclasses add the counters that mean something for their task and say
+ which of them is `done`. Everything the runner adds up is here.
+ """
+
+ skipped: bool = False
+ pending: int = 0
+ reason: str | None = None
+ batches: int = 0
+ #: Batches that asked the model and got nothing back. Kept apart from
+ #: `batches` because a run with 0 done and 0 failed had nothing to do,
+ #: while 0 done and 4 failed is a broken backend — and reporting both as
+ #: "0 done" is exactly the silence this is here to end.
+ failed: int = 0
+ #: What the model answered that couldn't be used: a slug not in the
+ #: vocabulary, a uid not in the batch, an abstract too short to be one.
+ rejected: list[str] = []
+ #: Rows this run stopped offering: handed to the model often enough,
+ #: without an answer, that asking again is only spending. Reported rather
+ #: than left to be inferred from a backlog that quietly got smaller — see
+ #: `ModelTask.missed`.
+ given_up: int = 0
+ #: Which of desk/tools/agent.py's backends answered — `api` or
+ #: `claude_code`, and empty when nothing did. Worth keeping: it is the
+ #: difference between a run that cost money and one that didn't.
+ backend: str = ""
+
+ @property
+ def done(self) -> int:
+ """How many rows this run finished. The task's own counter."""
+ raise NotImplementedError
+
+ def absorb(self, other: ModelTaskResult) -> None:
+ """Add one chunk's result into a running total.
+
+ Counts add, lists extend; `skipped`, `pending`, `reason` and
+ `backend` are about the run, not the chunk, and are left to the
+ runner. Written once here so a new task's counters count without it
+ saying how.
+ """
+ for name in type(other).model_fields:
+ if name in ("skipped", "pending", "reason", "backend"):
+ continue
+ mine, theirs = getattr(self, name), getattr(other, name)
+ if isinstance(mine, list):
+ mine.extend(theirs)
+ elif isinstance(mine, int) and not isinstance(mine, bool):
+ setattr(self, name, mine + theirs)
+
+
+class ItemLink(Model):
+ """A source-supplied link with a shared meaning, separate from the destination."""
+
+ url: str
+ title: str = ""
+ role: Literal["reading", "discussion", "source"] = "source"
+
+ @field_validator("url")
+ @classmethod
+ def http_url(cls, value: str) -> str:
+ parsed = urlsplit(value)
+ if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+ raise ValueError("item links must be absolute HTTP(S) URLs")
+ return value
+
+
+@dataclass(slots=True)
+class Item:
+ """One thing that showed up: an article, a post, a link."""
+
+ #: The source's own id for this thing. With `source`, it makes the `uid`,
+ #: so refetching the same thing recognises it rather than duplicating it.
+ source_id: str
+ #: Where the original lives. Any URI — `https://` for a feed entry,
+ #: `file:///…` for something added by hand.
+ location: str
+ title: str
+ #: The short version the source handed over: a feed summary, a post.
+ excerpt: str = ""
+ #: The whole thing, when the source already has it. Left empty by every
+ #: source that doesn't: `ingest.fetch_bodies` looks for an empty body
+ #: and fetches the page with trafilatura, so filling this in is also how a
+ #: source says "don't go and scrape this" — which is what a podcast
+ #: segment wants, its body being a transcript rather than a web page.
+ body: str = ""
+ author: str = ""
+ #: Always set and always UTC once `__post_init__` has run — a plugin may
+ #: omit it, and gets `now`; a plugin that supplies one must make it aware,
+ #: and it is converted. Declared as a plain `datetime` rather than
+ #: `datetime | None` because that is what every reader is handed; the
+ #: `None` only ever exists between the two.
+ #:
+ #: UTC and not merely aware, because this is the one field that is written
+ #: to storage as its own text and then *compared as text* — ordered,
+ #: `MAX`ed, ranged over. A stamp keeping the offset its API sent sorts by
+ #: the digits it happens to start with, so 10:00+02:00 comes out newer than
+ #: 09:00+00:00 when it is an hour older. Converting here rather than at the
+ #: `INSERT` is what makes every reader of the field agree with the column.
+ published_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+ #: Which feed or handle you follow that produced it. The only attribution
+ #: an item carries: who publishes it — the outlet, and the section of it —
+ #: is a fact about *this subscription*, held once on the subscription row
+ #: and joined when something needs it. It used to be copied onto every
+ #: item, which meant a rename left four hundred rows disagreeing with the
+ #: feed they came from until a repair pass walked the table.
+ subscription: str = ""
+ source: str = "" # plugin name; filled in by the runner
+ #: What this source made, in the app's own words. A free string, and
+ #: deliberately: which shapes exist is the app's business, not the plugin
+ #: contract's. newsdesk reads `article`, `post` and `segment` and lays each
+ #: out its own way; a jobs board on this core will say `role` and `desk`
+ #: has no opinion about either list.
+ #:
+ #: What was *made*, not what carried it — an RSS feed of a podcast, once
+ #: the audio is fetched and cut, yields segments and should say so. This
+ #: used to read `article | video | audio`, which named the medium; the
+ #: medium is not the shape, and every row in the table said `article`
+ #: anyway because no plugin ever set it.
+ kind: str = "article"
+ #: Which of its plugin's declared profiles made this, by name. A plugin
+ #: that declares one uses `default` and never sets this; a show that
+ #: publishes both whole episodes and cut segments names the profile each
+ #: item belongs to, and that recorded name is what decides how the item is
+ #: processed and laid out for the rest of its life. Never worked out later
+ #: from what the item happens to carry — see `desk/sources/spec.py`.
+ profile: str = "default"
+ #: What this is, when it is not an ordinary article: `paywalled`,
+ #: `sponsored`. Usually left alone — `ingest` reads them off the title
+ #: using the tokens the subscription declares (see desk/tools/marks.py).
+ #: A plugin talking to an API that says so outright can set them directly.
+ marks: list[str] = field(default_factory=list)
+ #: The title as it was published. `marks.classify` writes it onto every
+ #: item it looks at, whether or not it cut anything, because a title with
+ #: a token already cut off cannot be reclassified later — the evidence
+ #: goes with it, and "we looked and there was nothing to cut" is a fact
+ #: worth having rather than an absence to re-derive.
+ #:
+ #: Empty therefore means *unknown*: a row stored before originals were
+ #: kept, or one no rule pass has seen. `Store` reads it that way — see the
+ #: unknown-original count `reclassify_marks` reports.
+ source_title: str = ""
+ #: Whatever else the source returned, kept as JSON on the row. One key is
+ #: read by name: `categories`, the source's own labels for this item, which
+ #: the tagger takes as a weak hint. Deliberately not merged into `tags` —
+ #: every outlet's scheme is its own ("Television & radio", "Putin, Vladimir
+ #: V"), and a filter built from the union of all of them is unusable. A
+ #: plugin that sets it caps its own list; a dozen is plenty.
+ raw: dict[str, Any] = field(default_factory=dict)
+ links: list[ItemLink] = field(default_factory=list)
+
+ def __post_init__(self) -> None:
+ # A plugin passing `published_at=None` explicitly bypasses the default
+ # factory. Unreachable as far as the annotation is concerned, and the
+ # annotation is what every *reader* should go by — but the plugin
+ # contract is a published interface and this used to be allowed, so a
+ # drop-in source under `plugin_paths` must not start raising.
+ if self.published_at is None:
+ self.published_at = datetime.now(UTC) # type: ignore[unreachable]
+ # A naive stamp is a source bug, and the only one of these three that
+ # cannot be repaired here: which hour it means is a fact about the API
+ # that sent it, and this end has nothing to read it off. It used to be
+ # taken as UTC, which is a guess wearing a fixed offset — wrong by up
+ # to a day for anything that sends local time, and wrong silently,
+ # since the row that comes back looks like every other row. A source
+ # that knows its API's convention says so where it parses; see
+ # `desk/sources/base.py`.
+ elif self.published_at.utcoffset() is None:
+ raise ValueError(
+ f"{self.source or 'a source'} gave a naive published_at "
+ f"({self.published_at.isoformat()}) for {self.source_id!r}. "
+ "A source must hand over an aware datetime: attach the offset "
+ "the API sent, or UTC when it documents UTC and says nothing."
+ )
+ else:
+ self.published_at = self.published_at.astimezone(UTC)
+ self.title = (self.title or "").strip()
+ self.excerpt = (self.excerpt or "").strip()
+ if not self.title and self.excerpt:
+ # Social posts have no title. Use the first line, capped.
+ first = self.excerpt.split("\n", 1)[0]
+ self.title = first[:200] + ("…" if len(first) > 200 else "")
+
+ @property
+ def uid(self) -> str:
+ return hashlib.sha1(
+ f"{self.source}\x00{self.source_id}".encode()
+ ).hexdigest()
+
+
+__all__ = [
+ "TERMINAL",
+ "CascadeProgress",
+ "Item",
+ "ItemLink",
+ "Job",
+ "JobState",
+ "JobsSnapshot",
+ "JsonDict",
+ "JsonIds",
+ "Model",
+ "ModelTaskResult",
+ "Notice",
+ "NoticeKind",
+ "NoticeLevel",
+ "Run",
+ "TaskProgress",
+]
diff --git a/desk/modeltask.py b/desk/modeltask.py
new file mode 100644
index 0000000..9588178
--- /dev/null
+++ b/desk/modeltask.py
@@ -0,0 +1,505 @@
+"""A task for the model: rows that are waiting, one call per chunk, answers
+written back. The machine tagging and summarising both run in.
+
+`agent.py` answers *how do I reach a model* — the API, or the local `claude`.
+This module answers the layer above: *what do I send it, and what do I do with
+what comes back*. Everything between those two lives here, once.
+
+ ModelTask the parts: waiting, build_batch, system, schema, apply,
+ missed
+ one one batch: one model call, one apply. What a job is.
+ of that as a `Task`: the batches, and how to find them
+ run every batch here and now, for a caller with no queue
+
+Tagging and summarising used to be near-twins — the same loop, the same
+failure handling, the same job body and the same resume, spelled out twice
+and drifting by a line each merge. What actually differs between them is
+small and is all *content*: which rows are waiting, what goes in the batch,
+what the model is told, what shape it must answer in, what to do with the
+answer. That is a `ModelTask`; this module is everything that is not one.
+
+The third one — clustering, or whatever comes next — is a prompt, a schema
+and two functions over the store, and no loop.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+from desk.config import HasAgent, ModelTaskConfig
+from pipeline.context import NULL, RunContext
+from pipeline.task import Batch, Tally, Task, resolve
+from desk.models import ModelTaskResult, Notice
+from desk.notices import classify, stops_the_run
+from desk.tools import agent
+
+log = logging.getLogger(__name__)
+
+#: `notices.py`'s code for "the model's safeguards would not take this text".
+#: Named here because `one` has to recognise it by more than its wording: it
+#: is the only failure that says something about a particular row, and so the
+#: only one worth asking about again with fewer rows.
+REFUSED = "agent.refused"
+
+
+@dataclass(frozen=True, slots=True)
+class ModelTask[R: ModelTaskResult]:
+ """What one task for the model consists of. See tagging.TASK for one."""
+
+ #: The job kind.
+ kind: str
+ label: str
+ #: For progress lines and the idle result: "tagging" / "tags" /
+ #: "nothing untagged".
+ verb: str
+ noun: str
+ idle: str
+ #: The key under which `build_batch` lists its rows. Both of newsdesk's
+ #: say "items", and they used to say "articles" and "segments": a batch is
+ #: a mixed archive and naming it after one shape was a claim about the
+ #: rows in it that stopped being true.
+ items: str
+ #: Which config section governs it.
+ settings: Callable[[Any], ModelTaskConfig]
+ #: How many rows are waiting: `(config, store)`.
+ pending: Callable[[Any, Any], int]
+ #: Which rows are waiting, by uid, most recent first: `(config, store, limit)`.
+ #: What the task splits over — one batch per model call — so it asks for
+ #: identifiers rather than the payload, which each batch builds for itself
+ #: when it runs.
+ waiting: Callable[[Any, Any, int], list[str]]
+ #: The payload the model works from, its prompt included:
+ #: `(config, store, limit, uids)`. Given uids it builds that batch, and
+ #: asks the store again rather than trusting the list — a row may have been
+ #: done in the seconds since the task shared the work out.
+ build_batch: Callable[..., dict[str, Any]]
+
+ # All three take the config because *which rows are waiting* is no longer a
+ # question the store can answer on its own: summarising is governed by what
+ # each installed plugin declares about its own items, and the plugins are
+ # found through the config. A task that does not care ignores the argument.
+ #: The system prompt, given the store rather than fixed, because tagging's
+ #: carries the vocabulary and that grows between one run and the next.
+ system: Callable[[Any], str]
+ schema: dict[str, Any]
+ #: Write an answer back: `(store, answer, batch, config)`. Tolerant of a
+ #: sloppy response, and the only place the answer's shape is trusted.
+ apply: Callable[[Any, dict[str, Any], dict[str, Any] | None, Any], R]
+ #: `(store, uids, why="")`: the model was handed these rows and said
+ #: nothing usable about them. Count that against each one still waiting,
+ #: and answer how many of them that has now given up on. `why` is
+ #: `refused` when the safeguards turned the row down, so the Given up list
+ #: can say so.
+ #:
+ #: Every task needs one and there is no default, because the default is the
+ #: bug. A row the model cannot satisfy — a segment whose only honest
+ #: abstract is shorter than the guard, a uid it drops from every reply — is
+ #: otherwise found by the next `waiting`, sent again, and paid for again,
+ #: on every pull, for as long as the row exists. Where the count is kept is
+ #: the store's business; that there is a bound at all is this module's.
+ missed: Callable[..., int]
+ #: The result type, so totals start as the right kind of zero.
+ result: type[R]
+ #: `module:function` run once when every batch of the run is over, given
+ #: `(config, store, tally)` and the run context. "" for a task with
+ #: nothing to settle, which summarising is: an abstract is written for one
+ #: segment by the batch that read it, and no other batch has an opinion.
+ #:
+ #: For the decision a batch is in no position to make. Tagging's new tags
+ #: are the case: the cap is *per run*, and a batch that can see neither
+ #: the other thirty-nine nor what they proposed can only enforce it
+ #: forty times over. So a batch reports what it wants, and this is where
+ #: the run's own answer is worked out — once, over the whole tally.
+ #:
+ #: It returns the fields of `result` that settling added, and both paths
+ #: take it from there: queued, the final job reports it; direct, `_fold`
+ #: absorbs it into the total. Same function either way.
+ settle: str = ""
+ #: The field every answer in the reply names its row by. Both tasks say
+ #: `uid`, which is not a coincidence — an answer that cannot say which row
+ #: it is about is not applicable — and counting it in the streamed reply is
+ #: what moves the progress bar per item rather than per batch. A task whose
+ #: answers are not one-per-row sets it to "" and gets the old behaviour.
+ answer_key: str = "uid"
+
+
+class _Answered:
+ """Counts finished answers in a reply that is still arriving.
+
+ The model writes its answers one after another, so the `"uid"` that opens
+ each one is a reliable "another row is done" marker tens of seconds before
+ the request that carries it completes. Counting a substring is a blunt
+ instrument and deliberately so: the alternative is incremental JSON
+ parsing, which is a great deal of machinery to be *approximately as* right
+ about a number that only drives a bar. It cannot overcount — the key
+ appears once per answer, and the payload it is counted in is the model's
+ own output, not the batch we sent — and if it undercounts, the bar catches
+ up when the chunk lands and the real total is known.
+
+ `tail` is why this is a class rather than a closure: a key can be split
+ across two deltas, so the last few characters of each are carried into the
+ next. Carrying `len(needle) - 1` is exactly enough to catch one that spans
+ the join, and too few for anything already counted to be seen twice.
+ """
+
+ def __init__(self, key: str, report: Callable[[int], None]) -> None:
+ self._needle = f'"{key}"'
+ self._report = report
+ self._tail = ""
+ self.count = 0
+
+ def __call__(self, piece: str) -> None:
+ window = self._tail + piece
+ found = window.count(self._needle)
+ self._tail = window[1 - len(self._needle):] if len(self._needle) > 1 else ""
+ if found:
+ self.count += found
+ self._report(self.count)
+
+
+def one[R: ModelTaskResult](task: ModelTask[R], config: HasAgent, store: Any,
+ uids: Sequence[str], *, context: RunContext = NULL) -> R:
+ """One batch: one model call, one apply. What a job is.
+
+ The unit the queue admits. It used to be a whole backlog inside a single
+ job, which opened four threads of its own — so the queue counted one thing
+ running where four requests were in the air, and four model workers was a
+ number nobody was keeping. Now the job and the request are
+ the same thing and the count is true.
+
+ The prompt is built here rather than when the batch was handed out: for
+ tagging that carries the vocabulary as it stands when this batch starts,
+ which is what somebody adding a tag by hand mid-run should reach. What it
+ does *not* carry is what a sibling batch proposed a second ago — nothing
+ of this run's proposals exists yet, because they are settled once, when
+ every batch is over. See `settle`.
+
+ Raises `agent.AgentError` when no model can be reached, or when the one
+ that could gave nothing. The job fails, saying which — and the notice says
+ what to do about it, because both of those are things you fix rather than
+ things to work around.
+
+ A row the model answers is done with. One it leaves out, or answers in a
+ way `apply` will not take, is charged a turn against `task.missed` — so it
+ comes round again a bounded number of times rather than on every pull for
+ as long as it exists.
+
+ A batch the model's safeguards refuse outright is split and asked again in
+ halves rather than failed, because a refusal is about the text of one row
+ and the other rows in it are owed their answers. Down to a single row it
+ is charged like any other unanswered one and the run carries on. See
+ `_narrow`.
+
+ It counts its own rows and nothing else. As a job that is all there is; run
+ through `Task.run` instead, the caller shifts the progress into the whole
+ backlog, which is the only place that knows how big it is.
+ """
+ context.check()
+ cfg = task.settings(config)
+ batch = task.build_batch(config, store, len(uids) or cfg.batch_size,
+ list(uids))
+ rows = batch[task.items]
+ if not rows:
+ # Done by someone else between the task sharing the work out and
+ # now, which is ordinary:
+ # every task rediscovers its backlog and the batches overlap at the
+ # edges. Nothing to ask, nothing to write.
+ return task.result(reason=task.idle)
+
+ name = agent.backend(config.agent)
+ if not name:
+ # The notice goes out from here rather than being left to the failure,
+ # because nothing was asked and so the `except` below never runs. Left
+ # to itself this reached the screen only as "Tagging a batch failed",
+ # filed as a bug, about a machine that is working correctly and is one
+ # install or one config line from working here too. The configured
+ # backend is what is passed, not the one that ran: there wasn't one,
+ # and `off` is the difference between a missing install and a switch.
+ unreachable = agent.AgentError("no model is reachable from here")
+ _report(context, classify(unreachable, backend=config.agent.backend))
+ raise unreachable
+
+ total = len(rows)
+
+ def report(done: int) -> None:
+ done = min(done, total)
+ context.progress(done, total, f"{task.verb} {task.items}, {done} of {total}")
+
+ context.progress(0, total, f"asking {name} about {total} {task.items}")
+ counted = (_Answered(task.answer_key, report) if task.answer_key else None)
+
+ try:
+ context.check()
+ reply = agent.ask_json(
+ config.agent,
+ system=task.system(store),
+ question={task.items: rows},
+ schema=task.schema,
+ model=config.agent.model or cfg.model,
+ thinking=cfg.thinking,
+ using=name,
+ on_partial=counted,
+ )
+ except agent.AgentError as exc:
+ notice = classify(exc, backend=name)
+ # A refusal is the one failure that is about the rows rather than
+ # about the call, and so the one that answers "which of them?" by
+ # being asked again with fewer. Everything else falls through to the
+ # raise below unchanged.
+ if notice.code == REFUSED and len(rows) > 1:
+ return _narrow(task, config, store, _uids(task, rows),
+ context=context)
+ log.exception("%s batch of %d failed on %s", task.verb, len(rows), name)
+ # The notice carries the sentence and the fix — "Claude Code is signed
+ # out" and the command that mends it, rather than the token that
+ # expired — to where you will see it. Then the batch fails: its rows
+ # keep their watermark and come round on the next run — uncharged,
+ # because nothing was asked of a model.
+ #
+ # Its siblings are unaffected only while the failure was this batch's.
+ # When it is the account's, `_report` stops the ones still queued: see
+ # `notices.stops_the_run`.
+ if notice.code == REFUSED:
+ # Down to one row, so the notice can name it — and has to. "The
+ # safeguards wouldn't take this text" is not a sentence anyone can
+ # act on without knowing which of the day's articles it is about,
+ # and the row is about to leave the backlog quietly. `key` is the
+ # title too, so two refusals in one run are two lines rather than
+ # one line with a count of two.
+ about = rows[0].get("title") or _uids(task, rows)[0]
+ notice = notice.model_copy(update={
+ "text": f"{notice.text} — {about}", "key": str(about)})
+ _report(context, notice)
+ if notice.code == REFUSED:
+ # Down to one row, and that row is the one. Charged and not
+ # raised: the rule about never charging a failure is a rule about
+ # failures that prove nothing, and this one has just proved
+ # exactly which row it is about. Left uncharged it is found by
+ # the next `waiting`, sent again, refused again, and poisons
+ # whichever batch it lands in for as long as the row exists —
+ # which is how two articles held eighteen others' abstracts.
+ totals = task.result(reason=notice.text)
+ totals.failed = 1
+ totals.given_up = task.missed(store, _uids(task, rows), why="refused")
+ totals.pending = task.pending(config, store)
+ return totals
+ raise
+
+ # Apply even if killed during the call: the spend has happened. The worker
+ # records cancellation at return while preserving the paid-for answer.
+ totals = task.apply(store, reply.data, batch, config)
+ # Whatever is still waiting now is what the model was asked about and did
+ # not settle, so it is charged one turn — before `pending` is read, so a
+ # row this call gave up on has already left the number the run reports.
+ # Charged on a reply and never on a failure: a batch that never reached a
+ # model cost nothing and proved nothing about its rows.
+ totals.given_up = task.missed(store, uids)
+ totals.batches = 1
+ totals.backend = reply.backend
+ totals.pending = task.pending(config, store)
+ report(total)
+ return totals
+
+
+def _uids(task: ModelTask[Any], rows: Sequence[dict[str, Any]]) -> list[str]:
+ """Which rows a built batch is about, from the batch itself.
+
+ `one` is given uids, but what it sent is `rows` — `build_batch` asks the
+ store again and drops whatever was finished in between. On the path this
+ serves, that difference is the whole point: a row nobody sent is a row
+ nobody may charge.
+ """
+ return [str(row[task.answer_key]) for row in rows if row.get(task.answer_key)]
+
+
+def _narrow[R: ModelTaskResult](task: ModelTask[R], config: HasAgent, store: Any,
+ uids: Sequence[str], *,
+ context: RunContext) -> R:
+ """Ask about these rows again in two halves, and add up what comes back.
+
+ A refusal names no row. It is one answer about the whole request, so the
+ only way to learn which row it is about is to send fewer — and the reason
+ to bother is that the rest of the batch is fine and is owed its abstracts.
+ The split costs a handful of extra calls, once, against one call that
+ summarises nobody and is made again on every pull.
+
+ Halves rather than one row at a time, because a clean half lands whole:
+ the cost is set by how many rows are flagged rather than by how many are
+ in the batch, and two flagged rows in one batch is ordinary — they arrive
+ from the same day's news.
+
+ Each half is a whole `one`, so a half that meets an unrelated failure
+ raises it exactly as it would have unsplit, and the batch fails the way it
+ always did.
+
+ The bad case, if the classifier ever took against the *prompt* rather than
+ any row: every single-row call is refused, every row is charged, and three
+ pulls empty the backlog into given-up. Bounded and visible — the status
+ line counts them and `reopen_given_up` asks again — but it is why the
+ notice names the text rather than the row.
+ """
+ middle = len(uids) // 2
+ totals = task.result()
+ for half in (uids[:middle], uids[middle:]):
+ if not half:
+ continue
+ landed = one(task, config, store, half, context=context)
+ totals.absorb(landed)
+ # `absorb` leaves the fields that are about the run rather than the
+ # chunk, and which backend answered is one of them — the same reading
+ # `_fold` takes of the same shape.
+ totals.backend = landed.backend or totals.backend
+ totals.pending = task.pending(config, store)
+ return totals
+
+
+def _report(context: RunContext, notice: Notice) -> None:
+ """Put one failure on screen, and stop the run if it will meet the same one.
+
+ The two halves belong together because they are one judgement made once. A
+ batch that cannot reach a model, or is told the spend limit is reached, has
+ learned something about the *account* and not about its own rows — and the
+ twenty-two batches queued behind it would each pay a call to be told it
+ again, and each put the same line on screen. One batch fails, one notice
+ goes out, and the rest are cancelled rather than spent.
+
+ Which failures those are is the catalogue's to say and not this module's: a
+ timeout or a reply that was not JSON is this batch's bad luck, and the next
+ batch deserves its turn. See `notices.stops_the_run`.
+ """
+ context.notice(notice)
+ if stops_the_run(notice):
+ context.stop(notice.text)
+
+
+# ----------------------------------------------------------------- the task
+
+def batches(task: ModelTask[Any], limit: int | None = None) -> Any:
+ """The finder: what is waiting, cut into one payload per model call.
+
+ How many to a call is the task's own business — twenty-five articles is
+ what one call and one schema hold, and no generic queue could know it. How
+ many run at once is the queue's, because that is how many workers it has.
+
+ It asks for identifiers rather than the rows, and each batch asks the store
+ again when it runs: a row may have been done in the seconds since the work
+ was shared out.
+
+ `enabled: false` is answered here, and answered as nothing to do rather
+ than as a refusal. Here, because this is the one place every route to the
+ work passes through: the stage of a cascade, the button, and `run("tag")`
+ are three callers of this same finder, and a switch only one of them read
+ would be a switch that turned the task off in one place. As nothing to do,
+ because a task that finds nothing still queues its final job — so a
+ cascade whose tagging is off carries on to whatever follows rather than
+ ending on it.
+ """
+
+ def find(config: HasAgent, store: Any) -> list[Batch]:
+ cfg = task.settings(config)
+ if not cfg.enabled:
+ return []
+ uids = task.waiting(config, store,
+ limit if limit is not None else 100_000)
+ size = max(cfg.batch_size, 1)
+ cut = [uids[at:at + size] for at in range(0, len(uids), size)]
+ return [Batch({"uids": chunk}, label=_label(task, len(chunk), n, len(cut)))
+ for n, chunk in enumerate(cut, start=1)]
+
+ return find
+
+
+def _label(task: ModelTask[Any], rows: int, n: int, of: int) -> str:
+ """What one batch is called in the queue: "Tagging 25 items, 2 of 9".
+
+ The kind's own label is "Tagging a batch" on all nine of them, which is
+ true and useless — the queue lists nine identical rows and nothing says
+ which is which or how much is left. The slice is the only thing that tells
+ them apart, and this is the one place it is known: the finder cut them, and
+ the batch that runs half an hour later knows only its own uids.
+
+ Its shape is the progress line's, capitalised because it is a name rather
+ than a sentence about what is happening. The tail is dropped for a task of
+ one batch, where "1 of 1" says nothing.
+ """
+ said = f"{task.verb} {rows} {task.items}".capitalize()
+ return said if of < 2 else f"{said}, {n} of {of}"
+
+
+def _fold(task: ModelTask[Any]) -> Any:
+ """Add the batches up, for the direct path only. Under the queue each batch
+ is its own job with its own row, and there is nothing to add.
+
+ The settle runs here too, off the same tally shape the final job builds,
+ because "what may this run add" is a question about the run and not about
+ the queue. Otherwise `run("tag")` in a notebook would be the one caller
+ with no cap at all.
+ """
+
+ def fold(config: HasAgent, store: Any, results: list[Any]) -> dict[str, Any]:
+ if not results:
+ return task.result(reason=task.idle).model_dump()
+ totals = task.result()
+ for result in results:
+ if isinstance(result, Exception):
+ # One batch that could not be answered is not a failed run: as
+ # eight jobs it would be one failing and seven landing. Its rows
+ # keep their watermark and come round next time.
+ totals.failed += 1
+ totals.reason = str(result)
+ continue
+ landed = task.result.model_validate(result)
+ totals.absorb(landed)
+ # `absorb` adds up counts and leaves the fields that are about the
+ # run rather than the batch. Which backend answered is one of
+ # those, and the run wants the last one to have said anything.
+ totals.backend = landed.backend or totals.backend
+ if task.settle:
+ answered = tuple(r for r in results if isinstance(r, dict))
+ settled = resolve(task.settle)(
+ config, store, Tally(done=len(answered), results=answered),
+ context=NULL) or {}
+ totals.absorb(task.result.model_validate(settled))
+ # After the settle, so a tag it added — and the rows it went back and
+ # tagged — are counted in what the run reports it did.
+ totals.pending = task.pending(config, store)
+ return totals.model_dump()
+
+ return fold
+
+
+def of(task: ModelTask[Any], *, limit: int | None = None, name: str = "",
+ callback: str = "", carries: dict[str, Any] | None = None) -> Task:
+ """The `Task` for one model task: its batches, and how to find them.
+
+ A `ModelTask` is the *content* — which rows are waiting, what the model is
+ told, what to do with the answer. This is that content as a step of the
+ pipeline, which is the same shape as every other step and is queued and
+ killed the same way.
+ """
+ return Task(
+ kind=f"{task.kind}_batch", name=name or task.kind,
+ finder=batches(task, limit), callback=callback, settle=task.settle,
+ carries=dict(carries or {}),
+ size=lambda payload: len(payload.get("uids") or ()),
+ fold=_fold(task),
+ )
+
+
+def run[R: ModelTaskResult](task: ModelTask[R], config: HasAgent, store: Any,
+ limit: int | None = None, *, context: RunContext = NULL) -> dict[str, Any]:
+ """Every batch, here and now, with no queue. The direct path and nothing else.
+
+ What a Claude Code session, a notebook and a test get: one call that blocks
+ and hands back the total. Under the queue this same task is `of(...)`
+ started against the queue, its batches are independent jobs, and there is
+ no total until the final job reads how they went — which is the whole point
+ of splitting it.
+
+ Same `one` either way, and the same batches from the same finder, so there
+ is no second implementation to drift.
+ """
+ return of(task, limit=limit).run(config, store, context=context)
diff --git a/desk/notices.py b/desk/notices.py
new file mode 100644
index 0000000..b2c54ad
--- /dev/null
+++ b/desk/notices.py
@@ -0,0 +1,382 @@
+"""What newsdesk will interrupt you about, and the sentence it uses.
+
+The generic half of this — the notice a failed job gets, the check that stops
+it being said twice, the rendering a terminal wants — is `pipeline/notices.py`,
+and the pipeline uses only that. This module is the half that could only be
+newsdesk's: a catalogue of the ways *its* model backends, *its* feeds and *its*
+config keys fail, and what to tell you to do about each.
+
+`classify` is the interface. Everything else is the table it reads.
+
+Deliberately string-matching rather than exception-type-matching. These errors
+arrive as text through two SDKs and a subprocess, and half of them are
+`ResultMessage.errors` from a CLI that owes us no stable exception hierarchy.
+Matching prose is the honest reflection of that. The fallback is a notice that
+still says something and is marked `bug` — because a failure nobody wrote a
+rule for is one to go and read about, and the property that matters most is
+that nothing fails silently.
+
+`job_failed`, `already_said` and `render` are re-exported so that a caller
+reaching for newsdesk's notices gets the whole vocabulary from one import
+rather than having to know which half a given name lives in.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Iterable
+
+from pipeline.models import Notice, NoticeKind, NoticeLevel
+from pipeline.notices import (
+ already_said, job_failed, render, whose_fault as _whose_fault,
+)
+
+log = logging.getLogger(__name__)
+
+
+
+class Rule:
+ """One row of the catalogue: how to recognise a failure and what to say.
+
+ `per_backend` exists for the one case where the same failure wants
+ genuinely different advice depending on who hit it. A 401 means "run
+ `claude auth login`" on the subscription path and "check your
+ ANTHROPIC_API_KEY" on the API one, and telling you to re-run a CLI login
+ when the problem is an environment variable is worse than saying nothing.
+
+ Its keys are usually the backend that ran. `agent.unreachable` is the
+ exception, because there nobody ran: the caller passes the configured
+ `agent.backend` instead, and `off` is a real answer that wants its own
+ sentence — "you turned this off" is not "you have nothing installed".
+
+ `standing` says the cause is the account or the machine rather than this
+ batch, so every sibling batch of the same run will be told exactly the
+ same thing. A spend limit does not lift between one call and the next and
+ a signed-out CLI does not sign itself back in; a timeout or a reply that
+ was not JSON says nothing at all about the next batch. It is what a run
+ reads to stop before it spends the other twenty-two calls — see
+ `stops_the_run`.
+ """
+
+ __slots__ = ("code", "fix", "kind", "level", "patterns", "per_backend",
+ "standing", "text")
+
+ def __init__(self, code: str, level: NoticeLevel, kind: NoticeKind,
+ text: str, fix: str, patterns: Iterable[str],
+ per_backend: dict[str, tuple[str, str]] | None = None,
+ *, standing: bool = False) -> None:
+ self.code = code
+ self.level = level
+ self.kind = kind
+ self.text = text
+ self.fix = fix
+ self.patterns = tuple(patterns)
+ self.per_backend = per_backend or {}
+ self.standing = standing
+
+ def matches(self, error: str) -> bool:
+ return any(p in error for p in self.patterns)
+
+ def say(self, backend: str) -> tuple[str, str]:
+ return self.per_backend.get(backend, (self.text, self.fix))
+
+
+#: The catalogue, most specific first — `classify` takes the first match, so
+#: "out of credit" must be tried before the generic "the API refused".
+#:
+#: Two backends produce these and they word things differently, so most rules
+#: carry patterns from both: `credit balance is too low` is the API's phrasing,
+#: `usage limit reached` is the subscription CLI's, and they are the same
+#: problem from where you are sitting.
+#:
+#: Every one of these is `expected`. That is not an accident — between them
+#: they are the ways a model call fails when nothing is wrong with newsdesk,
+#: and the point of enumerating them is precisely so that what falls through
+#: can be called a bug with a straight face.
+RULES: tuple[Rule, ...] = (
+ Rule(
+ "agent.auth", "error", "expected",
+ "Claude Code is signed out, so nothing that needs a model can run.",
+ # Two places, two fixes. At a keyboard the login is interactive; in the
+ # container there is nobody to prompt, so the CLI is authenticated by a
+ # token in the server's env file and that is the thing to replace. One
+ # line naming only the login sends whoever is holding the server to a
+ # command that cannot work where they are running it.
+ "Run `claude auth login` here, or paste a fresh token from "
+ "`claude setup-token` into Settings. New calls use it without a restart.",
+ ("oauth", "unauthor", "authentication", "not logged in", "sign in",
+ "log in", "login", "expired", "401", "invalid_api_key",
+ "invalid api key", "could not resolve authentication"),
+ {"api": ("The Claude API rejected the key.",
+ "Check ANTHROPIC_API_KEY, or set agent.backend: claude_code "
+ "in config.yaml to use your subscription instead.")},
+ standing=True,
+ ),
+ Rule(
+ "agent.credit", "error", "expected",
+ "The Claude API account is out of credit.",
+ "Top up at console.anthropic.com, or set agent.backend: claude_code "
+ "in config.yaml to use your subscription instead.",
+ ("credit balance", "insufficient_quota", "insufficient quota",
+ "billing", "payment"),
+ standing=True,
+ ),
+ # The subscription CLI has more than one way of saying "not until it
+ # resets", and the patterns have to carry all of them, because what falls
+ # through here is filed as a defect of ours. "You've hit your monthly
+ # spend limit · raise it at claude.ai/settings/usage?from=cc_cli_limit_
+ # message · your weekly limit resets 6am" matched none of the original
+ # words — `usage` occurs in it only inside the URL — so twenty-three
+ # batches of one run each reported a working machine as a bug. The text
+ # arrives verbatim from the CLI (`desk/tools/agent.py:_explain`), so
+ # matching its own words is the only thing there is to match.
+ Rule(
+ "agent.limit", "warning", "expected",
+ "Claude's usage limit is reached — tagging will pick up where it left "
+ "off once it resets.",
+ "Nothing to do; it resets on its own. A spend limit can be raised at "
+ "claude.ai/settings/usage.",
+ ("usage limit", "spend limit", "spend cap", "limit reached",
+ "limit resets", "weekly limit", "monthly limit", "hourly limit",
+ "hit your limit", "rate_limit", "rate limit", "429",
+ "too many requests", "overloaded", "quota"),
+ standing=True,
+ ),
+ # A refusal about the *rows*, not about us and not about the account. The
+ # platform runs a classifier over the request, and a news archive walks
+ # into it regularly: reporting on a worm or a supply-chain compromise is
+ # written in the vocabulary the classifier watches for, and an outlet that
+ # covers security covers it weekly.
+ #
+ # Deliberately not `standing`. The account is fine and the next batch is
+ # about different rows, so stopping the run here would throw away every
+ # abstract the rest of the backlog was about to get. What it costs instead
+ # is one split — see `modeltask.one`, which re-asks the batch in halves so
+ # the flagged row is found and charged rather than taking its batch-mates
+ # down with it.
+ Rule(
+ "agent.refused", "warning", "expected",
+ # Short, because the title is appended to it and the two together are
+ # what the band shows — on a phone, over the article being read. What
+ # it means and what to do are the fix's, which is behind `more` there
+ # and open in the log. It says "item" and not "abstract": tagging
+ # meets the same refusal.
+ "The model's safeguards refused this item",
+ # The URL spelled out rather than left to the raw error. The one
+ # decision this notice exists to inform — whether the verification is
+ # worth applying for — cannot be made from a line that says a link
+ # exists.
+ "Usually security reporting, and the item itself is still here and "
+ "readable. Nothing to do: after three tries it is left alone rather "
+ "than sent again on every pull, it is said on screen only the first "
+ "time, and Jobs & history lists it under Given up with a Try again. "
+ "These stop for an account that has applied to "
+ "Anthropic's Cyber Verification Program — "
+ "https://support.claude.com/en/articles/"
+ "14604842-real-time-cyber-safeguards-on-claude",
+ ("[cyber]", "safeguards flagged", "cyber verification program",
+ "flagged by safeguards"),
+ ),
+ Rule(
+ "agent.output", "warning", "expected",
+ "The model's answer was longer than agent.max_tokens allows, so the "
+ "batch came back with nothing.",
+ "Raise agent.max_tokens in config.yaml, or lower the task's batch_size "
+ "so there is less to answer in one call.",
+ ("output token maximum", "max_output_tokens", "max output tokens",
+ "claude_code_max_output_tokens", "exceed your max_tokens"),
+ ),
+ Rule(
+ "agent.key", "error", "expected",
+ "No Claude API key, and no local `claude` to fall back on.",
+ "Set ANTHROPIC_API_KEY, or install Claude Code and run "
+ "`claude auth login`.",
+ ("no model backend", "anthropic sdk isn't installed",
+ "claude agent sdk isn't installed"),
+ standing=True,
+ ),
+ # The two halves of what a job that cannot reach a model has to be able to
+ # say — "none is reachable" and "the one there is would not answer".
+ # (`docs/pipeline-levels.md` used to promise both; the file is gone and
+ # the promise is not.) Both used to fall past every rule into the
+ # fallback, so a scratch box with `backend: off` and a machine with no
+ # `claude` on PATH each reported every batch as a defect of ours and sent
+ # you to a traceback about a program that is working correctly.
+ Rule(
+ "agent.unreachable", "error", "expected",
+ "No model is reachable from here, so nothing that needs one can run.",
+ "Install Claude Code and run `claude auth login`, or set "
+ "ANTHROPIC_API_KEY.",
+ ("no model is reachable",),
+ {"off": ("agent.backend is `off`, so nothing that needs a model runs.",
+ "Set agent.backend to `auto` in config.yaml, or name `api` "
+ "or `claude_code` outright.")},
+ standing=True,
+ ),
+ Rule(
+ "agent.cli", "error", "expected",
+ "Claude Code is the backend, but there is no `claude` here to run.",
+ "Install the CLI — `irm https://claude.ai/install.ps1 | iex` on "
+ "Windows, `npm install -g @anthropic-ai/claude-code` elsewhere — or "
+ "point agent.cli_path at one in config.yaml.",
+ ("claude code not found", "cli path not resolved"),
+ standing=True,
+ ),
+ Rule(
+ "agent.timeout", "warning", "expected",
+ "The model didn't answer in time.",
+ "Usually a slow moment; it retries on the next run. Raise "
+ "agent.timeout in config.yaml if it keeps happening.",
+ ("no answer within", "timeout", "timed out"),
+ ),
+ Rule(
+ "agent.offline", "warning", "expected",
+ "Couldn't reach Claude — the network looks down.",
+ "",
+ ("connection", "getaddrinfo", "name or service not known",
+ "temporary failure in name resolution", "network is unreachable",
+ "ssl", "certificate"),
+ ),
+ Rule(
+ "agent.response", "warning", "bug",
+ "The model answered with something that wasn't the JSON asked for.",
+ "",
+ ("no json object", "wasn't valid json", "wasn't a json object",
+ "returned no result"),
+ ),
+)
+
+#: Where an unrecognised failure lands, and marked `bug` on purpose: the rules
+#: above are the ways this fails when nothing is wrong, so falling past all of
+#: them is itself the signal. Better to send you to the log for something that
+#: turns out to be mundane than to file a real defect under "expected".
+FALLBACK = Rule(
+ "agent.failed", "warning", "bug",
+ "A model call failed for a reason newsdesk doesn't recognise.",
+ "", (),
+)
+
+#: Exception names that mean the *remote end* misbehaved rather than that we
+#: mishandled it. A feed that 403s or times out is news about the feed; a
+#: KeyError parsing its XML is news about us. Everything not listed is treated
+#: as ours, which is the safer default — a bug filed as expected is a bug that
+#: never gets looked at.
+
+
+def classify(error: BaseException | str, *, backend: str = "",
+ key: str = "") -> Notice:
+ """The notice for a model-backend failure.
+
+ `backend` is folded into the sentence when it is known, because "Claude
+ Code is signed out" and "the API rejected the key" want different hands on
+ different keyboards, and the caller is the only one that knows which ran.
+ When nothing ran at all, the caller passes the configured `agent.backend`
+ instead — see `Rule.per_backend`.
+ """
+ detail = str(error).strip()
+ rule = next((r for r in RULES if r.matches(detail.lower())), FALLBACK)
+ text, fix = rule.say(backend)
+ if rule is FALLBACK:
+ text = _unrecognised(backend, detail)
+ return Notice(
+ code=rule.code, level=rule.level, kind=rule.kind, text=text, fix=fix,
+ detail=detail, key=key, where="the tagging lines in the server log",
+ )
+def stops_the_run(notice: Notice) -> bool:
+ """Whether the rest of this run should be abandoned rather than attempted.
+
+ True for the failures a `Rule` marks `standing`: the ones caused by the
+ account or the machine, which will answer every sibling batch of the run
+ identically. The batch that met it has already failed; what this decides
+ is whether the twenty-two behind it are worth the calls.
+
+ It is a question about the notice rather than about the exception because
+ the notice is where the catalogue's judgement already is. A caller that
+ classified a failure to say something useful about it has, by then, been
+ told everything it needs to know about whether to carry on.
+ """
+ return any(rule.code == notice.code and rule.standing for rule in RULES)
+
+
+def source_failed(source: str, error: BaseException | str) -> Notice:
+ """A subscription that blew up mid-fetch.
+
+ Keyed by source, so a feed that has been failing all week collapses into
+ one line rather than one per pull — and, more to the point, is *visible*,
+ which a source quietly returning nothing has never been. That is the one
+ failure you cannot spot by reading the feed: a dead source and a quiet news
+ day look identical.
+
+ One failure gets its own sentence, because the general one sends you the
+ wrong way. When YouTube refuses the machine the show's feed is fine, so
+ the Check job would report every subscription healthy; what is wrong is
+ the address the request left from. See `source_blocked`.
+ """
+ if any(said in str(error).lower() for said in BLOCKED):
+ return source_blocked(source, error)
+ return Notice(
+ code="source.failed", level="warning", kind=_whose_fault(error),
+ key=source,
+ text=f"{source} failed to fetch — its items are missing from this pull.",
+ fix="Run the Check job to see which subscriptions still resolve.",
+ detail=str(error).strip(),
+ where=f"the `source {source} blew up mid-fetch` traceback in the log",
+ )
+
+
+#: How `desk.tools.video.youtube.Blocked` reads once it is text. Matched as
+#: prose like everything else here, and for one more reason: `notices` sits
+#: below `tools` in the layering, so the type is not this module's to import.
+BLOCKED = ("youtube refused",)
+
+
+def source_blocked(source: str, error: BaseException | str) -> Notice:
+ """A source YouTube would not serve, because of where the request came from.
+
+ Always `expected`: nothing in newsdesk is wrong, and the episodes are not
+ lost. They were deliberately not marked done, so the first pull from an
+ address YouTube accepts reads them. Until then this is said on every pull,
+ which is the point — it used to be an INFO line that retried for ever.
+ """
+ return Notice(
+ code="source.blocked", level="warning", kind="expected", key=source,
+ text=f"YouTube refused {source} — it takes this address for a bot, "
+ "so no episodes were read.",
+ fix="Usually a VPN or a datacentre address: pull again without it. "
+ "Nothing is lost; unread episodes are tried on every pull.",
+ detail=str(error).strip(),
+ where=f"the `source {source} blew up mid-fetch` traceback in the log",
+ )
+
+
+
+def _unrecognised(backend: str, detail: str) -> str:
+ """The fallback's sentence, with the failure's own words in it.
+
+ It used to say only "failed for a reason newsdesk doesn't recognise",
+ which is true and useless. This is precisely the case where nobody wrote a
+ rule, so the raw text is the only thing anyone has to go on — and it was
+ reachable only by pressing `why` on a notice that fades in half a minute,
+ or by going to the log.
+
+ Trimmed to the first line and a screenful, because a stack trace does not
+ belong in a strip. `Notice.detail` still carries every character of it.
+ """
+ who = _name(backend) if backend else "A model call"
+ said = detail.splitlines()[0].strip() if detail else ""
+ if len(said) > 160:
+ said = said[:159].rstrip() + "…"
+ if not said:
+ return f"{who} failed for a reason newsdesk doesn't recognise."
+ return f"{who} failed: {said}"
+
+
+def _name(backend: str) -> str:
+ return {"api": "The Claude API",
+ "claude_code": "Claude Code"}.get(backend, backend or "The model")
+
+
+__all__ = [
+ "BLOCKED", "FALLBACK", "RULES", "Rule", "already_said", "classify",
+ "job_failed", "render", "source_blocked", "source_failed", "stops_the_run",
+]
diff --git a/desk/sources/__init__.py b/desk/sources/__init__.py
new file mode 100644
index 0000000..ff78e37
--- /dev/null
+++ b/desk/sources/__init__.py
@@ -0,0 +1,175 @@
+"""The plugins: where items come from. One door, used from both sides.
+
+ from desk.sources import Source, Item # writing one
+ from desk.sources import build, available # running them
+
+A plugin is a file with a `Source` subclass in it — `base.py` is the contract
+— dropped into this package or into a directory under `plugin_paths:` in
+config.yaml. It yields `Item`s and touches nothing else: no store, no queue,
+no config beyond its own `options`. What it needs to remember between pulls
+goes in `self.state`, a mapping the runner backs with the database. That is
+what makes a new source one file, and what lets everything above treat every
+source the same way.
+
+ base.py `Source`, `Item`, `Suggestion`, `CheckResult`, and the
+ helpers every plugin ends up wanting
+ spec.py `PluginSpec`: what a plugin *declares* — its profiles, the
+ shared tools that acquire its content, what the shared
+ summariser and tagger should do with it, how it is laid out
+ registry.py finds the plugins on disk and builds the configured ones
+ bluesky.py, mastodon.py, x.py, podcast.py the built-in ones
+ rss.py feed fetching, which a publisher's plugin subclasses
+
+Declaring a `spec` is optional in the contract and every plugin installed here
+declares one: `spec_of` adapts a plugin that declares none, and such a plugin
+runs exactly as it always has, but nothing about it is a claim its author made.
+
+`PodcastSource` and `RssSource` are exported next to `Source` because that is
+how a publisher's plugin is written: subclass the class that already does the
+fetching, give it a name and a `spec`, and add nothing else.
+`plugins/lastweekin_ai.py` and `plugins/guardian.py` are the two models.
+`RssSource` is `shared` and is therefore not a plugin itself — there is no
+generic "Any RSS feed" source to paste a masthead into.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Protocol
+from desk.models import Item, ItemLink
+from desk.sources.base import (
+ MULTIWORD_OUTLETS, USER_AGENT, CheckResult, Getter, Source, Suggestion,
+ entry_time, http_failure, outlet_name, parse_timestamp, split_label, strip_html,
+)
+from desk.sources.podcast import PodcastSource, segment_profile
+from desk.sources.registry import available
+from desk.sources.rss import RssSource, check_feed
+from desk.sources.spec import (
+ API_VERSION, ARTICLE_READER, FEED_READER, LINK_READER, POST_READER,
+ RECORDING_READER, RSS, Discovery,
+ ExternalLink, MarkRule, OriginalView, PluginSpec, Presentation, Profile,
+ QueueLayout, RetryPolicy, Section, Step, SummaryPolicy, TaggingPolicy,
+ Target, TargetSpec, Tasks, View, Vocabulary, Wall, problems, spec_of,
+)
+
+__all__ = [
+ "API_VERSION",
+ "ARTICLE_READER",
+ "FEED_READER",
+ "LINK_READER",
+ "MULTIWORD_OUTLETS",
+ "POST_READER",
+ "RECORDING_READER",
+ "RSS",
+ "USER_AGENT",
+ "CheckResult",
+ "Discovery",
+ "ExternalLink",
+ "Getter",
+ "Item",
+ "ItemLink",
+ "MarkRule",
+ "OriginalView",
+ "PluginSpec",
+ "PodcastSource",
+ "Presentation",
+ "Profile",
+ "QueueLayout",
+ "RetryPolicy",
+ "RssSource",
+ "Section",
+ "Source",
+ "SourcesConfig",
+ "Step",
+ "Suggestion",
+ "SummaryPolicy",
+ "TaggingPolicy",
+ "Target",
+ "TargetSpec",
+ "Tasks",
+ "View",
+ "Vocabulary",
+ "Wall",
+ "available",
+ "build",
+ "check_feed",
+ "entry_time",
+ "faults",
+ "http_failure",
+ "outlet_name",
+ "parse_timestamp",
+ "problems",
+ "segment_profile",
+ "spec_of",
+ "split_label",
+ "strip_html",
+]
+
+log = logging.getLogger(__name__)
+
+
+class SourcesConfig(Protocol):
+ """What `build` needs from an app's config: where plugins are found, and
+ the section each configured one gets. Anything with these two attributes
+ will do; newsdesk's `Config` is one."""
+
+ @property
+ def plugin_paths(self) -> list[str]: ...
+
+ @property
+ def sources(self) -> dict[str, Any]: ...
+
+
+def build(config: SourcesConfig,
+ vocabulary: Vocabulary = Vocabulary()) -> list[Source]:
+ """Instantiate every source that is both installed and configured.
+
+ `vocabulary` is what this application supports, and an empty one — the
+ default — restricts nothing, which is every caller today. When an app
+ passes its own, a plugin whose declared spec names a tool, kind or
+ component the app does not have is left out with its diagnostics in the
+ log, and the others are built. One outlet's mistake is not an evening
+ without news.
+
+ A plugin that declares no spec is never left out on these grounds: there
+ is nothing in an adapted spec that anyone wrote. See `spec.spec_of`.
+ """
+ classes = available(config.plugin_paths)
+ instances = []
+ for name, options in config.sources.items():
+ cls = classes.get(name)
+ if cls is None:
+ log.warning("config has `sources.%s` but no plugin defines it (have: %s)",
+ name, ", ".join(sorted(classes)) or "none")
+ continue
+ if broken := faults(cls, vocabulary):
+ log.error("source %s is not loadable here and is disabled:\n %s",
+ name, "\n ".join(broken))
+ continue
+ try:
+ instance = cls(options or {})
+ except Exception:
+ log.exception("source %s could not be constructed; continuing", name)
+ continue
+ if not instance.enabled:
+ log.info("source %s is disabled in config", name)
+ continue
+ instances.append(instance)
+ return instances
+
+
+def faults(cls: type[Source], vocabulary: Vocabulary) -> list[str]:
+ """Why this application cannot run this plugin, or nothing.
+
+ A spec that will not even build is caught here rather than at import,
+ because a plugin is free to construct its spec in `__init__`; either way
+ the plugin is disabled and the reason is the message.
+
+ Exported because a plugin left out of a pull is also a row in a list of
+ installed plugins, saying why it is unavailable — and asking that
+ question twice, in two places, is how the two answers start to differ.
+ """
+ try:
+ return problems(spec_of(cls), vocabulary)
+ except Exception as exc:
+ return [f"its spec could not be read: {exc}"]
diff --git a/desk/sources/base.py b/desk/sources/base.py
new file mode 100644
index 0000000..90f9d4f
--- /dev/null
+++ b/desk/sources/base.py
@@ -0,0 +1,325 @@
+"""The plugin contract.
+
+Every source lives in its own file. Drop a .py file into `desk/sources/`
+or into any directory listed under `plugin_paths:` in config.yaml, define a
+subclass of `Source`, and it will be discovered automatically.
+
+Minimum viable plugin:
+
+ from desk.sources import Source, Item
+
+ class MySource(Source):
+ name = "mysource"
+ description = "One line shown in the Sources tab."
+
+ def fetch(self):
+ yield Item(
+ source_id="unique-per-source",
+ location="https://example.com/story",
+ title="Headline",
+ excerpt="Summary or body",
+ author="someone",
+ published_at=datetime.now(timezone.utc), # aware; stored as UTC
+ )
+
+`self.options` is whatever you put under `sources.mysource:` in the config —
+by the time a pull runs, its subscription list has been replaced with the
+fetch-enabled rows from the Sources tab (see newsdesk/jobs/subscriptions.py).
+
+Two optional methods round out the contract, both with do-nothing defaults:
+`discover` proposes accounts to follow, and `check` says whether each
+subscription still resolves. Neither is required for a source to be useful.
+"""
+
+from __future__ import annotations
+
+import re
+from abc import ABC, abstractmethod
+from collections.abc import Iterator, MutableMapping
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any, ClassVar
+
+from desk import __version__
+from desk.models import Item
+from desk.tools.scrape import Getter
+
+#: Sent by every plugin that talks HTTP, so an outlet reading its logs can see
+#: who this is. (`desk/tools/framing.py` deliberately sends a browser string
+#: instead — it is asking what a browser would be told, so it has to ask as
+#: one.)
+USER_AGENT = f"desk/{__version__} (personal aggregator)"
+
+_HTML_TAGS = re.compile(r"<[^>]+>")
+_WHITESPACE = re.compile(r"\s+")
+#: Tidying up after the line above — see `strip_html`. Deliberately only the
+#: punctuation that always closes: an apostrophe or a dash can legitimately
+#: have a space in front of it.
+_SPACE_BEFORE_PUNCT = re.compile(r" +([.,;:!?%)\]}])")
+_SPACE_AFTER_OPENING = re.compile(r"([(\[{]) +")
+
+
+def parse_timestamp(value: str) -> datetime:
+ """An ISO-8601 stamp from a social API, as an aware datetime, or now.
+
+ Every one of these APIs writes UTC as a trailing `Z`, which
+ `datetime.fromisoformat` did not accept before 3.11 and which is cheaper to
+ translate than to special-case. A missing or malformed stamp is not worth
+ dropping a post over: `now` sorts it to the top of the queue, where you
+ will see it and can judge it yourself.
+
+ A stamp with no offset at all is read as UTC, and that is a decision this
+ helper is entitled to make where `Item` is not. Here we know whose stamp it
+ is — an API that documents UTC and then omits the `Z` on some of its rows —
+ whereas the model is handed a bare datetime by anything at all, and a wrong
+ guess there is an item filed hours out with nothing on the row to show it.
+ A plugin whose API sends local time should not be reaching for this.
+ """
+ try:
+ parsed = datetime.fromisoformat((value or "").replace("Z", "+00:00"))
+ except (ValueError, AttributeError):
+ return datetime.now(UTC)
+ return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
+
+
+def entry_time(entry: Any) -> datetime:
+ """When a feedparser entry was published, or now if it doesn't say.
+
+ feedparser hands back a 9-tuple in UTC under `published_parsed`, falling
+ back to `updated_parsed`. Three modules used to do this slice on their
+ own; the tuple is sliced to six here, once.
+ """
+ for key in ("published_parsed", "updated_parsed"):
+ parsed = entry.get(key)
+ if parsed:
+ return datetime(*parsed[:6], tzinfo=UTC) # type: ignore[misc] # 9-tuple, sliced to 6
+ return datetime.now(UTC)
+
+
+def http_failure(exc: Exception) -> str:
+ """One line for a failed request, for a check result: the status when the
+ server answered, the exception's name when it didn't."""
+ response = getattr(exc, "response", None)
+ if response is not None:
+ return f"HTTP {response.status_code}"
+ return f"unreachable: {type(exc).__name__}"
+
+
+def strip_html(html: str) -> str:
+ """Tags out, whitespace collapsed. Feeds and Mastodon both hand us HTML.
+
+ A tag becomes a *space* rather than nothing, because `
` has to stop
+ the last word of one paragraph running into the first word of the next.
+ That is right for block tags and wrong for inline ones — `a link.`
+ would come out as "a link ." — so the space is taken back off where it
+ landed against punctuation. It reads better, and the ranker and the tagger
+ are both looking at these strings.
+ """
+ text = _WHITESPACE.sub(" ", _HTML_TAGS.sub(" ", html or "")).strip()
+ return _SPACE_BEFORE_PUNCT.sub(r"\1", _SPACE_AFTER_OPENING.sub(r"\1", text))
+
+
+#: Outlets whose name is more than one word, so "Al Jazeera Sport" doesn't
+#: become outlet "Al". Anything not listed splits on the first space, which is
+#: right for "Guardian Tech" and "BBC World". Set `outlet:`/`section:` on the
+#: feed in config.yaml when you want neither guess.
+MULTIWORD_OUTLETS = (
+ "Al Jazeera", "Hacker News", "Ars Technica", "Washington Post",
+ "Financial Times", "New York Times", "Wall Street Journal",
+ "Der Spiegel", "Le Monde", "Foreign Policy", "MIT Technology Review",
+)
+
+
+def split_label(label: str) -> tuple[str, str]:
+ """'Guardian Tech' -> ('Guardian', 'Tech'). Fallback when config is silent.
+
+ A reading of a feed's *label*, which is why it is part of the source
+ contract rather than of any store: the RSS plugin applies it to a feed it
+ was given with no outlet, and an app applies it to a subscription seeded
+ the same way. One rule, and the plugin side owns it.
+ """
+ label = (label or "").strip()
+ if not label:
+ return "", ""
+ for outlet in MULTIWORD_OUTLETS:
+ if label == outlet:
+ return outlet, ""
+ if label.startswith(outlet + " "):
+ return outlet, label[len(outlet):].strip()
+ head, _, rest = label.partition(" ")
+ return head, rest.strip()
+
+
+#: What sits between a site's name and its tagline, for `outlet_name`. Spelled
+#: with escapes because a literal en dash next to a hyphen inside a character
+#: class is unreadable, and ruff is right to say so.
+_TAGLINE = re.compile("\\s+[-\u2013\u2014|:\u00b7]\\s+")
+
+
+def outlet_name(raw: str) -> str:
+ """A name a site gave for itself, cut back to something a sidebar can hold.
+
+ Both places that ask a server what it is called get the SEO title back,
+ tagline and all: WordPress keeps ODSC's as "Open Data Science - Your News
+ Source for AI, Machine Learning & more", and a feed's `
` is often
+ "Ars Technica - All content". Everything after the first separator is the
+ tagline, and the name is the part in front of it.
+
+ A reading of what a *server* said, which is why it is part of the source
+ contract next to `split_label` — that one reads a label you wrote, this
+ one reads a string you never saw.
+ """
+ return _TAGLINE.split((raw or "").strip(), 1)[0].strip() or (raw or "").strip()
+
+
+@dataclass(slots=True)
+class Suggestion:
+ """An account you don't follow yet, that the graph thinks you should."""
+
+ handle: str
+ display_name: str = ""
+ reason: str = ""
+ score: float = 0.0
+ url: str = ""
+
+
+@dataclass(slots=True)
+class CheckResult:
+ """Whether one subscription still resolves, and what the server said it is.
+
+ `target` is the feed URL or the handle — what the subscription row holds —
+ and `label` is what *you* call it, so the Jobs view can name a failing row.
+
+ `outlet` and `section` are the other half: what the server calls itself.
+ They are how a plugin that learns a publisher's real name from the network
+ hands it over — WordPress reads it off `/wp-json`, a feed off its own
+ `` — and they are written onto the subscription, once, by
+ `name_grouping`. Empty from every plugin that has nothing to add, which is
+ most of them: an outlet is normally known from the label before anything
+ is fetched.
+
+ Deliberately *not* `label`. That field already carries the subscription's
+ own name in every plugin and in the Jobs view's failing-row line, and a
+ publisher's name is a different fact about a different thing.
+ """
+
+ kind: str
+ target: str
+ ok: bool
+ detail: str
+ label: str = ""
+ outlet: str = ""
+ section: str = ""
+
+
+class Source(ABC):
+ """Base class for every ingest plugin."""
+
+ #: config key under `sources:` in config.yaml. Required, must be unique.
+ name: str = ""
+ #: True for a class that exists to be subclassed rather than to be run:
+ #: `RssSource` is the fetching every publisher's plugin inherits, not a
+ #: plugin of its own. The registry passes over one, so there is no
+ #: "Any RSS feed" row to paste a masthead into — see `desk.sources.rss`.
+ #:
+ #: Read off the class's own `__dict__` and never inherited, or declaring
+ #: it once would take every plugin that subclasses that class with it.
+ shared: ClassVar[bool] = False
+ #: shown in the Sources tab, under Plugins
+ description: str = ""
+ #: Which key under `sources.:` holds this plugin's subscriptions, and
+ #: therefore whether one is a URL or a handle. Declared here rather than
+ #: inferred, because everything downstream needs the answer: what
+ #: `subscriptions.apply` puts back into the config for a pull, what the Add
+ #: box in the Sources tab asks for, and what the UI validates.
+ #:
+ #: This used to be `name == "rss"` spelled out at each of those sites,
+ #: which meant the second URL-shaped plugin silently had its feed list
+ #: replaced by an empty account list and fetched nothing.
+ subscription_field: ClassVar[str] = "accounts" # "accounts" or "feeds"
+ #: The outlet every subscription of this plugin sits under, when the plugin
+ #: is a network rather than a publisher — a Bluesky handle is a channel of
+ #: Bluesky, not a masthead of its own. Written onto a subscription when it
+ #: is added, and only over a name nobody chose; `check` overrides it for a
+ #: plugin that can ask the network what a subscription is really called.
+ #: Empty means "read it off the label", which is right for a feed.
+ default_outlet: str = ""
+ #: The addresses this plugin knows how to read better than a web page, and
+ #: how: a `Getter` whose `handles` matches them and whose `claims` puts it
+ #: ahead of the plain request. The body fetch collects these from every
+ #: active plugin and hands them to `desk.tools.scrape`, so a Hacker News
+ #: link to a Bluesky post is read through Bluesky's API while the Bluesky
+ #: plugin is on, and as a web page while it is off. A getter only gets the
+ #: page: which link an item follows is decided when the plugin makes it.
+ getters: ClassVar[tuple[Getter, ...]] = ()
+
+ def __init__(self, options: dict[str, Any] | None = None) -> None:
+ self.options = options or {}
+ #: What this plugin remembers between pulls — a spend counter, which
+ #: episodes are already done — as a plain mapping of strings. A pull
+ #: hands in one backed by the database; driven directly (a test, the
+ #: dry-run `main` in a plugin file) it is this dict and lives as long
+ #: as the instance. It is the only thing a plugin is given by the
+ #: application, which is what keeps a plugin one file.
+ self.state: MutableMapping[str, str] = {}
+
+ @property
+ def enabled(self) -> bool:
+ return bool(self.options.get("enabled", True))
+
+ @abstractmethod
+ def fetch(self) -> Iterator[Item]:
+ """Yield everything new. Runner handles dedup, so over-fetching is fine."""
+ raise NotImplementedError
+
+ def discover(self) -> list[Suggestion]:
+ """Optional: propose accounts to follow. Return [] if unsupported."""
+ return []
+
+ def check(self) -> list[CheckResult]:
+ """Optional: does each subscription still resolve, and what is it called?
+
+ One result per feed or account, and never a paid request — the `check`
+ job runs this for every source at once. Return [] if there is nothing
+ to ask.
+
+ This is also the naming channel. A plugin that can only learn a
+ publisher's real name by asking the network says so here, in the
+ result's `outlet`/`section`, and the app writes it onto the
+ subscription. It is asked of one target at a time when you add a
+ subscription — build the source with that one entry in its options —
+ so a URL you paste comes back named.
+ """
+ return []
+
+ def params_for(self, target: str) -> dict[str, Any]:
+ """What this subscription asks for, when it is a query and not an address.
+
+ A feed URL and a handle need nothing; a search takes keywords, and
+ maybe a location or a floor. Those live on the subscription, not in the
+ plugin's own settings, because they are what distinguishes one of your
+ saved searches from the next — and so that adding one is the same act
+ as adding a feed, with the same switches on it afterwards.
+ """
+ for entry in self.options.get(self.subscription_field) or []:
+ if isinstance(entry, dict):
+ this = str(entry.get("url") or entry.get("handle") or "")
+ if this == target:
+ got = entry.get("params")
+ return dict(got) if isinstance(got, dict) else {}
+ return {}
+
+ def targets(self) -> list[tuple[str, str]]:
+ """The subscriptions this instance was configured with, as
+ `(target, label)` pairs — a URL or a handle, and what it is called.
+ A bare string in the list is its own label."""
+ out = []
+ for entry in self.options.get(self.subscription_field) or []:
+ if isinstance(entry, dict):
+ target = str(entry.get("url") or entry.get("handle") or "")
+ label = str(entry.get("label") or target)
+ else:
+ target = label = str(entry)
+ if target:
+ out.append((target, label))
+ return out
diff --git a/desk/sources/bluesky.py b/desk/sources/bluesky.py
new file mode 100644
index 0000000..d5542cc
--- /dev/null
+++ b/desk/sources/bluesky.py
@@ -0,0 +1,242 @@
+"""Bluesky via the public AppView. No auth, no key, no bill.
+
+Public app.bsky.* endpoints can be called directly against
+https://public.api.bsky.app, which is what this plugin does. Endpoints used:
+
+ app.bsky.feed.getAuthorFeed - posts by one account
+ app.bsky.graph.getFollows - who an account follows (for discovery)
+ app.bsky.graph.getSuggestedFollowsByActor - accounts similar to a given one
+"""
+
+from __future__ import annotations
+
+import logging
+from collections import defaultdict
+from collections.abc import Iterator
+from typing import Any
+
+import httpx
+
+from desk.sources.base import (
+ USER_AGENT, CheckResult, Getter, Item, Source, Suggestion, http_failure,
+ parse_timestamp,
+)
+from desk.sources.spec import (
+ API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy,
+ TaggingPolicy, TargetSpec, Tasks,
+)
+from desk.tools import posts
+
+log = logging.getLogger(__name__)
+API = "https://public.api.bsky.app/xrpc"
+
+
+class BlueskySource(Source):
+ name = "bluesky"
+ description = "Posts from Bluesky accounts (free public API)."
+ default_outlet = "Bluesky"
+ #: A `bsky.app/profile/.../post/...` link from any outlet is read through
+ #: the same public API, not as the web app's script shell.
+ getters = (Getter("bluesky_api", posts.bluesky, posts.is_bluesky, claims=True),)
+
+ #: A network declares no accounts, because which ones you follow are
+ #: yours. `mode="editable"` is that sentence in the contract, and it is
+ #: what puts an Add box under this plugin and nothing but Fetch and Show
+ #: under a publisher's — see §8 of docs/plugin-rework.md.
+ #:
+ #: Both policies are declared as these items already behave rather than
+ #: as anyone would design them fresh, which is step 11's rule that
+ #: silence is not a decision. Posts are tagged today and have been for as
+ #: long as there has been a tagger; a post is already the length an
+ #: abstract would be, so summarising is off and says so.
+ spec = PluginSpec(
+ api_version=API_VERSION, id=name, version="1", label="Bluesky",
+ targets=TargetSpec(mode="editable", kind="account", defaults=[],
+ validator="bluesky_handle"),
+ profiles={"default": Profile(
+ kind="post",
+ # The fetching a link post has always had: `links_only` keeps the
+ # ones that point somewhere, and where they point is extracted
+ # into the field the reader shows under the post itself. A post
+ # linking to nothing carries its own permalink as `location` and
+ # as `raw.post_url`, and the body fetch skips it.
+ acquisition=[Step(id="article", tool="extract_article",
+ inputs={"url": "inputs.original_url"},
+ outputs={"content": "inputs.article_text"})],
+ tasks=Tasks(
+ summary=SummaryPolicy(enabled=False),
+ tagging=TaggingPolicy(
+ enabled=True, input_fields=["inputs.feed_text"],
+ context_fields=["author"]),
+ ),
+ presentation=POST_READER,
+ )},
+ )
+
+ def __init__(self, options: dict[str, Any] | None = None) -> None:
+ super().__init__(options)
+ self.client = httpx.Client(
+ timeout=20,
+ headers={"User-Agent": USER_AGENT},
+ )
+
+ def _get(self, method: str, **params) -> dict[str, Any]:
+ r = self.client.get(f"{API}/{method}", params=params)
+ r.raise_for_status()
+ return r.json()
+
+ # ------------------------------------------------------------- ingest
+
+ def fetch(self) -> Iterator[Item]:
+ limit = int(self.options.get("posts_per_account", 40))
+ skip_replies = self.options.get("skip_replies", True)
+ links_only = self.options.get("links_only", False)
+
+ for handle, label in self.targets():
+ try:
+ data = self._get(
+ "app.bsky.feed.getAuthorFeed",
+ actor=handle,
+ limit=min(limit, 100),
+ filter="posts_no_replies" if skip_replies else "posts_with_replies",
+ )
+ except httpx.HTTPError as exc:
+ log.warning("bluesky: %s failed (%s)", handle, exc)
+ continue
+
+ for entry in data.get("feed", []):
+ post = entry.get("post") or {}
+ record = post.get("record") or {}
+ author = post.get("author") or {}
+ text = record.get("text", "")
+ link = self._external_link(post)
+
+ if links_only and not link:
+ continue
+
+ rkey = (post.get("uri") or "").rsplit("/", 1)[-1]
+ yield Item(
+ source_id=post.get("uri", ""),
+ location=link
+ or f"https://bsky.app/profile/{author.get('handle', handle)}/post/{rkey}",
+ title="", # base class derives one from the text
+ excerpt=text,
+ author=author.get("displayName") or author.get("handle", handle),
+ published_at=parse_timestamp(record.get("createdAt", "")),
+ subscription=label,
+ kind="post",
+ raw={
+ "likes": post.get("likeCount", 0),
+ "reposts": post.get("repostCount", 0),
+ "post_url": f"https://bsky.app/profile/"
+ f"{author.get('handle', handle)}/post/{rkey}",
+ },
+ )
+
+ @staticmethod
+ def _external_link(post: dict[str, Any]) -> str:
+ """Prefer the article a post links to over the post itself."""
+ embed = post.get("embed") or {}
+ external = embed.get("external") or {}
+ if external.get("uri"):
+ return external["uri"]
+ for facet in (post.get("record") or {}).get("facets", []) or []:
+ for feature in facet.get("features", []):
+ if feature.get("$type", "").endswith("#link") and feature.get("uri"):
+ return feature["uri"]
+ return ""
+
+ # --------------------------------------------------------------- check
+
+ def check(self) -> list[CheckResult]:
+ return [self._check_one(handle) for handle, _ in self.targets()]
+
+ def _check_one(self, handle: str) -> CheckResult:
+ try:
+ data = self._get("app.bsky.actor.getProfile", actor=handle)
+ except httpx.HTTPError as exc:
+ return CheckResult("bluesky", handle, False, http_failure(exc))
+ return CheckResult(
+ "bluesky", handle, True,
+ f"{data.get('postsCount', '?')} posts, "
+ f"{data.get('followersCount', '?')} followers",
+ )
+
+ # ----------------------------------------------------------- discovery
+
+ def discover(self) -> list[Suggestion]:
+ settings = self.options.get("discover") or {}
+ if not settings.get("enabled", True):
+ return []
+
+ seeds = [target for target, _ in self.targets()][: settings.get("max_seeds", 25)]
+ if not seeds:
+ return []
+
+ following = {s.lower() for s in seeds}
+ # Fractional, because signal 2 is weighted above signal 1 — so not a
+ # Counter, whose values typeshed (correctly) says are ints.
+ counts: defaultdict[str, float] = defaultdict(float)
+ meta: dict[str, dict[str, str]] = {}
+
+ # Signal 1: accounts that several of your accounts follow.
+ for seed in seeds:
+ for account in self._paged_follows(seed):
+ handle = account.get("handle", "").lower()
+ if not handle or handle in following:
+ continue
+ counts[handle] += 1
+ meta.setdefault(handle, {
+ "display_name": account.get("displayName", ""),
+ "description": (account.get("description") or "")[:160],
+ })
+
+ # Signal 2: Bluesky's own "similar to this account" endpoint.
+ for seed in seeds[:10]:
+ try:
+ data = self._get("app.bsky.graph.getSuggestedFollowsByActor", actor=seed)
+ except httpx.HTTPError:
+ continue
+ for account in data.get("suggestions", []):
+ handle = account.get("handle", "").lower()
+ if not handle or handle in following:
+ continue
+ counts[handle] += 1.5 # weight it slightly above a raw co-follow
+ meta.setdefault(handle, {
+ "display_name": account.get("displayName", ""),
+ "description": (account.get("description") or "")[:160],
+ })
+
+ threshold = settings.get("min_overlap", 3)
+ out = []
+ ranked = sorted(counts.items(), key=lambda kv: -kv[1])[:200]
+ for handle, score in ranked:
+ if score < threshold:
+ break
+ info = meta.get(handle, {})
+ out.append(Suggestion(
+ handle=handle,
+ display_name=info.get("display_name", ""),
+ reason=f"followed by {int(score)} of the accounts you follow",
+ score=float(score),
+ url=f"https://bsky.app/profile/{handle}",
+ ))
+ return out
+
+ def _paged_follows(self, actor: str, cap: int = 400) -> list[dict[str, Any]]:
+ out: list[dict[str, Any]] = []
+ cursor: str | None = None
+ while len(out) < cap:
+ params = {"actor": actor, "limit": 100}
+ if cursor:
+ params["cursor"] = cursor
+ try:
+ data = self._get("app.bsky.graph.getFollows", **params)
+ except httpx.HTTPError as exc:
+ log.warning("bluesky: follows for %s failed (%s)", actor, exc)
+ break
+ out.extend(data.get("follows", []))
+ cursor = data.get("cursor")
+ if not cursor:
+ break
+ return out
diff --git a/desk/sources/mastodon.py b/desk/sources/mastodon.py
new file mode 100644
index 0000000..6a157ef
--- /dev/null
+++ b/desk/sources/mastodon.py
@@ -0,0 +1,196 @@
+"""Mastodon via each instance's public REST API.
+
+Uses /api/v1/accounts/lookup, /api/v1/accounts/{id}/statuses and
+/api/v1/accounts/{id}/following. Public statuses are readable without a token
+on Mastodon 2.7.0 and later, so no app registration is needed for public
+accounts. Instances that run in whitelist mode will 401; those get skipped.
+
+Accounts are written as user@instance.tld.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from collections import Counter
+from collections.abc import Iterator
+from typing import Any
+
+import httpx
+
+from desk.sources.base import (
+ USER_AGENT, CheckResult, Getter, Item, Source, Suggestion, http_failure,
+ parse_timestamp, strip_html,
+)
+from desk.sources.spec import (
+ API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy,
+ TaggingPolicy, TargetSpec, Tasks,
+)
+from desk.tools import posts
+
+log = logging.getLogger(__name__)
+
+
+class MastodonSource(Source):
+ name = "mastodon"
+ description = "Public posts from Mastodon accounts (no token needed)."
+ default_outlet = "Mastodon"
+ #: A post link on any instance (`/@user/123`) is read through that
+ #: instance's API. The shape is not proof of Mastodon; a host that does not
+ #: answer the API in JSON gets the ordinary path.
+ getters = (Getter("mastodon_api", posts.mastodon, posts.is_mastodon, claims=True),)
+
+ #: The same declaration Bluesky makes, for the same reasons — read that
+ #: one. What differs is the handle: `user@instance.tld` names the server
+ #: as well as the person, so the check is its own.
+ spec = PluginSpec(
+ api_version=API_VERSION, id=name, version="1", label="Mastodon",
+ targets=TargetSpec(mode="editable", kind="account", defaults=[],
+ validator="mastodon_handle"),
+ profiles={"default": Profile(
+ kind="post",
+ acquisition=[Step(id="article", tool="extract_article",
+ inputs={"url": "inputs.original_url"},
+ outputs={"content": "inputs.article_text"})],
+ tasks=Tasks(
+ summary=SummaryPolicy(enabled=False),
+ tagging=TaggingPolicy(
+ enabled=True, input_fields=["inputs.feed_text"],
+ context_fields=["author"]),
+ ),
+ presentation=POST_READER,
+ )},
+ )
+
+ def __init__(self, options: dict[str, Any] | None = None) -> None:
+ super().__init__(options)
+ self.client = httpx.Client(
+ timeout=20,
+ follow_redirects=True,
+ headers={"User-Agent": USER_AGENT},
+ )
+ self._delay = float(self.options.get("request_delay", 0.3))
+
+ @staticmethod
+ def _split(acct: str) -> tuple[str, str]:
+ acct = acct.lstrip("@")
+ user, _, host = acct.partition("@")
+ return user, host
+
+ def _get(self, host: str, path: str, **params) -> Any:
+ time.sleep(self._delay)
+ r = self.client.get(f"https://{host}/api/v1/{path}", params=params)
+ r.raise_for_status()
+ return r.json()
+
+ def _lookup(self, acct: str) -> dict[str, Any] | None:
+ user, host = self._split(acct)
+ if not host:
+ log.warning("mastodon: %s is missing an instance (use user@instance)", acct)
+ return None
+ try:
+ return self._get(host, "accounts/lookup", acct=user)
+ except httpx.HTTPError as exc:
+ log.warning("mastodon: lookup %s failed (%s)", acct, exc)
+ return None
+
+ # ------------------------------------------------------------- ingest
+
+ def fetch(self) -> Iterator[Item]:
+ limit = int(self.options.get("posts_per_account", 40))
+ for acct, label in self.targets():
+ account = self._lookup(acct)
+ if not account:
+ continue
+ _, host = self._split(acct)
+ try:
+ statuses = self._get(
+ host,
+ f"accounts/{account['id']}/statuses",
+ limit=min(limit, 40),
+ exclude_replies=str(self.options.get("skip_replies", True)).lower(),
+ exclude_reblogs=str(self.options.get("skip_boosts", True)).lower(),
+ )
+ except httpx.HTTPError as exc:
+ log.warning("mastodon: statuses for %s failed (%s)", acct, exc)
+ continue
+
+ for status in statuses:
+ card = status.get("card") or {}
+ text = strip_html(status.get("content", ""))
+ yield Item(
+ source_id=status.get("uri") or status.get("url", ""),
+ location=card.get("url") or status.get("url", ""),
+ title=card.get("title", ""),
+ excerpt=text,
+ author=(status.get("account") or {}).get("display_name") or acct,
+ published_at=parse_timestamp(status.get("created_at", "")),
+ subscription=label,
+ kind="post",
+ raw={
+ "boosts": status.get("reblogs_count", 0),
+ "favourites": status.get("favourites_count", 0),
+ "post_url": status.get("url", ""),
+ },
+ )
+
+ # --------------------------------------------------------------- check
+
+ def check(self) -> list[CheckResult]:
+ return [self._check_one(acct) for acct, _ in self.targets()]
+
+ def _check_one(self, acct: str) -> CheckResult:
+ user, host = self._split(acct)
+ if not host:
+ return CheckResult("mastodon", acct, False,
+ "missing instance (use user@instance)")
+ try:
+ data = self._get(host, "accounts/lookup", acct=user)
+ except httpx.HTTPError as exc:
+ return CheckResult("mastodon", acct, False, http_failure(exc))
+ return CheckResult("mastodon", acct, True,
+ f"{data.get('statuses_count', '?')} posts")
+
+ # ----------------------------------------------------------- discovery
+
+ def discover(self) -> list[Suggestion]:
+ settings = self.options.get("discover") or {}
+ if not settings.get("enabled", True):
+ return []
+
+ seeds = [target for target, _ in self.targets()]
+ following = {s.lstrip("@").lower() for s in seeds}
+ counts: Counter[str] = Counter()
+ meta: dict[str, str] = {}
+
+ for acct in seeds:
+ account = self._lookup(acct)
+ if not account:
+ continue
+ _, host = self._split(acct)
+ try:
+ # Many accounts hide this list; a 401/403 just means skip.
+ follows = self._get(host, f"accounts/{account['id']}/following", limit=80)
+ except httpx.HTTPError:
+ continue
+ for other in follows:
+ handle = other.get("acct", "")
+ if "@" not in handle:
+ handle = f"{handle}@{host}"
+ if handle.lower() in following:
+ continue
+ counts[handle] += 1
+ meta.setdefault(handle, other.get("display_name", ""))
+
+ threshold = settings.get("min_overlap", 2)
+ return [
+ Suggestion(
+ handle=handle,
+ display_name=meta.get(handle, ""),
+ reason=f"followed by {n} of the accounts you follow",
+ score=float(n),
+ url=f"https://{handle.split('@')[-1]}/@{handle.split('@')[0]}",
+ )
+ for handle, n in counts.most_common(200)
+ if n >= threshold
+ ]
diff --git a/desk/sources/podcast.py b/desk/sources/podcast.py
new file mode 100644
index 0000000..247675d
--- /dev/null
+++ b/desk/sources/podcast.py
@@ -0,0 +1,546 @@
+"""A news podcast, read as one item per story rather than one per episode.
+
+The general half. `desk/tools/video/` below knows how to cut a video into
+chapters; this knows the things that are true of a *show*: it has a feed, it
+publishes on a cycle, its episodes are somewhere on YouTube, and its stories
+belong in your queue next to the RSS copies of those same articles.
+`plugins/lastweekin_ai.py` above knows one show.
+
+That three-layer split is not decoration. A show that publishes a timestamped
+outline works here with nothing show-specific involved — Dwarkesh and Latent
+Space both do. What varies between shows is smaller than it looks, and is
+declared as two methods rather than guessed at:
+
+ video_key(entry) where this show puts its video. The default reads
+ the feed body, which is the common case and is
+ wrong for several of the biggest shows.
+ episode_number(title) how this show writes an episode number.
+
+Where each piece comes from, for one episode:
+
+ the feed title, date, the audio enclosure, the video link,
+ and the show notes as HTML — one fetch, everything
+ the notes chapter marks, sometimes with the article links attached,
+ which is what makes a segment a story rather than a topic
+ YouTube the transcript, with timings
+ video.cut() one Segment per chapter
+ this file one Item per Segment
+
+**The subscription is just a feed.** `feeds:` is the same config key `rss`
+uses, so a show imports into the `subscriptions` table like any other
+subscription and gets add/remove/fetch/show, the Sources tab and the sidebar
+for free — none of which had to know podcasts exist.
+
+Config, for any show that is not one somebody has written a plugin for:
+
+ sources:
+ podcast:
+ feeds:
+ - url: https://api.substack.com/feed/podcast/69345.rss
+ label: Dwarkesh
+ episodes_per_feed: 2
+
+Two things this deliberately does not do. It does not summarise: `text` starts
+as the opening of the transcript and is replaced when a summary lands. What the
+hosts made of a story is the one thing here that no feed already gives you, and
+so the only part of this worth a model — which makes it a separate job, whose
+failure costs you the abstracts and not the pull. See
+newsdesk/jobs/summarise.py.
+
+And it does not store the episode as an item, or as anything else: an episode
+is a date and a video id, both of which live on every segment it produced, so
+"every segment of episode 255, in order" is already a query and needs no
+entity behind it.
+
+**A show with no outline gets nothing stored, on purpose.** That claim above
+used to hold only by luck: with no marks anywhere, `cut` hands back the whole
+episode as one segment — right for a link you added by hand, wrong here — and
+this stored it, so a two-hour recording arrived as a single eleven-thousand-word
+item whose standfirst was the first two sentences of the intro and which then
+queued itself for a summariser expecting one story. Now `_one_episode` stops
+before the transcript is even fetched and says so in the log. Only shows that
+publish a timestamped outline work here, which was always true and is now also
+enforced.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+import feedparser
+import httpx
+
+from desk.sources.base import USER_AGENT, CheckResult, Item, Source, entry_time
+from desk.sources.rss import check_feed
+from desk.sources.spec import (
+ API_VERSION, RECORDING_READER, RSS, PluginSpec, Profile, RetryPolicy, Step,
+ SummaryPolicy, TaggingPolicy, TargetSpec, Tasks,
+)
+from desk.tools.http import session
+from desk.tools.video import Segment, Video, cut, marks, media, opening, youtube
+
+log = logging.getLogger(__name__)
+
+#: "LWiAI Podcast #255 - Gemini 3.7, ..." -> 255. Every show numbers its
+#: episodes somewhere; the title is where they all agree to put it.
+_NUMBER = re.compile(r"(?:#|episode\s+|ep\.?\s*)(\d{1,5})\b", re.IGNORECASE)
+
+
+def segment_profile(instructions: str = "", tags: str = "",
+ target_words: int = 70) -> Profile:
+ """What a show that publishes a timestamped outline produces, declared.
+
+ A reusable typed preset, so that a plugin for one show is its feed, its
+ numbering and its editorial line — and not a second copy of this
+ machinery's description. `plugins/lastweekin_ai.py` is the model.
+
+ The two steps are what `_one_episode` below does, in the contract's words:
+ retrieve the recording's captions *once*, then cut them at the marks the
+ show published. Nothing executes them yet — the runner is a later step of
+ `docs/plugin-rework.md` — and they are written now because they are the
+ part a future reader has to be able to check this file against.
+
+ `instructions` is the one thing a show is expected to fill in: what its
+ own segments want said about them. What is general about summarising
+ speech — write what they made of it, keep the disagreement, do not quote a
+ mangled caption back — belongs to the shared task and is written once in
+ `newsdesk/jobs/summarise.py`, not per show.
+ """
+ return Profile(
+ # One profile, because one thing is stored: the stories, not the
+ # episodes they came out of. An episode is a date and a video id, both
+ # of which are on every segment it produced.
+ kind="segment",
+ acquisition=[
+ Step(id="captions", tool="retrieve_captions",
+ inputs={"media": "inputs.media"},
+ outputs={"transcript": "inputs.transcript"},
+ # Auto-captions on a two-hour upload can take hours to
+ # appear, so absence is a wait and not a failure. Today that
+ # wait is "the next pull, for ever", because the episode is
+ # simply not marked done; a bound is what this declares.
+ retry=RetryPolicy(max_attempts=4, delay_seconds=3600,
+ backoff=2.0, on=["not_ready"])),
+ Step(id="segments", tool="cut_segments", needs=["captions"],
+ # The marks come from the show's own notes, which arrive in
+ # the feed entry — HTML, so each story keeps the article link
+ # beside its timestamp.
+ inputs={"transcript": "inputs.transcript",
+ "marks": "inputs.feed_text"},
+ outputs={"transcript": "inputs.transcript",
+ "links": "inputs.links"}),
+ ],
+ tasks=Tasks(
+ summary=SummaryPolicy(
+ enabled=True, input_fields=["inputs.transcript"],
+ require_complete=True,
+ # No floor: a two-sentence housekeeping segment gets one flat
+ # sentence saying so, which is worth having and is what the
+ # shared prompt already asks for.
+ min_words=0, target_words=target_words,
+ instructions=instructions),
+ tagging=TaggingPolicy(
+ # The transcript, not the abstract: tagging a segment must not
+ # quietly wait on a summariser that failed. Naming it decides
+ # *which* text and not how much of it — the tagger reads the
+ # opening few hundred characters of whatever a profile names,
+ # because it is choosing from a closed list of labels and a
+ # two-hour transcript would be thirty times the tokens for
+ # the same answer. See `newsdesk/jobs/tagging.py`.
+ enabled=True, input_fields=["inputs.transcript"],
+ context_fields=["title", "inputs.links"], instructions=tags),
+ ),
+ # The player first, then what the hosts made of it; the articles
+ # they were reading stay above it in every view. See the preset.
+ presentation=RECORDING_READER,
+ )
+
+
+class PodcastSource(Source):
+ name = "podcast"
+ description = "A show's feed, cut into one item per story discussed."
+ subscription_field = "feeds"
+ default_outlet = "Podcast"
+
+ #: The generic source declares the same profile it hands its subclasses,
+ #: with no editorial line of its own: it is any show, and it cannot know
+ #: what one of them wants said. Its targets are editable because they are
+ #: whatever you put under `podcast:` in config.yaml — a show with a plugin
+ #: of its own declares a catalogue instead.
+ spec = PluginSpec(
+ api_version=API_VERSION, id=name, version="1", label="Podcast",
+ discovery=RSS(),
+ targets=TargetSpec(mode="editable", kind="rss_feed", defaults=[]),
+ profiles={"default": segment_profile()},
+ )
+
+ # -------------------------------------------------------- what a show is
+ #
+ # The two things that differ between shows, as methods rather than as
+ # config knobs — because neither has a value you could write down. Finding
+ # the video means knowing where that show puts it, and numbering an episode
+ # means knowing how that show writes a number. A subclass that knows one
+ # show answers both; everything else in this file is the same for all of
+ # them.
+
+ def video_key(self, entry: Any) -> str:
+ """The YouTube id for this episode, or "" if there isn't one to find.
+
+ **The one place a show's publishing habits really bite.** The default
+ reads the feed body, which works when the show embeds or links its own
+ video there — Last Week in AI, Dwarkesh, Latent Space all do.
+
+ It is not general, and it fails in a way worth naming: The Vergecast,
+ Hard Fork, TWiT, All-In and Lex Fridman all publish full video to
+ YouTube and none of them mentions it in the feed, so all five come back
+ empty here. Fixing that means matching a feed entry against a channel's
+ uploads — which title, which date, whose channel — and that is a
+ per-show judgement, not a parser. This is where it goes when someone
+ wants one of those shows: override this, return the id, and every other
+ line in this file already works.
+ """
+ try:
+ return youtube.video_id(_body(entry))
+ except ValueError:
+ return ""
+
+ def episode_number(self, title: str) -> int | None:
+ """Which episode this is, from its title, or None if it doesn't say.
+
+ Every show numbers its episodes and the title is where they all agree
+ to put it — but not in one format. `#255` and `Episode 42` and `Ep. 7`
+ are the common family and are what the default reads; `TWiT 1100` and
+ `SN 1094` are not, and an interview show may not number at all.
+
+ Nothing downstream depends on the answer: it is stored on the item for
+ display and for "every segment of episode 255, in order". So the honest
+ default is a regex that catches most of them and returns None rather
+ than guessing when it doesn't, and a subclass overrides it when its
+ show writes numbers its own way.
+ """
+ found = _NUMBER.search(title or "")
+ return int(found.group(1)) if found else None
+
+ # ------------------------------------------------------------------ fetch
+
+ def fetch(self) -> Iterator[Item]:
+ limit = int(self.options.get("episodes_per_feed", 2))
+ with session(None, timeout=40) as http:
+ for entry in self.options.get("feeds", []):
+ url = entry["url"] if isinstance(entry, dict) else entry
+ label = _label(entry, url)
+ try:
+ yield from self._one_feed(http, url, label, limit)
+ except youtube.Blocked:
+ # The one failure that is not this show's. YouTube refused
+ # the machine, so every other show here would be refused
+ # too, and each attempt is one more request from an address
+ # already taken for a bot. Out of `fetch` altogether: the
+ # runner keeps what was yielded and turns this into a
+ # notice, which is what the log line below never was.
+ raise
+ except Exception:
+ # One dead show must not cost the others their episodes.
+ log.exception("podcast feed %s failed", url)
+
+ def _one_feed(self, http: httpx.Client, url: str, label: str,
+ limit: int) -> Iterator[Item]:
+ response = http.get(url, headers={"User-Agent": USER_AGENT})
+ response.raise_for_status()
+ # The bytes, not `.text`. A feed declares its own encoding in its XML
+ # declaration and plenty of them are not UTF-8; handing the parser a
+ # string that httpx has already decoded by its own guess throws that
+ # declaration away, and what comes back is an episode title with a
+ # mojibake apostrophe in it. `check_feed` parses the bytes too.
+ feed = feedparser.parse(response.content)
+ if not feed.entries:
+ log.warning("no episodes in %s", url)
+ return
+
+ for entry in feed.entries[:limit]:
+ episode = self._episode(entry, label)
+ if episode is None:
+ continue
+ if self._already_done(episode.key):
+ log.debug("%s: already ingested", episode.title)
+ continue
+ try:
+ yield from self._one_episode(http, episode, entry, label)
+ except youtube.Blocked:
+ raise # not this episode's failure — see `fetch`
+ except Exception:
+ log.exception("episode %s failed; continuing the feed", episode.key)
+
+ def _one_episode(self, http: httpx.Client | None, episode: Video, entry: Any,
+ label: str) -> Iterator[Item]:
+ # The show's own notes first: same outline as the video description,
+ # but the links survived. Then the description, then the comments.
+ chapters = marks.best(episode.notes)
+ if not chapters:
+ described = youtube.details(episode.key, client=http)
+ chapters = marks.best(described["description"])
+ episode.duration = episode.duration or described["duration"]
+ if not chapters and self.options.get("read_comments", True):
+ chapters = marks.in_comments(youtube.comments(episode.key, client=http))
+
+ if not chapters:
+ # Nothing published an outline: not the notes, not the description,
+ # not a listener in the comments. `cut` would hand back the whole
+ # episode as one segment — correct of it, and what a hand-added
+ # link wants — but this source's contract is one item per *story*,
+ # and quietly substituting one item per episode is a different
+ # product wearing its name.
+ #
+ # It is also unusable downstream in a way that is easy to miss. The
+ # item's `text` would be `opening(transcript)`, the first two
+ # sentences of a two-hour recording, which is what the ranker and
+ # the tag filters would then judge it on; and eleven thousand words
+ # would go to a summariser whose prompt promises it a single story
+ # with the article the hosts were reading. Better to store nothing and say
+ # why than to store one wall and three wrong things about it.
+ #
+ # Marked done regardless. Reaching here means the episode was
+ # fetched and read successfully; there was simply nothing to cut
+ # along. A show that never publishes an outline would otherwise
+ # re-download a full transcript on every pull, for ever, to throw
+ # it away again — which is the waste `_already_done` exists to stop.
+ log.warning(
+ "%s: no timestamped outline in the notes, the description or "
+ "the comments, so there are no stories to cut it into — "
+ "storing nothing for this episode", episode.title)
+ self._mark_done(episode.key)
+ return
+
+ try:
+ cues = youtube.transcript(episode.key, client=http)
+ except youtube.Blocked:
+ # A `TranscriptUnavailable` too, and the one that must not be
+ # treated as one here. Waiting is the answer to captions that are
+ # still being made; to a refusal it is the same nothing on every
+ # pull, at INFO, with the episode never marked done and nobody
+ # told. Still not marked done, so the pull after the address is
+ # clean reads it — but it leaves as a failure, and reaches the
+ # screen. `details` above raises the same thing the same way.
+ raise
+ except youtube.TranscriptUnavailable as exc:
+ # Not done, so no marker is written and the next pull tries again.
+ # Auto-captions on a two-hour upload can take hours to appear, and
+ # an episode without them is worth waiting for rather than
+ # ingesting as twenty titles with nothing under them.
+ log.info("%s: %s", episode.title, exc)
+ return
+
+ segments = cut(episode, cues, chapters)
+ log.info("%s: %d segments", episode.title, len(segments))
+ for segment in segments:
+ yield _item(segment, entry, label,
+ number=self.episode_number(episode.title))
+ self._mark_done(episode.key)
+
+ def check(self) -> list[CheckResult]:
+ """A show is a feed, so the feed check is the right one."""
+ return [check_feed(url, label) for url, label in self.targets()]
+
+ def _episode(self, entry: Any, show: str) -> Video | None:
+ """A feed entry as a `Video`. None when there is no video to read.
+
+ A show whose video this cannot find is not a failure — see
+ `video_key`, which is the part a subclass replaces. Saying so once,
+ quietly, beats failing per episode.
+ """
+ key = self.video_key(entry)
+ if not key:
+ # Deliberately not "audio-only shows need local transcription",
+ # which is what this used to say and which was wrong for the five
+ # biggest shows it fires on: they all post video, they just do not
+ # say so in the feed. Naming the actual condition is what stops the
+ # next person going and building a transcriber they did not need.
+ log.info("no video id in the feed entry for %r — the show may still "
+ "publish one; see PodcastSource.video_key",
+ (entry.get("title") or "")[:60])
+ return None
+
+ return Video(
+ ref=media.youtube(key),
+ title=(entry.get("title") or "").strip(),
+ url=entry.get("link") or "",
+ author=entry.get("author") or show,
+ published_at=entry_time(entry),
+ notes=_body(entry),
+ )
+
+ # ------------------------------------------------------------ bookkeeping
+
+ def _already_done(self, key: str) -> bool:
+ """Whether this episode has been through here successfully.
+
+ In the `state` table rather than in a table of its own, because that is
+ genuinely all the state an episode has: it is done or it is not. An
+ episode is not a thing you read, so it needs no row of its own, and one
+ key in a table that already exists needs no migration.
+
+ The key is written only on success, so an episode whose captions
+ YouTube had not finished generating is left unmarked and comes round
+ again on the next pull.
+
+ Without this, a weekly pull would re-fetch and re-cut every transcript
+ in the window on every run and then discard the lot at the dedup check
+ — correct, and several wasted seconds per pull.
+ """
+ return bool(self.state.get(self._done_key(key)))
+
+ def _mark_done(self, key: str) -> None:
+ self.state[self._done_key(key)] = datetime.now(UTC).isoformat()
+
+ def _done_key(self, key: str) -> str:
+ """Namespaced by the plugin, not hard-coded to `podcast`.
+
+ Two shows can publish the same video id — a guest episode cross-posted,
+ a network re-run — and with one shared prefix the second would find the
+ first's marker and skip an episode it had never read. One subclass per
+ show means one namespace per show for free.
+ """
+ return f"{self.name}:done:{key}"
+
+
+# ------------------------------------------------------------------ internals
+
+def _label(entry: Any, url: str) -> str:
+ """What one configured show is called. Which outlet that stands for is the
+ subscription's business — see `Item.subscription`."""
+ if not isinstance(entry, dict):
+ return url
+ return entry.get("label") or url
+
+
+def _item(segment: Segment, entry: Any, label: str,
+ number: int | None = None) -> Item:
+ """One segment, as the row newsdesk stores.
+
+ Three of the field choices are not obvious:
+
+ `body` is the transcript, which is both where it belongs and how this
+ tells `fetch_bodies` not to go and scrape a YouTube page for one. It
+ is not the `excerpt`, because the excerpt is what the queue row renders,
+ what the ranker's document is built from and what search indexes, and
+ sixteen hundred words of speech would swamp all three.
+
+ `excerpt` is a placeholder until a summary is written over it. It is the
+ opening of the transcript rather than empty, because an item with no
+ excerpt is an empty row in the queue, and the ranker and the tagger both
+ read it. It used to be accompanied by a `summarised: False` flag, which
+ was this source telling the summariser that the row was waiting for it.
+ Nothing needs telling now: the profile declares that these items are
+ summarised, and an empty `summary` column is the whole of "not yet".
+
+ `published_at` is the episode's stamp plus the segment's offset, so twenty
+ items published in the same instant still sort in the order they were said.
+
+ `links` is the reason a segment is a story rather than a topic: the
+ articles the outline attached to this stretch, typed and roled, so the
+ reading pane and the summariser read them from the same field an outlet's
+ links live in rather than from this plugin's own corner of `raw`.
+ """
+ episode = segment.video
+ return Item(
+ source_id=segment.external_id,
+ location=segment.url(),
+ title=segment.title,
+ excerpt=opening(segment.transcript),
+ body=segment.transcript,
+ author=episode.author,
+ published_at=episode.published_at + timedelta(seconds=segment.start),
+ subscription=label,
+ links=segment.reading(),
+ # Not `article`, which is what every one of these used to be stored as
+ # — twenty-three transcripts filed as prose. What the pipeline made,
+ # not what carried it: this arrived over RSS and left as segments.
+ kind="segment",
+ raw={
+ # The chapter this segment sits under. Per *item* — it changes
+ # down the episode — so it is here rather than on the
+ # subscription's `section`, which names a branch of an outlet and
+ # is the same for everything that subscription hands over.
+ "heading": segment.heading,
+ # Where it plays and what it linked to: the record every stored
+ # segment carries, defined once on `Segment`.
+ **segment.record(),
+ "show": label,
+ "episode": number,
+ "episode_title": episode.title,
+ "episode_url": episode.url,
+ "episode_published": episode.published_at.isoformat(),
+ "audio_url": _enclosure(entry),
+ },
+ )
+
+
+def _body(entry: Any) -> str:
+ """The richest form of the notes the feed offers."""
+ for content in entry.get("content") or []:
+ if value := content.get("value"):
+ return value
+ return entry.get("summary") or ""
+
+
+def _enclosure(entry: Any) -> str:
+ return next(
+ (e.get("href", "") for e in entry.get("enclosures") or []
+ if (e.get("type") or "").startswith("audio")),
+ "",
+ )
+
+
+# ----------------------------------------------------------------- dry run
+
+def main(argv: list[str] | None = None) -> int:
+ """`python -m desk.sources.podcast ` — what a pull would store.
+
+ Runs the plugin with no store behind it and prints the items instead of
+ writing them. This is the thing to run when a show changes the shape of
+ its notes: everything else in the pipeline is downstream of these fields.
+ """
+ import argparse
+ import sys
+
+ parser = argparse.ArgumentParser(prog="python -m desk.sources.podcast")
+ parser.add_argument("feed", help="the show's RSS feed")
+ parser.add_argument("--episodes", type=int, default=1)
+ parser.add_argument("-v", "--verbose", action="store_true")
+ args = parser.parse_args(argv)
+
+ logging.basicConfig(level=logging.INFO if args.verbose else logging.WARNING,
+ format="%(levelname)-7s %(message)s")
+ if hasattr(sys.stdout, "reconfigure"):
+ sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+
+ source = PodcastSource({
+ "feeds": [args.feed], "episodes_per_feed": args.episodes,
+ })
+ items = list(source.fetch())
+ if not items:
+ print("no items — run with -v to see why", file=sys.stderr)
+ return 1
+
+ print(f"{len(items)} items from {args.feed}\n")
+ heading = None
+ for item in items:
+ if item.raw["heading"] != heading:
+ heading = item.raw["heading"]
+ print(f" {heading or '(unfiled)'}")
+ stamp = marks.hhmmss(item.raw["start"])
+ print(f" {stamp:>8} {item.title[:80]}")
+ print(f" {item.location}")
+ for link in item.links:
+ print(f" → {link.url}")
+ print(f" {len(item.body.split()):,} words · {item.excerpt[:90]}")
+ print()
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/desk/sources/registry.py b/desk/sources/registry.py
new file mode 100644
index 0000000..8382304
--- /dev/null
+++ b/desk/sources/registry.py
@@ -0,0 +1,103 @@
+"""Finds Source subclasses on disk. `desk.sources.build` instantiates them."""
+
+from __future__ import annotations
+
+import importlib.util
+import inspect
+import logging
+import pkgutil
+import sys
+import hashlib
+import threading
+from functools import lru_cache
+from pathlib import Path
+
+from desk.sources.base import Source
+
+log = logging.getLogger(__name__)
+_load_lock = threading.RLock()
+
+
+def _classes_in_module(module) -> list[type[Source]]:
+ found = []
+ for _, obj in inspect.getmembers(module, inspect.isclass):
+ if not issubclass(obj, Source) or obj is Source:
+ continue
+ if inspect.isabstract(obj):
+ continue
+ # A class written to be subclassed is not a plugin. `RssSource` is
+ # the fetching a publisher's plugin inherits; it has no feeds, no
+ # archive and nothing to switch on.
+ if obj.__dict__.get("shared", False):
+ continue
+ if not getattr(obj, "name", ""):
+ log.warning("%s.%s has no `name`, skipping", module.__name__, obj.__name__)
+ continue
+ # Only take classes defined in this module, not imported into it.
+ if obj.__module__ != module.__name__:
+ continue
+ found.append(obj)
+ return found
+
+
+def _load_file(path: Path) -> list[type[Source]]:
+ path = path.resolve()
+ try:
+ stamp = path.stat()
+ # A concurrent status request must not execute the same plugin twice.
+ with _load_lock:
+ return _load_version(path, stamp.st_mtime_ns, stamp.st_size)
+ except OSError:
+ log.exception("cannot read plugin %s", path)
+ return []
+
+
+@lru_cache(maxsize=128)
+def _load_version(path: Path, modified: int, size: int) -> list[type[Source]]:
+ """Reuse unchanged plugins while allowing edits to take effect on next use."""
+ identity = hashlib.sha256(str(path).encode()).hexdigest()[:16]
+ mod_name = f"newsdesk_plugin_{path.stem}_{identity}"
+ spec = importlib.util.spec_from_file_location(mod_name, path)
+ if spec is None or spec.loader is None:
+ return []
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[mod_name] = module
+ try:
+ spec.loader.exec_module(module)
+ except Exception:
+ sys.modules.pop(mod_name, None)
+ log.exception("failed to load plugin %s", path)
+ return []
+ return _classes_in_module(module)
+
+
+def available(plugin_paths: list[str] | None = None) -> dict[str, type[Source]]:
+ """Map of name -> Source class. Later definitions win, so a user plugin
+ named `rss` overrides the built-in one, with a warning naming both classes."""
+ classes: dict[str, type[Source]] = {}
+
+ import desk.sources as builtin_pkg
+
+ for info in pkgutil.iter_modules(builtin_pkg.__path__):
+ if info.name == "base":
+ continue
+ module = importlib.import_module(f"desk.sources.{info.name}")
+ for cls in _classes_in_module(module):
+ classes[cls.name] = cls
+
+ for raw in plugin_paths or []:
+ directory = Path(raw).expanduser()
+ if not directory.is_dir():
+ log.warning("plugin_path %s is not a directory", directory)
+ continue
+ for path in sorted(directory.glob("*.py")):
+ if path.name.startswith("_"):
+ continue
+ for cls in _load_file(path):
+ if (previous := classes.get(cls.name)) and previous is not cls:
+ log.warning("plugin name %s: %s.%s overrides %s.%s",
+ cls.name, cls.__module__, cls.__name__,
+ previous.__module__, previous.__name__)
+ classes[cls.name] = cls
+
+ return classes
diff --git a/desk/sources/rss.py b/desk/sources/rss.py
new file mode 100644
index 0000000..4e96bc0
--- /dev/null
+++ b/desk/sources/rss.py
@@ -0,0 +1,236 @@
+"""RSS and Atom feed fetching, as a base class for a publisher's plugin.
+
+Not itself a source: see `RssSource`, which is what a masthead subclasses.
+`check_feed` is the same fetching asked whether one address still answers.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Iterator
+from datetime import UTC, datetime
+from typing import Any
+
+import feedparser
+import httpx
+
+from desk.sources.base import (
+ USER_AGENT, CheckResult, Item, Source, entry_time, http_failure, outlet_name,
+ strip_html,
+)
+from desk.tools.http import session
+
+log = logging.getLogger(__name__)
+
+#: Seconds one feed gets, per operation. Which number it is matters far less
+#: than there being one at all — see `_read`.
+FETCH_TIMEOUT = 30
+
+
+class FeedsFailed(Exception):
+ """Some of a pull's feeds could not be fetched, and the rest were.
+
+ Raised at the end of `fetch` rather than where it happened, and that is the
+ whole of the design: everything the live feeds gave has already been
+ yielded and stored by the time this lands, so one outlet returning 403
+ costs you that outlet and not the other nineteen. The runner turns it into
+ `notices.source_failed`, which is what puts a feed that has been dead all
+ week on screen — the one failure a reader cannot spot by eye, because a
+ dead source and a quiet news day look identical.
+
+ Worded through `http_failure`, so every line in it reads `HTTP 403` or
+ `unreachable: ReadTimeout`. That is also what has the notice file it as the
+ outlet's business rather than as a defect of ours; see
+ `desk/notices.py`.
+ """
+
+
+def _categories(entry) -> list[str]:
+ """Whatever the outlet put in .
+
+ The Guardian, Ars Technica, the NYT and Al Jazeera all populate these;
+ the BBC and Hacker News don't. They're inconsistent enough between outlets
+ to be useless as a filter — the NYT files people ('Putin, Vladimir V'),
+ the Guardian files desks ('Television & radio') — so they're carried as a
+ hint for the tagger rather than shown to you.
+ """
+ seen, out = set(), []
+ for tag in entry.get("tags") or []:
+ term = strip_html(tag.get("term", ""))
+ if term and term.lower() not in seen:
+ seen.add(term.lower())
+ out.append(term)
+ return out
+
+
+def check_feed(url: str, label: str = "", kind: str = "rss") -> CheckResult:
+ """Fetch one feed and say what is in it. Shared with the podcast source,
+ whose subscription is a feed too."""
+ try:
+ r = httpx.get(url, headers={"User-Agent": USER_AGENT}, timeout=20,
+ follow_redirects=True)
+ except httpx.HTTPError as exc:
+ return CheckResult(kind, url, False, f"unreachable: {type(exc).__name__}", label)
+ if r.status_code != 200:
+ return CheckResult(kind, url, False, f"HTTP {r.status_code}", label)
+
+ parsed = feedparser.parse(r.content)
+ n = len(parsed.entries)
+ if n == 0:
+ reason = parsed.get("bozo_exception")
+ return CheckResult(kind, url, False, f"0 entries ({reason or 'not a feed?'})", label)
+ return CheckResult(kind, url, True, f"{n} entries, {_age(parsed.entries)}", label,
+ outlet=_proposed_outlet(parsed, url, label))
+
+
+def _read(http: httpx.Client, url: str) -> Any:
+ """One feed, fetched here and parsed from the bytes it answered with.
+
+ Deliberately not `feedparser.parse(url)`, which is the same call with the
+ fetching left to somebody else and which goes through urllib with **no
+ timeout**. A
+ host that accepts the connection and then says nothing holds the thread for
+ ever, and the thread it holds is a worker's: the pull is `solo`, so nothing
+ else pulls behind it; cancellation is cooperative, so Kill cannot reach a
+ thread parked in a socket read; and a restart releases the lease only to
+ re-queue the job and stall it on the same feed. One bad feed stopped the
+ application from pulling at all until somebody edited the subscription.
+ feedparser offers no way to pass urllib a deadline, so the fetch comes
+ here, where the client has one.
+
+ Two things fall out of doing it here, both of which `check_feed` has had
+ all along. The request goes out as `USER_AGENT`, which `base.py` promises
+ every plugin sends and which feedparser was quietly replacing with its own.
+ And the status is read, so a 403, 410 or 429 — each of which arrives with
+ an empty body that parses perfectly well as a feed with nothing in it — is
+ a failure of that feed rather than a shrug about a slow news day.
+ """
+ response = http.get(url, headers={"User-Agent": USER_AGENT})
+ response.raise_for_status()
+ # The bytes rather than `.text`, so the document's own XML declaration
+ # decides its encoding instead of a guess made from a header the outlet
+ # may never have set.
+ return feedparser.parse(response.content)
+
+
+def _proposed_outlet(parsed: Any, url: str, label: str) -> str:
+ """What this feed calls itself — but only when you didn't say.
+
+ A label you wrote is the better name and the check must never talk you out
+ of it: "Guardian Tech" splits into an outlet and a section that a feed's
+ own `` ("Technology | The Guardian") cannot improve on. A label is
+ only ever *this* when you pasted a URL and named nothing, and then the
+ alternative is filing your reading under `https://…`, which the feed's
+ title beats outright.
+ """
+ if label and label != url:
+ return ""
+ return outlet_name(strip_html(parsed.feed.get("title", "")))
+
+
+def _age(entries) -> str:
+ stamps = [entry_time(e) for e in entries
+ if e.get("published_parsed") or e.get("updated_parsed")]
+ if not stamps:
+ return "no dates"
+ hours = (datetime.now(UTC) - max(stamps)).total_seconds() / 3600
+ return f"newest {hours:.0f}h ago" if hours < 48 else f"newest {hours / 24:.0f}d ago"
+
+
+class RssSource(Source):
+ """Feed fetching, for a publisher's plugin to inherit. Not a plugin.
+
+ `shared` is what says so, and retiring the generic "Any RSS feed" source
+ it used to be is §9 of the rework: an outlet is a plugin, so a masthead
+ has a file that declares its own feeds, marks and policies rather than a
+ row somebody pasted a URL into. Everything below is unchanged — this is
+ the same fetching the Guardian, the BBC, Ars Technica, Golem and Hacker
+ News have always run on, reached now by subclassing rather than by
+ configuring.
+
+ A subclass states its own `name`; the empty one here is deliberate, so
+ that forgetting it is a plugin the registry skips loudly rather than a
+ second class claiming the archive of the first.
+ """
+
+ name = ""
+ shared = True
+ #: Inherited by a subclass that states none of its own.
+ description = "An RSS or Atom feed."
+ subscription_field = "feeds"
+
+ def fetch(self) -> Iterator[Item]:
+ feeds = self.options.get("feeds", [])
+ failed: list[str] = []
+ # One client for the whole pull, so twenty feeds off one host are one
+ # connection rather than twenty. Closed on the way out however this
+ # generator ends.
+ with session(None, timeout=FETCH_TIMEOUT) as http:
+ for entry in feeds:
+ # The label is the whole of what an item says about where it came
+ # from. Which outlet and which section of it that label stands for
+ # is the subscription's business, settled once when you added it —
+ # see `CheckResult.outlet`, which is how this plugin has its say.
+ if isinstance(entry, dict):
+ url = entry["url"]
+ label = entry.get("label") or url
+ else:
+ url = label = entry
+ try:
+ parsed = _read(http, url)
+ except (httpx.HTTPError, httpx.InvalidURL) as exc:
+ # Everything this feed's own end can do to us: a status, a
+ # host that never answers, an address that was mistyped
+ # when it was pasted. Noted and carried past, because the
+ # next feed in the list has nothing to do with any of it.
+ #
+ # Nothing broader is caught. What else could land here is a
+ # wrong assumption of ours about the shape of a response,
+ # and notices.py is explicit that those have to reach you
+ # as bugs rather than be folded in with the weather.
+ log.warning("could not fetch %s: %s", url, http_failure(exc))
+ failed.append(f"{label} ({http_failure(exc)})")
+ continue
+ if parsed.bozo and not parsed.entries:
+ # It answered, and what came back is not a feed. Empty
+ # rather than failed: the server did its part, and there is
+ # nothing here for anyone to go and fix.
+ log.warning("no entries from %s (%s)", url, parsed.get("bozo_exception"))
+ continue
+
+ yield from self._items(parsed, url, label)
+
+ if failed:
+ raise FeedsFailed(
+ f"{len(failed)} of {len(feeds)} feeds failed: " + "; ".join(failed))
+
+ def _items(self, parsed: Any, url: str, label: str) -> Iterator[Item]:
+ """One feed's entries, as rows. Split out of `fetch` only so that the
+ loop over your subscriptions stays readable beside what it catches."""
+ for e in parsed.entries[: self.options.get("max_items", 60)]:
+ yield self.item_from_entry(e, url, label, parsed.feed.get("title", ""))
+
+ def item_from_entry(self, entry: Any, url: str, label: str, author: str = "") -> Item:
+ """Map one parsed entry; outlets may refine fields without owning HTTP."""
+ link = entry.get("link") or entry.get("id") or ""
+ body = ""
+ if entry.get("content"):
+ body = strip_html(entry.content[0].get("value", ""))
+ body = body or strip_html(entry.get("summary", ""))
+ raw: dict[str, Any] = {"feed": url}
+ if cats := _categories(entry):
+ raw["categories"] = cats[:12]
+ return Item(
+ source_id=entry.get("id") or link,
+ location=link,
+ title=strip_html(entry.get("title", "")),
+ excerpt=body,
+ author=entry.get("author", "") or author,
+ published_at=entry_time(entry),
+ subscription=label,
+ kind="article",
+ raw=raw,
+ )
+
+ def check(self) -> list[CheckResult]:
+ return [check_feed(url, label) for url, label in self.targets()]
diff --git a/desk/sources/sources.md b/desk/sources/sources.md
new file mode 100644
index 0000000..363c9ed
--- /dev/null
+++ b/desk/sources/sources.md
@@ -0,0 +1,110 @@
+# `desk.sources` — the plugins
+
+## What it is
+
+**"Source" and "plugin" are the same thing.** One `.py` file with a `Source` subclass in it. It answers one question: *where do items come from* — fetch, prepare, yield. Nothing else: no store, no queue, no config beyond its own options.
+
+| | |
+|---|---|
+| Contract | `base.py` — `Source`, `Item`, `Suggestion`, `CheckResult` |
+| Discovery | `registry.available()` — every module in `desk/sources/`, plus every `*.py` in the dirs under `plugin_paths:` in config.yaml (leading `_` skipped) |
+| Built-ins | `bluesky`, `mastodon`, `x`, `podcast`. `RssSource` is here too and is **not** a plugin: it is `shared`, which the registry passes over, and a publisher subclasses it |
+| Packages | `httpx` fetches, `feedparser` parses what a feed answered with. No plugin framework, no entry points |
+| Door | `from desk.sources import Source, Item, build, available, PodcastSource` |
+| Override | later definition wins — a user plugin named `bluesky` replaces the built-in, with a warning naming both |
+
+## The three steps
+
+1. **Fetch** — `fetch()` is a generator; hit the network, yield as you go. Put a timeout on every request you make — see Limits.
+2. **Prepare** — normalise into `Item`. Over-fetching is fine: the runner dedups on `uid = sha1(source + source_id)`.
+3. **Output** — `Item`s. The runner fills `source`, applies the default outlet, and stores them.
+
+`Item(source_id, location, title, excerpt="", body="", author="", published_at=now, subscription="", kind="article", marks=[], raw={}, links=[])`. `published_at` must be an aware datetime; it is stored as UTC. A non-empty `body` means "don't scrape this page" — that's how a podcast segment keeps its transcript. A post whose `location` equals `raw["post_url"]` links nothing and is not fetched either. The source's own labels go in `raw` under `categories`, which the tagger reads as a weak hint. `ItemLink`, exported through the same door, carries an absolute HTTP(S) URL, title and `reading`/`discussion`/`source` role separately from the item's primary destination.
+
+**A plugin may register how to read its own URLs.** `getters = (Getter("bluesky_api", posts.bluesky, posts.is_bluesky, claims=True),)` on the class says that a `bsky.app` post link, from whichever outlet, is read through that function rather than as a web page, while this plugin is active. It gets the page only; which link an item points to is decided in `fetch`. See `docs/architecture.md` §1.1.
+
+**A plugin declares what should be done with its items, and the app does it.** `spec.py` is that declaration — `PluginSpec`, its profiles, and a `SummaryPolicy` and `TaggingPolicy` per profile. Both shared jobs now run off one: a profile says whether its items are summarised and tagged, which stored text each task reads in order of preference, how short is too short for an abstract and how long one should be, and what this outlet wants said or emphasised about its own items. Those instructions travel with each item in a mixed batch and never apply to another outlet's. See `newsdesk/jobs/summarise.py` and `newsdesk/jobs/tagging.py`.
+
+**An outlet is a plugin.** There is no generic "Any RSS feed" source to paste a masthead into — step 19 retired it, because a feed belongs to a publisher and a publisher has extraction, marks, retries and a summary floor of its own. `RssSource` is the fetching those plugins share: subclass it, give it a `name` and a `spec`, declare the feeds. `plugins/ars_technica.py` is the smallest one, a catalogue of a single feed. A feed nobody has written a plugin for keeps its archive and its Show switch and is simply not fetched; the v28 migration names each one in the log.
+
+**Declaring nothing is not declaring no.** A plugin with no `spec` gets an adapted one with both tasks off, and the two jobs read that silence differently on purpose. Nothing undeclared has ever been summarised, so the summariser takes it as no and loses nothing. Tagging has run over every row in the archive since long before this contract, so it takes it as *today's behaviour* — the acquired text, then the feed's own, then an abstract as a last resort — and only a profile that says `enabled=False` out loud is left untagged. A converted outlet must therefore say `enabled=True` where its items were already tagged; Guardian and Hacker News say so. Which field a tagging policy names decides *which* text and not how much of it: the tagger reads the opening few hundred characters whatever is named, because it is choosing from a closed list of ~35 labels and a whole transcript would be thirty times the tokens for the same answer.
+
+**A profile also declares how its items are read.** The drawer is fixed: Summary, Text, Original and a new-tab action, in that order, for every plugin. The content is the plugin's: which slots it fills, what to call them (a segment's text is its Transcript, its original is Watch), and an ordered list of sections per view, each `prose`, `links` or `media` bound to a named field. It cannot add a fourth view, emit markup, or overrule an outlet that refuses framing. `header` holds links shown above every view: the host of the page the item is, the discussion thread of an aggregator's link, the articles a podcast segment discusses. `QueueLayout.destination` adds that host to the queue row too, and is for a plugin whose outlet is not the publisher. Name a preset rather than restating one: `ARTICLE_READER` for a publisher's own article, `LINK_READER` for a link to someone else's page, `FEED_READER` for a walled outlet, `POST_READER` for a social post, `RECORDING_READER` for a recording's segment. The table in `docs/architecture.md` §1.4 says which flow each is. A view may declare what to show `instead` when its own field is empty, with its own label: `ARTICLE_READER`'s Summary control falls through to the feed's own text and calls itself RSS feed, which is what the queue row beside it has always done. The rename is half of it: a publisher's teaser under a control marked Summary reads as our abstract of the piece, and as a poor one.
+
+**A mark and a wall are different facts, and only one of them changes the flow.** `framing` asks whether the browser will render a page in a frame, and the page behind a paywalled item renders perfectly — it is the paywall. `OriginalView.blocked_by` names the marks that refuse the embed for the items carrying one (`paywalled` by default); the layout does not change, because the piece beside it in the same feed is an ordinary article, and the control greys out with the mark's own words (`Mark.framed`). A `Wall` on a profile is the other thing: there is no article behind any URL this outlet publishes, no abstract will ever be written from one, and the destination frames the gate — so the flow differs rather than the ordinary one standing there with two permanently dead controls. `FEED_READER` is that flow, the feed's own text and the way out, and a walled profile declaring an Original view of its destination is refused where it is written. Its reason rides on `ReaderModel.note` as the one line in the bar, since there is no greyed button left to hover. A plugin that learns to get past its wall deletes the declaration and is an ordinary article profile again. The server resolves the declaration against one item and serves the answer; the browser renders it and decides nothing. See `newsdesk/web/reader.py`.
+
+A show subclasses `PodcastSource`, which finds the episode, retrieves its captions once and cuts them at the marks the show published. `plugins/lastweekin_ai.py` is the model: a catalog of one feed, `segment_profile()` — the machinery's own declaration of what it produces, exported through this door for a show to reuse — and nothing of its own beyond how this show numbers an episode and what it wants said about its segments. Each segment carries the articles its outline linked as `links` with the `reading` role, so the reading pane and the summariser read them from the same field an outlet's links live in.
+
+A show whose site already publishes the cut needs none of that. `plugins/ai_daily_brief.py` subclasses `Source` directly: the site lists its editions in `/agent.json`, each as JSON nuggets and a minute-marked transcript, so the plugin makes one segment per nugget from two requests, with no feed, no captions and no summariser.
+
+RSS outlets subclass `RssSource`. Its `item_from_entry(entry, url, label, author)` hook refines one parsed entry while shared code owns HTTP, feed parsing, limits and failure reporting. `plugins/hacker_news.py` uses it to preserve publisher destinations and HN discussion links while keeping points/comment metadata out of preview text. Guardian needs no override. Both declare fixed catalogs; unknown custom feeds remain supported through generic RSS.
+
+**There is no `outlet` and no `section`.** Who publishes a thing is a fact about the subscription it came from, held once on the subscription row and joined when something needs it — so `subscription` is the only attribution an item carries. They used to be fields, copied onto every item, and a rename left four hundred rows disagreeing with the feed they came from until somebody walked the table.
+
+## What a plugin declares about failing
+
+`spec.py` is the other half of the contract: not where items come from but what should be done with them. `Step.retry` is the part of it that runs today.
+
+```python
+Step(id="article", tool="extract_article",
+ inputs={"url": "inputs.original_url"},
+ outputs={"content": "inputs.article_text"},
+ retry=RetryPolicy(max_attempts=2, delay_seconds=300, backoff=1.0,
+ on=["timeout", "unreachable", "server_error", "rate_limited"],
+ manual=True))
+```
+
+**Per step and per plugin, which is the whole reason it is declared.** A transcript that was not ready an hour ago and a host that timed out are different waits, and retrying the whole outlet to get one of them is how a source ends up fetched four times an evening. `max_attempts` counts the first attempt, so the default of 1 is "no retry" and is what every plugin that declares nothing gets — behaviour unchanged from before any of this existed.
+
+**`on` names failure categories from `desk.tools.failures`**, and a name that is not one of them is refused where the plugin is written rather than becoming a policy that silently matches nothing. Name the ones that could answer differently next time. A consent wall is a category and belongs in nobody's list: it is the same answer however many times it is asked for. Where the server named its own wait — a `Retry-After` — that is honoured in place of `delay_seconds`, but it buys no extra attempt: the bound is the plugin's and the wait is the world's.
+
+**`manual` is whether the reader offers a person the retry** the policy would not take on its own. The app still checks that the offer means anything: the plugin must be active, and the item must actually have been tried and come back empty. A target's Fetch switch is *not* checked for a manual retry — Fetch governs the evening's automatic acquisition, not a repair somebody asked for.
+
+## Minimum example
+
+```python
+# plugins/mysource.py (or desk/sources/mysource.py)
+from desk.sources import Source, Item
+
+class MySource(Source):
+ name = "mysource" # the key under `sources:` in config.yaml
+ description = "One line shown in the Sources tab."
+ subscription_field = "accounts" # or "feeds" for URL-shaped subscriptions
+
+ def fetch(self):
+ for target, label in self.targets(): # subscriptions from config/UI
+ yield Item(source_id="abc123", location="https://example.com/x",
+ title="Headline", excerpt="Summary", subscription=target)
+```
+
+```yaml
+# config.yaml
+plugin_paths: ["plugins"]
+sources:
+ mysource:
+ enabled: true
+ accounts: ["someone"]
+```
+
+Optional, both defaulting to nothing: `discover() -> [Suggestion]` (accounts worth following) and `check() -> [CheckResult]` (does each subscription still resolve — free requests only, it runs for every source at once).
+
+## What a plugin is given
+
+- `self.options` — its own config block. By pull time the subscription list has been replaced with the fetch-enabled rows from the Sources tab.
+- `self.state` — a `MutableMapping[str, str]` backed by the database, for what it must remember between pulls (a spend counter, which episodes are done). Driven directly it's a plain dict.
+
+**`self.state` is one namespace shared by every plugin**, not one per source. Nothing prefixes your keys for you, so prefix them yourself: `x.py` writes `x_spend_2026_09`, and `PodcastSource` derives its done-marker key from the subclass name. A bare key like `last_seen` is a collision waiting for the second plugin that wants one, and the symptom is one source silently reading another's bookkeeping.
+
+Helpers worth knowing: `strip_html`, `parse_timestamp`, `entry_time`, `split_label`, `outlet_name`, `MULTIWORD_OUTLETS`, `http_failure`, `USER_AGENT`, `check_feed`.
+
+Two of those read names, and which one you want depends on who wrote the string. `split_label` reads a label *you* wrote and cuts it into outlet and section — "Guardian Tech" into the two — with `MULTIWORD_OUTLETS` as the list of names that contain a space themselves, so "Hacker News Best" does not come out as an outlet called Hacker. `outlet_name` reads a name a *server* gave for itself and cuts the tagline off it, because both a feed's `` and a site's own title tend to arrive as the SEO version.
+
+## Limits
+
+- **One file, no state but `self.state`.** No store, no queue, no cross-source access. That's the deal that keeps a new source one file.
+- **A failed target and an empty one are different answers.** Yielding nothing means there was nothing new; raising means something is wrong and you are owed a notice about it — `desk/notices.py` decides from the exception whether it reads as the world misbehaving or as a bug of ours. A source with several subscriptions gets both at once by yielding everything that worked and raising once at the end for what didn't, which is what stops one dead feed costing you the other nineteen. `rss.FeedsFailed` is that exception.
+- **`fetch` is synchronous and blocking.** Runs in the worker's thread. Async, rate-limiting and *timeouts* are yours. A slow source holds the pull; a source that fetches without a deadline holds it for ever, because cancellation is cooperative and Kill cannot reach a thread parked in a socket read. The case to watch for is a library that does its own fetching: `feedparser.parse(url)` and `requests` with no `timeout=` both wait indefinitely by default, so `rss.py` fetches the bytes itself and hands the parser a document.
+- **No sandbox, no versioning.** A plugin file is imported and trusted; a plugin that raises on import is logged and skipped, one that raises inside `fetch` fails that pull's source.
+- **`name` must be unique and set** — no `name`, not discovered.
+- **Discovery is flat**: `*.py` in a directory, not packages, not installed distributions.
+- **A source can't create jobs.** It yields items; what happens next (body, vectors, tags) is the app's ingest job.
diff --git a/desk/sources/spec.py b/desk/sources/spec.py
new file mode 100644
index 0000000..d09bdff
--- /dev/null
+++ b/desk/sources/spec.py
@@ -0,0 +1,1012 @@
+"""What a plugin declares about itself, as types rather than as convention.
+
+A plugin is still one file with a `Source` subclass in it — `registry.py`
+finds it and `base.py` is the contract for *fetching*. This is the other half,
+and it is declarative: instead of a plugin calling a model, cutting a
+transcript or emitting markup, it says which shared tools acquire its content,
+what the shared summariser and tagger should do with it, and how its fields
+are laid out. The application owns the doing. See `docs/plugin-rework.md`.
+
+ class HackerNews(Source):
+ name = "hacker_news"
+ spec = PluginSpec(api_version=1, id=name, label="Hacker News", ...)
+
+Nothing executes a spec yet. This step introduces the contract and its
+validation so that later steps have one thing to implement against, rather
+than each inventing its own declaration and then reconciling them.
+
+Three rules hold this together:
+
+*Versioned.* `api_version` is the whole contract, not a part of it, which is
+why every type it needs is declared here at once. A field a later step starts
+*reading* costs nothing; a field a later step has to *add* means every plugin
+written against version 1 was written against a contract that did not say what
+it claimed to say.
+
+*Validated twice, for two different reasons.* What is wrong on its face — a
+step that needs one that isn't there, a cycle, a step writing over a summary —
+is a pydantic error raised where the plugin is declared, so a bad spec fails
+at import and `registry._load_file` logs it against that file and keeps every
+other plugin. What can only be judged by the app — is `transcribe_media` a
+tool this application has, is `segment` a kind it lays out — is `problems()`,
+because `desk` has no list of either and inventing one here would be the
+contract deciding what an app on it supports. See `Vocabulary`.
+
+*Undeclared is a legitimate state.* Every plugin that exists today declares no
+spec, and `spec_of` gives it one marked `adapted`: honest about the identity
+it does have and silent about everything it never said. An adapted spec is
+never invalid, because there is nothing in it that a plugin author wrote.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Any, Literal
+
+from pydantic import model_validator
+
+from desk.models import Model
+from desk.tools import failures as failures_tool
+from desk.tools import marks as marks_tool
+
+#: The contract this module defines. A plugin naming any other number is not
+#: refused because it is wrong but because nothing here knows what it means.
+API_VERSION = 1
+
+#: A plugin id, and a target id: stable machine names. Not the displayed
+#: outlet name — that is `label`, it is written for people, and it changes.
+#: An id that changed would orphan every stored item that recorded it.
+_MACHINE_NAME = re.compile(r"^[a-z][a-z0-9_]*$")
+
+#: Where an acquisition step is allowed to write. The plugin and its steps own
+#: inputs; the shared summariser and tagger own outputs. A step that could
+#: write `outputs.summary` would be a plugin generating its own summary
+#: through the back door, which is the one thing this contract exists to stop.
+_OWNED_BY_ACQUISITION = "inputs."
+
+#: The marks are a shared vocabulary with shared glyphs and shared filters; a
+#: plugin declares the *evidence* for one, never a mark of its own.
+_MARK_SLUGS = frozenset(marks_tool.BY_SLUG)
+
+#: What a failed step can be called. Shared for the same reason the marks are:
+#: the categories are what the runner *produces*, so a policy naming one they
+#: do not contain is a policy that matches nothing, and the place to find that
+#: out is the plugin file rather than an evening of retries that never happen.
+_FAILURES = failures_tool.CATEGORIES
+
+
+# --------------------------------------------------------------- acquisition
+
+class RetryPolicy(Model):
+ """When a failed acquisition step is worth running again.
+
+ Per step, not per plugin and not per pull: a transcript that was not ready
+ an hour ago and an article whose host timed out are different waits, and
+ retrying the whole outlet to get one of them is how a source ends up
+ fetched four times an evening.
+ """
+
+ #: Counting the first attempt, so 1 means "no retry" and reads as it says.
+ max_attempts: int = 1
+ delay_seconds: int = 0
+ #: Multiplied into the delay for each further attempt. 1.0 is a fixed wait.
+ backoff: float = 1.0
+ #: Which failures are worth another attempt — `timeout`, `server_error`.
+ #: Empty retries none of them, which is what `max_attempts=1` already says.
+ #: A consent wall is deliberately not in any app's list by default: it is
+ #: the same answer however many times it is asked for. The names are
+ #: `desk.tools.failures.CATEGORIES`; a misspelt one is refused here rather
+ #: than becoming a policy that quietly matches nothing.
+ on: list[str] = []
+ #: Whether the reader offers a person the retry this policy would not take
+ #: on its own.
+ manual: bool = False
+
+ @model_validator(mode="after")
+ def _bounded(self) -> RetryPolicy:
+ if not 1 <= self.max_attempts <= 10:
+ raise ValueError(
+ f"max_attempts is {self.max_attempts}; it counts the first "
+ "attempt, so it must be between 1 and 10")
+ if self.delay_seconds < 0 or self.backoff < 1.0:
+ raise ValueError("a retry delay cannot be negative and backoff "
+ "cannot shrink it")
+ unknown = sorted(set(self.on) - _FAILURES)
+ if unknown:
+ raise ValueError(
+ f"retry on {', '.join(unknown)}: not a failure category "
+ f"({', '.join(sorted(_FAILURES))})")
+ return self
+
+
+class Step(Model):
+ """One named piece of acquisition: a shared tool, and what it reads and
+ writes.
+
+ `inputs` maps the tool's own parameters to where their values come from;
+ `outputs` maps what it produced to the item fields it lands in. Both sides
+ named, so a reader of the plugin can see what a step will change without
+ knowing the tool, and so the runner can refuse a step that would write
+ somewhere it does not own.
+ """
+
+ id: str
+ tool: str
+ inputs: dict[str, str] = {}
+ outputs: dict[str, str] = {}
+ #: Step ids in the same profile that must succeed first. The extraction of
+ #: an article needs the destination URL a discovery hook worked out.
+ needs: list[str] = []
+ #: A field path that must have a value for this step to run at all. Empty
+ #: runs it always.
+ when: str = ""
+ retry: RetryPolicy = RetryPolicy()
+
+ @model_validator(mode="after")
+ def _writes_only_inputs(self) -> Step:
+ if not _MACHINE_NAME.match(self.id):
+ raise ValueError(f"step id {self.id!r} is not a machine name")
+ for produced, destination in self.outputs.items():
+ if not destination.startswith(_OWNED_BY_ACQUISITION):
+ raise ValueError(
+ f"step {self.id!r} writes {produced!r} to {destination!r}; "
+ "an acquisition step may only write `inputs.*` — outputs "
+ "belong to the shared summariser and tagger")
+ return self
+
+
+class Discovery(Model):
+ """How a plugin finds items in the first place.
+
+ Separate from acquisition because it answers a different question: not
+ "what is in this item" but "which items are there". `plugin` means the
+ class's own `fetch`, which is what every plugin does today.
+ """
+
+ tool: Literal["plugin", "rss", "api"] = "plugin"
+ #: One fixed feed, for a plugin whose whole catalogue is a single address.
+ #: A plugin with more than one declares them in `targets` instead, and the
+ #: discovery helper reads them from there; two competing lists of feeds is
+ #: the thing this must not become.
+ feed_url: str = ""
+ options: dict[str, Any] = {}
+
+
+class RSS(Discovery):
+ """`discovery=RSS(feed_url=...)`, which is the common case spelled out."""
+
+ tool: Literal["plugin", "rss", "api"] = "rss"
+
+
+# ---------------------------------------------------------------- the tasks
+
+class SummaryPolicy(Model):
+ """What the shared summariser should do with this profile's items.
+
+ Structured parameters settle length and eligibility; `instructions` is
+ editorial and cannot change the schema, the fields written, or whether an
+ item is eligible at all. It is added to the same model call rather than
+ being a second pass over the answer.
+ """
+
+ #: Required, and required even when false: a profile that says nothing
+ #: about summarising is a profile whose author has not decided, and the
+ #: cost of guessing is a model bill or a queue of items silently skipped.
+ enabled: bool
+ #: In order; the first field with usable content is the one summarised.
+ #: Not concatenated — the same words twice is not more context.
+ input_fields: list[str] = []
+ #: Whether a partial article — a feed's first paragraph — may be summarised
+ #: as though it were the piece.
+ require_complete: bool = True
+ #: Below this, the item is skipped with a reason rather than summarised.
+ #: A long article and a two-minute segment want different floors.
+ min_words: int = 0
+ target_words: int = 140
+ instructions: str = ""
+
+ @model_validator(mode="after")
+ def _has_something_to_read(self) -> SummaryPolicy:
+ return _needs_inputs(self, "summary")
+
+
+class TaggingPolicy(Model):
+ """What the shared tagger should do with this profile's items.
+
+ The vocabulary stays central: a plugin adds emphasis, never a tag
+ namespace of its own. Prefer acquired content as the input rather than the
+ summary, so tagging does not silently wait on a summary that failed.
+
+ Naming a field says *which* text is read and never how much of it. An app
+ that picks from a fixed list of labels may read the opening of it and no
+ more — newsdesk does, at four hundred characters — so naming a transcript
+ here is not a request to send a transcript.
+ """
+
+ enabled: bool
+ input_fields: list[str] = []
+ #: Read for context but not as the thing being tagged: the title, the
+ #: publisher's own categories.
+ context_fields: list[str] = []
+ instructions: str = ""
+
+ @model_validator(mode="after")
+ def _has_something_to_read(self) -> TaggingPolicy:
+ return _needs_inputs(self, "tagging")
+
+
+def _needs_inputs[T: SummaryPolicy | TaggingPolicy](policy: T, name: str) -> T:
+ """An enabled task with no input field is a task that will look at every
+ item and find nothing to read — reported here rather than as an empty
+ result for the whole archive."""
+ if policy.enabled and not policy.input_fields:
+ raise ValueError(f"{name} is enabled but names no input_fields")
+ return policy
+
+
+class Tasks(Model):
+ """The shared tasks, both declared explicitly. See `SummaryPolicy.enabled`
+ for why neither has a default."""
+
+ summary: SummaryPolicy
+ tagging: TaggingPolicy
+
+
+# ---------------------------------------------------------- how it is shown
+
+class Section(Model):
+ """One band of a view: a standard field, bound to a shared component.
+
+ Not markup. Ordering the list is what moves a player above or below the
+ links, and that is the whole of the layout language on purpose — safe
+ links, responsive behaviour, loading and error states belong to the
+ renderer, and every plugin that could express them would express them
+ differently.
+ """
+
+ component: str
+ field: str
+ label: str = ""
+ #: For a `links` section: show only links with this role — `reading`,
+ #: `discussion`, `source`.
+ role: str = ""
+ #: For a `media` section: which media reference, when an item has several.
+ media: str = ""
+ collapsed: bool = False
+
+
+class View(Model):
+ """One of the reader's exclusive views, as this profile fills it in."""
+
+ label: str
+ sections: list[Section] = []
+ #: Field paths that must have a value for this view to be offered. A
+ #: Summary view with no summary is disabled and says why, rather than
+ #: rendering an empty panel or a placeholder dressed as an abstract.
+ requires: list[str] = []
+ #: What to show here when `requires` cannot be met: another view, with its
+ #: own label, its own requirement and its own sections. The control keeps
+ #: its place in the drawer and renames itself to say what is actually in
+ #: it, which is the difference between a reader who sees the feed's own
+ #: two sentences and one who sees a greyed-out button.
+ #:
+ #: The queue has done this since it existed — `QueueLayout.preview` is an
+ #: ordered list and the first non-empty field wins. A row that falls
+ #: through to the feed's text while the pane behind it refuses to show
+ #: anything is the same item answering two different ways.
+ instead: View | None = None
+
+ def chain(self) -> list[View]:
+ """This view and every fallback under it, best first."""
+ return [self, *(self.instead.chain() if self.instead else [])]
+
+
+class QueueLayout(Model):
+ """The row, which is a decision rather than a read: a title, the best
+ preview available, and whatever metadata makes the decision."""
+
+ #: In order, first non-empty wins. A generated summary, then the feed's
+ #: own text — and a placeholder is never one of these.
+ preview: list[str] = []
+ metadata: list[str] = []
+ #: Whether the row names the host its item points at, beside the outlet.
+ #: True where the outlet is not the publisher — an aggregator's front page
+ #: — because there the host is half of what the decision to read is made
+ #: on, and "Hacker News" alone does not say whether this is a kernel
+ #: mailing list or a newspaper. False everywhere the two are the same
+ #: name twice.
+ destination: bool = False
+
+
+class OriginalView(Model):
+ """The destination itself, embedded. Hidden on mobile, and never shown
+ over an outlet that refuses framing however this is labelled."""
+
+ field: str
+ label: str = "Original"
+ #: The marks whose items this view is not offered for. Per item, because
+ #: the mark is: a piece behind the paywall frames the paywall, and the
+ #: piece beside it in the same feed frames the article. The layout does
+ #: not change — the control stays where it is, greyed, with the reason on
+ #: it — because which items are readable is not a fact about the outlet.
+ #: An outlet where *nothing* is readable is a `Wall`, which is a different
+ #: thing and has a different flow. Mark slugs, from the shared vocabulary.
+ blocked_by: list[str] = ["paywalled"]
+
+ @model_validator(mode="after")
+ def _known_marks(self) -> OriginalView:
+ unknown = sorted(set(self.blocked_by) - _MARK_SLUGS)
+ if unknown:
+ raise ValueError(
+ f"blocked_by names {', '.join(repr(m) for m in unknown)}, "
+ f"which is not among the shared marks "
+ f"({', '.join(sorted(_MARK_SLUGS))})")
+ return self
+
+
+class ExternalLink(Model):
+ """The new-tab action beside the views. Not a view: it leaves."""
+
+ field: str
+ label: str = ""
+
+
+class Presentation(Model):
+ """Declared once per profile, never per stored item.
+
+ The drawer is fixed and the content is not. Every item is read through the
+ same four controls in the same order, whatever produced it:
+
+ - `summary`, the short version: an abstract, or the feed's own teaser
+ - `text`, the whole words: an article, a transcript, a post
+ - `original`, the thing itself inside the pane: the page framed, or the
+ recording playing
+ - `external`, the way out to a new tab
+
+ A plugin decides which of them it fills, what each is called, what sits
+ in each and what each falls back to. It never adds a fifth: a reader
+ moving from a newspaper to a podcast to a front-page link keeps one hand
+ position, and a sticky choice of view means the same thing on every item.
+
+ `header` is what holds whichever view is on: where an item points and
+ where it is discussed. An aggregator's destination is not its own outlet,
+ and a reader in the Text view of a Hacker News link is owed both names
+ without leaving it.
+ """
+
+ queue: QueueLayout = QueueLayout()
+ #: Shown above the pane in every view, including one that cannot be
+ #: shown. `links` sections only: `inputs.original_url` is the item's own
+ #: destination, `inputs.links` with a `role` is the typed links. A section
+ #: with nothing in it drops out, so a show whose notes link nothing has no
+ #: header rather than an empty one.
+ header: list[Section] = []
+ default_view: str = ""
+ views: dict[str, View] = {}
+ original: OriginalView | None = None
+ external: ExternalLink | None = None
+
+ @model_validator(mode="after")
+ def _default_is_a_view(self) -> Presentation:
+ if self.default_view and self.default_view not in self.views:
+ raise ValueError(
+ f"default_view is {self.default_view!r}, which is not one of "
+ f"the declared views ({', '.join(sorted(self.views)) or 'none'})")
+ return self
+
+ @model_validator(mode="after")
+ def _header_is_links(self) -> Presentation:
+ wrong = sorted({s.component for s in self.header} - {"links"})
+ if wrong:
+ raise ValueError(
+ f"the header holds links, and this one declares "
+ f"{', '.join(repr(c) for c in wrong)}; prose and media belong "
+ "in a view")
+ return self
+
+
+#: What an ordinary published article looks like in a reader, as a preset a
+#: plugin can name instead of restating. The shapes are the contract's; which
+#: of them an application actually renders is its own business, and
+#: `problems()` is where the two meet.
+#:
+#: Summary before Text is the order the shell has, not a preference: a reader
+#: who wants the piece presses Text, and the view they were last in is
+#: remembered. A profile that wants its own arrangement declares one rather
+#: than editing this.
+ARTICLE_READER = Presentation(
+ queue=QueueLayout(preview=["outputs.summary", "inputs.feed_text"],
+ metadata=["outlet"]),
+ #: The page this item is, named by its host. What a reader in the Text
+ #: view is owed without leaving it: which page these words came from.
+ header=[Section(component="links", field="inputs.original_url")],
+ default_view="text",
+ views={
+ # An outlet that is never summarised, or one whose article we could
+ # not fetch, still published two sentences in its feed. They are the
+ # shortest version of the piece we have, which is what this control is
+ # for, so it shows them rather than greying out — and says "RSS feed",
+ # because they are the publisher's own teaser and calling a teaser a
+ # summary is how a reader comes to think our summaries are terrible.
+ "summary": View(label="Summary", requires=["outputs.summary"], sections=[
+ Section(component="prose", field="outputs.summary"),
+ ], instead=View(label="RSS feed", requires=["inputs.feed_text"],
+ sections=[
+ Section(component="prose", field="inputs.feed_text"),
+ ])),
+ "text": View(label="Text", requires=["inputs.article_text"], sections=[
+ Section(component="prose", field="inputs.article_text"),
+ Section(component="links", field="inputs.links", label="Links"),
+ ]),
+ },
+ original=OriginalView(field="inputs.original_url"),
+ external=ExternalLink(field="inputs.original_url"),
+)
+
+
+#: What a social post looks like, as the preset a network names. A post is
+#: its own text and a link out of it: there is no article to extract and no
+#: abstract to wait for, so the profile declares one view rather than
+#: offering a Summary that could never become ready.
+#:
+#: `inputs.feed_text` is the post itself and not a teaser for something
+#: else — the same field an outlet's feed fills, read here as what the
+#: network published rather than as a stub of an article behind it.
+#:
+#: The article section below it is the link post: a post that pointed at
+#: something and had it fetched shows both, in the order they were written.
+#: It is optional and a post without one simply has one band — which is why
+#: `requires` names the post text and not the article.
+POST_READER = Presentation(
+ # The same order every row is built in, and not a narrower claim: the
+ # queue falls through an abstract to the feed's own text whatever the
+ # profile, and a post declaring only the second half would be describing
+ # its own policy rather than the queue. There is never a summary here to
+ # come first.
+ queue=QueueLayout(preview=["outputs.summary", "inputs.feed_text"],
+ metadata=["outlet"]),
+ #: The page this item is, named by its host. What a reader in the Text
+ #: view is owed without leaving it: which page these words came from.
+ header=[Section(component="links", field="inputs.original_url")],
+ default_view="text",
+ views={
+ "text": View(label="Post", requires=["inputs.feed_text"], sections=[
+ Section(component="prose", field="inputs.feed_text"),
+ Section(component="prose", field="inputs.article_text",
+ label="What it linked to"),
+ Section(component="links", field="inputs.links", label="Links"),
+ ]),
+ },
+ original=OriginalView(field="inputs.original_url"),
+ external=ExternalLink(field="inputs.original_url"),
+)
+
+
+#: What an outlet we cannot read at all looks like: the text its feed gave us,
+#: and the way out to the page. The flow a profile declaring a `Wall` wants —
+#: there is no article to put under a Text control, no abstract will ever be
+#: written, and the destination frames the gate, so none of the three is
+#: offered. `ARTICLE_READER` with two permanently disabled buttons says the
+#: same thing, and says it again on every item for as long as the wall stands.
+#:
+#: The feed's text sits in the summary slot, where `ARTICLE_READER` puts it
+#: when an abstract is missing, so that the control is in the same place and
+#: keeps the same sticky choice as a reader moves between outlets.
+FEED_READER = Presentation(
+ queue=QueueLayout(preview=["outputs.summary", "inputs.feed_text"],
+ metadata=["outlet"]),
+ #: The page this item is, named by its host. What a reader in the Text
+ #: view is owed without leaving it: which page these words came from.
+ header=[Section(component="links", field="inputs.original_url")],
+ default_view="summary",
+ views={
+ "summary": View(label="RSS feed", requires=["inputs.feed_text"],
+ sections=[
+ Section(component="prose", field="inputs.feed_text"),
+ Section(component="links", field="inputs.links", label="Links"),
+ ]),
+ },
+ external=ExternalLink(field="inputs.original_url"),
+)
+
+
+#: What a link to somebody else's page looks like: an aggregator's front page,
+#: where the item is a pointer and the outlet is not the publisher. The views
+#: are `ARTICLE_READER`'s, because what is behind the link is an article, or
+#: fails to be one, like any other. The header is what differs: the host the
+#: link goes to and the thread it is discussed in, above every view — so both
+#: are there when the page would not extract, which is exactly when a reader
+#: needs them, and the Text view is the article and nothing else.
+LINK_READER = Presentation(
+ queue=QueueLayout(preview=["outputs.summary", "inputs.feed_text"],
+ metadata=["outlet"], destination=True),
+ header=[
+ Section(component="links", field="inputs.original_url"),
+ Section(component="links", field="inputs.links", role="discussion"),
+ ],
+ default_view="text",
+ views={
+ "summary": View(label="Summary", requires=["outputs.summary"], sections=[
+ Section(component="prose", field="outputs.summary"),
+ ], instead=View(label="RSS feed", requires=["inputs.feed_text"],
+ sections=[
+ Section(component="prose", field="inputs.feed_text"),
+ ])),
+ "text": View(label="Text", requires=["inputs.article_text"], sections=[
+ Section(component="prose", field="inputs.article_text"),
+ ]),
+ },
+ original=OriginalView(field="inputs.original_url"),
+ external=ExternalLink(field="inputs.original_url"),
+)
+
+
+#: What a stretch of a recording looks like: the player's strip, then what
+#: was said. Summary, Transcript, Watch — the same three places as an article,
+#: under the names of what is in them.
+#:
+#: No fallback to `inputs.feed_text` under Summary, which is what an article's
+#: does: a segment's feed text is the *episode's* whole timestamped outline,
+#: so falling through to it would put the running order under every story cut
+#: out of it. The transcript is a view you choose and not the thing under the
+#: abstract, because it arrives as captions with the names mangled.
+#:
+#: The articles the hosts were reading are the header, so they stay in reach
+#: while the recording plays. A show whose notes link nothing has no header,
+#: and needs no second preset for it.
+RECORDING_READER = Presentation(
+ queue=QueueLayout(preview=["outputs.summary", "inputs.feed_text"],
+ metadata=["outlet", "inputs.media"]),
+ header=[Section(component="links", field="inputs.links", role="reading")],
+ default_view="summary",
+ views={
+ "summary": View(label="Summary", requires=["outputs.summary"], sections=[
+ Section(component="media", field="inputs.media"),
+ Section(component="prose", field="outputs.summary"),
+ ]),
+ "text": View(label="Transcript", requires=["inputs.transcript"],
+ sections=[
+ Section(component="media", field="inputs.media"),
+ Section(component="prose", field="inputs.transcript"),
+ ]),
+ },
+ # Watch, not Original: the destination is the recording, playing at this
+ # segment's own offset.
+ original=OriginalView(field="inputs.media", label="Watch"),
+ external=ExternalLink(field="inputs.original_url"),
+)
+
+
+# -------------------------------------------------------------- the targets
+
+class Target(Model):
+ """One feed or one account, as the plugin declares it.
+
+ `id` is the archive's name for it and never the URL or the label: a feed
+ that moves, or a handle that renames itself, must stay the same thing it
+ was, with its items, votes and read state on it.
+ """
+
+ id: str
+ label: str = ""
+ #: The address, handle or query this target is. Empty for an editable
+ #: target, which gets its locator when someone adds it.
+ locator: str = ""
+ section: str = ""
+ profile: str = "default"
+ #: What this target does before anyone touches the switches. Seeded once;
+ #: a later reload must not reassert them over a choice someone made.
+ fetch: bool = True
+ show: bool = True
+
+
+class TargetSpec(Model):
+ """Whether the plugin's targets are a fixed catalogue or a list a person
+ adds to — a plugin capability, not a default for every outlet.
+
+ A publisher declares its own feeds and there is no Add box: another
+ masthead's feed is another plugin, with that publisher's own extraction,
+ marks and retries. A social network declares no accounts and the box is
+ the point.
+ """
+
+ mode: Literal["catalog", "editable"]
+ #: What one of these *is*: `rss_feed`, `account`. The app decides which
+ #: types it can add and validate.
+ kind: str
+ defaults: list[Target] = []
+ #: Names the check a typed value must pass before it is accepted —
+ #: `bluesky_handle`. Empty accepts anything the app's own type check does.
+ validator: str = ""
+ default_fetch: bool = True
+ default_show: bool = True
+
+ @model_validator(mode="after")
+ def _ids_are_stable_and_unique(self) -> TargetSpec:
+ seen = [t.id for t in self.defaults]
+ if len(seen) != len(set(seen)):
+ raise ValueError("two targets share an id: "
+ f"{sorted({i for i in seen if seen.count(i) > 1})}")
+ for target in self.defaults:
+ if not _MACHINE_NAME.match(target.id):
+ raise ValueError(f"target id {target.id!r} is not a machine name")
+ if self.mode == "catalog" and not self.defaults:
+ raise ValueError("a catalog declares its targets, and this one "
+ "declares none; use mode='editable' for a plugin "
+ "whose targets are added by a person")
+ return self
+
+
+# ----------------------------------------------------------------- the wall
+
+class Wall(Model):
+ """What an outlet answers with instead of its articles — all of them.
+
+ Not a mark, and the difference is the whole reason this exists. A mark is
+ a fact about one item: this piece is behind the paywall, that one is an
+ advert. The archive filters on it, refuses the embed for it, and leaves
+ everything else about the outlet alone, because the piece next to it in
+ the same feed is an ordinary readable article. A wall is a fact about the
+ plugin: there is no article to fetch on any URL it produces, no abstract
+ will ever be written from one, and the destination frames the gate rather
+ than the piece. A walled outlet is not an outlet where everything is
+ paywalled — it is one we cannot read at all, and what it needs is a
+ different flow rather than the ordinary one with its controls greyed out
+ on every item for ever.
+
+ Declared rather than detected because the consequence is the *layout*.
+ `desk.tools.walls` learns a wall from three identical bodies and stops
+ fetching, which is right, is about a host, and arrives long after the
+ reading pane has had to decide what to offer. A plugin that learns to get
+ past its wall — a browser running the CMP's JavaScript for a
+ tracking-consent gate — deletes this declaration and its profile is an
+ ordinary article profile again, flow included.
+ """
+
+ #: In the vocabulary the runner already reports failures in, so a declared
+ #: wall and a detected one are the same word on the row: `consent_wall`
+ #: for a tracking-consent gate, `denied` for a login or a bot check.
+ kind: Literal["consent_wall", "denied"]
+ #: Why, in words. For whoever reads the plugin, and for the reader, who is
+ #: owed an answer to "where is the rest of this".
+ reason: str = ""
+
+ @model_validator(mode="after")
+ def _says_why(self) -> Wall:
+ if not self.reason.strip():
+ raise ValueError(
+ "a wall needs a reason: it is what the reading pane tells "
+ "somebody who can see there is no article here")
+ return self
+
+
+# ---------------------------------------------------------------- the marks
+
+class MarkRule(Model):
+ """How this outlet says "paid" or "advert" in its own text.
+
+ The vocabulary and the glyphs stay shared; only where the evidence is
+ found moves into the plugin. `strip` cuts a matched token out of the
+ displayed title — the original is kept, because a title with its token
+ already cut off cannot be reclassified later.
+ """
+
+ #: Where to look: a field path, or a publisher category list.
+ field: str
+ method: Literal["token", "prefix", "suffix", "contains", "category"]
+ value: str
+ mark: str
+ strip: bool = False
+
+ @model_validator(mode="after")
+ def _known_mark(self) -> MarkRule:
+ if self.field not in {"title", "inputs.source_title", "inputs.feed_text",
+ "inputs.source_categories"}:
+ raise ValueError(f"unsupported mark input {self.field!r}")
+ if not self.value.strip():
+ raise ValueError("a mark rule needs a non-empty value")
+ categories = self.field == "inputs.source_categories"
+ if categories != (self.method == "category"):
+ raise ValueError("category matching requires source_categories")
+ if self.strip and self.field not in {"title", "inputs.source_title"}:
+ raise ValueError("only a title token may be stripped")
+ if self.mark not in _MARK_SLUGS:
+ raise ValueError(
+ f"mark {self.mark!r} is not one of the shared marks "
+ f"({', '.join(sorted(_MARK_SLUGS))}); a plugin declares the "
+ "evidence for a mark, not a mark of its own")
+ return self
+
+
+# ------------------------------------------------------------- the contract
+
+class Profile(Model):
+ """One shape of content a plugin produces, end to end.
+
+ A show that publishes both whole episodes and cut segments declares two of
+ these rather than being two plugins, and each item records which one it
+ is. That recorded name is how an item is laid out and processed for the
+ rest of its life — never a guess made later from whether it happens to
+ have a video key, or from what its outlet is called.
+ """
+
+ #: `article`, `post`, `segment` in newsdesk. A free string here on
+ #: purpose: which shapes exist is the app's business — see `Item.kind` —
+ #: and the app's `Vocabulary` is what checks it.
+ kind: str
+ acquisition: list[Step] = []
+ #: None inherits the plugin rules; an explicit list replaces them.
+ marks: list[MarkRule] | None = None
+ #: What this outlet answers with instead of its articles, when it does.
+ #: None is the ordinary case and says nothing; a `Wall` says there is no
+ #: article behind any of these URLs, which is a different flow and not a
+ #: worse version of this one. See `Wall`, and `FEED_READER` for the flow.
+ #:
+ #: The acquisition step stays declared beside it on purpose: it is what
+ #: would run if the wall came down, and running it is how `desk.tools.walls`
+ #: finds out that it has not.
+ wall: Wall | None = None
+ tasks: Tasks
+ presentation: Presentation = Presentation()
+
+ @model_validator(mode="after")
+ def _a_wall_is_not_a_view(self) -> Profile:
+ """A walled profile may not offer the destination as a view.
+
+ The page frames perfectly well — a consent gate is a real page, and
+ `framing` has no way to tell it from an article — so this is not
+ something the app can find out for itself. And it is true of every
+ item from this outlet, for as long as the wall stands, which makes it
+ a layout decision rather than a control to grey out one item at a
+ time. The way to the page is `ExternalLink`: in a tab of their own a
+ reader can accept the gate and read the piece, which is the whole
+ difference between useless and the way out.
+ """
+ original = self.presentation.original
+ if self.wall and original and original.field.split(".")[-1] != "media":
+ raise ValueError(
+ f"this profile declares a {self.wall.kind} and an Original "
+ "view of the destination, which would frame the wall on every "
+ "item; declare the way out as `external` instead — see "
+ "`FEED_READER`")
+ return self
+
+ @model_validator(mode="after")
+ def _acquisition_is_a_graph(self) -> Profile:
+ steps = {s.id: s for s in self.acquisition}
+ if len(steps) != len(self.acquisition):
+ raise ValueError("two acquisition steps share an id")
+ for step in self.acquisition:
+ for needed in step.needs:
+ if needed not in steps:
+ raise ValueError(
+ f"step {step.id!r} needs {needed!r}, which is not a "
+ f"step of this profile ({', '.join(steps) or 'none'})")
+ _no_cycles(steps)
+ return self
+
+
+def _no_cycles(steps: dict[str, Step]) -> None:
+ """Refuse a dependency loop where it is written rather than where it
+ hangs. Depth-first with three colours: an edge back into the path being
+ walked is the cycle, and the path is the useful half of the message."""
+ walking: list[str] = []
+ done: set[str] = set()
+
+ def walk(current: str) -> None:
+ if current in done:
+ return
+ if current in walking:
+ loop = " -> ".join([*walking[walking.index(current):], current])
+ raise ValueError(f"acquisition steps depend on each other: {loop}")
+ walking.append(current)
+ for needed in steps[current].needs:
+ walk(needed)
+ walking.pop()
+ done.add(current)
+
+ for step_id in steps:
+ walk(step_id)
+
+
+class PluginSpec(Model):
+ """Everything a plugin declares. One version, one id, one set of profiles.
+
+ Structural faults raise here, where the plugin is written. What only the
+ application can judge is `problems()`.
+ """
+
+ api_version: int
+ #: Stable and machine-readable. Conventionally the class's `name`, which
+ #: is already the config key and already on every stored item.
+ id: str
+ #: The plugin's own revision. Bumping it makes outputs produced under the
+ #: old one recognisably stale without regenerating the archive.
+ version: str = "1"
+ #: What a person is shown. Free to change; `id` is not.
+ label: str = ""
+ discovery: Discovery = Discovery()
+ targets: TargetSpec | None = None
+ marks: list[MarkRule] = []
+ profiles: dict[str, Profile]
+ #: True when nothing in the plugin file declared this and `spec_of` read
+ #: it off the class. Such a plugin runs exactly as it did before the
+ #: contract existed; nothing about it is a claim its author made.
+ adapted: bool = False
+
+ @model_validator(mode="after")
+ def _coherent(self) -> PluginSpec:
+ if self.api_version != API_VERSION:
+ raise ValueError(
+ f"plugin {self.id!r} declares api_version {self.api_version}; "
+ f"this application implements version {API_VERSION}")
+ if not _MACHINE_NAME.match(self.id):
+ raise ValueError(
+ f"plugin id {self.id!r} is not a machine name: lowercase "
+ "letters, digits and underscores, starting with a letter")
+ if not self.profiles:
+ raise ValueError(f"plugin {self.id!r} declares no profiles; every "
+ "plugin has at least a `default`")
+ for name, target in _targets_of(self):
+ if target.profile not in self.profiles:
+ raise ValueError(
+ f"target {name!r} names profile {target.profile!r}, which "
+ f"this plugin does not declare "
+ f"({', '.join(sorted(self.profiles))})")
+ return self
+
+
+def _targets_of(spec: PluginSpec) -> list[tuple[str, Target]]:
+ return [(t.id, t) for t in (spec.targets.defaults if spec.targets else [])]
+
+
+# ------------------------------------------------------- what the app allows
+
+@dataclass(frozen=True)
+class Vocabulary:
+ """The names one application answers to, against which a spec is checked.
+
+ Deliberately not a constant in this module. `desk` does not know which
+ acquisition tools an app has wired up, which kinds it lays out, or which
+ components it renders — and a list here would be this contract deciding
+ what an application on it supports, which is the opposite of the point.
+ An empty set means the app does not restrict that name, which is the
+ right answer for `fields` until the typed item fields exist.
+ """
+
+ tools: frozenset[str] = frozenset()
+ kinds: frozenset[str] = frozenset()
+ components: frozenset[str] = frozenset()
+ views: frozenset[str] = frozenset()
+ fields: frozenset[str] = frozenset()
+ #: Target types the app can store and validate: `rss_feed`, `account`.
+ target_kinds: frozenset[str] = frozenset()
+ #: The named value checks the app implements for a target somebody types
+ #: in — `bluesky_handle`. A plugin naming one the app has not written is
+ #: asking for a guarantee nothing would enforce, which is worse than
+ #: declaring none. Empty does not restrict it.
+ validators: frozenset[str] = frozenset()
+ #: The preview order the app's queue can actually produce, in order, with
+ #: the leading group dropped. A queue row is one column in a list of
+ #: hundreds, so what it can show is the app's to say — an app whose
+ #: preview is a generated column can produce exactly one order, and a
+ #: profile declaring another has to be told rather than quietly given the
+ #: one order there is. Empty does not restrict it.
+ queue_preview: tuple[str, ...] = ()
+
+
+def problems(spec: PluginSpec, vocabulary: Vocabulary) -> list[str]:
+ """Every reason this application cannot run this plugin, in one list.
+
+ All of them, not the first: a plugin naming three tools an app does not
+ have should say so once rather than over three edits. An empty list means
+ it can be loaded.
+
+ An adapted spec is always empty — see `spec_of`. Nothing in one was
+ declared, so there is nothing in it to be wrong.
+ """
+ if spec.adapted:
+ return []
+
+ found: list[str] = []
+ for name, profile in sorted(spec.profiles.items()):
+ where = f"{spec.id}.{name}"
+ found += _unknown(vocabulary.kinds, [profile.kind],
+ f"{where}: kind", "kinds this application lays out")
+ found += _unknown(vocabulary.tools, [s.tool for s in profile.acquisition],
+ f"{where}: acquisition tool", "tools it can run")
+ found += _unknown(
+ vocabulary.components,
+ [s.component for view in profile.presentation.views.values()
+ for shown in view.chain() for s in shown.sections]
+ + [s.component for s in profile.presentation.header],
+ f"{where}: component", "components it renders")
+ found += _unknown(vocabulary.views, list(profile.presentation.views),
+ f"{where}: view", "views its reader has")
+ found += _unknown(vocabulary.fields, _fields_named(profile),
+ f"{where}: field", "fields an item has")
+ declared = tuple(name.split(".")[-1]
+ for name in profile.presentation.queue.preview)
+ if vocabulary.queue_preview and declared and declared != vocabulary.queue_preview:
+ found.append(
+ f"{where}: queue preview {' then '.join(declared)} is not an "
+ f"order this application's queue can produce "
+ f"({' then '.join(vocabulary.queue_preview)})")
+ if spec.targets:
+ found += _unknown(vocabulary.target_kinds, [spec.targets.kind],
+ f"{spec.id}: target kind",
+ "target types it can store")
+ found += _unknown(vocabulary.validators,
+ [v for v in [spec.targets.validator] if v],
+ f"{spec.id}: target validator",
+ "value checks it implements")
+ return found
+
+
+def _unknown(allowed: frozenset[str], used: list[str], what: str,
+ supported: str) -> list[str]:
+ """One diagnostic per unrecognised name, naming what was expected. An
+ empty `allowed` is "this app does not restrict these" and never a
+ vocabulary in which every name is wrong."""
+ if not allowed:
+ return []
+ return [f"{what} {name!r} is not among the {supported} "
+ f"({', '.join(sorted(allowed))})"
+ for name in sorted(set(used)) if name not in allowed]
+
+
+def _fields_named(profile: Profile) -> list[str]:
+ """Every item field this profile binds, from any of the places one can be
+ named. The leading group is dropped: `inputs.transcript` is the transcript
+ wherever it is read from."""
+ named = [*profile.tasks.summary.input_fields,
+ *profile.tasks.tagging.input_fields,
+ *profile.tasks.tagging.context_fields,
+ *profile.presentation.queue.preview,
+ *profile.presentation.queue.metadata,
+ *[section.field for section in profile.presentation.header]]
+ for step in profile.acquisition:
+ named += [*step.outputs.values(), *([step.when] if step.when else [])]
+ for view in profile.presentation.views.values():
+ for shown in view.chain():
+ named += [section.field for section in shown.sections]
+ named += shown.requires
+ return [name.split(".")[-1] for name in named if name]
+
+
+# ------------------------------------------------ plugins that declare none
+
+def spec_of(source: type) -> PluginSpec:
+ """The plugin's own spec, or one adapted from the class it already is.
+
+ Every plugin in this codebase is undeclared today, and most will still be
+ after the first few are converted, so this is the ordinary path and not a
+ fallback. It reads only what the class actually states — its name, its
+ description, and whether its subscriptions are feeds or handles — and
+ states nothing else on the author's behalf. `adapted` is how everything
+ downstream tells the difference between a plugin that declared no
+ instructions and one that declared none *deliberately*.
+
+ Read off the class itself and never off a parent: a show that subclasses
+ `PodcastSource` is its own plugin with its own id and its own archive, and
+ inheriting a declaration would quietly file its items under the class it
+ borrowed from. A subclass that wants its parent's acquisition says so.
+
+ The one profile it carries is `default` with both tasks disabled. That is
+ not a claim that the plugin is not summarised or tagged — it is today,
+ by the pipeline, exactly as before. It is this contract saying it has been
+ told nothing about either, which is the truth.
+ """
+ declared = source.__dict__.get("spec")
+ if isinstance(declared, PluginSpec):
+ return declared
+ name = getattr(source, "name", "") or ""
+ return PluginSpec(
+ api_version=API_VERSION, id=name, version="0",
+ label=getattr(source, "default_outlet", "") or name,
+ profiles={"default": Profile(
+ kind="", tasks=Tasks(summary=SummaryPolicy(enabled=False),
+ tagging=TaggingPolicy(enabled=False)))},
+ adapted=True,
+ )
diff --git a/desk/sources/x.py b/desk/sources/x.py
new file mode 100644
index 0000000..9353586
--- /dev/null
+++ b/desk/sources/x.py
@@ -0,0 +1,203 @@
+"""X (Twitter) via the official API v2. Off by default, because it costs money.
+
+X moved to pay-per-usage billing. As published on the official pricing page,
+post reads are $0.005 per resource and user reads are $0.010 per resource,
+with pay-per-usage plans capped at 3 million post reads per billing cycle.
+Reads of your *own* account's data ("Owned Reads") are $0.001 per resource.
+
+So a config with 30 accounts x 20 posts x 4 polls a day is roughly
+30*20*4*30 = 72,000 reads/month = about $360/month. Check the estimate this
+plugin prints before you enable it.
+
+This plugin therefore:
+ - is disabled unless you set `enabled: true`
+ - refuses to run without `budget_usd_per_month`
+ - tracks spend in the `state` table and stops when the budget is gone
+ - caches user IDs so it never pays for a lookup twice
+
+Source for rates: https://docs.x.com/x-api/getting-started/pricing
+"""
+
+from __future__ import annotations
+
+from urllib.parse import urlsplit
+
+import logging
+import os
+from collections.abc import Iterator
+from datetime import UTC, datetime
+from typing import Any
+
+import httpx
+
+from desk.sources.base import USER_AGENT, Item, Source, Suggestion, parse_timestamp
+from desk.sources.spec import (
+ API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy,
+ TaggingPolicy, TargetSpec, Tasks,
+)
+
+log = logging.getLogger(__name__)
+API = "https://api.x.com/2"
+
+POST_READ_USD = 0.005
+USER_READ_USD = 0.010
+
+
+def _is_internal(url: str) -> bool:
+ """Match the host boundary so linux.com remains an external article."""
+ try:
+ host = (urlsplit(url).hostname or "").lower().rstrip(".")
+ except ValueError:
+ return False
+ return any(host == domain or host.endswith("." + domain)
+ for domain in ("x.com", "twitter.com"))
+
+
+class XSource(Source):
+ name = "x"
+ default_outlet = "X"
+ description = "X/Twitter timelines (paid API — off by default)."
+
+ #: Declared like the other two networks. The budget, the token and the
+ #: refusal to run without either stay in the code above: they are what
+ #: this plugin *does*, and none of them is a thing the shared jobs or the
+ #: reader need to be told about.
+ spec = PluginSpec(
+ api_version=API_VERSION, id=name, version="1", label="X",
+ targets=TargetSpec(mode="editable", kind="account", defaults=[],
+ validator="x_handle"),
+ profiles={"default": Profile(
+ kind="post",
+ acquisition=[Step(id="article", tool="extract_article",
+ inputs={"url": "inputs.original_url"},
+ outputs={"content": "inputs.article_text"})],
+ tasks=Tasks(
+ summary=SummaryPolicy(enabled=False),
+ tagging=TaggingPolicy(
+ enabled=True, input_fields=["inputs.feed_text"],
+ context_fields=["author"]),
+ ),
+ presentation=POST_READER,
+ )},
+ )
+
+ def __init__(self, options: dict[str, Any] | None = None) -> None:
+ super().__init__(options)
+ token_env = self.options.get("bearer_token_env", "X_BEARER_TOKEN")
+ self.token = os.environ.get(token_env, "")
+ self.client = httpx.Client(
+ timeout=25,
+ headers={
+ "Authorization": f"Bearer {self.token}",
+ "User-Agent": USER_AGENT,
+ },
+ )
+
+ @property
+ def enabled(self) -> bool:
+ return bool(self.options.get("enabled", False))
+
+ def estimate_monthly_usd(self, polls_per_day: int = 4) -> float:
+ n = len(self.options.get("accounts", []))
+ per_poll = n * int(self.options.get("posts_per_account", 10))
+ return per_poll * polls_per_day * 30 * POST_READ_USD
+
+ # ------------------------------------------------------------ budget
+
+ def _spent(self) -> float:
+ month = datetime.now(UTC).strftime("%Y-%m")
+ return float(self.state.get(f"x_spend_{month}", "0") or 0)
+
+ def _charge(self, usd: float) -> None:
+ month = datetime.now(UTC).strftime("%Y-%m")
+ self.state[f"x_spend_{month}"] = str(self._spent() + usd)
+
+ def _user_id(self, handle: str) -> str | None:
+ cache_key = f"x_userid_{handle.lower()}"
+ cached = self.state.get(cache_key)
+ if cached:
+ return cached
+ r = self.client.get(f"{API}/users/by/username/{handle}")
+ if r.status_code != 200:
+ log.warning("x: lookup %s returned %s", handle, r.status_code)
+ return None
+ self._charge(USER_READ_USD)
+ uid = (r.json().get("data") or {}).get("id")
+ if uid:
+ self.state[cache_key] = uid
+ return uid
+
+ # ------------------------------------------------------------- ingest
+
+ def fetch(self) -> Iterator[Item]:
+ if not self.token:
+ log.error("x: no bearer token in $%s",
+ self.options.get("bearer_token_env", "X_BEARER_TOKEN"))
+ return
+ budget = self.options.get("budget_usd_per_month")
+ if budget is None:
+ log.error("x: refusing to run without `budget_usd_per_month` in config")
+ return
+
+ per_account = int(self.options.get("posts_per_account", 10))
+ exclude = ",".join(self.options.get("exclude", ["retweets", "replies"]))
+
+ for handle, label in self.targets():
+ spent = self._spent()
+ if spent >= float(budget):
+ log.warning("x: monthly budget of $%.2f exhausted ($%.2f spent), stopping",
+ float(budget), spent)
+ return
+
+ uid = self._user_id(handle)
+ if not uid:
+ continue
+
+ r = self.client.get(
+ f"{API}/users/{uid}/tweets",
+ params={
+ "max_results": max(5, min(per_account, 100)),
+ "exclude": exclude,
+ "tweet.fields": "created_at,entities,public_metrics",
+ },
+ )
+ if r.status_code != 200:
+ log.warning("x: timeline %s returned %s: %s",
+ handle, r.status_code, r.text[:200])
+ continue
+
+ payload = r.json()
+ posts = payload.get("data", []) or []
+ self._charge(len(posts) * POST_READ_USD)
+
+ for post in posts:
+ urls = ((post.get("entities") or {}).get("urls") or [])
+ external = next(
+ (u.get("expanded_url") for u in urls
+ if u.get("expanded_url")
+ and not _is_internal(u["expanded_url"])),
+ "",
+ )
+ yield Item(
+ source_id=post["id"],
+ location=external or f"https://x.com/{handle}/status/{post['id']}",
+ title="",
+ excerpt=post.get("text", ""),
+ author=handle,
+ published_at=parse_timestamp(post.get("created_at", "")),
+ subscription=label,
+ kind="post",
+ raw={
+ "metrics": post.get("public_metrics", {}),
+ "post_url": f"https://x.com/{handle}/status/{post['id']}",
+ },
+ )
+
+ def discover(self) -> list[Suggestion]:
+ # The follow-graph endpoints are billed per user read, which makes
+ # graph-walking on X expensive enough that it is not worth doing
+ # automatically. Use Bluesky/Mastodon discovery instead.
+ return []
+
+ # `check` is left at the default too: a user lookup is a billed read, and
+ # a check job must never cost money.
diff --git a/desk/tools/__init__.py b/desk/tools/__init__.py
new file mode 100644
index 0000000..2a5daa9
--- /dev/null
+++ b/desk/tools/__init__.py
@@ -0,0 +1,20 @@
+"""Reusable machinery with no opinion about news.
+
+A tool is something a pipeline stage picks up and puts down: it takes plain
+inputs, returns plain outputs, and knows nothing about subscriptions, feeds,
+or what a job is for. That is the test for whether a module belongs here —
+if it would make sense in a different application, it is a tool.
+
+ agent.py who answers when something needs a model: the Claude API
+ with a key, the `claude` CLI on a subscription, or nobody
+ embedding.py text in, vector out
+ framing.py will this page load inside an iframe?
+ http.py one shared HTTP client per call chain
+ marks.py what kind of thing is this, if not an article?
+ video/ a recording, read as text with timings on it
+ walls.py was that the article, or the page shown instead of it?
+
+Nothing is imported here on purpose. `video` pulls in httpx and the caption
+parsers, `embedding` a sentence-transformer, `agent` an SDK; each is paid for
+by the caller that wants it and by nobody else.
+"""
diff --git a/desk/tools/agent.py b/desk/tools/agent.py
new file mode 100644
index 0000000..2c52881
--- /dev/null
+++ b/desk/tools/agent.py
@@ -0,0 +1,461 @@
+"""Who answers when a job needs a model.
+
+There is one question this module exists to answer — *ask Claude this, get JSON
+back* — and two ways to answer it. They are interchangeable, so a handler
+asks without knowing which one ran:
+
+ api One call to the Claude API. Wants ANTHROPIC_API_KEY (or an
+ `ant auth login` profile). No local install, no subprocess,
+ per-token billing.
+ claude_code The Claude Agent SDK drives the `claude` CLI already on this
+ machine, on the subscription it is already signed in with.
+ No key, nothing to bill. This is what the server runs on
+ today, authenticated by CLAUDE_CODE_OAUTH_TOKEN.
+
+`backend()` answers `""` when neither can run, and that is a failure rather
+than a third way: the caller says so with a notice naming the reason — no
+key, no CLI, `backend: off` — and the batch fails without asking anybody.
+Nothing waits for a person to answer it by hand.
+
+The seam is deliberately narrow: `ask_json` takes a system prompt, a payload
+and a JSON schema, and returns parsed JSON. Both backends take the *same*
+schema — the Agent SDK accepts the Messages API's `json_schema` output format
+and hands the parsed object back on `ResultMessage.structured_output` — so
+neither path is the degraded one, and moving between them is a config line
+rather than a second implementation.
+
+Both also *stream*, and hand the text out through `on_partial` as it arrives.
+Nothing here does anything with it — this module still knows only "JSON in,
+JSON out" — but a batch of twenty-five articles is one request that takes the
+best part of a minute, and the answers inside it are finished one at a time
+long before the request is. Passing the pieces up is what lets the layer that
+knows what an answer looks like count them, and a progress bar move per
+article instead of per batch.
+
+What the `claude_code` backend deliberately does not get: no tools
+(`tools=[]`), no filesystem settings (`setting_sources=None`, so no CLAUDE.md,
+no skills, no MCP servers leak into a tagging call), no session to resume. One
+text-in, JSON-out turn — the same shape as the API call it stands in for.
+
+`thinking` is the one knob a caller turns, and it is stated to both backends
+rather than left to either. It has to be: the CLI is built for agent work and
+thinks unprompted, the Messages API thinks only when asked, so "say nothing"
+means opposite things on the two sides. Saying `disabled` out loud is what
+makes them the interchangeable pair this module claims — and it is worth
+saying, because on a batch of twenty-five headlines the CLI's default cost
+128s against 17s for the same tags. What to think about is the task's business
+(`ModelTaskConfig.thinking`); making both backends agree on it is this one's.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import logging
+import os
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from typing import Any
+
+from desk.config import AgentConfig
+from desk.tools import credentials
+
+log = logging.getLogger(__name__)
+
+#: The backends `backend()` can name, best first. `auto` walks this list.
+BACKENDS = ("api", "claude_code")
+
+#: Where the answer goes as it arrives, a piece at a time. Called with raw
+#: text, not parsed anything — half a JSON object is not a JSON object, and
+#: what to make of a partial answer is the caller's business, not this
+#: module's.
+Partial = Callable[[str], None]
+
+
+class AgentError(Exception):
+ """The backend was there and the call still didn't produce an answer.
+
+ Raised for every failure mode of both paths — no SDK installed, no CLI on
+ PATH, an API error, a timeout, a response that isn't the shape asked for.
+ Callers treat it as "no answer this time", which for a job means either
+ moving on to the next batch or handing off to a human-driven agent.
+ """
+
+
+@dataclass(slots=True)
+class Reply:
+ """One answer, and enough about where it came from to say so in the UI."""
+
+ data: dict[str, Any]
+ backend: str = ""
+ model: str = ""
+ #: Filled by the `claude_code` path only, which is the reverse of what you
+ #: would expect: the SDK's `ResultMessage` prices the turn, so a
+ #: subscription that is billed nothing for it still reports a figure, while
+ #: the API path — the one actually billed — leaves this at zero because the
+ #: Messages response does not price itself. An estimate of what the run
+ #: would have cost on the API, then, and never a bill. `usage` is the field
+ #: to trust if you want to price it yourself.
+ cost_usd: float = 0.0
+ usage: dict[str, Any] = field(default_factory=dict)
+
+
+# ------------------------------------------------------------ which backend
+
+def has_api_credentials() -> bool:
+ """Whether the API path can run unattended.
+
+ An unset ANTHROPIC_API_KEY isn't proof there are no credentials — the SDK
+ also reads ANTHROPIC_AUTH_TOKEN and an `ant auth login` profile — but env
+ vars are the only part we can check without importing the SDK.
+ """
+ return any(os.environ.get(name) for name in (
+ "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_PROFILE",
+ ))
+
+
+def has_claude_code() -> bool:
+ """Whether the Agent SDK is here, and so whether there is a CLI to drive.
+
+ The SDK bundles its own `claude`, so having the package is enough — a
+ binary on PATH is a bonus, not a requirement, and checking for one would
+ wrongly rule out a machine where the SDK is the only install.
+
+ Not proof the call will work — the CLI may be signed out — but it is the
+ check that can be made in microseconds, which is what a job asking "can I
+ do this myself?" needs. A signed-out CLI surfaces as an `AgentError` on
+ the call, and the caller falls back exactly as if this had returned False.
+ """
+ return importlib.util.find_spec("claude_agent_sdk") is not None
+
+
+def backend(settings: AgentConfig) -> str:
+ """Which backend will answer, or `""` for nobody.
+
+ `agent.backend: auto` prefers the API when there are credentials, because
+ that is the unattended path and the one production will use; it falls back
+ to the local Claude Code subscription, which is the one that costs nothing
+ while you are building. Naming a backend skips the detection — including
+ `off`, which is how you make everything that wants a model fail.
+ """
+ want = settings.backend
+ if want == "off":
+ return ""
+ if want in BACKENDS:
+ return want
+ if has_api_credentials():
+ return "api"
+ if has_claude_code():
+ return "claude_code"
+ return ""
+
+
+# ------------------------------------------------------------------ the ask
+
+def _thinking(setting: str) -> Any:
+ """One `thinking` setting in the form both SDKs take.
+
+ Returns `Any` on purpose. Each SDK declares its own TypedDict for the same
+ two-word object — `ThinkingConfigDisabledParam` on one side,
+ `ThinkingConfigDisabled` on the other — and neither can be named here
+ without importing an SDK this module is careful to keep optional. One
+ honest `Any` at the seam beats a hard dependency at the top of the file.
+
+ Anything that is not a mode we offer is off, rather than an error. This is
+ reached from config pydantic has already constrained to the same two words,
+ so a third one arriving here would be a bug in our own plumbing — and the
+ honest answer to that is the same one the default gives, not a failed run
+ the model would happily have answered.
+ """
+ return {"type": "adaptive"} if setting == "adaptive" else {"type": "disabled"}
+
+
+def ask_json(
+ settings: AgentConfig,
+ *,
+ system: str,
+ question: dict[str, Any],
+ schema: dict[str, Any],
+ model: str = "",
+ using: str = "",
+ thinking: str = "off",
+ on_partial: Partial | None = None,
+) -> Reply:
+ """Ask Claude `question`, constrained to `schema`, and parse the answer.
+
+ `system` is expected to be byte-identical across the calls in a run — it is
+ what the API path caches. `question` is the part that changes. `using`
+ forces a backend for one call; otherwise `backend()` decides. `thinking` is
+ `off` or `adaptive`, and defaults to off because the caller who wants it is
+ rarer than the caller who would be surprised to be waiting for it.
+
+ `on_partial` is called with each piece of the answer as it arrives, in
+ order, on the thread this was called on. It is a courtesy, not a contract:
+ a backend that cannot stream simply never calls it, and the return value
+ is the same either way.
+ """
+ name = using or backend(settings)
+ model = model or settings.model
+ if thinking == "adaptive" and model.startswith(
+ ("claude-haiku-4-5", "claude-sonnet-4-5", "claude-opus-4-5")):
+ # These models support manual extended thinking only; fail before spend.
+ # https://platform.claude.com/docs/en/build-with-claude/extended-thinking
+ raise AgentError(f"{model} does not support adaptive thinking; set thinking: off "
+ "or choose a model supporting adaptive thinking")
+ if not name:
+ raise AgentError("no model backend is available")
+ if name == "api":
+ return _ask_api(settings, system, question, schema, model, thinking, on_partial)
+ if name == "claude_code":
+ return _ask_claude_code(settings, system, question, schema, model, thinking,
+ on_partial)
+ raise AgentError(f"unknown backend {name!r}")
+
+
+def _ask_api(settings: AgentConfig, system: str, question: dict[str, Any],
+ schema: dict[str, Any], model: str, thinking: str = "off",
+ on_partial: Partial | None = None) -> Reply:
+ try:
+ import anthropic
+ except ImportError as exc:
+ raise AgentError(
+ "the anthropic SDK isn't installed — `uv pip install anthropic`"
+ ) from exc
+
+ try:
+ client = anthropic.Anthropic(timeout=settings.timeout)
+ # Streamed rather than awaited whole, always — not only when someone
+ # passed `on_partial`. One code path is worth more than the few lines a
+ # non-streaming branch would save, and a minute-long request held open
+ # by a single response is the shape that trips idle timeouts in
+ # whatever sits between here and the API.
+ with client.messages.stream(
+ model=model,
+ max_tokens=settings.max_tokens,
+ system=[{
+ "type": "text",
+ "text": system,
+ # The instructions are byte-identical across every batch and
+ # every run; only the payload below changes.
+ "cache_control": {"type": "ephemeral"},
+ }],
+ messages=[{"role": "user", "content": _dumps(question)}],
+ thinking=_thinking(thinking),
+ output_config={"format": {"type": "json_schema", "schema": schema}},
+ ) as stream:
+ for event in stream:
+ if on_partial is not None and (piece := _delta(event)):
+ on_partial(piece)
+ response = stream.get_final_message()
+ except (anthropic.APIError, TypeError) as exc:
+ # The status code carried through in the text on purpose: it is what
+ # `notices.classify` reads to tell "your key is wrong" (401) apart from
+ # "you are being rate limited" (429), which want different reactions
+ # from you and which the message body alone does not always separate.
+ status = getattr(exc, "status_code", "")
+ raise AgentError(
+ f"the API call failed: {f'{status} ' if status else ''}{exc}") from exc
+
+ text = next((b.text for b in response.content if b.type == "text"), "")
+ return Reply(
+ data=_parse(text),
+ backend="api",
+ model=response.model or model,
+ usage=response.usage.model_dump() if response.usage else {},
+ )
+
+
+def _ask_claude_code(settings: AgentConfig, system: str, question: dict[str, Any],
+ schema: dict[str, Any], model: str, thinking: str = "off",
+ on_partial: Partial | None = None) -> Reply:
+ try:
+ from claude_agent_sdk import (
+ ClaudeAgentOptions, ClaudeSDKError, ResultMessage, StreamEvent, query,
+ )
+ except ImportError as exc:
+ raise AgentError(
+ "the Claude Agent SDK isn't installed — "
+ "`uv pip install claude-agent-sdk`"
+ ) from exc
+
+ options = ClaudeAgentOptions(
+ system_prompt=system,
+ model=model or None,
+ # A classification call, not an agent: nothing to run, nothing to
+ # read, and no project settings — so no CLAUDE.md, no skills and no
+ # MCP servers get in front of the model on the way to a tag list.
+ tools=[],
+ allowed_tools=[],
+ permission_mode="dontAsk",
+ setting_sources=None,
+ # Always stated, never left to the CLI's default, which is on. With
+ # it on the progress bar also stops meaning anything: thinking emits
+ # nothing `_Answered` can count, so a chunk streams no `"uid"` until
+ # the thinking is done, and the bar sits on the previous chunk's
+ # number until the answer arrives in a rush.
+ thinking=_thinking(thinking),
+ output_format={"type": "json_schema", "schema": schema},
+ # Only when someone is listening. The partials are the same bytes as
+ # the final answer sent again in pieces, and asking the CLI for them
+ # when nothing reads them is noise on the pipe for no one's benefit.
+ include_partial_messages=on_partial is not None,
+ cli_path=settings.cli_path or None,
+ # `max_tokens` has to mean the same thing on both backends or it means
+ # nothing. The Messages API takes it as a parameter and the CLI takes
+ # it only as an environment variable, so without this the setting was
+ # quietly API-only — which is how a call configured for 8,192 tokens
+ # answered with 11,706 and nothing objected. The SDK merges this into
+ # the inherited environment rather than replacing it, so naming one
+ # variable does not cost the CLI its PATH.
+ env={"CLAUDE_CODE_MAX_OUTPUT_TOKENS": str(settings.max_tokens),
+ **({"CLAUDE_CODE_OAUTH_TOKEN": oauth} if (oauth := credentials.token(settings)) else {})},
+ stderr=lambda line: log.debug("claude: %s", line.rstrip()),
+ )
+
+ async def ask() -> Reply:
+ answer: Reply | None = None
+ async for message in query(prompt=_dumps(question), options=options):
+ if isinstance(message, StreamEvent) and on_partial is not None:
+ piece = _delta(message.event)
+ if piece:
+ on_partial(piece)
+ continue
+ if not isinstance(message, ResultMessage):
+ continue
+ if message.is_error:
+ raise AgentError(_explain(
+ "; ".join(message.errors or []) or message.result
+ or f"the agent ended as {message.subtype}"))
+ data = message.structured_output
+ if not isinstance(data, dict):
+ # A run that ended without structured output — a limit, an
+ # interruption. `result` is the raw text if there is any, and
+ # that is worth one attempt at parsing before giving up.
+ data = _parse(message.result or "")
+ answer = Reply(
+ data=data,
+ backend="claude_code",
+ model=model,
+ cost_usd=message.total_cost_usd or 0.0,
+ usage=message.usage or {},
+ )
+ if answer is None:
+ raise AgentError("the agent returned no result")
+ return answer
+
+ try:
+ return _sync(ask(), settings.timeout)
+ except ClaudeSDKError as exc:
+ # The SDK has its own exception hierarchy — a CLI that exited, a line
+ # it could not parse, an answer that overran `max_tokens` — and none of
+ # it is an `AgentError`. Left alone those escape the per-chunk `except
+ # AgentError` in `with_model`, so one bad chunk takes down a run that
+ # is contractually supposed to leave its rows for next time. Every
+ # failure of this backend is an `AgentError` or none of them are.
+ #
+ # The text is passed through intact rather than summarised: `notices`
+ # matches on prose, by its own deliberate design, so the wording *is*
+ # the thing that decides which sentence you get told.
+ raise AgentError(_explain(str(exc))) from exc
+
+
+# --------------------------------------------------------------- plumbing
+
+def _field(event: Any, name: str) -> Any:
+ """One field of a stream event, whichever shape it arrived in.
+
+ The two backends carry the same events in different clothes: the API SDK
+ parses them into objects, and the CLI passes the wire format straight
+ through as dicts. Reading both through here is what lets `_delta` be
+ written once.
+ """
+ return event.get(name) if isinstance(event, dict) else getattr(event, name, None)
+
+
+def _delta(event: Any) -> str:
+ """The answer text one stream event carries, or "" for the many with none.
+
+ The events worth anything here are `content_block_delta`, whose delta is
+ `text` for a plain answer and `partial_json` when the schema is enforced
+ through a tool. Everything else — message start and stop, block
+ boundaries, usage — is dropped.
+
+ Deliberately reads defensively rather than by type. An event shape nobody
+ here recognises must be nothing, not an exception: this drives a progress
+ bar, and it does not get to fail a run the model is answering correctly.
+ """
+ if _field(event, "type") != "content_block_delta":
+ return ""
+ delta = _field(event, "delta")
+ if delta is None:
+ return ""
+ piece = _field(delta, "text") or _field(delta, "partial_json") or ""
+ return piece if isinstance(piece, str) else ""
+
+
+def _sync(coro: Any, timeout: float) -> Reply:
+ """Run one coroutine to completion from synchronous code.
+
+ Jobs are synchronous by design and the worker runs them on a thread, so
+ there is normally no loop here and `asyncio.run` is exactly right. The
+ thread pool is for the other caller — anything asking from inside a
+ running loop, where `asyncio.run` would raise.
+ """
+ import asyncio
+
+ async def bounded() -> Reply:
+ try:
+ return await asyncio.wait_for(coro, timeout)
+ except TimeoutError as exc:
+ raise AgentError(f"no answer within {timeout:g}s") from exc
+
+ try:
+ asyncio.get_running_loop()
+ except RuntimeError:
+ return asyncio.run(bounded())
+ with ThreadPoolExecutor(max_workers=1) as pool:
+ return pool.submit(asyncio.run, bounded()).result()
+
+
+def _explain(error: str) -> str:
+ """Keep the raw failure text intact for the layer that classifies it.
+
+ This used to append advice ("the `claude` login looks expired, run…"),
+ which was the right instinct in the wrong place: the message was built here
+ and then swallowed three frames up by a `except AgentError: continue`, so
+ the advice never reached anyone. Recognising the failure and deciding what
+ to say about it now lives in `desk/notices.py`, which has somewhere to
+ put the answer.
+
+ What stays here is the part only this module knows: the CLI can end a run
+ with an empty error and a subtype, and "" is not something to classify.
+ """
+ return error.strip() or "the agent ended without saying why"
+
+
+def _dumps(payload: dict[str, Any]) -> str:
+ return json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
+
+
+def _parse(text: str) -> dict[str, Any]:
+ """The answer as a dict. Tolerant of a fenced or chatty response.
+
+ Both backends are asked for JSON through a schema, so this should be a
+ plain `json.loads`. It isn't, because "should be" is not a guarantee worth
+ losing a batch over — a model that wrapped its answer in a code fence has
+ still done the work.
+ """
+ body = text.strip()
+ if body.startswith("```"):
+ body = body.split("\n", 1)[-1].rsplit("```", 1)[0]
+ start, end = body.find("{"), body.rfind("}")
+ if start == -1 or end <= start:
+ raise AgentError(f"no JSON object in the response: {text[:200]!r}")
+ try:
+ data = json.loads(body[start:end + 1])
+ except ValueError as exc:
+ raise AgentError(f"the response wasn't valid JSON: {exc}") from exc
+ if not isinstance(data, dict):
+ raise AgentError("the response wasn't a JSON object")
+ return data
diff --git a/desk/tools/article.py b/desk/tools/article.py
new file mode 100644
index 0000000..6773ff9
--- /dev/null
+++ b/desk/tools/article.py
@@ -0,0 +1,212 @@
+"""Structured article extraction, independent of fetching and persistence.
+
+The tree is data, never HTML. Code is kept verbatim; plain text is a derived
+view of the same tree. Custom extractors can return ArticleContent; callers
+own persistence and renderers consume the closed vocabulary below.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from datetime import UTC, datetime
+from html.parser import HTMLParser
+from typing import Literal, Self
+from urllib.parse import urljoin, urlsplit
+from xml.etree import ElementTree as ET
+
+from pydantic import Field, model_validator
+
+from desk.models import Model
+
+
+class ContentNode(Model):
+ kind: Literal[
+ "document", "text", "paragraph", "heading", "link", "code", "list",
+ "item", "table", "row", "cell", "strong", "emphasis", "quote", "break",
+ "span", "inline_code",
+ ]
+ text: str = ""
+ children: list[ContentNode] = Field(default_factory=list)
+ level: int = Field(default=1, ge=1, le=6)
+ ordered: bool = False
+ header: bool = False
+ url: str = ""
+
+ @model_validator(mode="after")
+ def consistent_node(self) -> Self:
+ if self.kind in {"text", "code", "inline_code", "break"}:
+ if self.children:
+ raise ValueError("leaf nodes cannot have children")
+ elif self.text:
+ raise ValueError("container text belongs in text children")
+ if self.url and (self.kind != "link" or not _link(self.url, "")):
+ raise ValueError("only links can carry absolute HTTP(S) URLs")
+ return self
+
+ def plain_text(self) -> str:
+ if self.kind == "break":
+ return "\n"
+ if self.kind in {"code", "inline_code", "text"}:
+ return self.text
+ separator = {"document": "\n\n", "list": "\n", "table": "\n", "row": "\t"}.get(self.kind, "")
+ return separator.join(child.plain_text() for child in self.children)
+
+
+class ArticleContent(Model):
+ format: Literal["plain", "structured-v1"]
+ content: str | ContentNode
+ source_url: str
+ method: str
+ acquired_at: str = Field(default_factory=lambda: datetime.now(UTC).isoformat())
+ # Finding a readable article does not prove that it was not truncated by
+ # the publisher. The caller must supply evidence before declaring it full.
+ completeness: Literal["unknown", "complete", "partial"] = "unknown"
+ removed_gutters: int = Field(default=0, ge=0)
+
+ @model_validator(mode="after")
+ def consistent_format(self) -> Self:
+ if self.format == "plain":
+ if not isinstance(self.content, str):
+ raise ValueError("plain content must be a string")
+ elif not isinstance(self.content, ContentNode) or self.content.kind != "document":
+ raise ValueError("structured-v1 content must be a document node")
+ return self
+
+ @property
+ def plain_text(self) -> str:
+ return self.content if isinstance(self.content, str) else self.content.plain_text()
+
+ @property
+ def revision(self) -> str:
+ value = self.content if isinstance(self.content, str) else self.content.model_dump_json()
+ return hashlib.sha256(f"{self.format}\n{value}".encode()).hexdigest()
+
+
+class _Gutters(HTMLParser):
+ """Remove only explicit code-gutter nodes, retaining their following text.
+
+ No numeric-line heuristics. Unknown publisher markup is preserved. The
+ void-element handling matters: a br inside a gutter must not eat the rest
+ of the article by leaving the suppression depth unbalanced.
+ """
+
+ VOID = frozenset({"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"})
+ MARKERS = frozenset({"line-number", "line-numbers-rows", "lineno", "linenos"})
+
+ def __init__(self) -> None:
+ super().__init__(convert_charrefs=False)
+ self.parts: list[str] = []
+ self.stack: list[tuple[str, bool, bool]] = []
+ self.removed = 0
+
+ def handle_starttag(self, tag, attrs):
+ classes = set((dict(attrs).get("class") or "").split())
+ context = tag in {"pre", "code"} or bool(classes & {"highlight", "highlighttable"}) or any(c for _, c, _ in self.stack)
+ suppressed = any(s for _, _, s in self.stack)
+ gutter = context and bool(classes & self.MARKERS)
+ if gutter and not suppressed:
+ self.removed += 1
+ if not (gutter or suppressed):
+ self.parts.append(self.get_starttag_text() or "")
+ if tag not in self.VOID:
+ self.stack.append((tag, context, gutter or suppressed))
+
+ def handle_startendtag(self, tag, attrs):
+ self.handle_starttag(tag, attrs)
+ if tag not in self.VOID:
+ self.handle_endtag(tag)
+
+ def handle_endtag(self, tag):
+ suppressed = any(s for _, _, s in self.stack)
+ for index in range(len(self.stack) - 1, -1, -1):
+ if self.stack[index][0] == tag:
+ del self.stack[index:]
+ break
+ if not suppressed:
+ self.parts.append(f"{tag}>")
+
+ def handle_data(self, data):
+ if not any(s for _, _, s in self.stack):
+ self.parts.append(data)
+
+ def handle_entityref(self, name):
+ self.handle_data(f"&{name};")
+
+ def handle_charref(self, name):
+ self.handle_data(f"{name};")
+
+
+def _link(value: str, source_url: str) -> str:
+ try:
+ resolved = urljoin(source_url, value)
+ return resolved if urlsplit(resolved).scheme in {"http", "https"} else ""
+ except ValueError:
+ return ""
+
+
+def _node(element: ET.Element, source_url: str) -> ContentNode:
+ children: list[ContentNode] = []
+ if element.text:
+ children.append(ContentNode(kind="text", text=element.text))
+ for child in element:
+ children.append(_node(child, source_url))
+ if child.tail:
+ children.append(ContentNode(kind="text", text=child.tail))
+ match element.tag:
+ case "code":
+ return ContentNode(kind="code", text="".join(element.itertext()))
+ case "head":
+ level = element.get("rend", "h1").removeprefix("h")
+ return ContentNode(kind="heading", level=int(level) if level in "123456" and len(level) == 1 else 1, children=children)
+ case "ref":
+ return ContentNode(kind="link", url=_link(element.get("target", ""), source_url), children=children)
+ case "list":
+ return ContentNode(kind="list", ordered=element.get("rend") == "ol", children=children)
+ case "cell":
+ return ContentNode(kind="cell", header=element.get("role") == "head", children=children)
+ case "hi":
+ if element.get("rend") == "#t":
+ return ContentNode(kind="inline_code", text="".join(element.itertext()))
+ kind = {"#b": "strong", "#i": "emphasis"}.get(element.get("rend", ""), "span")
+ return ContentNode.model_validate({"kind": kind, "children": children})
+ case "lb":
+ return ContentNode(kind="break")
+ case _:
+ kinds = {"main": "document", "p": "paragraph", "item": "item", "table": "table", "row": "row", "quote": "quote"}
+ return ContentNode.model_validate({"kind": kinds.get(element.tag, "paragraph"), "children": children})
+
+
+def extract_article(html: str, source_url: str) -> ArticleContent | None:
+ """Extract supplied HTML without I/O; return None when no article is found.
+
+ XML is an intermediate representation only. Its tags and attributes never
+ escape into the content contract, and no raw HTML is passed to a renderer.
+ """
+ import trafilatura
+
+ cleaned = _Gutters()
+ cleaned.feed(html)
+ cleaned.close()
+ extracted = trafilatura.extract(
+ "".join(cleaned.parts), url=source_url, output_format="xml",
+ include_comments=False, include_tables=True, include_links=True,
+ include_formatting=True,
+ )
+ if not extracted:
+ return None
+ main = ET.fromstring(extracted).find("main")
+ if main is None:
+ return None
+ # XML pretty-printing inserts whitespace between blocks. It is not source
+ # content; inside code, however, every byte of whitespace belongs to it.
+ for element in main.iter():
+ if element.tag in {"main", "table", "row", "list"}:
+ if not (element.text or "").strip():
+ element.text = None
+ for child in element:
+ if not (child.tail or "").strip():
+ child.tail = None
+ return ArticleContent(
+ format="structured-v1", content=_node(main, source_url), source_url=source_url,
+ method="trafilatura-xml", removed_gutters=cleaned.removed,
+ )
diff --git a/desk/tools/browser.py b/desk/tools/browser.py
new file mode 100644
index 0000000..1e04a64
--- /dev/null
+++ b/desk/tools/browser.py
@@ -0,0 +1,254 @@
+"""Render a page in headless Chromium, accepting a tracking-consent gate on the way.
+
+The getter a check sends a page to when plain HTTP brought back a placeholder:
+an app shell whose article is drawn by a script, or a consent gate standing in
+front of every article on a host. Golem is the second kind. Every article URL
+redirects to `/sonstiges/zustimmung/auswahl.html`, where Sourcepoint's CMP
+draws its dialog in an iframe; accepting it sets the cookies and redirects
+back to the article.
+
+ from desk.tools import browser
+
+ got = browser.render("https://www.golem.de/news/...")
+ got.status, got.html, got.url, got.consent, got.cookies
+
+ browser.remember(cache, got.url, got.cookies)
+ browser.cookie_header(cache, url) # "golem_consent20=cmp|250101; ..."
+
+**Consent is accepted once and then reused.** Chromium costs a few seconds
+and a few hundred megabytes per page. The cookies it earns open the gate for
+a plain request too, so the caller stores them per host with `remember` and
+sends them with `cookie_header` on the next HTTP fetch. The browser runs again
+only when HTTP meets the gate again, which is when the cookies expired or the
+outlet changed what it accepts. Golem's last three months.
+
+**A CMP is one entry in `CONSENTS`**: the selector of the iframe it draws in
+and a pattern for the accept button's name. A page with no such frame is
+rendered without clicking anything, after waiting `gate_wait` seconds for one
+to appear. Only accept buttons are listed. A bot check that wants a human is
+not something this clicks; it stays `js_required` or `denied`.
+
+Needs the `browser` extra and a Chromium build: `playwright install chromium`.
+Imported only inside `render`, so the rest of `desk.tools` does not pay for it.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import importlib.util
+import json
+import re
+import threading
+import time
+from collections.abc import Mapping, MutableMapping, Sequence
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import urlsplit
+
+from desk.tools.framing import host_of
+
+#: Sits alongside `embed:` and `body:` in the same table.
+CACHE_PREFIX = "cookies:"
+
+#: Seconds for navigation and for the click. A gate that has not redirected
+#: back by then is not going to.
+TIMEOUT = 30.0
+
+#: Seconds to wait for a consent frame on a page that may not have one. The
+#: CMP injects its iframe from a script; on golem.de it is there within a
+#: second.
+GATE_WAIT = 5.0
+
+#: Seconds to let a script-drawn page settle after `domcontentloaded`. Not
+#: `networkidle` as a condition: an ad-carrying page never goes idle, so this is
+#: a ceiling.
+SETTLE = 3.0
+
+#: One Chromium at a time in this process. Body batches run side by side, and
+#: the first articles of a gated host would otherwise each start a browser
+#: before the first one has earned the cookies.
+_ONE_AT_A_TIME = threading.Lock()
+
+
+def installed() -> bool:
+ """Whether the `browser` extra is. Chromium itself is checked by launching."""
+ return importlib.util.find_spec("playwright") is not None
+
+
+@dataclass(frozen=True)
+class Consent:
+ """How to accept one CMP's dialog."""
+
+ name: str
+ #: CSS selector of the iframe the dialog is drawn in.
+ frame: str
+ #: Case-insensitive regex, matched against the accept button's
+ #: accessible name. Anchored, so "Accept" does not match "Accept only
+ #: necessary" and a reject button is never the one clicked.
+ accept: str
+ #: A page this was found on.
+ seen: str = ""
+
+ def __post_init__(self) -> None:
+ re.compile(self.accept)
+
+
+CONSENTS: tuple[Consent, ...] = (
+ Consent("sourcepoint", "iframe[id^='sp_message_iframe']",
+ r"^(Zustimmen und weiter|Alle akzeptieren|Akzeptieren|Zustimmen"
+ r"|Accept all|Accept|I agree|Agree)$",
+ seen="https://www.golem.de/sonstiges/zustimmung/auswahl.html"),
+)
+
+
+@dataclass(frozen=True)
+class Cookie:
+ name: str
+ value: str
+ domain: str
+ path: str = "/"
+ #: Unix seconds; -1 for a session cookie, which is kept until the gate
+ #: comes back rather than until a browser that no longer exists closes.
+ expires: float = -1
+
+
+@dataclass(frozen=True)
+class Rendered:
+ """What Chromium ended up showing. `url` is where it ended up."""
+
+ status: int
+ html: str
+ url: str
+ headers: Mapping[str, str] = field(default_factory=dict)
+ #: Every cookie the host of `url` would be sent.
+ cookies: tuple[Cookie, ...] = ()
+ #: The `Consent.name` that was accepted, or "" when no gate was met.
+ consent: str = ""
+
+
+def render(url: str, *, consents: Sequence[Consent] = CONSENTS,
+ timeout: float = TIMEOUT, gate_wait: float = GATE_WAIT,
+ settle: float = SETTLE) -> Rendered:
+ """Load `url`, accept a consent dialog if one appears, return the page.
+
+ Raises what Playwright raises: its `TimeoutError` names itself a timeout
+ to `failures.of_exception`, and a missing Chromium build is `unknown`.
+ One browser per call. A pull renders a page only where HTTP failed, so a
+ browser kept alive between calls would sit idle in a worker.
+ """
+ from playwright.sync_api import TimeoutError as PlaywrightTimeout
+ from playwright.sync_api import sync_playwright
+
+ ms = int(timeout * 1000)
+ with _ONE_AT_A_TIME, sync_playwright() as playwright:
+ # /dev/shm is 64 MB in a container, which Chromium outgrows and
+ # crashes on; this has it use /tmp instead.
+ chromium = playwright.chromium.launch(
+ headless=True, args=["--disable-dev-shm-usage"])
+ try:
+ context = chromium.new_context()
+ page = context.new_page()
+ # The last document response of the main frame, so the status is
+ # the article's after a gate's redirect rather than the gate's.
+ documents: list[Any] = []
+ page.on("response", lambda response: documents.append(response)
+ if response.request.is_navigation_request()
+ and response.frame == page.main_frame else None)
+ # `domcontentloaded`, not `load`: an ad-carrying page such as a
+ # golem.de article may not fire `load` within any useful timeout.
+ page.goto(url, wait_until="domcontentloaded", timeout=ms)
+ accepted = _accept(page, consents, ms, gate_wait, PlaywrightTimeout)
+ with contextlib.suppress(PlaywrightTimeout):
+ page.wait_for_load_state("networkidle", timeout=int(settle * 1000))
+ last = documents[-1] if documents else None
+ final = page.url
+ return Rendered(
+ status=last.status if last else 0,
+ html=page.content(),
+ url=final,
+ headers=dict(last.headers) if last else {},
+ cookies=_for_host(_hostname(final), context.cookies()),
+ consent=accepted)
+ finally:
+ chromium.close()
+
+
+def _accept(page: Any, consents: Sequence[Consent], ms: int, gate_wait: float,
+ timeout_error: type[Exception]) -> str:
+ """Click the first consent dialog's accept button. The CMP's name, or ""."""
+ deadline = time.monotonic() + gate_wait
+ for consent in consents:
+ left = max(0, int((deadline - time.monotonic()) * 1000))
+ frame = page.locator(consent.frame).first
+ try:
+ frame.wait_for(state="attached", timeout=max(left, 1))
+ except timeout_error:
+ continue
+ button = page.frame_locator(consent.frame).first.get_by_role(
+ "button", name=re.compile(consent.accept, re.IGNORECASE))
+ button.first.click(timeout=ms)
+ # Gone either way it is dismissed: an overlay removes its frame, a
+ # gate page is navigated away from.
+ frame.wait_for(state="detached", timeout=ms)
+ page.wait_for_load_state("domcontentloaded", timeout=ms)
+ return consent.name
+ return ""
+
+
+def _for_host(host: str, cookies: Sequence[Mapping[str, Any]]) -> tuple[Cookie, ...]:
+ return tuple(
+ Cookie(str(c["name"]), str(c["value"]), str(c["domain"]),
+ str(c.get("path", "/")), float(c.get("expires", -1)))
+ for c in cookies if _matches(host, str(c.get("domain", ""))))
+
+
+def _hostname(url: str) -> str:
+ """The host without a port, which is what a cookie's domain is matched to."""
+ try:
+ return (urlsplit(url).hostname or "").lower()
+ except ValueError:
+ return ""
+
+
+def _matches(host: str, domain: str) -> bool:
+ domain = domain.lstrip(".").lower()
+ return bool(host) and (host == domain or host.endswith("." + domain))
+
+
+# ------------------------------------------------------------- the cookie jar
+
+def remember(cache: MutableMapping[str, str], url: str,
+ cookies: Sequence[Cookie]) -> None:
+ """Keep the cookies a render earned, for the next plain fetch from this host.
+
+ Replaces whatever the host had: a render that met the gate again earned
+ the current set, and the old one is what stopped working.
+ """
+ kept = [c for c in cookies if _matches(_hostname(url), c.domain)]
+ if kept:
+ cache[CACHE_PREFIX + host_of(url)] = json.dumps(
+ [[c.name, c.value, c.domain, c.path, c.expires] for c in kept])
+
+
+def cookie_header(cache: Mapping[str, str], url: str, *,
+ now: float | None = None) -> str:
+ """The `Cookie` header for a plain request to `url`, or "" for none.
+
+ Expired cookies are left out rather than deleted. A host whose cookies
+ have all expired sends none, meets the gate, and is rendered again.
+ """
+ stored = cache.get(CACHE_PREFIX + host_of(url), "")
+ if not stored:
+ return ""
+ now = time.time() if now is None else now
+ try:
+ rows = json.loads(stored)
+ except ValueError:
+ return ""
+ return "; ".join(f"{name}={value}" for name, value, _domain, _path, expires in rows
+ if expires < 0 or expires > now)
+
+
+def forget(cache: MutableMapping[str, str], url: str) -> None:
+ """Drop a host's cookies, so its next fetch goes without them."""
+ cache.pop(CACHE_PREFIX + host_of(url), None)
diff --git a/desk/tools/checks.py b/desk/tools/checks.py
new file mode 100644
index 0000000..9442b73
--- /dev/null
+++ b/desk/tools/checks.py
@@ -0,0 +1,227 @@
+"""Did the getter bring back a page worth converting, and if not, what next?
+
+`scrape` fetches, this judges, `article` converts. A 200 is not an article
+because it answered 200: an app shell serves a `