Bring in desk from news

The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-23 12:29:17 +02:00
1 parent 9d666d0937
commit 248ea1636d
36 files changed
+8393

No files matched your search

+203
View File
@@ -0,0 +1,203 @@
"""X (Twitter) via the official API v2. Off by default, because it costs money.
X moved to pay-per-usage billing. As published on the official pricing page,
post reads are $0.005 per resource and user reads are $0.010 per resource,
with pay-per-usage plans capped at 3 million post reads per billing cycle.
Reads of your *own* account's data ("Owned Reads") are $0.001 per resource.
So a config with 30 accounts x 20 posts x 4 polls a day is roughly
30*20*4*30 = 72,000 reads/month = about $360/month. Check the estimate this
plugin prints before you enable it.
This plugin therefore:
- is disabled unless you set `enabled: true`
- refuses to run without `budget_usd_per_month`
- tracks spend in the `state` table and stops when the budget is gone
- caches user IDs so it never pays for a lookup twice
Source for rates: https://docs.x.com/x-api/getting-started/pricing
"""
from __future__ import annotations
from urllib.parse import urlsplit
import logging
import os
from collections.abc import Iterator
from datetime import UTC, datetime
from typing import Any
import httpx
from desk.sources.base import USER_AGENT, Item, Source, Suggestion, parse_timestamp
from desk.sources.spec import (
API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy,
TaggingPolicy, TargetSpec, Tasks,
)
log = logging.getLogger(__name__)
API = "https://api.x.com/2"
POST_READ_USD = 0.005
USER_READ_USD = 0.010
def _is_internal(url: str) -> bool:
"""Match the host boundary so linux.com remains an external article."""
try:
host = (urlsplit(url).hostname or "").lower().rstrip(".")
except ValueError:
return False
return any(host == domain or host.endswith("." + domain)
for domain in ("x.com", "twitter.com"))
class XSource(Source):
name = "x"
default_outlet = "X"
description = "X/Twitter timelines (paid API — off by default)."
#: Declared like the other two networks. The budget, the token and the
#: refusal to run without either stay in the code above: they are what
#: this plugin *does*, and none of them is a thing the shared jobs or the
#: reader need to be told about.
spec = PluginSpec(
api_version=API_VERSION, id=name, version="1", label="X",
targets=TargetSpec(mode="editable", kind="account", defaults=[],
validator="x_handle"),
profiles={"default": Profile(
kind="post",
acquisition=[Step(id="article", tool="extract_article",
inputs={"url": "inputs.original_url"},
outputs={"content": "inputs.article_text"})],
tasks=Tasks(
summary=SummaryPolicy(enabled=False),
tagging=TaggingPolicy(
enabled=True, input_fields=["inputs.feed_text"],
context_fields=["author"]),
),
presentation=POST_READER,
)},
)
def __init__(self, options: dict[str, Any] | None = None) -> None:
super().__init__(options)
token_env = self.options.get("bearer_token_env", "X_BEARER_TOKEN")
self.token = os.environ.get(token_env, "")
self.client = httpx.Client(
timeout=25,
headers={
"Authorization": f"Bearer {self.token}",
"User-Agent": USER_AGENT,
},
)
@property
def enabled(self) -> bool:
return bool(self.options.get("enabled", False))
def estimate_monthly_usd(self, polls_per_day: int = 4) -> float:
n = len(self.options.get("accounts", []))
per_poll = n * int(self.options.get("posts_per_account", 10))
return per_poll * polls_per_day * 30 * POST_READ_USD
# ------------------------------------------------------------ budget
def _spent(self) -> float:
month = datetime.now(UTC).strftime("%Y-%m")
return float(self.state.get(f"x_spend_{month}", "0") or 0)
def _charge(self, usd: float) -> None:
month = datetime.now(UTC).strftime("%Y-%m")
self.state[f"x_spend_{month}"] = str(self._spent() + usd)
def _user_id(self, handle: str) -> str | None:
cache_key = f"x_userid_{handle.lower()}"
cached = self.state.get(cache_key)
if cached:
return cached
r = self.client.get(f"{API}/users/by/username/{handle}")
if r.status_code != 200:
log.warning("x: lookup %s returned %s", handle, r.status_code)
return None
self._charge(USER_READ_USD)
uid = (r.json().get("data") or {}).get("id")
if uid:
self.state[cache_key] = uid
return uid
# ------------------------------------------------------------- ingest
def fetch(self) -> Iterator[Item]:
if not self.token:
log.error("x: no bearer token in $%s",
self.options.get("bearer_token_env", "X_BEARER_TOKEN"))
return
budget = self.options.get("budget_usd_per_month")
if budget is None:
log.error("x: refusing to run without `budget_usd_per_month` in config")
return
per_account = int(self.options.get("posts_per_account", 10))
exclude = ",".join(self.options.get("exclude", ["retweets", "replies"]))
for handle, label in self.targets():
spent = self._spent()
if spent >= float(budget):
log.warning("x: monthly budget of $%.2f exhausted ($%.2f spent), stopping",
float(budget), spent)
return
uid = self._user_id(handle)
if not uid:
continue
r = self.client.get(
f"{API}/users/{uid}/tweets",
params={
"max_results": max(5, min(per_account, 100)),
"exclude": exclude,
"tweet.fields": "created_at,entities,public_metrics",
},
)
if r.status_code != 200:
log.warning("x: timeline %s returned %s: %s",
handle, r.status_code, r.text[:200])
continue
payload = r.json()
posts = payload.get("data", []) or []
self._charge(len(posts) * POST_READ_USD)
for post in posts:
urls = ((post.get("entities") or {}).get("urls") or [])
external = next(
(u.get("expanded_url") for u in urls
if u.get("expanded_url")
and not _is_internal(u["expanded_url"])),
"",
)
yield Item(
source_id=post["id"],
location=external or f"https://x.com/{handle}/status/{post['id']}",
title="",
excerpt=post.get("text", ""),
author=handle,
published_at=parse_timestamp(post.get("created_at", "")),
subscription=label,
kind="post",
raw={
"metrics": post.get("public_metrics", {}),
"post_url": f"https://x.com/{handle}/status/{post['id']}",
},
)
def discover(self) -> list[Suggestion]:
# The follow-graph endpoints are billed per user read, which makes
# graph-walking on X expensive enough that it is not worth doing
# automatically. Use Bluesky/Mastodon discovery instead.
return []
# `check` is left at the default too: a user lookup is a billed read, and
# a check job must never cost money.