Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d666d0937
commit
248ea1636d
36 files changed
+8393
No files matched your search
@@ -0,0 +1,203 @@
|
||||
"""X (Twitter) via the official API v2. Off by default, because it costs money.
|
||||
|
||||
X moved to pay-per-usage billing. As published on the official pricing page,
|
||||
post reads are $0.005 per resource and user reads are $0.010 per resource,
|
||||
with pay-per-usage plans capped at 3 million post reads per billing cycle.
|
||||
Reads of your *own* account's data ("Owned Reads") are $0.001 per resource.
|
||||
|
||||
So a config with 30 accounts x 20 posts x 4 polls a day is roughly
|
||||
30*20*4*30 = 72,000 reads/month = about $360/month. Check the estimate this
|
||||
plugin prints before you enable it.
|
||||
|
||||
This plugin therefore:
|
||||
- is disabled unless you set `enabled: true`
|
||||
- refuses to run without `budget_usd_per_month`
|
||||
- tracks spend in the `state` table and stops when the budget is gone
|
||||
- caches user IDs so it never pays for a lookup twice
|
||||
|
||||
Source for rates: https://docs.x.com/x-api/getting-started/pricing
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from desk.sources.base import USER_AGENT, Item, Source, Suggestion, parse_timestamp
|
||||
from desk.sources.spec import (
|
||||
API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy,
|
||||
TaggingPolicy, TargetSpec, Tasks,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
API = "https://api.x.com/2"
|
||||
|
||||
POST_READ_USD = 0.005
|
||||
USER_READ_USD = 0.010
|
||||
|
||||
|
||||
def _is_internal(url: str) -> bool:
|
||||
"""Match the host boundary so linux.com remains an external article."""
|
||||
try:
|
||||
host = (urlsplit(url).hostname or "").lower().rstrip(".")
|
||||
except ValueError:
|
||||
return False
|
||||
return any(host == domain or host.endswith("." + domain)
|
||||
for domain in ("x.com", "twitter.com"))
|
||||
|
||||
|
||||
class XSource(Source):
|
||||
name = "x"
|
||||
default_outlet = "X"
|
||||
description = "X/Twitter timelines (paid API — off by default)."
|
||||
|
||||
#: Declared like the other two networks. The budget, the token and the
|
||||
#: refusal to run without either stay in the code above: they are what
|
||||
#: this plugin *does*, and none of them is a thing the shared jobs or the
|
||||
#: reader need to be told about.
|
||||
spec = PluginSpec(
|
||||
api_version=API_VERSION, id=name, version="1", label="X",
|
||||
targets=TargetSpec(mode="editable", kind="account", defaults=[],
|
||||
validator="x_handle"),
|
||||
profiles={"default": Profile(
|
||||
kind="post",
|
||||
acquisition=[Step(id="article", tool="extract_article",
|
||||
inputs={"url": "inputs.original_url"},
|
||||
outputs={"content": "inputs.article_text"})],
|
||||
tasks=Tasks(
|
||||
summary=SummaryPolicy(enabled=False),
|
||||
tagging=TaggingPolicy(
|
||||
enabled=True, input_fields=["inputs.feed_text"],
|
||||
context_fields=["author"]),
|
||||
),
|
||||
presentation=POST_READER,
|
||||
)},
|
||||
)
|
||||
|
||||
def __init__(self, options: dict[str, Any] | None = None) -> None:
|
||||
super().__init__(options)
|
||||
token_env = self.options.get("bearer_token_env", "X_BEARER_TOKEN")
|
||||
self.token = os.environ.get(token_env, "")
|
||||
self.client = httpx.Client(
|
||||
timeout=25,
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.token}",
|
||||
"User-Agent": USER_AGENT,
|
||||
},
|
||||
)
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return bool(self.options.get("enabled", False))
|
||||
|
||||
def estimate_monthly_usd(self, polls_per_day: int = 4) -> float:
|
||||
n = len(self.options.get("accounts", []))
|
||||
per_poll = n * int(self.options.get("posts_per_account", 10))
|
||||
return per_poll * polls_per_day * 30 * POST_READ_USD
|
||||
|
||||
# ------------------------------------------------------------ budget
|
||||
|
||||
def _spent(self) -> float:
|
||||
month = datetime.now(UTC).strftime("%Y-%m")
|
||||
return float(self.state.get(f"x_spend_{month}", "0") or 0)
|
||||
|
||||
def _charge(self, usd: float) -> None:
|
||||
month = datetime.now(UTC).strftime("%Y-%m")
|
||||
self.state[f"x_spend_{month}"] = str(self._spent() + usd)
|
||||
|
||||
def _user_id(self, handle: str) -> str | None:
|
||||
cache_key = f"x_userid_{handle.lower()}"
|
||||
cached = self.state.get(cache_key)
|
||||
if cached:
|
||||
return cached
|
||||
r = self.client.get(f"{API}/users/by/username/{handle}")
|
||||
if r.status_code != 200:
|
||||
log.warning("x: lookup %s returned %s", handle, r.status_code)
|
||||
return None
|
||||
self._charge(USER_READ_USD)
|
||||
uid = (r.json().get("data") or {}).get("id")
|
||||
if uid:
|
||||
self.state[cache_key] = uid
|
||||
return uid
|
||||
|
||||
# ------------------------------------------------------------- ingest
|
||||
|
||||
def fetch(self) -> Iterator[Item]:
|
||||
if not self.token:
|
||||
log.error("x: no bearer token in $%s",
|
||||
self.options.get("bearer_token_env", "X_BEARER_TOKEN"))
|
||||
return
|
||||
budget = self.options.get("budget_usd_per_month")
|
||||
if budget is None:
|
||||
log.error("x: refusing to run without `budget_usd_per_month` in config")
|
||||
return
|
||||
|
||||
per_account = int(self.options.get("posts_per_account", 10))
|
||||
exclude = ",".join(self.options.get("exclude", ["retweets", "replies"]))
|
||||
|
||||
for handle, label in self.targets():
|
||||
spent = self._spent()
|
||||
if spent >= float(budget):
|
||||
log.warning("x: monthly budget of $%.2f exhausted ($%.2f spent), stopping",
|
||||
float(budget), spent)
|
||||
return
|
||||
|
||||
uid = self._user_id(handle)
|
||||
if not uid:
|
||||
continue
|
||||
|
||||
r = self.client.get(
|
||||
f"{API}/users/{uid}/tweets",
|
||||
params={
|
||||
"max_results": max(5, min(per_account, 100)),
|
||||
"exclude": exclude,
|
||||
"tweet.fields": "created_at,entities,public_metrics",
|
||||
},
|
||||
)
|
||||
if r.status_code != 200:
|
||||
log.warning("x: timeline %s returned %s: %s",
|
||||
handle, r.status_code, r.text[:200])
|
||||
continue
|
||||
|
||||
payload = r.json()
|
||||
posts = payload.get("data", []) or []
|
||||
self._charge(len(posts) * POST_READ_USD)
|
||||
|
||||
for post in posts:
|
||||
urls = ((post.get("entities") or {}).get("urls") or [])
|
||||
external = next(
|
||||
(u.get("expanded_url") for u in urls
|
||||
if u.get("expanded_url")
|
||||
and not _is_internal(u["expanded_url"])),
|
||||
"",
|
||||
)
|
||||
yield Item(
|
||||
source_id=post["id"],
|
||||
location=external or f"https://x.com/{handle}/status/{post['id']}",
|
||||
title="",
|
||||
excerpt=post.get("text", ""),
|
||||
author=handle,
|
||||
published_at=parse_timestamp(post.get("created_at", "")),
|
||||
subscription=label,
|
||||
kind="post",
|
||||
raw={
|
||||
"metrics": post.get("public_metrics", {}),
|
||||
"post_url": f"https://x.com/{handle}/status/{post['id']}",
|
||||
},
|
||||
)
|
||||
|
||||
def discover(self) -> list[Suggestion]:
|
||||
# The follow-graph endpoints are billed per user read, which makes
|
||||
# graph-walking on X expensive enough that it is not worth doing
|
||||
# automatically. Use Bluesky/Mastodon discovery instead.
|
||||
return []
|
||||
|
||||
# `check` is left at the default too: a user lookup is a billed read, and
|
||||
# a check job must never cost money.
|
||||
Reference in new issue
Block a user