The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
204 lines
7.5 KiB
Python
204 lines
7.5 KiB
Python
"""X (Twitter) via the official API v2. Off by default, because it costs money.
|
|
|
|
X moved to pay-per-usage billing. As published on the official pricing page,
|
|
post reads are $0.005 per resource and user reads are $0.010 per resource,
|
|
with pay-per-usage plans capped at 3 million post reads per billing cycle.
|
|
Reads of your *own* account's data ("Owned Reads") are $0.001 per resource.
|
|
|
|
So a config with 30 accounts x 20 posts x 4 polls a day is roughly
|
|
30*20*4*30 = 72,000 reads/month = about $360/month. Check the estimate this
|
|
plugin prints before you enable it.
|
|
|
|
This plugin therefore:
|
|
- is disabled unless you set `enabled: true`
|
|
- refuses to run without `budget_usd_per_month`
|
|
- tracks spend in the `state` table and stops when the budget is gone
|
|
- caches user IDs so it never pays for a lookup twice
|
|
|
|
Source for rates: https://docs.x.com/x-api/getting-started/pricing
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from urllib.parse import urlsplit
|
|
|
|
import logging
|
|
import os
|
|
from collections.abc import Iterator
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from desk.sources.base import USER_AGENT, Item, Source, Suggestion, parse_timestamp
|
|
from desk.sources.spec import (
|
|
API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy,
|
|
TaggingPolicy, TargetSpec, Tasks,
|
|
)
|
|
|
|
log = logging.getLogger(__name__)
|
|
API = "https://api.x.com/2"
|
|
|
|
POST_READ_USD = 0.005
|
|
USER_READ_USD = 0.010
|
|
|
|
|
|
def _is_internal(url: str) -> bool:
|
|
"""Match the host boundary so linux.com remains an external article."""
|
|
try:
|
|
host = (urlsplit(url).hostname or "").lower().rstrip(".")
|
|
except ValueError:
|
|
return False
|
|
return any(host == domain or host.endswith("." + domain)
|
|
for domain in ("x.com", "twitter.com"))
|
|
|
|
|
|
class XSource(Source):
|
|
name = "x"
|
|
default_outlet = "X"
|
|
description = "X/Twitter timelines (paid API — off by default)."
|
|
|
|
#: Declared like the other two networks. The budget, the token and the
|
|
#: refusal to run without either stay in the code above: they are what
|
|
#: this plugin *does*, and none of them is a thing the shared jobs or the
|
|
#: reader need to be told about.
|
|
spec = PluginSpec(
|
|
api_version=API_VERSION, id=name, version="1", label="X",
|
|
targets=TargetSpec(mode="editable", kind="account", defaults=[],
|
|
validator="x_handle"),
|
|
profiles={"default": Profile(
|
|
kind="post",
|
|
acquisition=[Step(id="article", tool="extract_article",
|
|
inputs={"url": "inputs.original_url"},
|
|
outputs={"content": "inputs.article_text"})],
|
|
tasks=Tasks(
|
|
summary=SummaryPolicy(enabled=False),
|
|
tagging=TaggingPolicy(
|
|
enabled=True, input_fields=["inputs.feed_text"],
|
|
context_fields=["author"]),
|
|
),
|
|
presentation=POST_READER,
|
|
)},
|
|
)
|
|
|
|
def __init__(self, options: dict[str, Any] | None = None) -> None:
|
|
super().__init__(options)
|
|
token_env = self.options.get("bearer_token_env", "X_BEARER_TOKEN")
|
|
self.token = os.environ.get(token_env, "")
|
|
self.client = httpx.Client(
|
|
timeout=25,
|
|
headers={
|
|
"Authorization": f"Bearer {self.token}",
|
|
"User-Agent": USER_AGENT,
|
|
},
|
|
)
|
|
|
|
@property
|
|
def enabled(self) -> bool:
|
|
return bool(self.options.get("enabled", False))
|
|
|
|
def estimate_monthly_usd(self, polls_per_day: int = 4) -> float:
|
|
n = len(self.options.get("accounts", []))
|
|
per_poll = n * int(self.options.get("posts_per_account", 10))
|
|
return per_poll * polls_per_day * 30 * POST_READ_USD
|
|
|
|
# ------------------------------------------------------------ budget
|
|
|
|
def _spent(self) -> float:
|
|
month = datetime.now(UTC).strftime("%Y-%m")
|
|
return float(self.state.get(f"x_spend_{month}", "0") or 0)
|
|
|
|
def _charge(self, usd: float) -> None:
|
|
month = datetime.now(UTC).strftime("%Y-%m")
|
|
self.state[f"x_spend_{month}"] = str(self._spent() + usd)
|
|
|
|
def _user_id(self, handle: str) -> str | None:
|
|
cache_key = f"x_userid_{handle.lower()}"
|
|
cached = self.state.get(cache_key)
|
|
if cached:
|
|
return cached
|
|
r = self.client.get(f"{API}/users/by/username/{handle}")
|
|
if r.status_code != 200:
|
|
log.warning("x: lookup %s returned %s", handle, r.status_code)
|
|
return None
|
|
self._charge(USER_READ_USD)
|
|
uid = (r.json().get("data") or {}).get("id")
|
|
if uid:
|
|
self.state[cache_key] = uid
|
|
return uid
|
|
|
|
# ------------------------------------------------------------- ingest
|
|
|
|
def fetch(self) -> Iterator[Item]:
|
|
if not self.token:
|
|
log.error("x: no bearer token in $%s",
|
|
self.options.get("bearer_token_env", "X_BEARER_TOKEN"))
|
|
return
|
|
budget = self.options.get("budget_usd_per_month")
|
|
if budget is None:
|
|
log.error("x: refusing to run without `budget_usd_per_month` in config")
|
|
return
|
|
|
|
per_account = int(self.options.get("posts_per_account", 10))
|
|
exclude = ",".join(self.options.get("exclude", ["retweets", "replies"]))
|
|
|
|
for handle, label in self.targets():
|
|
spent = self._spent()
|
|
if spent >= float(budget):
|
|
log.warning("x: monthly budget of $%.2f exhausted ($%.2f spent), stopping",
|
|
float(budget), spent)
|
|
return
|
|
|
|
uid = self._user_id(handle)
|
|
if not uid:
|
|
continue
|
|
|
|
r = self.client.get(
|
|
f"{API}/users/{uid}/tweets",
|
|
params={
|
|
"max_results": max(5, min(per_account, 100)),
|
|
"exclude": exclude,
|
|
"tweet.fields": "created_at,entities,public_metrics",
|
|
},
|
|
)
|
|
if r.status_code != 200:
|
|
log.warning("x: timeline %s returned %s: %s",
|
|
handle, r.status_code, r.text[:200])
|
|
continue
|
|
|
|
payload = r.json()
|
|
posts = payload.get("data", []) or []
|
|
self._charge(len(posts) * POST_READ_USD)
|
|
|
|
for post in posts:
|
|
urls = ((post.get("entities") or {}).get("urls") or [])
|
|
external = next(
|
|
(u.get("expanded_url") for u in urls
|
|
if u.get("expanded_url")
|
|
and not _is_internal(u["expanded_url"])),
|
|
"",
|
|
)
|
|
yield Item(
|
|
source_id=post["id"],
|
|
location=external or f"https://x.com/{handle}/status/{post['id']}",
|
|
title="",
|
|
excerpt=post.get("text", ""),
|
|
author=handle,
|
|
published_at=parse_timestamp(post.get("created_at", "")),
|
|
subscription=label,
|
|
kind="post",
|
|
raw={
|
|
"metrics": post.get("public_metrics", {}),
|
|
"post_url": f"https://x.com/{handle}/status/{post['id']}",
|
|
},
|
|
)
|
|
|
|
def discover(self) -> list[Suggestion]:
|
|
# The follow-graph endpoints are billed per user read, which makes
|
|
# graph-walking on X expensive enough that it is not worth doing
|
|
# automatically. Use Bluesky/Mastodon discovery instead.
|
|
return []
|
|
|
|
# `check` is left at the default too: a user lookup is a billed read, and
|
|
# a check job must never cost money.
|