"""X (Twitter) via the official API v2. Off by default, because it costs money. X moved to pay-per-usage billing. As published on the official pricing page, post reads are $0.005 per resource and user reads are $0.010 per resource, with pay-per-usage plans capped at 3 million post reads per billing cycle. Reads of your *own* account's data ("Owned Reads") are $0.001 per resource. So a config with 30 accounts x 20 posts x 4 polls a day is roughly 30*20*4*30 = 72,000 reads/month = about $360/month. Check the estimate this plugin prints before you enable it. This plugin therefore: - is disabled unless you set `enabled: true` - refuses to run without `budget_usd_per_month` - tracks spend in the `state` table and stops when the budget is gone - caches user IDs so it never pays for a lookup twice Source for rates: https://docs.x.com/x-api/getting-started/pricing """ from __future__ import annotations from urllib.parse import urlsplit import logging import os from collections.abc import Iterator from datetime import UTC, datetime from typing import Any import httpx from desk.sources.base import USER_AGENT, Item, Source, Suggestion, parse_timestamp from desk.sources.spec import ( API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, ) log = logging.getLogger(__name__) API = "https://api.x.com/2" POST_READ_USD = 0.005 USER_READ_USD = 0.010 def _is_internal(url: str) -> bool: """Match the host boundary so linux.com remains an external article.""" try: host = (urlsplit(url).hostname or "").lower().rstrip(".") except ValueError: return False return any(host == domain or host.endswith("." + domain) for domain in ("x.com", "twitter.com")) class XSource(Source): name = "x" default_outlet = "X" description = "X/Twitter timelines (paid API — off by default)." #: Declared like the other two networks. The budget, the token and the #: refusal to run without either stay in the code above: they are what #: this plugin *does*, and none of them is a thing the shared jobs or the #: reader need to be told about. spec = PluginSpec( api_version=API_VERSION, id=name, version="1", label="X", targets=TargetSpec(mode="editable", kind="account", defaults=[], validator="x_handle"), profiles={"default": Profile( kind="post", acquisition=[Step(id="article", tool="extract_article", inputs={"url": "inputs.original_url"}, outputs={"content": "inputs.article_text"})], tasks=Tasks( summary=SummaryPolicy(enabled=False), tagging=TaggingPolicy( enabled=True, input_fields=["inputs.feed_text"], context_fields=["author"]), ), presentation=POST_READER, )}, ) def __init__(self, options: dict[str, Any] | None = None) -> None: super().__init__(options) token_env = self.options.get("bearer_token_env", "X_BEARER_TOKEN") self.token = os.environ.get(token_env, "") self.client = httpx.Client( timeout=25, headers={ "Authorization": f"Bearer {self.token}", "User-Agent": USER_AGENT, }, ) @property def enabled(self) -> bool: return bool(self.options.get("enabled", False)) def estimate_monthly_usd(self, polls_per_day: int = 4) -> float: n = len(self.options.get("accounts", [])) per_poll = n * int(self.options.get("posts_per_account", 10)) return per_poll * polls_per_day * 30 * POST_READ_USD # ------------------------------------------------------------ budget def _spent(self) -> float: month = datetime.now(UTC).strftime("%Y-%m") return float(self.state.get(f"x_spend_{month}", "0") or 0) def _charge(self, usd: float) -> None: month = datetime.now(UTC).strftime("%Y-%m") self.state[f"x_spend_{month}"] = str(self._spent() + usd) def _user_id(self, handle: str) -> str | None: cache_key = f"x_userid_{handle.lower()}" cached = self.state.get(cache_key) if cached: return cached r = self.client.get(f"{API}/users/by/username/{handle}") if r.status_code != 200: log.warning("x: lookup %s returned %s", handle, r.status_code) return None self._charge(USER_READ_USD) uid = (r.json().get("data") or {}).get("id") if uid: self.state[cache_key] = uid return uid # ------------------------------------------------------------- ingest def fetch(self) -> Iterator[Item]: if not self.token: log.error("x: no bearer token in $%s", self.options.get("bearer_token_env", "X_BEARER_TOKEN")) return budget = self.options.get("budget_usd_per_month") if budget is None: log.error("x: refusing to run without `budget_usd_per_month` in config") return per_account = int(self.options.get("posts_per_account", 10)) exclude = ",".join(self.options.get("exclude", ["retweets", "replies"])) for handle, label in self.targets(): spent = self._spent() if spent >= float(budget): log.warning("x: monthly budget of $%.2f exhausted ($%.2f spent), stopping", float(budget), spent) return uid = self._user_id(handle) if not uid: continue r = self.client.get( f"{API}/users/{uid}/tweets", params={ "max_results": max(5, min(per_account, 100)), "exclude": exclude, "tweet.fields": "created_at,entities,public_metrics", }, ) if r.status_code != 200: log.warning("x: timeline %s returned %s: %s", handle, r.status_code, r.text[:200]) continue payload = r.json() posts = payload.get("data", []) or [] self._charge(len(posts) * POST_READ_USD) for post in posts: urls = ((post.get("entities") or {}).get("urls") or []) external = next( (u.get("expanded_url") for u in urls if u.get("expanded_url") and not _is_internal(u["expanded_url"])), "", ) yield Item( source_id=post["id"], location=external or f"https://x.com/{handle}/status/{post['id']}", title="", excerpt=post.get("text", ""), author=handle, published_at=parse_timestamp(post.get("created_at", "")), subscription=label, kind="post", raw={ "metrics": post.get("public_metrics", {}), "post_url": f"https://x.com/{handle}/status/{post['id']}", }, ) def discover(self) -> list[Suggestion]: # The follow-graph endpoints are billed per user read, which makes # graph-walking on X expensive enough that it is not worth doing # automatically. Use Bluesky/Mastodon discovery instead. return [] # `check` is left at the default too: a user lookup is a billed read, and # a check job must never cost money.