Files
jobboard/desk/sources/x.py
T
WolfandClaude Opus 5.5 248ea1636d Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:17 +02:00

204 lines
7.5 KiB
Python

"""X (Twitter) via the official API v2. Off by default, because it costs money.
X moved to pay-per-usage billing. As published on the official pricing page,
post reads are $0.005 per resource and user reads are $0.010 per resource,
with pay-per-usage plans capped at 3 million post reads per billing cycle.
Reads of your *own* account's data ("Owned Reads") are $0.001 per resource.
So a config with 30 accounts x 20 posts x 4 polls a day is roughly
30*20*4*30 = 72,000 reads/month = about $360/month. Check the estimate this
plugin prints before you enable it.
This plugin therefore:
- is disabled unless you set `enabled: true`
- refuses to run without `budget_usd_per_month`
- tracks spend in the `state` table and stops when the budget is gone
- caches user IDs so it never pays for a lookup twice
Source for rates: https://docs.x.com/x-api/getting-started/pricing
"""
from __future__ import annotations
from urllib.parse import urlsplit
import logging
import os
from collections.abc import Iterator
from datetime import UTC, datetime
from typing import Any
import httpx
from desk.sources.base import USER_AGENT, Item, Source, Suggestion, parse_timestamp
from desk.sources.spec import (
API_VERSION, POST_READER, PluginSpec, Profile, Step, SummaryPolicy,
TaggingPolicy, TargetSpec, Tasks,
)
log = logging.getLogger(__name__)
API = "https://api.x.com/2"
POST_READ_USD = 0.005
USER_READ_USD = 0.010
def _is_internal(url: str) -> bool:
"""Match the host boundary so linux.com remains an external article."""
try:
host = (urlsplit(url).hostname or "").lower().rstrip(".")
except ValueError:
return False
return any(host == domain or host.endswith("." + domain)
for domain in ("x.com", "twitter.com"))
class XSource(Source):
name = "x"
default_outlet = "X"
description = "X/Twitter timelines (paid API — off by default)."
#: Declared like the other two networks. The budget, the token and the
#: refusal to run without either stay in the code above: they are what
#: this plugin *does*, and none of them is a thing the shared jobs or the
#: reader need to be told about.
spec = PluginSpec(
api_version=API_VERSION, id=name, version="1", label="X",
targets=TargetSpec(mode="editable", kind="account", defaults=[],
validator="x_handle"),
profiles={"default": Profile(
kind="post",
acquisition=[Step(id="article", tool="extract_article",
inputs={"url": "inputs.original_url"},
outputs={"content": "inputs.article_text"})],
tasks=Tasks(
summary=SummaryPolicy(enabled=False),
tagging=TaggingPolicy(
enabled=True, input_fields=["inputs.feed_text"],
context_fields=["author"]),
),
presentation=POST_READER,
)},
)
def __init__(self, options: dict[str, Any] | None = None) -> None:
super().__init__(options)
token_env = self.options.get("bearer_token_env", "X_BEARER_TOKEN")
self.token = os.environ.get(token_env, "")
self.client = httpx.Client(
timeout=25,
headers={
"Authorization": f"Bearer {self.token}",
"User-Agent": USER_AGENT,
},
)
@property
def enabled(self) -> bool:
return bool(self.options.get("enabled", False))
def estimate_monthly_usd(self, polls_per_day: int = 4) -> float:
n = len(self.options.get("accounts", []))
per_poll = n * int(self.options.get("posts_per_account", 10))
return per_poll * polls_per_day * 30 * POST_READ_USD
# ------------------------------------------------------------ budget
def _spent(self) -> float:
month = datetime.now(UTC).strftime("%Y-%m")
return float(self.state.get(f"x_spend_{month}", "0") or 0)
def _charge(self, usd: float) -> None:
month = datetime.now(UTC).strftime("%Y-%m")
self.state[f"x_spend_{month}"] = str(self._spent() + usd)
def _user_id(self, handle: str) -> str | None:
cache_key = f"x_userid_{handle.lower()}"
cached = self.state.get(cache_key)
if cached:
return cached
r = self.client.get(f"{API}/users/by/username/{handle}")
if r.status_code != 200:
log.warning("x: lookup %s returned %s", handle, r.status_code)
return None
self._charge(USER_READ_USD)
uid = (r.json().get("data") or {}).get("id")
if uid:
self.state[cache_key] = uid
return uid
# ------------------------------------------------------------- ingest
def fetch(self) -> Iterator[Item]:
if not self.token:
log.error("x: no bearer token in $%s",
self.options.get("bearer_token_env", "X_BEARER_TOKEN"))
return
budget = self.options.get("budget_usd_per_month")
if budget is None:
log.error("x: refusing to run without `budget_usd_per_month` in config")
return
per_account = int(self.options.get("posts_per_account", 10))
exclude = ",".join(self.options.get("exclude", ["retweets", "replies"]))
for handle, label in self.targets():
spent = self._spent()
if spent >= float(budget):
log.warning("x: monthly budget of $%.2f exhausted ($%.2f spent), stopping",
float(budget), spent)
return
uid = self._user_id(handle)
if not uid:
continue
r = self.client.get(
f"{API}/users/{uid}/tweets",
params={
"max_results": max(5, min(per_account, 100)),
"exclude": exclude,
"tweet.fields": "created_at,entities,public_metrics",
},
)
if r.status_code != 200:
log.warning("x: timeline %s returned %s: %s",
handle, r.status_code, r.text[:200])
continue
payload = r.json()
posts = payload.get("data", []) or []
self._charge(len(posts) * POST_READ_USD)
for post in posts:
urls = ((post.get("entities") or {}).get("urls") or [])
external = next(
(u.get("expanded_url") for u in urls
if u.get("expanded_url")
and not _is_internal(u["expanded_url"])),
"",
)
yield Item(
source_id=post["id"],
location=external or f"https://x.com/{handle}/status/{post['id']}",
title="",
excerpt=post.get("text", ""),
author=handle,
published_at=parse_timestamp(post.get("created_at", "")),
subscription=label,
kind="post",
raw={
"metrics": post.get("public_metrics", {}),
"post_url": f"https://x.com/{handle}/status/{post['id']}",
},
)
def discover(self) -> list[Suggestion]:
# The follow-graph endpoints are billed per user read, which makes
# graph-walking on X expensive enough that it is not worth doing
# automatically. Use Bluesky/Mastodon discovery instead.
return []
# `check` is left at the default too: a user lookup is a billed read, and
# a check job must never cost money.