Bring in desk from news

The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-23 12:29:17 +02:00
1 parent 9d666d0937
commit 248ea1636d
36 files changed
+8393

No files matched your search

+227
View File
@@ -0,0 +1,227 @@
"""Did the getter bring back a page worth converting, and if not, what next?
`scrape` fetches, this judges, `article` converts. A 200 is not an article
because it answered 200: an app shell serves a `<noscript>` notice asking for
JavaScript, a bot check serves a few words and a script. Converting either
stores the notice as the article.
A check answers with a `Verdict`: the failure category from
`desk.tools.failures`, the name of what matched, and which getters are worth
trying next. That last part is where what we learn goes. A Mastodon post is
a shell over HTTP and a status over Mastodon's API, so its signature names
`mastodon_api`; a bot check names `browser`. A getter the app does not have is
skipped, so a signature can name one before it is built.
Checks run in order and the first match decides:
1. the status code
2. no HTML at all
3. `SIGNATURES`, the table below, most specific first
4. the structural rule, for a shell nobody has written a signature for yet
**Adding a signature** is one entry in `SIGNATURES` and one test page in
`tests/test_checks.py`. The entry names the page it was found on, so whoever
reads it later can fetch that page and see whether it still says this.
Signature(
name="mastodon_shell", # logged, and on the attempt
where="noscript", # title | text | noscript | html | url
pattern=r"To use the Mastodon web application", # regex, case-insensitive
category="js_required", # from desk.tools.failures
next=("mastodon_api", "browser"), # getters to try, in order
seen="https://grapheneos.social/@GrapheneOS/117225539756835649",
)
`where` is what the pattern is matched against: `title` the `<title>`, `text`
the words a reader without JavaScript sees outside `<noscript>`, `noscript`
the words inside it, `html` the raw markup (script sources, class names),
`url` the address the getter ended up at after redirects. Prefer the narrowest
one that works: a phrase in `text` can appear in an article about the thing
it describes, a class name in `html` usually cannot.
A pattern that does not compile, a category the failure vocabulary does not
have, or two signatures with one name is refused at import.
"""
from __future__ import annotations
import re
from collections.abc import Mapping
from dataclasses import dataclass, field
from html.parser import HTMLParser
from typing import Literal
from desk.tools import failures
Where = Literal["title", "text", "noscript", "html", "url"]
@dataclass(frozen=True)
class Page:
"""What one getter brought back. `url` is where it ended up."""
status: int
html: str
url: str = ""
headers: Mapping[str, str] = field(default_factory=dict)
@dataclass(frozen=True)
class Verdict:
"""Why this page is not worth converting, and what might get the real one."""
category: str
#: The signature or rule that matched. What a person reads in the log to
#: decide whether the table needs a new entry or a narrower one.
name: str
next: tuple[str, ...] = ()
@dataclass(frozen=True)
class Signature:
name: str
where: Where
pattern: str
category: str
next: tuple[str, ...] = ()
#: A page this was found on.
seen: str = ""
def __post_init__(self) -> None:
re.compile(self.pattern)
if self.category not in failures.TRANSIENT | failures.SETTLED:
raise ValueError(f"signature {self.name!r} names category "
f"{self.category!r}, which desk.tools.failures has not")
#: Most specific first. See the module docstring for the format.
SIGNATURES: tuple[Signature, ...] = (
Signature("mastodon_shell", "noscript", r"To use the Mastodon web application",
"js_required", ("mastodon_api", "browser"),
seen="https://grapheneos.social/@GrapheneOS/117225539756835649"),
Signature("bluesky_shell", "noscript", r"This is a heavily interactive web application, and JavaScript is required",
"js_required", ("bluesky_api", "browser"),
seen="https://bsky.app/profile/bsky.app/post/3mv3shqdfuc2e"),
Signature("create_react_app_shell", "noscript", r"You need to enable JavaScript to run this app",
"js_required", ("browser",),
seen="https://rune.build/blog/rune-is-now-open-source"),
Signature("cloudflare_turnstile", "html", r"challenges\.cloudflare\.com/turnstile",
"js_required", ("browser",),
seen="https://openreview.net/forum?id=pc7fqaOcAH"),
Signature("anubis_check", "text", r"automated check to get rid of most bots|Making sure you're not a bot",
"js_required", ("browser",),
seen="https://codeberg.org/forgejo/forgejo/src/branch/forgejo/release-notes-published/16.0.4.md"),
# Every article redirects here without the consent cookie. The browser
# accepts Sourcepoint's dialog, and the cookies it earns open the gate for
# HTTP until they expire. See `desk.tools.browser`.
Signature("golem_consent_gate", "html", r"GolemConsent\.init",
"consent_wall", ("browser",),
seen="https://www.golem.de/sonstiges/zustimmung/auswahl.html"),
)
#: Below this many words outside `<noscript>`, a page carrying a script is a
#: placeholder. OpenReview's check is 36 words, Anubis 42; a real article is
#: several hundred.
SCRIPT_PAGE_WORDS = 80
#: The longest `<noscript>` read as a notice. Mastodon's is 21 words,
#: Bluesky's 76. A longer one is a site serving its content without scripts.
NOTICE_WORDS = 150
#: Where the structural rule sends a page nobody has a signature for.
UNRECOGNISED_SHELL_NEXT = ("browser",)
def _unique_names(signatures: tuple[Signature, ...]) -> None:
names = [s.name for s in signatures]
if len(names) != len(set(names)):
raise ValueError(f"two signatures share a name: "
f"{sorted({n for n in names if names.count(n) > 1})}")
_unique_names(SIGNATURES)
def check(page: Page | None, signatures: tuple[Signature, ...] = SIGNATURES) -> Verdict | None:
"""None when the page is worth converting, else why not."""
if page is None:
# The downloader logs which way it failed and returns nothing: DNS, a
# refused connection, a reset and TLS all arrive here as one.
return Verdict("unreachable", "no_response")
if why := failures.of_status(page.status):
return Verdict(why, f"status_{page.status}")
if not page.html.strip():
return Verdict("empty", "no_html")
parts = _Parts.of(page)
for signature in signatures:
if re.search(signature.pattern, parts[signature.where], re.IGNORECASE):
return Verdict(signature.category, signature.name, signature.next)
if (parts.scripts and len(_words(parts["text"])) < SCRIPT_PAGE_WORDS
and len(_words(parts["noscript"])) <= NOTICE_WORDS):
return Verdict("js_required", "script_page", UNRECOGNISED_SHELL_NEXT)
return None
def _flat(pieces: list[str]) -> str:
return " ".join(" ".join(pieces).split())
def _words(text: str) -> list[str]:
return re.findall(r"\w+", text)
class _Parts(dict[str, str]):
"""The five things a signature can match against, read in one pass."""
scripts: int
@classmethod
def of(cls, page: Page) -> _Parts:
reader = _Reader()
try:
reader.feed(page.html)
reader.close()
except Exception: # malformed markup: match what was read so far
pass
# Whitespace collapsed in the three text parts, so a signature is
# written with single spaces however the page wrapped its lines.
parts = cls(title=_flat(reader.title), text=_flat(reader.text),
noscript=_flat(reader.noscript), html=page.html, url=page.url)
parts.scripts = reader.scripts
return parts
class _Reader(HTMLParser):
HIDDEN = frozenset({"script", "style", "template", "svg"})
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.title: list[str] = []
self.text: list[str] = []
self.noscript: list[str] = []
self.scripts = 0
self._hidden = 0
self._in: dict[str, int] = {"title": 0, "noscript": 0}
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
self.scripts += tag == "script"
if tag in self.HIDDEN:
self._hidden += 1
elif tag in self._in:
self._in[tag] += 1
def handle_endtag(self, tag: str) -> None:
if tag in self.HIDDEN:
self._hidden = max(0, self._hidden - 1)
elif tag in self._in:
self._in[tag] = max(0, self._in[tag] - 1)
def handle_data(self, data: str) -> None:
if self._hidden:
return
if self._in["title"]:
self.title.append(data)
elif self._in["noscript"]:
self.noscript.append(data)
else:
self.text.append(data)