Files
jobboard/desk/tools/checks.py
T
WolfandClaude Opus 5.5 248ea1636d Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:17 +02:00

228 lines
8.9 KiB
Python

"""Did the getter bring back a page worth converting, and if not, what next?
`scrape` fetches, this judges, `article` converts. A 200 is not an article
because it answered 200: an app shell serves a `<noscript>` notice asking for
JavaScript, a bot check serves a few words and a script. Converting either
stores the notice as the article.
A check answers with a `Verdict`: the failure category from
`desk.tools.failures`, the name of what matched, and which getters are worth
trying next. That last part is where what we learn goes. A Mastodon post is
a shell over HTTP and a status over Mastodon's API, so its signature names
`mastodon_api`; a bot check names `browser`. A getter the app does not have is
skipped, so a signature can name one before it is built.
Checks run in order and the first match decides:
1. the status code
2. no HTML at all
3. `SIGNATURES`, the table below, most specific first
4. the structural rule, for a shell nobody has written a signature for yet
**Adding a signature** is one entry in `SIGNATURES` and one test page in
`tests/test_checks.py`. The entry names the page it was found on, so whoever
reads it later can fetch that page and see whether it still says this.
Signature(
name="mastodon_shell", # logged, and on the attempt
where="noscript", # title | text | noscript | html | url
pattern=r"To use the Mastodon web application", # regex, case-insensitive
category="js_required", # from desk.tools.failures
next=("mastodon_api", "browser"), # getters to try, in order
seen="https://grapheneos.social/@GrapheneOS/117225539756835649",
)
`where` is what the pattern is matched against: `title` the `<title>`, `text`
the words a reader without JavaScript sees outside `<noscript>`, `noscript`
the words inside it, `html` the raw markup (script sources, class names),
`url` the address the getter ended up at after redirects. Prefer the narrowest
one that works: a phrase in `text` can appear in an article about the thing
it describes, a class name in `html` usually cannot.
A pattern that does not compile, a category the failure vocabulary does not
have, or two signatures with one name is refused at import.
"""
from __future__ import annotations
import re
from collections.abc import Mapping
from dataclasses import dataclass, field
from html.parser import HTMLParser
from typing import Literal
from desk.tools import failures
Where = Literal["title", "text", "noscript", "html", "url"]
@dataclass(frozen=True)
class Page:
"""What one getter brought back. `url` is where it ended up."""
status: int
html: str
url: str = ""
headers: Mapping[str, str] = field(default_factory=dict)
@dataclass(frozen=True)
class Verdict:
"""Why this page is not worth converting, and what might get the real one."""
category: str
#: The signature or rule that matched. What a person reads in the log to
#: decide whether the table needs a new entry or a narrower one.
name: str
next: tuple[str, ...] = ()
@dataclass(frozen=True)
class Signature:
name: str
where: Where
pattern: str
category: str
next: tuple[str, ...] = ()
#: A page this was found on.
seen: str = ""
def __post_init__(self) -> None:
re.compile(self.pattern)
if self.category not in failures.TRANSIENT | failures.SETTLED:
raise ValueError(f"signature {self.name!r} names category "
f"{self.category!r}, which desk.tools.failures has not")
#: Most specific first. See the module docstring for the format.
SIGNATURES: tuple[Signature, ...] = (
Signature("mastodon_shell", "noscript", r"To use the Mastodon web application",
"js_required", ("mastodon_api", "browser"),
seen="https://grapheneos.social/@GrapheneOS/117225539756835649"),
Signature("bluesky_shell", "noscript", r"This is a heavily interactive web application, and JavaScript is required",
"js_required", ("bluesky_api", "browser"),
seen="https://bsky.app/profile/bsky.app/post/3mv3shqdfuc2e"),
Signature("create_react_app_shell", "noscript", r"You need to enable JavaScript to run this app",
"js_required", ("browser",),
seen="https://rune.build/blog/rune-is-now-open-source"),
Signature("cloudflare_turnstile", "html", r"challenges\.cloudflare\.com/turnstile",
"js_required", ("browser",),
seen="https://openreview.net/forum?id=pc7fqaOcAH"),
Signature("anubis_check", "text", r"automated check to get rid of most bots|Making sure you're not a bot",
"js_required", ("browser",),
seen="https://codeberg.org/forgejo/forgejo/src/branch/forgejo/release-notes-published/16.0.4.md"),
# Every article redirects here without the consent cookie. The browser
# accepts Sourcepoint's dialog, and the cookies it earns open the gate for
# HTTP until they expire. See `desk.tools.browser`.
Signature("golem_consent_gate", "html", r"GolemConsent\.init",
"consent_wall", ("browser",),
seen="https://www.golem.de/sonstiges/zustimmung/auswahl.html"),
)
#: Below this many words outside `<noscript>`, a page carrying a script is a
#: placeholder. OpenReview's check is 36 words, Anubis 42; a real article is
#: several hundred.
SCRIPT_PAGE_WORDS = 80
#: The longest `<noscript>` read as a notice. Mastodon's is 21 words,
#: Bluesky's 76. A longer one is a site serving its content without scripts.
NOTICE_WORDS = 150
#: Where the structural rule sends a page nobody has a signature for.
UNRECOGNISED_SHELL_NEXT = ("browser",)
def _unique_names(signatures: tuple[Signature, ...]) -> None:
names = [s.name for s in signatures]
if len(names) != len(set(names)):
raise ValueError(f"two signatures share a name: "
f"{sorted({n for n in names if names.count(n) > 1})}")
_unique_names(SIGNATURES)
def check(page: Page | None, signatures: tuple[Signature, ...] = SIGNATURES) -> Verdict | None:
"""None when the page is worth converting, else why not."""
if page is None:
# The downloader logs which way it failed and returns nothing: DNS, a
# refused connection, a reset and TLS all arrive here as one.
return Verdict("unreachable", "no_response")
if why := failures.of_status(page.status):
return Verdict(why, f"status_{page.status}")
if not page.html.strip():
return Verdict("empty", "no_html")
parts = _Parts.of(page)
for signature in signatures:
if re.search(signature.pattern, parts[signature.where], re.IGNORECASE):
return Verdict(signature.category, signature.name, signature.next)
if (parts.scripts and len(_words(parts["text"])) < SCRIPT_PAGE_WORDS
and len(_words(parts["noscript"])) <= NOTICE_WORDS):
return Verdict("js_required", "script_page", UNRECOGNISED_SHELL_NEXT)
return None
def _flat(pieces: list[str]) -> str:
return " ".join(" ".join(pieces).split())
def _words(text: str) -> list[str]:
return re.findall(r"\w+", text)
class _Parts(dict[str, str]):
"""The five things a signature can match against, read in one pass."""
scripts: int
@classmethod
def of(cls, page: Page) -> _Parts:
reader = _Reader()
try:
reader.feed(page.html)
reader.close()
except Exception: # malformed markup: match what was read so far
pass
# Whitespace collapsed in the three text parts, so a signature is
# written with single spaces however the page wrapped its lines.
parts = cls(title=_flat(reader.title), text=_flat(reader.text),
noscript=_flat(reader.noscript), html=page.html, url=page.url)
parts.scripts = reader.scripts
return parts
class _Reader(HTMLParser):
HIDDEN = frozenset({"script", "style", "template", "svg"})
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.title: list[str] = []
self.text: list[str] = []
self.noscript: list[str] = []
self.scripts = 0
self._hidden = 0
self._in: dict[str, int] = {"title": 0, "noscript": 0}
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
self.scripts += tag == "script"
if tag in self.HIDDEN:
self._hidden += 1
elif tag in self._in:
self._in[tag] += 1
def handle_endtag(self, tag: str) -> None:
if tag in self.HIDDEN:
self._hidden = max(0, self._hidden - 1)
elif tag in self._in:
self._in[tag] = max(0, self._in[tag] - 1)
def handle_data(self, data: str) -> None:
if self._hidden:
return
if self._in["title"]:
self.title.append(data)
elif self._in["noscript"]:
self.noscript.append(data)
else:
self.text.append(data)