The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
228 lines
8.9 KiB
Python
228 lines
8.9 KiB
Python
"""Did the getter bring back a page worth converting, and if not, what next?
|
|
|
|
`scrape` fetches, this judges, `article` converts. A 200 is not an article
|
|
because it answered 200: an app shell serves a `<noscript>` notice asking for
|
|
JavaScript, a bot check serves a few words and a script. Converting either
|
|
stores the notice as the article.
|
|
|
|
A check answers with a `Verdict`: the failure category from
|
|
`desk.tools.failures`, the name of what matched, and which getters are worth
|
|
trying next. That last part is where what we learn goes. A Mastodon post is
|
|
a shell over HTTP and a status over Mastodon's API, so its signature names
|
|
`mastodon_api`; a bot check names `browser`. A getter the app does not have is
|
|
skipped, so a signature can name one before it is built.
|
|
|
|
Checks run in order and the first match decides:
|
|
|
|
1. the status code
|
|
2. no HTML at all
|
|
3. `SIGNATURES`, the table below, most specific first
|
|
4. the structural rule, for a shell nobody has written a signature for yet
|
|
|
|
**Adding a signature** is one entry in `SIGNATURES` and one test page in
|
|
`tests/test_checks.py`. The entry names the page it was found on, so whoever
|
|
reads it later can fetch that page and see whether it still says this.
|
|
|
|
Signature(
|
|
name="mastodon_shell", # logged, and on the attempt
|
|
where="noscript", # title | text | noscript | html | url
|
|
pattern=r"To use the Mastodon web application", # regex, case-insensitive
|
|
category="js_required", # from desk.tools.failures
|
|
next=("mastodon_api", "browser"), # getters to try, in order
|
|
seen="https://grapheneos.social/@GrapheneOS/117225539756835649",
|
|
)
|
|
|
|
`where` is what the pattern is matched against: `title` the `<title>`, `text`
|
|
the words a reader without JavaScript sees outside `<noscript>`, `noscript`
|
|
the words inside it, `html` the raw markup (script sources, class names),
|
|
`url` the address the getter ended up at after redirects. Prefer the narrowest
|
|
one that works: a phrase in `text` can appear in an article about the thing
|
|
it describes, a class name in `html` usually cannot.
|
|
|
|
A pattern that does not compile, a category the failure vocabulary does not
|
|
have, or two signatures with one name is refused at import.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from collections.abc import Mapping
|
|
from dataclasses import dataclass, field
|
|
from html.parser import HTMLParser
|
|
from typing import Literal
|
|
|
|
from desk.tools import failures
|
|
|
|
Where = Literal["title", "text", "noscript", "html", "url"]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Page:
|
|
"""What one getter brought back. `url` is where it ended up."""
|
|
|
|
status: int
|
|
html: str
|
|
url: str = ""
|
|
headers: Mapping[str, str] = field(default_factory=dict)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Verdict:
|
|
"""Why this page is not worth converting, and what might get the real one."""
|
|
|
|
category: str
|
|
#: The signature or rule that matched. What a person reads in the log to
|
|
#: decide whether the table needs a new entry or a narrower one.
|
|
name: str
|
|
next: tuple[str, ...] = ()
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Signature:
|
|
name: str
|
|
where: Where
|
|
pattern: str
|
|
category: str
|
|
next: tuple[str, ...] = ()
|
|
#: A page this was found on.
|
|
seen: str = ""
|
|
|
|
def __post_init__(self) -> None:
|
|
re.compile(self.pattern)
|
|
if self.category not in failures.TRANSIENT | failures.SETTLED:
|
|
raise ValueError(f"signature {self.name!r} names category "
|
|
f"{self.category!r}, which desk.tools.failures has not")
|
|
|
|
|
|
#: Most specific first. See the module docstring for the format.
|
|
SIGNATURES: tuple[Signature, ...] = (
|
|
Signature("mastodon_shell", "noscript", r"To use the Mastodon web application",
|
|
"js_required", ("mastodon_api", "browser"),
|
|
seen="https://grapheneos.social/@GrapheneOS/117225539756835649"),
|
|
Signature("bluesky_shell", "noscript", r"This is a heavily interactive web application, and JavaScript is required",
|
|
"js_required", ("bluesky_api", "browser"),
|
|
seen="https://bsky.app/profile/bsky.app/post/3mv3shqdfuc2e"),
|
|
Signature("create_react_app_shell", "noscript", r"You need to enable JavaScript to run this app",
|
|
"js_required", ("browser",),
|
|
seen="https://rune.build/blog/rune-is-now-open-source"),
|
|
Signature("cloudflare_turnstile", "html", r"challenges\.cloudflare\.com/turnstile",
|
|
"js_required", ("browser",),
|
|
seen="https://openreview.net/forum?id=pc7fqaOcAH"),
|
|
Signature("anubis_check", "text", r"automated check to get rid of most bots|Making sure you're not a bot",
|
|
"js_required", ("browser",),
|
|
seen="https://codeberg.org/forgejo/forgejo/src/branch/forgejo/release-notes-published/16.0.4.md"),
|
|
# Every article redirects here without the consent cookie. The browser
|
|
# accepts Sourcepoint's dialog, and the cookies it earns open the gate for
|
|
# HTTP until they expire. See `desk.tools.browser`.
|
|
Signature("golem_consent_gate", "html", r"GolemConsent\.init",
|
|
"consent_wall", ("browser",),
|
|
seen="https://www.golem.de/sonstiges/zustimmung/auswahl.html"),
|
|
)
|
|
|
|
#: Below this many words outside `<noscript>`, a page carrying a script is a
|
|
#: placeholder. OpenReview's check is 36 words, Anubis 42; a real article is
|
|
#: several hundred.
|
|
SCRIPT_PAGE_WORDS = 80
|
|
|
|
#: The longest `<noscript>` read as a notice. Mastodon's is 21 words,
|
|
#: Bluesky's 76. A longer one is a site serving its content without scripts.
|
|
NOTICE_WORDS = 150
|
|
|
|
#: Where the structural rule sends a page nobody has a signature for.
|
|
UNRECOGNISED_SHELL_NEXT = ("browser",)
|
|
|
|
|
|
def _unique_names(signatures: tuple[Signature, ...]) -> None:
|
|
names = [s.name for s in signatures]
|
|
if len(names) != len(set(names)):
|
|
raise ValueError(f"two signatures share a name: "
|
|
f"{sorted({n for n in names if names.count(n) > 1})}")
|
|
|
|
|
|
_unique_names(SIGNATURES)
|
|
|
|
|
|
def check(page: Page | None, signatures: tuple[Signature, ...] = SIGNATURES) -> Verdict | None:
|
|
"""None when the page is worth converting, else why not."""
|
|
if page is None:
|
|
# The downloader logs which way it failed and returns nothing: DNS, a
|
|
# refused connection, a reset and TLS all arrive here as one.
|
|
return Verdict("unreachable", "no_response")
|
|
if why := failures.of_status(page.status):
|
|
return Verdict(why, f"status_{page.status}")
|
|
if not page.html.strip():
|
|
return Verdict("empty", "no_html")
|
|
parts = _Parts.of(page)
|
|
for signature in signatures:
|
|
if re.search(signature.pattern, parts[signature.where], re.IGNORECASE):
|
|
return Verdict(signature.category, signature.name, signature.next)
|
|
if (parts.scripts and len(_words(parts["text"])) < SCRIPT_PAGE_WORDS
|
|
and len(_words(parts["noscript"])) <= NOTICE_WORDS):
|
|
return Verdict("js_required", "script_page", UNRECOGNISED_SHELL_NEXT)
|
|
return None
|
|
|
|
|
|
def _flat(pieces: list[str]) -> str:
|
|
return " ".join(" ".join(pieces).split())
|
|
|
|
|
|
def _words(text: str) -> list[str]:
|
|
return re.findall(r"\w+", text)
|
|
|
|
|
|
class _Parts(dict[str, str]):
|
|
"""The five things a signature can match against, read in one pass."""
|
|
|
|
scripts: int
|
|
|
|
@classmethod
|
|
def of(cls, page: Page) -> _Parts:
|
|
reader = _Reader()
|
|
try:
|
|
reader.feed(page.html)
|
|
reader.close()
|
|
except Exception: # malformed markup: match what was read so far
|
|
pass
|
|
# Whitespace collapsed in the three text parts, so a signature is
|
|
# written with single spaces however the page wrapped its lines.
|
|
parts = cls(title=_flat(reader.title), text=_flat(reader.text),
|
|
noscript=_flat(reader.noscript), html=page.html, url=page.url)
|
|
parts.scripts = reader.scripts
|
|
return parts
|
|
|
|
|
|
class _Reader(HTMLParser):
|
|
HIDDEN = frozenset({"script", "style", "template", "svg"})
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(convert_charrefs=True)
|
|
self.title: list[str] = []
|
|
self.text: list[str] = []
|
|
self.noscript: list[str] = []
|
|
self.scripts = 0
|
|
self._hidden = 0
|
|
self._in: dict[str, int] = {"title": 0, "noscript": 0}
|
|
|
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
self.scripts += tag == "script"
|
|
if tag in self.HIDDEN:
|
|
self._hidden += 1
|
|
elif tag in self._in:
|
|
self._in[tag] += 1
|
|
|
|
def handle_endtag(self, tag: str) -> None:
|
|
if tag in self.HIDDEN:
|
|
self._hidden = max(0, self._hidden - 1)
|
|
elif tag in self._in:
|
|
self._in[tag] = max(0, self._in[tag] - 1)
|
|
|
|
def handle_data(self, data: str) -> None:
|
|
if self._hidden:
|
|
return
|
|
if self._in["title"]:
|
|
self.title.append(data)
|
|
elif self._in["noscript"]:
|
|
self.noscript.append(data)
|
|
else:
|
|
self.text.append(data)
|