Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d666d0937
commit
248ea1636d
36 files changed
+8393
No files matched your search
@@ -0,0 +1,227 @@
|
||||
"""Did the getter bring back a page worth converting, and if not, what next?
|
||||
|
||||
`scrape` fetches, this judges, `article` converts. A 200 is not an article
|
||||
because it answered 200: an app shell serves a `<noscript>` notice asking for
|
||||
JavaScript, a bot check serves a few words and a script. Converting either
|
||||
stores the notice as the article.
|
||||
|
||||
A check answers with a `Verdict`: the failure category from
|
||||
`desk.tools.failures`, the name of what matched, and which getters are worth
|
||||
trying next. That last part is where what we learn goes. A Mastodon post is
|
||||
a shell over HTTP and a status over Mastodon's API, so its signature names
|
||||
`mastodon_api`; a bot check names `browser`. A getter the app does not have is
|
||||
skipped, so a signature can name one before it is built.
|
||||
|
||||
Checks run in order and the first match decides:
|
||||
|
||||
1. the status code
|
||||
2. no HTML at all
|
||||
3. `SIGNATURES`, the table below, most specific first
|
||||
4. the structural rule, for a shell nobody has written a signature for yet
|
||||
|
||||
**Adding a signature** is one entry in `SIGNATURES` and one test page in
|
||||
`tests/test_checks.py`. The entry names the page it was found on, so whoever
|
||||
reads it later can fetch that page and see whether it still says this.
|
||||
|
||||
Signature(
|
||||
name="mastodon_shell", # logged, and on the attempt
|
||||
where="noscript", # title | text | noscript | html | url
|
||||
pattern=r"To use the Mastodon web application", # regex, case-insensitive
|
||||
category="js_required", # from desk.tools.failures
|
||||
next=("mastodon_api", "browser"), # getters to try, in order
|
||||
seen="https://grapheneos.social/@GrapheneOS/117225539756835649",
|
||||
)
|
||||
|
||||
`where` is what the pattern is matched against: `title` the `<title>`, `text`
|
||||
the words a reader without JavaScript sees outside `<noscript>`, `noscript`
|
||||
the words inside it, `html` the raw markup (script sources, class names),
|
||||
`url` the address the getter ended up at after redirects. Prefer the narrowest
|
||||
one that works: a phrase in `text` can appear in an article about the thing
|
||||
it describes, a class name in `html` usually cannot.
|
||||
|
||||
A pattern that does not compile, a category the failure vocabulary does not
|
||||
have, or two signatures with one name is refused at import.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from html.parser import HTMLParser
|
||||
from typing import Literal
|
||||
|
||||
from desk.tools import failures
|
||||
|
||||
Where = Literal["title", "text", "noscript", "html", "url"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Page:
|
||||
"""What one getter brought back. `url` is where it ended up."""
|
||||
|
||||
status: int
|
||||
html: str
|
||||
url: str = ""
|
||||
headers: Mapping[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Verdict:
|
||||
"""Why this page is not worth converting, and what might get the real one."""
|
||||
|
||||
category: str
|
||||
#: The signature or rule that matched. What a person reads in the log to
|
||||
#: decide whether the table needs a new entry or a narrower one.
|
||||
name: str
|
||||
next: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Signature:
|
||||
name: str
|
||||
where: Where
|
||||
pattern: str
|
||||
category: str
|
||||
next: tuple[str, ...] = ()
|
||||
#: A page this was found on.
|
||||
seen: str = ""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
re.compile(self.pattern)
|
||||
if self.category not in failures.TRANSIENT | failures.SETTLED:
|
||||
raise ValueError(f"signature {self.name!r} names category "
|
||||
f"{self.category!r}, which desk.tools.failures has not")
|
||||
|
||||
|
||||
#: Most specific first. See the module docstring for the format.
|
||||
SIGNATURES: tuple[Signature, ...] = (
|
||||
Signature("mastodon_shell", "noscript", r"To use the Mastodon web application",
|
||||
"js_required", ("mastodon_api", "browser"),
|
||||
seen="https://grapheneos.social/@GrapheneOS/117225539756835649"),
|
||||
Signature("bluesky_shell", "noscript", r"This is a heavily interactive web application, and JavaScript is required",
|
||||
"js_required", ("bluesky_api", "browser"),
|
||||
seen="https://bsky.app/profile/bsky.app/post/3mv3shqdfuc2e"),
|
||||
Signature("create_react_app_shell", "noscript", r"You need to enable JavaScript to run this app",
|
||||
"js_required", ("browser",),
|
||||
seen="https://rune.build/blog/rune-is-now-open-source"),
|
||||
Signature("cloudflare_turnstile", "html", r"challenges\.cloudflare\.com/turnstile",
|
||||
"js_required", ("browser",),
|
||||
seen="https://openreview.net/forum?id=pc7fqaOcAH"),
|
||||
Signature("anubis_check", "text", r"automated check to get rid of most bots|Making sure you're not a bot",
|
||||
"js_required", ("browser",),
|
||||
seen="https://codeberg.org/forgejo/forgejo/src/branch/forgejo/release-notes-published/16.0.4.md"),
|
||||
# Every article redirects here without the consent cookie. The browser
|
||||
# accepts Sourcepoint's dialog, and the cookies it earns open the gate for
|
||||
# HTTP until they expire. See `desk.tools.browser`.
|
||||
Signature("golem_consent_gate", "html", r"GolemConsent\.init",
|
||||
"consent_wall", ("browser",),
|
||||
seen="https://www.golem.de/sonstiges/zustimmung/auswahl.html"),
|
||||
)
|
||||
|
||||
#: Below this many words outside `<noscript>`, a page carrying a script is a
|
||||
#: placeholder. OpenReview's check is 36 words, Anubis 42; a real article is
|
||||
#: several hundred.
|
||||
SCRIPT_PAGE_WORDS = 80
|
||||
|
||||
#: The longest `<noscript>` read as a notice. Mastodon's is 21 words,
|
||||
#: Bluesky's 76. A longer one is a site serving its content without scripts.
|
||||
NOTICE_WORDS = 150
|
||||
|
||||
#: Where the structural rule sends a page nobody has a signature for.
|
||||
UNRECOGNISED_SHELL_NEXT = ("browser",)
|
||||
|
||||
|
||||
def _unique_names(signatures: tuple[Signature, ...]) -> None:
|
||||
names = [s.name for s in signatures]
|
||||
if len(names) != len(set(names)):
|
||||
raise ValueError(f"two signatures share a name: "
|
||||
f"{sorted({n for n in names if names.count(n) > 1})}")
|
||||
|
||||
|
||||
_unique_names(SIGNATURES)
|
||||
|
||||
|
||||
def check(page: Page | None, signatures: tuple[Signature, ...] = SIGNATURES) -> Verdict | None:
|
||||
"""None when the page is worth converting, else why not."""
|
||||
if page is None:
|
||||
# The downloader logs which way it failed and returns nothing: DNS, a
|
||||
# refused connection, a reset and TLS all arrive here as one.
|
||||
return Verdict("unreachable", "no_response")
|
||||
if why := failures.of_status(page.status):
|
||||
return Verdict(why, f"status_{page.status}")
|
||||
if not page.html.strip():
|
||||
return Verdict("empty", "no_html")
|
||||
parts = _Parts.of(page)
|
||||
for signature in signatures:
|
||||
if re.search(signature.pattern, parts[signature.where], re.IGNORECASE):
|
||||
return Verdict(signature.category, signature.name, signature.next)
|
||||
if (parts.scripts and len(_words(parts["text"])) < SCRIPT_PAGE_WORDS
|
||||
and len(_words(parts["noscript"])) <= NOTICE_WORDS):
|
||||
return Verdict("js_required", "script_page", UNRECOGNISED_SHELL_NEXT)
|
||||
return None
|
||||
|
||||
|
||||
def _flat(pieces: list[str]) -> str:
|
||||
return " ".join(" ".join(pieces).split())
|
||||
|
||||
|
||||
def _words(text: str) -> list[str]:
|
||||
return re.findall(r"\w+", text)
|
||||
|
||||
|
||||
class _Parts(dict[str, str]):
|
||||
"""The five things a signature can match against, read in one pass."""
|
||||
|
||||
scripts: int
|
||||
|
||||
@classmethod
|
||||
def of(cls, page: Page) -> _Parts:
|
||||
reader = _Reader()
|
||||
try:
|
||||
reader.feed(page.html)
|
||||
reader.close()
|
||||
except Exception: # malformed markup: match what was read so far
|
||||
pass
|
||||
# Whitespace collapsed in the three text parts, so a signature is
|
||||
# written with single spaces however the page wrapped its lines.
|
||||
parts = cls(title=_flat(reader.title), text=_flat(reader.text),
|
||||
noscript=_flat(reader.noscript), html=page.html, url=page.url)
|
||||
parts.scripts = reader.scripts
|
||||
return parts
|
||||
|
||||
|
||||
class _Reader(HTMLParser):
|
||||
HIDDEN = frozenset({"script", "style", "template", "svg"})
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.title: list[str] = []
|
||||
self.text: list[str] = []
|
||||
self.noscript: list[str] = []
|
||||
self.scripts = 0
|
||||
self._hidden = 0
|
||||
self._in: dict[str, int] = {"title": 0, "noscript": 0}
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
self.scripts += tag == "script"
|
||||
if tag in self.HIDDEN:
|
||||
self._hidden += 1
|
||||
elif tag in self._in:
|
||||
self._in[tag] += 1
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag in self.HIDDEN:
|
||||
self._hidden = max(0, self._hidden - 1)
|
||||
elif tag in self._in:
|
||||
self._in[tag] = max(0, self._in[tag] - 1)
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._hidden:
|
||||
return
|
||||
if self._in["title"]:
|
||||
self.title.append(data)
|
||||
elif self._in["noscript"]:
|
||||
self.noscript.append(data)
|
||||
else:
|
||||
self.text.append(data)
|
||||
Reference in new issue
Block a user