Match robots.txt rules by RFC 9309, the longest rule winning, with wildcards, end anchors and case-sensitive paths
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a9fb4397ac
commit
13ce99304b
2 files changed
+118
-7
No files matched your search
+76
-7
@@ -36,7 +36,7 @@ import time
|
||||
from collections.abc import Callable, Iterator, MutableMapping
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from urllib.robotparser import RobotFileParser
|
||||
from urllib.parse import unquote
|
||||
|
||||
import httpx
|
||||
|
||||
@@ -422,13 +422,82 @@ def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
ROBOTS_TTL = timedelta(days=1)
|
||||
|
||||
|
||||
#: This app's product token, the name a robots.txt group addresses it by:
|
||||
#: the `desk` of `desk/<version> (personal aggregator)`.
|
||||
ROBOTS_TOKEN = USER_AGENT.split("/", 1)[0].strip().lower()
|
||||
|
||||
#: One rule of a group: `(allow, path pattern)`.
|
||||
_Rule = tuple[bool, str]
|
||||
|
||||
|
||||
def _robots_groups(text: str) -> list[tuple[set[str], list[_Rule]]]:
|
||||
"""The groups of a robots.txt, per RFC 9309: one or more `user-agent`
|
||||
lines (their tokens lowercased) and the `allow` and `disallow` rules that
|
||||
follow them, up to the next `user-agent` line after a rule. Comments,
|
||||
other keys and rules before any `user-agent` line are left out, and so is
|
||||
an empty rule, which allows everything and so says nothing."""
|
||||
groups: list[tuple[set[str], list[_Rule]]] = []
|
||||
in_agents = False
|
||||
for line in text.splitlines():
|
||||
key, sep, value = line.split("#", 1)[0].partition(":")
|
||||
key, value = key.strip().lower(), value.strip()
|
||||
if not sep:
|
||||
continue
|
||||
if key == "user-agent":
|
||||
if not in_agents:
|
||||
groups.append((set(), []))
|
||||
in_agents = True
|
||||
groups[-1][0].add(value.lower())
|
||||
elif key in ("allow", "disallow") and groups:
|
||||
in_agents = False
|
||||
if value:
|
||||
groups[-1][1].append((key == "allow", value))
|
||||
return groups
|
||||
|
||||
|
||||
def _robots_rules(groups: list[tuple[set[str], list[_Rule]]], token: str) -> list[_Rule]:
|
||||
"""The rules for `token`: every group that names it (matched whole and
|
||||
ignoring case, as RFC 9309 says, not as a substring), merged; else every
|
||||
`*` group, merged; else none, which allows everything."""
|
||||
for wanted in (token, "*"):
|
||||
rules = [rule for agents, group in groups if wanted in agents for rule in group]
|
||||
if rules or any(wanted in agents for agents, _group in groups):
|
||||
return rules
|
||||
return []
|
||||
|
||||
|
||||
def _robots_pattern(path: str) -> re.Pattern[str]:
|
||||
"""A rule's path as a regex anchored at the start: `*` is any run of
|
||||
characters, and a `$` at the end anchors the end."""
|
||||
anchored = path.endswith("$")
|
||||
body = ".*".join(re.escape(part) for part in path.rstrip("$").split("*"))
|
||||
return re.compile(body + (r"\Z" if anchored else ""), re.DOTALL)
|
||||
|
||||
|
||||
def _robots_verdict(rules: list[_Rule], path: str) -> bool:
|
||||
"""Whether `path` is allowed by `rules`: the longest matching rule wins,
|
||||
an allow wins a tie, and no match allows. Both sides are compared
|
||||
percent-decoded, so `%7E` and `~` are one character."""
|
||||
path = unquote(path)
|
||||
best: tuple[int, bool] | None = None
|
||||
for allow, pattern in rules:
|
||||
if _robots_pattern(unquote(pattern)).match(path):
|
||||
weight = (len(pattern), allow)
|
||||
best = weight if best is None or weight > best else best
|
||||
return best is None or best[1]
|
||||
|
||||
|
||||
def _robots_allows(body: bytes, page: str) -> bool:
|
||||
"""Whether a robots.txt lets `page` be fetched: by its rules for every
|
||||
agent (`*`), and by any it has for this app's own User-Agent token, so a
|
||||
site that shuts out this app by name is honoured as well."""
|
||||
parser = RobotFileParser()
|
||||
parser.parse(body.decode("utf-8", "replace").splitlines())
|
||||
return parser.can_fetch("*", page) and parser.can_fetch(USER_AGENT, page)
|
||||
"""Whether a robots.txt lets `page` be fetched, by RFC 9309: by its rules
|
||||
for every agent (`*`), and by those it has for this app's own token
|
||||
(`ROBOTS_TOKEN`, which fall back to `*`), so a site that shuts out this
|
||||
app by name is honoured as well. Paths are case-sensitive; a byte order
|
||||
mark is dropped."""
|
||||
groups = _robots_groups(body.decode("utf-8-sig", "replace"))
|
||||
url = httpx.URL(page)
|
||||
path = (url.raw_path or b"/").decode("ascii", "replace")
|
||||
return all(_robots_verdict(_robots_rules(groups, token), path)
|
||||
for token in ("*", ROBOTS_TOKEN))
|
||||
|
||||
|
||||
def _check_robots(client: httpx.Client, target: str,
|
||||
|
||||
Reference in new issue
Block a user