Match robots.txt rules by RFC 9309, the longest rule winning, with wildcards, end anchors and case-sensitive paths
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a9fb4397ac
commit
13ce99304b
2 files changed
+118
-7
No files matched your search
+76
-7
@@ -36,7 +36,7 @@ import time
|
||||
from collections.abc import Callable, Iterator, MutableMapping
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from urllib.robotparser import RobotFileParser
|
||||
from urllib.parse import unquote
|
||||
|
||||
import httpx
|
||||
|
||||
@@ -422,13 +422,82 @@ def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
ROBOTS_TTL = timedelta(days=1)
|
||||
|
||||
|
||||
#: This app's product token, the name a robots.txt group addresses it by:
|
||||
#: the `desk` of `desk/<version> (personal aggregator)`.
|
||||
ROBOTS_TOKEN = USER_AGENT.split("/", 1)[0].strip().lower()
|
||||
|
||||
#: One rule of a group: `(allow, path pattern)`.
|
||||
_Rule = tuple[bool, str]
|
||||
|
||||
|
||||
def _robots_groups(text: str) -> list[tuple[set[str], list[_Rule]]]:
|
||||
"""The groups of a robots.txt, per RFC 9309: one or more `user-agent`
|
||||
lines (their tokens lowercased) and the `allow` and `disallow` rules that
|
||||
follow them, up to the next `user-agent` line after a rule. Comments,
|
||||
other keys and rules before any `user-agent` line are left out, and so is
|
||||
an empty rule, which allows everything and so says nothing."""
|
||||
groups: list[tuple[set[str], list[_Rule]]] = []
|
||||
in_agents = False
|
||||
for line in text.splitlines():
|
||||
key, sep, value = line.split("#", 1)[0].partition(":")
|
||||
key, value = key.strip().lower(), value.strip()
|
||||
if not sep:
|
||||
continue
|
||||
if key == "user-agent":
|
||||
if not in_agents:
|
||||
groups.append((set(), []))
|
||||
in_agents = True
|
||||
groups[-1][0].add(value.lower())
|
||||
elif key in ("allow", "disallow") and groups:
|
||||
in_agents = False
|
||||
if value:
|
||||
groups[-1][1].append((key == "allow", value))
|
||||
return groups
|
||||
|
||||
|
||||
def _robots_rules(groups: list[tuple[set[str], list[_Rule]]], token: str) -> list[_Rule]:
|
||||
"""The rules for `token`: every group that names it (matched whole and
|
||||
ignoring case, as RFC 9309 says, not as a substring), merged; else every
|
||||
`*` group, merged; else none, which allows everything."""
|
||||
for wanted in (token, "*"):
|
||||
rules = [rule for agents, group in groups if wanted in agents for rule in group]
|
||||
if rules or any(wanted in agents for agents, _group in groups):
|
||||
return rules
|
||||
return []
|
||||
|
||||
|
||||
def _robots_pattern(path: str) -> re.Pattern[str]:
|
||||
"""A rule's path as a regex anchored at the start: `*` is any run of
|
||||
characters, and a `$` at the end anchors the end."""
|
||||
anchored = path.endswith("$")
|
||||
body = ".*".join(re.escape(part) for part in path.rstrip("$").split("*"))
|
||||
return re.compile(body + (r"\Z" if anchored else ""), re.DOTALL)
|
||||
|
||||
|
||||
def _robots_verdict(rules: list[_Rule], path: str) -> bool:
|
||||
"""Whether `path` is allowed by `rules`: the longest matching rule wins,
|
||||
an allow wins a tie, and no match allows. Both sides are compared
|
||||
percent-decoded, so `%7E` and `~` are one character."""
|
||||
path = unquote(path)
|
||||
best: tuple[int, bool] | None = None
|
||||
for allow, pattern in rules:
|
||||
if _robots_pattern(unquote(pattern)).match(path):
|
||||
weight = (len(pattern), allow)
|
||||
best = weight if best is None or weight > best else best
|
||||
return best is None or best[1]
|
||||
|
||||
|
||||
def _robots_allows(body: bytes, page: str) -> bool:
|
||||
"""Whether a robots.txt lets `page` be fetched: by its rules for every
|
||||
agent (`*`), and by any it has for this app's own User-Agent token, so a
|
||||
site that shuts out this app by name is honoured as well."""
|
||||
parser = RobotFileParser()
|
||||
parser.parse(body.decode("utf-8", "replace").splitlines())
|
||||
return parser.can_fetch("*", page) and parser.can_fetch(USER_AGENT, page)
|
||||
"""Whether a robots.txt lets `page` be fetched, by RFC 9309: by its rules
|
||||
for every agent (`*`), and by those it has for this app's own token
|
||||
(`ROBOTS_TOKEN`, which fall back to `*`), so a site that shuts out this
|
||||
app by name is honoured as well. Paths are case-sensitive; a byte order
|
||||
mark is dropped."""
|
||||
groups = _robots_groups(body.decode("utf-8-sig", "replace"))
|
||||
url = httpx.URL(page)
|
||||
path = (url.raw_path or b"/").decode("ascii", "replace")
|
||||
return all(_robots_verdict(_robots_rules(groups, token), path)
|
||||
for token in ("*", ROBOTS_TOKEN))
|
||||
|
||||
|
||||
def _check_robots(client: httpx.Client, target: str,
|
||||
|
||||
@@ -364,6 +364,48 @@ def test_robots_that_disallow_the_page_fail_the_board(
|
||||
assert "ats:ok:fake:acme" not in state
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("robots", "page", "allowed"), [
|
||||
# The longest matching rule wins, whatever the order of the lines.
|
||||
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",
|
||||
"https://jobs.infineon.com/api/pcsx/search?domain=infineon.com", True),
|
||||
("User-agent: *\nAllow: /careers\nDisallow: /\n", "https://x.example/careers/list", True),
|
||||
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", "https://x.example/other", False),
|
||||
# An allow wins a tie.
|
||||
("User-agent: *\nDisallow: /jobs\nAllow: /jobs\n", "https://x.example/jobs", True),
|
||||
# `*` inside a path, and the query counts as part of it.
|
||||
("User-agent: *\nDisallow: /*company\n",
|
||||
"https://career5.successfactors.eu/career?company=x", False),
|
||||
# `$` anchors the end.
|
||||
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json", False),
|
||||
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json?page=2", True),
|
||||
# An empty Disallow allows.
|
||||
("User-agent: *\nDisallow:\n", "https://x.example/anything", True),
|
||||
# A byte order mark and CRLF line ends.
|
||||
("\ufeffUser-agent: *\r\nDisallow: /\r\n", "https://x.example/jobs", False),
|
||||
# Paths are case-sensitive.
|
||||
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
|
||||
"https://novonesis.wd103.myworkdayjobs.com/Novonesis_Careers/", False),
|
||||
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
|
||||
"https://novonesis.wd103.myworkdayjobs.com/novonesis_careers/", True),
|
||||
# A group for this app by name counts, a group for someone else does
|
||||
# not, and a token matches whole, not as a substring.
|
||||
(f"User-agent: {ats.ROBOTS_TOKEN.upper()}\nDisallow: /\n\nUser-agent: *\nAllow: /\n",
|
||||
"https://x.example/jobs", False),
|
||||
("User-agent: Googlebot\nDisallow: /\n", "https://x.example/jobs", True),
|
||||
("User-agent: sk\nDisallow: /\n\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
|
||||
# Two agents share one group, and a comment is no rule.
|
||||
("User-agent: a\nUser-agent: *\nDisallow: /jobs # not these\n",
|
||||
"https://x.example/jobs/1", False),
|
||||
# A group for this app that allows does not undo the `*` group's refusal.
|
||||
(f"User-agent: {ats.ROBOTS_TOKEN}\nAllow: /\n\nUser-agent: *\nDisallow: /\n",
|
||||
"https://x.example/jobs", False),
|
||||
# Rules before any user-agent line belong to no group.
|
||||
("Disallow: /\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
|
||||
])
|
||||
def test_robots_rules_follow_rfc_9309(robots: str, page: str, allowed: bool) -> None:
|
||||
assert ats._robots_allows(robots.encode(), page) is allowed
|
||||
|
||||
|
||||
def test_a_robots_verdict_is_kept_for_a_day(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
|
||||
|
||||
Reference in new issue
Block a user