Match robots.txt rules by RFC 9309, the longest rule winning, with wildcards, end anchors and case-sensitive paths

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-29 19:39:11 +02:00
1 parent a9fb4397ac
commit 13ce99304b
2 files changed
+118 -7

No files matched your search

+76 -7
View File
@@ -36,7 +36,7 @@ import time
from collections.abc import Callable, Iterator, MutableMapping
from datetime import UTC, datetime, timedelta
from typing import Any
from urllib.robotparser import RobotFileParser
from urllib.parse import unquote
import httpx
@@ -422,13 +422,82 @@ def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
ROBOTS_TTL = timedelta(days=1)
#: This app's product token, the name a robots.txt group addresses it by:
#: the `desk` of `desk/<version> (personal aggregator)`.
ROBOTS_TOKEN = USER_AGENT.split("/", 1)[0].strip().lower()
#: One rule of a group: `(allow, path pattern)`.
_Rule = tuple[bool, str]
def _robots_groups(text: str) -> list[tuple[set[str], list[_Rule]]]:
"""The groups of a robots.txt, per RFC 9309: one or more `user-agent`
lines (their tokens lowercased) and the `allow` and `disallow` rules that
follow them, up to the next `user-agent` line after a rule. Comments,
other keys and rules before any `user-agent` line are left out, and so is
an empty rule, which allows everything and so says nothing."""
groups: list[tuple[set[str], list[_Rule]]] = []
in_agents = False
for line in text.splitlines():
key, sep, value = line.split("#", 1)[0].partition(":")
key, value = key.strip().lower(), value.strip()
if not sep:
continue
if key == "user-agent":
if not in_agents:
groups.append((set(), []))
in_agents = True
groups[-1][0].add(value.lower())
elif key in ("allow", "disallow") and groups:
in_agents = False
if value:
groups[-1][1].append((key == "allow", value))
return groups
def _robots_rules(groups: list[tuple[set[str], list[_Rule]]], token: str) -> list[_Rule]:
"""The rules for `token`: every group that names it (matched whole and
ignoring case, as RFC 9309 says, not as a substring), merged; else every
`*` group, merged; else none, which allows everything."""
for wanted in (token, "*"):
rules = [rule for agents, group in groups if wanted in agents for rule in group]
if rules or any(wanted in agents for agents, _group in groups):
return rules
return []
def _robots_pattern(path: str) -> re.Pattern[str]:
"""A rule's path as a regex anchored at the start: `*` is any run of
characters, and a `$` at the end anchors the end."""
anchored = path.endswith("$")
body = ".*".join(re.escape(part) for part in path.rstrip("$").split("*"))
return re.compile(body + (r"\Z" if anchored else ""), re.DOTALL)
def _robots_verdict(rules: list[_Rule], path: str) -> bool:
"""Whether `path` is allowed by `rules`: the longest matching rule wins,
an allow wins a tie, and no match allows. Both sides are compared
percent-decoded, so `%7E` and `~` are one character."""
path = unquote(path)
best: tuple[int, bool] | None = None
for allow, pattern in rules:
if _robots_pattern(unquote(pattern)).match(path):
weight = (len(pattern), allow)
best = weight if best is None or weight > best else best
return best is None or best[1]
def _robots_allows(body: bytes, page: str) -> bool:
"""Whether a robots.txt lets `page` be fetched: by its rules for every
agent (`*`), and by any it has for this app's own User-Agent token, so a
site that shuts out this app by name is honoured as well."""
parser = RobotFileParser()
parser.parse(body.decode("utf-8", "replace").splitlines())
return parser.can_fetch("*", page) and parser.can_fetch(USER_AGENT, page)
"""Whether a robots.txt lets `page` be fetched, by RFC 9309: by its rules
for every agent (`*`), and by those it has for this app's own token
(`ROBOTS_TOKEN`, which fall back to `*`), so a site that shuts out this
app by name is honoured as well. Paths are case-sensitive; a byte order
mark is dropped."""
groups = _robots_groups(body.decode("utf-8-sig", "replace"))
url = httpx.URL(page)
path = (url.raw_path or b"/").decode("ascii", "replace")
return all(_robots_verdict(_robots_rules(groups, token), path)
for token in ("*", ROBOTS_TOKEN))
def _check_robots(client: httpx.Client, target: str,
+42
View File
@@ -364,6 +364,48 @@ def test_robots_that_disallow_the_page_fail_the_board(
assert "ats:ok:fake:acme" not in state
@pytest.mark.parametrize(("robots", "page", "allowed"), [
# The longest matching rule wins, whatever the order of the lines.
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",
"https://jobs.infineon.com/api/pcsx/search?domain=infineon.com", True),
("User-agent: *\nAllow: /careers\nDisallow: /\n", "https://x.example/careers/list", True),
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", "https://x.example/other", False),
# An allow wins a tie.
("User-agent: *\nDisallow: /jobs\nAllow: /jobs\n", "https://x.example/jobs", True),
# `*` inside a path, and the query counts as part of it.
("User-agent: *\nDisallow: /*company\n",
"https://career5.successfactors.eu/career?company=x", False),
# `$` anchors the end.
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json", False),
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json?page=2", True),
# An empty Disallow allows.
("User-agent: *\nDisallow:\n", "https://x.example/anything", True),
# A byte order mark and CRLF line ends.
("\ufeffUser-agent: *\r\nDisallow: /\r\n", "https://x.example/jobs", False),
# Paths are case-sensitive.
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
"https://novonesis.wd103.myworkdayjobs.com/Novonesis_Careers/", False),
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
"https://novonesis.wd103.myworkdayjobs.com/novonesis_careers/", True),
# A group for this app by name counts, a group for someone else does
# not, and a token matches whole, not as a substring.
(f"User-agent: {ats.ROBOTS_TOKEN.upper()}\nDisallow: /\n\nUser-agent: *\nAllow: /\n",
"https://x.example/jobs", False),
("User-agent: Googlebot\nDisallow: /\n", "https://x.example/jobs", True),
("User-agent: sk\nDisallow: /\n\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
# Two agents share one group, and a comment is no rule.
("User-agent: a\nUser-agent: *\nDisallow: /jobs # not these\n",
"https://x.example/jobs/1", False),
# A group for this app that allows does not undo the `*` group's refusal.
(f"User-agent: {ats.ROBOTS_TOKEN}\nAllow: /\n\nUser-agent: *\nDisallow: /\n",
"https://x.example/jobs", False),
# Rules before any user-agent line belong to no group.
("Disallow: /\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
])
def test_robots_rules_follow_rfc_9309(robots: str, page: str, allowed: bool) -> None:
assert ats._robots_allows(robots.encode(), page) is allowed
def test_a_robots_verdict_is_kept_for_a_day(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",