Match robots.txt rules by RFC 9309, the longest rule winning, with wildcards, end anchors and case-sensitive paths

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-29 19:39:11 +02:00
1 parent a9fb4397ac
commit 13ce99304b
2 files changed
+118 -7

No files matched your search

+76 -7
View File
@@ -36,7 +36,7 @@ import time
from collections.abc import Callable, Iterator, MutableMapping from collections.abc import Callable, Iterator, MutableMapping
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
from typing import Any from typing import Any
from urllib.robotparser import RobotFileParser from urllib.parse import unquote
import httpx import httpx
@@ -422,13 +422,82 @@ def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
ROBOTS_TTL = timedelta(days=1) ROBOTS_TTL = timedelta(days=1)
#: This app's product token, the name a robots.txt group addresses it by:
#: the `desk` of `desk/<version> (personal aggregator)`.
ROBOTS_TOKEN = USER_AGENT.split("/", 1)[0].strip().lower()
#: One rule of a group: `(allow, path pattern)`.
_Rule = tuple[bool, str]
def _robots_groups(text: str) -> list[tuple[set[str], list[_Rule]]]:
"""The groups of a robots.txt, per RFC 9309: one or more `user-agent`
lines (their tokens lowercased) and the `allow` and `disallow` rules that
follow them, up to the next `user-agent` line after a rule. Comments,
other keys and rules before any `user-agent` line are left out, and so is
an empty rule, which allows everything and so says nothing."""
groups: list[tuple[set[str], list[_Rule]]] = []
in_agents = False
for line in text.splitlines():
key, sep, value = line.split("#", 1)[0].partition(":")
key, value = key.strip().lower(), value.strip()
if not sep:
continue
if key == "user-agent":
if not in_agents:
groups.append((set(), []))
in_agents = True
groups[-1][0].add(value.lower())
elif key in ("allow", "disallow") and groups:
in_agents = False
if value:
groups[-1][1].append((key == "allow", value))
return groups
def _robots_rules(groups: list[tuple[set[str], list[_Rule]]], token: str) -> list[_Rule]:
"""The rules for `token`: every group that names it (matched whole and
ignoring case, as RFC 9309 says, not as a substring), merged; else every
`*` group, merged; else none, which allows everything."""
for wanted in (token, "*"):
rules = [rule for agents, group in groups if wanted in agents for rule in group]
if rules or any(wanted in agents for agents, _group in groups):
return rules
return []
def _robots_pattern(path: str) -> re.Pattern[str]:
"""A rule's path as a regex anchored at the start: `*` is any run of
characters, and a `$` at the end anchors the end."""
anchored = path.endswith("$")
body = ".*".join(re.escape(part) for part in path.rstrip("$").split("*"))
return re.compile(body + (r"\Z" if anchored else ""), re.DOTALL)
def _robots_verdict(rules: list[_Rule], path: str) -> bool:
"""Whether `path` is allowed by `rules`: the longest matching rule wins,
an allow wins a tie, and no match allows. Both sides are compared
percent-decoded, so `%7E` and `~` are one character."""
path = unquote(path)
best: tuple[int, bool] | None = None
for allow, pattern in rules:
if _robots_pattern(unquote(pattern)).match(path):
weight = (len(pattern), allow)
best = weight if best is None or weight > best else best
return best is None or best[1]
def _robots_allows(body: bytes, page: str) -> bool: def _robots_allows(body: bytes, page: str) -> bool:
"""Whether a robots.txt lets `page` be fetched: by its rules for every """Whether a robots.txt lets `page` be fetched, by RFC 9309: by its rules
agent (`*`), and by any it has for this app's own User-Agent token, so a for every agent (`*`), and by those it has for this app's own token
site that shuts out this app by name is honoured as well.""" (`ROBOTS_TOKEN`, which fall back to `*`), so a site that shuts out this
parser = RobotFileParser() app by name is honoured as well. Paths are case-sensitive; a byte order
parser.parse(body.decode("utf-8", "replace").splitlines()) mark is dropped."""
return parser.can_fetch("*", page) and parser.can_fetch(USER_AGENT, page) groups = _robots_groups(body.decode("utf-8-sig", "replace"))
url = httpx.URL(page)
path = (url.raw_path or b"/").decode("ascii", "replace")
return all(_robots_verdict(_robots_rules(groups, token), path)
for token in ("*", ROBOTS_TOKEN))
def _check_robots(client: httpx.Client, target: str, def _check_robots(client: httpx.Client, target: str,
+42
View File
@@ -364,6 +364,48 @@ def test_robots_that_disallow_the_page_fail_the_board(
assert "ats:ok:fake:acme" not in state assert "ats:ok:fake:acme" not in state
@pytest.mark.parametrize(("robots", "page", "allowed"), [
# The longest matching rule wins, whatever the order of the lines.
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",
"https://jobs.infineon.com/api/pcsx/search?domain=infineon.com", True),
("User-agent: *\nAllow: /careers\nDisallow: /\n", "https://x.example/careers/list", True),
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", "https://x.example/other", False),
# An allow wins a tie.
("User-agent: *\nDisallow: /jobs\nAllow: /jobs\n", "https://x.example/jobs", True),
# `*` inside a path, and the query counts as part of it.
("User-agent: *\nDisallow: /*company\n",
"https://career5.successfactors.eu/career?company=x", False),
# `$` anchors the end.
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json", False),
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json?page=2", True),
# An empty Disallow allows.
("User-agent: *\nDisallow:\n", "https://x.example/anything", True),
# A byte order mark and CRLF line ends.
("\ufeffUser-agent: *\r\nDisallow: /\r\n", "https://x.example/jobs", False),
# Paths are case-sensitive.
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
"https://novonesis.wd103.myworkdayjobs.com/Novonesis_Careers/", False),
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
"https://novonesis.wd103.myworkdayjobs.com/novonesis_careers/", True),
# A group for this app by name counts, a group for someone else does
# not, and a token matches whole, not as a substring.
(f"User-agent: {ats.ROBOTS_TOKEN.upper()}\nDisallow: /\n\nUser-agent: *\nAllow: /\n",
"https://x.example/jobs", False),
("User-agent: Googlebot\nDisallow: /\n", "https://x.example/jobs", True),
("User-agent: sk\nDisallow: /\n\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
# Two agents share one group, and a comment is no rule.
("User-agent: a\nUser-agent: *\nDisallow: /jobs # not these\n",
"https://x.example/jobs/1", False),
# A group for this app that allows does not undo the `*` group's refusal.
(f"User-agent: {ats.ROBOTS_TOKEN}\nAllow: /\n\nUser-agent: *\nDisallow: /\n",
"https://x.example/jobs", False),
# Rules before any user-agent line belong to no group.
("Disallow: /\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
])
def test_robots_rules_follow_rfc_9309(robots: str, page: str, allowed: bool) -> None:
assert ats._robots_allows(robots.encode(), page) is allowed
def test_a_robots_verdict_is_kept_for_a_day( def test_a_robots_verdict_is_kept_for_a_day(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n", web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",