Match robots.txt rules by RFC 9309, the longest rule winning, with wildcards, end anchors and case-sensitive paths

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-29 19:39:11 +02:00
1 parent a9fb4397ac
commit 13ce99304b
2 files changed
+118 -7

No files matched your search

+42
View File
@@ -364,6 +364,48 @@ def test_robots_that_disallow_the_page_fail_the_board(
assert "ats:ok:fake:acme" not in state
@pytest.mark.parametrize(("robots", "page", "allowed"), [
# The longest matching rule wins, whatever the order of the lines.
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",
"https://jobs.infineon.com/api/pcsx/search?domain=infineon.com", True),
("User-agent: *\nAllow: /careers\nDisallow: /\n", "https://x.example/careers/list", True),
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", "https://x.example/other", False),
# An allow wins a tie.
("User-agent: *\nDisallow: /jobs\nAllow: /jobs\n", "https://x.example/jobs", True),
# `*` inside a path, and the query counts as part of it.
("User-agent: *\nDisallow: /*company\n",
"https://career5.successfactors.eu/career?company=x", False),
# `$` anchors the end.
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json", False),
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json?page=2", True),
# An empty Disallow allows.
("User-agent: *\nDisallow:\n", "https://x.example/anything", True),
# A byte order mark and CRLF line ends.
("\ufeffUser-agent: *\r\nDisallow: /\r\n", "https://x.example/jobs", False),
# Paths are case-sensitive.
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
"https://novonesis.wd103.myworkdayjobs.com/Novonesis_Careers/", False),
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
"https://novonesis.wd103.myworkdayjobs.com/novonesis_careers/", True),
# A group for this app by name counts, a group for someone else does
# not, and a token matches whole, not as a substring.
(f"User-agent: {ats.ROBOTS_TOKEN.upper()}\nDisallow: /\n\nUser-agent: *\nAllow: /\n",
"https://x.example/jobs", False),
("User-agent: Googlebot\nDisallow: /\n", "https://x.example/jobs", True),
("User-agent: sk\nDisallow: /\n\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
# Two agents share one group, and a comment is no rule.
("User-agent: a\nUser-agent: *\nDisallow: /jobs # not these\n",
"https://x.example/jobs/1", False),
# A group for this app that allows does not undo the `*` group's refusal.
(f"User-agent: {ats.ROBOTS_TOKEN}\nAllow: /\n\nUser-agent: *\nDisallow: /\n",
"https://x.example/jobs", False),
# Rules before any user-agent line belong to no group.
("Disallow: /\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
])
def test_robots_rules_follow_rfc_9309(robots: str, page: str, allowed: bool) -> None:
assert ats._robots_allows(robots.encode(), page) is allowed
def test_a_robots_verdict_is_kept_for_a_day(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",