Read robots.txt without case for a host that serves a path in any case, as Workday does

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-29 20:07:14 +02:00
1 parent bd1ce44c6f
commit bb8a9a3b8f
5 files changed
+30 -2

No files matched your search

+14
View File
@@ -588,6 +588,20 @@ def test_robots_that_disallow_the_page_fail_the_board(
assert "ats:ok:fake:acme" not in state
def test_a_host_that_serves_any_case_is_forbidden_by_a_rule_in_any_case(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /ACME/\n",
"https://jobs.fake.example/acme/list": b'["1"]'})
host = _fake_host(robots=True)
items, _state = _run(monkeypatch, host)
assert len(items) == 1 # paths are case-sensitive by default
host.ROBOTS_ANY_CASE = True
web["asked"].clear()
items, state = _run(monkeypatch, host)
assert items == []
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
@pytest.mark.parametrize(("robots", "page", "allowed"), [
# The longest matching rule wins, whatever the order of the lines.
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",