Read robots.txt without case for a host that serves a path in any case, as Workday does
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
bd1ce44c6f
commit
bb8a9a3b8f
5 files changed
+30
-2
No files matched your search
@@ -588,6 +588,20 @@ def test_robots_that_disallow_the_page_fail_the_board(
|
||||
assert "ats:ok:fake:acme" not in state
|
||||
|
||||
|
||||
def test_a_host_that_serves_any_case_is_forbidden_by_a_rule_in_any_case(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /ACME/\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
host = _fake_host(robots=True)
|
||||
items, _state = _run(monkeypatch, host)
|
||||
assert len(items) == 1 # paths are case-sensitive by default
|
||||
host.ROBOTS_ANY_CASE = True
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, host)
|
||||
assert items == []
|
||||
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("robots", "page", "allowed"), [
|
||||
# The longest matching rule wins, whatever the order of the lines.
|
||||
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",
|
||||
|
||||
Reference in new issue
Block a user