Read robots.txt without case for a host that serves a path in any case, as Workday does

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-29 20:07:14 +02:00
1 parent bd1ce44c6f
commit bb8a9a3b8f
5 files changed
+30 -2

No files matched your search

+6 -1
View File
@@ -593,7 +593,12 @@ def _check_robots(client: httpx.Client, target: str,
except Exception as exc:
raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc
else:
allowed = _robots_allows(body, page)
# A host that serves a path whatever its case is matched without
# case, so a rule written `/Jobs/` forbids a site named `jobs` too.
if getattr(module, "ROBOTS_ANY_CASE", False):
allowed = _robots_allows(body.lower(), page.lower())
else:
allowed = _robots_allows(body, page)
if state is not None:
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
"allowed": allowed})