Read robots.txt without case for a host that serves a path in any case, as Workday does
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
bd1ce44c6f
commit
bb8a9a3b8f
5 files changed
+30
-2
No files matched your search
+6
-1
@@ -593,7 +593,12 @@ def _check_robots(client: httpx.Client, target: str,
|
||||
except Exception as exc:
|
||||
raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc
|
||||
else:
|
||||
allowed = _robots_allows(body, page)
|
||||
# A host that serves a path whatever its case is matched without
|
||||
# case, so a rule written `/Jobs/` forbids a site named `jobs` too.
|
||||
if getattr(module, "ROBOTS_ANY_CASE", False):
|
||||
allowed = _robots_allows(body.lower(), page.lower())
|
||||
else:
|
||||
allowed = _robots_allows(body, page)
|
||||
if state is not None:
|
||||
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
|
||||
"allowed": allowed})
|
||||
|
||||
Reference in new issue
Block a user