Read robots.txt without case for a host that serves a path in any case, as Workday does
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
bd1ce44c6f
commit
bb8a9a3b8f
5 files changed
+30
-2
No files matched your search
@@ -30,6 +30,9 @@ from ._shared import Get, Page, Row, only, parse_dt
|
||||
NAME = "workday"
|
||||
LABEL = "Workday"
|
||||
KEEP_CASE = True
|
||||
#: A tenant answers `/jobs` and `/Jobs` alike, so a rule in either case
|
||||
#: forbids the site.
|
||||
ROBOTS_ANY_CASE = True
|
||||
|
||||
#: One group for `<tenant>.<wdN>` and one for the site, which discovery joins
|
||||
#: into the slug with "/". A locale segment (`/en-US/`, `/de-DE/`) may come
|
||||
|
||||
Reference in new issue
Block a user