Read robots.txt without case for a host that serves a path in any case, as Workday does

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-29 20:07:14 +02:00
1 parent bd1ce44c6f
commit bb8a9a3b8f
5 files changed
+30 -2

No files matched your search

+3
View File
@@ -30,6 +30,9 @@ from ._shared import Get, Page, Row, only, parse_dt
NAME = "workday"
LABEL = "Workday"
KEEP_CASE = True
#: A tenant answers `/jobs` and `/Jobs` alike, so a rule in either case
#: forbids the site.
ROBOTS_ANY_CASE = True
#: One group for `<tenant>.<wdN>` and one for the site, which discovery joins
#: into the slug with "/". A locale segment (`/en-US/`, `/de-DE/`) may come