Read robots.txt without case for a host that serves a path in any case, as Workday does

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-29 20:07:14 +02:00
1 parent bd1ce44c6f
commit bb8a9a3b8f
5 files changed
+30 -2

No files matched your search

+1
View File
@@ -54,6 +54,7 @@ LABEL = "Workday" # the Sources page's select
PATTERNS = (r"https?://...",) # discovery: regexes over a URL, compiled with IGNORECASE
RANK = 100 # optional int, default 100; lower comes first
KEEP_CASE = True # optional, default False; keep a discovered slug's case
ROBOTS_ANY_CASE = True # optional, default False; read robots.txt paths without case
def slug_ok(slug: str) -> bool: ... # strict, re.fullmatch
def allowed(slug: str) -> Callable[[str], bool]: ... # host names a fetch may reach