Read robots.txt without case for a host that serves a path in any case, as Workday does
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
bd1ce44c6f
commit
bb8a9a3b8f
5 files changed
+30
-2
No files matched your search
@@ -54,6 +54,7 @@ LABEL = "Workday" # the Sources page's select
|
||||
PATTERNS = (r"https?://...",) # discovery: regexes over a URL, compiled with IGNORECASE
|
||||
RANK = 100 # optional int, default 100; lower comes first
|
||||
KEEP_CASE = True # optional, default False; keep a discovered slug's case
|
||||
ROBOTS_ANY_CASE = True # optional, default False; read robots.txt paths without case
|
||||
|
||||
def slug_ok(slug: str) -> bool: ... # strict, re.fullmatch
|
||||
def allowed(slug: str) -> Callable[[str], bool]: ... # host names a fetch may reach
|
||||
|
||||
Reference in new issue
Block a user