Read robots.txt without case for a host that serves a path in any case, as Workday does

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-29 20:07:14 +02:00
1 parent bd1ce44c6f
commit bb8a9a3b8f
5 files changed
+29 -1

No files matched your search

+1
View File
@@ -54,6 +54,7 @@ LABEL = "Workday" # the Sources page's select
PATTERNS = (r"https?://...",) # discovery: regexes over a URL, compiled with IGNORECASE PATTERNS = (r"https?://...",) # discovery: regexes over a URL, compiled with IGNORECASE
RANK = 100 # optional int, default 100; lower comes first RANK = 100 # optional int, default 100; lower comes first
KEEP_CASE = True # optional, default False; keep a discovered slug's case KEEP_CASE = True # optional, default False; keep a discovered slug's case
ROBOTS_ANY_CASE = True # optional, default False; read robots.txt paths without case
def slug_ok(slug: str) -> bool: ... # strict, re.fullmatch def slug_ok(slug: str) -> bool: ... # strict, re.fullmatch
def allowed(slug: str) -> Callable[[str], bool]: ... # host names a fetch may reach def allowed(slug: str) -> Callable[[str], bool]: ... # host names a fetch may reach
+5
View File
@@ -592,6 +592,11 @@ def _check_robots(client: httpx.Client, target: str,
allowed = True allowed = True
except Exception as exc: except Exception as exc:
raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc
else:
# A host that serves a path whatever its case is matched without
# case, so a rule written `/Jobs/` forbids a site named `jobs` too.
if getattr(module, "ROBOTS_ANY_CASE", False):
allowed = _robots_allows(body.lower(), page.lower())
else: else:
allowed = _robots_allows(body, page) allowed = _robots_allows(body, page)
if state is not None: if state is not None:
+6 -1
View File
@@ -19,6 +19,7 @@ A module is named after its host (`plugins/atshosts/workday.py`) and imports
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
RANK = 100 # optional int; lower comes first in the select and the probes RANK = 100 # optional int; lower comes first in the select and the probes
KEEP_CASE = False # optional; True keeps a discovered slug's case KEEP_CASE = False # optional; True keeps a discovered slug's case
ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case
def slug_ok(slug: str) -> bool # strict, see below def slug_ok(slug: str) -> bool # strict, see below
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
@@ -120,7 +121,9 @@ once a day per board (`ats:robots:<target>`), following a redirect to the
host's `www.` twin or back and no other. The board fails with "robots.txt host's `www.` twin or back and no other. The board fails with "robots.txt
disallows it" when the rules for `*` or for the app's own token disallow the disallows it" when the rules for `*` or for the app's own token disallow the
page (RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a page (RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a
final `$` in paths, case-sensitive paths). A 4xx other than 429 means no final `$` in paths, case-sensitive paths, or without case for a host with
`ROBOTS_ANY_CASE`, which is for one that serves a path in any case, as
Workday does). A 4xx other than 429 means no
rules, which allows; a 5xx, a 429, a timeout or a redirect elsewhere fails rules, which allows; a 5xx, a 429, a timeout or a redirect elsewhere fails
the board with "robots.txt could not be read" and is asked again next pull. the board with "robots.txt could not be read" and is asked again next pull.
`robots_page` must be on a host `allowed(slug)` admits. `robots_page` must be on a host `allowed(slug)` admits.
@@ -163,6 +166,8 @@ def problems(module: ModuleType, name: str | None = None) -> list[str]:
out.append("LABEL is not a non-empty string") out.append("LABEL is not a non-empty string")
if not isinstance(getattr(module, "KEEP_CASE", False), bool): if not isinstance(getattr(module, "KEEP_CASE", False), bool):
out.append("KEEP_CASE is not a bool") out.append("KEEP_CASE is not a bool")
if not isinstance(getattr(module, "ROBOTS_ANY_CASE", False), bool):
out.append("ROBOTS_ANY_CASE is not a bool")
rank = getattr(module, "RANK", DEFAULT_RANK) rank = getattr(module, "RANK", DEFAULT_RANK)
if not isinstance(rank, int) or isinstance(rank, bool): if not isinstance(rank, int) or isinstance(rank, bool):
out.append("RANK is not an int") out.append("RANK is not an int")
+3
View File
@@ -30,6 +30,9 @@ from ._shared import Get, Page, Row, only, parse_dt
NAME = "workday" NAME = "workday"
LABEL = "Workday" LABEL = "Workday"
KEEP_CASE = True KEEP_CASE = True
#: A tenant answers `/jobs` and `/Jobs` alike, so a rule in either case
#: forbids the site.
ROBOTS_ANY_CASE = True
#: One group for `<tenant>.<wdN>` and one for the site, which discovery joins #: One group for `<tenant>.<wdN>` and one for the site, which discovery joins
#: into the slug with "/". A locale segment (`/en-US/`, `/de-DE/`) may come #: into the slug with "/". A locale segment (`/en-US/`, `/de-DE/`) may come
+14
View File
@@ -588,6 +588,20 @@ def test_robots_that_disallow_the_page_fail_the_board(
assert "ats:ok:fake:acme" not in state assert "ats:ok:fake:acme" not in state
def test_a_host_that_serves_any_case_is_forbidden_by_a_rule_in_any_case(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /ACME/\n",
"https://jobs.fake.example/acme/list": b'["1"]'})
host = _fake_host(robots=True)
items, _state = _run(monkeypatch, host)
assert len(items) == 1 # paths are case-sensitive by default
host.ROBOTS_ANY_CASE = True
web["asked"].clear()
items, state = _run(monkeypatch, host)
assert items == []
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
@pytest.mark.parametrize(("robots", "page", "allowed"), [ @pytest.mark.parametrize(("robots", "page", "allowed"), [
# The longest matching rule wins, whatever the order of the lines. # The longest matching rule wins, whatever the order of the lines.
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", ("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",