From bb8a9a3b8f4815865534d99cf6a640216e8402fd Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 20:07:14 +0200 Subject: [PATCH] Read robots.txt without case for a host that serves a path in any case, as Workday does Co-Authored-By: Claude Fable 5.1 --- docs/plan-hosts-and-by-hand.md | 1 + plugins/ats.py | 7 ++++++- plugins/atshosts/__init__.py | 7 ++++++- plugins/atshosts/workday.py | 3 +++ tests/test_atshosts.py | 14 ++++++++++++++ 5 files changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/plan-hosts-and-by-hand.md b/docs/plan-hosts-and-by-hand.md index 6247864..fde9de5 100644 --- a/docs/plan-hosts-and-by-hand.md +++ b/docs/plan-hosts-and-by-hand.md @@ -54,6 +54,7 @@ LABEL = "Workday" # the Sources page's select PATTERNS = (r"https?://...",) # discovery: regexes over a URL, compiled with IGNORECASE RANK = 100 # optional int, default 100; lower comes first KEEP_CASE = True # optional, default False; keep a discovered slug's case +ROBOTS_ANY_CASE = True # optional, default False; read robots.txt paths without case def slug_ok(slug: str) -> bool: ... # strict, re.fullmatch def allowed(slug: str) -> Callable[[str], bool]: ... # host names a fetch may reach diff --git a/plugins/ats.py b/plugins/ats.py index 14bfd7e..97992f8 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -593,7 +593,12 @@ def _check_robots(client: httpx.Client, target: str, except Exception as exc: raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc else: - allowed = _robots_allows(body, page) + # A host that serves a path whatever its case is matched without + # case, so a rule written `/Jobs/` forbids a site named `jobs` too. + if getattr(module, "ROBOTS_ANY_CASE", False): + allowed = _robots_allows(body.lower(), page.lower()) + else: + allowed = _robots_allows(body, page) if state is not None: state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"), "allowed": allowed}) diff --git a/plugins/atshosts/__init__.py b/plugins/atshosts/__init__.py index efe0623..f5661cd 100644 --- a/plugins/atshosts/__init__.py +++ b/plugins/atshosts/__init__.py @@ -19,6 +19,7 @@ A module is named after its host (`plugins/atshosts/workday.py`) and imports PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE RANK = 100 # optional int; lower comes first in the select and the probes KEEP_CASE = False # optional; True keeps a discovered slug's case + ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case def slug_ok(slug: str) -> bool # strict, see below def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach @@ -120,7 +121,9 @@ once a day per board (`ats:robots:`), following a redirect to the host's `www.` twin or back and no other. The board fails with "robots.txt disallows it" when the rules for `*` or for the app's own token disallow the page (RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a -final `$` in paths, case-sensitive paths). A 4xx other than 429 means no +final `$` in paths, case-sensitive paths, or without case for a host with +`ROBOTS_ANY_CASE`, which is for one that serves a path in any case, as +Workday does). A 4xx other than 429 means no rules, which allows; a 5xx, a 429, a timeout or a redirect elsewhere fails the board with "robots.txt could not be read" and is asked again next pull. `robots_page` must be on a host `allowed(slug)` admits. @@ -163,6 +166,8 @@ def problems(module: ModuleType, name: str | None = None) -> list[str]: out.append("LABEL is not a non-empty string") if not isinstance(getattr(module, "KEEP_CASE", False), bool): out.append("KEEP_CASE is not a bool") + if not isinstance(getattr(module, "ROBOTS_ANY_CASE", False), bool): + out.append("ROBOTS_ANY_CASE is not a bool") rank = getattr(module, "RANK", DEFAULT_RANK) if not isinstance(rank, int) or isinstance(rank, bool): out.append("RANK is not an int") diff --git a/plugins/atshosts/workday.py b/plugins/atshosts/workday.py index faf45c4..5aad007 100644 --- a/plugins/atshosts/workday.py +++ b/plugins/atshosts/workday.py @@ -30,6 +30,9 @@ from ._shared import Get, Page, Row, only, parse_dt NAME = "workday" LABEL = "Workday" KEEP_CASE = True +#: A tenant answers `/jobs` and `/Jobs` alike, so a rule in either case +#: forbids the site. +ROBOTS_ANY_CASE = True #: One group for `.` and one for the site, which discovery joins #: into the slug with "/". A locale segment (`/en-US/`, `/de-DE/`) may come diff --git a/tests/test_atshosts.py b/tests/test_atshosts.py index 4d9e1ee..f4443c5 100644 --- a/tests/test_atshosts.py +++ b/tests/test_atshosts.py @@ -588,6 +588,20 @@ def test_robots_that_disallow_the_page_fail_the_board( assert "ats:ok:fake:acme" not in state +def test_a_host_that_serves_any_case_is_forbidden_by_a_rule_in_any_case( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /ACME/\n", + "https://jobs.fake.example/acme/list": b'["1"]'}) + host = _fake_host(robots=True) + items, _state = _run(monkeypatch, host) + assert len(items) == 1 # paths are case-sensitive by default + host.ROBOTS_ANY_CASE = True + web["asked"].clear() + items, state = _run(monkeypatch, host) + assert items == [] + assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it" + + @pytest.mark.parametrize(("robots", "page", "allowed"), [ # The longest matching rule wins, whatever the order of the lines. ("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",