Read robots.txt without case for a host that serves a path in any case, as Workday does
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
bd1ce44c6f
commit
bb8a9a3b8f
5 files changed
+30
-2
No files matched your search
@@ -54,6 +54,7 @@ LABEL = "Workday" # the Sources page's select
|
||||
PATTERNS = (r"https?://...",) # discovery: regexes over a URL, compiled with IGNORECASE
|
||||
RANK = 100 # optional int, default 100; lower comes first
|
||||
KEEP_CASE = True # optional, default False; keep a discovered slug's case
|
||||
ROBOTS_ANY_CASE = True # optional, default False; read robots.txt paths without case
|
||||
|
||||
def slug_ok(slug: str) -> bool: ... # strict, re.fullmatch
|
||||
def allowed(slug: str) -> Callable[[str], bool]: ... # host names a fetch may reach
|
||||
|
||||
+6
-1
@@ -593,7 +593,12 @@ def _check_robots(client: httpx.Client, target: str,
|
||||
except Exception as exc:
|
||||
raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc
|
||||
else:
|
||||
allowed = _robots_allows(body, page)
|
||||
# A host that serves a path whatever its case is matched without
|
||||
# case, so a rule written `/Jobs/` forbids a site named `jobs` too.
|
||||
if getattr(module, "ROBOTS_ANY_CASE", False):
|
||||
allowed = _robots_allows(body.lower(), page.lower())
|
||||
else:
|
||||
allowed = _robots_allows(body, page)
|
||||
if state is not None:
|
||||
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
|
||||
"allowed": allowed})
|
||||
|
||||
@@ -19,6 +19,7 @@ A module is named after its host (`plugins/atshosts/workday.py`) and imports
|
||||
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
|
||||
RANK = 100 # optional int; lower comes first in the select and the probes
|
||||
KEEP_CASE = False # optional; True keeps a discovered slug's case
|
||||
ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case
|
||||
|
||||
def slug_ok(slug: str) -> bool # strict, see below
|
||||
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
|
||||
@@ -120,7 +121,9 @@ once a day per board (`ats:robots:<target>`), following a redirect to the
|
||||
host's `www.` twin or back and no other. The board fails with "robots.txt
|
||||
disallows it" when the rules for `*` or for the app's own token disallow the
|
||||
page (RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a
|
||||
final `$` in paths, case-sensitive paths). A 4xx other than 429 means no
|
||||
final `$` in paths, case-sensitive paths, or without case for a host with
|
||||
`ROBOTS_ANY_CASE`, which is for one that serves a path in any case, as
|
||||
Workday does). A 4xx other than 429 means no
|
||||
rules, which allows; a 5xx, a 429, a timeout or a redirect elsewhere fails
|
||||
the board with "robots.txt could not be read" and is asked again next pull.
|
||||
`robots_page` must be on a host `allowed(slug)` admits.
|
||||
@@ -163,6 +166,8 @@ def problems(module: ModuleType, name: str | None = None) -> list[str]:
|
||||
out.append("LABEL is not a non-empty string")
|
||||
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
|
||||
out.append("KEEP_CASE is not a bool")
|
||||
if not isinstance(getattr(module, "ROBOTS_ANY_CASE", False), bool):
|
||||
out.append("ROBOTS_ANY_CASE is not a bool")
|
||||
rank = getattr(module, "RANK", DEFAULT_RANK)
|
||||
if not isinstance(rank, int) or isinstance(rank, bool):
|
||||
out.append("RANK is not an int")
|
||||
|
||||
@@ -30,6 +30,9 @@ from ._shared import Get, Page, Row, only, parse_dt
|
||||
NAME = "workday"
|
||||
LABEL = "Workday"
|
||||
KEEP_CASE = True
|
||||
#: A tenant answers `/jobs` and `/Jobs` alike, so a rule in either case
|
||||
#: forbids the site.
|
||||
ROBOTS_ANY_CASE = True
|
||||
|
||||
#: One group for `<tenant>.<wdN>` and one for the site, which discovery joins
|
||||
#: into the slug with "/". A locale segment (`/en-US/`, `/de-DE/`) may come
|
||||
|
||||
@@ -588,6 +588,20 @@ def test_robots_that_disallow_the_page_fail_the_board(
|
||||
assert "ats:ok:fake:acme" not in state
|
||||
|
||||
|
||||
def test_a_host_that_serves_any_case_is_forbidden_by_a_rule_in_any_case(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /ACME/\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
host = _fake_host(robots=True)
|
||||
items, _state = _run(monkeypatch, host)
|
||||
assert len(items) == 1 # paths are case-sensitive by default
|
||||
host.ROBOTS_ANY_CASE = True
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, host)
|
||||
assert items == []
|
||||
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("robots", "page", "allowed"), [
|
||||
# The longest matching rule wins, whatever the order of the lines.
|
||||
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",
|
||||
|
||||
Reference in new issue
Block a user