Read robots.txt without case for a host that serves a path in any case, as Workday does
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
bd1ce44c6f
commit
bb8a9a3b8f
5 files changed
+29
-1
No files matched your search
@@ -54,6 +54,7 @@ LABEL = "Workday" # the Sources page's select
|
|||||||
PATTERNS = (r"https?://...",) # discovery: regexes over a URL, compiled with IGNORECASE
|
PATTERNS = (r"https?://...",) # discovery: regexes over a URL, compiled with IGNORECASE
|
||||||
RANK = 100 # optional int, default 100; lower comes first
|
RANK = 100 # optional int, default 100; lower comes first
|
||||||
KEEP_CASE = True # optional, default False; keep a discovered slug's case
|
KEEP_CASE = True # optional, default False; keep a discovered slug's case
|
||||||
|
ROBOTS_ANY_CASE = True # optional, default False; read robots.txt paths without case
|
||||||
|
|
||||||
def slug_ok(slug: str) -> bool: ... # strict, re.fullmatch
|
def slug_ok(slug: str) -> bool: ... # strict, re.fullmatch
|
||||||
def allowed(slug: str) -> Callable[[str], bool]: ... # host names a fetch may reach
|
def allowed(slug: str) -> Callable[[str], bool]: ... # host names a fetch may reach
|
||||||
|
|||||||
@@ -592,6 +592,11 @@ def _check_robots(client: httpx.Client, target: str,
|
|||||||
allowed = True
|
allowed = True
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc
|
raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc
|
||||||
|
else:
|
||||||
|
# A host that serves a path whatever its case is matched without
|
||||||
|
# case, so a rule written `/Jobs/` forbids a site named `jobs` too.
|
||||||
|
if getattr(module, "ROBOTS_ANY_CASE", False):
|
||||||
|
allowed = _robots_allows(body.lower(), page.lower())
|
||||||
else:
|
else:
|
||||||
allowed = _robots_allows(body, page)
|
allowed = _robots_allows(body, page)
|
||||||
if state is not None:
|
if state is not None:
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ A module is named after its host (`plugins/atshosts/workday.py`) and imports
|
|||||||
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
|
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
|
||||||
RANK = 100 # optional int; lower comes first in the select and the probes
|
RANK = 100 # optional int; lower comes first in the select and the probes
|
||||||
KEEP_CASE = False # optional; True keeps a discovered slug's case
|
KEEP_CASE = False # optional; True keeps a discovered slug's case
|
||||||
|
ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case
|
||||||
|
|
||||||
def slug_ok(slug: str) -> bool # strict, see below
|
def slug_ok(slug: str) -> bool # strict, see below
|
||||||
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
|
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
|
||||||
@@ -120,7 +121,9 @@ once a day per board (`ats:robots:<target>`), following a redirect to the
|
|||||||
host's `www.` twin or back and no other. The board fails with "robots.txt
|
host's `www.` twin or back and no other. The board fails with "robots.txt
|
||||||
disallows it" when the rules for `*` or for the app's own token disallow the
|
disallows it" when the rules for `*` or for the app's own token disallow the
|
||||||
page (RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a
|
page (RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a
|
||||||
final `$` in paths, case-sensitive paths). A 4xx other than 429 means no
|
final `$` in paths, case-sensitive paths, or without case for a host with
|
||||||
|
`ROBOTS_ANY_CASE`, which is for one that serves a path in any case, as
|
||||||
|
Workday does). A 4xx other than 429 means no
|
||||||
rules, which allows; a 5xx, a 429, a timeout or a redirect elsewhere fails
|
rules, which allows; a 5xx, a 429, a timeout or a redirect elsewhere fails
|
||||||
the board with "robots.txt could not be read" and is asked again next pull.
|
the board with "robots.txt could not be read" and is asked again next pull.
|
||||||
`robots_page` must be on a host `allowed(slug)` admits.
|
`robots_page` must be on a host `allowed(slug)` admits.
|
||||||
@@ -163,6 +166,8 @@ def problems(module: ModuleType, name: str | None = None) -> list[str]:
|
|||||||
out.append("LABEL is not a non-empty string")
|
out.append("LABEL is not a non-empty string")
|
||||||
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
|
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
|
||||||
out.append("KEEP_CASE is not a bool")
|
out.append("KEEP_CASE is not a bool")
|
||||||
|
if not isinstance(getattr(module, "ROBOTS_ANY_CASE", False), bool):
|
||||||
|
out.append("ROBOTS_ANY_CASE is not a bool")
|
||||||
rank = getattr(module, "RANK", DEFAULT_RANK)
|
rank = getattr(module, "RANK", DEFAULT_RANK)
|
||||||
if not isinstance(rank, int) or isinstance(rank, bool):
|
if not isinstance(rank, int) or isinstance(rank, bool):
|
||||||
out.append("RANK is not an int")
|
out.append("RANK is not an int")
|
||||||
|
|||||||
@@ -30,6 +30,9 @@ from ._shared import Get, Page, Row, only, parse_dt
|
|||||||
NAME = "workday"
|
NAME = "workday"
|
||||||
LABEL = "Workday"
|
LABEL = "Workday"
|
||||||
KEEP_CASE = True
|
KEEP_CASE = True
|
||||||
|
#: A tenant answers `/jobs` and `/Jobs` alike, so a rule in either case
|
||||||
|
#: forbids the site.
|
||||||
|
ROBOTS_ANY_CASE = True
|
||||||
|
|
||||||
#: One group for `<tenant>.<wdN>` and one for the site, which discovery joins
|
#: One group for `<tenant>.<wdN>` and one for the site, which discovery joins
|
||||||
#: into the slug with "/". A locale segment (`/en-US/`, `/de-DE/`) may come
|
#: into the slug with "/". A locale segment (`/en-US/`, `/de-DE/`) may come
|
||||||
|
|||||||
@@ -588,6 +588,20 @@ def test_robots_that_disallow_the_page_fail_the_board(
|
|||||||
assert "ats:ok:fake:acme" not in state
|
assert "ats:ok:fake:acme" not in state
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_host_that_serves_any_case_is_forbidden_by_a_rule_in_any_case(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /ACME/\n",
|
||||||
|
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||||
|
host = _fake_host(robots=True)
|
||||||
|
items, _state = _run(monkeypatch, host)
|
||||||
|
assert len(items) == 1 # paths are case-sensitive by default
|
||||||
|
host.ROBOTS_ANY_CASE = True
|
||||||
|
web["asked"].clear()
|
||||||
|
items, state = _run(monkeypatch, host)
|
||||||
|
assert items == []
|
||||||
|
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(("robots", "page", "allowed"), [
|
@pytest.mark.parametrize(("robots", "page", "allowed"), [
|
||||||
# The longest matching rule wins, whatever the order of the lines.
|
# The longest matching rule wins, whatever the order of the lines.
|
||||||
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",
|
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",
|
||||||
|
|||||||
Reference in new issue
Block a user