diff --git a/plugins/ats.py b/plugins/ats.py index 5c3f94c..2f1317f 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -36,7 +36,7 @@ import time from collections.abc import Callable, Iterator, MutableMapping from datetime import UTC, datetime, timedelta from typing import Any -from urllib.robotparser import RobotFileParser +from urllib.parse import unquote import httpx @@ -422,13 +422,82 @@ def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]: ROBOTS_TTL = timedelta(days=1) +#: This app's product token, the name a robots.txt group addresses it by: +#: the `desk` of `desk/ (personal aggregator)`. +ROBOTS_TOKEN = USER_AGENT.split("/", 1)[0].strip().lower() + +#: One rule of a group: `(allow, path pattern)`. +_Rule = tuple[bool, str] + + +def _robots_groups(text: str) -> list[tuple[set[str], list[_Rule]]]: + """The groups of a robots.txt, per RFC 9309: one or more `user-agent` + lines (their tokens lowercased) and the `allow` and `disallow` rules that + follow them, up to the next `user-agent` line after a rule. Comments, + other keys and rules before any `user-agent` line are left out, and so is + an empty rule, which allows everything and so says nothing.""" + groups: list[tuple[set[str], list[_Rule]]] = [] + in_agents = False + for line in text.splitlines(): + key, sep, value = line.split("#", 1)[0].partition(":") + key, value = key.strip().lower(), value.strip() + if not sep: + continue + if key == "user-agent": + if not in_agents: + groups.append((set(), [])) + in_agents = True + groups[-1][0].add(value.lower()) + elif key in ("allow", "disallow") and groups: + in_agents = False + if value: + groups[-1][1].append((key == "allow", value)) + return groups + + +def _robots_rules(groups: list[tuple[set[str], list[_Rule]]], token: str) -> list[_Rule]: + """The rules for `token`: every group that names it (matched whole and + ignoring case, as RFC 9309 says, not as a substring), merged; else every + `*` group, merged; else none, which allows everything.""" + for wanted in (token, "*"): + rules = [rule for agents, group in groups if wanted in agents for rule in group] + if rules or any(wanted in agents for agents, _group in groups): + return rules + return [] + + +def _robots_pattern(path: str) -> re.Pattern[str]: + """A rule's path as a regex anchored at the start: `*` is any run of + characters, and a `$` at the end anchors the end.""" + anchored = path.endswith("$") + body = ".*".join(re.escape(part) for part in path.rstrip("$").split("*")) + return re.compile(body + (r"\Z" if anchored else ""), re.DOTALL) + + +def _robots_verdict(rules: list[_Rule], path: str) -> bool: + """Whether `path` is allowed by `rules`: the longest matching rule wins, + an allow wins a tie, and no match allows. Both sides are compared + percent-decoded, so `%7E` and `~` are one character.""" + path = unquote(path) + best: tuple[int, bool] | None = None + for allow, pattern in rules: + if _robots_pattern(unquote(pattern)).match(path): + weight = (len(pattern), allow) + best = weight if best is None or weight > best else best + return best is None or best[1] + + def _robots_allows(body: bytes, page: str) -> bool: - """Whether a robots.txt lets `page` be fetched: by its rules for every - agent (`*`), and by any it has for this app's own User-Agent token, so a - site that shuts out this app by name is honoured as well.""" - parser = RobotFileParser() - parser.parse(body.decode("utf-8", "replace").splitlines()) - return parser.can_fetch("*", page) and parser.can_fetch(USER_AGENT, page) + """Whether a robots.txt lets `page` be fetched, by RFC 9309: by its rules + for every agent (`*`), and by those it has for this app's own token + (`ROBOTS_TOKEN`, which fall back to `*`), so a site that shuts out this + app by name is honoured as well. Paths are case-sensitive; a byte order + mark is dropped.""" + groups = _robots_groups(body.decode("utf-8-sig", "replace")) + url = httpx.URL(page) + path = (url.raw_path or b"/").decode("ascii", "replace") + return all(_robots_verdict(_robots_rules(groups, token), path) + for token in ("*", ROBOTS_TOKEN)) def _check_robots(client: httpx.Client, target: str, diff --git a/tests/test_atshosts.py b/tests/test_atshosts.py index 0495fd3..ed3e948 100644 --- a/tests/test_atshosts.py +++ b/tests/test_atshosts.py @@ -364,6 +364,48 @@ def test_robots_that_disallow_the_page_fail_the_board( assert "ats:ok:fake:acme" not in state +@pytest.mark.parametrize(("robots", "page", "allowed"), [ + # The longest matching rule wins, whatever the order of the lines. + ("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", + "https://jobs.infineon.com/api/pcsx/search?domain=infineon.com", True), + ("User-agent: *\nAllow: /careers\nDisallow: /\n", "https://x.example/careers/list", True), + ("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", "https://x.example/other", False), + # An allow wins a tie. + ("User-agent: *\nDisallow: /jobs\nAllow: /jobs\n", "https://x.example/jobs", True), + # `*` inside a path, and the query counts as part of it. + ("User-agent: *\nDisallow: /*company\n", + "https://career5.successfactors.eu/career?company=x", False), + # `$` anchors the end. + ("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json", False), + ("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json?page=2", True), + # An empty Disallow allows. + ("User-agent: *\nDisallow:\n", "https://x.example/anything", True), + # A byte order mark and CRLF line ends. + ("\ufeffUser-agent: *\r\nDisallow: /\r\n", "https://x.example/jobs", False), + # Paths are case-sensitive. + ("User-agent: *\nDisallow: /Novonesis_Careers/\n", + "https://novonesis.wd103.myworkdayjobs.com/Novonesis_Careers/", False), + ("User-agent: *\nDisallow: /Novonesis_Careers/\n", + "https://novonesis.wd103.myworkdayjobs.com/novonesis_careers/", True), + # A group for this app by name counts, a group for someone else does + # not, and a token matches whole, not as a substring. + (f"User-agent: {ats.ROBOTS_TOKEN.upper()}\nDisallow: /\n\nUser-agent: *\nAllow: /\n", + "https://x.example/jobs", False), + ("User-agent: Googlebot\nDisallow: /\n", "https://x.example/jobs", True), + ("User-agent: sk\nDisallow: /\n\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True), + # Two agents share one group, and a comment is no rule. + ("User-agent: a\nUser-agent: *\nDisallow: /jobs # not these\n", + "https://x.example/jobs/1", False), + # A group for this app that allows does not undo the `*` group's refusal. + (f"User-agent: {ats.ROBOTS_TOKEN}\nAllow: /\n\nUser-agent: *\nDisallow: /\n", + "https://x.example/jobs", False), + # Rules before any user-agent line belong to no group. + ("Disallow: /\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True), +]) +def test_robots_rules_follow_rfc_9309(robots: str, page: str, allowed: bool) -> None: + assert ats._robots_allows(robots.encode(), page) is allowed + + def test_a_robots_verdict_is_kept_for_a_day( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",