From 1e47192abab5d5111ec973dfa8d552a7647bc871 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 19:40:14 +0200 Subject: [PATCH] Fail a board whose robots.txt cannot be read, and follow a robots.txt redirect to the host's www. twin only Co-Authored-By: Claude Opus 5.5 --- plugins/ats.py | 55 ++++++++++++++++++++++++------------ tests/test_atshosts.py | 63 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 92 insertions(+), 26 deletions(-) diff --git a/plugins/ats.py b/plugins/ats.py index 2f1317f..ccf8e3e 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -117,15 +117,23 @@ class _Disallowed(Exception): `_check_robots`).""" +class _RobotsUnreachable(Exception): + """The site's robots.txt could not be read: a 5xx, a 429, a timeout, a + connection error or a redirect off the site. RFC 9309 says to assume + the site disallows everything then, so the board is not read this pull.""" + + def _get(client: httpx.Client, url: str, host: str, slug: str, *, method: str = "GET", payload: dict[str, Any] | None = None, - headers: dict[str, str] | None = None) -> bytes: + headers: dict[str, str] | None = None, www_twin: bool = False) -> bytes: """Fetch one URL for a board: https only, the host's own allow-list, at most three redirect hops each checked the same way, and the body cut off at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST sends `payload` as JSON, `{}` when there is none. `headers` are sent on top of the User-Agent and Accept every request carries, and replace - either where they name it. + either where they name it. With `www_twin`, a redirect may also land on + the first URL's host with `www.` added or taken off, which only the + robots.txt read asks for (see `_check_robots`). The body is streamed and counted as it arrives, so an oversized one is refused at `MAX_BYTES` rather than after it has been read whole, and @@ -134,6 +142,10 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *, good. The deadline is checked between chunks, so it can be overrun by at most one read timeout.""" allow = _allowed_host(host, slug) + if www_twin: + bare = httpx.URL(url).host.removeprefix("www.") + board_allow = allow + allow = lambda h: board_allow(h) or h.removeprefix("www.") == bare # noqa: E731 sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {}) deadline = time.monotonic() + DEADLINE current = url @@ -503,13 +515,20 @@ def _robots_allows(body: bytes, page: str) -> bool: def _check_robots(client: httpx.Client, target: str, state: MutableMapping[str, str] | None = None) -> None: """Raise `_Disallowed` when the site of a host with a `robots_page` has a - robots.txt that disallows that page. Read through `_get`, so from the - board's own allowed hosts only, and at most once per `ROBOTS_TTL` for a - board when `state` is given: the verdict and when it was reached are kept - under `ats:robots:`. A robots.txt that is missing (404 or 410) - allows, and that verdict is kept like any other; one that cannot be read - for another reason allows too, but is not kept, so the next pull asks - again.""" + robots.txt that disallows that page, and `_RobotsUnreachable` when it + cannot be read. Read through `_get`, so from the board's own allowed + hosts only, except that a redirect between the host and its `www.` twin + is followed (at most three hops, https only); any other redirect off the + site is unreachable. With `state`, the verdict and when it was reached + are kept under `ats:robots:` and held for `ROBOTS_TTL`. + + By RFC 9309: a 4xx other than 429 means there is no robots.txt to obey, + which allows, and is kept like any other verdict. A 5xx, a 429, a + timeout or a connection error means the file is unreachable, which the + RFC says to treat as disallowing everything: the board is not read on + this pull, fails with "robots.txt could not be read", and nothing is + kept, so the next pull asks again. A 429 counts as unreachable although + it is a 4xx, since it says "not now" rather than "there is none".""" host, _, slug = target.partition(":") module = atshosts.HOSTS.get(host) if module is None or not hasattr(module, "robots_page") or not is_known_target(target): @@ -529,17 +548,14 @@ def _check_robots(client: httpx.Client, target: str, return url = f"https://{httpx.URL(page).host}/robots.txt" try: - body = _getter(client, host, slug)(url, headers={"Accept": "text/plain"}) + body = _get(client, url, host, slug, headers={"Accept": "text/plain"}, www_twin=True) except httpx.HTTPStatusError as exc: - if exc.response.status_code not in (404, 410): - log.warning("ats: %s: robots.txt answered %s; allowed until the next pull", - target, exc.response.status_code) - return + code = exc.response.status_code + if not 400 <= code < 500 or code == 429: + raise _RobotsUnreachable(f"robots.txt answered {code}") from exc allowed = True except Exception as exc: - log.warning("ats: %s: robots.txt not read (%s); allowed until the next pull", - target, exc) - return + raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc else: allowed = _robots_allows(body, page) if state is not None: @@ -610,7 +626,8 @@ def fetch_board(target: str) -> tuple[list[Row], bool]: `app companies add|check`, which ask that question about one board at a time outside any pull. The listing only, with no detail reads, and robots.txt read afresh. Raises the same way `_fetch_rows` does, and - `_Disallowed` for a board robots.txt shuts out.""" + `_Disallowed` or `_RobotsUnreachable` for a board robots.txt shuts out or + that cannot be checked.""" with httpx.Client() as client: _check_robots(client, target) return _fetch_rows(client, target) @@ -621,6 +638,8 @@ def failure_reason(exc: BaseException) -> str: error, else the exception's class and the first line of its message.""" if isinstance(exc, _Disallowed): return "robots.txt disallows it" + if isinstance(exc, _RobotsUnreachable): + return "robots.txt could not be read" if isinstance(exc, httpx.HTTPStatusError): return f"{exc.response.status_code} {exc.response.reason_phrase}".strip() lines = str(exc).strip().splitlines() diff --git a/tests/test_atshosts.py b/tests/test_atshosts.py index ed3e948..4b2b1be 100644 --- a/tests/test_atshosts.py +++ b/tests/test_atshosts.py @@ -200,7 +200,8 @@ def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: answers: dict[str, Any] = {} asked: list[str] = [] - def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None): + def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None, + www_twin=False): asked.append(url) if not ats._allowed_host(host, slug)(httpx.URL(url).host): raise ats._Refused(url) @@ -429,19 +430,65 @@ def test_a_robots_verdict_is_kept_for_a_day( assert len(items) == 1 and "ats:failing:fake:acme" not in state -def test_a_missing_robots_txt_allows_and_is_kept( - monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: - web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]' # robots.txt: 404 +@pytest.mark.parametrize("code", [404, 410, 401, 403]) +def test_a_robots_txt_answering_4xx_allows_and_is_kept( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], code: int) -> None: + web["answers"].update({ROBOTS: _status(code), "https://jobs.fake.example/acme/list": b'["1"]'}) items, state = _run(monkeypatch, _fake_host(robots=True)) assert len(items) == 1 assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True -def test_a_robots_txt_that_cannot_be_read_allows_and_is_asked_again( - monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: - web["answers"].update({ROBOTS: _status(503), "https://jobs.fake.example/acme/list": b'["1"]'}) +@pytest.mark.parametrize("failure", [ + _status(500), _status(503), _status(429), httpx.ConnectTimeout("timed out"), + httpx.ConnectError("refused"), ats._Refused("https://elsewhere.example/robots.txt"), +]) +def test_a_robots_txt_that_cannot_be_read_fails_the_board_and_is_asked_again( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None: + web["answers"].update({ROBOTS: failure, "https://jobs.fake.example/acme/list": b'["1"]'}) items, state = _run(monkeypatch, _fake_host(robots=True)) - assert len(items) == 1 and "ats:robots:fake:acme" not in state + assert items == [] and web["asked"] == [ROBOTS] + assert json.loads(state["ats:failing:fake:acme"])["reason"] == \ + "robots.txt could not be read" + assert "ats:robots:fake:acme" not in state + + +def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]: + """A site whose robots.txt on jobs.fake.example redirects to `location`, + where a robots.txt that disallows everything is served.""" + def handler(request: httpx.Request) -> httpx.Response: + if str(request.url) == ROBOTS: + return httpx.Response(301, headers={"location": location}) + if request.url.path == "/robots.txt": + return httpx.Response(200, text="User-agent: *\nDisallow: /\n") + return httpx.Response(200, text='["1"]') + return handler + + +def test_a_robots_txt_redirect_to_the_www_twin_is_followed( + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) + with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \ + pytest.raises(ats._Disallowed): + ats._check_robots(client, "fake:acme", {}) + + +@pytest.mark.parametrize("location", [ + "https://evil.example/robots.txt", "https://other.fake.example/robots.txt", + "http://www.jobs.fake.example/robots.txt", +]) +def test_a_robots_txt_redirect_anywhere_else_is_unreachable( + monkeypatch: pytest.MonkeyPatch, location: str) -> None: + monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) + with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable): + ats._check_robots(client, "fake:acme", {}) + + +def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True)) + with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \ + pytest.raises(ats._Refused): + ats._get(client, ROBOTS, "fake", "acme") def test_fetch_board_honours_robots_txt(