Fail a board whose robots.txt cannot be read, and follow a robots.txt redirect to the host's www. twin only

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-29 19:40:14 +02:00
1 parent 13ce99304b
commit 1e47192aba
2 files changed
+92 -26

No files matched your search

+37 -18
View File
@@ -117,15 +117,23 @@ class _Disallowed(Exception):
`_check_robots`)."""
class _RobotsUnreachable(Exception):
"""The site's robots.txt could not be read: a 5xx, a 429, a timeout, a
connection error or a redirect off the site. RFC 9309 says to assume
the site disallows everything then, so the board is not read this pull."""
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None) -> bytes:
headers: dict[str, str] | None = None, www_twin: bool = False) -> bytes:
"""Fetch one URL for a board: https only, the host's own allow-list, at
most three redirect hops each checked the same way, and the body cut off
at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
sends `payload` as JSON, `{}` when there is none. `headers` are sent on
top of the User-Agent and Accept every request carries, and replace
either where they name it.
either where they name it. With `www_twin`, a redirect may also land on
the first URL's host with `www.` added or taken off, which only the
robots.txt read asks for (see `_check_robots`).
The body is streamed and counted as it arrives, so an oversized one is
refused at `MAX_BYTES` rather than after it has been read whole, and
@@ -134,6 +142,10 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *,
good. The deadline is checked between chunks, so it can be overrun by
at most one read timeout."""
allow = _allowed_host(host, slug)
if www_twin:
bare = httpx.URL(url).host.removeprefix("www.")
board_allow = allow
allow = lambda h: board_allow(h) or h.removeprefix("www.") == bare # noqa: E731
sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {})
deadline = time.monotonic() + DEADLINE
current = url
@@ -503,13 +515,20 @@ def _robots_allows(body: bytes, page: str) -> bool:
def _check_robots(client: httpx.Client, target: str,
state: MutableMapping[str, str] | None = None) -> None:
"""Raise `_Disallowed` when the site of a host with a `robots_page` has a
robots.txt that disallows that page. Read through `_get`, so from the
board's own allowed hosts only, and at most once per `ROBOTS_TTL` for a
board when `state` is given: the verdict and when it was reached are kept
under `ats:robots:<target>`. A robots.txt that is missing (404 or 410)
allows, and that verdict is kept like any other; one that cannot be read
for another reason allows too, but is not kept, so the next pull asks
again."""
robots.txt that disallows that page, and `_RobotsUnreachable` when it
cannot be read. Read through `_get`, so from the board's own allowed
hosts only, except that a redirect between the host and its `www.` twin
is followed (at most three hops, https only); any other redirect off the
site is unreachable. With `state`, the verdict and when it was reached
are kept under `ats:robots:<target>` and held for `ROBOTS_TTL`.
By RFC 9309: a 4xx other than 429 means there is no robots.txt to obey,
which allows, and is kept like any other verdict. A 5xx, a 429, a
timeout or a connection error means the file is unreachable, which the
RFC says to treat as disallowing everything: the board is not read on
this pull, fails with "robots.txt could not be read", and nothing is
kept, so the next pull asks again. A 429 counts as unreachable although
it is a 4xx, since it says "not now" rather than "there is none"."""
host, _, slug = target.partition(":")
module = atshosts.HOSTS.get(host)
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
@@ -529,17 +548,14 @@ def _check_robots(client: httpx.Client, target: str,
return
url = f"https://{httpx.URL(page).host}/robots.txt"
try:
body = _getter(client, host, slug)(url, headers={"Accept": "text/plain"})
body = _get(client, url, host, slug, headers={"Accept": "text/plain"}, www_twin=True)
except httpx.HTTPStatusError as exc:
if exc.response.status_code not in (404, 410):
log.warning("ats: %s: robots.txt answered %s; allowed until the next pull",
target, exc.response.status_code)
return
code = exc.response.status_code
if not 400 <= code < 500 or code == 429:
raise _RobotsUnreachable(f"robots.txt answered {code}") from exc
allowed = True
except Exception as exc:
log.warning("ats: %s: robots.txt not read (%s); allowed until the next pull",
target, exc)
return
raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc
else:
allowed = _robots_allows(body, page)
if state is not None:
@@ -610,7 +626,8 @@ def fetch_board(target: str) -> tuple[list[Row], bool]:
`app companies add|check`, which ask that question about one board at a
time outside any pull. The listing only, with no detail reads, and
robots.txt read afresh. Raises the same way `_fetch_rows` does, and
`_Disallowed` for a board robots.txt shuts out."""
`_Disallowed` or `_RobotsUnreachable` for a board robots.txt shuts out or
that cannot be checked."""
with httpx.Client() as client:
_check_robots(client, target)
return _fetch_rows(client, target)
@@ -621,6 +638,8 @@ def failure_reason(exc: BaseException) -> str:
error, else the exception's class and the first line of its message."""
if isinstance(exc, _Disallowed):
return "robots.txt disallows it"
if isinstance(exc, _RobotsUnreachable):
return "robots.txt could not be read"
if isinstance(exc, httpx.HTTPStatusError):
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
lines = str(exc).strip().splitlines()