Fail a board whose robots.txt cannot be read, and follow a robots.txt redirect to the host's www. twin only
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
13ce99304b
commit
1e47192aba
2 files changed
+92
-26
No files matched your search
+37
-18
@@ -117,15 +117,23 @@ class _Disallowed(Exception):
|
||||
`_check_robots`)."""
|
||||
|
||||
|
||||
class _RobotsUnreachable(Exception):
|
||||
"""The site's robots.txt could not be read: a 5xx, a 429, a timeout, a
|
||||
connection error or a redirect off the site. RFC 9309 says to assume
|
||||
the site disallows everything then, so the board is not read this pull."""
|
||||
|
||||
|
||||
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
method: str = "GET", payload: dict[str, Any] | None = None,
|
||||
headers: dict[str, str] | None = None) -> bytes:
|
||||
headers: dict[str, str] | None = None, www_twin: bool = False) -> bytes:
|
||||
"""Fetch one URL for a board: https only, the host's own allow-list, at
|
||||
most three redirect hops each checked the same way, and the body cut off
|
||||
at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
|
||||
sends `payload` as JSON, `{}` when there is none. `headers` are sent on
|
||||
top of the User-Agent and Accept every request carries, and replace
|
||||
either where they name it.
|
||||
either where they name it. With `www_twin`, a redirect may also land on
|
||||
the first URL's host with `www.` added or taken off, which only the
|
||||
robots.txt read asks for (see `_check_robots`).
|
||||
|
||||
The body is streamed and counted as it arrives, so an oversized one is
|
||||
refused at `MAX_BYTES` rather than after it has been read whole, and
|
||||
@@ -134,6 +142,10 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
good. The deadline is checked between chunks, so it can be overrun by
|
||||
at most one read timeout."""
|
||||
allow = _allowed_host(host, slug)
|
||||
if www_twin:
|
||||
bare = httpx.URL(url).host.removeprefix("www.")
|
||||
board_allow = allow
|
||||
allow = lambda h: board_allow(h) or h.removeprefix("www.") == bare # noqa: E731
|
||||
sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {})
|
||||
deadline = time.monotonic() + DEADLINE
|
||||
current = url
|
||||
@@ -503,13 +515,20 @@ def _robots_allows(body: bytes, page: str) -> bool:
|
||||
def _check_robots(client: httpx.Client, target: str,
|
||||
state: MutableMapping[str, str] | None = None) -> None:
|
||||
"""Raise `_Disallowed` when the site of a host with a `robots_page` has a
|
||||
robots.txt that disallows that page. Read through `_get`, so from the
|
||||
board's own allowed hosts only, and at most once per `ROBOTS_TTL` for a
|
||||
board when `state` is given: the verdict and when it was reached are kept
|
||||
under `ats:robots:<target>`. A robots.txt that is missing (404 or 410)
|
||||
allows, and that verdict is kept like any other; one that cannot be read
|
||||
for another reason allows too, but is not kept, so the next pull asks
|
||||
again."""
|
||||
robots.txt that disallows that page, and `_RobotsUnreachable` when it
|
||||
cannot be read. Read through `_get`, so from the board's own allowed
|
||||
hosts only, except that a redirect between the host and its `www.` twin
|
||||
is followed (at most three hops, https only); any other redirect off the
|
||||
site is unreachable. With `state`, the verdict and when it was reached
|
||||
are kept under `ats:robots:<target>` and held for `ROBOTS_TTL`.
|
||||
|
||||
By RFC 9309: a 4xx other than 429 means there is no robots.txt to obey,
|
||||
which allows, and is kept like any other verdict. A 5xx, a 429, a
|
||||
timeout or a connection error means the file is unreachable, which the
|
||||
RFC says to treat as disallowing everything: the board is not read on
|
||||
this pull, fails with "robots.txt could not be read", and nothing is
|
||||
kept, so the next pull asks again. A 429 counts as unreachable although
|
||||
it is a 4xx, since it says "not now" rather than "there is none"."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = atshosts.HOSTS.get(host)
|
||||
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
|
||||
@@ -529,17 +548,14 @@ def _check_robots(client: httpx.Client, target: str,
|
||||
return
|
||||
url = f"https://{httpx.URL(page).host}/robots.txt"
|
||||
try:
|
||||
body = _getter(client, host, slug)(url, headers={"Accept": "text/plain"})
|
||||
body = _get(client, url, host, slug, headers={"Accept": "text/plain"}, www_twin=True)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if exc.response.status_code not in (404, 410):
|
||||
log.warning("ats: %s: robots.txt answered %s; allowed until the next pull",
|
||||
target, exc.response.status_code)
|
||||
return
|
||||
code = exc.response.status_code
|
||||
if not 400 <= code < 500 or code == 429:
|
||||
raise _RobotsUnreachable(f"robots.txt answered {code}") from exc
|
||||
allowed = True
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s: robots.txt not read (%s); allowed until the next pull",
|
||||
target, exc)
|
||||
return
|
||||
raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc
|
||||
else:
|
||||
allowed = _robots_allows(body, page)
|
||||
if state is not None:
|
||||
@@ -610,7 +626,8 @@ def fetch_board(target: str) -> tuple[list[Row], bool]:
|
||||
`app companies add|check`, which ask that question about one board at a
|
||||
time outside any pull. The listing only, with no detail reads, and
|
||||
robots.txt read afresh. Raises the same way `_fetch_rows` does, and
|
||||
`_Disallowed` for a board robots.txt shuts out."""
|
||||
`_Disallowed` or `_RobotsUnreachable` for a board robots.txt shuts out or
|
||||
that cannot be checked."""
|
||||
with httpx.Client() as client:
|
||||
_check_robots(client, target)
|
||||
return _fetch_rows(client, target)
|
||||
@@ -621,6 +638,8 @@ def failure_reason(exc: BaseException) -> str:
|
||||
error, else the exception's class and the first line of its message."""
|
||||
if isinstance(exc, _Disallowed):
|
||||
return "robots.txt disallows it"
|
||||
if isinstance(exc, _RobotsUnreachable):
|
||||
return "robots.txt could not be read"
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
|
||||
lines = str(exc).strip().splitlines()
|
||||
|
||||
Reference in new issue
Block a user