Fail a board whose robots.txt cannot be read, and follow a robots.txt redirect to the host's www. twin only

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-29 19:40:14 +02:00
1 parent 13ce99304b
commit 1e47192aba
2 files changed
+92 -26

No files matched your search

+37 -18
View File
@@ -117,15 +117,23 @@ class _Disallowed(Exception):
`_check_robots`)."""
class _RobotsUnreachable(Exception):
"""The site's robots.txt could not be read: a 5xx, a 429, a timeout, a
connection error or a redirect off the site. RFC 9309 says to assume
the site disallows everything then, so the board is not read this pull."""
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None) -> bytes:
headers: dict[str, str] | None = None, www_twin: bool = False) -> bytes:
"""Fetch one URL for a board: https only, the host's own allow-list, at
most three redirect hops each checked the same way, and the body cut off
at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
sends `payload` as JSON, `{}` when there is none. `headers` are sent on
top of the User-Agent and Accept every request carries, and replace
either where they name it.
either where they name it. With `www_twin`, a redirect may also land on
the first URL's host with `www.` added or taken off, which only the
robots.txt read asks for (see `_check_robots`).
The body is streamed and counted as it arrives, so an oversized one is
refused at `MAX_BYTES` rather than after it has been read whole, and
@@ -134,6 +142,10 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *,
good. The deadline is checked between chunks, so it can be overrun by
at most one read timeout."""
allow = _allowed_host(host, slug)
if www_twin:
bare = httpx.URL(url).host.removeprefix("www.")
board_allow = allow
allow = lambda h: board_allow(h) or h.removeprefix("www.") == bare # noqa: E731
sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {})
deadline = time.monotonic() + DEADLINE
current = url
@@ -503,13 +515,20 @@ def _robots_allows(body: bytes, page: str) -> bool:
def _check_robots(client: httpx.Client, target: str,
state: MutableMapping[str, str] | None = None) -> None:
"""Raise `_Disallowed` when the site of a host with a `robots_page` has a
robots.txt that disallows that page. Read through `_get`, so from the
board's own allowed hosts only, and at most once per `ROBOTS_TTL` for a
board when `state` is given: the verdict and when it was reached are kept
under `ats:robots:<target>`. A robots.txt that is missing (404 or 410)
allows, and that verdict is kept like any other; one that cannot be read
for another reason allows too, but is not kept, so the next pull asks
again."""
robots.txt that disallows that page, and `_RobotsUnreachable` when it
cannot be read. Read through `_get`, so from the board's own allowed
hosts only, except that a redirect between the host and its `www.` twin
is followed (at most three hops, https only); any other redirect off the
site is unreachable. With `state`, the verdict and when it was reached
are kept under `ats:robots:<target>` and held for `ROBOTS_TTL`.
By RFC 9309: a 4xx other than 429 means there is no robots.txt to obey,
which allows, and is kept like any other verdict. A 5xx, a 429, a
timeout or a connection error means the file is unreachable, which the
RFC says to treat as disallowing everything: the board is not read on
this pull, fails with "robots.txt could not be read", and nothing is
kept, so the next pull asks again. A 429 counts as unreachable although
it is a 4xx, since it says "not now" rather than "there is none"."""
host, _, slug = target.partition(":")
module = atshosts.HOSTS.get(host)
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
@@ -529,17 +548,14 @@ def _check_robots(client: httpx.Client, target: str,
return
url = f"https://{httpx.URL(page).host}/robots.txt"
try:
body = _getter(client, host, slug)(url, headers={"Accept": "text/plain"})
body = _get(client, url, host, slug, headers={"Accept": "text/plain"}, www_twin=True)
except httpx.HTTPStatusError as exc:
if exc.response.status_code not in (404, 410):
log.warning("ats: %s: robots.txt answered %s; allowed until the next pull",
target, exc.response.status_code)
return
code = exc.response.status_code
if not 400 <= code < 500 or code == 429:
raise _RobotsUnreachable(f"robots.txt answered {code}") from exc
allowed = True
except Exception as exc:
log.warning("ats: %s: robots.txt not read (%s); allowed until the next pull",
target, exc)
return
raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc
else:
allowed = _robots_allows(body, page)
if state is not None:
@@ -610,7 +626,8 @@ def fetch_board(target: str) -> tuple[list[Row], bool]:
`app companies add|check`, which ask that question about one board at a
time outside any pull. The listing only, with no detail reads, and
robots.txt read afresh. Raises the same way `_fetch_rows` does, and
`_Disallowed` for a board robots.txt shuts out."""
`_Disallowed` or `_RobotsUnreachable` for a board robots.txt shuts out or
that cannot be checked."""
with httpx.Client() as client:
_check_robots(client, target)
return _fetch_rows(client, target)
@@ -621,6 +638,8 @@ def failure_reason(exc: BaseException) -> str:
error, else the exception's class and the first line of its message."""
if isinstance(exc, _Disallowed):
return "robots.txt disallows it"
if isinstance(exc, _RobotsUnreachable):
return "robots.txt could not be read"
if isinstance(exc, httpx.HTTPStatusError):
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
lines = str(exc).strip().splitlines()
+55 -8
View File
@@ -200,7 +200,8 @@ def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
answers: dict[str, Any] = {}
asked: list[str] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None):
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None,
www_twin=False):
asked.append(url)
if not ats._allowed_host(host, slug)(httpx.URL(url).host):
raise ats._Refused(url)
@@ -429,19 +430,65 @@ def test_a_robots_verdict_is_kept_for_a_day(
assert len(items) == 1 and "ats:failing:fake:acme" not in state
def test_a_missing_robots_txt_allows_and_is_kept(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]' # robots.txt: 404
@pytest.mark.parametrize("code", [404, 410, 401, 403])
def test_a_robots_txt_answering_4xx_allows_and_is_kept(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], code: int) -> None:
web["answers"].update({ROBOTS: _status(code), "https://jobs.fake.example/acme/list": b'["1"]'})
items, state = _run(monkeypatch, _fake_host(robots=True))
assert len(items) == 1
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
def test_a_robots_txt_that_cannot_be_read_allows_and_is_asked_again(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: _status(503), "https://jobs.fake.example/acme/list": b'["1"]'})
@pytest.mark.parametrize("failure", [
_status(500), _status(503), _status(429), httpx.ConnectTimeout("timed out"),
httpx.ConnectError("refused"), ats._Refused("https://elsewhere.example/robots.txt"),
])
def test_a_robots_txt_that_cannot_be_read_fails_the_board_and_is_asked_again(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
web["answers"].update({ROBOTS: failure, "https://jobs.fake.example/acme/list": b'["1"]'})
items, state = _run(monkeypatch, _fake_host(robots=True))
assert len(items) == 1 and "ats:robots:fake:acme" not in state
assert items == [] and web["asked"] == [ROBOTS]
assert json.loads(state["ats:failing:fake:acme"])["reason"] == \
"robots.txt could not be read"
assert "ats:robots:fake:acme" not in state
def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]:
"""A site whose robots.txt on jobs.fake.example redirects to `location`,
where a robots.txt that disallows everything is served."""
def handler(request: httpx.Request) -> httpx.Response:
if str(request.url) == ROBOTS:
return httpx.Response(301, headers={"location": location})
if request.url.path == "/robots.txt":
return httpx.Response(200, text="User-agent: *\nDisallow: /\n")
return httpx.Response(200, text='["1"]')
return handler
def test_a_robots_txt_redirect_to_the_www_twin_is_followed(
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
pytest.raises(ats._Disallowed):
ats._check_robots(client, "fake:acme", {})
@pytest.mark.parametrize("location", [
"https://evil.example/robots.txt", "https://other.fake.example/robots.txt",
"http://www.jobs.fake.example/robots.txt",
])
def test_a_robots_txt_redirect_anywhere_else_is_unreachable(
monkeypatch: pytest.MonkeyPatch, location: str) -> None:
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable):
ats._check_robots(client, "fake:acme", {})
def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
pytest.raises(ats._Refused):
ats._get(client, ROBOTS, "fake", "acme")
def test_fetch_board_honours_robots_txt(