Fail a board whose robots.txt cannot be read, and follow a robots.txt redirect to the host's www. twin only

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-29 19:40:14 +02:00
1 parent 13ce99304b
commit 1e47192aba
2 files changed
+92 -26

No files matched your search

+37 -18
View File
@@ -117,15 +117,23 @@ class _Disallowed(Exception):
`_check_robots`).""" `_check_robots`)."""
class _RobotsUnreachable(Exception):
"""The site's robots.txt could not be read: a 5xx, a 429, a timeout, a
connection error or a redirect off the site. RFC 9309 says to assume
the site disallows everything then, so the board is not read this pull."""
def _get(client: httpx.Client, url: str, host: str, slug: str, *, def _get(client: httpx.Client, url: str, host: str, slug: str, *,
method: str = "GET", payload: dict[str, Any] | None = None, method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None) -> bytes: headers: dict[str, str] | None = None, www_twin: bool = False) -> bytes:
"""Fetch one URL for a board: https only, the host's own allow-list, at """Fetch one URL for a board: https only, the host's own allow-list, at
most three redirect hops each checked the same way, and the body cut off most three redirect hops each checked the same way, and the body cut off
at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
sends `payload` as JSON, `{}` when there is none. `headers` are sent on sends `payload` as JSON, `{}` when there is none. `headers` are sent on
top of the User-Agent and Accept every request carries, and replace top of the User-Agent and Accept every request carries, and replace
either where they name it. either where they name it. With `www_twin`, a redirect may also land on
the first URL's host with `www.` added or taken off, which only the
robots.txt read asks for (see `_check_robots`).
The body is streamed and counted as it arrives, so an oversized one is The body is streamed and counted as it arrives, so an oversized one is
refused at `MAX_BYTES` rather than after it has been read whole, and refused at `MAX_BYTES` rather than after it has been read whole, and
@@ -134,6 +142,10 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *,
good. The deadline is checked between chunks, so it can be overrun by good. The deadline is checked between chunks, so it can be overrun by
at most one read timeout.""" at most one read timeout."""
allow = _allowed_host(host, slug) allow = _allowed_host(host, slug)
if www_twin:
bare = httpx.URL(url).host.removeprefix("www.")
board_allow = allow
allow = lambda h: board_allow(h) or h.removeprefix("www.") == bare # noqa: E731
sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {}) sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {})
deadline = time.monotonic() + DEADLINE deadline = time.monotonic() + DEADLINE
current = url current = url
@@ -503,13 +515,20 @@ def _robots_allows(body: bytes, page: str) -> bool:
def _check_robots(client: httpx.Client, target: str, def _check_robots(client: httpx.Client, target: str,
state: MutableMapping[str, str] | None = None) -> None: state: MutableMapping[str, str] | None = None) -> None:
"""Raise `_Disallowed` when the site of a host with a `robots_page` has a """Raise `_Disallowed` when the site of a host with a `robots_page` has a
robots.txt that disallows that page. Read through `_get`, so from the robots.txt that disallows that page, and `_RobotsUnreachable` when it
board's own allowed hosts only, and at most once per `ROBOTS_TTL` for a cannot be read. Read through `_get`, so from the board's own allowed
board when `state` is given: the verdict and when it was reached are kept hosts only, except that a redirect between the host and its `www.` twin
under `ats:robots:<target>`. A robots.txt that is missing (404 or 410) is followed (at most three hops, https only); any other redirect off the
allows, and that verdict is kept like any other; one that cannot be read site is unreachable. With `state`, the verdict and when it was reached
for another reason allows too, but is not kept, so the next pull asks are kept under `ats:robots:<target>` and held for `ROBOTS_TTL`.
again."""
By RFC 9309: a 4xx other than 429 means there is no robots.txt to obey,
which allows, and is kept like any other verdict. A 5xx, a 429, a
timeout or a connection error means the file is unreachable, which the
RFC says to treat as disallowing everything: the board is not read on
this pull, fails with "robots.txt could not be read", and nothing is
kept, so the next pull asks again. A 429 counts as unreachable although
it is a 4xx, since it says "not now" rather than "there is none"."""
host, _, slug = target.partition(":") host, _, slug = target.partition(":")
module = atshosts.HOSTS.get(host) module = atshosts.HOSTS.get(host)
if module is None or not hasattr(module, "robots_page") or not is_known_target(target): if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
@@ -529,17 +548,14 @@ def _check_robots(client: httpx.Client, target: str,
return return
url = f"https://{httpx.URL(page).host}/robots.txt" url = f"https://{httpx.URL(page).host}/robots.txt"
try: try:
body = _getter(client, host, slug)(url, headers={"Accept": "text/plain"}) body = _get(client, url, host, slug, headers={"Accept": "text/plain"}, www_twin=True)
except httpx.HTTPStatusError as exc: except httpx.HTTPStatusError as exc:
if exc.response.status_code not in (404, 410): code = exc.response.status_code
log.warning("ats: %s: robots.txt answered %s; allowed until the next pull", if not 400 <= code < 500 or code == 429:
target, exc.response.status_code) raise _RobotsUnreachable(f"robots.txt answered {code}") from exc
return
allowed = True allowed = True
except Exception as exc: except Exception as exc:
log.warning("ats: %s: robots.txt not read (%s); allowed until the next pull", raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc
target, exc)
return
else: else:
allowed = _robots_allows(body, page) allowed = _robots_allows(body, page)
if state is not None: if state is not None:
@@ -610,7 +626,8 @@ def fetch_board(target: str) -> tuple[list[Row], bool]:
`app companies add|check`, which ask that question about one board at a `app companies add|check`, which ask that question about one board at a
time outside any pull. The listing only, with no detail reads, and time outside any pull. The listing only, with no detail reads, and
robots.txt read afresh. Raises the same way `_fetch_rows` does, and robots.txt read afresh. Raises the same way `_fetch_rows` does, and
`_Disallowed` for a board robots.txt shuts out.""" `_Disallowed` or `_RobotsUnreachable` for a board robots.txt shuts out or
that cannot be checked."""
with httpx.Client() as client: with httpx.Client() as client:
_check_robots(client, target) _check_robots(client, target)
return _fetch_rows(client, target) return _fetch_rows(client, target)
@@ -621,6 +638,8 @@ def failure_reason(exc: BaseException) -> str:
error, else the exception's class and the first line of its message.""" error, else the exception's class and the first line of its message."""
if isinstance(exc, _Disallowed): if isinstance(exc, _Disallowed):
return "robots.txt disallows it" return "robots.txt disallows it"
if isinstance(exc, _RobotsUnreachable):
return "robots.txt could not be read"
if isinstance(exc, httpx.HTTPStatusError): if isinstance(exc, httpx.HTTPStatusError):
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip() return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
lines = str(exc).strip().splitlines() lines = str(exc).strip().splitlines()
+55 -8
View File
@@ -200,7 +200,8 @@ def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
answers: dict[str, Any] = {} answers: dict[str, Any] = {}
asked: list[str] = [] asked: list[str] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None): def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None,
www_twin=False):
asked.append(url) asked.append(url)
if not ats._allowed_host(host, slug)(httpx.URL(url).host): if not ats._allowed_host(host, slug)(httpx.URL(url).host):
raise ats._Refused(url) raise ats._Refused(url)
@@ -429,19 +430,65 @@ def test_a_robots_verdict_is_kept_for_a_day(
assert len(items) == 1 and "ats:failing:fake:acme" not in state assert len(items) == 1 and "ats:failing:fake:acme" not in state
def test_a_missing_robots_txt_allows_and_is_kept( @pytest.mark.parametrize("code", [404, 410, 401, 403])
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: def test_a_robots_txt_answering_4xx_allows_and_is_kept(
web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]' # robots.txt: 404 monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], code: int) -> None:
web["answers"].update({ROBOTS: _status(code), "https://jobs.fake.example/acme/list": b'["1"]'})
items, state = _run(monkeypatch, _fake_host(robots=True)) items, state = _run(monkeypatch, _fake_host(robots=True))
assert len(items) == 1 assert len(items) == 1
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
def test_a_robots_txt_that_cannot_be_read_allows_and_is_asked_again( @pytest.mark.parametrize("failure", [
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: _status(500), _status(503), _status(429), httpx.ConnectTimeout("timed out"),
web["answers"].update({ROBOTS: _status(503), "https://jobs.fake.example/acme/list": b'["1"]'}) httpx.ConnectError("refused"), ats._Refused("https://elsewhere.example/robots.txt"),
])
def test_a_robots_txt_that_cannot_be_read_fails_the_board_and_is_asked_again(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
web["answers"].update({ROBOTS: failure, "https://jobs.fake.example/acme/list": b'["1"]'})
items, state = _run(monkeypatch, _fake_host(robots=True)) items, state = _run(monkeypatch, _fake_host(robots=True))
assert len(items) == 1 and "ats:robots:fake:acme" not in state assert items == [] and web["asked"] == [ROBOTS]
assert json.loads(state["ats:failing:fake:acme"])["reason"] == \
"robots.txt could not be read"
assert "ats:robots:fake:acme" not in state
def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]:
"""A site whose robots.txt on jobs.fake.example redirects to `location`,
where a robots.txt that disallows everything is served."""
def handler(request: httpx.Request) -> httpx.Response:
if str(request.url) == ROBOTS:
return httpx.Response(301, headers={"location": location})
if request.url.path == "/robots.txt":
return httpx.Response(200, text="User-agent: *\nDisallow: /\n")
return httpx.Response(200, text='["1"]')
return handler
def test_a_robots_txt_redirect_to_the_www_twin_is_followed(
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
pytest.raises(ats._Disallowed):
ats._check_robots(client, "fake:acme", {})
@pytest.mark.parametrize("location", [
"https://evil.example/robots.txt", "https://other.fake.example/robots.txt",
"http://www.jobs.fake.example/robots.txt",
])
def test_a_robots_txt_redirect_anywhere_else_is_unreachable(
monkeypatch: pytest.MonkeyPatch, location: str) -> None:
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable):
ats._check_robots(client, "fake:acme", {})
def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
pytest.raises(ats._Refused):
ats._get(client, ROBOTS, "fake", "acme")
def test_fetch_board_honours_robots_txt( def test_fetch_board_honours_robots_txt(