Fail a board whose robots.txt cannot be read, and follow a robots.txt redirect to the host's www. twin only
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
13ce99304b
commit
1e47192aba
2 files changed
+92
-26
No files matched your search
+37
-18
@@ -117,15 +117,23 @@ class _Disallowed(Exception):
|
||||
`_check_robots`)."""
|
||||
|
||||
|
||||
class _RobotsUnreachable(Exception):
|
||||
"""The site's robots.txt could not be read: a 5xx, a 429, a timeout, a
|
||||
connection error or a redirect off the site. RFC 9309 says to assume
|
||||
the site disallows everything then, so the board is not read this pull."""
|
||||
|
||||
|
||||
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
method: str = "GET", payload: dict[str, Any] | None = None,
|
||||
headers: dict[str, str] | None = None) -> bytes:
|
||||
headers: dict[str, str] | None = None, www_twin: bool = False) -> bytes:
|
||||
"""Fetch one URL for a board: https only, the host's own allow-list, at
|
||||
most three redirect hops each checked the same way, and the body cut off
|
||||
at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
|
||||
sends `payload` as JSON, `{}` when there is none. `headers` are sent on
|
||||
top of the User-Agent and Accept every request carries, and replace
|
||||
either where they name it.
|
||||
either where they name it. With `www_twin`, a redirect may also land on
|
||||
the first URL's host with `www.` added or taken off, which only the
|
||||
robots.txt read asks for (see `_check_robots`).
|
||||
|
||||
The body is streamed and counted as it arrives, so an oversized one is
|
||||
refused at `MAX_BYTES` rather than after it has been read whole, and
|
||||
@@ -134,6 +142,10 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
good. The deadline is checked between chunks, so it can be overrun by
|
||||
at most one read timeout."""
|
||||
allow = _allowed_host(host, slug)
|
||||
if www_twin:
|
||||
bare = httpx.URL(url).host.removeprefix("www.")
|
||||
board_allow = allow
|
||||
allow = lambda h: board_allow(h) or h.removeprefix("www.") == bare # noqa: E731
|
||||
sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {})
|
||||
deadline = time.monotonic() + DEADLINE
|
||||
current = url
|
||||
@@ -503,13 +515,20 @@ def _robots_allows(body: bytes, page: str) -> bool:
|
||||
def _check_robots(client: httpx.Client, target: str,
|
||||
state: MutableMapping[str, str] | None = None) -> None:
|
||||
"""Raise `_Disallowed` when the site of a host with a `robots_page` has a
|
||||
robots.txt that disallows that page. Read through `_get`, so from the
|
||||
board's own allowed hosts only, and at most once per `ROBOTS_TTL` for a
|
||||
board when `state` is given: the verdict and when it was reached are kept
|
||||
under `ats:robots:<target>`. A robots.txt that is missing (404 or 410)
|
||||
allows, and that verdict is kept like any other; one that cannot be read
|
||||
for another reason allows too, but is not kept, so the next pull asks
|
||||
again."""
|
||||
robots.txt that disallows that page, and `_RobotsUnreachable` when it
|
||||
cannot be read. Read through `_get`, so from the board's own allowed
|
||||
hosts only, except that a redirect between the host and its `www.` twin
|
||||
is followed (at most three hops, https only); any other redirect off the
|
||||
site is unreachable. With `state`, the verdict and when it was reached
|
||||
are kept under `ats:robots:<target>` and held for `ROBOTS_TTL`.
|
||||
|
||||
By RFC 9309: a 4xx other than 429 means there is no robots.txt to obey,
|
||||
which allows, and is kept like any other verdict. A 5xx, a 429, a
|
||||
timeout or a connection error means the file is unreachable, which the
|
||||
RFC says to treat as disallowing everything: the board is not read on
|
||||
this pull, fails with "robots.txt could not be read", and nothing is
|
||||
kept, so the next pull asks again. A 429 counts as unreachable although
|
||||
it is a 4xx, since it says "not now" rather than "there is none"."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = atshosts.HOSTS.get(host)
|
||||
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
|
||||
@@ -529,17 +548,14 @@ def _check_robots(client: httpx.Client, target: str,
|
||||
return
|
||||
url = f"https://{httpx.URL(page).host}/robots.txt"
|
||||
try:
|
||||
body = _getter(client, host, slug)(url, headers={"Accept": "text/plain"})
|
||||
body = _get(client, url, host, slug, headers={"Accept": "text/plain"}, www_twin=True)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if exc.response.status_code not in (404, 410):
|
||||
log.warning("ats: %s: robots.txt answered %s; allowed until the next pull",
|
||||
target, exc.response.status_code)
|
||||
return
|
||||
code = exc.response.status_code
|
||||
if not 400 <= code < 500 or code == 429:
|
||||
raise _RobotsUnreachable(f"robots.txt answered {code}") from exc
|
||||
allowed = True
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s: robots.txt not read (%s); allowed until the next pull",
|
||||
target, exc)
|
||||
return
|
||||
raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc
|
||||
else:
|
||||
allowed = _robots_allows(body, page)
|
||||
if state is not None:
|
||||
@@ -610,7 +626,8 @@ def fetch_board(target: str) -> tuple[list[Row], bool]:
|
||||
`app companies add|check`, which ask that question about one board at a
|
||||
time outside any pull. The listing only, with no detail reads, and
|
||||
robots.txt read afresh. Raises the same way `_fetch_rows` does, and
|
||||
`_Disallowed` for a board robots.txt shuts out."""
|
||||
`_Disallowed` or `_RobotsUnreachable` for a board robots.txt shuts out or
|
||||
that cannot be checked."""
|
||||
with httpx.Client() as client:
|
||||
_check_robots(client, target)
|
||||
return _fetch_rows(client, target)
|
||||
@@ -621,6 +638,8 @@ def failure_reason(exc: BaseException) -> str:
|
||||
error, else the exception's class and the first line of its message."""
|
||||
if isinstance(exc, _Disallowed):
|
||||
return "robots.txt disallows it"
|
||||
if isinstance(exc, _RobotsUnreachable):
|
||||
return "robots.txt could not be read"
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
|
||||
lines = str(exc).strip().splitlines()
|
||||
|
||||
+55
-8
@@ -200,7 +200,8 @@ def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
|
||||
answers: dict[str, Any] = {}
|
||||
asked: list[str] = []
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None):
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None,
|
||||
www_twin=False):
|
||||
asked.append(url)
|
||||
if not ats._allowed_host(host, slug)(httpx.URL(url).host):
|
||||
raise ats._Refused(url)
|
||||
@@ -429,19 +430,65 @@ def test_a_robots_verdict_is_kept_for_a_day(
|
||||
assert len(items) == 1 and "ats:failing:fake:acme" not in state
|
||||
|
||||
|
||||
def test_a_missing_robots_txt_allows_and_is_kept(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]' # robots.txt: 404
|
||||
@pytest.mark.parametrize("code", [404, 410, 401, 403])
|
||||
def test_a_robots_txt_answering_4xx_allows_and_is_kept(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], code: int) -> None:
|
||||
web["answers"].update({ROBOTS: _status(code), "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert len(items) == 1
|
||||
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
|
||||
|
||||
|
||||
def test_a_robots_txt_that_cannot_be_read_allows_and_is_asked_again(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: _status(503), "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
@pytest.mark.parametrize("failure", [
|
||||
_status(500), _status(503), _status(429), httpx.ConnectTimeout("timed out"),
|
||||
httpx.ConnectError("refused"), ats._Refused("https://elsewhere.example/robots.txt"),
|
||||
])
|
||||
def test_a_robots_txt_that_cannot_be_read_fails_the_board_and_is_asked_again(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
|
||||
web["answers"].update({ROBOTS: failure, "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert len(items) == 1 and "ats:robots:fake:acme" not in state
|
||||
assert items == [] and web["asked"] == [ROBOTS]
|
||||
assert json.loads(state["ats:failing:fake:acme"])["reason"] == \
|
||||
"robots.txt could not be read"
|
||||
assert "ats:robots:fake:acme" not in state
|
||||
|
||||
|
||||
def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]:
|
||||
"""A site whose robots.txt on jobs.fake.example redirects to `location`,
|
||||
where a robots.txt that disallows everything is served."""
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if str(request.url) == ROBOTS:
|
||||
return httpx.Response(301, headers={"location": location})
|
||||
if request.url.path == "/robots.txt":
|
||||
return httpx.Response(200, text="User-agent: *\nDisallow: /\n")
|
||||
return httpx.Response(200, text='["1"]')
|
||||
return handler
|
||||
|
||||
|
||||
def test_a_robots_txt_redirect_to_the_www_twin_is_followed(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
||||
pytest.raises(ats._Disallowed):
|
||||
ats._check_robots(client, "fake:acme", {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("location", [
|
||||
"https://evil.example/robots.txt", "https://other.fake.example/robots.txt",
|
||||
"http://www.jobs.fake.example/robots.txt",
|
||||
])
|
||||
def test_a_robots_txt_redirect_anywhere_else_is_unreachable(
|
||||
monkeypatch: pytest.MonkeyPatch, location: str) -> None:
|
||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable):
|
||||
ats._check_robots(client, "fake:acme", {})
|
||||
|
||||
|
||||
def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
||||
pytest.raises(ats._Refused):
|
||||
ats._get(client, ROBOTS, "fake", "acme")
|
||||
|
||||
|
||||
def test_fetch_board_honours_robots_txt(
|
||||
|
||||
Reference in new issue
Block a user