Fail a board whose robots.txt cannot be read, and follow a robots.txt redirect to the host's www. twin only
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
13ce99304b
commit
1e47192aba
2 files changed
+92
-26
No files matched your search
+55
-8
@@ -200,7 +200,8 @@ def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
|
||||
answers: dict[str, Any] = {}
|
||||
asked: list[str] = []
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None):
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None,
|
||||
www_twin=False):
|
||||
asked.append(url)
|
||||
if not ats._allowed_host(host, slug)(httpx.URL(url).host):
|
||||
raise ats._Refused(url)
|
||||
@@ -429,19 +430,65 @@ def test_a_robots_verdict_is_kept_for_a_day(
|
||||
assert len(items) == 1 and "ats:failing:fake:acme" not in state
|
||||
|
||||
|
||||
def test_a_missing_robots_txt_allows_and_is_kept(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]' # robots.txt: 404
|
||||
@pytest.mark.parametrize("code", [404, 410, 401, 403])
|
||||
def test_a_robots_txt_answering_4xx_allows_and_is_kept(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], code: int) -> None:
|
||||
web["answers"].update({ROBOTS: _status(code), "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert len(items) == 1
|
||||
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
|
||||
|
||||
|
||||
def test_a_robots_txt_that_cannot_be_read_allows_and_is_asked_again(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: _status(503), "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
@pytest.mark.parametrize("failure", [
|
||||
_status(500), _status(503), _status(429), httpx.ConnectTimeout("timed out"),
|
||||
httpx.ConnectError("refused"), ats._Refused("https://elsewhere.example/robots.txt"),
|
||||
])
|
||||
def test_a_robots_txt_that_cannot_be_read_fails_the_board_and_is_asked_again(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
|
||||
web["answers"].update({ROBOTS: failure, "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert len(items) == 1 and "ats:robots:fake:acme" not in state
|
||||
assert items == [] and web["asked"] == [ROBOTS]
|
||||
assert json.loads(state["ats:failing:fake:acme"])["reason"] == \
|
||||
"robots.txt could not be read"
|
||||
assert "ats:robots:fake:acme" not in state
|
||||
|
||||
|
||||
def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]:
|
||||
"""A site whose robots.txt on jobs.fake.example redirects to `location`,
|
||||
where a robots.txt that disallows everything is served."""
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if str(request.url) == ROBOTS:
|
||||
return httpx.Response(301, headers={"location": location})
|
||||
if request.url.path == "/robots.txt":
|
||||
return httpx.Response(200, text="User-agent: *\nDisallow: /\n")
|
||||
return httpx.Response(200, text='["1"]')
|
||||
return handler
|
||||
|
||||
|
||||
def test_a_robots_txt_redirect_to_the_www_twin_is_followed(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
||||
pytest.raises(ats._Disallowed):
|
||||
ats._check_robots(client, "fake:acme", {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("location", [
|
||||
"https://evil.example/robots.txt", "https://other.fake.example/robots.txt",
|
||||
"http://www.jobs.fake.example/robots.txt",
|
||||
])
|
||||
def test_a_robots_txt_redirect_anywhere_else_is_unreachable(
|
||||
monkeypatch: pytest.MonkeyPatch, location: str) -> None:
|
||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable):
|
||||
ats._check_robots(client, "fake:acme", {})
|
||||
|
||||
|
||||
def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
||||
pytest.raises(ats._Refused):
|
||||
ats._get(client, ROBOTS, "fake", "acme")
|
||||
|
||||
|
||||
def test_fetch_board_honours_robots_txt(
|
||||
|
||||
Reference in new issue
Block a user