Refuse a fetch to another port than 443, an IP address or a local host name, at every hop

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-29 19:42:58 +02:00
1 parent f61caddced
commit 60136765b2
2 files changed
+71 -2

No files matched your search

+27 -2
View File
@@ -29,6 +29,7 @@ with the whole board at once is complete whenever it parses.
from __future__ import annotations
import ipaddress
import json
import logging
import re
@@ -123,10 +124,33 @@ class _RobotsUnreachable(Exception):
the site disallows everything then, so the board is not read this pull."""
#: Host name suffixes that name a machine on a private network, never a
#: company's careers site.
_LOCAL_SUFFIXES = (".localhost", ".local", ".internal", ".lan", ".home.arpa")
def _local_host(name: str) -> bool:
"""Whether a host name is an address or a local name rather than a public
DNS name: an IP literal (v4 or v6), anything whose last label is a number
(`2130706433`, `127.1` and `0x7f000001` all reach 127.0.0.1 through the
resolver), `localhost`, or a private suffix. A Teamtailor slug can be any
host name, so the allow-list alone would let one point the fetch here."""
name = name.strip("[]").rstrip(".").lower()
try:
ipaddress.ip_address(name)
return True
except ValueError:
pass
last = name.rsplit(".", 1)[-1]
return (last.isdigit() or bool(re.fullmatch(r"0x[0-9a-f]+", last))
or name == "localhost" or name.endswith(_LOCAL_SUFFIXES))
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None, www_twin: bool = False) -> bytes:
"""Fetch one URL for a board: https only, the host's own allow-list, at
"""Fetch one URL for a board: https only on the default port, never an
address or a local name (`_local_host`), the host's own allow-list, at
most three redirect hops each checked the same way, and the body cut off
at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
sends `payload` as JSON, `{}` when there is none. `headers` are sent on
@@ -151,7 +175,8 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *,
current = url
for _hop in range(4):
parsed = httpx.URL(current)
if parsed.scheme != "https" or not allow(parsed.host):
if (parsed.scheme != "https" or parsed.port not in (None, 443)
or _local_host(parsed.host) or not allow(parsed.host)):
raise _Refused(f"{current} is not an allowed host for {host}")
with client.stream(
method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent,
+44
View File
@@ -156,6 +156,50 @@ def test_get_sends_extra_headers_on_top_of_its_own() -> None:
assert "x-board" not in seen[1]
@pytest.mark.parametrize("site", [
"metadata.google.internal", "printer.local", "router.lan", "nas.home.arpa",
"app.localhost", "foo.0x7f000001", "printer.local.",
])
def test_get_refuses_a_local_name_before_any_request(site: str) -> None:
"""A Teamtailor slug may be any host name, so the allow-list admits
these; `_get` refuses them before a request is made."""
asked: list[str] = []
def handler(request: httpx.Request) -> httpx.Response:
asked.append(str(request.url))
return httpx.Response(200, content=b"<rss><channel/></rss>")
with _client(handler) as client:
# A trailing dot is no Teamtailor slug at all; the rest are, and are refused.
with pytest.raises(ValueError if site.endswith(".") else ats._Refused):
ats._fetch_rows(client, f"teamtailor:{site}")
for url in (f"https://{site}/jobs.rss", "https://127.0.0.1/", "https://[::1]/",
"https://2130706433/", "https://127.1/", "https://localhost/"):
with pytest.raises(ats._Refused):
ats._get(client, url, "teamtailor", site)
assert asked == []
@pytest.mark.parametrize("location", [
"https://api.lever.co:8443/x", "https://127.0.0.1/x", "https://[::1]/x",
"https://localhost/x",
])
def test_get_refuses_a_redirect_to_another_port_or_a_local_host(location: str) -> None:
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(302, headers={"location": location})
with _client(handler) as client, pytest.raises(ats._Refused):
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
def test_get_takes_port_443_written_out_and_refuses_another() -> None:
with _client(lambda request: httpx.Response(200, content=b"[]")) as client:
assert ats._get(client, "https://api.lever.co:443/v0/postings/acme",
"lever", "acme") == b"[]"
with pytest.raises(ats._Refused):
ats._get(client, "https://api.lever.co:8443/v0/postings/acme", "lever", "acme")
def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatch) -> None:
"""Bound before the replacement, and still reaching it: `_get` is looked
up when a request is made."""