Refuse a fetch to another port than 443, an IP address or a local host name, at every hop
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f61caddced
commit
60136765b2
2 files changed
+71
-2
No files matched your search
+27
-2
@@ -29,6 +29,7 @@ with the whole board at once is complete whenever it parses.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
@@ -123,10 +124,33 @@ class _RobotsUnreachable(Exception):
|
||||
the site disallows everything then, so the board is not read this pull."""
|
||||
|
||||
|
||||
#: Host name suffixes that name a machine on a private network, never a
|
||||
#: company's careers site.
|
||||
_LOCAL_SUFFIXES = (".localhost", ".local", ".internal", ".lan", ".home.arpa")
|
||||
|
||||
|
||||
def _local_host(name: str) -> bool:
|
||||
"""Whether a host name is an address or a local name rather than a public
|
||||
DNS name: an IP literal (v4 or v6), anything whose last label is a number
|
||||
(`2130706433`, `127.1` and `0x7f000001` all reach 127.0.0.1 through the
|
||||
resolver), `localhost`, or a private suffix. A Teamtailor slug can be any
|
||||
host name, so the allow-list alone would let one point the fetch here."""
|
||||
name = name.strip("[]").rstrip(".").lower()
|
||||
try:
|
||||
ipaddress.ip_address(name)
|
||||
return True
|
||||
except ValueError:
|
||||
pass
|
||||
last = name.rsplit(".", 1)[-1]
|
||||
return (last.isdigit() or bool(re.fullmatch(r"0x[0-9a-f]+", last))
|
||||
or name == "localhost" or name.endswith(_LOCAL_SUFFIXES))
|
||||
|
||||
|
||||
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
method: str = "GET", payload: dict[str, Any] | None = None,
|
||||
headers: dict[str, str] | None = None, www_twin: bool = False) -> bytes:
|
||||
"""Fetch one URL for a board: https only, the host's own allow-list, at
|
||||
"""Fetch one URL for a board: https only on the default port, never an
|
||||
address or a local name (`_local_host`), the host's own allow-list, at
|
||||
most three redirect hops each checked the same way, and the body cut off
|
||||
at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
|
||||
sends `payload` as JSON, `{}` when there is none. `headers` are sent on
|
||||
@@ -151,7 +175,8 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
current = url
|
||||
for _hop in range(4):
|
||||
parsed = httpx.URL(current)
|
||||
if parsed.scheme != "https" or not allow(parsed.host):
|
||||
if (parsed.scheme != "https" or parsed.port not in (None, 443)
|
||||
or _local_host(parsed.host) or not allow(parsed.host)):
|
||||
raise _Refused(f"{current} is not an allowed host for {host}")
|
||||
with client.stream(
|
||||
method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent,
|
||||
|
||||
@@ -156,6 +156,50 @@ def test_get_sends_extra_headers_on_top_of_its_own() -> None:
|
||||
assert "x-board" not in seen[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("site", [
|
||||
"metadata.google.internal", "printer.local", "router.lan", "nas.home.arpa",
|
||||
"app.localhost", "foo.0x7f000001", "printer.local.",
|
||||
])
|
||||
def test_get_refuses_a_local_name_before_any_request(site: str) -> None:
|
||||
"""A Teamtailor slug may be any host name, so the allow-list admits
|
||||
these; `_get` refuses them before a request is made."""
|
||||
asked: list[str] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
asked.append(str(request.url))
|
||||
return httpx.Response(200, content=b"<rss><channel/></rss>")
|
||||
|
||||
with _client(handler) as client:
|
||||
# A trailing dot is no Teamtailor slug at all; the rest are, and are refused.
|
||||
with pytest.raises(ValueError if site.endswith(".") else ats._Refused):
|
||||
ats._fetch_rows(client, f"teamtailor:{site}")
|
||||
for url in (f"https://{site}/jobs.rss", "https://127.0.0.1/", "https://[::1]/",
|
||||
"https://2130706433/", "https://127.1/", "https://localhost/"):
|
||||
with pytest.raises(ats._Refused):
|
||||
ats._get(client, url, "teamtailor", site)
|
||||
assert asked == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("location", [
|
||||
"https://api.lever.co:8443/x", "https://127.0.0.1/x", "https://[::1]/x",
|
||||
"https://localhost/x",
|
||||
])
|
||||
def test_get_refuses_a_redirect_to_another_port_or_a_local_host(location: str) -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(302, headers={"location": location})
|
||||
|
||||
with _client(handler) as client, pytest.raises(ats._Refused):
|
||||
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
|
||||
|
||||
|
||||
def test_get_takes_port_443_written_out_and_refuses_another() -> None:
|
||||
with _client(lambda request: httpx.Response(200, content=b"[]")) as client:
|
||||
assert ats._get(client, "https://api.lever.co:443/v0/postings/acme",
|
||||
"lever", "acme") == b"[]"
|
||||
with pytest.raises(ats._Refused):
|
||||
ats._get(client, "https://api.lever.co:8443/v0/postings/acme", "lever", "acme")
|
||||
|
||||
|
||||
def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Bound before the replacement, and still reaching it: `_get` is looked
|
||||
up when a request is made."""
|
||||
|
||||
Reference in new issue
Block a user