Refuse a fetch to another port than 443, an IP address or a local host name, at every hop
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f61caddced
commit
60136765b2
2 files changed
+71
-2
No files matched your search
+27
-2
@@ -29,6 +29,7 @@ with the whole board at once is complete whenever it parses.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import ipaddress
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
@@ -123,10 +124,33 @@ class _RobotsUnreachable(Exception):
|
|||||||
the site disallows everything then, so the board is not read this pull."""
|
the site disallows everything then, so the board is not read this pull."""
|
||||||
|
|
||||||
|
|
||||||
|
#: Host name suffixes that name a machine on a private network, never a
|
||||||
|
#: company's careers site.
|
||||||
|
_LOCAL_SUFFIXES = (".localhost", ".local", ".internal", ".lan", ".home.arpa")
|
||||||
|
|
||||||
|
|
||||||
|
def _local_host(name: str) -> bool:
|
||||||
|
"""Whether a host name is an address or a local name rather than a public
|
||||||
|
DNS name: an IP literal (v4 or v6), anything whose last label is a number
|
||||||
|
(`2130706433`, `127.1` and `0x7f000001` all reach 127.0.0.1 through the
|
||||||
|
resolver), `localhost`, or a private suffix. A Teamtailor slug can be any
|
||||||
|
host name, so the allow-list alone would let one point the fetch here."""
|
||||||
|
name = name.strip("[]").rstrip(".").lower()
|
||||||
|
try:
|
||||||
|
ipaddress.ip_address(name)
|
||||||
|
return True
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
last = name.rsplit(".", 1)[-1]
|
||||||
|
return (last.isdigit() or bool(re.fullmatch(r"0x[0-9a-f]+", last))
|
||||||
|
or name == "localhost" or name.endswith(_LOCAL_SUFFIXES))
|
||||||
|
|
||||||
|
|
||||||
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||||
method: str = "GET", payload: dict[str, Any] | None = None,
|
method: str = "GET", payload: dict[str, Any] | None = None,
|
||||||
headers: dict[str, str] | None = None, www_twin: bool = False) -> bytes:
|
headers: dict[str, str] | None = None, www_twin: bool = False) -> bytes:
|
||||||
"""Fetch one URL for a board: https only, the host's own allow-list, at
|
"""Fetch one URL for a board: https only on the default port, never an
|
||||||
|
address or a local name (`_local_host`), the host's own allow-list, at
|
||||||
most three redirect hops each checked the same way, and the body cut off
|
most three redirect hops each checked the same way, and the body cut off
|
||||||
at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
|
at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
|
||||||
sends `payload` as JSON, `{}` when there is none. `headers` are sent on
|
sends `payload` as JSON, `{}` when there is none. `headers` are sent on
|
||||||
@@ -151,7 +175,8 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
|||||||
current = url
|
current = url
|
||||||
for _hop in range(4):
|
for _hop in range(4):
|
||||||
parsed = httpx.URL(current)
|
parsed = httpx.URL(current)
|
||||||
if parsed.scheme != "https" or not allow(parsed.host):
|
if (parsed.scheme != "https" or parsed.port not in (None, 443)
|
||||||
|
or _local_host(parsed.host) or not allow(parsed.host)):
|
||||||
raise _Refused(f"{current} is not an allowed host for {host}")
|
raise _Refused(f"{current} is not an allowed host for {host}")
|
||||||
with client.stream(
|
with client.stream(
|
||||||
method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent,
|
method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent,
|
||||||
|
|||||||
@@ -156,6 +156,50 @@ def test_get_sends_extra_headers_on_top_of_its_own() -> None:
|
|||||||
assert "x-board" not in seen[1]
|
assert "x-board" not in seen[1]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("site", [
|
||||||
|
"metadata.google.internal", "printer.local", "router.lan", "nas.home.arpa",
|
||||||
|
"app.localhost", "foo.0x7f000001", "printer.local.",
|
||||||
|
])
|
||||||
|
def test_get_refuses_a_local_name_before_any_request(site: str) -> None:
|
||||||
|
"""A Teamtailor slug may be any host name, so the allow-list admits
|
||||||
|
these; `_get` refuses them before a request is made."""
|
||||||
|
asked: list[str] = []
|
||||||
|
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
asked.append(str(request.url))
|
||||||
|
return httpx.Response(200, content=b"<rss><channel/></rss>")
|
||||||
|
|
||||||
|
with _client(handler) as client:
|
||||||
|
# A trailing dot is no Teamtailor slug at all; the rest are, and are refused.
|
||||||
|
with pytest.raises(ValueError if site.endswith(".") else ats._Refused):
|
||||||
|
ats._fetch_rows(client, f"teamtailor:{site}")
|
||||||
|
for url in (f"https://{site}/jobs.rss", "https://127.0.0.1/", "https://[::1]/",
|
||||||
|
"https://2130706433/", "https://127.1/", "https://localhost/"):
|
||||||
|
with pytest.raises(ats._Refused):
|
||||||
|
ats._get(client, url, "teamtailor", site)
|
||||||
|
assert asked == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("location", [
|
||||||
|
"https://api.lever.co:8443/x", "https://127.0.0.1/x", "https://[::1]/x",
|
||||||
|
"https://localhost/x",
|
||||||
|
])
|
||||||
|
def test_get_refuses_a_redirect_to_another_port_or_a_local_host(location: str) -> None:
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
return httpx.Response(302, headers={"location": location})
|
||||||
|
|
||||||
|
with _client(handler) as client, pytest.raises(ats._Refused):
|
||||||
|
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_takes_port_443_written_out_and_refuses_another() -> None:
|
||||||
|
with _client(lambda request: httpx.Response(200, content=b"[]")) as client:
|
||||||
|
assert ats._get(client, "https://api.lever.co:443/v0/postings/acme",
|
||||||
|
"lever", "acme") == b"[]"
|
||||||
|
with pytest.raises(ats._Refused):
|
||||||
|
ats._get(client, "https://api.lever.co:8443/v0/postings/acme", "lever", "acme")
|
||||||
|
|
||||||
|
|
||||||
def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatch) -> None:
|
def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
"""Bound before the replacement, and still reaching it: `_get` is looked
|
"""Bound before the replacement, and still reaching it: `_get` is looked
|
||||||
up when a request is made."""
|
up when a request is made."""
|
||||||
|
|||||||
Reference in new issue
Block a user