347 lines
14 KiB
Python
347 lines
14 KiB
Python
"""The egress proxy with raw sockets against local test servers: what it forwards, where, and what it refuses."""
|
|
|
|
import socket
|
|
import threading
|
|
import time
|
|
|
|
import pytest
|
|
|
|
import egress
|
|
import guard
|
|
from egress import EgressProxy
|
|
|
|
OK_REPLY = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"
|
|
|
|
|
|
class Upstream:
|
|
"""Raw TCP server on 127.0.0.1. Records the bytes of every accepted connection; answers each request head with
|
|
a canned reply (or echoes everything back), then keeps reading for a moment so stray bytes are recorded too."""
|
|
|
|
def __init__(self, reply=OK_REPLY, echo=False):
|
|
self.reply, self.echo = reply, echo
|
|
self.connections: list[bytearray] = []
|
|
self.sock = socket.socket()
|
|
self.sock.bind(("127.0.0.1", 0))
|
|
self.sock.listen()
|
|
self.port = self.sock.getsockname()[1]
|
|
threading.Thread(target=self._accept, daemon=True).start()
|
|
|
|
def _accept(self):
|
|
while True:
|
|
try:
|
|
conn, _ = self.sock.accept()
|
|
except OSError:
|
|
return
|
|
data = bytearray()
|
|
self.connections.append(data)
|
|
threading.Thread(target=self._serve, args=(conn, data), daemon=True).start()
|
|
|
|
def _serve(self, conn, data):
|
|
conn.settimeout(1)
|
|
answered = False
|
|
with conn:
|
|
try:
|
|
while True:
|
|
chunk = conn.recv(65536)
|
|
if not chunk:
|
|
return
|
|
data += chunk
|
|
if self.echo:
|
|
conn.sendall(chunk)
|
|
elif not answered and b"\r\n\r\n" in data:
|
|
conn.sendall(self.reply)
|
|
answered = True
|
|
except OSError:
|
|
return
|
|
|
|
def close(self):
|
|
self.sock.close()
|
|
|
|
|
|
@pytest.fixture
|
|
def upstreams():
|
|
made = []
|
|
|
|
def make(**kw):
|
|
u = Upstream(**kw)
|
|
made.append(u)
|
|
return u
|
|
yield make
|
|
for u in made:
|
|
u.close()
|
|
|
|
|
|
@pytest.fixture
|
|
def proxy_for():
|
|
made = []
|
|
|
|
def make(addresses_for):
|
|
p = EgressProxy(addresses_for, [], timeout=3).start()
|
|
made.append(p)
|
|
return p
|
|
yield make
|
|
for p in made:
|
|
p.stop()
|
|
|
|
|
|
def allow(*urls_to_address):
|
|
"""addresses_for that allows exactly the given (url prefix, address) pairs and records every call."""
|
|
calls = []
|
|
|
|
def addresses_for(url):
|
|
calls.append(url)
|
|
for prefix, address in urls_to_address:
|
|
if url.startswith(prefix):
|
|
return [address]
|
|
return []
|
|
addresses_for.calls = calls
|
|
return addresses_for
|
|
|
|
|
|
def exchange(proxy, data: bytes, wait: float = 3.0) -> bytes:
|
|
"""Send data to the proxy and read until it closes the connection (or wait runs out)."""
|
|
port = int(proxy.address.rsplit(":", 1)[1])
|
|
with socket.create_connection(("127.0.0.1", port)) as s:
|
|
s.settimeout(wait)
|
|
s.sendall(data)
|
|
out = bytearray()
|
|
try:
|
|
while True:
|
|
chunk = s.recv(65536)
|
|
if not chunk:
|
|
break
|
|
out += chunk
|
|
except TimeoutError:
|
|
pass
|
|
return bytes(out)
|
|
|
|
|
|
def settle():
|
|
time.sleep(0.3) # let the upstream threads record what they got
|
|
|
|
|
|
# --- DNS rebinding: the proxy connects only to the address the check returned ---------------------------------
|
|
|
|
def test_proxy_connects_to_the_checked_address_and_never_resolves_the_name(upstreams, proxy_for):
|
|
up = upstreams()
|
|
addresses_for = allow((f"http://never-resolvable.invalid:{up.port}/", "127.0.0.1"))
|
|
reply = exchange(proxy_for(addresses_for),
|
|
f"GET http://never-resolvable.invalid:{up.port}/x HTTP/1.1\r\nHost: never-resolvable.invalid\r\n\r\n"
|
|
.encode())
|
|
assert reply.startswith(b"HTTP/1.1 200 OK") and reply.endswith(b"ok")
|
|
settle()
|
|
assert len(up.connections) == 1 and up.connections[0].startswith(b"GET /x HTTP/1.1\r\n")
|
|
|
|
|
|
def test_rebinding_resolver_cannot_redirect_the_connection(monkeypatch, proxy_for):
|
|
answers = iter([["93.184.216.34"]])
|
|
lookups = []
|
|
|
|
def flip(host): # public on the first lookup, loopback on every later one
|
|
lookups.append(host)
|
|
return next(answers, ["127.0.0.1"])
|
|
|
|
connects = []
|
|
|
|
def fake_connect(address, timeout=None, *args, **kw):
|
|
connects.append(address)
|
|
raise OSError("no network in this test")
|
|
|
|
real_getaddrinfo = socket.getaddrinfo
|
|
name_lookups = []
|
|
|
|
def spy_getaddrinfo(host, *a, **kw):
|
|
name_lookups.append(host)
|
|
return real_getaddrinfo(host, *a, **kw)
|
|
|
|
monkeypatch.setattr(egress, "_connect", fake_connect)
|
|
monkeypatch.setattr(socket, "getaddrinfo", spy_getaddrinfo)
|
|
proxy = proxy_for(lambda url: guard.public_addresses(url, resolve=flip, blocked=()))
|
|
assert exchange(proxy, b"GET http://rebind.example/ HTTP/1.1\r\nHost: rebind.example\r\n\r\n") == b""
|
|
assert lookups == ["rebind.example"] # one check, one lookup
|
|
assert connects == [("93.184.216.34", 80)] # to the vetted address, not to the name
|
|
assert "rebind.example" not in name_lookups
|
|
|
|
|
|
def test_next_checked_address_is_tried_when_the_first_fails(upstreams, proxy_for):
|
|
up = upstreams() # listens on 127.0.0.1 only: 127.0.0.2 on the same port refuses
|
|
|
|
def addresses_for(url):
|
|
return ["127.0.0.2", "127.0.0.1"]
|
|
reply = exchange(proxy_for(addresses_for), f"GET http://two.example:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode())
|
|
assert reply.endswith(b"ok")
|
|
|
|
|
|
def test_denied_url_makes_no_upstream_connection_and_gets_no_answer(upstreams, proxy_for):
|
|
up = upstreams()
|
|
proxy = proxy_for(lambda url: [])
|
|
assert exchange(proxy, f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode()) == b""
|
|
settle()
|
|
assert up.connections == [] and proxy.blocked == [f"http://127.0.0.1:{up.port}/"]
|
|
|
|
|
|
def test_failing_check_counts_as_denied(upstreams, proxy_for):
|
|
up = upstreams()
|
|
|
|
def broken(url):
|
|
raise RuntimeError("resolver crashed")
|
|
assert exchange(proxy_for(broken), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode()) == b""
|
|
settle()
|
|
assert up.connections == []
|
|
|
|
|
|
# --- CONNECT ------------------------------------------------------------------------------------------------
|
|
|
|
def test_connect_allowed_tunnels_bytes(upstreams, proxy_for):
|
|
up = upstreams(echo=True)
|
|
addresses_for = allow((f"https://tunnel.example:{up.port}/", "127.0.0.1"))
|
|
port = int(proxy_for(addresses_for).address.rsplit(":", 1)[1])
|
|
with socket.create_connection(("127.0.0.1", port)) as s:
|
|
s.settimeout(3)
|
|
s.sendall(f"CONNECT tunnel.example:{up.port} HTTP/1.1\r\nHost: tunnel.example:{up.port}\r\n\r\n".encode())
|
|
head = s.recv(1024)
|
|
assert head == b"HTTP/1.1 200 Connection established\r\n\r\n"
|
|
s.sendall(b"opaque tls bytes")
|
|
assert s.recv(1024) == b"opaque tls bytes"
|
|
assert addresses_for.calls == [f"https://tunnel.example:{up.port}/"]
|
|
|
|
|
|
def test_connect_denied_sends_nothing_and_connects_nowhere(upstreams, proxy_for):
|
|
up = upstreams(echo=True)
|
|
proxy = proxy_for(lambda url: [])
|
|
assert exchange(proxy, f"CONNECT 127.0.0.1:{up.port} HTTP/1.1\r\n\r\n".encode()) == b""
|
|
settle()
|
|
assert up.connections == []
|
|
|
|
|
|
@pytest.mark.parametrize("authority", ["tunnel.example", "tunnel.example:abc", "tunnel.example:0",
|
|
"tunnel.example:70000", "user@tunnel.example:443", "user:pw@tunnel.example:443",
|
|
":443", "[::1", "::1:443", "tunnel.example:+443", "tunnel.example: 443"])
|
|
def test_malformed_connect_is_refused_without_a_check(proxy_for, authority):
|
|
addresses_for = allow(("https://", "127.0.0.1"))
|
|
assert exchange(proxy_for(addresses_for), f"CONNECT {authority} HTTP/1.1\r\n\r\n".encode()) == b""
|
|
assert addresses_for.calls == []
|
|
|
|
|
|
def test_connect_to_bracketed_ipv6_literal_is_checked_with_brackets(proxy_for):
|
|
addresses_for = allow()
|
|
assert exchange(proxy_for(addresses_for), b"CONNECT [::1]:443 HTTP/1.1\r\n\r\n") == b""
|
|
assert addresses_for.calls == ["https://[::1]:443/"]
|
|
|
|
|
|
# --- plain HTTP (absolute form) --------------------------------------------------------------------------------
|
|
|
|
def test_absolute_form_request_is_forwarded_in_origin_form(upstreams, proxy_for):
|
|
up = upstreams()
|
|
addresses_for = allow((f"http://site.example:{up.port}/", "127.0.0.1"))
|
|
reply = exchange(proxy_for(addresses_for),
|
|
f"POST http://site.example:{up.port}/u?x=1 HTTP/1.1\r\nHost: site.example:{up.port}\r\n"
|
|
"Proxy-Connection: keep-alive\r\nConnection: keep-alive, X-Secret\r\nX-Secret: 1\r\n"
|
|
"Content-Length: 5\r\n\r\nhello".encode())
|
|
assert reply.endswith(b"ok")
|
|
settle()
|
|
head, _, body = bytes(up.connections[0]).partition(b"\r\n\r\n")
|
|
lines = head.decode().split("\r\n")
|
|
assert lines[0] == "POST /u?x=1 HTTP/1.1" and body == b"hello"
|
|
names = [line.split(":")[0].lower() for line in lines[1:]]
|
|
assert "host" in names and "proxy-connection" not in names and "x-secret" not in names
|
|
assert "Connection: close" in lines and "Connection: keep-alive, X-Secret" not in lines
|
|
assert addresses_for.calls == [f"http://site.example:{up.port}/u?x=1"]
|
|
|
|
|
|
def test_absolute_form_denied(upstreams, proxy_for):
|
|
up = upstreams()
|
|
assert exchange(proxy_for(allow()), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\n\r\n".encode()) == b""
|
|
settle()
|
|
assert up.connections == []
|
|
|
|
|
|
@pytest.mark.parametrize("target", ["/relative", "https://site.example/", "ftp://site.example/", "http:///x",
|
|
"http://site.example:0/", "http://site.example:abc/", "http://u:p@site.example/"])
|
|
def test_other_request_targets_are_refused_without_a_check(proxy_for, target):
|
|
addresses_for = allow(("http://", "127.0.0.1"))
|
|
assert exchange(proxy_for(addresses_for), f"GET {target} HTTP/1.1\r\nHost: x\r\n\r\n".encode()) == b""
|
|
assert addresses_for.calls == []
|
|
|
|
|
|
def test_request_with_transfer_encoding_is_dropped(upstreams, proxy_for):
|
|
up = upstreams()
|
|
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
|
|
reply = exchange(proxy_for(addresses_for), f"POST http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n"
|
|
"Transfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n".encode())
|
|
settle()
|
|
assert reply == b"" and up.connections == []
|
|
|
|
|
|
@pytest.mark.parametrize("lengths", [("5", "5"), ("5", "7"), ("5, 5",), ("-1",), ("abc",)])
|
|
def test_duplicate_or_bad_content_length_is_rejected(upstreams, proxy_for, lengths):
|
|
up = upstreams()
|
|
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
|
|
headers = "".join(f"Content-Length: {v}\r\n" for v in lengths)
|
|
reply = exchange(proxy_for(addresses_for),
|
|
f"POST http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n{headers}\r\nhello".encode())
|
|
settle()
|
|
assert reply == b"" and up.connections == []
|
|
|
|
|
|
def test_pipelined_second_request_is_never_forwarded(upstreams, proxy_for):
|
|
a, b = upstreams(), upstreams()
|
|
addresses_for = allow((f"http://127.0.0.1:{a.port}/", "127.0.0.1"), (f"http://127.0.0.1:{b.port}/", "127.0.0.1"))
|
|
first = f"GET http://127.0.0.1:{a.port}/first HTTP/1.1\r\nHost: a\r\n\r\n"
|
|
second = f"GET http://127.0.0.1:{b.port}/second HTTP/1.1\r\nHost: b\r\n\r\n"
|
|
reply = exchange(proxy_for(addresses_for), (first + second).encode())
|
|
settle()
|
|
assert reply.count(b"HTTP/1.1 200 OK") == 1
|
|
assert b.connections == [] and len(a.connections) == 1
|
|
assert bytes(a.connections[0]).count(b" HTTP/1.1\r\n") == 1 and b"/second" not in a.connections[0]
|
|
|
|
|
|
def test_second_request_on_the_same_connection_is_never_forwarded(upstreams, proxy_for):
|
|
a, b = upstreams(), upstreams()
|
|
addresses_for = allow((f"http://127.0.0.1:{a.port}/", "127.0.0.1"), (f"http://127.0.0.1:{b.port}/", "127.0.0.1"))
|
|
port = int(proxy_for(addresses_for).address.rsplit(":", 1)[1])
|
|
with socket.create_connection(("127.0.0.1", port)) as s:
|
|
s.settimeout(3)
|
|
s.sendall(f"GET http://127.0.0.1:{a.port}/first HTTP/1.1\r\nHost: a\r\n\r\n".encode())
|
|
got = bytearray()
|
|
while not got.endswith(b"ok"):
|
|
got += s.recv(65536)
|
|
assert b"Connection: close" in got
|
|
try:
|
|
s.sendall(f"GET http://127.0.0.1:{b.port}/second HTTP/1.1\r\nHost: b\r\n\r\n".encode())
|
|
assert s.recv(65536) == b""
|
|
except (BrokenPipeError, ConnectionResetError):
|
|
pass
|
|
settle()
|
|
assert b.connections == [] and b"/second" not in a.connections[0]
|
|
|
|
|
|
def test_chunked_response_passes_through_intact(upstreams, proxy_for):
|
|
body = b"4\r\nWiki\r\n5\r\npedia\r\n0\r\n\r\n"
|
|
up = upstreams(reply=b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nContent-Type: text/plain\r\n\r\n" + body)
|
|
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
|
|
reply = exchange(proxy_for(addresses_for), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode())
|
|
head, _, rest = reply.partition(b"\r\n\r\n")
|
|
assert b"Transfer-Encoding: chunked" in head and rest == body
|
|
|
|
|
|
def test_interim_1xx_responses_are_skipped_and_hop_by_hop_headers_of_the_final_one_stripped(upstreams, proxy_for):
|
|
up = upstreams(reply=b"HTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 103 Early Hints\r\nLink: </a.css>\r\n\r\n"
|
|
b"HTTP/1.1 200 OK\r\nConnection: keep-alive, X-Hop\r\nKeep-Alive: timeout=5\r\nX-Hop: 1\r\n"
|
|
b"Proxy-Connection: keep-alive\r\nContent-Length: 2\r\n\r\nok")
|
|
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
|
|
reply = exchange(proxy_for(addresses_for), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode())
|
|
head, _, rest = reply.partition(b"\r\n\r\n")
|
|
lines = head.decode().split("\r\n")
|
|
assert lines[0] == "HTTP/1.1 200 OK" and rest == b"ok" and reply.count(b"HTTP/1.1 ") == 1
|
|
names = [line.split(":")[0].lower() for line in lines[1:]]
|
|
assert "keep-alive" not in names and "x-hop" not in names and "proxy-connection" not in names
|
|
assert lines.count("Connection: close") == 1 and names.count("connection") == 1
|
|
|
|
|
|
def test_switching_protocols_is_not_passed_on(upstreams, proxy_for):
|
|
up = upstreams(reply=b"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\nraw")
|
|
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
|
|
reply = exchange(proxy_for(addresses_for), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode())
|
|
assert reply == b""
|