Agent: Befunde aus dem Review der Browser-Schleife behoben
This commit is contained in:
10 files changed
+976
-74
No files matched your search
+120
-39
@@ -1,10 +1,15 @@
|
||||
"""Filtering forward proxy for one case: every request of the browser, redirect hops included, passes url_ok here.
|
||||
"""Filtering forward proxy for one case: every request of the browser, redirect hops included, is checked here.
|
||||
|
||||
Playwright's route handler never sees the hops of a redirect (Chromium follows them itself), so the route gate
|
||||
alone cannot keep the browser away from private addresses. The browser context is pointed at this proxy, which
|
||||
checks each plain-HTTP request by its full URL and each CONNECT tunnel (HTTPS, WebSocket) by its host and port.
|
||||
checks each plain-HTTP request by its full URL and each CONNECT tunnel (HTTPS, WebSocket) by host and port.
|
||||
|
||||
The check is addresses_for(url) -> list of IP addresses: the addresses the host may be reached at, or an empty
|
||||
list if the URL is refused. The proxy connects only to those addresses and never resolves the name itself, so a
|
||||
DNS answer that changes between check and connection (DNS rebinding) cannot redirect it.
|
||||
"""
|
||||
|
||||
import re
|
||||
import socket
|
||||
import socketserver
|
||||
import threading
|
||||
@@ -14,6 +19,12 @@ MAX_HEADER_LINES = 100
|
||||
MAX_LINE = 16384
|
||||
HOP_BY_HOP = {"connection", "proxy-connection", "keep-alive", "proxy-authorization", "proxy-authenticate", "te",
|
||||
"trailer", "upgrade"}
|
||||
_HOST = re.compile(r"[A-Za-z0-9._-]+")
|
||||
_PORT = re.compile(r"[0-9]{1,5}")
|
||||
|
||||
|
||||
def _connect(address: tuple[str, int], timeout: float) -> socket.socket:
|
||||
return socket.create_connection(address, timeout=timeout)
|
||||
|
||||
|
||||
def _read_head(rfile) -> list[bytes] | None:
|
||||
@@ -32,17 +43,64 @@ def _headers(lines: list[bytes]) -> list[tuple[str, str]]:
|
||||
out = []
|
||||
for line in lines:
|
||||
name, sep, value = line.decode("latin-1").partition(":")
|
||||
if not sep:
|
||||
if not sep or not name or name != name.strip():
|
||||
raise ValueError("bad header line")
|
||||
out.append((name.strip(), value.strip()))
|
||||
out.append((name, value.strip()))
|
||||
return out
|
||||
|
||||
|
||||
def _keep(headers: list[tuple[str, str]]) -> list[tuple[str, str]]:
|
||||
"""Headers without the hop-by-hop ones, including those the Connection header names."""
|
||||
named = {token.strip().lower() for k, v in headers if k.lower() == "connection" for token in v.split(",")}
|
||||
return [(k, v) for k, v in headers if k.lower() not in HOP_BY_HOP and k.lower() not in named]
|
||||
|
||||
|
||||
def _port(text: str) -> int | None:
|
||||
if not _PORT.fullmatch(text):
|
||||
return None
|
||||
port = int(text)
|
||||
return port if 0 < port < 65536 else None
|
||||
|
||||
|
||||
def _authority(target: str) -> tuple[str, int] | None:
|
||||
"""Host and port of a CONNECT target: host:port or [ipv6]:port, nothing else (no userinfo, no default port)."""
|
||||
if target.startswith("["):
|
||||
host, sep, port = target[1:].partition("]:")
|
||||
if not sep or not host or not all(c in "0123456789abcdefABCDEF:." for c in host):
|
||||
return None
|
||||
else:
|
||||
host, sep, port = target.rpartition(":")
|
||||
if not sep or not _HOST.fullmatch(host):
|
||||
return None
|
||||
number = _port(port)
|
||||
return (host, number) if number else None
|
||||
|
||||
|
||||
def _absolute(target: str) -> tuple[str, int, str] | None:
|
||||
"""Host, port and origin-form path of an absolute http:// request target."""
|
||||
try:
|
||||
parts = urlsplit(target)
|
||||
port = parts.port
|
||||
except ValueError:
|
||||
return None
|
||||
if parts.scheme != "http" or not parts.hostname or "@" in parts.netloc or port == 0:
|
||||
return None
|
||||
path = (parts.path or "/") + (f"?{parts.query}" if parts.query else "")
|
||||
return parts.hostname, port or 80, path
|
||||
|
||||
|
||||
def _content_length(headers: list[tuple[str, str]]) -> int | None:
|
||||
"""The request body length; None if the request must be refused (chunked, duplicate or bad length)."""
|
||||
if any(k.lower() == "transfer-encoding" for k, _ in headers):
|
||||
return None
|
||||
values = [v for k, v in headers if k.lower() == "content-length"]
|
||||
if not values:
|
||||
return 0
|
||||
if len(values) > 1 or not values[0].isdigit():
|
||||
return None
|
||||
return int(values[0])
|
||||
|
||||
|
||||
def _pump(src, dst) -> None:
|
||||
"""Copy bytes from src (socket or buffered file) to dst until src ends or either side fails."""
|
||||
try:
|
||||
@@ -61,6 +119,9 @@ def _pump(src, dst) -> None:
|
||||
|
||||
|
||||
class _Handler(socketserver.StreamRequestHandler):
|
||||
"""One client connection carries exactly one request (or one tunnel); then both sides close. A kept-alive
|
||||
connection could otherwise carry the browser's next request, meant for another host, to this upstream."""
|
||||
|
||||
def handle(self):
|
||||
proxy = self.server.egress
|
||||
self.connection.settimeout(proxy.timeout)
|
||||
@@ -73,40 +134,53 @@ class _Handler(socketserver.StreamRequestHandler):
|
||||
except ValueError:
|
||||
return
|
||||
if method == "CONNECT":
|
||||
host, _, port = target.rpartition(":")
|
||||
url = f"https://{host}:{port}/"
|
||||
else:
|
||||
url = target
|
||||
parts = urlsplit(target)
|
||||
host, port = parts.hostname, parts.port or 80
|
||||
if parts.scheme != "http" or not host:
|
||||
authority = _authority(target)
|
||||
if not authority:
|
||||
return
|
||||
host = f"[{host}]" if ":" in host else host
|
||||
if not proxy.url_ok(url):
|
||||
host, port = authority
|
||||
url = f"https://[{host}]:{port}/" if ":" in host else f"https://{host}:{port}/"
|
||||
length = path = None
|
||||
else:
|
||||
absolute = _absolute(target)
|
||||
length = _content_length(headers)
|
||||
if not absolute or length is None or not version.startswith("HTTP/1."):
|
||||
return
|
||||
host, port, path = absolute
|
||||
url = target
|
||||
try:
|
||||
addresses = list(proxy.addresses_for(url))
|
||||
except Exception:
|
||||
addresses = []
|
||||
if not addresses:
|
||||
proxy.blocked.append(url)
|
||||
return # no answer at all: the browser sees a failed request, as with a blocked route
|
||||
try:
|
||||
upstream = socket.create_connection((host.strip("[]"), int(port)), timeout=proxy.timeout)
|
||||
except (OSError, ValueError):
|
||||
upstream = self._open(addresses, port, proxy.timeout)
|
||||
if upstream is None:
|
||||
return
|
||||
with upstream:
|
||||
if method == "CONNECT":
|
||||
self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
|
||||
self.wfile.flush()
|
||||
t = threading.Thread(target=_pump, args=(self.rfile, upstream), daemon=True)
|
||||
t.start()
|
||||
_pump(upstream, self.connection)
|
||||
t.join(proxy.timeout)
|
||||
return
|
||||
self._forward(upstream, method, parts, version, headers)
|
||||
self._tunnel(upstream, proxy.timeout)
|
||||
else:
|
||||
self._forward(upstream, method, path, version, headers, length)
|
||||
|
||||
def _forward(self, upstream, method, parts, version, headers):
|
||||
"""One plain-HTTP request, then both connections close: a kept-alive connection could carry the next
|
||||
request of the browser, meant for another host, to this upstream without a check."""
|
||||
if any(k.lower() == "transfer-encoding" for k, _ in headers):
|
||||
return # chunked request bodies are not supported; the request fails
|
||||
length = next((int(v) for k, v in headers if k.lower() == "content-length" and v.isdigit()), 0)
|
||||
path = (parts.path or "/") + (f"?{parts.query}" if parts.query else "")
|
||||
@staticmethod
|
||||
def _open(addresses: list[str], port: int, timeout: float) -> socket.socket | None:
|
||||
for address in addresses:
|
||||
try:
|
||||
return _connect((address, port), timeout)
|
||||
except OSError:
|
||||
continue
|
||||
return None
|
||||
|
||||
def _tunnel(self, upstream, timeout):
|
||||
self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
|
||||
self.wfile.flush()
|
||||
t = threading.Thread(target=_pump, args=(self.rfile, upstream), daemon=True)
|
||||
t.start()
|
||||
_pump(upstream, self.connection)
|
||||
t.join(timeout)
|
||||
|
||||
def _forward(self, upstream, method, path, version, headers, length):
|
||||
lines = [f"{method} {path} {version}"] + [f"{k}: {v}" for k, v in _keep(headers)] + ["Connection: close"]
|
||||
upstream.sendall(("\r\n".join(lines) + "\r\n\r\n").encode("latin-1"))
|
||||
while length > 0:
|
||||
@@ -116,14 +190,21 @@ class _Handler(socketserver.StreamRequestHandler):
|
||||
upstream.sendall(chunk)
|
||||
length -= len(chunk)
|
||||
ufile = upstream.makefile("rb")
|
||||
head = _read_head(ufile)
|
||||
if not head:
|
||||
return
|
||||
while True:
|
||||
head = _read_head(ufile)
|
||||
if not head:
|
||||
return
|
||||
status = head[0].decode("latin-1")
|
||||
code = status.split(" ")[1] if status.count(" ") else ""
|
||||
if code == "101":
|
||||
return # no protocol switches: the Upgrade header was not passed on, so this is not ours
|
||||
if not (len(code) == 3 and code.startswith("1")):
|
||||
break # interim 1xx answers (100 Continue, 103 Early Hints) are dropped; the final one counts
|
||||
try:
|
||||
status, response_headers = head[0], _keep(_headers(head[1:]))
|
||||
response_headers = _keep(_headers(head[1:]))
|
||||
except ValueError:
|
||||
return
|
||||
out = [status.decode("latin-1")] + [f"{k}: {v}" for k, v in response_headers] + ["Connection: close"]
|
||||
out = [status] + [f"{k}: {v}" for k, v in response_headers] + ["Connection: close"]
|
||||
self.wfile.write(("\r\n".join(out) + "\r\n\r\n").encode("latin-1"))
|
||||
self.wfile.flush()
|
||||
_pump(ufile, self.connection)
|
||||
@@ -138,10 +219,10 @@ class _Server(socketserver.ThreadingTCPServer):
|
||||
|
||||
|
||||
class EgressProxy:
|
||||
"""Proxy on 127.0.0.1 with a random port, between start() and stop(). Blocked URLs are appended to blocked."""
|
||||
"""Proxy on 127.0.0.1 with a random port, between start() and stop(). Refused URLs are appended to blocked."""
|
||||
|
||||
def __init__(self, url_ok, blocked: list, timeout: float = 30):
|
||||
self.url_ok = url_ok
|
||||
def __init__(self, addresses_for, blocked: list, timeout: float = 30):
|
||||
self.addresses_for = addresses_for
|
||||
self.blocked = blocked
|
||||
self.timeout = timeout
|
||||
self._server = _Server(("127.0.0.1", 0), _Handler)
|
||||
|
||||
Reference in new issue
Block a user