Agent: Befunde aus dem Review der Browser-Schleife behoben

This commit is contained in:
Rsclub2_2 committed 2026-11-15 02:06:41 +01:00
1 parent 7e50983d85
commit 6556087cf2
10 files changed
+976 -74

No files matched your search

+2 -1
View File
@@ -4,5 +4,6 @@
set -eu
# --load: with a docker-container buildx builder the image would otherwise stay in the build cache only
docker build -q --load -t unsubscriber-agent-test agent >/dev/null
docker run --rm --network none --user pwuser -v "$PWD/agent/tests:/app/tests:ro" unsubscriber-agent-test \
# --init: python would be PID 1 and ignore SIGTERM, so a hanging run could not be stopped
docker run --init --rm --network none --user pwuser -v "$PWD/agent/tests:/app/tests:ro" unsubscriber-agent-test \
python -m pytest -q -p no:cacheprovider /app/tests
+59 -1
View File
@@ -1,6 +1,8 @@
"""Tiny web server with imitated unsubscribe pages for the browser tests."""
import html
import threading
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, urlsplit
@@ -27,6 +29,14 @@ PAGES = {
"/download": "<p>Bitte bestätigen</p><a href='/file.bin' download>Abmelden bestätigen</a>",
"/home": "<p>Welcome</p>",
"/bought": "<p>Thank you for your purchase</p>",
"/busy": "<p>Bitte bestätigen</p><button>Abmelden bestätigen</button>"
"<script>addEventListener('load', () => setTimeout(() => { while (true) {} }, 50))</script>",
"/rtc": "<p id='r'>?</p><button>Abmelden</button><script>document.getElementById('r').textContent ="
" 'rtc:' + typeof RTCPeerConnection + ',' + typeof webkitRTCPeerConnection</script>",
"/many-links-password": "".join(f"<a href='/home?{i}'>Link {i}</a> " for i in range(70)) +
"<form><input type='password' name='pw'><button>Abmelden</button></form>",
"/frame-password": "<p>Abmelden</p><iframe src='/pwframe'></iframe><button>Abmelden bestätigen</button>",
"/pwframe": "<form><input type='text' name='u'><input type='password' name='pw'></form>",
}
@@ -44,8 +54,31 @@ class Handler(BaseHTTPRequestHandler):
self.wfile.write(data)
def do_GET(self):
path = urlsplit(self.path).path
parts = urlsplit(self.path)
path, query = parts.path, parse_qs(parts.query)
Handler.log.append(("GET", path, {}))
if path == "/redirect":
return self._send("", 302, {"Location": query["to"][0]})
if path == "/img":
return self._send(f"<p>Bild</p><img src='{html.escape(query['src'][0])}'>")
if path == "/fetch":
target = html.escape(query["to"][0])
return self._send(f"<p>Skript</p><script>fetch('{target}', {{mode: 'no-cors'}}).catch(() => 0);"
f"navigator.sendBeacon('{target}', 'x')</script>")
if path == "/hang-img":
# loads a blocked image, then never finishes the document (a byte a second keeps the connection busy)
self.send_response(200)
self.send_header("Content-Type", "text/html; charset=utf-8")
self.end_headers()
self.wfile.write(b"<!doctype html><html><body><p>Laedt ...</p><img src='http://192.168.8.111/p.gif'>")
for _ in range(10):
try:
self.wfile.write(b" ")
self.wfile.flush()
except OSError:
return
time.sleep(1)
return
if path == "/to-private":
return self._send("", 302, {"Location": "http://192.168.8.111/admin"})
if path == "/file.bin":
@@ -78,3 +111,28 @@ def start() -> tuple[ThreadingHTTPServer, str]:
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
threading.Thread(target=server.serve_forever, daemon=True).start()
return server, f"http://127.0.0.1:{server.server_address[1]}"
def start_other() -> tuple[ThreadingHTTPServer, str, list]:
"""A second server on 127.0.0.1 with its own request log: the browser must never reach it."""
class Other(BaseHTTPRequestHandler):
log = []
def do_GET(self):
Other.log.append((self.command, self.path))
data = b"<p>other server</p>"
self.send_response(200)
self.send_header("Content-Type", "text/html")
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
do_POST = do_GET
def log_message(self, *args):
pass
server = ThreadingHTTPServer(("127.0.0.1", 0), Other)
threading.Thread(target=server.serve_forever, daemon=True).start()
return server, f"http://127.0.0.1:{server.server_address[1]}", Other.log
+346
View File
@@ -0,0 +1,346 @@
"""The egress proxy with raw sockets against local test servers: what it forwards, where, and what it refuses."""
import socket
import threading
import time
import pytest
import egress
import guard
from egress import EgressProxy
OK_REPLY = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"
class Upstream:
"""Raw TCP server on 127.0.0.1. Records the bytes of every accepted connection; answers each request head with
a canned reply (or echoes everything back), then keeps reading for a moment so stray bytes are recorded too."""
def __init__(self, reply=OK_REPLY, echo=False):
self.reply, self.echo = reply, echo
self.connections: list[bytearray] = []
self.sock = socket.socket()
self.sock.bind(("127.0.0.1", 0))
self.sock.listen()
self.port = self.sock.getsockname()[1]
threading.Thread(target=self._accept, daemon=True).start()
def _accept(self):
while True:
try:
conn, _ = self.sock.accept()
except OSError:
return
data = bytearray()
self.connections.append(data)
threading.Thread(target=self._serve, args=(conn, data), daemon=True).start()
def _serve(self, conn, data):
conn.settimeout(1)
answered = False
with conn:
try:
while True:
chunk = conn.recv(65536)
if not chunk:
return
data += chunk
if self.echo:
conn.sendall(chunk)
elif not answered and b"\r\n\r\n" in data:
conn.sendall(self.reply)
answered = True
except OSError:
return
def close(self):
self.sock.close()
@pytest.fixture
def upstreams():
made = []
def make(**kw):
u = Upstream(**kw)
made.append(u)
return u
yield make
for u in made:
u.close()
@pytest.fixture
def proxy_for():
made = []
def make(addresses_for):
p = EgressProxy(addresses_for, [], timeout=3).start()
made.append(p)
return p
yield make
for p in made:
p.stop()
def allow(*urls_to_address):
"""addresses_for that allows exactly the given (url prefix, address) pairs and records every call."""
calls = []
def addresses_for(url):
calls.append(url)
for prefix, address in urls_to_address:
if url.startswith(prefix):
return [address]
return []
addresses_for.calls = calls
return addresses_for
def exchange(proxy, data: bytes, wait: float = 3.0) -> bytes:
"""Send data to the proxy and read until it closes the connection (or wait runs out)."""
port = int(proxy.address.rsplit(":", 1)[1])
with socket.create_connection(("127.0.0.1", port)) as s:
s.settimeout(wait)
s.sendall(data)
out = bytearray()
try:
while True:
chunk = s.recv(65536)
if not chunk:
break
out += chunk
except TimeoutError:
pass
return bytes(out)
def settle():
time.sleep(0.3) # let the upstream threads record what they got
# --- DNS rebinding: the proxy connects only to the address the check returned ---------------------------------
def test_proxy_connects_to_the_checked_address_and_never_resolves_the_name(upstreams, proxy_for):
up = upstreams()
addresses_for = allow((f"http://never-resolvable.invalid:{up.port}/", "127.0.0.1"))
reply = exchange(proxy_for(addresses_for),
f"GET http://never-resolvable.invalid:{up.port}/x HTTP/1.1\r\nHost: never-resolvable.invalid\r\n\r\n"
.encode())
assert reply.startswith(b"HTTP/1.1 200 OK") and reply.endswith(b"ok")
settle()
assert len(up.connections) == 1 and up.connections[0].startswith(b"GET /x HTTP/1.1\r\n")
def test_rebinding_resolver_cannot_redirect_the_connection(monkeypatch, proxy_for):
answers = iter([["93.184.216.34"]])
lookups = []
def flip(host): # public on the first lookup, loopback on every later one
lookups.append(host)
return next(answers, ["127.0.0.1"])
connects = []
def fake_connect(address, timeout=None, *args, **kw):
connects.append(address)
raise OSError("no network in this test")
real_getaddrinfo = socket.getaddrinfo
name_lookups = []
def spy_getaddrinfo(host, *a, **kw):
name_lookups.append(host)
return real_getaddrinfo(host, *a, **kw)
monkeypatch.setattr(egress, "_connect", fake_connect)
monkeypatch.setattr(socket, "getaddrinfo", spy_getaddrinfo)
proxy = proxy_for(lambda url: guard.public_addresses(url, resolve=flip, blocked=()))
assert exchange(proxy, b"GET http://rebind.example/ HTTP/1.1\r\nHost: rebind.example\r\n\r\n") == b""
assert lookups == ["rebind.example"] # one check, one lookup
assert connects == [("93.184.216.34", 80)] # to the vetted address, not to the name
assert "rebind.example" not in name_lookups
def test_next_checked_address_is_tried_when_the_first_fails(upstreams, proxy_for):
up = upstreams() # listens on 127.0.0.1 only: 127.0.0.2 on the same port refuses
def addresses_for(url):
return ["127.0.0.2", "127.0.0.1"]
reply = exchange(proxy_for(addresses_for), f"GET http://two.example:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode())
assert reply.endswith(b"ok")
def test_denied_url_makes_no_upstream_connection_and_gets_no_answer(upstreams, proxy_for):
up = upstreams()
proxy = proxy_for(lambda url: [])
assert exchange(proxy, f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode()) == b""
settle()
assert up.connections == [] and proxy.blocked == [f"http://127.0.0.1:{up.port}/"]
def test_failing_check_counts_as_denied(upstreams, proxy_for):
up = upstreams()
def broken(url):
raise RuntimeError("resolver crashed")
assert exchange(proxy_for(broken), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode()) == b""
settle()
assert up.connections == []
# --- CONNECT ------------------------------------------------------------------------------------------------
def test_connect_allowed_tunnels_bytes(upstreams, proxy_for):
up = upstreams(echo=True)
addresses_for = allow((f"https://tunnel.example:{up.port}/", "127.0.0.1"))
port = int(proxy_for(addresses_for).address.rsplit(":", 1)[1])
with socket.create_connection(("127.0.0.1", port)) as s:
s.settimeout(3)
s.sendall(f"CONNECT tunnel.example:{up.port} HTTP/1.1\r\nHost: tunnel.example:{up.port}\r\n\r\n".encode())
head = s.recv(1024)
assert head == b"HTTP/1.1 200 Connection established\r\n\r\n"
s.sendall(b"opaque tls bytes")
assert s.recv(1024) == b"opaque tls bytes"
assert addresses_for.calls == [f"https://tunnel.example:{up.port}/"]
def test_connect_denied_sends_nothing_and_connects_nowhere(upstreams, proxy_for):
up = upstreams(echo=True)
proxy = proxy_for(lambda url: [])
assert exchange(proxy, f"CONNECT 127.0.0.1:{up.port} HTTP/1.1\r\n\r\n".encode()) == b""
settle()
assert up.connections == []
@pytest.mark.parametrize("authority", ["tunnel.example", "tunnel.example:abc", "tunnel.example:0",
"tunnel.example:70000", "user@tunnel.example:443", "user:pw@tunnel.example:443",
":443", "[::1", "::1:443", "tunnel.example:+443", "tunnel.example: 443"])
def test_malformed_connect_is_refused_without_a_check(proxy_for, authority):
addresses_for = allow(("https://", "127.0.0.1"))
assert exchange(proxy_for(addresses_for), f"CONNECT {authority} HTTP/1.1\r\n\r\n".encode()) == b""
assert addresses_for.calls == []
def test_connect_to_bracketed_ipv6_literal_is_checked_with_brackets(proxy_for):
addresses_for = allow()
assert exchange(proxy_for(addresses_for), b"CONNECT [::1]:443 HTTP/1.1\r\n\r\n") == b""
assert addresses_for.calls == ["https://[::1]:443/"]
# --- plain HTTP (absolute form) --------------------------------------------------------------------------------
def test_absolute_form_request_is_forwarded_in_origin_form(upstreams, proxy_for):
up = upstreams()
addresses_for = allow((f"http://site.example:{up.port}/", "127.0.0.1"))
reply = exchange(proxy_for(addresses_for),
f"POST http://site.example:{up.port}/u?x=1 HTTP/1.1\r\nHost: site.example:{up.port}\r\n"
"Proxy-Connection: keep-alive\r\nConnection: keep-alive, X-Secret\r\nX-Secret: 1\r\n"
"Content-Length: 5\r\n\r\nhello".encode())
assert reply.endswith(b"ok")
settle()
head, _, body = bytes(up.connections[0]).partition(b"\r\n\r\n")
lines = head.decode().split("\r\n")
assert lines[0] == "POST /u?x=1 HTTP/1.1" and body == b"hello"
names = [line.split(":")[0].lower() for line in lines[1:]]
assert "host" in names and "proxy-connection" not in names and "x-secret" not in names
assert "Connection: close" in lines and "Connection: keep-alive, X-Secret" not in lines
assert addresses_for.calls == [f"http://site.example:{up.port}/u?x=1"]
def test_absolute_form_denied(upstreams, proxy_for):
up = upstreams()
assert exchange(proxy_for(allow()), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\n\r\n".encode()) == b""
settle()
assert up.connections == []
@pytest.mark.parametrize("target", ["/relative", "https://site.example/", "ftp://site.example/", "http:///x",
"http://site.example:0/", "http://site.example:abc/", "http://u:p@site.example/"])
def test_other_request_targets_are_refused_without_a_check(proxy_for, target):
addresses_for = allow(("http://", "127.0.0.1"))
assert exchange(proxy_for(addresses_for), f"GET {target} HTTP/1.1\r\nHost: x\r\n\r\n".encode()) == b""
assert addresses_for.calls == []
def test_request_with_transfer_encoding_is_dropped(upstreams, proxy_for):
up = upstreams()
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
reply = exchange(proxy_for(addresses_for), f"POST http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n"
"Transfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n".encode())
settle()
assert reply == b"" and up.connections == []
@pytest.mark.parametrize("lengths", [("5", "5"), ("5", "7"), ("5, 5",), ("-1",), ("abc",)])
def test_duplicate_or_bad_content_length_is_rejected(upstreams, proxy_for, lengths):
up = upstreams()
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
headers = "".join(f"Content-Length: {v}\r\n" for v in lengths)
reply = exchange(proxy_for(addresses_for),
f"POST http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n{headers}\r\nhello".encode())
settle()
assert reply == b"" and up.connections == []
def test_pipelined_second_request_is_never_forwarded(upstreams, proxy_for):
a, b = upstreams(), upstreams()
addresses_for = allow((f"http://127.0.0.1:{a.port}/", "127.0.0.1"), (f"http://127.0.0.1:{b.port}/", "127.0.0.1"))
first = f"GET http://127.0.0.1:{a.port}/first HTTP/1.1\r\nHost: a\r\n\r\n"
second = f"GET http://127.0.0.1:{b.port}/second HTTP/1.1\r\nHost: b\r\n\r\n"
reply = exchange(proxy_for(addresses_for), (first + second).encode())
settle()
assert reply.count(b"HTTP/1.1 200 OK") == 1
assert b.connections == [] and len(a.connections) == 1
assert bytes(a.connections[0]).count(b" HTTP/1.1\r\n") == 1 and b"/second" not in a.connections[0]
def test_second_request_on_the_same_connection_is_never_forwarded(upstreams, proxy_for):
a, b = upstreams(), upstreams()
addresses_for = allow((f"http://127.0.0.1:{a.port}/", "127.0.0.1"), (f"http://127.0.0.1:{b.port}/", "127.0.0.1"))
port = int(proxy_for(addresses_for).address.rsplit(":", 1)[1])
with socket.create_connection(("127.0.0.1", port)) as s:
s.settimeout(3)
s.sendall(f"GET http://127.0.0.1:{a.port}/first HTTP/1.1\r\nHost: a\r\n\r\n".encode())
got = bytearray()
while not got.endswith(b"ok"):
got += s.recv(65536)
assert b"Connection: close" in got
try:
s.sendall(f"GET http://127.0.0.1:{b.port}/second HTTP/1.1\r\nHost: b\r\n\r\n".encode())
assert s.recv(65536) == b""
except (BrokenPipeError, ConnectionResetError):
pass
settle()
assert b.connections == [] and b"/second" not in a.connections[0]
def test_chunked_response_passes_through_intact(upstreams, proxy_for):
body = b"4\r\nWiki\r\n5\r\npedia\r\n0\r\n\r\n"
up = upstreams(reply=b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nContent-Type: text/plain\r\n\r\n" + body)
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
reply = exchange(proxy_for(addresses_for), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode())
head, _, rest = reply.partition(b"\r\n\r\n")
assert b"Transfer-Encoding: chunked" in head and rest == body
def test_interim_1xx_responses_are_skipped_and_hop_by_hop_headers_of_the_final_one_stripped(upstreams, proxy_for):
up = upstreams(reply=b"HTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 103 Early Hints\r\nLink: </a.css>\r\n\r\n"
b"HTTP/1.1 200 OK\r\nConnection: keep-alive, X-Hop\r\nKeep-Alive: timeout=5\r\nX-Hop: 1\r\n"
b"Proxy-Connection: keep-alive\r\nContent-Length: 2\r\n\r\nok")
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
reply = exchange(proxy_for(addresses_for), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode())
head, _, rest = reply.partition(b"\r\n\r\n")
lines = head.decode().split("\r\n")
assert lines[0] == "HTTP/1.1 200 OK" and rest == b"ok" and reply.count(b"HTTP/1.1 ") == 1
names = [line.split(":")[0].lower() for line in lines[1:]]
assert "keep-alive" not in names and "x-hop" not in names and "proxy-connection" not in names
assert lines.count("Connection: close") == 1 and names.count("connection") == 1
def test_switching_protocols_is_not_passed_on(upstreams, proxy_for):
up = upstreams(reply=b"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\nraw")
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
reply = exchange(proxy_for(addresses_for), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode())
assert reply == b""
+13
View File
@@ -55,3 +55,16 @@ def test_error_text_never_contains_the_key():
def test_invalid_endpoint_url_means_unavailable(base_url):
with pytest.raises(LLMUnavailable):
LLM(base_url, "m", "k", client=client(lambda r: httpx.Response(200, json={}))).ask("s", "u")
@pytest.mark.parametrize("given,expected", [(None, 60), (5.5, 5.5), (120, 60)])
def test_timeout_can_be_capped_per_call_but_never_raised(given, expected):
seen = {}
def handler(request):
seen["timeout"] = request.extensions["timeout"]
return httpx.Response(200, json={"choices": [{"message": {"content": "x"}}]})
llm = LLM("https://ai.example", "m", "k", timeout=60, client=client(handler))
llm.ask("s", "u") if given is None else llm.ask("s", "u", timeout=given)
assert seen["timeout"] == {"connect": expected, "read": expected, "write": expected, "pool": expected}
+123
View File
@@ -0,0 +1,123 @@
"""The HTTP service: answers for broken cases and bodies, model configuration, and what it logs.
Needs Playwright and Flask for the import: run with tests/run-in-docker.sh."""
import importlib
import logging
import pytest
pytest.importorskip("playwright.sync_api")
pytest.importorskip("flask")
ENV = {"LLM_BASE_URL": "https://ai.example", "LLM_MODEL": "deepseek-chat", "LLM_API_KEY": "sk-test"}
KEYS = ["evidence", "evidence_url", "reason", "result", "stage", "steps"]
URL = "https://shop.example/u?token=secret"
ADDRESS = "anne@wagnersnetz.de"
@pytest.fixture
def server(monkeypatch):
for k, v in ENV.items():
monkeypatch.setenv(k, v)
import server as module
return importlib.reload(module)
@pytest.fixture
def client(server):
return server.app.test_client()
def result(reason, result="unsubscribed"):
return {"result": result, "reason": reason, "evidence": "Sie wurden abgemeldet", "evidence_url": URL,
"steps": 2, "stage": 2}
def test_health(client):
r = client.get("/health")
assert r.status_code == 200 and r.get_json() == {"status": "ok"}
@pytest.mark.parametrize("body", ["not json", "[1, 2]", "null", "{}", '{"url": 1}', '{"url": "https://x.example"'
', "address": 5}', '"https://x.example"'])
def test_bad_bodies_answer_400(client, body):
assert client.post("/run", data=body, content_type="application/json").status_code == 400
def test_exception_in_a_case_is_unclear_and_logs_nothing_personal(server, client, monkeypatch, caplog):
def boom(*args, **kw):
raise RuntimeError(f"{URL} {ADDRESS}")
monkeypatch.setattr(server, "run_case", boom)
caplog.set_level(logging.DEBUG)
r = client.post("/run", json={"url": URL, "address": ADDRESS})
assert r.status_code == 200
body = r.get_json()
assert sorted(body) == KEYS
assert (body["result"], body["reason"], body["steps"], body["stage"]) == ("unclear", "agent_failure", 0, 1)
assert "RuntimeError" in caplog.text
for secret in ("shop.example", "token", "secret", ADDRESS, "anne"):
assert secret not in caplog.text
def test_browser_launch_failure_is_unclear_too(server, client, monkeypatch, caplog):
def no_browser():
raise OSError(f"cannot start browser for {URL}")
monkeypatch.setattr(server, "sync_playwright", no_browser)
caplog.set_level(logging.DEBUG)
body = client.post("/run", json={"url": URL, "address": ADDRESS}).get_json()
assert (body["result"], body["reason"]) == ("unclear", "agent_failure")
assert "shop.example" not in caplog.text and "OSError" in caplog.text
def test_model_reason_is_answered_but_logged_only_as_a_category(server, client, monkeypatch, caplog):
reason = "unsubscribed anne@wagnersnetz.de at https://x.example/u?t=1"
monkeypatch.setattr(server, "run_case", lambda *args, **kw: result(reason))
caplog.set_level(logging.DEBUG)
body = client.post("/run", json={"url": URL, "address": ADDRESS}).get_json()
assert body["reason"] == reason and body["evidence_url"] == URL
assert "anne" not in caplog.text and "x.example" not in caplog.text and "shop.example" not in caplog.text
assert "case finished: unsubscribed (model), 2 step(s), stage 2" in caplog.text
@pytest.mark.parametrize("reason", ["login", "captcha", "blocked_address", "budget", "page_error", "llm_unavailable",
"page confirms without interaction",
"three invalid replies from the model in a row"])
def test_reasons_set_by_code_are_logged_as_they_are(server, client, monkeypatch, caplog, reason):
monkeypatch.setattr(server, "run_case", lambda *args, **kw: result(reason, "not_possible"))
caplog.set_level(logging.INFO)
client.post("/run", json={"url": URL, "address": ADDRESS})
assert f"case finished: not_possible ({reason})" in caplog.text
def test_case_gets_the_configured_model_and_limits(server, client, monkeypatch):
seen = {}
def fake_run_case(browser, url, address, llm, **kw):
seen.update(url=url, address=address, llm=llm, **kw)
return result("budget", "not_possible")
monkeypatch.setattr(server, "run_case", fake_run_case)
client.post("/run", json={"url": URL, "address": ADDRESS})
assert (seen["url"], seen["address"]) == (URL, ADDRESS)
assert seen["max_steps"] == server.MAX_STEPS and seen["case_timeout"] == server.CASE_TIMEOUT
assert seen["llm"]._model == "deepseek-chat"
@pytest.mark.parametrize("missing", sorted(ENV))
@pytest.mark.parametrize("value", [None, "", " "])
def test_missing_model_configuration_stops_the_start(monkeypatch, missing, value):
for k, v in ENV.items():
monkeypatch.setenv(k, v)
import server as module
if value is None:
monkeypatch.delenv(missing)
else:
monkeypatch.setenv(missing, value)
try:
with pytest.raises(RuntimeError, match=missing):
importlib.reload(module)
finally:
monkeypatch.setenv(missing, ENV[missing])
importlib.reload(module)
+113 -5
View File
@@ -1,12 +1,18 @@
"""The browser loop against imitated unsubscribe pages. Needs Playwright: run with tests/run-in-docker.sh."""
import logging
import os
import tempfile
import time
from urllib.parse import quote
import pytest
playwright_api = pytest.importorskip("playwright.sync_api")
import guard # noqa: E402
from llm import LLMUnavailable # noqa: E402
from runner import run_case # noqa: E402
from runner import BROWSER_ARGS, run_case # noqa: E402
from tests import sites # noqa: E402
@@ -36,7 +42,7 @@ def element(prompt: str, label: str) -> int:
@pytest.fixture(scope="module")
def browser():
with playwright_api.sync_playwright() as p:
b = p.chromium.launch()
b = p.chromium.launch(args=BROWSER_ARGS)
yield b
b.close()
@@ -48,13 +54,26 @@ def site():
server.shutdown()
@pytest.fixture
def other():
server, base, log = sites.start_other()
yield base, log
server.shutdown()
def local_ok(base):
"""Guard for the tests: the test server on 127.0.0.1 counts as public, everything else goes to the real guard."""
return lambda url: url.startswith(base) or guard.is_public_url(url)
def local_addresses(base):
"""The matching check for the egress proxy: the test server is reached at 127.0.0.1, all else as in production."""
return lambda url: ["127.0.0.1"] if url.startswith(base + "/") else guard.public_addresses(url)
def run(browser, site, path, llm, address="philipp@wagnersnetz.de", **kw):
return run_case(browser, site + path, address, llm, url_ok=local_ok(site), nav_timeout=10, **kw)
return run_case(browser, site + path, address, llm, url_ok=local_ok(site), addresses_for=local_addresses(site),
nav_timeout=10, **kw)
def test_one_click_page_needs_no_model(browser, site):
@@ -202,7 +221,24 @@ def test_embedded_private_content_is_blocked_but_the_page_still_works(browser, s
assert run(browser, site, "/embeds-private", Confirm())["result"] == "unsubscribed"
def test_download_is_not_kept(browser, site, tmp_path):
def _files_with(content: bytes) -> set[str]:
"""Every file below the temp directory (where Playwright keeps downloads) whose content is exactly content."""
found = set()
for root, _, files in os.walk(tempfile.gettempdir()):
for name in files:
path = os.path.join(root, name)
try:
if os.path.getsize(path) == len(content) and open(path, "rb").read() == content:
found.add(path)
except OSError:
pass
return found
def test_download_is_not_kept(browser, site, tmp_path, caplog):
caplog.set_level(logging.INFO, logger="runner")
before = _files_with(b"data")
class Download(Script):
def ask(self, system, user):
self.prompts.append(user)
@@ -212,6 +248,9 @@ def test_download_is_not_kept(browser, site, tmp_path):
r = run(browser, site, "/download", Download())
assert r["result"] == "unclear"
assert ("GET", "/file.bin", {}) in sites.Handler.log # the download really was attempted
assert "download refused" in caplog.text # the browser offered it and it was refused
assert _files_with(b"data") - before == set() # and no file with its content appeared
def test_model_unavailable_is_an_error_not_a_result(browser, site):
@@ -227,5 +266,74 @@ def test_no_address_means_the_address_action_is_refused(browser, site):
def test_page_that_does_not_load_is_a_page_error(browser, site):
r = run_case(browser, "http://127.0.0.1:9/gone", "", Script(), url_ok=lambda url: True, nav_timeout=5)
r = run_case(browser, "http://127.0.0.1:9/gone", "", Script(), url_ok=lambda url: True,
addresses_for=lambda url: ["127.0.0.1"], nav_timeout=5)
assert (r["result"], r["reason"]) == ("not_possible", "page_error")
def test_redirect_to_loopback_never_leaves_the_proxy(browser, site, other):
base2, log2 = other
r = run(browser, site, "/redirect?to=" + quote(base2 + "/admin"), Script())
assert log2 == []
assert (r["result"], r["reason"]) == ("not_possible", "blocked_address")
@pytest.mark.parametrize("page", ["/img?src=", "/fetch?to="])
@pytest.mark.parametrize("via_redirect", [False, True])
def test_subresources_to_loopback_never_reach_it(browser, site, other, page, via_redirect):
base2, log2 = other
target = base2 + "/pixel.gif"
if via_redirect: # the route gate sees only the first hop; the proxy must stop the second
target = site + "/redirect?to=" + quote(target)
run(browser, site, page + quote(target), Script())
time.sleep(0.5)
assert log2 == []
if via_redirect:
assert ("GET", "/redirect", {}) in sites.Handler.log # the first hop did go out
def test_endless_script_ends_at_the_time_budget(browser, site):
start = time.monotonic()
r = run(browser, site, "/busy", Script(), case_timeout=5)
assert time.monotonic() - start < 15
assert (r["result"], r["reason"]) == ("not_possible", "budget")
def test_slow_model_ends_at_the_time_budget_and_gets_a_capped_timeout(browser, site):
class Slow(Script):
timeouts = []
def ask(self, system, user, timeout=None):
self.prompts.append(user)
self.timeouts.append(timeout)
time.sleep(4)
return '{"action":"click","element":%d}' % element(user, "Abmelden bestätigen")
llm = Slow()
start = time.monotonic()
r = run(browser, site, "/confirm", llm, case_timeout=3)
assert time.monotonic() - start < 10
assert (r["result"], r["reason"]) == ("not_possible", "budget")
assert llm.timeouts and all(t is not None and 0 < t <= 3 for t in llm.timeouts)
assert not any(method == "POST" for method, _, _ in sites.Handler.log)
def test_webrtc_is_not_available_to_pages(browser, site):
llm = Script()
run(browser, site, "/rtc", llm)
assert "rtc:undefined,undefined" in llm.prompts[0]
def test_blocked_image_on_a_page_that_never_loads_is_a_page_error(browser, site):
start = time.monotonic()
r = run_case(browser, site + "/hang-img", "", Script(), url_ok=local_ok(site), addresses_for=local_addresses(site),
nav_timeout=3)
assert (r["result"], r["reason"]) == ("not_possible", "page_error")
assert time.monotonic() - start < 8
@pytest.mark.parametrize("path", ["/many-links-password", "/frame-password"])
def test_password_field_anywhere_ends_without_the_model(browser, site, path):
llm = Script()
r = run(browser, site, path, llm)
assert (r["result"], r["reason"]) == ("not_possible", "login") and llm.prompts == []