Agent: Befunde aus dem Review der Browser-Schleife behoben

This commit is contained in:
Rsclub2_2 committed 2026-11-15 02:06:41 +01:00
1 parent 7e50983d85
commit 6556087cf2
10 files changed
+976 -74

No files matched your search

+120 -39
View File
@@ -1,10 +1,15 @@
"""Filtering forward proxy for one case: every request of the browser, redirect hops included, passes url_ok here. """Filtering forward proxy for one case: every request of the browser, redirect hops included, is checked here.
Playwright's route handler never sees the hops of a redirect (Chromium follows them itself), so the route gate Playwright's route handler never sees the hops of a redirect (Chromium follows them itself), so the route gate
alone cannot keep the browser away from private addresses. The browser context is pointed at this proxy, which alone cannot keep the browser away from private addresses. The browser context is pointed at this proxy, which
checks each plain-HTTP request by its full URL and each CONNECT tunnel (HTTPS, WebSocket) by its host and port. checks each plain-HTTP request by its full URL and each CONNECT tunnel (HTTPS, WebSocket) by host and port.
The check is addresses_for(url) -> list of IP addresses: the addresses the host may be reached at, or an empty
list if the URL is refused. The proxy connects only to those addresses and never resolves the name itself, so a
DNS answer that changes between check and connection (DNS rebinding) cannot redirect it.
""" """
import re
import socket import socket
import socketserver import socketserver
import threading import threading
@@ -14,6 +19,12 @@ MAX_HEADER_LINES = 100
MAX_LINE = 16384 MAX_LINE = 16384
HOP_BY_HOP = {"connection", "proxy-connection", "keep-alive", "proxy-authorization", "proxy-authenticate", "te", HOP_BY_HOP = {"connection", "proxy-connection", "keep-alive", "proxy-authorization", "proxy-authenticate", "te",
"trailer", "upgrade"} "trailer", "upgrade"}
_HOST = re.compile(r"[A-Za-z0-9._-]+")
_PORT = re.compile(r"[0-9]{1,5}")
def _connect(address: tuple[str, int], timeout: float) -> socket.socket:
return socket.create_connection(address, timeout=timeout)
def _read_head(rfile) -> list[bytes] | None: def _read_head(rfile) -> list[bytes] | None:
@@ -32,17 +43,64 @@ def _headers(lines: list[bytes]) -> list[tuple[str, str]]:
out = [] out = []
for line in lines: for line in lines:
name, sep, value = line.decode("latin-1").partition(":") name, sep, value = line.decode("latin-1").partition(":")
if not sep: if not sep or not name or name != name.strip():
raise ValueError("bad header line") raise ValueError("bad header line")
out.append((name.strip(), value.strip())) out.append((name, value.strip()))
return out return out
def _keep(headers: list[tuple[str, str]]) -> list[tuple[str, str]]: def _keep(headers: list[tuple[str, str]]) -> list[tuple[str, str]]:
"""Headers without the hop-by-hop ones, including those the Connection header names."""
named = {token.strip().lower() for k, v in headers if k.lower() == "connection" for token in v.split(",")} named = {token.strip().lower() for k, v in headers if k.lower() == "connection" for token in v.split(",")}
return [(k, v) for k, v in headers if k.lower() not in HOP_BY_HOP and k.lower() not in named] return [(k, v) for k, v in headers if k.lower() not in HOP_BY_HOP and k.lower() not in named]
def _port(text: str) -> int | None:
if not _PORT.fullmatch(text):
return None
port = int(text)
return port if 0 < port < 65536 else None
def _authority(target: str) -> tuple[str, int] | None:
"""Host and port of a CONNECT target: host:port or [ipv6]:port, nothing else (no userinfo, no default port)."""
if target.startswith("["):
host, sep, port = target[1:].partition("]:")
if not sep or not host or not all(c in "0123456789abcdefABCDEF:." for c in host):
return None
else:
host, sep, port = target.rpartition(":")
if not sep or not _HOST.fullmatch(host):
return None
number = _port(port)
return (host, number) if number else None
def _absolute(target: str) -> tuple[str, int, str] | None:
"""Host, port and origin-form path of an absolute http:// request target."""
try:
parts = urlsplit(target)
port = parts.port
except ValueError:
return None
if parts.scheme != "http" or not parts.hostname or "@" in parts.netloc or port == 0:
return None
path = (parts.path or "/") + (f"?{parts.query}" if parts.query else "")
return parts.hostname, port or 80, path
def _content_length(headers: list[tuple[str, str]]) -> int | None:
"""The request body length; None if the request must be refused (chunked, duplicate or bad length)."""
if any(k.lower() == "transfer-encoding" for k, _ in headers):
return None
values = [v for k, v in headers if k.lower() == "content-length"]
if not values:
return 0
if len(values) > 1 or not values[0].isdigit():
return None
return int(values[0])
def _pump(src, dst) -> None: def _pump(src, dst) -> None:
"""Copy bytes from src (socket or buffered file) to dst until src ends or either side fails.""" """Copy bytes from src (socket or buffered file) to dst until src ends or either side fails."""
try: try:
@@ -61,6 +119,9 @@ def _pump(src, dst) -> None:
class _Handler(socketserver.StreamRequestHandler): class _Handler(socketserver.StreamRequestHandler):
"""One client connection carries exactly one request (or one tunnel); then both sides close. A kept-alive
connection could otherwise carry the browser's next request, meant for another host, to this upstream."""
def handle(self): def handle(self):
proxy = self.server.egress proxy = self.server.egress
self.connection.settimeout(proxy.timeout) self.connection.settimeout(proxy.timeout)
@@ -73,40 +134,53 @@ class _Handler(socketserver.StreamRequestHandler):
except ValueError: except ValueError:
return return
if method == "CONNECT": if method == "CONNECT":
host, _, port = target.rpartition(":") authority = _authority(target)
url = f"https://{host}:{port}/" if not authority:
else:
url = target
parts = urlsplit(target)
host, port = parts.hostname, parts.port or 80
if parts.scheme != "http" or not host:
return return
host = f"[{host}]" if ":" in host else host host, port = authority
if not proxy.url_ok(url): url = f"https://[{host}]:{port}/" if ":" in host else f"https://{host}:{port}/"
length = path = None
else:
absolute = _absolute(target)
length = _content_length(headers)
if not absolute or length is None or not version.startswith("HTTP/1."):
return
host, port, path = absolute
url = target
try:
addresses = list(proxy.addresses_for(url))
except Exception:
addresses = []
if not addresses:
proxy.blocked.append(url) proxy.blocked.append(url)
return # no answer at all: the browser sees a failed request, as with a blocked route return # no answer at all: the browser sees a failed request, as with a blocked route
try: upstream = self._open(addresses, port, proxy.timeout)
upstream = socket.create_connection((host.strip("[]"), int(port)), timeout=proxy.timeout) if upstream is None:
except (OSError, ValueError):
return return
with upstream: with upstream:
if method == "CONNECT": if method == "CONNECT":
self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n") self._tunnel(upstream, proxy.timeout)
self.wfile.flush() else:
t = threading.Thread(target=_pump, args=(self.rfile, upstream), daemon=True) self._forward(upstream, method, path, version, headers, length)
t.start()
_pump(upstream, self.connection)
t.join(proxy.timeout)
return
self._forward(upstream, method, parts, version, headers)
def _forward(self, upstream, method, parts, version, headers): @staticmethod
"""One plain-HTTP request, then both connections close: a kept-alive connection could carry the next def _open(addresses: list[str], port: int, timeout: float) -> socket.socket | None:
request of the browser, meant for another host, to this upstream without a check.""" for address in addresses:
if any(k.lower() == "transfer-encoding" for k, _ in headers): try:
return # chunked request bodies are not supported; the request fails return _connect((address, port), timeout)
length = next((int(v) for k, v in headers if k.lower() == "content-length" and v.isdigit()), 0) except OSError:
path = (parts.path or "/") + (f"?{parts.query}" if parts.query else "") continue
return None
def _tunnel(self, upstream, timeout):
self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
self.wfile.flush()
t = threading.Thread(target=_pump, args=(self.rfile, upstream), daemon=True)
t.start()
_pump(upstream, self.connection)
t.join(timeout)
def _forward(self, upstream, method, path, version, headers, length):
lines = [f"{method} {path} {version}"] + [f"{k}: {v}" for k, v in _keep(headers)] + ["Connection: close"] lines = [f"{method} {path} {version}"] + [f"{k}: {v}" for k, v in _keep(headers)] + ["Connection: close"]
upstream.sendall(("\r\n".join(lines) + "\r\n\r\n").encode("latin-1")) upstream.sendall(("\r\n".join(lines) + "\r\n\r\n").encode("latin-1"))
while length > 0: while length > 0:
@@ -116,14 +190,21 @@ class _Handler(socketserver.StreamRequestHandler):
upstream.sendall(chunk) upstream.sendall(chunk)
length -= len(chunk) length -= len(chunk)
ufile = upstream.makefile("rb") ufile = upstream.makefile("rb")
head = _read_head(ufile) while True:
if not head: head = _read_head(ufile)
return if not head:
return
status = head[0].decode("latin-1")
code = status.split(" ")[1] if status.count(" ") else ""
if code == "101":
return # no protocol switches: the Upgrade header was not passed on, so this is not ours
if not (len(code) == 3 and code.startswith("1")):
break # interim 1xx answers (100 Continue, 103 Early Hints) are dropped; the final one counts
try: try:
status, response_headers = head[0], _keep(_headers(head[1:])) response_headers = _keep(_headers(head[1:]))
except ValueError: except ValueError:
return return
out = [status.decode("latin-1")] + [f"{k}: {v}" for k, v in response_headers] + ["Connection: close"] out = [status] + [f"{k}: {v}" for k, v in response_headers] + ["Connection: close"]
self.wfile.write(("\r\n".join(out) + "\r\n\r\n").encode("latin-1")) self.wfile.write(("\r\n".join(out) + "\r\n\r\n").encode("latin-1"))
self.wfile.flush() self.wfile.flush()
_pump(ufile, self.connection) _pump(ufile, self.connection)
@@ -138,10 +219,10 @@ class _Server(socketserver.ThreadingTCPServer):
class EgressProxy: class EgressProxy:
"""Proxy on 127.0.0.1 with a random port, between start() and stop(). Blocked URLs are appended to blocked.""" """Proxy on 127.0.0.1 with a random port, between start() and stop(). Refused URLs are appended to blocked."""
def __init__(self, url_ok, blocked: list, timeout: float = 30): def __init__(self, addresses_for, blocked: list, timeout: float = 30):
self.url_ok = url_ok self.addresses_for = addresses_for
self.blocked = blocked self.blocked = blocked
self.timeout = timeout self.timeout = timeout
self._server = _Server(("127.0.0.1", 0), _Handler) self._server = _Server(("127.0.0.1", 0), _Handler)
+5 -2
View File
@@ -15,14 +15,17 @@ class LLM:
self._timeout = timeout self._timeout = timeout
self._client = client or httpx.Client() self._client = client or httpx.Client()
def ask(self, system: str, user: str) -> str: def ask(self, system: str, user: str, timeout: float | None = None) -> str:
"""The model's reply text. timeout (seconds) can only shorten the configured one, for example to the
remaining budget of a case."""
timeout = self._timeout if timeout is None else min(timeout, self._timeout)
body = { body = {
"model": self._model, "stream": False, "temperature": 0, "model": self._model, "stream": False, "temperature": 0,
"messages": [{"role": "system", "content": system}, {"role": "user", "content": user}], "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}],
} }
try: try:
r = self._client.post(self._url, json=body, headers={"Authorization": f"Bearer {self._key}"}, r = self._client.post(self._url, json=body, headers={"Authorization": f"Bearer {self._key}"},
timeout=self._timeout) timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL, ValueError) as e: # InvalidURL is no HTTPError except (httpx.HTTPError, httpx.InvalidURL, ValueError) as e: # InvalidURL is no HTTPError
raise LLMUnavailable(f"model endpoint not reachable: {type(e).__name__}") from None raise LLMUnavailable(f"model endpoint not reachable: {type(e).__name__}") from None
if r.status_code != 200: if r.status_code != 200:
+159 -13
View File
@@ -1,20 +1,41 @@
"""Browser loop of the unsubscribe agent: stage 1 without the model, stage 2 with one action per step.""" """Browser loop of the unsubscribe agent: stage 1 without the model, stage 2 with one action per step."""
import asyncio
import inspect
import logging import logging
import threading
import time import time
from urllib.parse import urlsplit
from egress import EgressProxy from egress import EgressProxy
from guard import is_public_url from guard import is_public_url, public_addresses
from llm import LLMUnavailable from llm import LLMUnavailable
from pagecheck import has_password, is_login_url, looks_unsubscribed, needs_input from pagecheck import has_password, is_login_url, looks_unsubscribed, needs_input
from protocol import SYSTEM_PROMPT, ProtocolError, build_prompt, parse_action from protocol import SYSTEM_PROMPT, ProtocolError, build_prompt, parse_action
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# Launch arguments for every browser that runs cases (the service and the tests use the same).
# WebRTC may only use proxied connections: no UDP (STUN/TURN) past the egress proxy.
BROWSER_ARGS = ["--disable-dev-shm-usage", "--force-webrtc-ip-handling-policy=disable_non_proxied_udp"]
WATCHDOG_MARGIN = 2.0 # seconds past the case budget after which the browser context is closed by force
MAX_TEXT = 6000 MAX_TEXT = 6000
MAX_ELEMENTS = 60 MAX_ELEMENTS = 60
EVIDENCE = 400 EVIDENCE = 400
# Reasons set by this code; any other reason was written by the model (and may contain personal data).
CODE_REASONS = frozenset({
"blocked_address", "page_error", "login", "captcha", "budget", "llm_unavailable", "agent_failure",
"page confirms without interaction", "three invalid replies from the model in a row"})
# Removes the WebRTC constructors before any page script runs (the launch flag above is the second line).
NO_WEBRTC_JS = """
for (const name of ['RTCPeerConnection', 'webkitRTCPeerConnection', 'RTCDataChannel']) {
try { delete window[name]; } catch (e) {}
try { Object.defineProperty(window, name, {value: undefined, configurable: false, writable: false}); } catch (e) {}
}
"""
# Lists the visible controls and marks each with data-ua-idx so the next action can find it again. # Lists the visible controls and marks each with data-ua-idx so the next action can find it again.
SNAPSHOT_JS = """ SNAPSHOT_JS = """
([maxElements, maxText]) => { ([maxElements, maxText]) => {
@@ -80,73 +101,188 @@ def _act(page, action, address: str) -> str:
return f"{action.kind} {action.element}" return f"{action.kind} {action.element}"
def _authority(url: str) -> tuple:
try:
parts = urlsplit(url)
return (parts.hostname, parts.port or {"http": 80, "https": 443}.get(parts.scheme))
except ValueError:
return (None, None)
def _password_anywhere(page) -> bool:
"""A visible password field in any frame, not only among the controls listed for the model."""
for frame in page.frames:
try:
if frame.locator("input[type=password]:visible").count():
return True
except Exception:
continue # detached or crashed frame
return False
def _ask(llm, prompt: str, timeout: float) -> str:
"""Ask the model with a timeout capped by the remaining budget, if its ask() takes one."""
try:
takes_timeout = "timeout" in inspect.signature(llm.ask).parameters
except (TypeError, ValueError):
takes_timeout = False
if takes_timeout:
return llm.ask(SYSTEM_PROMPT, prompt, timeout=timeout)
return llm.ask(SYSTEM_PROMPT, prompt)
def _refuse_download(download) -> None:
logger.info("download refused")
try:
download.cancel()
except Exception:
pass
def _report_close(future) -> None:
if not future.cancelled() and future.exception() is not None:
logger.warning("watchdog could not close the browser context: %s", type(future.exception()).__name__)
class _Watchdog:
"""Hard wall clock for one case. Playwright's sync API may only be used from its own thread, so the timer
schedules the context's close on Playwright's event loop; a call that hangs in the page (an endless script
blocks page.evaluate forever) then fails at once, and the case ends with budget."""
def __init__(self, seconds: float):
self.expired = threading.Event()
self._context = None
self._timer = threading.Timer(seconds, self._fire)
self._timer.daemon = True
def watch(self, context) -> None:
self._context = context
self._timer.start()
def _fire(self) -> None:
self.expired.set()
try:
# private parts of Playwright 1.65 (pinned): the async implementation object and its event loop
future = asyncio.run_coroutine_threadsafe(self._context._impl_obj.close(None), self._context._loop)
except Exception as e:
logger.warning("watchdog could not close the browser context: %s", type(e).__name__)
return
future.add_done_callback(_report_close)
def cancel(self) -> None:
self._timer.cancel()
def run_case(browser, url: str, address: str, llm, *, max_steps: int = 20, case_timeout: float = 300, def run_case(browser, url: str, address: str, llm, *, max_steps: int = 20, case_timeout: float = 300,
nav_timeout: float = 30, url_ok=is_public_url, clock=time.monotonic) -> dict: nav_timeout: float = 30, url_ok=is_public_url, addresses_for=public_addresses, clock=time.monotonic) -> dict:
"""Try to unsubscribe at url. Returns result unsubscribed | not_possible | unclear | error with a reason.""" """Try to unsubscribe at url. Returns result unsubscribed | not_possible | unclear | error with a reason.
url_ok checks the start address, every request the route gate sees and the page address after each step.
addresses_for is the egress proxy's check: the addresses a URL may be reached at ([] = refused); the proxy
connects only to those. Both must agree on what is allowed.
"""
if not url_ok(url): if not url_ok(url):
return _result("not_possible", "blocked_address") return _result("not_possible", "blocked_address")
deadline = clock() + case_timeout deadline = clock() + case_timeout
blocked: list[str] = [] watchdog = _Watchdog(case_timeout + WATCHDOG_MARGIN)
# Every request leaves through the egress proxy, which checks url_ok also for redirect hops and tunnels; steps = 0
def over() -> bool:
return watchdog.expired.is_set() or clock() >= deadline
def budget(evidence: str = "", evidence_url: str = "") -> dict:
return _result("not_possible", "budget", evidence, evidence_url, steps, 2 if steps else 1)
blocked: list[str] = [] # every refused URL (route gate and proxy)
main_navigation: list[str] = [] # URLs of the main frame's navigation requests, redirect hops included
main_blocked = threading.Event() # the route gate refused a main-frame navigation
# Every request leaves through the egress proxy, which checks also redirect hops and tunnels;
# the route gate below sees only the first request of each redirect chain, but with the full https URL. # the route gate below sees only the first request of each redirect chain, but with the full https URL.
proxy = EgressProxy(url_ok, blocked, timeout=nav_timeout).start() proxy = EgressProxy(addresses_for, blocked, timeout=nav_timeout).start()
context = None context = None
page = None
try: try:
context = browser.new_context(accept_downloads=False, service_workers="block", context = browser.new_context(accept_downloads=False, service_workers="block",
proxy={"server": proxy.address}) proxy={"server": proxy.address})
watchdog.watch(context)
context.add_init_script(NO_WEBRTC_JS)
def is_main_navigation(request) -> bool:
try:
return request.is_navigation_request() and page is not None and request.frame == page.main_frame
except Exception:
return False
def gate(route): def gate(route):
if url_ok(route.request.url): if url_ok(route.request.url):
route.continue_() route.continue_()
else: else:
blocked.append(route.request.url) blocked.append(route.request.url)
if is_main_navigation(route.request):
main_blocked.set()
route.abort("blockedbyclient") route.abort("blockedbyclient")
def navigation_blocked() -> bool:
refused = {_authority(u) for u in list(blocked)}
return main_blocked.is_set() or any(_authority(u) in refused for u in main_navigation)
context.route("**/*", gate) context.route("**/*", gate)
page = context.new_page() page = context.new_page()
context.on("page", lambda extra: extra.close() if extra != page else None) # pop-ups and new tabs context.on("page", lambda extra: extra.close() if extra != page else None) # pop-ups and new tabs
page.on("dialog", lambda dialog: dialog.dismiss()) page.on("dialog", lambda dialog: dialog.dismiss())
page.on("download", lambda download: download.cancel()) page.on("download", _refuse_download)
page.on("request", lambda request: main_navigation.append(request.url) if is_main_navigation(request) else None)
try: try:
page.goto(url, timeout=nav_timeout * 1000, wait_until="domcontentloaded") page.goto(url, timeout=nav_timeout * 1000, wait_until="domcontentloaded")
except Exception as e: except Exception as e:
reason = "blocked_address" if blocked else "page_error" if over():
return budget()
reason = "blocked_address" if navigation_blocked() else "page_error"
logger.info("first load failed (%s): %s", reason, type(e).__name__) logger.info("first load failed (%s): %s", reason, type(e).__name__)
return _result("not_possible", reason) return _result("not_possible", reason)
_settle(page, nav_timeout) _settle(page, nav_timeout)
history: list[str] = [] history: list[str] = []
bad_replies = 0 bad_replies = 0
steps = 0
while True: while True:
if over():
return budget()
try: try:
snap = page.evaluate(SNAPSHOT_JS, [MAX_ELEMENTS, MAX_TEXT]) snap = page.evaluate(SNAPSHOT_JS, [MAX_ELEMENTS, MAX_TEXT])
except Exception: except Exception:
if over():
return budget()
_settle(page, nav_timeout) # the page was still navigating; one more try _settle(page, nav_timeout) # the page was still navigating; one more try
try: try:
snap = page.evaluate(SNAPSHOT_JS, [MAX_ELEMENTS, MAX_TEXT]) snap = page.evaluate(SNAPSHOT_JS, [MAX_ELEMENTS, MAX_TEXT])
except Exception: except Exception:
if over():
return budget()
return _result("not_possible", "page_error", steps=steps, stage=2 if steps else 1) return _result("not_possible", "page_error", steps=steps, stage=2 if steps else 1)
stage = 2 if steps else 1 stage = 2 if steps else 1
here = page.url here = page.url
if not url_ok(here) and here != "about:blank": if not url_ok(here) and here != "about:blank":
return _result("not_possible", "blocked_address", steps=steps, stage=stage) return _result("not_possible", "blocked_address", steps=steps, stage=stage)
if has_password(snap["elements"]) or is_login_url(here): if has_password(snap["elements"]) or is_login_url(here) or _password_anywhere(page):
return _result("not_possible", "login", snap["tail"], here, steps, stage) return _result("not_possible", "login", snap["tail"], here, steps, stage)
if snap["captcha"]: if snap["captcha"]:
return _result("not_possible", "captcha", snap["tail"], here, steps, stage) return _result("not_possible", "captcha", snap["tail"], here, steps, stage)
if steps == 0 and looks_unsubscribed(snap["text"]) and not needs_input(snap["elements"]): if steps == 0 and looks_unsubscribed(snap["text"]) and not needs_input(snap["elements"]):
return _result("unsubscribed", "page confirms without interaction", snap["tail"], here, 0, 1) return _result("unsubscribed", "page confirms without interaction", snap["tail"], here, 0, 1)
if steps >= max_steps or clock() >= deadline: if steps >= max_steps or over():
return _result("not_possible", "budget", snap["tail"], here, steps, 2) return _result("not_possible", "budget", snap["tail"], here, steps, 2)
steps += 1 steps += 1
prompt = build_prompt(snap["title"], here, snap["text"], snap["elements"], history, bool(address)) prompt = build_prompt(snap["title"], here, snap["text"], snap["elements"], history, bool(address))
try: try:
reply = llm.ask(SYSTEM_PROMPT, prompt) reply = _ask(llm, prompt, max(deadline - clock(), 0.001))
except LLMUnavailable as e: except LLMUnavailable as e:
if over():
return budget(snap["tail"], here)
logger.warning("model unavailable: %s", e) logger.warning("model unavailable: %s", e)
return _result("error", "llm_unavailable", steps=steps - 1, stage=2) return _result("error", "llm_unavailable", steps=steps - 1, stage=2)
if over():
return budget(snap["tail"], here)
try: try:
action = parse_action(reply, snap["elements"], bool(address)) action = parse_action(reply, snap["elements"], bool(address))
except ProtocolError as e: except ProtocolError as e:
@@ -161,9 +297,19 @@ def run_case(browser, url: str, address: str, llm, *, max_steps: int = 20, case_
try: try:
history.append(_act(page, action, address) + " -> ok") history.append(_act(page, action, address) + " -> ok")
except Exception as e: except Exception as e:
if over():
return budget(snap["tail"], here)
history.append(f"{action.kind} {action.element} -> failed ({type(e).__name__})") history.append(f"{action.kind} {action.element} -> failed ({type(e).__name__})")
_settle(page, nav_timeout) _settle(page, nav_timeout)
except Exception:
if over():
return budget()
raise
finally: finally:
watchdog.cancel()
if context is not None: if context is not None:
context.close() try:
context.close()
except Exception:
pass
proxy.stop() proxy.stop()
+36 -13
View File
@@ -8,19 +8,32 @@ from flask import Flask, jsonify, request
from playwright.sync_api import sync_playwright from playwright.sync_api import sync_playwright
from llm import LLM from llm import LLM
from runner import run_case from runner import BROWSER_ARGS, CODE_REASONS, run_case
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO")) logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"))
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
app = Flask(__name__)
LOCK = threading.Lock() # one case at a time def _required(name: str) -> str:
"""A setting the service cannot work without: a missing one stops the start instead of failing every case."""
value = os.environ.get(name, "").strip()
if not value:
raise RuntimeError(f"{name} is not set: the agent cannot reach the model without it")
return value
MODEL = LLM(_required("LLM_BASE_URL"), _required("LLM_MODEL"), _required("LLM_API_KEY")) # one at a time: shared
MAX_STEPS = int(os.environ.get("MAX_STEPS", "20")) MAX_STEPS = int(os.environ.get("MAX_STEPS", "20"))
CASE_TIMEOUT = float(os.environ.get("CASE_TIMEOUT", "300")) CASE_TIMEOUT = float(os.environ.get("CASE_TIMEOUT", "300"))
app = Flask(__name__)
LOCK = threading.Lock() # one case at a time
def _llm() -> LLM:
return LLM(os.environ["LLM_BASE_URL"], os.environ["LLM_MODEL"], os.environ["LLM_API_KEY"]) def _failure() -> dict:
"""Result for a case the agent itself failed on. Unclear, not error: the attempt is used up, so one bad page
cannot block the queue of the caller."""
return {"result": "unclear", "reason": "agent_failure", "evidence": "", "evidence_url": "", "steps": 0, "stage": 1}
@app.route("/health") @app.route("/health")
@@ -30,16 +43,26 @@ def health():
@app.route("/run", methods=["POST"]) @app.route("/run", methods=["POST"])
def run(): def run():
data = request.get_json(silent=True) or {} data = request.get_json(silent=True)
if not isinstance(data, dict):
return jsonify({"error": "body must be a JSON object"}), 400
url, address = data.get("url"), data.get("address", "") url, address = data.get("url"), data.get("address", "")
if not isinstance(url, str) or not isinstance(address, str): if not isinstance(url, str) or not isinstance(address, str):
return jsonify({"error": "url and address must be strings"}), 400 return jsonify({"error": "url and address must be strings"}), 400
with LOCK, sync_playwright() as p: with LOCK:
browser = p.chromium.launch(args=["--disable-dev-shm-usage"])
try: try:
result = run_case(browser, url, address, _llm(), max_steps=MAX_STEPS, case_timeout=CASE_TIMEOUT) with sync_playwright() as p:
finally: browser = p.chromium.launch(args=BROWSER_ARGS)
browser.close() try:
logger.info("case finished: %s (%s), %d step(s), stage %d", result["result"], result["reason"], result["steps"], result = run_case(browser, url, address, MODEL, max_steps=MAX_STEPS, case_timeout=CASE_TIMEOUT)
result["stage"]) # never log the url or the address: both are personal data of the users finally:
browser.close()
except Exception as e:
# only the type: messages of browser and library errors may contain the url
logger.warning("case failed in the agent: %s", type(e).__name__)
result = _failure()
# never log the url, the address or a reason the model wrote: all may contain personal data of the users
category = result["reason"] if result["reason"] in CODE_REASONS else "model"
logger.info("case finished: %s (%s), %d step(s), stage %d", result["result"], category, result["steps"],
result["stage"])
return jsonify(result) return jsonify(result)
+2 -1
View File
@@ -4,5 +4,6 @@
set -eu set -eu
# --load: with a docker-container buildx builder the image would otherwise stay in the build cache only # --load: with a docker-container buildx builder the image would otherwise stay in the build cache only
docker build -q --load -t unsubscriber-agent-test agent >/dev/null docker build -q --load -t unsubscriber-agent-test agent >/dev/null
docker run --rm --network none --user pwuser -v "$PWD/agent/tests:/app/tests:ro" unsubscriber-agent-test \ # --init: python would be PID 1 and ignore SIGTERM, so a hanging run could not be stopped
docker run --init --rm --network none --user pwuser -v "$PWD/agent/tests:/app/tests:ro" unsubscriber-agent-test \
python -m pytest -q -p no:cacheprovider /app/tests python -m pytest -q -p no:cacheprovider /app/tests
+59 -1
View File
@@ -1,6 +1,8 @@
"""Tiny web server with imitated unsubscribe pages for the browser tests.""" """Tiny web server with imitated unsubscribe pages for the browser tests."""
import html
import threading import threading
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, urlsplit from urllib.parse import parse_qs, urlsplit
@@ -27,6 +29,14 @@ PAGES = {
"/download": "<p>Bitte bestätigen</p><a href='/file.bin' download>Abmelden bestätigen</a>", "/download": "<p>Bitte bestätigen</p><a href='/file.bin' download>Abmelden bestätigen</a>",
"/home": "<p>Welcome</p>", "/home": "<p>Welcome</p>",
"/bought": "<p>Thank you for your purchase</p>", "/bought": "<p>Thank you for your purchase</p>",
"/busy": "<p>Bitte bestätigen</p><button>Abmelden bestätigen</button>"
"<script>addEventListener('load', () => setTimeout(() => { while (true) {} }, 50))</script>",
"/rtc": "<p id='r'>?</p><button>Abmelden</button><script>document.getElementById('r').textContent ="
" 'rtc:' + typeof RTCPeerConnection + ',' + typeof webkitRTCPeerConnection</script>",
"/many-links-password": "".join(f"<a href='/home?{i}'>Link {i}</a> " for i in range(70)) +
"<form><input type='password' name='pw'><button>Abmelden</button></form>",
"/frame-password": "<p>Abmelden</p><iframe src='/pwframe'></iframe><button>Abmelden bestätigen</button>",
"/pwframe": "<form><input type='text' name='u'><input type='password' name='pw'></form>",
} }
@@ -44,8 +54,31 @@ class Handler(BaseHTTPRequestHandler):
self.wfile.write(data) self.wfile.write(data)
def do_GET(self): def do_GET(self):
path = urlsplit(self.path).path parts = urlsplit(self.path)
path, query = parts.path, parse_qs(parts.query)
Handler.log.append(("GET", path, {})) Handler.log.append(("GET", path, {}))
if path == "/redirect":
return self._send("", 302, {"Location": query["to"][0]})
if path == "/img":
return self._send(f"<p>Bild</p><img src='{html.escape(query['src'][0])}'>")
if path == "/fetch":
target = html.escape(query["to"][0])
return self._send(f"<p>Skript</p><script>fetch('{target}', {{mode: 'no-cors'}}).catch(() => 0);"
f"navigator.sendBeacon('{target}', 'x')</script>")
if path == "/hang-img":
# loads a blocked image, then never finishes the document (a byte a second keeps the connection busy)
self.send_response(200)
self.send_header("Content-Type", "text/html; charset=utf-8")
self.end_headers()
self.wfile.write(b"<!doctype html><html><body><p>Laedt ...</p><img src='http://192.168.8.111/p.gif'>")
for _ in range(10):
try:
self.wfile.write(b" ")
self.wfile.flush()
except OSError:
return
time.sleep(1)
return
if path == "/to-private": if path == "/to-private":
return self._send("", 302, {"Location": "http://192.168.8.111/admin"}) return self._send("", 302, {"Location": "http://192.168.8.111/admin"})
if path == "/file.bin": if path == "/file.bin":
@@ -78,3 +111,28 @@ def start() -> tuple[ThreadingHTTPServer, str]:
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
threading.Thread(target=server.serve_forever, daemon=True).start() threading.Thread(target=server.serve_forever, daemon=True).start()
return server, f"http://127.0.0.1:{server.server_address[1]}" return server, f"http://127.0.0.1:{server.server_address[1]}"
def start_other() -> tuple[ThreadingHTTPServer, str, list]:
"""A second server on 127.0.0.1 with its own request log: the browser must never reach it."""
class Other(BaseHTTPRequestHandler):
log = []
def do_GET(self):
Other.log.append((self.command, self.path))
data = b"<p>other server</p>"
self.send_response(200)
self.send_header("Content-Type", "text/html")
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
do_POST = do_GET
def log_message(self, *args):
pass
server = ThreadingHTTPServer(("127.0.0.1", 0), Other)
threading.Thread(target=server.serve_forever, daemon=True).start()
return server, f"http://127.0.0.1:{server.server_address[1]}", Other.log
+346
View File
@@ -0,0 +1,346 @@
"""The egress proxy with raw sockets against local test servers: what it forwards, where, and what it refuses."""
import socket
import threading
import time
import pytest
import egress
import guard
from egress import EgressProxy
OK_REPLY = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"
class Upstream:
"""Raw TCP server on 127.0.0.1. Records the bytes of every accepted connection; answers each request head with
a canned reply (or echoes everything back), then keeps reading for a moment so stray bytes are recorded too."""
def __init__(self, reply=OK_REPLY, echo=False):
self.reply, self.echo = reply, echo
self.connections: list[bytearray] = []
self.sock = socket.socket()
self.sock.bind(("127.0.0.1", 0))
self.sock.listen()
self.port = self.sock.getsockname()[1]
threading.Thread(target=self._accept, daemon=True).start()
def _accept(self):
while True:
try:
conn, _ = self.sock.accept()
except OSError:
return
data = bytearray()
self.connections.append(data)
threading.Thread(target=self._serve, args=(conn, data), daemon=True).start()
def _serve(self, conn, data):
conn.settimeout(1)
answered = False
with conn:
try:
while True:
chunk = conn.recv(65536)
if not chunk:
return
data += chunk
if self.echo:
conn.sendall(chunk)
elif not answered and b"\r\n\r\n" in data:
conn.sendall(self.reply)
answered = True
except OSError:
return
def close(self):
self.sock.close()
@pytest.fixture
def upstreams():
made = []
def make(**kw):
u = Upstream(**kw)
made.append(u)
return u
yield make
for u in made:
u.close()
@pytest.fixture
def proxy_for():
made = []
def make(addresses_for):
p = EgressProxy(addresses_for, [], timeout=3).start()
made.append(p)
return p
yield make
for p in made:
p.stop()
def allow(*urls_to_address):
"""addresses_for that allows exactly the given (url prefix, address) pairs and records every call."""
calls = []
def addresses_for(url):
calls.append(url)
for prefix, address in urls_to_address:
if url.startswith(prefix):
return [address]
return []
addresses_for.calls = calls
return addresses_for
def exchange(proxy, data: bytes, wait: float = 3.0) -> bytes:
"""Send data to the proxy and read until it closes the connection (or wait runs out)."""
port = int(proxy.address.rsplit(":", 1)[1])
with socket.create_connection(("127.0.0.1", port)) as s:
s.settimeout(wait)
s.sendall(data)
out = bytearray()
try:
while True:
chunk = s.recv(65536)
if not chunk:
break
out += chunk
except TimeoutError:
pass
return bytes(out)
def settle():
time.sleep(0.3) # let the upstream threads record what they got
# --- DNS rebinding: the proxy connects only to the address the check returned ---------------------------------
def test_proxy_connects_to_the_checked_address_and_never_resolves_the_name(upstreams, proxy_for):
up = upstreams()
addresses_for = allow((f"http://never-resolvable.invalid:{up.port}/", "127.0.0.1"))
reply = exchange(proxy_for(addresses_for),
f"GET http://never-resolvable.invalid:{up.port}/x HTTP/1.1\r\nHost: never-resolvable.invalid\r\n\r\n"
.encode())
assert reply.startswith(b"HTTP/1.1 200 OK") and reply.endswith(b"ok")
settle()
assert len(up.connections) == 1 and up.connections[0].startswith(b"GET /x HTTP/1.1\r\n")
def test_rebinding_resolver_cannot_redirect_the_connection(monkeypatch, proxy_for):
answers = iter([["93.184.216.34"]])
lookups = []
def flip(host): # public on the first lookup, loopback on every later one
lookups.append(host)
return next(answers, ["127.0.0.1"])
connects = []
def fake_connect(address, timeout=None, *args, **kw):
connects.append(address)
raise OSError("no network in this test")
real_getaddrinfo = socket.getaddrinfo
name_lookups = []
def spy_getaddrinfo(host, *a, **kw):
name_lookups.append(host)
return real_getaddrinfo(host, *a, **kw)
monkeypatch.setattr(egress, "_connect", fake_connect)
monkeypatch.setattr(socket, "getaddrinfo", spy_getaddrinfo)
proxy = proxy_for(lambda url: guard.public_addresses(url, resolve=flip, blocked=()))
assert exchange(proxy, b"GET http://rebind.example/ HTTP/1.1\r\nHost: rebind.example\r\n\r\n") == b""
assert lookups == ["rebind.example"] # one check, one lookup
assert connects == [("93.184.216.34", 80)] # to the vetted address, not to the name
assert "rebind.example" not in name_lookups
def test_next_checked_address_is_tried_when_the_first_fails(upstreams, proxy_for):
up = upstreams() # listens on 127.0.0.1 only: 127.0.0.2 on the same port refuses
def addresses_for(url):
return ["127.0.0.2", "127.0.0.1"]
reply = exchange(proxy_for(addresses_for), f"GET http://two.example:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode())
assert reply.endswith(b"ok")
def test_denied_url_makes_no_upstream_connection_and_gets_no_answer(upstreams, proxy_for):
up = upstreams()
proxy = proxy_for(lambda url: [])
assert exchange(proxy, f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode()) == b""
settle()
assert up.connections == [] and proxy.blocked == [f"http://127.0.0.1:{up.port}/"]
def test_failing_check_counts_as_denied(upstreams, proxy_for):
up = upstreams()
def broken(url):
raise RuntimeError("resolver crashed")
assert exchange(proxy_for(broken), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode()) == b""
settle()
assert up.connections == []
# --- CONNECT ------------------------------------------------------------------------------------------------
def test_connect_allowed_tunnels_bytes(upstreams, proxy_for):
up = upstreams(echo=True)
addresses_for = allow((f"https://tunnel.example:{up.port}/", "127.0.0.1"))
port = int(proxy_for(addresses_for).address.rsplit(":", 1)[1])
with socket.create_connection(("127.0.0.1", port)) as s:
s.settimeout(3)
s.sendall(f"CONNECT tunnel.example:{up.port} HTTP/1.1\r\nHost: tunnel.example:{up.port}\r\n\r\n".encode())
head = s.recv(1024)
assert head == b"HTTP/1.1 200 Connection established\r\n\r\n"
s.sendall(b"opaque tls bytes")
assert s.recv(1024) == b"opaque tls bytes"
assert addresses_for.calls == [f"https://tunnel.example:{up.port}/"]
def test_connect_denied_sends_nothing_and_connects_nowhere(upstreams, proxy_for):
up = upstreams(echo=True)
proxy = proxy_for(lambda url: [])
assert exchange(proxy, f"CONNECT 127.0.0.1:{up.port} HTTP/1.1\r\n\r\n".encode()) == b""
settle()
assert up.connections == []
@pytest.mark.parametrize("authority", ["tunnel.example", "tunnel.example:abc", "tunnel.example:0",
"tunnel.example:70000", "user@tunnel.example:443", "user:pw@tunnel.example:443",
":443", "[::1", "::1:443", "tunnel.example:+443", "tunnel.example: 443"])
def test_malformed_connect_is_refused_without_a_check(proxy_for, authority):
addresses_for = allow(("https://", "127.0.0.1"))
assert exchange(proxy_for(addresses_for), f"CONNECT {authority} HTTP/1.1\r\n\r\n".encode()) == b""
assert addresses_for.calls == []
def test_connect_to_bracketed_ipv6_literal_is_checked_with_brackets(proxy_for):
addresses_for = allow()
assert exchange(proxy_for(addresses_for), b"CONNECT [::1]:443 HTTP/1.1\r\n\r\n") == b""
assert addresses_for.calls == ["https://[::1]:443/"]
# --- plain HTTP (absolute form) --------------------------------------------------------------------------------
def test_absolute_form_request_is_forwarded_in_origin_form(upstreams, proxy_for):
up = upstreams()
addresses_for = allow((f"http://site.example:{up.port}/", "127.0.0.1"))
reply = exchange(proxy_for(addresses_for),
f"POST http://site.example:{up.port}/u?x=1 HTTP/1.1\r\nHost: site.example:{up.port}\r\n"
"Proxy-Connection: keep-alive\r\nConnection: keep-alive, X-Secret\r\nX-Secret: 1\r\n"
"Content-Length: 5\r\n\r\nhello".encode())
assert reply.endswith(b"ok")
settle()
head, _, body = bytes(up.connections[0]).partition(b"\r\n\r\n")
lines = head.decode().split("\r\n")
assert lines[0] == "POST /u?x=1 HTTP/1.1" and body == b"hello"
names = [line.split(":")[0].lower() for line in lines[1:]]
assert "host" in names and "proxy-connection" not in names and "x-secret" not in names
assert "Connection: close" in lines and "Connection: keep-alive, X-Secret" not in lines
assert addresses_for.calls == [f"http://site.example:{up.port}/u?x=1"]
def test_absolute_form_denied(upstreams, proxy_for):
up = upstreams()
assert exchange(proxy_for(allow()), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\n\r\n".encode()) == b""
settle()
assert up.connections == []
@pytest.mark.parametrize("target", ["/relative", "https://site.example/", "ftp://site.example/", "http:///x",
"http://site.example:0/", "http://site.example:abc/", "http://u:p@site.example/"])
def test_other_request_targets_are_refused_without_a_check(proxy_for, target):
addresses_for = allow(("http://", "127.0.0.1"))
assert exchange(proxy_for(addresses_for), f"GET {target} HTTP/1.1\r\nHost: x\r\n\r\n".encode()) == b""
assert addresses_for.calls == []
def test_request_with_transfer_encoding_is_dropped(upstreams, proxy_for):
up = upstreams()
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
reply = exchange(proxy_for(addresses_for), f"POST http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n"
"Transfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n".encode())
settle()
assert reply == b"" and up.connections == []
@pytest.mark.parametrize("lengths", [("5", "5"), ("5", "7"), ("5, 5",), ("-1",), ("abc",)])
def test_duplicate_or_bad_content_length_is_rejected(upstreams, proxy_for, lengths):
up = upstreams()
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
headers = "".join(f"Content-Length: {v}\r\n" for v in lengths)
reply = exchange(proxy_for(addresses_for),
f"POST http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n{headers}\r\nhello".encode())
settle()
assert reply == b"" and up.connections == []
def test_pipelined_second_request_is_never_forwarded(upstreams, proxy_for):
a, b = upstreams(), upstreams()
addresses_for = allow((f"http://127.0.0.1:{a.port}/", "127.0.0.1"), (f"http://127.0.0.1:{b.port}/", "127.0.0.1"))
first = f"GET http://127.0.0.1:{a.port}/first HTTP/1.1\r\nHost: a\r\n\r\n"
second = f"GET http://127.0.0.1:{b.port}/second HTTP/1.1\r\nHost: b\r\n\r\n"
reply = exchange(proxy_for(addresses_for), (first + second).encode())
settle()
assert reply.count(b"HTTP/1.1 200 OK") == 1
assert b.connections == [] and len(a.connections) == 1
assert bytes(a.connections[0]).count(b" HTTP/1.1\r\n") == 1 and b"/second" not in a.connections[0]
def test_second_request_on_the_same_connection_is_never_forwarded(upstreams, proxy_for):
a, b = upstreams(), upstreams()
addresses_for = allow((f"http://127.0.0.1:{a.port}/", "127.0.0.1"), (f"http://127.0.0.1:{b.port}/", "127.0.0.1"))
port = int(proxy_for(addresses_for).address.rsplit(":", 1)[1])
with socket.create_connection(("127.0.0.1", port)) as s:
s.settimeout(3)
s.sendall(f"GET http://127.0.0.1:{a.port}/first HTTP/1.1\r\nHost: a\r\n\r\n".encode())
got = bytearray()
while not got.endswith(b"ok"):
got += s.recv(65536)
assert b"Connection: close" in got
try:
s.sendall(f"GET http://127.0.0.1:{b.port}/second HTTP/1.1\r\nHost: b\r\n\r\n".encode())
assert s.recv(65536) == b""
except (BrokenPipeError, ConnectionResetError):
pass
settle()
assert b.connections == [] and b"/second" not in a.connections[0]
def test_chunked_response_passes_through_intact(upstreams, proxy_for):
body = b"4\r\nWiki\r\n5\r\npedia\r\n0\r\n\r\n"
up = upstreams(reply=b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nContent-Type: text/plain\r\n\r\n" + body)
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
reply = exchange(proxy_for(addresses_for), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode())
head, _, rest = reply.partition(b"\r\n\r\n")
assert b"Transfer-Encoding: chunked" in head and rest == body
def test_interim_1xx_responses_are_skipped_and_hop_by_hop_headers_of_the_final_one_stripped(upstreams, proxy_for):
up = upstreams(reply=b"HTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 103 Early Hints\r\nLink: </a.css>\r\n\r\n"
b"HTTP/1.1 200 OK\r\nConnection: keep-alive, X-Hop\r\nKeep-Alive: timeout=5\r\nX-Hop: 1\r\n"
b"Proxy-Connection: keep-alive\r\nContent-Length: 2\r\n\r\nok")
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
reply = exchange(proxy_for(addresses_for), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode())
head, _, rest = reply.partition(b"\r\n\r\n")
lines = head.decode().split("\r\n")
assert lines[0] == "HTTP/1.1 200 OK" and rest == b"ok" and reply.count(b"HTTP/1.1 ") == 1
names = [line.split(":")[0].lower() for line in lines[1:]]
assert "keep-alive" not in names and "x-hop" not in names and "proxy-connection" not in names
assert lines.count("Connection: close") == 1 and names.count("connection") == 1
def test_switching_protocols_is_not_passed_on(upstreams, proxy_for):
up = upstreams(reply=b"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\nraw")
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
reply = exchange(proxy_for(addresses_for), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode())
assert reply == b""
+13
View File
@@ -55,3 +55,16 @@ def test_error_text_never_contains_the_key():
def test_invalid_endpoint_url_means_unavailable(base_url): def test_invalid_endpoint_url_means_unavailable(base_url):
with pytest.raises(LLMUnavailable): with pytest.raises(LLMUnavailable):
LLM(base_url, "m", "k", client=client(lambda r: httpx.Response(200, json={}))).ask("s", "u") LLM(base_url, "m", "k", client=client(lambda r: httpx.Response(200, json={}))).ask("s", "u")
@pytest.mark.parametrize("given,expected", [(None, 60), (5.5, 5.5), (120, 60)])
def test_timeout_can_be_capped_per_call_but_never_raised(given, expected):
seen = {}
def handler(request):
seen["timeout"] = request.extensions["timeout"]
return httpx.Response(200, json={"choices": [{"message": {"content": "x"}}]})
llm = LLM("https://ai.example", "m", "k", timeout=60, client=client(handler))
llm.ask("s", "u") if given is None else llm.ask("s", "u", timeout=given)
assert seen["timeout"] == {"connect": expected, "read": expected, "write": expected, "pool": expected}
+123
View File
@@ -0,0 +1,123 @@
"""The HTTP service: answers for broken cases and bodies, model configuration, and what it logs.
Needs Playwright and Flask for the import: run with tests/run-in-docker.sh."""
import importlib
import logging
import pytest
pytest.importorskip("playwright.sync_api")
pytest.importorskip("flask")
ENV = {"LLM_BASE_URL": "https://ai.example", "LLM_MODEL": "deepseek-chat", "LLM_API_KEY": "sk-test"}
KEYS = ["evidence", "evidence_url", "reason", "result", "stage", "steps"]
URL = "https://shop.example/u?token=secret"
ADDRESS = "anne@wagnersnetz.de"
@pytest.fixture
def server(monkeypatch):
for k, v in ENV.items():
monkeypatch.setenv(k, v)
import server as module
return importlib.reload(module)
@pytest.fixture
def client(server):
return server.app.test_client()
def result(reason, result="unsubscribed"):
return {"result": result, "reason": reason, "evidence": "Sie wurden abgemeldet", "evidence_url": URL,
"steps": 2, "stage": 2}
def test_health(client):
r = client.get("/health")
assert r.status_code == 200 and r.get_json() == {"status": "ok"}
@pytest.mark.parametrize("body", ["not json", "[1, 2]", "null", "{}", '{"url": 1}', '{"url": "https://x.example"'
', "address": 5}', '"https://x.example"'])
def test_bad_bodies_answer_400(client, body):
assert client.post("/run", data=body, content_type="application/json").status_code == 400
def test_exception_in_a_case_is_unclear_and_logs_nothing_personal(server, client, monkeypatch, caplog):
def boom(*args, **kw):
raise RuntimeError(f"{URL} {ADDRESS}")
monkeypatch.setattr(server, "run_case", boom)
caplog.set_level(logging.DEBUG)
r = client.post("/run", json={"url": URL, "address": ADDRESS})
assert r.status_code == 200
body = r.get_json()
assert sorted(body) == KEYS
assert (body["result"], body["reason"], body["steps"], body["stage"]) == ("unclear", "agent_failure", 0, 1)
assert "RuntimeError" in caplog.text
for secret in ("shop.example", "token", "secret", ADDRESS, "anne"):
assert secret not in caplog.text
def test_browser_launch_failure_is_unclear_too(server, client, monkeypatch, caplog):
def no_browser():
raise OSError(f"cannot start browser for {URL}")
monkeypatch.setattr(server, "sync_playwright", no_browser)
caplog.set_level(logging.DEBUG)
body = client.post("/run", json={"url": URL, "address": ADDRESS}).get_json()
assert (body["result"], body["reason"]) == ("unclear", "agent_failure")
assert "shop.example" not in caplog.text and "OSError" in caplog.text
def test_model_reason_is_answered_but_logged_only_as_a_category(server, client, monkeypatch, caplog):
reason = "unsubscribed anne@wagnersnetz.de at https://x.example/u?t=1"
monkeypatch.setattr(server, "run_case", lambda *args, **kw: result(reason))
caplog.set_level(logging.DEBUG)
body = client.post("/run", json={"url": URL, "address": ADDRESS}).get_json()
assert body["reason"] == reason and body["evidence_url"] == URL
assert "anne" not in caplog.text and "x.example" not in caplog.text and "shop.example" not in caplog.text
assert "case finished: unsubscribed (model), 2 step(s), stage 2" in caplog.text
@pytest.mark.parametrize("reason", ["login", "captcha", "blocked_address", "budget", "page_error", "llm_unavailable",
"page confirms without interaction",
"three invalid replies from the model in a row"])
def test_reasons_set_by_code_are_logged_as_they_are(server, client, monkeypatch, caplog, reason):
monkeypatch.setattr(server, "run_case", lambda *args, **kw: result(reason, "not_possible"))
caplog.set_level(logging.INFO)
client.post("/run", json={"url": URL, "address": ADDRESS})
assert f"case finished: not_possible ({reason})" in caplog.text
def test_case_gets_the_configured_model_and_limits(server, client, monkeypatch):
seen = {}
def fake_run_case(browser, url, address, llm, **kw):
seen.update(url=url, address=address, llm=llm, **kw)
return result("budget", "not_possible")
monkeypatch.setattr(server, "run_case", fake_run_case)
client.post("/run", json={"url": URL, "address": ADDRESS})
assert (seen["url"], seen["address"]) == (URL, ADDRESS)
assert seen["max_steps"] == server.MAX_STEPS and seen["case_timeout"] == server.CASE_TIMEOUT
assert seen["llm"]._model == "deepseek-chat"
@pytest.mark.parametrize("missing", sorted(ENV))
@pytest.mark.parametrize("value", [None, "", " "])
def test_missing_model_configuration_stops_the_start(monkeypatch, missing, value):
for k, v in ENV.items():
monkeypatch.setenv(k, v)
import server as module
if value is None:
monkeypatch.delenv(missing)
else:
monkeypatch.setenv(missing, value)
try:
with pytest.raises(RuntimeError, match=missing):
importlib.reload(module)
finally:
monkeypatch.setenv(missing, ENV[missing])
importlib.reload(module)
+113 -5
View File
@@ -1,12 +1,18 @@
"""The browser loop against imitated unsubscribe pages. Needs Playwright: run with tests/run-in-docker.sh.""" """The browser loop against imitated unsubscribe pages. Needs Playwright: run with tests/run-in-docker.sh."""
import logging
import os
import tempfile
import time
from urllib.parse import quote
import pytest import pytest
playwright_api = pytest.importorskip("playwright.sync_api") playwright_api = pytest.importorskip("playwright.sync_api")
import guard # noqa: E402 import guard # noqa: E402
from llm import LLMUnavailable # noqa: E402 from llm import LLMUnavailable # noqa: E402
from runner import run_case # noqa: E402 from runner import BROWSER_ARGS, run_case # noqa: E402
from tests import sites # noqa: E402 from tests import sites # noqa: E402
@@ -36,7 +42,7 @@ def element(prompt: str, label: str) -> int:
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def browser(): def browser():
with playwright_api.sync_playwright() as p: with playwright_api.sync_playwright() as p:
b = p.chromium.launch() b = p.chromium.launch(args=BROWSER_ARGS)
yield b yield b
b.close() b.close()
@@ -48,13 +54,26 @@ def site():
server.shutdown() server.shutdown()
@pytest.fixture
def other():
server, base, log = sites.start_other()
yield base, log
server.shutdown()
def local_ok(base): def local_ok(base):
"""Guard for the tests: the test server on 127.0.0.1 counts as public, everything else goes to the real guard.""" """Guard for the tests: the test server on 127.0.0.1 counts as public, everything else goes to the real guard."""
return lambda url: url.startswith(base) or guard.is_public_url(url) return lambda url: url.startswith(base) or guard.is_public_url(url)
def local_addresses(base):
"""The matching check for the egress proxy: the test server is reached at 127.0.0.1, all else as in production."""
return lambda url: ["127.0.0.1"] if url.startswith(base + "/") else guard.public_addresses(url)
def run(browser, site, path, llm, address="philipp@wagnersnetz.de", **kw): def run(browser, site, path, llm, address="philipp@wagnersnetz.de", **kw):
return run_case(browser, site + path, address, llm, url_ok=local_ok(site), nav_timeout=10, **kw) return run_case(browser, site + path, address, llm, url_ok=local_ok(site), addresses_for=local_addresses(site),
nav_timeout=10, **kw)
def test_one_click_page_needs_no_model(browser, site): def test_one_click_page_needs_no_model(browser, site):
@@ -202,7 +221,24 @@ def test_embedded_private_content_is_blocked_but_the_page_still_works(browser, s
assert run(browser, site, "/embeds-private", Confirm())["result"] == "unsubscribed" assert run(browser, site, "/embeds-private", Confirm())["result"] == "unsubscribed"
def test_download_is_not_kept(browser, site, tmp_path): def _files_with(content: bytes) -> set[str]:
"""Every file below the temp directory (where Playwright keeps downloads) whose content is exactly content."""
found = set()
for root, _, files in os.walk(tempfile.gettempdir()):
for name in files:
path = os.path.join(root, name)
try:
if os.path.getsize(path) == len(content) and open(path, "rb").read() == content:
found.add(path)
except OSError:
pass
return found
def test_download_is_not_kept(browser, site, tmp_path, caplog):
caplog.set_level(logging.INFO, logger="runner")
before = _files_with(b"data")
class Download(Script): class Download(Script):
def ask(self, system, user): def ask(self, system, user):
self.prompts.append(user) self.prompts.append(user)
@@ -212,6 +248,9 @@ def test_download_is_not_kept(browser, site, tmp_path):
r = run(browser, site, "/download", Download()) r = run(browser, site, "/download", Download())
assert r["result"] == "unclear" assert r["result"] == "unclear"
assert ("GET", "/file.bin", {}) in sites.Handler.log # the download really was attempted
assert "download refused" in caplog.text # the browser offered it and it was refused
assert _files_with(b"data") - before == set() # and no file with its content appeared
def test_model_unavailable_is_an_error_not_a_result(browser, site): def test_model_unavailable_is_an_error_not_a_result(browser, site):
@@ -227,5 +266,74 @@ def test_no_address_means_the_address_action_is_refused(browser, site):
def test_page_that_does_not_load_is_a_page_error(browser, site): def test_page_that_does_not_load_is_a_page_error(browser, site):
r = run_case(browser, "http://127.0.0.1:9/gone", "", Script(), url_ok=lambda url: True, nav_timeout=5) r = run_case(browser, "http://127.0.0.1:9/gone", "", Script(), url_ok=lambda url: True,
addresses_for=lambda url: ["127.0.0.1"], nav_timeout=5)
assert (r["result"], r["reason"]) == ("not_possible", "page_error") assert (r["result"], r["reason"]) == ("not_possible", "page_error")
def test_redirect_to_loopback_never_leaves_the_proxy(browser, site, other):
base2, log2 = other
r = run(browser, site, "/redirect?to=" + quote(base2 + "/admin"), Script())
assert log2 == []
assert (r["result"], r["reason"]) == ("not_possible", "blocked_address")
@pytest.mark.parametrize("page", ["/img?src=", "/fetch?to="])
@pytest.mark.parametrize("via_redirect", [False, True])
def test_subresources_to_loopback_never_reach_it(browser, site, other, page, via_redirect):
base2, log2 = other
target = base2 + "/pixel.gif"
if via_redirect: # the route gate sees only the first hop; the proxy must stop the second
target = site + "/redirect?to=" + quote(target)
run(browser, site, page + quote(target), Script())
time.sleep(0.5)
assert log2 == []
if via_redirect:
assert ("GET", "/redirect", {}) in sites.Handler.log # the first hop did go out
def test_endless_script_ends_at_the_time_budget(browser, site):
start = time.monotonic()
r = run(browser, site, "/busy", Script(), case_timeout=5)
assert time.monotonic() - start < 15
assert (r["result"], r["reason"]) == ("not_possible", "budget")
def test_slow_model_ends_at_the_time_budget_and_gets_a_capped_timeout(browser, site):
class Slow(Script):
timeouts = []
def ask(self, system, user, timeout=None):
self.prompts.append(user)
self.timeouts.append(timeout)
time.sleep(4)
return '{"action":"click","element":%d}' % element(user, "Abmelden bestätigen")
llm = Slow()
start = time.monotonic()
r = run(browser, site, "/confirm", llm, case_timeout=3)
assert time.monotonic() - start < 10
assert (r["result"], r["reason"]) == ("not_possible", "budget")
assert llm.timeouts and all(t is not None and 0 < t <= 3 for t in llm.timeouts)
assert not any(method == "POST" for method, _, _ in sites.Handler.log)
def test_webrtc_is_not_available_to_pages(browser, site):
llm = Script()
run(browser, site, "/rtc", llm)
assert "rtc:undefined,undefined" in llm.prompts[0]
def test_blocked_image_on_a_page_that_never_loads_is_a_page_error(browser, site):
start = time.monotonic()
r = run_case(browser, site + "/hang-img", "", Script(), url_ok=local_ok(site), addresses_for=local_addresses(site),
nav_timeout=3)
assert (r["result"], r["reason"]) == ("not_possible", "page_error")
assert time.monotonic() - start < 8
@pytest.mark.parametrize("path", ["/many-links-password", "/frame-password"])
def test_password_field_anywhere_ends_without_the_model(browser, site, path):
llm = Script()
r = run(browser, site, path, llm)
assert (r["result"], r["reason"]) == ("not_possible", "login") and llm.prompts == []