Agent: Befunde aus dem Review der Browser-Schleife behoben
This commit is contained in:
10 files changed
+965
-63
No files matched your search
+112
-31
@@ -1,10 +1,15 @@
|
|||||||
"""Filtering forward proxy for one case: every request of the browser, redirect hops included, passes url_ok here.
|
"""Filtering forward proxy for one case: every request of the browser, redirect hops included, is checked here.
|
||||||
|
|
||||||
Playwright's route handler never sees the hops of a redirect (Chromium follows them itself), so the route gate
|
Playwright's route handler never sees the hops of a redirect (Chromium follows them itself), so the route gate
|
||||||
alone cannot keep the browser away from private addresses. The browser context is pointed at this proxy, which
|
alone cannot keep the browser away from private addresses. The browser context is pointed at this proxy, which
|
||||||
checks each plain-HTTP request by its full URL and each CONNECT tunnel (HTTPS, WebSocket) by its host and port.
|
checks each plain-HTTP request by its full URL and each CONNECT tunnel (HTTPS, WebSocket) by host and port.
|
||||||
|
|
||||||
|
The check is addresses_for(url) -> list of IP addresses: the addresses the host may be reached at, or an empty
|
||||||
|
list if the URL is refused. The proxy connects only to those addresses and never resolves the name itself, so a
|
||||||
|
DNS answer that changes between check and connection (DNS rebinding) cannot redirect it.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
import socket
|
import socket
|
||||||
import socketserver
|
import socketserver
|
||||||
import threading
|
import threading
|
||||||
@@ -14,6 +19,12 @@ MAX_HEADER_LINES = 100
|
|||||||
MAX_LINE = 16384
|
MAX_LINE = 16384
|
||||||
HOP_BY_HOP = {"connection", "proxy-connection", "keep-alive", "proxy-authorization", "proxy-authenticate", "te",
|
HOP_BY_HOP = {"connection", "proxy-connection", "keep-alive", "proxy-authorization", "proxy-authenticate", "te",
|
||||||
"trailer", "upgrade"}
|
"trailer", "upgrade"}
|
||||||
|
_HOST = re.compile(r"[A-Za-z0-9._-]+")
|
||||||
|
_PORT = re.compile(r"[0-9]{1,5}")
|
||||||
|
|
||||||
|
|
||||||
|
def _connect(address: tuple[str, int], timeout: float) -> socket.socket:
|
||||||
|
return socket.create_connection(address, timeout=timeout)
|
||||||
|
|
||||||
|
|
||||||
def _read_head(rfile) -> list[bytes] | None:
|
def _read_head(rfile) -> list[bytes] | None:
|
||||||
@@ -32,17 +43,64 @@ def _headers(lines: list[bytes]) -> list[tuple[str, str]]:
|
|||||||
out = []
|
out = []
|
||||||
for line in lines:
|
for line in lines:
|
||||||
name, sep, value = line.decode("latin-1").partition(":")
|
name, sep, value = line.decode("latin-1").partition(":")
|
||||||
if not sep:
|
if not sep or not name or name != name.strip():
|
||||||
raise ValueError("bad header line")
|
raise ValueError("bad header line")
|
||||||
out.append((name.strip(), value.strip()))
|
out.append((name, value.strip()))
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
def _keep(headers: list[tuple[str, str]]) -> list[tuple[str, str]]:
|
def _keep(headers: list[tuple[str, str]]) -> list[tuple[str, str]]:
|
||||||
|
"""Headers without the hop-by-hop ones, including those the Connection header names."""
|
||||||
named = {token.strip().lower() for k, v in headers if k.lower() == "connection" for token in v.split(",")}
|
named = {token.strip().lower() for k, v in headers if k.lower() == "connection" for token in v.split(",")}
|
||||||
return [(k, v) for k, v in headers if k.lower() not in HOP_BY_HOP and k.lower() not in named]
|
return [(k, v) for k, v in headers if k.lower() not in HOP_BY_HOP and k.lower() not in named]
|
||||||
|
|
||||||
|
|
||||||
|
def _port(text: str) -> int | None:
|
||||||
|
if not _PORT.fullmatch(text):
|
||||||
|
return None
|
||||||
|
port = int(text)
|
||||||
|
return port if 0 < port < 65536 else None
|
||||||
|
|
||||||
|
|
||||||
|
def _authority(target: str) -> tuple[str, int] | None:
|
||||||
|
"""Host and port of a CONNECT target: host:port or [ipv6]:port, nothing else (no userinfo, no default port)."""
|
||||||
|
if target.startswith("["):
|
||||||
|
host, sep, port = target[1:].partition("]:")
|
||||||
|
if not sep or not host or not all(c in "0123456789abcdefABCDEF:." for c in host):
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
host, sep, port = target.rpartition(":")
|
||||||
|
if not sep or not _HOST.fullmatch(host):
|
||||||
|
return None
|
||||||
|
number = _port(port)
|
||||||
|
return (host, number) if number else None
|
||||||
|
|
||||||
|
|
||||||
|
def _absolute(target: str) -> tuple[str, int, str] | None:
|
||||||
|
"""Host, port and origin-form path of an absolute http:// request target."""
|
||||||
|
try:
|
||||||
|
parts = urlsplit(target)
|
||||||
|
port = parts.port
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
if parts.scheme != "http" or not parts.hostname or "@" in parts.netloc or port == 0:
|
||||||
|
return None
|
||||||
|
path = (parts.path or "/") + (f"?{parts.query}" if parts.query else "")
|
||||||
|
return parts.hostname, port or 80, path
|
||||||
|
|
||||||
|
|
||||||
|
def _content_length(headers: list[tuple[str, str]]) -> int | None:
|
||||||
|
"""The request body length; None if the request must be refused (chunked, duplicate or bad length)."""
|
||||||
|
if any(k.lower() == "transfer-encoding" for k, _ in headers):
|
||||||
|
return None
|
||||||
|
values = [v for k, v in headers if k.lower() == "content-length"]
|
||||||
|
if not values:
|
||||||
|
return 0
|
||||||
|
if len(values) > 1 or not values[0].isdigit():
|
||||||
|
return None
|
||||||
|
return int(values[0])
|
||||||
|
|
||||||
|
|
||||||
def _pump(src, dst) -> None:
|
def _pump(src, dst) -> None:
|
||||||
"""Copy bytes from src (socket or buffered file) to dst until src ends or either side fails."""
|
"""Copy bytes from src (socket or buffered file) to dst until src ends or either side fails."""
|
||||||
try:
|
try:
|
||||||
@@ -61,6 +119,9 @@ def _pump(src, dst) -> None:
|
|||||||
|
|
||||||
|
|
||||||
class _Handler(socketserver.StreamRequestHandler):
|
class _Handler(socketserver.StreamRequestHandler):
|
||||||
|
"""One client connection carries exactly one request (or one tunnel); then both sides close. A kept-alive
|
||||||
|
connection could otherwise carry the browser's next request, meant for another host, to this upstream."""
|
||||||
|
|
||||||
def handle(self):
|
def handle(self):
|
||||||
proxy = self.server.egress
|
proxy = self.server.egress
|
||||||
self.connection.settimeout(proxy.timeout)
|
self.connection.settimeout(proxy.timeout)
|
||||||
@@ -73,40 +134,53 @@ class _Handler(socketserver.StreamRequestHandler):
|
|||||||
except ValueError:
|
except ValueError:
|
||||||
return
|
return
|
||||||
if method == "CONNECT":
|
if method == "CONNECT":
|
||||||
host, _, port = target.rpartition(":")
|
authority = _authority(target)
|
||||||
url = f"https://{host}:{port}/"
|
if not authority:
|
||||||
else:
|
|
||||||
url = target
|
|
||||||
parts = urlsplit(target)
|
|
||||||
host, port = parts.hostname, parts.port or 80
|
|
||||||
if parts.scheme != "http" or not host:
|
|
||||||
return
|
return
|
||||||
host = f"[{host}]" if ":" in host else host
|
host, port = authority
|
||||||
if not proxy.url_ok(url):
|
url = f"https://[{host}]:{port}/" if ":" in host else f"https://{host}:{port}/"
|
||||||
|
length = path = None
|
||||||
|
else:
|
||||||
|
absolute = _absolute(target)
|
||||||
|
length = _content_length(headers)
|
||||||
|
if not absolute or length is None or not version.startswith("HTTP/1."):
|
||||||
|
return
|
||||||
|
host, port, path = absolute
|
||||||
|
url = target
|
||||||
|
try:
|
||||||
|
addresses = list(proxy.addresses_for(url))
|
||||||
|
except Exception:
|
||||||
|
addresses = []
|
||||||
|
if not addresses:
|
||||||
proxy.blocked.append(url)
|
proxy.blocked.append(url)
|
||||||
return # no answer at all: the browser sees a failed request, as with a blocked route
|
return # no answer at all: the browser sees a failed request, as with a blocked route
|
||||||
try:
|
upstream = self._open(addresses, port, proxy.timeout)
|
||||||
upstream = socket.create_connection((host.strip("[]"), int(port)), timeout=proxy.timeout)
|
if upstream is None:
|
||||||
except (OSError, ValueError):
|
|
||||||
return
|
return
|
||||||
with upstream:
|
with upstream:
|
||||||
if method == "CONNECT":
|
if method == "CONNECT":
|
||||||
|
self._tunnel(upstream, proxy.timeout)
|
||||||
|
else:
|
||||||
|
self._forward(upstream, method, path, version, headers, length)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _open(addresses: list[str], port: int, timeout: float) -> socket.socket | None:
|
||||||
|
for address in addresses:
|
||||||
|
try:
|
||||||
|
return _connect((address, port), timeout)
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _tunnel(self, upstream, timeout):
|
||||||
self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
|
self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
|
||||||
self.wfile.flush()
|
self.wfile.flush()
|
||||||
t = threading.Thread(target=_pump, args=(self.rfile, upstream), daemon=True)
|
t = threading.Thread(target=_pump, args=(self.rfile, upstream), daemon=True)
|
||||||
t.start()
|
t.start()
|
||||||
_pump(upstream, self.connection)
|
_pump(upstream, self.connection)
|
||||||
t.join(proxy.timeout)
|
t.join(timeout)
|
||||||
return
|
|
||||||
self._forward(upstream, method, parts, version, headers)
|
|
||||||
|
|
||||||
def _forward(self, upstream, method, parts, version, headers):
|
def _forward(self, upstream, method, path, version, headers, length):
|
||||||
"""One plain-HTTP request, then both connections close: a kept-alive connection could carry the next
|
|
||||||
request of the browser, meant for another host, to this upstream without a check."""
|
|
||||||
if any(k.lower() == "transfer-encoding" for k, _ in headers):
|
|
||||||
return # chunked request bodies are not supported; the request fails
|
|
||||||
length = next((int(v) for k, v in headers if k.lower() == "content-length" and v.isdigit()), 0)
|
|
||||||
path = (parts.path or "/") + (f"?{parts.query}" if parts.query else "")
|
|
||||||
lines = [f"{method} {path} {version}"] + [f"{k}: {v}" for k, v in _keep(headers)] + ["Connection: close"]
|
lines = [f"{method} {path} {version}"] + [f"{k}: {v}" for k, v in _keep(headers)] + ["Connection: close"]
|
||||||
upstream.sendall(("\r\n".join(lines) + "\r\n\r\n").encode("latin-1"))
|
upstream.sendall(("\r\n".join(lines) + "\r\n\r\n").encode("latin-1"))
|
||||||
while length > 0:
|
while length > 0:
|
||||||
@@ -116,14 +190,21 @@ class _Handler(socketserver.StreamRequestHandler):
|
|||||||
upstream.sendall(chunk)
|
upstream.sendall(chunk)
|
||||||
length -= len(chunk)
|
length -= len(chunk)
|
||||||
ufile = upstream.makefile("rb")
|
ufile = upstream.makefile("rb")
|
||||||
|
while True:
|
||||||
head = _read_head(ufile)
|
head = _read_head(ufile)
|
||||||
if not head:
|
if not head:
|
||||||
return
|
return
|
||||||
|
status = head[0].decode("latin-1")
|
||||||
|
code = status.split(" ")[1] if status.count(" ") else ""
|
||||||
|
if code == "101":
|
||||||
|
return # no protocol switches: the Upgrade header was not passed on, so this is not ours
|
||||||
|
if not (len(code) == 3 and code.startswith("1")):
|
||||||
|
break # interim 1xx answers (100 Continue, 103 Early Hints) are dropped; the final one counts
|
||||||
try:
|
try:
|
||||||
status, response_headers = head[0], _keep(_headers(head[1:]))
|
response_headers = _keep(_headers(head[1:]))
|
||||||
except ValueError:
|
except ValueError:
|
||||||
return
|
return
|
||||||
out = [status.decode("latin-1")] + [f"{k}: {v}" for k, v in response_headers] + ["Connection: close"]
|
out = [status] + [f"{k}: {v}" for k, v in response_headers] + ["Connection: close"]
|
||||||
self.wfile.write(("\r\n".join(out) + "\r\n\r\n").encode("latin-1"))
|
self.wfile.write(("\r\n".join(out) + "\r\n\r\n").encode("latin-1"))
|
||||||
self.wfile.flush()
|
self.wfile.flush()
|
||||||
_pump(ufile, self.connection)
|
_pump(ufile, self.connection)
|
||||||
@@ -138,10 +219,10 @@ class _Server(socketserver.ThreadingTCPServer):
|
|||||||
|
|
||||||
|
|
||||||
class EgressProxy:
|
class EgressProxy:
|
||||||
"""Proxy on 127.0.0.1 with a random port, between start() and stop(). Blocked URLs are appended to blocked."""
|
"""Proxy on 127.0.0.1 with a random port, between start() and stop(). Refused URLs are appended to blocked."""
|
||||||
|
|
||||||
def __init__(self, url_ok, blocked: list, timeout: float = 30):
|
def __init__(self, addresses_for, blocked: list, timeout: float = 30):
|
||||||
self.url_ok = url_ok
|
self.addresses_for = addresses_for
|
||||||
self.blocked = blocked
|
self.blocked = blocked
|
||||||
self.timeout = timeout
|
self.timeout = timeout
|
||||||
self._server = _Server(("127.0.0.1", 0), _Handler)
|
self._server = _Server(("127.0.0.1", 0), _Handler)
|
||||||
|
|||||||
+5
-2
@@ -15,14 +15,17 @@ class LLM:
|
|||||||
self._timeout = timeout
|
self._timeout = timeout
|
||||||
self._client = client or httpx.Client()
|
self._client = client or httpx.Client()
|
||||||
|
|
||||||
def ask(self, system: str, user: str) -> str:
|
def ask(self, system: str, user: str, timeout: float | None = None) -> str:
|
||||||
|
"""The model's reply text. timeout (seconds) can only shorten the configured one, for example to the
|
||||||
|
remaining budget of a case."""
|
||||||
|
timeout = self._timeout if timeout is None else min(timeout, self._timeout)
|
||||||
body = {
|
body = {
|
||||||
"model": self._model, "stream": False, "temperature": 0,
|
"model": self._model, "stream": False, "temperature": 0,
|
||||||
"messages": [{"role": "system", "content": system}, {"role": "user", "content": user}],
|
"messages": [{"role": "system", "content": system}, {"role": "user", "content": user}],
|
||||||
}
|
}
|
||||||
try:
|
try:
|
||||||
r = self._client.post(self._url, json=body, headers={"Authorization": f"Bearer {self._key}"},
|
r = self._client.post(self._url, json=body, headers={"Authorization": f"Bearer {self._key}"},
|
||||||
timeout=self._timeout)
|
timeout=timeout)
|
||||||
except (httpx.HTTPError, httpx.InvalidURL, ValueError) as e: # InvalidURL is no HTTPError
|
except (httpx.HTTPError, httpx.InvalidURL, ValueError) as e: # InvalidURL is no HTTPError
|
||||||
raise LLMUnavailable(f"model endpoint not reachable: {type(e).__name__}") from None
|
raise LLMUnavailable(f"model endpoint not reachable: {type(e).__name__}") from None
|
||||||
if r.status_code != 200:
|
if r.status_code != 200:
|
||||||
|
|||||||
+158
-12
@@ -1,20 +1,41 @@
|
|||||||
"""Browser loop of the unsubscribe agent: stage 1 without the model, stage 2 with one action per step."""
|
"""Browser loop of the unsubscribe agent: stage 1 without the model, stage 2 with one action per step."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import inspect
|
||||||
import logging
|
import logging
|
||||||
|
import threading
|
||||||
import time
|
import time
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
from egress import EgressProxy
|
from egress import EgressProxy
|
||||||
from guard import is_public_url
|
from guard import is_public_url, public_addresses
|
||||||
from llm import LLMUnavailable
|
from llm import LLMUnavailable
|
||||||
from pagecheck import has_password, is_login_url, looks_unsubscribed, needs_input
|
from pagecheck import has_password, is_login_url, looks_unsubscribed, needs_input
|
||||||
from protocol import SYSTEM_PROMPT, ProtocolError, build_prompt, parse_action
|
from protocol import SYSTEM_PROMPT, ProtocolError, build_prompt, parse_action
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Launch arguments for every browser that runs cases (the service and the tests use the same).
|
||||||
|
# WebRTC may only use proxied connections: no UDP (STUN/TURN) past the egress proxy.
|
||||||
|
BROWSER_ARGS = ["--disable-dev-shm-usage", "--force-webrtc-ip-handling-policy=disable_non_proxied_udp"]
|
||||||
|
WATCHDOG_MARGIN = 2.0 # seconds past the case budget after which the browser context is closed by force
|
||||||
MAX_TEXT = 6000
|
MAX_TEXT = 6000
|
||||||
MAX_ELEMENTS = 60
|
MAX_ELEMENTS = 60
|
||||||
EVIDENCE = 400
|
EVIDENCE = 400
|
||||||
|
|
||||||
|
# Reasons set by this code; any other reason was written by the model (and may contain personal data).
|
||||||
|
CODE_REASONS = frozenset({
|
||||||
|
"blocked_address", "page_error", "login", "captcha", "budget", "llm_unavailable", "agent_failure",
|
||||||
|
"page confirms without interaction", "three invalid replies from the model in a row"})
|
||||||
|
|
||||||
|
# Removes the WebRTC constructors before any page script runs (the launch flag above is the second line).
|
||||||
|
NO_WEBRTC_JS = """
|
||||||
|
for (const name of ['RTCPeerConnection', 'webkitRTCPeerConnection', 'RTCDataChannel']) {
|
||||||
|
try { delete window[name]; } catch (e) {}
|
||||||
|
try { Object.defineProperty(window, name, {value: undefined, configurable: false, writable: false}); } catch (e) {}
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
|
||||||
# Lists the visible controls and marks each with data-ua-idx so the next action can find it again.
|
# Lists the visible controls and marks each with data-ua-idx so the next action can find it again.
|
||||||
SNAPSHOT_JS = """
|
SNAPSHOT_JS = """
|
||||||
([maxElements, maxText]) => {
|
([maxElements, maxText]) => {
|
||||||
@@ -80,73 +101,188 @@ def _act(page, action, address: str) -> str:
|
|||||||
return f"{action.kind} {action.element}"
|
return f"{action.kind} {action.element}"
|
||||||
|
|
||||||
|
|
||||||
|
def _authority(url: str) -> tuple:
|
||||||
|
try:
|
||||||
|
parts = urlsplit(url)
|
||||||
|
return (parts.hostname, parts.port or {"http": 80, "https": 443}.get(parts.scheme))
|
||||||
|
except ValueError:
|
||||||
|
return (None, None)
|
||||||
|
|
||||||
|
|
||||||
|
def _password_anywhere(page) -> bool:
|
||||||
|
"""A visible password field in any frame, not only among the controls listed for the model."""
|
||||||
|
for frame in page.frames:
|
||||||
|
try:
|
||||||
|
if frame.locator("input[type=password]:visible").count():
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
continue # detached or crashed frame
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _ask(llm, prompt: str, timeout: float) -> str:
|
||||||
|
"""Ask the model with a timeout capped by the remaining budget, if its ask() takes one."""
|
||||||
|
try:
|
||||||
|
takes_timeout = "timeout" in inspect.signature(llm.ask).parameters
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
takes_timeout = False
|
||||||
|
if takes_timeout:
|
||||||
|
return llm.ask(SYSTEM_PROMPT, prompt, timeout=timeout)
|
||||||
|
return llm.ask(SYSTEM_PROMPT, prompt)
|
||||||
|
|
||||||
|
|
||||||
|
def _refuse_download(download) -> None:
|
||||||
|
logger.info("download refused")
|
||||||
|
try:
|
||||||
|
download.cancel()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _report_close(future) -> None:
|
||||||
|
if not future.cancelled() and future.exception() is not None:
|
||||||
|
logger.warning("watchdog could not close the browser context: %s", type(future.exception()).__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class _Watchdog:
|
||||||
|
"""Hard wall clock for one case. Playwright's sync API may only be used from its own thread, so the timer
|
||||||
|
schedules the context's close on Playwright's event loop; a call that hangs in the page (an endless script
|
||||||
|
blocks page.evaluate forever) then fails at once, and the case ends with budget."""
|
||||||
|
|
||||||
|
def __init__(self, seconds: float):
|
||||||
|
self.expired = threading.Event()
|
||||||
|
self._context = None
|
||||||
|
self._timer = threading.Timer(seconds, self._fire)
|
||||||
|
self._timer.daemon = True
|
||||||
|
|
||||||
|
def watch(self, context) -> None:
|
||||||
|
self._context = context
|
||||||
|
self._timer.start()
|
||||||
|
|
||||||
|
def _fire(self) -> None:
|
||||||
|
self.expired.set()
|
||||||
|
try:
|
||||||
|
# private parts of Playwright 1.65 (pinned): the async implementation object and its event loop
|
||||||
|
future = asyncio.run_coroutine_threadsafe(self._context._impl_obj.close(None), self._context._loop)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("watchdog could not close the browser context: %s", type(e).__name__)
|
||||||
|
return
|
||||||
|
future.add_done_callback(_report_close)
|
||||||
|
|
||||||
|
def cancel(self) -> None:
|
||||||
|
self._timer.cancel()
|
||||||
|
|
||||||
|
|
||||||
def run_case(browser, url: str, address: str, llm, *, max_steps: int = 20, case_timeout: float = 300,
|
def run_case(browser, url: str, address: str, llm, *, max_steps: int = 20, case_timeout: float = 300,
|
||||||
nav_timeout: float = 30, url_ok=is_public_url, clock=time.monotonic) -> dict:
|
nav_timeout: float = 30, url_ok=is_public_url, addresses_for=public_addresses, clock=time.monotonic) -> dict:
|
||||||
"""Try to unsubscribe at url. Returns result unsubscribed | not_possible | unclear | error with a reason."""
|
"""Try to unsubscribe at url. Returns result unsubscribed | not_possible | unclear | error with a reason.
|
||||||
|
|
||||||
|
url_ok checks the start address, every request the route gate sees and the page address after each step.
|
||||||
|
addresses_for is the egress proxy's check: the addresses a URL may be reached at ([] = refused); the proxy
|
||||||
|
connects only to those. Both must agree on what is allowed.
|
||||||
|
"""
|
||||||
if not url_ok(url):
|
if not url_ok(url):
|
||||||
return _result("not_possible", "blocked_address")
|
return _result("not_possible", "blocked_address")
|
||||||
deadline = clock() + case_timeout
|
deadline = clock() + case_timeout
|
||||||
blocked: list[str] = []
|
watchdog = _Watchdog(case_timeout + WATCHDOG_MARGIN)
|
||||||
# Every request leaves through the egress proxy, which checks url_ok also for redirect hops and tunnels;
|
steps = 0
|
||||||
|
|
||||||
|
def over() -> bool:
|
||||||
|
return watchdog.expired.is_set() or clock() >= deadline
|
||||||
|
|
||||||
|
def budget(evidence: str = "", evidence_url: str = "") -> dict:
|
||||||
|
return _result("not_possible", "budget", evidence, evidence_url, steps, 2 if steps else 1)
|
||||||
|
|
||||||
|
blocked: list[str] = [] # every refused URL (route gate and proxy)
|
||||||
|
main_navigation: list[str] = [] # URLs of the main frame's navigation requests, redirect hops included
|
||||||
|
main_blocked = threading.Event() # the route gate refused a main-frame navigation
|
||||||
|
# Every request leaves through the egress proxy, which checks also redirect hops and tunnels;
|
||||||
# the route gate below sees only the first request of each redirect chain, but with the full https URL.
|
# the route gate below sees only the first request of each redirect chain, but with the full https URL.
|
||||||
proxy = EgressProxy(url_ok, blocked, timeout=nav_timeout).start()
|
proxy = EgressProxy(addresses_for, blocked, timeout=nav_timeout).start()
|
||||||
context = None
|
context = None
|
||||||
|
page = None
|
||||||
try:
|
try:
|
||||||
context = browser.new_context(accept_downloads=False, service_workers="block",
|
context = browser.new_context(accept_downloads=False, service_workers="block",
|
||||||
proxy={"server": proxy.address})
|
proxy={"server": proxy.address})
|
||||||
|
watchdog.watch(context)
|
||||||
|
context.add_init_script(NO_WEBRTC_JS)
|
||||||
|
|
||||||
|
def is_main_navigation(request) -> bool:
|
||||||
|
try:
|
||||||
|
return request.is_navigation_request() and page is not None and request.frame == page.main_frame
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
def gate(route):
|
def gate(route):
|
||||||
if url_ok(route.request.url):
|
if url_ok(route.request.url):
|
||||||
route.continue_()
|
route.continue_()
|
||||||
else:
|
else:
|
||||||
blocked.append(route.request.url)
|
blocked.append(route.request.url)
|
||||||
|
if is_main_navigation(route.request):
|
||||||
|
main_blocked.set()
|
||||||
route.abort("blockedbyclient")
|
route.abort("blockedbyclient")
|
||||||
|
|
||||||
|
def navigation_blocked() -> bool:
|
||||||
|
refused = {_authority(u) for u in list(blocked)}
|
||||||
|
return main_blocked.is_set() or any(_authority(u) in refused for u in main_navigation)
|
||||||
|
|
||||||
context.route("**/*", gate)
|
context.route("**/*", gate)
|
||||||
page = context.new_page()
|
page = context.new_page()
|
||||||
context.on("page", lambda extra: extra.close() if extra != page else None) # pop-ups and new tabs
|
context.on("page", lambda extra: extra.close() if extra != page else None) # pop-ups and new tabs
|
||||||
page.on("dialog", lambda dialog: dialog.dismiss())
|
page.on("dialog", lambda dialog: dialog.dismiss())
|
||||||
page.on("download", lambda download: download.cancel())
|
page.on("download", _refuse_download)
|
||||||
|
page.on("request", lambda request: main_navigation.append(request.url) if is_main_navigation(request) else None)
|
||||||
try:
|
try:
|
||||||
page.goto(url, timeout=nav_timeout * 1000, wait_until="domcontentloaded")
|
page.goto(url, timeout=nav_timeout * 1000, wait_until="domcontentloaded")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
reason = "blocked_address" if blocked else "page_error"
|
if over():
|
||||||
|
return budget()
|
||||||
|
reason = "blocked_address" if navigation_blocked() else "page_error"
|
||||||
logger.info("first load failed (%s): %s", reason, type(e).__name__)
|
logger.info("first load failed (%s): %s", reason, type(e).__name__)
|
||||||
return _result("not_possible", reason)
|
return _result("not_possible", reason)
|
||||||
_settle(page, nav_timeout)
|
_settle(page, nav_timeout)
|
||||||
|
|
||||||
history: list[str] = []
|
history: list[str] = []
|
||||||
bad_replies = 0
|
bad_replies = 0
|
||||||
steps = 0
|
|
||||||
while True:
|
while True:
|
||||||
|
if over():
|
||||||
|
return budget()
|
||||||
try:
|
try:
|
||||||
snap = page.evaluate(SNAPSHOT_JS, [MAX_ELEMENTS, MAX_TEXT])
|
snap = page.evaluate(SNAPSHOT_JS, [MAX_ELEMENTS, MAX_TEXT])
|
||||||
except Exception:
|
except Exception:
|
||||||
|
if over():
|
||||||
|
return budget()
|
||||||
_settle(page, nav_timeout) # the page was still navigating; one more try
|
_settle(page, nav_timeout) # the page was still navigating; one more try
|
||||||
try:
|
try:
|
||||||
snap = page.evaluate(SNAPSHOT_JS, [MAX_ELEMENTS, MAX_TEXT])
|
snap = page.evaluate(SNAPSHOT_JS, [MAX_ELEMENTS, MAX_TEXT])
|
||||||
except Exception:
|
except Exception:
|
||||||
|
if over():
|
||||||
|
return budget()
|
||||||
return _result("not_possible", "page_error", steps=steps, stage=2 if steps else 1)
|
return _result("not_possible", "page_error", steps=steps, stage=2 if steps else 1)
|
||||||
stage = 2 if steps else 1
|
stage = 2 if steps else 1
|
||||||
here = page.url
|
here = page.url
|
||||||
if not url_ok(here) and here != "about:blank":
|
if not url_ok(here) and here != "about:blank":
|
||||||
return _result("not_possible", "blocked_address", steps=steps, stage=stage)
|
return _result("not_possible", "blocked_address", steps=steps, stage=stage)
|
||||||
if has_password(snap["elements"]) or is_login_url(here):
|
if has_password(snap["elements"]) or is_login_url(here) or _password_anywhere(page):
|
||||||
return _result("not_possible", "login", snap["tail"], here, steps, stage)
|
return _result("not_possible", "login", snap["tail"], here, steps, stage)
|
||||||
if snap["captcha"]:
|
if snap["captcha"]:
|
||||||
return _result("not_possible", "captcha", snap["tail"], here, steps, stage)
|
return _result("not_possible", "captcha", snap["tail"], here, steps, stage)
|
||||||
if steps == 0 and looks_unsubscribed(snap["text"]) and not needs_input(snap["elements"]):
|
if steps == 0 and looks_unsubscribed(snap["text"]) and not needs_input(snap["elements"]):
|
||||||
return _result("unsubscribed", "page confirms without interaction", snap["tail"], here, 0, 1)
|
return _result("unsubscribed", "page confirms without interaction", snap["tail"], here, 0, 1)
|
||||||
if steps >= max_steps or clock() >= deadline:
|
if steps >= max_steps or over():
|
||||||
return _result("not_possible", "budget", snap["tail"], here, steps, 2)
|
return _result("not_possible", "budget", snap["tail"], here, steps, 2)
|
||||||
|
|
||||||
steps += 1
|
steps += 1
|
||||||
prompt = build_prompt(snap["title"], here, snap["text"], snap["elements"], history, bool(address))
|
prompt = build_prompt(snap["title"], here, snap["text"], snap["elements"], history, bool(address))
|
||||||
try:
|
try:
|
||||||
reply = llm.ask(SYSTEM_PROMPT, prompt)
|
reply = _ask(llm, prompt, max(deadline - clock(), 0.001))
|
||||||
except LLMUnavailable as e:
|
except LLMUnavailable as e:
|
||||||
|
if over():
|
||||||
|
return budget(snap["tail"], here)
|
||||||
logger.warning("model unavailable: %s", e)
|
logger.warning("model unavailable: %s", e)
|
||||||
return _result("error", "llm_unavailable", steps=steps - 1, stage=2)
|
return _result("error", "llm_unavailable", steps=steps - 1, stage=2)
|
||||||
|
if over():
|
||||||
|
return budget(snap["tail"], here)
|
||||||
try:
|
try:
|
||||||
action = parse_action(reply, snap["elements"], bool(address))
|
action = parse_action(reply, snap["elements"], bool(address))
|
||||||
except ProtocolError as e:
|
except ProtocolError as e:
|
||||||
@@ -161,9 +297,19 @@ def run_case(browser, url: str, address: str, llm, *, max_steps: int = 20, case_
|
|||||||
try:
|
try:
|
||||||
history.append(_act(page, action, address) + " -> ok")
|
history.append(_act(page, action, address) + " -> ok")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
if over():
|
||||||
|
return budget(snap["tail"], here)
|
||||||
history.append(f"{action.kind} {action.element} -> failed ({type(e).__name__})")
|
history.append(f"{action.kind} {action.element} -> failed ({type(e).__name__})")
|
||||||
_settle(page, nav_timeout)
|
_settle(page, nav_timeout)
|
||||||
|
except Exception:
|
||||||
|
if over():
|
||||||
|
return budget()
|
||||||
|
raise
|
||||||
finally:
|
finally:
|
||||||
|
watchdog.cancel()
|
||||||
if context is not None:
|
if context is not None:
|
||||||
|
try:
|
||||||
context.close()
|
context.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
proxy.stop()
|
proxy.stop()
|
||||||
+34
-11
@@ -8,19 +8,32 @@ from flask import Flask, jsonify, request
|
|||||||
from playwright.sync_api import sync_playwright
|
from playwright.sync_api import sync_playwright
|
||||||
|
|
||||||
from llm import LLM
|
from llm import LLM
|
||||||
from runner import run_case
|
from runner import BROWSER_ARGS, CODE_REASONS, run_case
|
||||||
|
|
||||||
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"))
|
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"))
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
app = Flask(__name__)
|
|
||||||
LOCK = threading.Lock() # one case at a time
|
def _required(name: str) -> str:
|
||||||
|
"""A setting the service cannot work without: a missing one stops the start instead of failing every case."""
|
||||||
|
value = os.environ.get(name, "").strip()
|
||||||
|
if not value:
|
||||||
|
raise RuntimeError(f"{name} is not set: the agent cannot reach the model without it")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
MODEL = LLM(_required("LLM_BASE_URL"), _required("LLM_MODEL"), _required("LLM_API_KEY")) # one at a time: shared
|
||||||
MAX_STEPS = int(os.environ.get("MAX_STEPS", "20"))
|
MAX_STEPS = int(os.environ.get("MAX_STEPS", "20"))
|
||||||
CASE_TIMEOUT = float(os.environ.get("CASE_TIMEOUT", "300"))
|
CASE_TIMEOUT = float(os.environ.get("CASE_TIMEOUT", "300"))
|
||||||
|
|
||||||
|
app = Flask(__name__)
|
||||||
|
LOCK = threading.Lock() # one case at a time
|
||||||
|
|
||||||
def _llm() -> LLM:
|
|
||||||
return LLM(os.environ["LLM_BASE_URL"], os.environ["LLM_MODEL"], os.environ["LLM_API_KEY"])
|
def _failure() -> dict:
|
||||||
|
"""Result for a case the agent itself failed on. Unclear, not error: the attempt is used up, so one bad page
|
||||||
|
cannot block the queue of the caller."""
|
||||||
|
return {"result": "unclear", "reason": "agent_failure", "evidence": "", "evidence_url": "", "steps": 0, "stage": 1}
|
||||||
|
|
||||||
|
|
||||||
@app.route("/health")
|
@app.route("/health")
|
||||||
@@ -30,16 +43,26 @@ def health():
|
|||||||
|
|
||||||
@app.route("/run", methods=["POST"])
|
@app.route("/run", methods=["POST"])
|
||||||
def run():
|
def run():
|
||||||
data = request.get_json(silent=True) or {}
|
data = request.get_json(silent=True)
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return jsonify({"error": "body must be a JSON object"}), 400
|
||||||
url, address = data.get("url"), data.get("address", "")
|
url, address = data.get("url"), data.get("address", "")
|
||||||
if not isinstance(url, str) or not isinstance(address, str):
|
if not isinstance(url, str) or not isinstance(address, str):
|
||||||
return jsonify({"error": "url and address must be strings"}), 400
|
return jsonify({"error": "url and address must be strings"}), 400
|
||||||
with LOCK, sync_playwright() as p:
|
with LOCK:
|
||||||
browser = p.chromium.launch(args=["--disable-dev-shm-usage"])
|
|
||||||
try:
|
try:
|
||||||
result = run_case(browser, url, address, _llm(), max_steps=MAX_STEPS, case_timeout=CASE_TIMEOUT)
|
with sync_playwright() as p:
|
||||||
|
browser = p.chromium.launch(args=BROWSER_ARGS)
|
||||||
|
try:
|
||||||
|
result = run_case(browser, url, address, MODEL, max_steps=MAX_STEPS, case_timeout=CASE_TIMEOUT)
|
||||||
finally:
|
finally:
|
||||||
browser.close()
|
browser.close()
|
||||||
logger.info("case finished: %s (%s), %d step(s), stage %d", result["result"], result["reason"], result["steps"],
|
except Exception as e:
|
||||||
result["stage"]) # never log the url or the address: both are personal data of the users
|
# only the type: messages of browser and library errors may contain the url
|
||||||
|
logger.warning("case failed in the agent: %s", type(e).__name__)
|
||||||
|
result = _failure()
|
||||||
|
# never log the url, the address or a reason the model wrote: all may contain personal data of the users
|
||||||
|
category = result["reason"] if result["reason"] in CODE_REASONS else "model"
|
||||||
|
logger.info("case finished: %s (%s), %d step(s), stage %d", result["result"], category, result["steps"],
|
||||||
|
result["stage"])
|
||||||
return jsonify(result)
|
return jsonify(result)
|
||||||
@@ -4,5 +4,6 @@
|
|||||||
set -eu
|
set -eu
|
||||||
# --load: with a docker-container buildx builder the image would otherwise stay in the build cache only
|
# --load: with a docker-container buildx builder the image would otherwise stay in the build cache only
|
||||||
docker build -q --load -t unsubscriber-agent-test agent >/dev/null
|
docker build -q --load -t unsubscriber-agent-test agent >/dev/null
|
||||||
docker run --rm --network none --user pwuser -v "$PWD/agent/tests:/app/tests:ro" unsubscriber-agent-test \
|
# --init: python would be PID 1 and ignore SIGTERM, so a hanging run could not be stopped
|
||||||
|
docker run --init --rm --network none --user pwuser -v "$PWD/agent/tests:/app/tests:ro" unsubscriber-agent-test \
|
||||||
python -m pytest -q -p no:cacheprovider /app/tests
|
python -m pytest -q -p no:cacheprovider /app/tests
|
||||||
+59
-1
@@ -1,6 +1,8 @@
|
|||||||
"""Tiny web server with imitated unsubscribe pages for the browser tests."""
|
"""Tiny web server with imitated unsubscribe pages for the browser tests."""
|
||||||
|
|
||||||
|
import html
|
||||||
import threading
|
import threading
|
||||||
|
import time
|
||||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
from urllib.parse import parse_qs, urlsplit
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
|
||||||
@@ -27,6 +29,14 @@ PAGES = {
|
|||||||
"/download": "<p>Bitte bestätigen</p><a href='/file.bin' download>Abmelden bestätigen</a>",
|
"/download": "<p>Bitte bestätigen</p><a href='/file.bin' download>Abmelden bestätigen</a>",
|
||||||
"/home": "<p>Welcome</p>",
|
"/home": "<p>Welcome</p>",
|
||||||
"/bought": "<p>Thank you for your purchase</p>",
|
"/bought": "<p>Thank you for your purchase</p>",
|
||||||
|
"/busy": "<p>Bitte bestätigen</p><button>Abmelden bestätigen</button>"
|
||||||
|
"<script>addEventListener('load', () => setTimeout(() => { while (true) {} }, 50))</script>",
|
||||||
|
"/rtc": "<p id='r'>?</p><button>Abmelden</button><script>document.getElementById('r').textContent ="
|
||||||
|
" 'rtc:' + typeof RTCPeerConnection + ',' + typeof webkitRTCPeerConnection</script>",
|
||||||
|
"/many-links-password": "".join(f"<a href='/home?{i}'>Link {i}</a> " for i in range(70)) +
|
||||||
|
"<form><input type='password' name='pw'><button>Abmelden</button></form>",
|
||||||
|
"/frame-password": "<p>Abmelden</p><iframe src='/pwframe'></iframe><button>Abmelden bestätigen</button>",
|
||||||
|
"/pwframe": "<form><input type='text' name='u'><input type='password' name='pw'></form>",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -44,8 +54,31 @@ class Handler(BaseHTTPRequestHandler):
|
|||||||
self.wfile.write(data)
|
self.wfile.write(data)
|
||||||
|
|
||||||
def do_GET(self):
|
def do_GET(self):
|
||||||
path = urlsplit(self.path).path
|
parts = urlsplit(self.path)
|
||||||
|
path, query = parts.path, parse_qs(parts.query)
|
||||||
Handler.log.append(("GET", path, {}))
|
Handler.log.append(("GET", path, {}))
|
||||||
|
if path == "/redirect":
|
||||||
|
return self._send("", 302, {"Location": query["to"][0]})
|
||||||
|
if path == "/img":
|
||||||
|
return self._send(f"<p>Bild</p><img src='{html.escape(query['src'][0])}'>")
|
||||||
|
if path == "/fetch":
|
||||||
|
target = html.escape(query["to"][0])
|
||||||
|
return self._send(f"<p>Skript</p><script>fetch('{target}', {{mode: 'no-cors'}}).catch(() => 0);"
|
||||||
|
f"navigator.sendBeacon('{target}', 'x')</script>")
|
||||||
|
if path == "/hang-img":
|
||||||
|
# loads a blocked image, then never finishes the document (a byte a second keeps the connection busy)
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "text/html; charset=utf-8")
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(b"<!doctype html><html><body><p>Laedt ...</p><img src='http://192.168.8.111/p.gif'>")
|
||||||
|
for _ in range(10):
|
||||||
|
try:
|
||||||
|
self.wfile.write(b" ")
|
||||||
|
self.wfile.flush()
|
||||||
|
except OSError:
|
||||||
|
return
|
||||||
|
time.sleep(1)
|
||||||
|
return
|
||||||
if path == "/to-private":
|
if path == "/to-private":
|
||||||
return self._send("", 302, {"Location": "http://192.168.8.111/admin"})
|
return self._send("", 302, {"Location": "http://192.168.8.111/admin"})
|
||||||
if path == "/file.bin":
|
if path == "/file.bin":
|
||||||
@@ -78,3 +111,28 @@ def start() -> tuple[ThreadingHTTPServer, str]:
|
|||||||
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||||
return server, f"http://127.0.0.1:{server.server_address[1]}"
|
return server, f"http://127.0.0.1:{server.server_address[1]}"
|
||||||
|
|
||||||
|
|
||||||
|
def start_other() -> tuple[ThreadingHTTPServer, str, list]:
|
||||||
|
"""A second server on 127.0.0.1 with its own request log: the browser must never reach it."""
|
||||||
|
|
||||||
|
class Other(BaseHTTPRequestHandler):
|
||||||
|
log = []
|
||||||
|
|
||||||
|
def do_GET(self):
|
||||||
|
Other.log.append((self.command, self.path))
|
||||||
|
data = b"<p>other server</p>"
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "text/html")
|
||||||
|
self.send_header("Content-Length", str(len(data)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(data)
|
||||||
|
|
||||||
|
do_POST = do_GET
|
||||||
|
|
||||||
|
def log_message(self, *args):
|
||||||
|
pass
|
||||||
|
|
||||||
|
server = ThreadingHTTPServer(("127.0.0.1", 0), Other)
|
||||||
|
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||||
|
return server, f"http://127.0.0.1:{server.server_address[1]}", Other.log
|
||||||
@@ -0,0 +1,346 @@
|
|||||||
|
"""The egress proxy with raw sockets against local test servers: what it forwards, where, and what it refuses."""
|
||||||
|
|
||||||
|
import socket
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import egress
|
||||||
|
import guard
|
||||||
|
from egress import EgressProxy
|
||||||
|
|
||||||
|
OK_REPLY = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"
|
||||||
|
|
||||||
|
|
||||||
|
class Upstream:
|
||||||
|
"""Raw TCP server on 127.0.0.1. Records the bytes of every accepted connection; answers each request head with
|
||||||
|
a canned reply (or echoes everything back), then keeps reading for a moment so stray bytes are recorded too."""
|
||||||
|
|
||||||
|
def __init__(self, reply=OK_REPLY, echo=False):
|
||||||
|
self.reply, self.echo = reply, echo
|
||||||
|
self.connections: list[bytearray] = []
|
||||||
|
self.sock = socket.socket()
|
||||||
|
self.sock.bind(("127.0.0.1", 0))
|
||||||
|
self.sock.listen()
|
||||||
|
self.port = self.sock.getsockname()[1]
|
||||||
|
threading.Thread(target=self._accept, daemon=True).start()
|
||||||
|
|
||||||
|
def _accept(self):
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
conn, _ = self.sock.accept()
|
||||||
|
except OSError:
|
||||||
|
return
|
||||||
|
data = bytearray()
|
||||||
|
self.connections.append(data)
|
||||||
|
threading.Thread(target=self._serve, args=(conn, data), daemon=True).start()
|
||||||
|
|
||||||
|
def _serve(self, conn, data):
|
||||||
|
conn.settimeout(1)
|
||||||
|
answered = False
|
||||||
|
with conn:
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
chunk = conn.recv(65536)
|
||||||
|
if not chunk:
|
||||||
|
return
|
||||||
|
data += chunk
|
||||||
|
if self.echo:
|
||||||
|
conn.sendall(chunk)
|
||||||
|
elif not answered and b"\r\n\r\n" in data:
|
||||||
|
conn.sendall(self.reply)
|
||||||
|
answered = True
|
||||||
|
except OSError:
|
||||||
|
return
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.sock.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def upstreams():
|
||||||
|
made = []
|
||||||
|
|
||||||
|
def make(**kw):
|
||||||
|
u = Upstream(**kw)
|
||||||
|
made.append(u)
|
||||||
|
return u
|
||||||
|
yield make
|
||||||
|
for u in made:
|
||||||
|
u.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def proxy_for():
|
||||||
|
made = []
|
||||||
|
|
||||||
|
def make(addresses_for):
|
||||||
|
p = EgressProxy(addresses_for, [], timeout=3).start()
|
||||||
|
made.append(p)
|
||||||
|
return p
|
||||||
|
yield make
|
||||||
|
for p in made:
|
||||||
|
p.stop()
|
||||||
|
|
||||||
|
|
||||||
|
def allow(*urls_to_address):
|
||||||
|
"""addresses_for that allows exactly the given (url prefix, address) pairs and records every call."""
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def addresses_for(url):
|
||||||
|
calls.append(url)
|
||||||
|
for prefix, address in urls_to_address:
|
||||||
|
if url.startswith(prefix):
|
||||||
|
return [address]
|
||||||
|
return []
|
||||||
|
addresses_for.calls = calls
|
||||||
|
return addresses_for
|
||||||
|
|
||||||
|
|
||||||
|
def exchange(proxy, data: bytes, wait: float = 3.0) -> bytes:
|
||||||
|
"""Send data to the proxy and read until it closes the connection (or wait runs out)."""
|
||||||
|
port = int(proxy.address.rsplit(":", 1)[1])
|
||||||
|
with socket.create_connection(("127.0.0.1", port)) as s:
|
||||||
|
s.settimeout(wait)
|
||||||
|
s.sendall(data)
|
||||||
|
out = bytearray()
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
chunk = s.recv(65536)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
out += chunk
|
||||||
|
except TimeoutError:
|
||||||
|
pass
|
||||||
|
return bytes(out)
|
||||||
|
|
||||||
|
|
||||||
|
def settle():
|
||||||
|
time.sleep(0.3) # let the upstream threads record what they got
|
||||||
|
|
||||||
|
|
||||||
|
# --- DNS rebinding: the proxy connects only to the address the check returned ---------------------------------
|
||||||
|
|
||||||
|
def test_proxy_connects_to_the_checked_address_and_never_resolves_the_name(upstreams, proxy_for):
|
||||||
|
up = upstreams()
|
||||||
|
addresses_for = allow((f"http://never-resolvable.invalid:{up.port}/", "127.0.0.1"))
|
||||||
|
reply = exchange(proxy_for(addresses_for),
|
||||||
|
f"GET http://never-resolvable.invalid:{up.port}/x HTTP/1.1\r\nHost: never-resolvable.invalid\r\n\r\n"
|
||||||
|
.encode())
|
||||||
|
assert reply.startswith(b"HTTP/1.1 200 OK") and reply.endswith(b"ok")
|
||||||
|
settle()
|
||||||
|
assert len(up.connections) == 1 and up.connections[0].startswith(b"GET /x HTTP/1.1\r\n")
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebinding_resolver_cannot_redirect_the_connection(monkeypatch, proxy_for):
|
||||||
|
answers = iter([["93.184.216.34"]])
|
||||||
|
lookups = []
|
||||||
|
|
||||||
|
def flip(host): # public on the first lookup, loopback on every later one
|
||||||
|
lookups.append(host)
|
||||||
|
return next(answers, ["127.0.0.1"])
|
||||||
|
|
||||||
|
connects = []
|
||||||
|
|
||||||
|
def fake_connect(address, timeout=None, *args, **kw):
|
||||||
|
connects.append(address)
|
||||||
|
raise OSError("no network in this test")
|
||||||
|
|
||||||
|
real_getaddrinfo = socket.getaddrinfo
|
||||||
|
name_lookups = []
|
||||||
|
|
||||||
|
def spy_getaddrinfo(host, *a, **kw):
|
||||||
|
name_lookups.append(host)
|
||||||
|
return real_getaddrinfo(host, *a, **kw)
|
||||||
|
|
||||||
|
monkeypatch.setattr(egress, "_connect", fake_connect)
|
||||||
|
monkeypatch.setattr(socket, "getaddrinfo", spy_getaddrinfo)
|
||||||
|
proxy = proxy_for(lambda url: guard.public_addresses(url, resolve=flip, blocked=()))
|
||||||
|
assert exchange(proxy, b"GET http://rebind.example/ HTTP/1.1\r\nHost: rebind.example\r\n\r\n") == b""
|
||||||
|
assert lookups == ["rebind.example"] # one check, one lookup
|
||||||
|
assert connects == [("93.184.216.34", 80)] # to the vetted address, not to the name
|
||||||
|
assert "rebind.example" not in name_lookups
|
||||||
|
|
||||||
|
|
||||||
|
def test_next_checked_address_is_tried_when_the_first_fails(upstreams, proxy_for):
|
||||||
|
up = upstreams() # listens on 127.0.0.1 only: 127.0.0.2 on the same port refuses
|
||||||
|
|
||||||
|
def addresses_for(url):
|
||||||
|
return ["127.0.0.2", "127.0.0.1"]
|
||||||
|
reply = exchange(proxy_for(addresses_for), f"GET http://two.example:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode())
|
||||||
|
assert reply.endswith(b"ok")
|
||||||
|
|
||||||
|
|
||||||
|
def test_denied_url_makes_no_upstream_connection_and_gets_no_answer(upstreams, proxy_for):
|
||||||
|
up = upstreams()
|
||||||
|
proxy = proxy_for(lambda url: [])
|
||||||
|
assert exchange(proxy, f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode()) == b""
|
||||||
|
settle()
|
||||||
|
assert up.connections == [] and proxy.blocked == [f"http://127.0.0.1:{up.port}/"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_failing_check_counts_as_denied(upstreams, proxy_for):
|
||||||
|
up = upstreams()
|
||||||
|
|
||||||
|
def broken(url):
|
||||||
|
raise RuntimeError("resolver crashed")
|
||||||
|
assert exchange(proxy_for(broken), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode()) == b""
|
||||||
|
settle()
|
||||||
|
assert up.connections == []
|
||||||
|
|
||||||
|
|
||||||
|
# --- CONNECT ------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_connect_allowed_tunnels_bytes(upstreams, proxy_for):
|
||||||
|
up = upstreams(echo=True)
|
||||||
|
addresses_for = allow((f"https://tunnel.example:{up.port}/", "127.0.0.1"))
|
||||||
|
port = int(proxy_for(addresses_for).address.rsplit(":", 1)[1])
|
||||||
|
with socket.create_connection(("127.0.0.1", port)) as s:
|
||||||
|
s.settimeout(3)
|
||||||
|
s.sendall(f"CONNECT tunnel.example:{up.port} HTTP/1.1\r\nHost: tunnel.example:{up.port}\r\n\r\n".encode())
|
||||||
|
head = s.recv(1024)
|
||||||
|
assert head == b"HTTP/1.1 200 Connection established\r\n\r\n"
|
||||||
|
s.sendall(b"opaque tls bytes")
|
||||||
|
assert s.recv(1024) == b"opaque tls bytes"
|
||||||
|
assert addresses_for.calls == [f"https://tunnel.example:{up.port}/"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_denied_sends_nothing_and_connects_nowhere(upstreams, proxy_for):
|
||||||
|
up = upstreams(echo=True)
|
||||||
|
proxy = proxy_for(lambda url: [])
|
||||||
|
assert exchange(proxy, f"CONNECT 127.0.0.1:{up.port} HTTP/1.1\r\n\r\n".encode()) == b""
|
||||||
|
settle()
|
||||||
|
assert up.connections == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("authority", ["tunnel.example", "tunnel.example:abc", "tunnel.example:0",
|
||||||
|
"tunnel.example:70000", "user@tunnel.example:443", "user:pw@tunnel.example:443",
|
||||||
|
":443", "[::1", "::1:443", "tunnel.example:+443", "tunnel.example: 443"])
|
||||||
|
def test_malformed_connect_is_refused_without_a_check(proxy_for, authority):
|
||||||
|
addresses_for = allow(("https://", "127.0.0.1"))
|
||||||
|
assert exchange(proxy_for(addresses_for), f"CONNECT {authority} HTTP/1.1\r\n\r\n".encode()) == b""
|
||||||
|
assert addresses_for.calls == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_to_bracketed_ipv6_literal_is_checked_with_brackets(proxy_for):
|
||||||
|
addresses_for = allow()
|
||||||
|
assert exchange(proxy_for(addresses_for), b"CONNECT [::1]:443 HTTP/1.1\r\n\r\n") == b""
|
||||||
|
assert addresses_for.calls == ["https://[::1]:443/"]
|
||||||
|
|
||||||
|
|
||||||
|
# --- plain HTTP (absolute form) --------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_absolute_form_request_is_forwarded_in_origin_form(upstreams, proxy_for):
|
||||||
|
up = upstreams()
|
||||||
|
addresses_for = allow((f"http://site.example:{up.port}/", "127.0.0.1"))
|
||||||
|
reply = exchange(proxy_for(addresses_for),
|
||||||
|
f"POST http://site.example:{up.port}/u?x=1 HTTP/1.1\r\nHost: site.example:{up.port}\r\n"
|
||||||
|
"Proxy-Connection: keep-alive\r\nConnection: keep-alive, X-Secret\r\nX-Secret: 1\r\n"
|
||||||
|
"Content-Length: 5\r\n\r\nhello".encode())
|
||||||
|
assert reply.endswith(b"ok")
|
||||||
|
settle()
|
||||||
|
head, _, body = bytes(up.connections[0]).partition(b"\r\n\r\n")
|
||||||
|
lines = head.decode().split("\r\n")
|
||||||
|
assert lines[0] == "POST /u?x=1 HTTP/1.1" and body == b"hello"
|
||||||
|
names = [line.split(":")[0].lower() for line in lines[1:]]
|
||||||
|
assert "host" in names and "proxy-connection" not in names and "x-secret" not in names
|
||||||
|
assert "Connection: close" in lines and "Connection: keep-alive, X-Secret" not in lines
|
||||||
|
assert addresses_for.calls == [f"http://site.example:{up.port}/u?x=1"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_absolute_form_denied(upstreams, proxy_for):
|
||||||
|
up = upstreams()
|
||||||
|
assert exchange(proxy_for(allow()), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\n\r\n".encode()) == b""
|
||||||
|
settle()
|
||||||
|
assert up.connections == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("target", ["/relative", "https://site.example/", "ftp://site.example/", "http:///x",
|
||||||
|
"http://site.example:0/", "http://site.example:abc/", "http://u:p@site.example/"])
|
||||||
|
def test_other_request_targets_are_refused_without_a_check(proxy_for, target):
|
||||||
|
addresses_for = allow(("http://", "127.0.0.1"))
|
||||||
|
assert exchange(proxy_for(addresses_for), f"GET {target} HTTP/1.1\r\nHost: x\r\n\r\n".encode()) == b""
|
||||||
|
assert addresses_for.calls == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_request_with_transfer_encoding_is_dropped(upstreams, proxy_for):
|
||||||
|
up = upstreams()
|
||||||
|
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
|
||||||
|
reply = exchange(proxy_for(addresses_for), f"POST http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n"
|
||||||
|
"Transfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n".encode())
|
||||||
|
settle()
|
||||||
|
assert reply == b"" and up.connections == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("lengths", [("5", "5"), ("5", "7"), ("5, 5",), ("-1",), ("abc",)])
|
||||||
|
def test_duplicate_or_bad_content_length_is_rejected(upstreams, proxy_for, lengths):
|
||||||
|
up = upstreams()
|
||||||
|
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
|
||||||
|
headers = "".join(f"Content-Length: {v}\r\n" for v in lengths)
|
||||||
|
reply = exchange(proxy_for(addresses_for),
|
||||||
|
f"POST http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n{headers}\r\nhello".encode())
|
||||||
|
settle()
|
||||||
|
assert reply == b"" and up.connections == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_pipelined_second_request_is_never_forwarded(upstreams, proxy_for):
|
||||||
|
a, b = upstreams(), upstreams()
|
||||||
|
addresses_for = allow((f"http://127.0.0.1:{a.port}/", "127.0.0.1"), (f"http://127.0.0.1:{b.port}/", "127.0.0.1"))
|
||||||
|
first = f"GET http://127.0.0.1:{a.port}/first HTTP/1.1\r\nHost: a\r\n\r\n"
|
||||||
|
second = f"GET http://127.0.0.1:{b.port}/second HTTP/1.1\r\nHost: b\r\n\r\n"
|
||||||
|
reply = exchange(proxy_for(addresses_for), (first + second).encode())
|
||||||
|
settle()
|
||||||
|
assert reply.count(b"HTTP/1.1 200 OK") == 1
|
||||||
|
assert b.connections == [] and len(a.connections) == 1
|
||||||
|
assert bytes(a.connections[0]).count(b" HTTP/1.1\r\n") == 1 and b"/second" not in a.connections[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_second_request_on_the_same_connection_is_never_forwarded(upstreams, proxy_for):
|
||||||
|
a, b = upstreams(), upstreams()
|
||||||
|
addresses_for = allow((f"http://127.0.0.1:{a.port}/", "127.0.0.1"), (f"http://127.0.0.1:{b.port}/", "127.0.0.1"))
|
||||||
|
port = int(proxy_for(addresses_for).address.rsplit(":", 1)[1])
|
||||||
|
with socket.create_connection(("127.0.0.1", port)) as s:
|
||||||
|
s.settimeout(3)
|
||||||
|
s.sendall(f"GET http://127.0.0.1:{a.port}/first HTTP/1.1\r\nHost: a\r\n\r\n".encode())
|
||||||
|
got = bytearray()
|
||||||
|
while not got.endswith(b"ok"):
|
||||||
|
got += s.recv(65536)
|
||||||
|
assert b"Connection: close" in got
|
||||||
|
try:
|
||||||
|
s.sendall(f"GET http://127.0.0.1:{b.port}/second HTTP/1.1\r\nHost: b\r\n\r\n".encode())
|
||||||
|
assert s.recv(65536) == b""
|
||||||
|
except (BrokenPipeError, ConnectionResetError):
|
||||||
|
pass
|
||||||
|
settle()
|
||||||
|
assert b.connections == [] and b"/second" not in a.connections[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_chunked_response_passes_through_intact(upstreams, proxy_for):
|
||||||
|
body = b"4\r\nWiki\r\n5\r\npedia\r\n0\r\n\r\n"
|
||||||
|
up = upstreams(reply=b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nContent-Type: text/plain\r\n\r\n" + body)
|
||||||
|
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
|
||||||
|
reply = exchange(proxy_for(addresses_for), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode())
|
||||||
|
head, _, rest = reply.partition(b"\r\n\r\n")
|
||||||
|
assert b"Transfer-Encoding: chunked" in head and rest == body
|
||||||
|
|
||||||
|
|
||||||
|
def test_interim_1xx_responses_are_skipped_and_hop_by_hop_headers_of_the_final_one_stripped(upstreams, proxy_for):
|
||||||
|
up = upstreams(reply=b"HTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 103 Early Hints\r\nLink: </a.css>\r\n\r\n"
|
||||||
|
b"HTTP/1.1 200 OK\r\nConnection: keep-alive, X-Hop\r\nKeep-Alive: timeout=5\r\nX-Hop: 1\r\n"
|
||||||
|
b"Proxy-Connection: keep-alive\r\nContent-Length: 2\r\n\r\nok")
|
||||||
|
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
|
||||||
|
reply = exchange(proxy_for(addresses_for), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode())
|
||||||
|
head, _, rest = reply.partition(b"\r\n\r\n")
|
||||||
|
lines = head.decode().split("\r\n")
|
||||||
|
assert lines[0] == "HTTP/1.1 200 OK" and rest == b"ok" and reply.count(b"HTTP/1.1 ") == 1
|
||||||
|
names = [line.split(":")[0].lower() for line in lines[1:]]
|
||||||
|
assert "keep-alive" not in names and "x-hop" not in names and "proxy-connection" not in names
|
||||||
|
assert lines.count("Connection: close") == 1 and names.count("connection") == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_switching_protocols_is_not_passed_on(upstreams, proxy_for):
|
||||||
|
up = upstreams(reply=b"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\nraw")
|
||||||
|
addresses_for = allow((f"http://127.0.0.1:{up.port}/", "127.0.0.1"))
|
||||||
|
reply = exchange(proxy_for(addresses_for), f"GET http://127.0.0.1:{up.port}/ HTTP/1.1\r\nHost: x\r\n\r\n".encode())
|
||||||
|
assert reply == b""
|
||||||
@@ -55,3 +55,16 @@ def test_error_text_never_contains_the_key():
|
|||||||
def test_invalid_endpoint_url_means_unavailable(base_url):
|
def test_invalid_endpoint_url_means_unavailable(base_url):
|
||||||
with pytest.raises(LLMUnavailable):
|
with pytest.raises(LLMUnavailable):
|
||||||
LLM(base_url, "m", "k", client=client(lambda r: httpx.Response(200, json={}))).ask("s", "u")
|
LLM(base_url, "m", "k", client=client(lambda r: httpx.Response(200, json={}))).ask("s", "u")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("given,expected", [(None, 60), (5.5, 5.5), (120, 60)])
|
||||||
|
def test_timeout_can_be_capped_per_call_but_never_raised(given, expected):
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
def handler(request):
|
||||||
|
seen["timeout"] = request.extensions["timeout"]
|
||||||
|
return httpx.Response(200, json={"choices": [{"message": {"content": "x"}}]})
|
||||||
|
|
||||||
|
llm = LLM("https://ai.example", "m", "k", timeout=60, client=client(handler))
|
||||||
|
llm.ask("s", "u") if given is None else llm.ask("s", "u", timeout=given)
|
||||||
|
assert seen["timeout"] == {"connect": expected, "read": expected, "write": expected, "pool": expected}
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
"""The HTTP service: answers for broken cases and bodies, model configuration, and what it logs.
|
||||||
|
Needs Playwright and Flask for the import: run with tests/run-in-docker.sh."""
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import logging
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
pytest.importorskip("playwright.sync_api")
|
||||||
|
pytest.importorskip("flask")
|
||||||
|
|
||||||
|
ENV = {"LLM_BASE_URL": "https://ai.example", "LLM_MODEL": "deepseek-chat", "LLM_API_KEY": "sk-test"}
|
||||||
|
KEYS = ["evidence", "evidence_url", "reason", "result", "stage", "steps"]
|
||||||
|
URL = "https://shop.example/u?token=secret"
|
||||||
|
ADDRESS = "anne@wagnersnetz.de"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def server(monkeypatch):
|
||||||
|
for k, v in ENV.items():
|
||||||
|
monkeypatch.setenv(k, v)
|
||||||
|
import server as module
|
||||||
|
return importlib.reload(module)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def client(server):
|
||||||
|
return server.app.test_client()
|
||||||
|
|
||||||
|
|
||||||
|
def result(reason, result="unsubscribed"):
|
||||||
|
return {"result": result, "reason": reason, "evidence": "Sie wurden abgemeldet", "evidence_url": URL,
|
||||||
|
"steps": 2, "stage": 2}
|
||||||
|
|
||||||
|
|
||||||
|
def test_health(client):
|
||||||
|
r = client.get("/health")
|
||||||
|
assert r.status_code == 200 and r.get_json() == {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("body", ["not json", "[1, 2]", "null", "{}", '{"url": 1}', '{"url": "https://x.example"'
|
||||||
|
', "address": 5}', '"https://x.example"'])
|
||||||
|
def test_bad_bodies_answer_400(client, body):
|
||||||
|
assert client.post("/run", data=body, content_type="application/json").status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
def test_exception_in_a_case_is_unclear_and_logs_nothing_personal(server, client, monkeypatch, caplog):
|
||||||
|
def boom(*args, **kw):
|
||||||
|
raise RuntimeError(f"{URL} {ADDRESS}")
|
||||||
|
|
||||||
|
monkeypatch.setattr(server, "run_case", boom)
|
||||||
|
caplog.set_level(logging.DEBUG)
|
||||||
|
r = client.post("/run", json={"url": URL, "address": ADDRESS})
|
||||||
|
assert r.status_code == 200
|
||||||
|
body = r.get_json()
|
||||||
|
assert sorted(body) == KEYS
|
||||||
|
assert (body["result"], body["reason"], body["steps"], body["stage"]) == ("unclear", "agent_failure", 0, 1)
|
||||||
|
assert "RuntimeError" in caplog.text
|
||||||
|
for secret in ("shop.example", "token", "secret", ADDRESS, "anne"):
|
||||||
|
assert secret not in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_browser_launch_failure_is_unclear_too(server, client, monkeypatch, caplog):
|
||||||
|
def no_browser():
|
||||||
|
raise OSError(f"cannot start browser for {URL}")
|
||||||
|
|
||||||
|
monkeypatch.setattr(server, "sync_playwright", no_browser)
|
||||||
|
caplog.set_level(logging.DEBUG)
|
||||||
|
body = client.post("/run", json={"url": URL, "address": ADDRESS}).get_json()
|
||||||
|
assert (body["result"], body["reason"]) == ("unclear", "agent_failure")
|
||||||
|
assert "shop.example" not in caplog.text and "OSError" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_model_reason_is_answered_but_logged_only_as_a_category(server, client, monkeypatch, caplog):
|
||||||
|
reason = "unsubscribed anne@wagnersnetz.de at https://x.example/u?t=1"
|
||||||
|
monkeypatch.setattr(server, "run_case", lambda *args, **kw: result(reason))
|
||||||
|
caplog.set_level(logging.DEBUG)
|
||||||
|
body = client.post("/run", json={"url": URL, "address": ADDRESS}).get_json()
|
||||||
|
assert body["reason"] == reason and body["evidence_url"] == URL
|
||||||
|
assert "anne" not in caplog.text and "x.example" not in caplog.text and "shop.example" not in caplog.text
|
||||||
|
assert "case finished: unsubscribed (model), 2 step(s), stage 2" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("reason", ["login", "captcha", "blocked_address", "budget", "page_error", "llm_unavailable",
|
||||||
|
"page confirms without interaction",
|
||||||
|
"three invalid replies from the model in a row"])
|
||||||
|
def test_reasons_set_by_code_are_logged_as_they_are(server, client, monkeypatch, caplog, reason):
|
||||||
|
monkeypatch.setattr(server, "run_case", lambda *args, **kw: result(reason, "not_possible"))
|
||||||
|
caplog.set_level(logging.INFO)
|
||||||
|
client.post("/run", json={"url": URL, "address": ADDRESS})
|
||||||
|
assert f"case finished: not_possible ({reason})" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_case_gets_the_configured_model_and_limits(server, client, monkeypatch):
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
def fake_run_case(browser, url, address, llm, **kw):
|
||||||
|
seen.update(url=url, address=address, llm=llm, **kw)
|
||||||
|
return result("budget", "not_possible")
|
||||||
|
|
||||||
|
monkeypatch.setattr(server, "run_case", fake_run_case)
|
||||||
|
client.post("/run", json={"url": URL, "address": ADDRESS})
|
||||||
|
assert (seen["url"], seen["address"]) == (URL, ADDRESS)
|
||||||
|
assert seen["max_steps"] == server.MAX_STEPS and seen["case_timeout"] == server.CASE_TIMEOUT
|
||||||
|
assert seen["llm"]._model == "deepseek-chat"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("missing", sorted(ENV))
|
||||||
|
@pytest.mark.parametrize("value", [None, "", " "])
|
||||||
|
def test_missing_model_configuration_stops_the_start(monkeypatch, missing, value):
|
||||||
|
for k, v in ENV.items():
|
||||||
|
monkeypatch.setenv(k, v)
|
||||||
|
import server as module
|
||||||
|
if value is None:
|
||||||
|
monkeypatch.delenv(missing)
|
||||||
|
else:
|
||||||
|
monkeypatch.setenv(missing, value)
|
||||||
|
try:
|
||||||
|
with pytest.raises(RuntimeError, match=missing):
|
||||||
|
importlib.reload(module)
|
||||||
|
finally:
|
||||||
|
monkeypatch.setenv(missing, ENV[missing])
|
||||||
|
importlib.reload(module)
|
||||||
+113
-5
@@ -1,12 +1,18 @@
|
|||||||
"""The browser loop against imitated unsubscribe pages. Needs Playwright: run with tests/run-in-docker.sh."""
|
"""The browser loop against imitated unsubscribe pages. Needs Playwright: run with tests/run-in-docker.sh."""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
playwright_api = pytest.importorskip("playwright.sync_api")
|
playwright_api = pytest.importorskip("playwright.sync_api")
|
||||||
|
|
||||||
import guard # noqa: E402
|
import guard # noqa: E402
|
||||||
from llm import LLMUnavailable # noqa: E402
|
from llm import LLMUnavailable # noqa: E402
|
||||||
from runner import run_case # noqa: E402
|
from runner import BROWSER_ARGS, run_case # noqa: E402
|
||||||
from tests import sites # noqa: E402
|
from tests import sites # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
@@ -36,7 +42,7 @@ def element(prompt: str, label: str) -> int:
|
|||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def browser():
|
def browser():
|
||||||
with playwright_api.sync_playwright() as p:
|
with playwright_api.sync_playwright() as p:
|
||||||
b = p.chromium.launch()
|
b = p.chromium.launch(args=BROWSER_ARGS)
|
||||||
yield b
|
yield b
|
||||||
b.close()
|
b.close()
|
||||||
|
|
||||||
@@ -48,13 +54,26 @@ def site():
|
|||||||
server.shutdown()
|
server.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def other():
|
||||||
|
server, base, log = sites.start_other()
|
||||||
|
yield base, log
|
||||||
|
server.shutdown()
|
||||||
|
|
||||||
|
|
||||||
def local_ok(base):
|
def local_ok(base):
|
||||||
"""Guard for the tests: the test server on 127.0.0.1 counts as public, everything else goes to the real guard."""
|
"""Guard for the tests: the test server on 127.0.0.1 counts as public, everything else goes to the real guard."""
|
||||||
return lambda url: url.startswith(base) or guard.is_public_url(url)
|
return lambda url: url.startswith(base) or guard.is_public_url(url)
|
||||||
|
|
||||||
|
|
||||||
|
def local_addresses(base):
|
||||||
|
"""The matching check for the egress proxy: the test server is reached at 127.0.0.1, all else as in production."""
|
||||||
|
return lambda url: ["127.0.0.1"] if url.startswith(base + "/") else guard.public_addresses(url)
|
||||||
|
|
||||||
|
|
||||||
def run(browser, site, path, llm, address="philipp@wagnersnetz.de", **kw):
|
def run(browser, site, path, llm, address="philipp@wagnersnetz.de", **kw):
|
||||||
return run_case(browser, site + path, address, llm, url_ok=local_ok(site), nav_timeout=10, **kw)
|
return run_case(browser, site + path, address, llm, url_ok=local_ok(site), addresses_for=local_addresses(site),
|
||||||
|
nav_timeout=10, **kw)
|
||||||
|
|
||||||
|
|
||||||
def test_one_click_page_needs_no_model(browser, site):
|
def test_one_click_page_needs_no_model(browser, site):
|
||||||
@@ -202,7 +221,24 @@ def test_embedded_private_content_is_blocked_but_the_page_still_works(browser, s
|
|||||||
assert run(browser, site, "/embeds-private", Confirm())["result"] == "unsubscribed"
|
assert run(browser, site, "/embeds-private", Confirm())["result"] == "unsubscribed"
|
||||||
|
|
||||||
|
|
||||||
def test_download_is_not_kept(browser, site, tmp_path):
|
def _files_with(content: bytes) -> set[str]:
|
||||||
|
"""Every file below the temp directory (where Playwright keeps downloads) whose content is exactly content."""
|
||||||
|
found = set()
|
||||||
|
for root, _, files in os.walk(tempfile.gettempdir()):
|
||||||
|
for name in files:
|
||||||
|
path = os.path.join(root, name)
|
||||||
|
try:
|
||||||
|
if os.path.getsize(path) == len(content) and open(path, "rb").read() == content:
|
||||||
|
found.add(path)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
def test_download_is_not_kept(browser, site, tmp_path, caplog):
|
||||||
|
caplog.set_level(logging.INFO, logger="runner")
|
||||||
|
before = _files_with(b"data")
|
||||||
|
|
||||||
class Download(Script):
|
class Download(Script):
|
||||||
def ask(self, system, user):
|
def ask(self, system, user):
|
||||||
self.prompts.append(user)
|
self.prompts.append(user)
|
||||||
@@ -212,6 +248,9 @@ def test_download_is_not_kept(browser, site, tmp_path):
|
|||||||
|
|
||||||
r = run(browser, site, "/download", Download())
|
r = run(browser, site, "/download", Download())
|
||||||
assert r["result"] == "unclear"
|
assert r["result"] == "unclear"
|
||||||
|
assert ("GET", "/file.bin", {}) in sites.Handler.log # the download really was attempted
|
||||||
|
assert "download refused" in caplog.text # the browser offered it and it was refused
|
||||||
|
assert _files_with(b"data") - before == set() # and no file with its content appeared
|
||||||
|
|
||||||
|
|
||||||
def test_model_unavailable_is_an_error_not_a_result(browser, site):
|
def test_model_unavailable_is_an_error_not_a_result(browser, site):
|
||||||
@@ -227,5 +266,74 @@ def test_no_address_means_the_address_action_is_refused(browser, site):
|
|||||||
|
|
||||||
|
|
||||||
def test_page_that_does_not_load_is_a_page_error(browser, site):
|
def test_page_that_does_not_load_is_a_page_error(browser, site):
|
||||||
r = run_case(browser, "http://127.0.0.1:9/gone", "", Script(), url_ok=lambda url: True, nav_timeout=5)
|
r = run_case(browser, "http://127.0.0.1:9/gone", "", Script(), url_ok=lambda url: True,
|
||||||
|
addresses_for=lambda url: ["127.0.0.1"], nav_timeout=5)
|
||||||
assert (r["result"], r["reason"]) == ("not_possible", "page_error")
|
assert (r["result"], r["reason"]) == ("not_possible", "page_error")
|
||||||
|
|
||||||
|
|
||||||
|
def test_redirect_to_loopback_never_leaves_the_proxy(browser, site, other):
|
||||||
|
base2, log2 = other
|
||||||
|
r = run(browser, site, "/redirect?to=" + quote(base2 + "/admin"), Script())
|
||||||
|
assert log2 == []
|
||||||
|
assert (r["result"], r["reason"]) == ("not_possible", "blocked_address")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("page", ["/img?src=", "/fetch?to="])
|
||||||
|
@pytest.mark.parametrize("via_redirect", [False, True])
|
||||||
|
def test_subresources_to_loopback_never_reach_it(browser, site, other, page, via_redirect):
|
||||||
|
base2, log2 = other
|
||||||
|
target = base2 + "/pixel.gif"
|
||||||
|
if via_redirect: # the route gate sees only the first hop; the proxy must stop the second
|
||||||
|
target = site + "/redirect?to=" + quote(target)
|
||||||
|
run(browser, site, page + quote(target), Script())
|
||||||
|
time.sleep(0.5)
|
||||||
|
assert log2 == []
|
||||||
|
if via_redirect:
|
||||||
|
assert ("GET", "/redirect", {}) in sites.Handler.log # the first hop did go out
|
||||||
|
|
||||||
|
|
||||||
|
def test_endless_script_ends_at_the_time_budget(browser, site):
|
||||||
|
start = time.monotonic()
|
||||||
|
r = run(browser, site, "/busy", Script(), case_timeout=5)
|
||||||
|
assert time.monotonic() - start < 15
|
||||||
|
assert (r["result"], r["reason"]) == ("not_possible", "budget")
|
||||||
|
|
||||||
|
|
||||||
|
def test_slow_model_ends_at_the_time_budget_and_gets_a_capped_timeout(browser, site):
|
||||||
|
class Slow(Script):
|
||||||
|
timeouts = []
|
||||||
|
|
||||||
|
def ask(self, system, user, timeout=None):
|
||||||
|
self.prompts.append(user)
|
||||||
|
self.timeouts.append(timeout)
|
||||||
|
time.sleep(4)
|
||||||
|
return '{"action":"click","element":%d}' % element(user, "Abmelden bestätigen")
|
||||||
|
|
||||||
|
llm = Slow()
|
||||||
|
start = time.monotonic()
|
||||||
|
r = run(browser, site, "/confirm", llm, case_timeout=3)
|
||||||
|
assert time.monotonic() - start < 10
|
||||||
|
assert (r["result"], r["reason"]) == ("not_possible", "budget")
|
||||||
|
assert llm.timeouts and all(t is not None and 0 < t <= 3 for t in llm.timeouts)
|
||||||
|
assert not any(method == "POST" for method, _, _ in sites.Handler.log)
|
||||||
|
|
||||||
|
|
||||||
|
def test_webrtc_is_not_available_to_pages(browser, site):
|
||||||
|
llm = Script()
|
||||||
|
run(browser, site, "/rtc", llm)
|
||||||
|
assert "rtc:undefined,undefined" in llm.prompts[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_blocked_image_on_a_page_that_never_loads_is_a_page_error(browser, site):
|
||||||
|
start = time.monotonic()
|
||||||
|
r = run_case(browser, site + "/hang-img", "", Script(), url_ok=local_ok(site), addresses_for=local_addresses(site),
|
||||||
|
nav_timeout=3)
|
||||||
|
assert (r["result"], r["reason"]) == ("not_possible", "page_error")
|
||||||
|
assert time.monotonic() - start < 8
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("path", ["/many-links-password", "/frame-password"])
|
||||||
|
def test_password_field_anywhere_ends_without_the_model(browser, site, path):
|
||||||
|
llm = Script()
|
||||||
|
r = run(browser, site, path, llm)
|
||||||
|
assert (r["result"], r["reason"]) == ("not_possible", "login") and llm.prompts == []
|
||||||
Reference in new issue
Block a user