Agent: Befunde aus dem Review der Browser-Schleife behoben
This commit is contained in:
10 files changed
+976
-74
No files matched your search
+113
-5
@@ -1,12 +1,18 @@
|
||||
"""The browser loop against imitated unsubscribe pages. Needs Playwright: run with tests/run-in-docker.sh."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
|
||||
playwright_api = pytest.importorskip("playwright.sync_api")
|
||||
|
||||
import guard # noqa: E402
|
||||
from llm import LLMUnavailable # noqa: E402
|
||||
from runner import run_case # noqa: E402
|
||||
from runner import BROWSER_ARGS, run_case # noqa: E402
|
||||
from tests import sites # noqa: E402
|
||||
|
||||
|
||||
@@ -36,7 +42,7 @@ def element(prompt: str, label: str) -> int:
|
||||
@pytest.fixture(scope="module")
|
||||
def browser():
|
||||
with playwright_api.sync_playwright() as p:
|
||||
b = p.chromium.launch()
|
||||
b = p.chromium.launch(args=BROWSER_ARGS)
|
||||
yield b
|
||||
b.close()
|
||||
|
||||
@@ -48,13 +54,26 @@ def site():
|
||||
server.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def other():
|
||||
server, base, log = sites.start_other()
|
||||
yield base, log
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def local_ok(base):
|
||||
"""Guard for the tests: the test server on 127.0.0.1 counts as public, everything else goes to the real guard."""
|
||||
return lambda url: url.startswith(base) or guard.is_public_url(url)
|
||||
|
||||
|
||||
def local_addresses(base):
|
||||
"""The matching check for the egress proxy: the test server is reached at 127.0.0.1, all else as in production."""
|
||||
return lambda url: ["127.0.0.1"] if url.startswith(base + "/") else guard.public_addresses(url)
|
||||
|
||||
|
||||
def run(browser, site, path, llm, address="philipp@wagnersnetz.de", **kw):
|
||||
return run_case(browser, site + path, address, llm, url_ok=local_ok(site), nav_timeout=10, **kw)
|
||||
return run_case(browser, site + path, address, llm, url_ok=local_ok(site), addresses_for=local_addresses(site),
|
||||
nav_timeout=10, **kw)
|
||||
|
||||
|
||||
def test_one_click_page_needs_no_model(browser, site):
|
||||
@@ -202,7 +221,24 @@ def test_embedded_private_content_is_blocked_but_the_page_still_works(browser, s
|
||||
assert run(browser, site, "/embeds-private", Confirm())["result"] == "unsubscribed"
|
||||
|
||||
|
||||
def test_download_is_not_kept(browser, site, tmp_path):
|
||||
def _files_with(content: bytes) -> set[str]:
|
||||
"""Every file below the temp directory (where Playwright keeps downloads) whose content is exactly content."""
|
||||
found = set()
|
||||
for root, _, files in os.walk(tempfile.gettempdir()):
|
||||
for name in files:
|
||||
path = os.path.join(root, name)
|
||||
try:
|
||||
if os.path.getsize(path) == len(content) and open(path, "rb").read() == content:
|
||||
found.add(path)
|
||||
except OSError:
|
||||
pass
|
||||
return found
|
||||
|
||||
|
||||
def test_download_is_not_kept(browser, site, tmp_path, caplog):
|
||||
caplog.set_level(logging.INFO, logger="runner")
|
||||
before = _files_with(b"data")
|
||||
|
||||
class Download(Script):
|
||||
def ask(self, system, user):
|
||||
self.prompts.append(user)
|
||||
@@ -212,6 +248,9 @@ def test_download_is_not_kept(browser, site, tmp_path):
|
||||
|
||||
r = run(browser, site, "/download", Download())
|
||||
assert r["result"] == "unclear"
|
||||
assert ("GET", "/file.bin", {}) in sites.Handler.log # the download really was attempted
|
||||
assert "download refused" in caplog.text # the browser offered it and it was refused
|
||||
assert _files_with(b"data") - before == set() # and no file with its content appeared
|
||||
|
||||
|
||||
def test_model_unavailable_is_an_error_not_a_result(browser, site):
|
||||
@@ -227,5 +266,74 @@ def test_no_address_means_the_address_action_is_refused(browser, site):
|
||||
|
||||
|
||||
def test_page_that_does_not_load_is_a_page_error(browser, site):
|
||||
r = run_case(browser, "http://127.0.0.1:9/gone", "", Script(), url_ok=lambda url: True, nav_timeout=5)
|
||||
r = run_case(browser, "http://127.0.0.1:9/gone", "", Script(), url_ok=lambda url: True,
|
||||
addresses_for=lambda url: ["127.0.0.1"], nav_timeout=5)
|
||||
assert (r["result"], r["reason"]) == ("not_possible", "page_error")
|
||||
|
||||
|
||||
def test_redirect_to_loopback_never_leaves_the_proxy(browser, site, other):
|
||||
base2, log2 = other
|
||||
r = run(browser, site, "/redirect?to=" + quote(base2 + "/admin"), Script())
|
||||
assert log2 == []
|
||||
assert (r["result"], r["reason"]) == ("not_possible", "blocked_address")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("page", ["/img?src=", "/fetch?to="])
|
||||
@pytest.mark.parametrize("via_redirect", [False, True])
|
||||
def test_subresources_to_loopback_never_reach_it(browser, site, other, page, via_redirect):
|
||||
base2, log2 = other
|
||||
target = base2 + "/pixel.gif"
|
||||
if via_redirect: # the route gate sees only the first hop; the proxy must stop the second
|
||||
target = site + "/redirect?to=" + quote(target)
|
||||
run(browser, site, page + quote(target), Script())
|
||||
time.sleep(0.5)
|
||||
assert log2 == []
|
||||
if via_redirect:
|
||||
assert ("GET", "/redirect", {}) in sites.Handler.log # the first hop did go out
|
||||
|
||||
|
||||
def test_endless_script_ends_at_the_time_budget(browser, site):
|
||||
start = time.monotonic()
|
||||
r = run(browser, site, "/busy", Script(), case_timeout=5)
|
||||
assert time.monotonic() - start < 15
|
||||
assert (r["result"], r["reason"]) == ("not_possible", "budget")
|
||||
|
||||
|
||||
def test_slow_model_ends_at_the_time_budget_and_gets_a_capped_timeout(browser, site):
|
||||
class Slow(Script):
|
||||
timeouts = []
|
||||
|
||||
def ask(self, system, user, timeout=None):
|
||||
self.prompts.append(user)
|
||||
self.timeouts.append(timeout)
|
||||
time.sleep(4)
|
||||
return '{"action":"click","element":%d}' % element(user, "Abmelden bestätigen")
|
||||
|
||||
llm = Slow()
|
||||
start = time.monotonic()
|
||||
r = run(browser, site, "/confirm", llm, case_timeout=3)
|
||||
assert time.monotonic() - start < 10
|
||||
assert (r["result"], r["reason"]) == ("not_possible", "budget")
|
||||
assert llm.timeouts and all(t is not None and 0 < t <= 3 for t in llm.timeouts)
|
||||
assert not any(method == "POST" for method, _, _ in sites.Handler.log)
|
||||
|
||||
|
||||
def test_webrtc_is_not_available_to_pages(browser, site):
|
||||
llm = Script()
|
||||
run(browser, site, "/rtc", llm)
|
||||
assert "rtc:undefined,undefined" in llm.prompts[0]
|
||||
|
||||
|
||||
def test_blocked_image_on_a_page_that_never_loads_is_a_page_error(browser, site):
|
||||
start = time.monotonic()
|
||||
r = run_case(browser, site + "/hang-img", "", Script(), url_ok=local_ok(site), addresses_for=local_addresses(site),
|
||||
nav_timeout=3)
|
||||
assert (r["result"], r["reason"]) == ("not_possible", "page_error")
|
||||
assert time.monotonic() - start < 8
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/many-links-password", "/frame-password"])
|
||||
def test_password_field_anywhere_ends_without_the_model(browser, site, path):
|
||||
llm = Script()
|
||||
r = run(browser, site, path, llm)
|
||||
assert (r["result"], r["reason"]) == ("not_possible", "login") and llm.prompts == []
|
||||
Reference in new issue
Block a user