diff --git a/agent/guard.py b/agent/guard.py new file mode 100644 index 0000000..c956532 --- /dev/null +++ b/agent/guard.py @@ -0,0 +1,38 @@ +"""URL guard for the agent's browser: only public web addresses. The host firewall is the second line.""" + +import ipaddress +import socket +from urllib.parse import urlsplit + + +def _resolve(host: str) -> list[str]: + return [info[4][0] for info in socket.getaddrinfo(host, None, type=socket.SOCK_STREAM)] + + +# Publicly routable-looking range that is the homelab's own container network (see Ansible host_vars). +_EXTRA_BLOCKED = (ipaddress.ip_network("172.60.0.0/16"),) + + +def _is_global(address: str) -> bool: + ip = ipaddress.ip_address(address.split("%", 1)[0]) + if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None: + ip = ip.ipv4_mapped + if any(ip in net for net in _EXTRA_BLOCKED): + return False + return ip.is_global and not ip.is_multicast + + +def is_public_url(url: str, resolve=None) -> bool: + """http(s) only, no credentials in the URL, and every address the name resolves to must be global.""" + try: + parts = urlsplit(url) + host = parts.hostname + except ValueError: + return False + if parts.scheme not in ("http", "https") or not host or parts.username or parts.password: + return False + try: + addresses = (resolve or _resolve)(host) + return bool(addresses) and all(_is_global(a) for a in addresses) + except (OSError, ValueError): + return False diff --git a/agent/pagecheck.py b/agent/pagecheck.py new file mode 100644 index 0000000..40a9465 --- /dev/null +++ b/agent/pagecheck.py @@ -0,0 +1,45 @@ +"""Checks on a page snapshot that need no model: success texts, required input, login pages.""" + +import re +from urllib.parse import urlsplit + +_SUCCESS = re.compile( + r"(you('ve| have)? (been |successfully |now )*unsubscribed" + r"|you are (now )?unsubscribed" + r"|unsubscribe(d)? success(ful|fully)" + r"|successfully unsubscribed" + r"|you will no longer receive" + r"|(wurden?|wurdest) (erfolgreich )?(vom newsletter )?abgemeldet" + r"|erfolgreich (vom newsletter |von unserem newsletter )?abgemeldet" + r"|abmeldung (war |ist )?(erfolgreich|bestätigt)" + r"|erfolgreich abbestellt" + r"|erhalten (ab sofort |künftig |in zukunft )?keine (e-?mails|newsletter|nachrichten) mehr)", + re.IGNORECASE) +_CONFIRM = re.compile( + r"(abmelden|abbestellen|bestätigen|absenden|speichern|austragen|unsubscribe|confirm|submit|save|opt.?out|update)", + re.IGNORECASE) +_LOGIN_PATH = re.compile(r"(^|/)(login|log-in|signin|sign-in|sign_in|anmelden|einloggen|auth)(/|$|\.)", re.IGNORECASE) + + +def looks_unsubscribed(text: str) -> bool: + return bool(_SUCCESS.search(text or "")) + + +def needs_input(elements: list[dict]) -> bool: + """True if the page still offers something to fill in or a button that looks like a confirmation.""" + for e in elements: + if e["tag"] in ("select", "textarea"): + return True + if e["tag"] == "input" and e["type"] not in ("button", "submit", "reset", "image"): + return True + if (e["tag"] == "button" or e["type"] in ("button", "submit")) and _CONFIRM.search(e.get("label", "")): + return True + return False + + +def has_password(elements: list[dict]) -> bool: + return any(e["tag"] == "input" and e["type"] == "password" for e in elements) + + +def is_login_url(url: str) -> bool: + return bool(_LOGIN_PATH.search(urlsplit(url).path)) diff --git a/agent/tests/__init__.py b/agent/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/agent/tests/conftest.py b/agent/tests/conftest.py new file mode 100644 index 0000000..e55f1e0 --- /dev/null +++ b/agent/tests/conftest.py @@ -0,0 +1,4 @@ +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) diff --git a/agent/tests/test_guard.py b/agent/tests/test_guard.py new file mode 100644 index 0000000..b2ffa04 --- /dev/null +++ b/agent/tests/test_guard.py @@ -0,0 +1,48 @@ +"""URL guard: the agent's browser may only talk to public web addresses.""" + +import pytest + +from guard import is_public_url + + +def resolver(table): + def resolve(host): + if host not in table: + raise OSError("unknown host") + return table[host] + return resolve + + +PUBLIC = resolver({"shop.example": ["93.184.216.34"], "v6.example": ["2606:2800:220:1:248:1893:25c8:1946"]}) + + +@pytest.mark.parametrize("url", ["https://shop.example/u?id=1", "http://shop.example", "https://v6.example/x"]) +def test_public_addresses_pass(url): + assert is_public_url(url, PUBLIC) is True + + +@pytest.mark.parametrize("url", [ + "ftp://shop.example/x", "javascript:alert(1)", "data:text/html,x", "file:///etc/passwd", "about:blank", + "https://user:pw@shop.example/", "https:///nohost", "not a url", ""]) +def test_other_schemes_credentials_and_garbage_fail(url): + assert is_public_url(url, PUBLIC) is False + + +@pytest.mark.parametrize("ip", [ + "127.0.0.1", "10.1.2.3", "172.16.5.4", "172.60.0.127", "192.168.8.111", "100.81.0.1", "169.254.169.254", + "0.0.0.0", "::1", "fe80::1", "fd7a:115c:a1e0::53", "::ffff:192.168.8.111", "224.0.0.1"]) +def test_private_and_special_addresses_fail(ip): + host = f"[{ip}]" if ":" in ip else ip + assert is_public_url(f"http://{host}/", resolver({ip: [ip]})) is False + + +def test_name_resolving_to_a_private_address_fails(): + assert is_public_url("https://intern.example/", resolver({"intern.example": ["192.168.8.111"]})) is False + + +def test_one_private_address_among_several_fails(): + assert is_public_url("https://mixed.example/", resolver({"mixed.example": ["93.184.216.34", "10.0.0.5"]})) is False + + +def test_unresolvable_name_fails(): + assert is_public_url("https://nowhere.example/", PUBLIC) is False diff --git a/agent/tests/test_pagecheck.py b/agent/tests/test_pagecheck.py new file mode 100644 index 0000000..2c8f15f --- /dev/null +++ b/agent/tests/test_pagecheck.py @@ -0,0 +1,51 @@ +"""Checks on a page snapshot that need no model.""" + +import pytest + +from pagecheck import has_password, is_login_url, looks_unsubscribed, needs_input + + +@pytest.mark.parametrize("text", [ + "You have been unsubscribed.", "You have successfully unsubscribed from our newsletter", + "Sie wurden erfolgreich abgemeldet", "Ihre Abmeldung war erfolgreich", "Du wurdest abgemeldet.", + "Sie haben sich erfolgreich vom Newsletter abgemeldet", "Unsubscribe successful", "You are now unsubscribed", + "Sie erhalten ab sofort keine E-Mails mehr", "You will no longer receive emails from us", + "Abmeldung bestätigt"]) +def test_success_texts(text): + assert looks_unsubscribed("Header\n" + text + "\nFooter") is True + + +@pytest.mark.parametrize("text", [ + "Unsubscribe", "Möchten Sie sich wirklich abmelden?", "Click the button to unsubscribe", + "Are you sure you want to unsubscribe?", "Bitte bestätigen Sie Ihre Abmeldung", "Newsletter abbestellen", ""]) +def test_no_false_success(text): + assert looks_unsubscribed(text) is False + + +def el(tag, type_="", label=""): + return {"n": 1, "tag": tag, "type": type_, "label": label} + + +def test_needs_input_for_fields_and_confirm_buttons(): + assert needs_input([el("button", "submit", "Abmelden bestätigen")]) is True + assert needs_input([el("input", "email", "E-Mail")]) is True + assert needs_input([el("select", "", "Grund")]) is True + assert needs_input([el("input", "checkbox", "Alle Newsletter")]) is True + + +def test_no_input_needed_for_plain_links_and_unrelated_buttons(): + assert needs_input([el("a", "", "Startseite"), el("button", "", "Cookie-Einstellungen"), el("a", "", "Impressum")]) is False + assert needs_input([]) is False + + +def test_password_field_detected(): + assert has_password([el("input", "text", "User"), el("input", "password", "Passwort")]) is True + assert has_password([el("input", "text", "User")]) is False + + +@pytest.mark.parametrize("url,expected", [ + ("https://shop.example/login?next=/prefs", True), ("https://accounts.google.com/signin/v2", True), + ("https://shop.example/account/sign-in", True), ("https://shop.example/anmelden", True), + ("https://shop.example/unsubscribe/123", False), ("https://shop.example/blog/login-tips-article", False)]) +def test_login_urls(url, expected): + assert is_login_url(url) is expected