Agent: Adressfilter und Seitenprüfungen ohne Modell
This commit is contained in:
6 files changed
+186
No files matched your search
@@ -0,0 +1,38 @@
|
||||
"""URL guard for the agent's browser: only public web addresses. The host firewall is the second line."""
|
||||
|
||||
import ipaddress
|
||||
import socket
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
def _resolve(host: str) -> list[str]:
|
||||
return [info[4][0] for info in socket.getaddrinfo(host, None, type=socket.SOCK_STREAM)]
|
||||
|
||||
|
||||
# Publicly routable-looking range that is the homelab's own container network (see Ansible host_vars).
|
||||
_EXTRA_BLOCKED = (ipaddress.ip_network("172.60.0.0/16"),)
|
||||
|
||||
|
||||
def _is_global(address: str) -> bool:
|
||||
ip = ipaddress.ip_address(address.split("%", 1)[0])
|
||||
if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
|
||||
ip = ip.ipv4_mapped
|
||||
if any(ip in net for net in _EXTRA_BLOCKED):
|
||||
return False
|
||||
return ip.is_global and not ip.is_multicast
|
||||
|
||||
|
||||
def is_public_url(url: str, resolve=None) -> bool:
|
||||
"""http(s) only, no credentials in the URL, and every address the name resolves to must be global."""
|
||||
try:
|
||||
parts = urlsplit(url)
|
||||
host = parts.hostname
|
||||
except ValueError:
|
||||
return False
|
||||
if parts.scheme not in ("http", "https") or not host or parts.username or parts.password:
|
||||
return False
|
||||
try:
|
||||
addresses = (resolve or _resolve)(host)
|
||||
return bool(addresses) and all(_is_global(a) for a in addresses)
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Checks on a page snapshot that need no model: success texts, required input, login pages."""
|
||||
|
||||
import re
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
_SUCCESS = re.compile(
|
||||
r"(you('ve| have)? (been |successfully |now )*unsubscribed"
|
||||
r"|you are (now )?unsubscribed"
|
||||
r"|unsubscribe(d)? success(ful|fully)"
|
||||
r"|successfully unsubscribed"
|
||||
r"|you will no longer receive"
|
||||
r"|(wurden?|wurdest) (erfolgreich )?(vom newsletter )?abgemeldet"
|
||||
r"|erfolgreich (vom newsletter |von unserem newsletter )?abgemeldet"
|
||||
r"|abmeldung (war |ist )?(erfolgreich|bestätigt)"
|
||||
r"|erfolgreich abbestellt"
|
||||
r"|erhalten (ab sofort |künftig |in zukunft )?keine (e-?mails|newsletter|nachrichten) mehr)",
|
||||
re.IGNORECASE)
|
||||
_CONFIRM = re.compile(
|
||||
r"(abmelden|abbestellen|bestätigen|absenden|speichern|austragen|unsubscribe|confirm|submit|save|opt.?out|update)",
|
||||
re.IGNORECASE)
|
||||
_LOGIN_PATH = re.compile(r"(^|/)(login|log-in|signin|sign-in|sign_in|anmelden|einloggen|auth)(/|$|\.)", re.IGNORECASE)
|
||||
|
||||
|
||||
def looks_unsubscribed(text: str) -> bool:
|
||||
return bool(_SUCCESS.search(text or ""))
|
||||
|
||||
|
||||
def needs_input(elements: list[dict]) -> bool:
|
||||
"""True if the page still offers something to fill in or a button that looks like a confirmation."""
|
||||
for e in elements:
|
||||
if e["tag"] in ("select", "textarea"):
|
||||
return True
|
||||
if e["tag"] == "input" and e["type"] not in ("button", "submit", "reset", "image"):
|
||||
return True
|
||||
if (e["tag"] == "button" or e["type"] in ("button", "submit")) and _CONFIRM.search(e.get("label", "")):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def has_password(elements: list[dict]) -> bool:
|
||||
return any(e["tag"] == "input" and e["type"] == "password" for e in elements)
|
||||
|
||||
|
||||
def is_login_url(url: str) -> bool:
|
||||
return bool(_LOGIN_PATH.search(urlsplit(url).path))
|
||||
Whitespace-only changes.
@@ -0,0 +1,4 @@
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
@@ -0,0 +1,48 @@
|
||||
"""URL guard: the agent's browser may only talk to public web addresses."""
|
||||
|
||||
import pytest
|
||||
|
||||
from guard import is_public_url
|
||||
|
||||
|
||||
def resolver(table):
|
||||
def resolve(host):
|
||||
if host not in table:
|
||||
raise OSError("unknown host")
|
||||
return table[host]
|
||||
return resolve
|
||||
|
||||
|
||||
PUBLIC = resolver({"shop.example": ["93.184.216.34"], "v6.example": ["2606:2800:220:1:248:1893:25c8:1946"]})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("url", ["https://shop.example/u?id=1", "http://shop.example", "https://v6.example/x"])
|
||||
def test_public_addresses_pass(url):
|
||||
assert is_public_url(url, PUBLIC) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("url", [
|
||||
"ftp://shop.example/x", "javascript:alert(1)", "data:text/html,x", "file:///etc/passwd", "about:blank",
|
||||
"https://user:pw@shop.example/", "https:///nohost", "not a url", ""])
|
||||
def test_other_schemes_credentials_and_garbage_fail(url):
|
||||
assert is_public_url(url, PUBLIC) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ip", [
|
||||
"127.0.0.1", "10.1.2.3", "172.16.5.4", "172.60.0.127", "192.168.8.111", "100.81.0.1", "169.254.169.254",
|
||||
"0.0.0.0", "::1", "fe80::1", "fd7a:115c:a1e0::53", "::ffff:192.168.8.111", "224.0.0.1"])
|
||||
def test_private_and_special_addresses_fail(ip):
|
||||
host = f"[{ip}]" if ":" in ip else ip
|
||||
assert is_public_url(f"http://{host}/", resolver({ip: [ip]})) is False
|
||||
|
||||
|
||||
def test_name_resolving_to_a_private_address_fails():
|
||||
assert is_public_url("https://intern.example/", resolver({"intern.example": ["192.168.8.111"]})) is False
|
||||
|
||||
|
||||
def test_one_private_address_among_several_fails():
|
||||
assert is_public_url("https://mixed.example/", resolver({"mixed.example": ["93.184.216.34", "10.0.0.5"]})) is False
|
||||
|
||||
|
||||
def test_unresolvable_name_fails():
|
||||
assert is_public_url("https://nowhere.example/", PUBLIC) is False
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Checks on a page snapshot that need no model."""
|
||||
|
||||
import pytest
|
||||
|
||||
from pagecheck import has_password, is_login_url, looks_unsubscribed, needs_input
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", [
|
||||
"You have been unsubscribed.", "You have successfully unsubscribed from our newsletter",
|
||||
"Sie wurden erfolgreich abgemeldet", "Ihre Abmeldung war erfolgreich", "Du wurdest abgemeldet.",
|
||||
"Sie haben sich erfolgreich vom Newsletter abgemeldet", "Unsubscribe successful", "You are now unsubscribed",
|
||||
"Sie erhalten ab sofort keine E-Mails mehr", "You will no longer receive emails from us",
|
||||
"Abmeldung bestätigt"])
|
||||
def test_success_texts(text):
|
||||
assert looks_unsubscribed("Header\n" + text + "\nFooter") is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", [
|
||||
"Unsubscribe", "Möchten Sie sich wirklich abmelden?", "Click the button to unsubscribe",
|
||||
"Are you sure you want to unsubscribe?", "Bitte bestätigen Sie Ihre Abmeldung", "Newsletter abbestellen", ""])
|
||||
def test_no_false_success(text):
|
||||
assert looks_unsubscribed(text) is False
|
||||
|
||||
|
||||
def el(tag, type_="", label=""):
|
||||
return {"n": 1, "tag": tag, "type": type_, "label": label}
|
||||
|
||||
|
||||
def test_needs_input_for_fields_and_confirm_buttons():
|
||||
assert needs_input([el("button", "submit", "Abmelden bestätigen")]) is True
|
||||
assert needs_input([el("input", "email", "E-Mail")]) is True
|
||||
assert needs_input([el("select", "", "Grund")]) is True
|
||||
assert needs_input([el("input", "checkbox", "Alle Newsletter")]) is True
|
||||
|
||||
|
||||
def test_no_input_needed_for_plain_links_and_unrelated_buttons():
|
||||
assert needs_input([el("a", "", "Startseite"), el("button", "", "Cookie-Einstellungen"), el("a", "", "Impressum")]) is False
|
||||
assert needs_input([]) is False
|
||||
|
||||
|
||||
def test_password_field_detected():
|
||||
assert has_password([el("input", "text", "User"), el("input", "password", "Passwort")]) is True
|
||||
assert has_password([el("input", "text", "User")]) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("url,expected", [
|
||||
("https://shop.example/login?next=/prefs", True), ("https://accounts.google.com/signin/v2", True),
|
||||
("https://shop.example/account/sign-in", True), ("https://shop.example/anmelden", True),
|
||||
("https://shop.example/unsubscribe/123", False), ("https://shop.example/blog/login-tips-article", False)])
|
||||
def test_login_urls(url, expected):
|
||||
assert is_login_url(url) is expected
|
||||
Reference in new issue
Block a user