Author SHA1 Message Date
Rsclub2_2 734e4f0320 Abmelde-Link auch finden, wenn nur "hier" verlinkt ist
Hat ein Link einen kurzen Text ohne Stichwort, entscheidet der Satz, in
dem er steht (bis Punkt, Zeilenumbruch oder Rand des Absatzes). Text
anderer Links und ausgeschriebene Adressen zählen nicht zum Satz.
2026-11-16 09:25:56 +01:00
Rsclub2_2 bb64133b36 Erlaubte Absender: ganze Domain als Eintrag (Vikunja 97) (#4) 2026-11-16 08:22:07 +01:00
Rsclub2_2 e0e8cad4de Abmelde-Agent: Fälle, Zeitlauf und Agent-Dienst (Vikunja 97) (#3) 2026-11-16 07:41:08 +01:00
7 changed files with 139 additions and 6 deletions

No files matched your search

+1 -1
View File
@@ -6,4 +6,4 @@ EMAIL_SERVER=example.com
# DATA_DIR=/data # DATA_DIR=/data
# AGENT_URL=http://unsubscriber-agent:8080 # AGENT_URL=http://unsubscriber-agent:8080
# AGENT_ALLOWED_DOMAINS=example.org,gmail.com # AGENT_ALLOWED_DOMAINS=example.org,gmail.com
# AGENT_ALLOWED_SENDERS=jens@example.org # AGENT_ALLOWED_SENDERS=example.org,jens@gmail.com
+3 -1
View File
@@ -56,7 +56,9 @@ of entries: a bare domain (`wagnersnetz.de`) allows every address of that domain
as full addresses, never their domain: the recipient is read from the forwarded text, and a newsletter body with a as full addresses, never their domain: the recipient is read from the forwarded text, and a newsletter body with a
line like `To: stranger@gmail.com` would otherwise make the agent type a stranger's address. line like `To: stranger@gmail.com` would otherwise make the agent type a stranger's address.
`AGENT_ALLOWED_SENDERS` (comma separated) lists the forwarders whose mails become cases; mails from other `AGENT_ALLOWED_SENDERS` (comma separated) lists the forwarders whose mails become cases; mails from other
senders are ignored. With the agent on and an empty list nothing is processed. senders are ignored. Entries work as above: a bare domain allows every sender of that domain (not of its
subdomains), a full address exactly that sender. Use a domain only for domains whose mail you control, since the
From header decides. With the agent on and an empty list nothing is processed.
The agent container needs `LLM_BASE_URL` (Open WebUI, `/api/chat/completions` is appended), `LLM_MODEL`, The agent container needs `LLM_BASE_URL` (Open WebUI, `/api/chat/completions` is appended), `LLM_MODEL`,
`LLM_API_KEY` and `BLOCKED_NETS`, optionally `MAX_STEPS` (20) and `CASE_TIMEOUT` (300). `BLOCKED_NETS` is required: `LLM_API_KEY` and `BLOCKED_NETS`, optionally `MAX_STEPS` (20) and `CASE_TIMEOUT` (300). `BLOCKED_NETS` is required:
+45 -2
View File
@@ -19,6 +19,7 @@ from email.header import decode_header, make_header
import httpx import httpx
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from bs4.element import Tag
from dotenv import load_dotenv from dotenv import load_dotenv
from flask import Flask, abort, redirect, render_template, request, url_for from flask import Flask, abort, redirect, render_template, request, url_for
@@ -57,6 +58,11 @@ URL_PATTERN = re.compile(r"https?://[^\s<>\"']+")
UID_PATTERN = re.compile(r"^\d+$") UID_PATTERN = re.compile(r"^\d+$")
# Characters that end a sentence or close a quote or bracket in running text, not a URL. # Characters that end a sentence or close a quote or bracket in running text, not a URL.
TRAILING_PUNCTUATION = ".,;:!?)\"'\u201d\u2019\u00bb\u203a" TRAILING_PUNCTUATION = ".,;:!?)\"'\u201d\u2019\u00bb\u203a"
# A link with a text this short ("hier", "click here") says nothing itself: the sentence around it decides.
SHORT_LINK_TEXT = 25
SENTENCE_END = re.compile(r"[.!?\u2026](?:\s|$)")
# Tags that stand inside a sentence; every other tag (br, p, td, div, ...) ends it.
INLINE_TAGS = frozenset("a abbr b big code del em font i ins mark nobr s small span strike strong sub sup tt u wbr".split())
def is_safe_link(url: str) -> bool: def is_safe_link(url: str) -> bool:
@@ -83,14 +89,51 @@ def get_subject(message) -> str:
return raw return raw
def _sentence_around(anchor) -> str:
"""The sentence a link stands in, without the link text: what is written before and after it, up to a full
stop, a line break or the edge of the paragraph or table cell."""
own = {id(parent) for parent in anchor.parents} | {id(anchor)}
def side(elements) -> list[str]:
parts = []
for element in elements:
if isinstance(element, Tag):
if element.name not in INLINE_TAGS:
break
continue
parents = [parent for parent in element.parents if id(parent) not in own]
if id(anchor) in {id(parent) for parent in element.parents}: # the link's own text
continue
if any(parent.name not in INLINE_TAGS for parent in parents): # text of a neighbouring block
break
if any(parent.name == "a" for parent in parents): # another link's text speaks for that link only
continue
parts.append(str(element))
return parts
before = " ".join("".join(reversed(side(anchor.previous_elements))).split())
after = " ".join("".join(side(anchor.next_elements)).split())
# a URL written out in the text is a link of its own (see below), not a word of this sentence
before, after = URL_PATTERN.sub(" ", before), URL_PATTERN.sub(" ", after)
return SENTENCE_END.split(before + " ")[-1] + " " + SENTENCE_END.split(after + " ")[0]
def find_unsubscribe_links(text: str) -> list[str]: def find_unsubscribe_links(text: str) -> list[str]:
"""Unsubscribe links from HTML anchors (link text or URL matches) and from plain-text URLs.""" """Unsubscribe links from HTML anchors (link text or URL matches, or a short link in a sentence that matches)
and from plain-text URLs."""
links: list[str] = [] links: list[str] = []
soup = BeautifulSoup(text, "html.parser") soup = BeautifulSoup(text, "html.parser")
for anchor in soup.find_all("a", href=True): anchors = soup.find_all("a", href=True)
for anchor in anchors:
href = str(anchor["href"]).strip() href = str(anchor["href"]).strip()
if UNSUBSCRIBE_PATTERN.search(str(anchor)) and is_safe_link(href) and href not in links: if UNSUBSCRIBE_PATTERN.search(str(anchor)) and is_safe_link(href) and href not in links:
links.append(href) links.append(href)
for anchor in anchors: # "you can unsubscribe <a>here</a>": the word stands next to the link
href = str(anchor["href"]).strip()
label = " ".join(anchor.get_text(" ").split())
if (href not in links and is_safe_link(href) and 0 < len(label) <= SHORT_LINK_TEXT
and UNSUBSCRIBE_PATTERN.search(_sentence_around(anchor))):
links.append(href)
for url in URL_PATTERN.findall(text): for url in URL_PATTERN.findall(text):
# the text search also runs over raw HTML: unescape (&amp;) like the parser does for hrefs, so the same # the text search also runs over raw HTML: unescape (&amp;) like the parser does for hrefs, so the same
# link is not taken twice, and drop punctuation that follows a link in running text # link is not taken twice, and drop punctuation that follows a link in running text
+8
View File
@@ -18,6 +18,14 @@ def _allowed(address: str, entries) -> bool:
return address in entries or address.rsplit("@", 1)[-1] in entries # a domain entry never contains "@" return address in entries or address.rsplit("@", 1)[-1] in entries # a domain entry never contains "@"
def sender_allowed(address: str, entries) -> bool:
"""Whether a forwarder (the address of the From header) is on the list; entries as in _allowed. Anything that
is not one plain address is refused, so that odd forms like "@allowed.example" or a quoted local part cannot
pass by their last part."""
address = address.lower()
return bool(_ADDRESS.fullmatch(address)) and _allowed(address, entries)
def original_recipient(text: str, forwarder: str, allowed_domains) -> str: def original_recipient(text: str, forwarder: str, allowed_domains) -> str:
"""First allowed address from the quoted 'To:'/'An:' lines, else the forwarder, else ''. """First allowed address from the quoted 'To:'/'An:' lines, else the forwarder, else ''.
+54
View File
@@ -104,3 +104,57 @@ def test_result_page_renders_hidden_ids_for_move_all(monkeypatch):
html = unsub.app.test_client().get("/result").get_data(as_text=True) html = unsub.app.test_client().get("/result").get_data(as_text=True)
assert html.count('name="email_id" value="41"') == 2 # single form + move-all form assert html.count('name="email_id" value="41"') == 2 # single form + move-all form
assert 'rel="noopener noreferrer"' in html assert 'rel="noopener noreferrer"' in html
# --- link named "hier": the word that says "unsubscribe" stands next to the link, in the same sentence ---
LIEFERANDO = (
'<table><tr><td>Wir schicken Dir diese E-Mail, weil Du mit der E-Mail-Adresse '
'<a href="mailto:philipp@example.org">philipp@example.org</a> unseren Newsletter abonniert hast.<br/>'
'Wenn Du keinen Newsletter von uns mehr bekommen möchtest, kannst Du ihn '
'<a href="https://click.example/u/1"><u>hier</u></a> abbestellen.<br/><br/>'
'Lieferando ist ein Angebot von Takeaway.com<br/>'
'E-Mail <a href="https://click.example/view/2"><u>online</u></a> ansehen.</td></tr></table>'
)
def test_link_named_hier_counts_when_its_sentence_says_unsubscribe():
assert unsub.find_unsubscribe_links(LIEFERANDO) == ["https://click.example/u/1"]
@pytest.mark.parametrize("html", [
'To unsubscribe, click <a href="https://click.example/u/1">here</a>.',
'<p>Du kannst dich <a href="https://click.example/u/1"><span>hier</span></a> <b>abmelden</b>.</p>',
'<div>Newsletter <a href="https://click.example/u/1">hier</a>\n abbestellen</div>',
])
def test_sentence_around_a_short_link_is_read_across_inline_tags(html):
assert unsub.find_unsubscribe_links(html) == ["https://click.example/u/1"]
@pytest.mark.parametrize("html", [
# another sentence of the same paragraph
'<p>Zum Abbestellen bitte antworten. Mehr dazu gibt es <a href="https://click.example/x">hier</a>.</p>',
'<p>Mehr dazu gibt es <a href="https://click.example/x">hier</a>. Abbestellen geht per Antwort.</p>',
# a line break ends the sentence
'<td>Newsletter abbestellen<br><a href="https://click.example/x">hier</a> geht es zum Shop</td>',
# another block
'<p>Newsletter abbestellen ist einfach</p><p><a href="https://click.example/x">hier</a> zum Shop</p>',
'<p><a href="https://click.example/x">hier</a> zum Shop</p><div>Newsletter abbestellen</div>',
# a link with a text of its own is judged by that text
'<p>Abbestellen kannst du jederzeit, lies dazu <a href="https://click.example/x">unsere ausführlichen '
'Hinweise zum Datenschutz</a>.</p>',
# the word belongs to another link (navigation row) or to a URL written out in the text
'<p><a href="https://ex.org/unsubscribe/1">Abmelden</a> | <a href="https://click.example/x">Home</a></p>',
'<p>https://ex.org/unsubscribe/1 oder <a href="https://click.example/x">Home</a></p>',
# not a web link
'<p>Zum Abbestellen <a href="mailto:x@example.org">hier</a> schreiben.</p>',
])
def test_short_link_outside_the_unsubscribe_sentence_is_not_taken(html):
assert "https://click.example/x" not in unsub.find_unsubscribe_links(html)
assert all("mailto" not in link for link in unsub.find_unsubscribe_links(html))
def test_link_found_by_its_own_text_is_not_listed_twice_and_comes_first():
html = ('<p>Du kannst dich <a href="https://click.example/a">hier</a> abmelden oder '
'<a href="https://click.example/b">Newsletter abbestellen</a>.</p>')
assert unsub.find_unsubscribe_links(html) == ["https://click.example/b", "https://click.example/a"]
+26
View File
@@ -296,6 +296,32 @@ def test_mail_from_unknown_sender_is_ignored(store):
assert store.get("<evil@x>") is None and b"2" in imap.folders["INBOX"] assert store.get("<evil@x>") is None and b"2" in imap.folders["INBOX"]
def test_sender_entry_without_at_allows_the_whole_domain(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender="Jens <JENS@Wagnersnetz.de>"),
b"2": mail_bytes("<b@x>", sender="philipp@wagnersnetz.de"),
b"3": mail_bytes("<c@x>", sender="someone@gmx.de")})
agent = Agent(OK, OK, OK)
summary = worker(store, imap, agent, allowed_senders=("Wagnersnetz.de", "someone@gmx.de")).run_once()
assert len(agent.calls) == 3 and summary["ignored"] == 0
@pytest.mark.parametrize("sender", [
"mallory@evil.example",
"mallory@sub.wagnersnetz.de", # a domain entry does not cover its subdomains
"mallory@wagnersnetz.de.evil.example",
"mallory@evilwagnersnetz.de",
"other@gmx.de", # a full address allows exactly that one
"@wagnersnetz.de", # not a plain address: must not pass by its last part
"a b@wagnersnetz.de",
'<"mallory@evil.example"@wagnersnetz.de>',
])
def test_sender_outside_the_allowed_domain_is_ignored(store, sender):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender=sender)})
agent = Agent(OK)
summary = worker(store, imap, agent, allowed_senders=("wagnersnetz.de", "someone@gmx.de")).run_once()
assert agent.calls == [] and summary["ignored"] == 1 and store.all() == []
def test_no_allowed_senders_with_agent_means_nobody(store): def test_no_allowed_senders_with_agent_means_nobody(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>")}) imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
agent = Agent(OK) agent = Agent(OK)
+2 -2
View File
@@ -8,7 +8,7 @@ import time
from email.utils import parseaddr from email.utils import parseaddr
from cases import case_key from cases import case_key
from forwarded import original_recipient from forwarded import original_recipient, sender_allowed
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -73,7 +73,7 @@ class Worker:
without += 1 without += 1
continue continue
forwarder = parseaddr(message.get("From", ""))[1] forwarder = parseaddr(message.get("From", ""))[1]
if (self.allowed_senders and forwarder.lower() not in self.allowed_senders) or ( if (self.allowed_senders and not sender_allowed(forwarder, self.allowed_senders)) or (
not self.allowed_senders and self.agent is not None): # fail closed not self.allowed_senders and self.agent is not None): # fail closed
ignored += 1 ignored += 1
continue continue