Author SHA1 Message Date
Rsclub2_2 734e4f0320 Abmelde-Link auch finden, wenn nur "hier" verlinkt ist
Hat ein Link einen kurzen Text ohne Stichwort, entscheidet der Satz, in
dem er steht (bis Punkt, Zeilenumbruch oder Rand des Absatzes). Text
anderer Links und ausgeschriebene Adressen zählen nicht zum Satz.
2026-11-16 09:25:56 +01:00
Rsclub2_2 bb64133b36 Erlaubte Absender: ganze Domain als Eintrag (Vikunja 97) (#4) 2026-11-16 08:22:07 +01:00
2 changed files with 99 additions and 2 deletions

No files matched your search

+45 -2
View File
@@ -19,6 +19,7 @@ from email.header import decode_header, make_header
import httpx
from bs4 import BeautifulSoup
from bs4.element import Tag
from dotenv import load_dotenv
from flask import Flask, abort, redirect, render_template, request, url_for
@@ -57,6 +58,11 @@ URL_PATTERN = re.compile(r"https?://[^\s<>\"']+")
UID_PATTERN = re.compile(r"^\d+$")
# Characters that end a sentence or close a quote or bracket in running text, not a URL.
TRAILING_PUNCTUATION = ".,;:!?)\"'\u201d\u2019\u00bb\u203a"
# A link with a text this short ("hier", "click here") says nothing itself: the sentence around it decides.
SHORT_LINK_TEXT = 25
SENTENCE_END = re.compile(r"[.!?\u2026](?:\s|$)")
# Tags that stand inside a sentence; every other tag (br, p, td, div, ...) ends it.
INLINE_TAGS = frozenset("a abbr b big code del em font i ins mark nobr s small span strike strong sub sup tt u wbr".split())
def is_safe_link(url: str) -> bool:
@@ -83,14 +89,51 @@ def get_subject(message) -> str:
return raw
def _sentence_around(anchor) -> str:
"""The sentence a link stands in, without the link text: what is written before and after it, up to a full
stop, a line break or the edge of the paragraph or table cell."""
own = {id(parent) for parent in anchor.parents} | {id(anchor)}
def side(elements) -> list[str]:
parts = []
for element in elements:
if isinstance(element, Tag):
if element.name not in INLINE_TAGS:
break
continue
parents = [parent for parent in element.parents if id(parent) not in own]
if id(anchor) in {id(parent) for parent in element.parents}: # the link's own text
continue
if any(parent.name not in INLINE_TAGS for parent in parents): # text of a neighbouring block
break
if any(parent.name == "a" for parent in parents): # another link's text speaks for that link only
continue
parts.append(str(element))
return parts
before = " ".join("".join(reversed(side(anchor.previous_elements))).split())
after = " ".join("".join(side(anchor.next_elements)).split())
# a URL written out in the text is a link of its own (see below), not a word of this sentence
before, after = URL_PATTERN.sub(" ", before), URL_PATTERN.sub(" ", after)
return SENTENCE_END.split(before + " ")[-1] + " " + SENTENCE_END.split(after + " ")[0]
def find_unsubscribe_links(text: str) -> list[str]:
"""Unsubscribe links from HTML anchors (link text or URL matches) and from plain-text URLs."""
"""Unsubscribe links from HTML anchors (link text or URL matches, or a short link in a sentence that matches)
and from plain-text URLs."""
links: list[str] = []
soup = BeautifulSoup(text, "html.parser")
for anchor in soup.find_all("a", href=True):
anchors = soup.find_all("a", href=True)
for anchor in anchors:
href = str(anchor["href"]).strip()
if UNSUBSCRIBE_PATTERN.search(str(anchor)) and is_safe_link(href) and href not in links:
links.append(href)
for anchor in anchors: # "you can unsubscribe <a>here</a>": the word stands next to the link
href = str(anchor["href"]).strip()
label = " ".join(anchor.get_text(" ").split())
if (href not in links and is_safe_link(href) and 0 < len(label) <= SHORT_LINK_TEXT
and UNSUBSCRIBE_PATTERN.search(_sentence_around(anchor))):
links.append(href)
for url in URL_PATTERN.findall(text):
# the text search also runs over raw HTML: unescape (&amp;) like the parser does for hrefs, so the same
# link is not taken twice, and drop punctuation that follows a link in running text
+54
View File
@@ -104,3 +104,57 @@ def test_result_page_renders_hidden_ids_for_move_all(monkeypatch):
html = unsub.app.test_client().get("/result").get_data(as_text=True)
assert html.count('name="email_id" value="41"') == 2 # single form + move-all form
assert 'rel="noopener noreferrer"' in html
# --- link named "hier": the word that says "unsubscribe" stands next to the link, in the same sentence ---
LIEFERANDO = (
'<table><tr><td>Wir schicken Dir diese E-Mail, weil Du mit der E-Mail-Adresse '
'<a href="mailto:philipp@example.org">philipp@example.org</a> unseren Newsletter abonniert hast.<br/>'
'Wenn Du keinen Newsletter von uns mehr bekommen möchtest, kannst Du ihn '
'<a href="https://click.example/u/1"><u>hier</u></a> abbestellen.<br/><br/>'
'Lieferando ist ein Angebot von Takeaway.com<br/>'
'E-Mail <a href="https://click.example/view/2"><u>online</u></a> ansehen.</td></tr></table>'
)
def test_link_named_hier_counts_when_its_sentence_says_unsubscribe():
assert unsub.find_unsubscribe_links(LIEFERANDO) == ["https://click.example/u/1"]
@pytest.mark.parametrize("html", [
'To unsubscribe, click <a href="https://click.example/u/1">here</a>.',
'<p>Du kannst dich <a href="https://click.example/u/1"><span>hier</span></a> <b>abmelden</b>.</p>',
'<div>Newsletter <a href="https://click.example/u/1">hier</a>\n abbestellen</div>',
])
def test_sentence_around_a_short_link_is_read_across_inline_tags(html):
assert unsub.find_unsubscribe_links(html) == ["https://click.example/u/1"]
@pytest.mark.parametrize("html", [
# another sentence of the same paragraph
'<p>Zum Abbestellen bitte antworten. Mehr dazu gibt es <a href="https://click.example/x">hier</a>.</p>',
'<p>Mehr dazu gibt es <a href="https://click.example/x">hier</a>. Abbestellen geht per Antwort.</p>',
# a line break ends the sentence
'<td>Newsletter abbestellen<br><a href="https://click.example/x">hier</a> geht es zum Shop</td>',
# another block
'<p>Newsletter abbestellen ist einfach</p><p><a href="https://click.example/x">hier</a> zum Shop</p>',
'<p><a href="https://click.example/x">hier</a> zum Shop</p><div>Newsletter abbestellen</div>',
# a link with a text of its own is judged by that text
'<p>Abbestellen kannst du jederzeit, lies dazu <a href="https://click.example/x">unsere ausführlichen '
'Hinweise zum Datenschutz</a>.</p>',
# the word belongs to another link (navigation row) or to a URL written out in the text
'<p><a href="https://ex.org/unsubscribe/1">Abmelden</a> | <a href="https://click.example/x">Home</a></p>',
'<p>https://ex.org/unsubscribe/1 oder <a href="https://click.example/x">Home</a></p>',
# not a web link
'<p>Zum Abbestellen <a href="mailto:x@example.org">hier</a> schreiben.</p>',
])
def test_short_link_outside_the_unsubscribe_sentence_is_not_taken(html):
assert "https://click.example/x" not in unsub.find_unsubscribe_links(html)
assert all("mailto" not in link for link in unsub.find_unsubscribe_links(html))
def test_link_found_by_its_own_text_is_not_listed_twice_and_comes_first():
html = ('<p>Du kannst dich <a href="https://click.example/a">hier</a> abmelden oder '
'<a href="https://click.example/b">Newsletter abbestellen</a>.</p>')
assert unsub.find_unsubscribe_links(html) == ["https://click.example/b", "https://click.example/a"]