From 734e4f0320c8223e031f7c54579f7a672792bac8 Mon Sep 17 00:00:00 2001 From: Philipp Wagner Date: Mon, 16 Nov 2026 09:25:56 +0100 Subject: [PATCH] Abmelde-Link auch finden, wenn nur "hier" verlinkt ist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hat ein Link einen kurzen Text ohne Stichwort, entscheidet der Satz, in dem er steht (bis Punkt, Zeilenumbruch oder Rand des Absatzes). Text anderer Links und ausgeschriebene Adressen zählen nicht zum Satz. --- app.py | 47 +++++++++++++++++++++++++++++++++++++++-- tests/test_app.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 7106af8..8f6fe66 100644 --- a/app.py +++ b/app.py @@ -19,6 +19,7 @@ from email.header import decode_header, make_header import httpx from bs4 import BeautifulSoup +from bs4.element import Tag from dotenv import load_dotenv from flask import Flask, abort, redirect, render_template, request, url_for @@ -57,6 +58,11 @@ URL_PATTERN = re.compile(r"https?://[^\s<>\"']+") UID_PATTERN = re.compile(r"^\d+$") # Characters that end a sentence or close a quote or bracket in running text, not a URL. TRAILING_PUNCTUATION = ".,;:!?)\"'\u201d\u2019\u00bb\u203a" +# A link with a text this short ("hier", "click here") says nothing itself: the sentence around it decides. +SHORT_LINK_TEXT = 25 +SENTENCE_END = re.compile(r"[.!?\u2026](?:\s|$)") +# Tags that stand inside a sentence; every other tag (br, p, td, div, ...) ends it. +INLINE_TAGS = frozenset("a abbr b big code del em font i ins mark nobr s small span strike strong sub sup tt u wbr".split()) def is_safe_link(url: str) -> bool: @@ -83,14 +89,51 @@ def get_subject(message) -> str: return raw +def _sentence_around(anchor) -> str: + """The sentence a link stands in, without the link text: what is written before and after it, up to a full + stop, a line break or the edge of the paragraph or table cell.""" + own = {id(parent) for parent in anchor.parents} | {id(anchor)} + + def side(elements) -> list[str]: + parts = [] + for element in elements: + if isinstance(element, Tag): + if element.name not in INLINE_TAGS: + break + continue + parents = [parent for parent in element.parents if id(parent) not in own] + if id(anchor) in {id(parent) for parent in element.parents}: # the link's own text + continue + if any(parent.name not in INLINE_TAGS for parent in parents): # text of a neighbouring block + break + if any(parent.name == "a" for parent in parents): # another link's text speaks for that link only + continue + parts.append(str(element)) + return parts + + before = " ".join("".join(reversed(side(anchor.previous_elements))).split()) + after = " ".join("".join(side(anchor.next_elements)).split()) + # a URL written out in the text is a link of its own (see below), not a word of this sentence + before, after = URL_PATTERN.sub(" ", before), URL_PATTERN.sub(" ", after) + return SENTENCE_END.split(before + " ")[-1] + " " + SENTENCE_END.split(after + " ")[0] + + def find_unsubscribe_links(text: str) -> list[str]: - """Unsubscribe links from HTML anchors (link text or URL matches) and from plain-text URLs.""" + """Unsubscribe links from HTML anchors (link text or URL matches, or a short link in a sentence that matches) + and from plain-text URLs.""" links: list[str] = [] soup = BeautifulSoup(text, "html.parser") - for anchor in soup.find_all("a", href=True): + anchors = soup.find_all("a", href=True) + for anchor in anchors: href = str(anchor["href"]).strip() if UNSUBSCRIBE_PATTERN.search(str(anchor)) and is_safe_link(href) and href not in links: links.append(href) + for anchor in anchors: # "you can unsubscribe here": the word stands next to the link + href = str(anchor["href"]).strip() + label = " ".join(anchor.get_text(" ").split()) + if (href not in links and is_safe_link(href) and 0 < len(label) <= SHORT_LINK_TEXT + and UNSUBSCRIBE_PATTERN.search(_sentence_around(anchor))): + links.append(href) for url in URL_PATTERN.findall(text): # the text search also runs over raw HTML: unescape (&) like the parser does for hrefs, so the same # link is not taken twice, and drop punctuation that follows a link in running text diff --git a/tests/test_app.py b/tests/test_app.py index e53071a..5bd8d71 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -104,3 +104,57 @@ def test_result_page_renders_hidden_ids_for_move_all(monkeypatch): html = unsub.app.test_client().get("/result").get_data(as_text=True) assert html.count('name="email_id" value="41"') == 2 # single form + move-all form assert 'rel="noopener noreferrer"' in html + + +# --- link named "hier": the word that says "unsubscribe" stands next to the link, in the same sentence --- + +LIEFERANDO = ( + '
Wir schicken Dir diese E-Mail, weil Du mit der E-Mail-Adresse ' + 'philipp@example.org unseren Newsletter abonniert hast.
' + 'Wenn Du keinen Newsletter von uns mehr bekommen möchtest, kannst Du ihn ' + 'hier abbestellen.

' + 'Lieferando ist ein Angebot von Takeaway.com
' + 'E-Mail online ansehen.
' +) + + +def test_link_named_hier_counts_when_its_sentence_says_unsubscribe(): + assert unsub.find_unsubscribe_links(LIEFERANDO) == ["https://click.example/u/1"] + + +@pytest.mark.parametrize("html", [ + 'To unsubscribe, click here.', + '

Du kannst dich hier abmelden.

', + '
Newsletter hier\n abbestellen
', +]) +def test_sentence_around_a_short_link_is_read_across_inline_tags(html): + assert unsub.find_unsubscribe_links(html) == ["https://click.example/u/1"] + + +@pytest.mark.parametrize("html", [ + # another sentence of the same paragraph + '

Zum Abbestellen bitte antworten. Mehr dazu gibt es hier.

', + '

Mehr dazu gibt es hier. Abbestellen geht per Antwort.

', + # a line break ends the sentence + 'Newsletter abbestellen
hier geht es zum Shop', + # another block + '

Newsletter abbestellen ist einfach

hier zum Shop

', + '

hier zum Shop

Newsletter abbestellen
', + # a link with a text of its own is judged by that text + '

Abbestellen kannst du jederzeit, lies dazu unsere ausführlichen ' + 'Hinweise zum Datenschutz.

', + # the word belongs to another link (navigation row) or to a URL written out in the text + '

Abmelden | Home

', + '

https://ex.org/unsubscribe/1 oder Home

', + # not a web link + '

Zum Abbestellen hier schreiben.

', +]) +def test_short_link_outside_the_unsubscribe_sentence_is_not_taken(html): + assert "https://click.example/x" not in unsub.find_unsubscribe_links(html) + assert all("mailto" not in link for link in unsub.find_unsubscribe_links(html)) + + +def test_link_found_by_its_own_text_is_not_listed_twice_and_comes_first(): + html = ('

Du kannst dich hier abmelden oder ' + 'Newsletter abbestellen.

') + assert unsub.find_unsubscribe_links(html) == ["https://click.example/b", "https://click.example/a"] -- 2.54.0