From 081f50f69af64a94e3da800be222bde293c7fc7b Mon Sep 17 00:00:00 2001 From: Philipp Wagner Date: Mon, 16 Nov 2026 09:36:17 +0100 Subject: [PATCH] =?UTF-8?q?Abmelde-Link=20auch=20finden,=20wenn=20nur=20?= =?UTF-8?q?=E2=80=9Ehier=E2=80=9C=20verlinkt=20ist=20(Vikunja=2097)=20(#5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app.py | 47 +++++++++++++++++++++++++++++++++++++++-- tests/test_app.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 7106af8..8f6fe66 100644 --- a/app.py +++ b/app.py @@ -19,6 +19,7 @@ from email.header import decode_header, make_header import httpx from bs4 import BeautifulSoup +from bs4.element import Tag from dotenv import load_dotenv from flask import Flask, abort, redirect, render_template, request, url_for @@ -57,6 +58,11 @@ URL_PATTERN = re.compile(r"https?://[^\s<>\"']+") UID_PATTERN = re.compile(r"^\d+$") # Characters that end a sentence or close a quote or bracket in running text, not a URL. TRAILING_PUNCTUATION = ".,;:!?)\"'\u201d\u2019\u00bb\u203a" +# A link with a text this short ("hier", "click here") says nothing itself: the sentence around it decides. +SHORT_LINK_TEXT = 25 +SENTENCE_END = re.compile(r"[.!?\u2026](?:\s|$)") +# Tags that stand inside a sentence; every other tag (br, p, td, div, ...) ends it. +INLINE_TAGS = frozenset("a abbr b big code del em font i ins mark nobr s small span strike strong sub sup tt u wbr".split()) def is_safe_link(url: str) -> bool: @@ -83,14 +89,51 @@ def get_subject(message) -> str: return raw +def _sentence_around(anchor) -> str: + """The sentence a link stands in, without the link text: what is written before and after it, up to a full + stop, a line break or the edge of the paragraph or table cell.""" + own = {id(parent) for parent in anchor.parents} | {id(anchor)} + + def side(elements) -> list[str]: + parts = [] + for element in elements: + if isinstance(element, Tag): + if element.name not in INLINE_TAGS: + break + continue + parents = [parent for parent in element.parents if id(parent) not in own] + if id(anchor) in {id(parent) for parent in element.parents}: # the link's own text + continue + if any(parent.name not in INLINE_TAGS for parent in parents): # text of a neighbouring block + break + if any(parent.name == "a" for parent in parents): # another link's text speaks for that link only + continue + parts.append(str(element)) + return parts + + before = " ".join("".join(reversed(side(anchor.previous_elements))).split()) + after = " ".join("".join(side(anchor.next_elements)).split()) + # a URL written out in the text is a link of its own (see below), not a word of this sentence + before, after = URL_PATTERN.sub(" ", before), URL_PATTERN.sub(" ", after) + return SENTENCE_END.split(before + " ")[-1] + " " + SENTENCE_END.split(after + " ")[0] + + def find_unsubscribe_links(text: str) -> list[str]: - """Unsubscribe links from HTML anchors (link text or URL matches) and from plain-text URLs.""" + """Unsubscribe links from HTML anchors (link text or URL matches, or a short link in a sentence that matches) + and from plain-text URLs.""" links: list[str] = [] soup = BeautifulSoup(text, "html.parser") - for anchor in soup.find_all("a", href=True): + anchors = soup.find_all("a", href=True) + for anchor in anchors: href = str(anchor["href"]).strip() if UNSUBSCRIBE_PATTERN.search(str(anchor)) and is_safe_link(href) and href not in links: links.append(href) + for anchor in anchors: # "you can unsubscribe here": the word stands next to the link + href = str(anchor["href"]).strip() + label = " ".join(anchor.get_text(" ").split()) + if (href not in links and is_safe_link(href) and 0 < len(label) <= SHORT_LINK_TEXT + and UNSUBSCRIBE_PATTERN.search(_sentence_around(anchor))): + links.append(href) for url in URL_PATTERN.findall(text): # the text search also runs over raw HTML: unescape (&) like the parser does for hrefs, so the same # link is not taken twice, and drop punctuation that follows a link in running text diff --git a/tests/test_app.py b/tests/test_app.py index e53071a..5bd8d71 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -104,3 +104,57 @@ def test_result_page_renders_hidden_ids_for_move_all(monkeypatch): html = unsub.app.test_client().get("/result").get_data(as_text=True) assert html.count('name="email_id" value="41"') == 2 # single form + move-all form assert 'rel="noopener noreferrer"' in html + + +# --- link named "hier": the word that says "unsubscribe" stands next to the link, in the same sentence --- + +LIEFERANDO = ( + '
Wir schicken Dir diese E-Mail, weil Du mit der E-Mail-Adresse ' + 'philipp@example.org unseren Newsletter abonniert hast.
' + 'Wenn Du keinen Newsletter von uns mehr bekommen möchtest, kannst Du ihn ' + 'hier abbestellen.

' + 'Lieferando ist ein Angebot von Takeaway.com
' + 'E-Mail online ansehen.
' +) + + +def test_link_named_hier_counts_when_its_sentence_says_unsubscribe(): + assert unsub.find_unsubscribe_links(LIEFERANDO) == ["https://click.example/u/1"] + + +@pytest.mark.parametrize("html", [ + 'To unsubscribe, click here.', + '

Du kannst dich hier abmelden.

', + '
Newsletter hier\n abbestellen
', +]) +def test_sentence_around_a_short_link_is_read_across_inline_tags(html): + assert unsub.find_unsubscribe_links(html) == ["https://click.example/u/1"] + + +@pytest.mark.parametrize("html", [ + # another sentence of the same paragraph + '

Zum Abbestellen bitte antworten. Mehr dazu gibt es hier.

', + '

Mehr dazu gibt es hier. Abbestellen geht per Antwort.

', + # a line break ends the sentence + 'Newsletter abbestellen
hier geht es zum Shop', + # another block + '

Newsletter abbestellen ist einfach

hier zum Shop

', + '

hier zum Shop

Newsletter abbestellen
', + # a link with a text of its own is judged by that text + '

Abbestellen kannst du jederzeit, lies dazu unsere ausführlichen ' + 'Hinweise zum Datenschutz.

', + # the word belongs to another link (navigation row) or to a URL written out in the text + '

Abmelden | Home

', + '

https://ex.org/unsubscribe/1 oder Home

', + # not a web link + '

Zum Abbestellen hier schreiben.

', +]) +def test_short_link_outside_the_unsubscribe_sentence_is_not_taken(html): + assert "https://click.example/x" not in unsub.find_unsubscribe_links(html) + assert all("mailto" not in link for link in unsub.find_unsubscribe_links(html)) + + +def test_link_found_by_its_own_text_is_not_listed_twice_and_comes_first(): + html = ('

Du kannst dich hier abmelden oder ' + 'Newsletter abbestellen.

') + assert unsub.find_unsubscribe_links(html) == ["https://click.example/b", "https://click.example/a"]