2 changed files with 99 additions and 2 deletions

No files matched your search

+45 -2
View File
@@ -19,6 +19,7 @@ from email.header import decode_header, make_header
import httpx
from bs4 import BeautifulSoup
from bs4.element import Tag
from dotenv import load_dotenv
from flask import Flask, abort, redirect, render_template, request, url_for
@@ -57,6 +58,11 @@ URL_PATTERN = re.compile(r"https?://[^\s<>\"']+")
UID_PATTERN = re.compile(r"^\d+$")
# Characters that end a sentence or close a quote or bracket in running text, not a URL.
TRAILING_PUNCTUATION = ".,;:!?)\"'\u201d\u2019\u00bb\u203a"
# A link with a text this short ("hier", "click here") says nothing itself: the sentence around it decides.
SHORT_LINK_TEXT = 25
SENTENCE_END = re.compile(r"[.!?\u2026](?:\s|$)")
# Tags that stand inside a sentence; every other tag (br, p, td, div, ...) ends it.
INLINE_TAGS = frozenset("a abbr b big code del em font i ins mark nobr s small span strike strong sub sup tt u wbr".split())
def is_safe_link(url: str) -> bool:
@@ -83,14 +89,51 @@ def get_subject(message) -> str:
return raw
def _sentence_around(anchor) -> str:
"""The sentence a link stands in, without the link text: what is written before and after it, up to a full
stop, a line break or the edge of the paragraph or table cell."""
own = {id(parent) for parent in anchor.parents} | {id(anchor)}
def side(elements) -> list[str]:
parts = []
for element in elements:
if isinstance(element, Tag):
if element.name not in INLINE_TAGS:
break
continue
parents = [parent for parent in element.parents if id(parent) not in own]
if id(anchor) in {id(parent) for parent in element.parents}: # the link's own text
continue
if any(parent.name not in INLINE_TAGS for parent in parents): # text of a neighbouring block
break
if any(parent.name == "a" for parent in parents): # another link's text speaks for that link only
continue
parts.append(str(element))
return parts
before = " ".join("".join(reversed(side(anchor.previous_elements))).split())
after = " ".join("".join(side(anchor.next_elements)).split())
# a URL written out in the text is a link of its own (see below), not a word of this sentence
before, after = URL_PATTERN.sub(" ", before), URL_PATTERN.sub(" ", after)
return SENTENCE_END.split(before + " ")[-1] + " " + SENTENCE_END.split(after + " ")[0]
def find_unsubscribe_links(text: str) -> list[str]:
"""Unsubscribe links from HTML anchors (link text or URL matches) and from plain-text URLs."""
"""Unsubscribe links from HTML anchors (link text or URL matches, or a short link in a sentence that matches)
and from plain-text URLs."""
links: list[str] = []
soup = BeautifulSoup(text, "html.parser")
for anchor in soup.find_all("a", href=True):
anchors = soup.find_all("a", href=True)
for anchor in anchors:
href = str(anchor["href"]).strip()
if UNSUBSCRIBE_PATTERN.search(str(anchor)) and is_safe_link(href) and href not in links:
links.append(href)
for anchor in anchors: # "you can unsubscribe <a>here</a>": the word stands next to the link
href = str(anchor["href"]).strip()
label = " ".join(anchor.get_text(" ").split())
if (href not in links and is_safe_link(href) and 0 < len(label) <= SHORT_LINK_TEXT
and UNSUBSCRIBE_PATTERN.search(_sentence_around(anchor))):
links.append(href)
for url in URL_PATTERN.findall(text):
# the text search also runs over raw HTML: unescape (&amp;) like the parser does for hrefs, so the same
# link is not taken twice, and drop punctuation that follows a link in running text
+54
View File
@@ -104,3 +104,57 @@ def test_result_page_renders_hidden_ids_for_move_all(monkeypatch):
html = unsub.app.test_client().get("/result").get_data(as_text=True)
assert html.count('name="email_id" value="41"') == 2 # single form + move-all form
assert 'rel="noopener noreferrer"' in html
# --- link named "hier": the word that says "unsubscribe" stands next to the link, in the same sentence ---
LIEFERANDO = (
'<table><tr><td>Wir schicken Dir diese E-Mail, weil Du mit der E-Mail-Adresse '
'<a href="mailto:philipp@example.org">philipp@example.org</a> unseren Newsletter abonniert hast.<br/>'
'Wenn Du keinen Newsletter von uns mehr bekommen möchtest, kannst Du ihn '
'<a href="https://click.example/u/1"><u>hier</u></a> abbestellen.<br/><br/>'
'Lieferando ist ein Angebot von Takeaway.com<br/>'
'E-Mail <a href="https://click.example/view/2"><u>online</u></a> ansehen.</td></tr></table>'
)
def test_link_named_hier_counts_when_its_sentence_says_unsubscribe():
assert unsub.find_unsubscribe_links(LIEFERANDO) == ["https://click.example/u/1"]
@pytest.mark.parametrize("html", [
'To unsubscribe, click <a href="https://click.example/u/1">here</a>.',
'<p>Du kannst dich <a href="https://click.example/u/1"><span>hier</span></a> <b>abmelden</b>.</p>',
'<div>Newsletter <a href="https://click.example/u/1">hier</a>\n abbestellen</div>',
])
def test_sentence_around_a_short_link_is_read_across_inline_tags(html):
assert unsub.find_unsubscribe_links(html) == ["https://click.example/u/1"]
@pytest.mark.parametrize("html", [
# another sentence of the same paragraph
'<p>Zum Abbestellen bitte antworten. Mehr dazu gibt es <a href="https://click.example/x">hier</a>.</p>',
'<p>Mehr dazu gibt es <a href="https://click.example/x">hier</a>. Abbestellen geht per Antwort.</p>',
# a line break ends the sentence
'<td>Newsletter abbestellen<br><a href="https://click.example/x">hier</a> geht es zum Shop</td>',
# another block
'<p>Newsletter abbestellen ist einfach</p><p><a href="https://click.example/x">hier</a> zum Shop</p>',
'<p><a href="https://click.example/x">hier</a> zum Shop</p><div>Newsletter abbestellen</div>',
# a link with a text of its own is judged by that text
'<p>Abbestellen kannst du jederzeit, lies dazu <a href="https://click.example/x">unsere ausführlichen '
'Hinweise zum Datenschutz</a>.</p>',
# the word belongs to another link (navigation row) or to a URL written out in the text
'<p><a href="https://ex.org/unsubscribe/1">Abmelden</a> | <a href="https://click.example/x">Home</a></p>',
'<p>https://ex.org/unsubscribe/1 oder <a href="https://click.example/x">Home</a></p>',
# not a web link
'<p>Zum Abbestellen <a href="mailto:x@example.org">hier</a> schreiben.</p>',
])
def test_short_link_outside_the_unsubscribe_sentence_is_not_taken(html):
assert "https://click.example/x" not in unsub.find_unsubscribe_links(html)
assert all("mailto" not in link for link in unsub.find_unsubscribe_links(html))
def test_link_found_by_its_own_text_is_not_listed_twice_and_comes_first():
html = ('<p>Du kannst dich <a href="https://click.example/a">hier</a> abmelden oder '
'<a href="https://click.example/b">Newsletter abbestellen</a>.</p>')
assert unsub.find_unsubscribe_links(html) == ["https://click.example/b", "https://click.example/a"]