Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
081f50f69a
|
||
|
|
bb64133b36
|
No files matched your search
+1
-1
@@ -6,4 +6,4 @@ EMAIL_SERVER=example.com
|
|||||||
# DATA_DIR=/data
|
# DATA_DIR=/data
|
||||||
# AGENT_URL=http://unsubscriber-agent:8080
|
# AGENT_URL=http://unsubscriber-agent:8080
|
||||||
# AGENT_ALLOWED_DOMAINS=example.org,gmail.com
|
# AGENT_ALLOWED_DOMAINS=example.org,gmail.com
|
||||||
# AGENT_ALLOWED_SENDERS=jens@example.org
|
# AGENT_ALLOWED_SENDERS=example.org,jens@gmail.com
|
||||||
@@ -56,7 +56,9 @@ of entries: a bare domain (`wagnersnetz.de`) allows every address of that domain
|
|||||||
as full addresses, never their domain: the recipient is read from the forwarded text, and a newsletter body with a
|
as full addresses, never their domain: the recipient is read from the forwarded text, and a newsletter body with a
|
||||||
line like `To: stranger@gmail.com` would otherwise make the agent type a stranger's address.
|
line like `To: stranger@gmail.com` would otherwise make the agent type a stranger's address.
|
||||||
`AGENT_ALLOWED_SENDERS` (comma separated) lists the forwarders whose mails become cases; mails from other
|
`AGENT_ALLOWED_SENDERS` (comma separated) lists the forwarders whose mails become cases; mails from other
|
||||||
senders are ignored. With the agent on and an empty list nothing is processed.
|
senders are ignored. Entries work as above: a bare domain allows every sender of that domain (not of its
|
||||||
|
subdomains), a full address exactly that sender. Use a domain only for domains whose mail you control, since the
|
||||||
|
From header decides. With the agent on and an empty list nothing is processed.
|
||||||
|
|
||||||
The agent container needs `LLM_BASE_URL` (Open WebUI, `/api/chat/completions` is appended), `LLM_MODEL`,
|
The agent container needs `LLM_BASE_URL` (Open WebUI, `/api/chat/completions` is appended), `LLM_MODEL`,
|
||||||
`LLM_API_KEY` and `BLOCKED_NETS`, optionally `MAX_STEPS` (20) and `CASE_TIMEOUT` (300). `BLOCKED_NETS` is required:
|
`LLM_API_KEY` and `BLOCKED_NETS`, optionally `MAX_STEPS` (20) and `CASE_TIMEOUT` (300). `BLOCKED_NETS` is required:
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ from email.header import decode_header, make_header
|
|||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
|
from bs4.element import Tag
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from flask import Flask, abort, redirect, render_template, request, url_for
|
from flask import Flask, abort, redirect, render_template, request, url_for
|
||||||
|
|
||||||
@@ -57,6 +58,11 @@ URL_PATTERN = re.compile(r"https?://[^\s<>\"']+")
|
|||||||
UID_PATTERN = re.compile(r"^\d+$")
|
UID_PATTERN = re.compile(r"^\d+$")
|
||||||
# Characters that end a sentence or close a quote or bracket in running text, not a URL.
|
# Characters that end a sentence or close a quote or bracket in running text, not a URL.
|
||||||
TRAILING_PUNCTUATION = ".,;:!?)\"'\u201d\u2019\u00bb\u203a"
|
TRAILING_PUNCTUATION = ".,;:!?)\"'\u201d\u2019\u00bb\u203a"
|
||||||
|
# A link with a text this short ("hier", "click here") says nothing itself: the sentence around it decides.
|
||||||
|
SHORT_LINK_TEXT = 25
|
||||||
|
SENTENCE_END = re.compile(r"[.!?\u2026](?:\s|$)")
|
||||||
|
# Tags that stand inside a sentence; every other tag (br, p, td, div, ...) ends it.
|
||||||
|
INLINE_TAGS = frozenset("a abbr b big code del em font i ins mark nobr s small span strike strong sub sup tt u wbr".split())
|
||||||
|
|
||||||
|
|
||||||
def is_safe_link(url: str) -> bool:
|
def is_safe_link(url: str) -> bool:
|
||||||
@@ -83,14 +89,51 @@ def get_subject(message) -> str:
|
|||||||
return raw
|
return raw
|
||||||
|
|
||||||
|
|
||||||
|
def _sentence_around(anchor) -> str:
|
||||||
|
"""The sentence a link stands in, without the link text: what is written before and after it, up to a full
|
||||||
|
stop, a line break or the edge of the paragraph or table cell."""
|
||||||
|
own = {id(parent) for parent in anchor.parents} | {id(anchor)}
|
||||||
|
|
||||||
|
def side(elements) -> list[str]:
|
||||||
|
parts = []
|
||||||
|
for element in elements:
|
||||||
|
if isinstance(element, Tag):
|
||||||
|
if element.name not in INLINE_TAGS:
|
||||||
|
break
|
||||||
|
continue
|
||||||
|
parents = [parent for parent in element.parents if id(parent) not in own]
|
||||||
|
if id(anchor) in {id(parent) for parent in element.parents}: # the link's own text
|
||||||
|
continue
|
||||||
|
if any(parent.name not in INLINE_TAGS for parent in parents): # text of a neighbouring block
|
||||||
|
break
|
||||||
|
if any(parent.name == "a" for parent in parents): # another link's text speaks for that link only
|
||||||
|
continue
|
||||||
|
parts.append(str(element))
|
||||||
|
return parts
|
||||||
|
|
||||||
|
before = " ".join("".join(reversed(side(anchor.previous_elements))).split())
|
||||||
|
after = " ".join("".join(side(anchor.next_elements)).split())
|
||||||
|
# a URL written out in the text is a link of its own (see below), not a word of this sentence
|
||||||
|
before, after = URL_PATTERN.sub(" ", before), URL_PATTERN.sub(" ", after)
|
||||||
|
return SENTENCE_END.split(before + " ")[-1] + " " + SENTENCE_END.split(after + " ")[0]
|
||||||
|
|
||||||
|
|
||||||
def find_unsubscribe_links(text: str) -> list[str]:
|
def find_unsubscribe_links(text: str) -> list[str]:
|
||||||
"""Unsubscribe links from HTML anchors (link text or URL matches) and from plain-text URLs."""
|
"""Unsubscribe links from HTML anchors (link text or URL matches, or a short link in a sentence that matches)
|
||||||
|
and from plain-text URLs."""
|
||||||
links: list[str] = []
|
links: list[str] = []
|
||||||
soup = BeautifulSoup(text, "html.parser")
|
soup = BeautifulSoup(text, "html.parser")
|
||||||
for anchor in soup.find_all("a", href=True):
|
anchors = soup.find_all("a", href=True)
|
||||||
|
for anchor in anchors:
|
||||||
href = str(anchor["href"]).strip()
|
href = str(anchor["href"]).strip()
|
||||||
if UNSUBSCRIBE_PATTERN.search(str(anchor)) and is_safe_link(href) and href not in links:
|
if UNSUBSCRIBE_PATTERN.search(str(anchor)) and is_safe_link(href) and href not in links:
|
||||||
links.append(href)
|
links.append(href)
|
||||||
|
for anchor in anchors: # "you can unsubscribe <a>here</a>": the word stands next to the link
|
||||||
|
href = str(anchor["href"]).strip()
|
||||||
|
label = " ".join(anchor.get_text(" ").split())
|
||||||
|
if (href not in links and is_safe_link(href) and 0 < len(label) <= SHORT_LINK_TEXT
|
||||||
|
and UNSUBSCRIBE_PATTERN.search(_sentence_around(anchor))):
|
||||||
|
links.append(href)
|
||||||
for url in URL_PATTERN.findall(text):
|
for url in URL_PATTERN.findall(text):
|
||||||
# the text search also runs over raw HTML: unescape (&) like the parser does for hrefs, so the same
|
# the text search also runs over raw HTML: unescape (&) like the parser does for hrefs, so the same
|
||||||
# link is not taken twice, and drop punctuation that follows a link in running text
|
# link is not taken twice, and drop punctuation that follows a link in running text
|
||||||
|
|||||||
@@ -18,6 +18,14 @@ def _allowed(address: str, entries) -> bool:
|
|||||||
return address in entries or address.rsplit("@", 1)[-1] in entries # a domain entry never contains "@"
|
return address in entries or address.rsplit("@", 1)[-1] in entries # a domain entry never contains "@"
|
||||||
|
|
||||||
|
|
||||||
|
def sender_allowed(address: str, entries) -> bool:
|
||||||
|
"""Whether a forwarder (the address of the From header) is on the list; entries as in _allowed. Anything that
|
||||||
|
is not one plain address is refused, so that odd forms like "@allowed.example" or a quoted local part cannot
|
||||||
|
pass by their last part."""
|
||||||
|
address = address.lower()
|
||||||
|
return bool(_ADDRESS.fullmatch(address)) and _allowed(address, entries)
|
||||||
|
|
||||||
|
|
||||||
def original_recipient(text: str, forwarder: str, allowed_domains) -> str:
|
def original_recipient(text: str, forwarder: str, allowed_domains) -> str:
|
||||||
"""First allowed address from the quoted 'To:'/'An:' lines, else the forwarder, else ''.
|
"""First allowed address from the quoted 'To:'/'An:' lines, else the forwarder, else ''.
|
||||||
|
|
||||||
|
|||||||
@@ -104,3 +104,57 @@ def test_result_page_renders_hidden_ids_for_move_all(monkeypatch):
|
|||||||
html = unsub.app.test_client().get("/result").get_data(as_text=True)
|
html = unsub.app.test_client().get("/result").get_data(as_text=True)
|
||||||
assert html.count('name="email_id" value="41"') == 2 # single form + move-all form
|
assert html.count('name="email_id" value="41"') == 2 # single form + move-all form
|
||||||
assert 'rel="noopener noreferrer"' in html
|
assert 'rel="noopener noreferrer"' in html
|
||||||
|
|
||||||
|
|
||||||
|
# --- link named "hier": the word that says "unsubscribe" stands next to the link, in the same sentence ---
|
||||||
|
|
||||||
|
LIEFERANDO = (
|
||||||
|
'<table><tr><td>Wir schicken Dir diese E-Mail, weil Du mit der E-Mail-Adresse '
|
||||||
|
'<a href="mailto:philipp@example.org">philipp@example.org</a> unseren Newsletter abonniert hast.<br/>'
|
||||||
|
'Wenn Du keinen Newsletter von uns mehr bekommen möchtest, kannst Du ihn '
|
||||||
|
'<a href="https://click.example/u/1"><u>hier</u></a> abbestellen.<br/><br/>'
|
||||||
|
'Lieferando ist ein Angebot von Takeaway.com<br/>'
|
||||||
|
'E-Mail <a href="https://click.example/view/2"><u>online</u></a> ansehen.</td></tr></table>'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_link_named_hier_counts_when_its_sentence_says_unsubscribe():
|
||||||
|
assert unsub.find_unsubscribe_links(LIEFERANDO) == ["https://click.example/u/1"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("html", [
|
||||||
|
'To unsubscribe, click <a href="https://click.example/u/1">here</a>.',
|
||||||
|
'<p>Du kannst dich <a href="https://click.example/u/1"><span>hier</span></a> <b>abmelden</b>.</p>',
|
||||||
|
'<div>Newsletter <a href="https://click.example/u/1">hier</a>\n abbestellen</div>',
|
||||||
|
])
|
||||||
|
def test_sentence_around_a_short_link_is_read_across_inline_tags(html):
|
||||||
|
assert unsub.find_unsubscribe_links(html) == ["https://click.example/u/1"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("html", [
|
||||||
|
# another sentence of the same paragraph
|
||||||
|
'<p>Zum Abbestellen bitte antworten. Mehr dazu gibt es <a href="https://click.example/x">hier</a>.</p>',
|
||||||
|
'<p>Mehr dazu gibt es <a href="https://click.example/x">hier</a>. Abbestellen geht per Antwort.</p>',
|
||||||
|
# a line break ends the sentence
|
||||||
|
'<td>Newsletter abbestellen<br><a href="https://click.example/x">hier</a> geht es zum Shop</td>',
|
||||||
|
# another block
|
||||||
|
'<p>Newsletter abbestellen ist einfach</p><p><a href="https://click.example/x">hier</a> zum Shop</p>',
|
||||||
|
'<p><a href="https://click.example/x">hier</a> zum Shop</p><div>Newsletter abbestellen</div>',
|
||||||
|
# a link with a text of its own is judged by that text
|
||||||
|
'<p>Abbestellen kannst du jederzeit, lies dazu <a href="https://click.example/x">unsere ausführlichen '
|
||||||
|
'Hinweise zum Datenschutz</a>.</p>',
|
||||||
|
# the word belongs to another link (navigation row) or to a URL written out in the text
|
||||||
|
'<p><a href="https://ex.org/unsubscribe/1">Abmelden</a> | <a href="https://click.example/x">Home</a></p>',
|
||||||
|
'<p>https://ex.org/unsubscribe/1 oder <a href="https://click.example/x">Home</a></p>',
|
||||||
|
# not a web link
|
||||||
|
'<p>Zum Abbestellen <a href="mailto:x@example.org">hier</a> schreiben.</p>',
|
||||||
|
])
|
||||||
|
def test_short_link_outside_the_unsubscribe_sentence_is_not_taken(html):
|
||||||
|
assert "https://click.example/x" not in unsub.find_unsubscribe_links(html)
|
||||||
|
assert all("mailto" not in link for link in unsub.find_unsubscribe_links(html))
|
||||||
|
|
||||||
|
|
||||||
|
def test_link_found_by_its_own_text_is_not_listed_twice_and_comes_first():
|
||||||
|
html = ('<p>Du kannst dich <a href="https://click.example/a">hier</a> abmelden oder '
|
||||||
|
'<a href="https://click.example/b">Newsletter abbestellen</a>.</p>')
|
||||||
|
assert unsub.find_unsubscribe_links(html) == ["https://click.example/b", "https://click.example/a"]
|
||||||
@@ -296,6 +296,32 @@ def test_mail_from_unknown_sender_is_ignored(store):
|
|||||||
assert store.get("<evil@x>") is None and b"2" in imap.folders["INBOX"]
|
assert store.get("<evil@x>") is None and b"2" in imap.folders["INBOX"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_sender_entry_without_at_allows_the_whole_domain(store):
|
||||||
|
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender="Jens <JENS@Wagnersnetz.de>"),
|
||||||
|
b"2": mail_bytes("<b@x>", sender="philipp@wagnersnetz.de"),
|
||||||
|
b"3": mail_bytes("<c@x>", sender="someone@gmx.de")})
|
||||||
|
agent = Agent(OK, OK, OK)
|
||||||
|
summary = worker(store, imap, agent, allowed_senders=("Wagnersnetz.de", "someone@gmx.de")).run_once()
|
||||||
|
assert len(agent.calls) == 3 and summary["ignored"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("sender", [
|
||||||
|
"mallory@evil.example",
|
||||||
|
"mallory@sub.wagnersnetz.de", # a domain entry does not cover its subdomains
|
||||||
|
"mallory@wagnersnetz.de.evil.example",
|
||||||
|
"mallory@evilwagnersnetz.de",
|
||||||
|
"other@gmx.de", # a full address allows exactly that one
|
||||||
|
"@wagnersnetz.de", # not a plain address: must not pass by its last part
|
||||||
|
"a b@wagnersnetz.de",
|
||||||
|
'<"mallory@evil.example"@wagnersnetz.de>',
|
||||||
|
])
|
||||||
|
def test_sender_outside_the_allowed_domain_is_ignored(store, sender):
|
||||||
|
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender=sender)})
|
||||||
|
agent = Agent(OK)
|
||||||
|
summary = worker(store, imap, agent, allowed_senders=("wagnersnetz.de", "someone@gmx.de")).run_once()
|
||||||
|
assert agent.calls == [] and summary["ignored"] == 1 and store.all() == []
|
||||||
|
|
||||||
|
|
||||||
def test_no_allowed_senders_with_agent_means_nobody(store):
|
def test_no_allowed_senders_with_agent_means_nobody(store):
|
||||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
|
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
|
||||||
agent = Agent(OK)
|
agent = Agent(OK)
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import time
|
|||||||
from email.utils import parseaddr
|
from email.utils import parseaddr
|
||||||
|
|
||||||
from cases import case_key
|
from cases import case_key
|
||||||
from forwarded import original_recipient
|
from forwarded import original_recipient, sender_allowed
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -73,7 +73,7 @@ class Worker:
|
|||||||
without += 1
|
without += 1
|
||||||
continue
|
continue
|
||||||
forwarder = parseaddr(message.get("From", ""))[1]
|
forwarder = parseaddr(message.get("From", ""))[1]
|
||||||
if (self.allowed_senders and forwarder.lower() not in self.allowed_senders) or (
|
if (self.allowed_senders and not sender_allowed(forwarder, self.allowed_senders)) or (
|
||||||
not self.allowed_senders and self.agent is not None): # fail closed
|
not self.allowed_senders and self.agent is not None): # fail closed
|
||||||
ignored += 1
|
ignored += 1
|
||||||
continue
|
continue
|
||||||
|
|||||||
Reference in new issue
Block a user