15 Commits
Author SHA1 Message Date
Rsclub2_2 89538ed67c Fallseite bei gescheitertem Scan und Schnappschuss bei untypischen Elementen 2026-11-15 03:23:22 +01:00
Rsclub2_2 95a23c0e45 Befunde der Abschlussprüfung behoben 2026-11-15 03:02:45 +01:00
Rsclub2_2 7bd3498501 Agent: Befunde aus der Nachprüfung behoben 2026-11-15 02:28:48 +01:00
Rsclub2_2 6556087cf2 Agent: Befunde aus dem Review der Browser-Schleife behoben 2026-11-15 02:06:41 +01:00
Rsclub2_2 7e50983d85 Agent: Sicherheitskern gehärtet (Adressfilter, Protokoll, Modellzugang) 2026-11-15 01:26:35 +01:00
Rsclub2_2 181eab40fb Agent: Browser-Schleife, HTTP-Dienst und Image 2026-11-15 01:15:50 +01:00
Rsclub2_2 6df39d6b93 Fall-Ansicht, Zeitlauf und Übergabe an den Agenten verdrahtet 2026-11-15 00:47:15 +01:00
Rsclub2_2 249293f382 Fallschlüssel auch bei Message-ID mit Sonderzeichen bilden 2026-11-15 00:46:12 +01:00
Rsclub2_2 57b93cefff Zeitlauf: Befunde aus dem Review behoben 2026-11-15 00:43:51 +01:00
Rsclub2_2 65c09d96e8 Zeitlauf: Fälle finden, an den Agenten geben, verschieben, aufräumen 2026-11-15 00:40:20 +01:00
Rsclub2_2 c7b054a3bb Weiterleiter-Adresse nur übernehmen, wenn sie wohlgeformt ist 2026-11-15 00:39:38 +01:00
Rsclub2_2 7f2b317dad Agent: Protokoll mit dem Modell und Modellzugang 2026-11-14 23:35:09 +01:00
Rsclub2_2 d546f82313 Agent: Adressfilter und Seitenprüfungen ohne Modell 2026-11-14 23:35:09 +01:00
Rsclub2_2 9b87962174 Empfängeradresse aus der weitergeleiteten Mail bestimmen 2026-11-14 23:31:57 +01:00
Rsclub2_2 f6a2f6c812 Fall-Tabelle für den Abmelde-Agenten 2026-11-14 23:31:20 +01:00
7 changed files with 6 additions and 139 deletions

No files matched your search

+1 -1
View File
@@ -6,4 +6,4 @@ EMAIL_SERVER=example.com
# DATA_DIR=/data
# AGENT_URL=http://unsubscriber-agent:8080
# AGENT_ALLOWED_DOMAINS=example.org,gmail.com
# AGENT_ALLOWED_SENDERS=example.org,jens@gmail.com
# AGENT_ALLOWED_SENDERS=jens@example.org
+1 -3
View File
@@ -56,9 +56,7 @@ of entries: a bare domain (`wagnersnetz.de`) allows every address of that domain
as full addresses, never their domain: the recipient is read from the forwarded text, and a newsletter body with a
line like `To: stranger@gmail.com` would otherwise make the agent type a stranger's address.
`AGENT_ALLOWED_SENDERS` (comma separated) lists the forwarders whose mails become cases; mails from other
senders are ignored. Entries work as above: a bare domain allows every sender of that domain (not of its
subdomains), a full address exactly that sender. Use a domain only for domains whose mail you control, since the
From header decides. With the agent on and an empty list nothing is processed.
senders are ignored. With the agent on and an empty list nothing is processed.
The agent container needs `LLM_BASE_URL` (Open WebUI, `/api/chat/completions` is appended), `LLM_MODEL`,
`LLM_API_KEY` and `BLOCKED_NETS`, optionally `MAX_STEPS` (20) and `CASE_TIMEOUT` (300). `BLOCKED_NETS` is required:
+2 -45
View File
@@ -19,7 +19,6 @@ from email.header import decode_header, make_header
import httpx
from bs4 import BeautifulSoup
from bs4.element import Tag
from dotenv import load_dotenv
from flask import Flask, abort, redirect, render_template, request, url_for
@@ -58,11 +57,6 @@ URL_PATTERN = re.compile(r"https?://[^\s<>\"']+")
UID_PATTERN = re.compile(r"^\d+$")
# Characters that end a sentence or close a quote or bracket in running text, not a URL.
TRAILING_PUNCTUATION = ".,;:!?)\"'\u201d\u2019\u00bb\u203a"
# A link with a text this short ("hier", "click here") says nothing itself: the sentence around it decides.
SHORT_LINK_TEXT = 25
SENTENCE_END = re.compile(r"[.!?\u2026](?:\s|$)")
# Tags that stand inside a sentence; every other tag (br, p, td, div, ...) ends it.
INLINE_TAGS = frozenset("a abbr b big code del em font i ins mark nobr s small span strike strong sub sup tt u wbr".split())
def is_safe_link(url: str) -> bool:
@@ -89,51 +83,14 @@ def get_subject(message) -> str:
return raw
def _sentence_around(anchor) -> str:
"""The sentence a link stands in, without the link text: what is written before and after it, up to a full
stop, a line break or the edge of the paragraph or table cell."""
own = {id(parent) for parent in anchor.parents} | {id(anchor)}
def side(elements) -> list[str]:
parts = []
for element in elements:
if isinstance(element, Tag):
if element.name not in INLINE_TAGS:
break
continue
parents = [parent for parent in element.parents if id(parent) not in own]
if id(anchor) in {id(parent) for parent in element.parents}: # the link's own text
continue
if any(parent.name not in INLINE_TAGS for parent in parents): # text of a neighbouring block
break
if any(parent.name == "a" for parent in parents): # another link's text speaks for that link only
continue
parts.append(str(element))
return parts
before = " ".join("".join(reversed(side(anchor.previous_elements))).split())
after = " ".join("".join(side(anchor.next_elements)).split())
# a URL written out in the text is a link of its own (see below), not a word of this sentence
before, after = URL_PATTERN.sub(" ", before), URL_PATTERN.sub(" ", after)
return SENTENCE_END.split(before + " ")[-1] + " " + SENTENCE_END.split(after + " ")[0]
def find_unsubscribe_links(text: str) -> list[str]:
"""Unsubscribe links from HTML anchors (link text or URL matches, or a short link in a sentence that matches)
and from plain-text URLs."""
"""Unsubscribe links from HTML anchors (link text or URL matches) and from plain-text URLs."""
links: list[str] = []
soup = BeautifulSoup(text, "html.parser")
anchors = soup.find_all("a", href=True)
for anchor in anchors:
for anchor in soup.find_all("a", href=True):
href = str(anchor["href"]).strip()
if UNSUBSCRIBE_PATTERN.search(str(anchor)) and is_safe_link(href) and href not in links:
links.append(href)
for anchor in anchors: # "you can unsubscribe <a>here</a>": the word stands next to the link
href = str(anchor["href"]).strip()
label = " ".join(anchor.get_text(" ").split())
if (href not in links and is_safe_link(href) and 0 < len(label) <= SHORT_LINK_TEXT
and UNSUBSCRIBE_PATTERN.search(_sentence_around(anchor))):
links.append(href)
for url in URL_PATTERN.findall(text):
# the text search also runs over raw HTML: unescape (&amp;) like the parser does for hrefs, so the same
# link is not taken twice, and drop punctuation that follows a link in running text
-8
View File
@@ -18,14 +18,6 @@ def _allowed(address: str, entries) -> bool:
return address in entries or address.rsplit("@", 1)[-1] in entries # a domain entry never contains "@"
def sender_allowed(address: str, entries) -> bool:
"""Whether a forwarder (the address of the From header) is on the list; entries as in _allowed. Anything that
is not one plain address is refused, so that odd forms like "@allowed.example" or a quoted local part cannot
pass by their last part."""
address = address.lower()
return bool(_ADDRESS.fullmatch(address)) and _allowed(address, entries)
def original_recipient(text: str, forwarder: str, allowed_domains) -> str:
"""First allowed address from the quoted 'To:'/'An:' lines, else the forwarder, else ''.
-54
View File
@@ -104,57 +104,3 @@ def test_result_page_renders_hidden_ids_for_move_all(monkeypatch):
html = unsub.app.test_client().get("/result").get_data(as_text=True)
assert html.count('name="email_id" value="41"') == 2 # single form + move-all form
assert 'rel="noopener noreferrer"' in html
# --- link named "hier": the word that says "unsubscribe" stands next to the link, in the same sentence ---
LIEFERANDO = (
'<table><tr><td>Wir schicken Dir diese E-Mail, weil Du mit der E-Mail-Adresse '
'<a href="mailto:philipp@example.org">philipp@example.org</a> unseren Newsletter abonniert hast.<br/>'
'Wenn Du keinen Newsletter von uns mehr bekommen möchtest, kannst Du ihn '
'<a href="https://click.example/u/1"><u>hier</u></a> abbestellen.<br/><br/>'
'Lieferando ist ein Angebot von Takeaway.com<br/>'
'E-Mail <a href="https://click.example/view/2"><u>online</u></a> ansehen.</td></tr></table>'
)
def test_link_named_hier_counts_when_its_sentence_says_unsubscribe():
assert unsub.find_unsubscribe_links(LIEFERANDO) == ["https://click.example/u/1"]
@pytest.mark.parametrize("html", [
'To unsubscribe, click <a href="https://click.example/u/1">here</a>.',
'<p>Du kannst dich <a href="https://click.example/u/1"><span>hier</span></a> <b>abmelden</b>.</p>',
'<div>Newsletter <a href="https://click.example/u/1">hier</a>\n abbestellen</div>',
])
def test_sentence_around_a_short_link_is_read_across_inline_tags(html):
assert unsub.find_unsubscribe_links(html) == ["https://click.example/u/1"]
@pytest.mark.parametrize("html", [
# another sentence of the same paragraph
'<p>Zum Abbestellen bitte antworten. Mehr dazu gibt es <a href="https://click.example/x">hier</a>.</p>',
'<p>Mehr dazu gibt es <a href="https://click.example/x">hier</a>. Abbestellen geht per Antwort.</p>',
# a line break ends the sentence
'<td>Newsletter abbestellen<br><a href="https://click.example/x">hier</a> geht es zum Shop</td>',
# another block
'<p>Newsletter abbestellen ist einfach</p><p><a href="https://click.example/x">hier</a> zum Shop</p>',
'<p><a href="https://click.example/x">hier</a> zum Shop</p><div>Newsletter abbestellen</div>',
# a link with a text of its own is judged by that text
'<p>Abbestellen kannst du jederzeit, lies dazu <a href="https://click.example/x">unsere ausführlichen '
'Hinweise zum Datenschutz</a>.</p>',
# the word belongs to another link (navigation row) or to a URL written out in the text
'<p><a href="https://ex.org/unsubscribe/1">Abmelden</a> | <a href="https://click.example/x">Home</a></p>',
'<p>https://ex.org/unsubscribe/1 oder <a href="https://click.example/x">Home</a></p>',
# not a web link
'<p>Zum Abbestellen <a href="mailto:x@example.org">hier</a> schreiben.</p>',
])
def test_short_link_outside_the_unsubscribe_sentence_is_not_taken(html):
assert "https://click.example/x" not in unsub.find_unsubscribe_links(html)
assert all("mailto" not in link for link in unsub.find_unsubscribe_links(html))
def test_link_found_by_its_own_text_is_not_listed_twice_and_comes_first():
html = ('<p>Du kannst dich <a href="https://click.example/a">hier</a> abmelden oder '
'<a href="https://click.example/b">Newsletter abbestellen</a>.</p>')
assert unsub.find_unsubscribe_links(html) == ["https://click.example/b", "https://click.example/a"]
-26
View File
@@ -296,32 +296,6 @@ def test_mail_from_unknown_sender_is_ignored(store):
assert store.get("<evil@x>") is None and b"2" in imap.folders["INBOX"]
def test_sender_entry_without_at_allows_the_whole_domain(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender="Jens <JENS@Wagnersnetz.de>"),
b"2": mail_bytes("<b@x>", sender="philipp@wagnersnetz.de"),
b"3": mail_bytes("<c@x>", sender="someone@gmx.de")})
agent = Agent(OK, OK, OK)
summary = worker(store, imap, agent, allowed_senders=("Wagnersnetz.de", "someone@gmx.de")).run_once()
assert len(agent.calls) == 3 and summary["ignored"] == 0
@pytest.mark.parametrize("sender", [
"mallory@evil.example",
"mallory@sub.wagnersnetz.de", # a domain entry does not cover its subdomains
"mallory@wagnersnetz.de.evil.example",
"mallory@evilwagnersnetz.de",
"other@gmx.de", # a full address allows exactly that one
"@wagnersnetz.de", # not a plain address: must not pass by its last part
"a b@wagnersnetz.de",
'<"mallory@evil.example"@wagnersnetz.de>',
])
def test_sender_outside_the_allowed_domain_is_ignored(store, sender):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender=sender)})
agent = Agent(OK)
summary = worker(store, imap, agent, allowed_senders=("wagnersnetz.de", "someone@gmx.de")).run_once()
assert agent.calls == [] and summary["ignored"] == 1 and store.all() == []
def test_no_allowed_senders_with_agent_means_nobody(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
agent = Agent(OK)
+2 -2
View File
@@ -8,7 +8,7 @@ import time
from email.utils import parseaddr
from cases import case_key
from forwarded import original_recipient, sender_allowed
from forwarded import original_recipient
logger = logging.getLogger(__name__)
@@ -73,7 +73,7 @@ class Worker:
without += 1
continue
forwarder = parseaddr(message.get("From", ""))[1]
if (self.allowed_senders and not sender_allowed(forwarder, self.allowed_senders)) or (
if (self.allowed_senders and forwarder.lower() not in self.allowed_senders) or (
not self.allowed_senders and self.agent is not None): # fail closed
ignored += 1
continue