From 6df39d6b934ee3a701ac4df639fa52cfda19992f Mon Sep 17 00:00:00 2001 From: Philipp Wagner Date: Sun, 15 Nov 2026 00:47:15 +0100 Subject: [PATCH] =?UTF-8?q?Fall-Ansicht,=20Zeitlauf=20und=20=C3=9Cbergabe?= =?UTF-8?q?=20an=20den=20Agenten=20verdrahtet?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 8 ++- Dockerfile | 2 +- README.md | 19 ++++++ app.py | 114 +++++++++++++++++++++++++++++++++-- cases.html | 69 +++++++++++++++++++++ tests/test_cases_view.py | 126 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 330 insertions(+), 8 deletions(-) create mode 100644 cases.html create mode 100644 tests/test_cases_view.py diff --git a/.env.example b/.env.example index 0402ebd..4f52750 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,9 @@ EMAIL_USERNAME=example@example.com EMAIL_PASSWORD=example -EMAIL_SERVER=example.com \ No newline at end of file +EMAIL_SERVER=example.com + +# Unsubscribe agent (optional) +# DATA_DIR=/data +# AGENT_URL=http://unsubscriber-agent:8080 +# AGENT_ALLOWED_DOMAINS=example.org,gmail.com +# AGENT_ALLOWED_SENDERS=jens@example.org diff --git a/Dockerfile b/Dockerfile index 248a33f..c8a737b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -COPY app.py oauth.py index.html result.html ./ +COPY app.py oauth.py cases.py forwarded.py worker.py index.html result.html cases.html ./ COPY static ./static RUN useradd -m -u 1000 app diff --git a/README.md b/README.md index 382006e..5bd6dde 100644 --- a/README.md +++ b/README.md @@ -34,3 +34,22 @@ Sammelpostfachs am SSO anmelden; das Refresh-Token liegt verschlüsselt in `DATA Der Container startet gunicorn auf Port 5000 (kein Flask-Debugger) und gehört hinter einen Reverse Proxy mit Anmeldung (bei Wagnersnetz Caddy `forward_auth` über Authelia). Tests: `pytest tests`. + +### Unsubscribe agent + +With `DATA_DIR` set the app keeps one case per mail in `DATA_DIR/cases.db` and shows them under `/result`. +With `AGENT_URL` (the agent container from `agent/`, `http://:8080`) a background run every +`AGENT_INTERVAL_MINUTES` (10) hands each new case to the agent: it opens the unsubscribe link in a sandboxed +browser, first checks for a confirmation without a model and otherwise lets a model operate the page with five +actions (click, select, check, insert the address, done). Successes are moved to the IMAP folder `DONE_FOLDER` +(`Erledigt`) and deleted after `RETENTION_DAYS` (30); everything else stays in the inbox with a reason. +"Run now" starts a run in the background and returns at once; the result page shows its outcome afterwards. +Further settings: `AGENT_CASE_TIMEOUT` (300), `AGENT_MAX_CASES` (10), `AGENT_ALLOWED_DOMAINS` (comma separated; +only addresses of these domains are ever typed into a page), `AGENT_SCHEDULER=0` (no background run). +`AGENT_ALLOWED_SENDERS` (comma separated) lists the forwarders whose mails become cases; mails from other +senders are ignored. With the agent on and an empty list nothing is processed. + +The agent container needs `LLM_BASE_URL` (Open WebUI, `/api/chat/completions` is appended), `LLM_MODEL`, +`LLM_API_KEY`, optionally `MAX_STEPS` (20) and `CASE_TIMEOUT` (300). It must run in a network from which private +addresses are not reachable; the URL guard in `agent/guard.py` is only the first line of defence. +Tests: `pytest tests`, `cd agent && pytest tests` and `sh agent/tests/run-in-docker.sh` (browser tests). diff --git a/app.py b/app.py index 8671eff..1887b6d 100644 --- a/app.py +++ b/app.py @@ -6,18 +6,24 @@ handled mails to the trash. Runs behind the reverse proxy (Caddy forward_auth / mailbox credentials come from the environment. """ +import datetime import email import imaplib +import json import logging import os import re +import threading from email.header import decode_header, make_header +import httpx from bs4 import BeautifulSoup from dotenv import load_dotenv from flask import Flask, abort, redirect, render_template, request, url_for +from cases import CaseStore from oauth import MailboxOAuth, OAuthError, xoauth2_string +from worker import AgentUnavailable, Worker, _tick, start_scheduler load_dotenv() # optional .env for local runs @@ -37,6 +43,12 @@ app = Flask(__name__, template_folder=template_dir) # OAuth (SSO) for the collection mailbox if OIDC_* and TOKEN_KEY are set; otherwise EMAIL_PASSWORD OAUTH = MailboxOAuth.from_env() +# Unsubscribe agent: cases live in DATA_DIR/cases.db; without AGENT_URL they are only recorded and shown +DATA_DIR = os.environ.get("DATA_DIR", "") +AGENT_URL = os.environ.get("AGENT_URL", "").rstrip("/") +AGENT_CASE_TIMEOUT = float(os.environ.get("AGENT_CASE_TIMEOUT", "300")) +DONE_FOLDER = os.environ.get("DONE_FOLDER", "Erledigt") + UNSUBSCRIBE_PATTERN = re.compile( r"\babbestellen\b|\bunsubscribe\b|\bdeabonnieren\b|\babbestellung\b|\babmelden\b", re.IGNORECASE ) @@ -126,6 +138,59 @@ def move_emails_to_trash(uids: list[str]) -> int: return len(uids) +def parse_mail(message) -> tuple[str, list[str], str]: + text = get_text_from_email(message) + return get_subject(message), find_unsubscribe_links(text) if text else [], text + + +def call_agent(url: str, address: str) -> dict: + """Hand one link to the agent container. It gets the link and the address, nothing else.""" + try: + # the agent takes up to AGENT_CASE_TIMEOUT per link; leave room for browser start and shutdown + r = httpx.post(AGENT_URL + "/run", json={"url": url, "address": address}, + timeout=2 * AGENT_CASE_TIMEOUT + 60) + r.raise_for_status() + return r.json() + except (httpx.HTTPError, ValueError) as e: + raise AgentUnavailable(f"agent not reachable: {type(e).__name__}") from None + + +def _env_list(name: str) -> list[str]: + return [v.strip().lower() for v in os.environ.get(name, "").split(",") if v.strip()] + + +STORE = CaseStore(os.path.join(DATA_DIR, "cases.db")) if DATA_DIR else None +WORKER = None +if STORE: + STORE.recover() + WORKER = Worker( + STORE, lambda: connect(), parse_mail, call_agent if AGENT_URL else None, + inbox=EMAIL_FOLDER, done_folder=DONE_FOLDER, + retention_days=int(os.environ.get("RETENTION_DAYS", "30")), + max_cases=int(os.environ.get("AGENT_MAX_CASES", "10")), + allowed_domains=_env_list("AGENT_ALLOWED_DOMAINS"), + allowed_senders=_env_list("AGENT_ALLOWED_SENDERS"), + ) + if os.environ.get("AGENT_SCHEDULER", "1") == "1": + start_scheduler(WORKER, 60 * int(os.environ.get("AGENT_INTERVAL_MINUTES", "10"))) + + +def scan_inbox() -> dict: + """{case key: uid} of the mails currently in the inbox; creates cases for new mails.""" + mail = connect() + try: + return WORKER.scan(mail)[0] + finally: + mail.logout() + + +def start_run() -> threading.Thread: + """One run in the background (it can take many minutes); _tick notes any error instead of raising.""" + thread = threading.Thread(target=_tick, args=(WORKER,), name="unsubscribe-run-now", daemon=True) + thread.start() + return thread + + @app.route("/", methods=["GET"]) def index(): return render_template( @@ -182,14 +247,51 @@ def oauth_error(e): return redirect(url_for("index", error="Mailbox not connected or token expired – connect again")) +STATE_LABELS = {"new": "new", "running": "running", "unsubscribed": "unsubscribed", + "not_possible": "not possible", "unclear": "unclear"} + + @app.route("/result") def result(): - mail = connect() - try: - links_data = collect_links(mail) - finally: - mail.logout() - return render_template("result.html", links_data=links_data) + if WORKER is None: + mail = connect() + try: + links_data = collect_links(mail) + finally: + mail.logout() + return render_template("result.html", links_data=links_data) + uids = scan_inbox() + cases = [dict(c, uid=uids[c["key"]].decode() if c["key"] in uids else "", label=STATE_LABELS[c["state"]]) + for c in STORE.all()] + last = json.loads(STORE.meta("last_run", "{}")) + at = STORE.meta("last_run_at") + return render_template( + "cases.html", cases=cases, last=last, agent=bool(AGENT_URL), paused=STORE.meta("paused") == "1", + last_at=datetime.datetime.fromtimestamp(int(at)).strftime("%Y-%m-%d %H:%M") if at else "never") + + +@app.route("/run-now", methods=["POST"]) +def run_now(): + if WORKER is None: + abort(404) + start_run() + return redirect(url_for("result")) + + +@app.route("/retry", methods=["POST"]) +def retry(): + if STORE is None: + abort(404) + STORE.retry(request.form.get("key", "")) + return redirect(url_for("result")) + + +@app.route("/pause", methods=["POST"]) +def pause(): + if STORE is None: + abort(404) + STORE.set_meta("paused", "1" if request.form.get("paused") == "1" else "0") + return redirect(url_for("result")) @app.route("/move-to-trash", methods=["POST"]) diff --git a/cases.html b/cases.html new file mode 100644 index 0000000..3ce802b --- /dev/null +++ b/cases.html @@ -0,0 +1,69 @@ + + + + E-Mail Unsubscribe Cases + + + +
+

E-Mail Unsubscribe Cases

+

+ Last run: {{ last_at }}{% if last %} – + {{ last.get('new', 0) }} new, {{ last.get('unsubscribed', 0) }} unsubscribed, + {{ last.get('moved', 0) }} moved, {{ last.get('cleaned', 0) }} cleaned up. + {{ last.get('without_link', 0) }} mail(s) without unsubscribe link. + {{ last.get('ignored', 0) }} mail(s) from senders that are not allowed.{% endif %} +

+ {% if last.get('error') %}

Last run: {{ last['error'] }}

{% endif %} + {% if not agent %}

The agent is switched off: cases are listed, nothing is unsubscribed automatically.

{% endif %} +
+ +
+
+ + +
+ + + + + + + + + {% for case in cases %} + + + + + + + + {% endfor %} +
SubjectStateReasonUnsubscribe LinkActions
{{ case.subject }}{{ case.label }} + {{ case.reason }} + {% if case.evidence %}
“…{{ case.evidence }}” ({{ case.evidence_url }}){% endif %} +
+ {% for link in case.links %} + {{ link[:30] + '...' }}
+ {% endfor %} +
+ {% if case.state in ('not_possible', 'unclear') %} +
+ + +
+ {% endif %} + {% if case.uid %} +
+ + +
+ {% endif %} +
+
+ +
+
+ + diff --git a/tests/test_cases_view.py b/tests/test_cases_view.py new file mode 100644 index 0000000..b906cd0 --- /dev/null +++ b/tests/test_cases_view.py @@ -0,0 +1,126 @@ +"""Case view and its actions (the store is real, the mailbox is faked).""" + +import importlib +import json + +import pytest + + +@pytest.fixture +def unsub(tmp_path, monkeypatch): + monkeypatch.setenv("DATA_DIR", str(tmp_path)) + monkeypatch.setenv("AGENT_SCHEDULER", "0") + monkeypatch.setenv("AGENT_URL", "http://agent.test:8080") + monkeypatch.setenv("AGENT_ALLOWED_DOMAINS", "wagnersnetz.de, gmail.com") + monkeypatch.setenv("AGENT_ALLOWED_SENDERS", "Jens@wagnersnetz.de, philipp@wagnersnetz.de") + for name in ("OIDC_ISSUER", "OIDC_CLIENT_ID", "TOKEN_KEY"): + monkeypatch.delenv(name, raising=False) + import app + module = importlib.reload(app) + yield module + monkeypatch.delenv("DATA_DIR") + monkeypatch.delenv("AGENT_URL") + importlib.reload(app) # the other test modules expect the app without a store + + +def test_settings_from_environment(unsub): + assert unsub.STORE is not None and unsub.WORKER is not None + assert unsub.WORKER.allowed_domains == ("wagnersnetz.de", "gmail.com") + assert unsub.WORKER.allowed_senders == ("jens@wagnersnetz.de", "philipp@wagnersnetz.de") + assert unsub.WORKER.agent is unsub.call_agent + + +def test_result_shows_cases_with_state_reason_evidence_and_links(unsub, monkeypatch): + s = unsub.STORE + s.add("", "Fwd: Shop News", ["https://shop.example/u/1"], "p@wagnersnetz.de") + s.claim("") + s.finish("", "unsubscribed", "page confirms", "Sie wurden abgemeldet", "https://shop.example/done") + s.add("", "Fwd: Bank", ["https://bank.example/u"], "") + s.claim("") + s.finish("", "not_possible", "login") + s.set_meta("last_run", json.dumps({"new": 2, "handled": 2, "unsubscribed": 1, "moved": 1, "cleaned": 0, + "without_link": 3, "ignored": 2, "error": ""})) + s.set_meta("last_run_at", "1800000000") + monkeypatch.setattr(unsub, "scan_inbox", lambda: {"": b"42"}) + html = unsub.app.test_client().get("/result").get_data(as_text=True) + assert "Fwd: Shop <b>News</b>" in html and "News" not in html + assert "unsubscribed" in html and "Sie wurden abgemeldet" in html and "https://shop.example/done" in html + assert "not possible" in html and "login" in html and 'href="https://bank.example/u"' in html + assert 'name="email_id" value="42"' in html # trash button only for mails still in the inbox + assert html.count('action="/retry"') == 1 # only the failed case can be retried + assert "3 mail(s) without unsubscribe link" in html + assert "2 mail(s) from senders that are not allowed" in html + + +def test_retry_resets_the_case(unsub): + s = unsub.STORE + s.add("", "s", ["https://bank.example/u"], "") + s.claim("") + s.finish("", "unclear", "x") + r = unsub.app.test_client().post("/retry", data={"key": ""}) + assert r.status_code == 302 and s.get("")["state"] == "new" and s.get("")["attempts"] == 0 + + +def test_pause_and_resume(unsub): + c = unsub.app.test_client() + c.post("/pause", data={"paused": "1"}) + assert unsub.STORE.meta("paused") == "1" + c.post("/pause", data={"paused": "0"}) + assert unsub.STORE.meta("paused") == "0" + + +def test_run_now_starts_a_background_run_and_returns_at_once(unsub, monkeypatch): + calls, threads = [], [] + real_start_run = unsub.start_run + + def keep_thread(): + threads.append(real_start_run()) + return threads[-1] + + monkeypatch.setattr(unsub, "start_run", keep_thread) + monkeypatch.setattr(unsub.WORKER, "run_once", lambda: calls.append(1) or {"error": ""}) + assert unsub.app.test_client().post("/run-now").status_code == 302 + threads[0].join(timeout=5) + assert calls == [1] + + def boom(): + raise RuntimeError("secret mail content") + + monkeypatch.setattr(unsub.WORKER, "run_once", boom) + assert unsub.app.test_client().post("/run-now").status_code == 302 + threads[1].join(timeout=5) + assert json.loads(unsub.STORE.meta("last_run"))["error"] == "RuntimeError" + + +def test_call_agent_maps_failures_to_unavailable(unsub, monkeypatch): + import httpx + + def boom(*a, **kw): + raise httpx.ConnectError("refused") + + monkeypatch.setattr(unsub.httpx, "post", boom) + with pytest.raises(unsub.AgentUnavailable): + unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de") + monkeypatch.setattr(unsub.httpx, "post", lambda *a, **kw: httpx.Response(500, text="x", request=httpx.Request("POST", a[0]))) + with pytest.raises(unsub.AgentUnavailable): + unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de") + + +def test_call_agent_sends_only_url_and_address(unsub, monkeypatch): + import httpx + seen = {} + + def post(url, json=None, timeout=None): + seen.update(url=url, json=json, timeout=timeout) + return httpx.Response(200, json={"result": "unclear", "reason": "x"}, request=httpx.Request("POST", url)) + + monkeypatch.setattr(unsub.httpx, "post", post) + assert unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de")["result"] == "unclear" + assert seen["url"] == "http://agent.test:8080/run" + assert seen["json"] == {"url": "https://shop.example/u", "address": "p@wagnersnetz.de"} + assert seen["timeout"] > 600 # two links, 5 minutes each + + +def test_without_data_dir_the_old_list_is_shown(): + import app + assert app.STORE is None and app.WORKER is None