Fall-Ansicht, Zeitlauf und Übergabe an den Agenten verdrahtet

This commit is contained in:
Rsclub2_2 committed 2026-11-15 00:47:15 +01:00
1 parent 249293f382
commit 6df39d6b93
6 files changed
+330 -8

No files matched your search

+7 -1
View File
@@ -1,3 +1,9 @@
EMAIL_USERNAME=example@example.com EMAIL_USERNAME=example@example.com
EMAIL_PASSWORD=example EMAIL_PASSWORD=example
EMAIL_SERVER=example.com EMAIL_SERVER=example.com
# Unsubscribe agent (optional)
# DATA_DIR=/data
# AGENT_URL=http://unsubscriber-agent:8080
# AGENT_ALLOWED_DOMAINS=example.org,gmail.com
# AGENT_ALLOWED_SENDERS=jens@example.org
+1 -1
View File
@@ -5,7 +5,7 @@ WORKDIR /app
COPY requirements.txt . COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt RUN pip install --no-cache-dir -r requirements.txt
COPY app.py oauth.py index.html result.html ./ COPY app.py oauth.py cases.py forwarded.py worker.py index.html result.html cases.html ./
COPY static ./static COPY static ./static
RUN useradd -m -u 1000 app RUN useradd -m -u 1000 app
+19
View File
@@ -34,3 +34,22 @@ Sammelpostfachs am SSO anmelden; das Refresh-Token liegt verschlüsselt in `DATA
Der Container startet gunicorn auf Port 5000 (kein Flask-Debugger) und gehört hinter einen Der Container startet gunicorn auf Port 5000 (kein Flask-Debugger) und gehört hinter einen
Reverse Proxy mit Anmeldung (bei Wagnersnetz Caddy `forward_auth` über Authelia). Reverse Proxy mit Anmeldung (bei Wagnersnetz Caddy `forward_auth` über Authelia).
Tests: `pytest tests`. Tests: `pytest tests`.
### Unsubscribe agent
With `DATA_DIR` set the app keeps one case per mail in `DATA_DIR/cases.db` and shows them under `/result`.
With `AGENT_URL` (the agent container from `agent/`, `http://<name>:8080`) a background run every
`AGENT_INTERVAL_MINUTES` (10) hands each new case to the agent: it opens the unsubscribe link in a sandboxed
browser, first checks for a confirmation without a model and otherwise lets a model operate the page with five
actions (click, select, check, insert the address, done). Successes are moved to the IMAP folder `DONE_FOLDER`
(`Erledigt`) and deleted after `RETENTION_DAYS` (30); everything else stays in the inbox with a reason.
"Run now" starts a run in the background and returns at once; the result page shows its outcome afterwards.
Further settings: `AGENT_CASE_TIMEOUT` (300), `AGENT_MAX_CASES` (10), `AGENT_ALLOWED_DOMAINS` (comma separated;
only addresses of these domains are ever typed into a page), `AGENT_SCHEDULER=0` (no background run).
`AGENT_ALLOWED_SENDERS` (comma separated) lists the forwarders whose mails become cases; mails from other
senders are ignored. With the agent on and an empty list nothing is processed.
The agent container needs `LLM_BASE_URL` (Open WebUI, `/api/chat/completions` is appended), `LLM_MODEL`,
`LLM_API_KEY`, optionally `MAX_STEPS` (20) and `CASE_TIMEOUT` (300). It must run in a network from which private
addresses are not reachable; the URL guard in `agent/guard.py` is only the first line of defence.
Tests: `pytest tests`, `cd agent && pytest tests` and `sh agent/tests/run-in-docker.sh` (browser tests).
+108 -6
View File
@@ -6,18 +6,24 @@ handled mails to the trash. Runs behind the reverse proxy (Caddy forward_auth /
mailbox credentials come from the environment. mailbox credentials come from the environment.
""" """
import datetime
import email import email
import imaplib import imaplib
import json
import logging import logging
import os import os
import re import re
import threading
from email.header import decode_header, make_header from email.header import decode_header, make_header
import httpx
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from dotenv import load_dotenv from dotenv import load_dotenv
from flask import Flask, abort, redirect, render_template, request, url_for from flask import Flask, abort, redirect, render_template, request, url_for
from cases import CaseStore
from oauth import MailboxOAuth, OAuthError, xoauth2_string from oauth import MailboxOAuth, OAuthError, xoauth2_string
from worker import AgentUnavailable, Worker, _tick, start_scheduler
load_dotenv() # optional .env for local runs load_dotenv() # optional .env for local runs
@@ -37,6 +43,12 @@ app = Flask(__name__, template_folder=template_dir)
# OAuth (SSO) for the collection mailbox if OIDC_* and TOKEN_KEY are set; otherwise EMAIL_PASSWORD # OAuth (SSO) for the collection mailbox if OIDC_* and TOKEN_KEY are set; otherwise EMAIL_PASSWORD
OAUTH = MailboxOAuth.from_env() OAUTH = MailboxOAuth.from_env()
# Unsubscribe agent: cases live in DATA_DIR/cases.db; without AGENT_URL they are only recorded and shown
DATA_DIR = os.environ.get("DATA_DIR", "")
AGENT_URL = os.environ.get("AGENT_URL", "").rstrip("/")
AGENT_CASE_TIMEOUT = float(os.environ.get("AGENT_CASE_TIMEOUT", "300"))
DONE_FOLDER = os.environ.get("DONE_FOLDER", "Erledigt")
UNSUBSCRIBE_PATTERN = re.compile( UNSUBSCRIBE_PATTERN = re.compile(
r"\babbestellen\b|\bunsubscribe\b|\bdeabonnieren\b|\babbestellung\b|\babmelden\b", re.IGNORECASE r"\babbestellen\b|\bunsubscribe\b|\bdeabonnieren\b|\babbestellung\b|\babmelden\b", re.IGNORECASE
) )
@@ -126,6 +138,59 @@ def move_emails_to_trash(uids: list[str]) -> int:
return len(uids) return len(uids)
def parse_mail(message) -> tuple[str, list[str], str]:
text = get_text_from_email(message)
return get_subject(message), find_unsubscribe_links(text) if text else [], text
def call_agent(url: str, address: str) -> dict:
"""Hand one link to the agent container. It gets the link and the address, nothing else."""
try:
# the agent takes up to AGENT_CASE_TIMEOUT per link; leave room for browser start and shutdown
r = httpx.post(AGENT_URL + "/run", json={"url": url, "address": address},
timeout=2 * AGENT_CASE_TIMEOUT + 60)
r.raise_for_status()
return r.json()
except (httpx.HTTPError, ValueError) as e:
raise AgentUnavailable(f"agent not reachable: {type(e).__name__}") from None
def _env_list(name: str) -> list[str]:
return [v.strip().lower() for v in os.environ.get(name, "").split(",") if v.strip()]
STORE = CaseStore(os.path.join(DATA_DIR, "cases.db")) if DATA_DIR else None
WORKER = None
if STORE:
STORE.recover()
WORKER = Worker(
STORE, lambda: connect(), parse_mail, call_agent if AGENT_URL else None,
inbox=EMAIL_FOLDER, done_folder=DONE_FOLDER,
retention_days=int(os.environ.get("RETENTION_DAYS", "30")),
max_cases=int(os.environ.get("AGENT_MAX_CASES", "10")),
allowed_domains=_env_list("AGENT_ALLOWED_DOMAINS"),
allowed_senders=_env_list("AGENT_ALLOWED_SENDERS"),
)
if os.environ.get("AGENT_SCHEDULER", "1") == "1":
start_scheduler(WORKER, 60 * int(os.environ.get("AGENT_INTERVAL_MINUTES", "10")))
def scan_inbox() -> dict:
"""{case key: uid} of the mails currently in the inbox; creates cases for new mails."""
mail = connect()
try:
return WORKER.scan(mail)[0]
finally:
mail.logout()
def start_run() -> threading.Thread:
"""One run in the background (it can take many minutes); _tick notes any error instead of raising."""
thread = threading.Thread(target=_tick, args=(WORKER,), name="unsubscribe-run-now", daemon=True)
thread.start()
return thread
@app.route("/", methods=["GET"]) @app.route("/", methods=["GET"])
def index(): def index():
return render_template( return render_template(
@@ -182,14 +247,51 @@ def oauth_error(e):
return redirect(url_for("index", error="Mailbox not connected or token expired – connect again")) return redirect(url_for("index", error="Mailbox not connected or token expired – connect again"))
STATE_LABELS = {"new": "new", "running": "running", "unsubscribed": "unsubscribed",
"not_possible": "not possible", "unclear": "unclear"}
@app.route("/result") @app.route("/result")
def result(): def result():
mail = connect() if WORKER is None:
try: mail = connect()
links_data = collect_links(mail) try:
finally: links_data = collect_links(mail)
mail.logout() finally:
return render_template("result.html", links_data=links_data) mail.logout()
return render_template("result.html", links_data=links_data)
uids = scan_inbox()
cases = [dict(c, uid=uids[c["key"]].decode() if c["key"] in uids else "", label=STATE_LABELS[c["state"]])
for c in STORE.all()]
last = json.loads(STORE.meta("last_run", "{}"))
at = STORE.meta("last_run_at")
return render_template(
"cases.html", cases=cases, last=last, agent=bool(AGENT_URL), paused=STORE.meta("paused") == "1",
last_at=datetime.datetime.fromtimestamp(int(at)).strftime("%Y-%m-%d %H:%M") if at else "never")
@app.route("/run-now", methods=["POST"])
def run_now():
if WORKER is None:
abort(404)
start_run()
return redirect(url_for("result"))
@app.route("/retry", methods=["POST"])
def retry():
if STORE is None:
abort(404)
STORE.retry(request.form.get("key", ""))
return redirect(url_for("result"))
@app.route("/pause", methods=["POST"])
def pause():
if STORE is None:
abort(404)
STORE.set_meta("paused", "1" if request.form.get("paused") == "1" else "0")
return redirect(url_for("result"))
@app.route("/move-to-trash", methods=["POST"]) @app.route("/move-to-trash", methods=["POST"])
+69
View File
@@ -0,0 +1,69 @@
<!DOCTYPE html>
<html>
<head>
<title>E-Mail Unsubscribe Cases</title>
<link rel="stylesheet" href="{{ url_for('static', filename='styles.css') }}">
</head>
<body>
<div class="container">
<h1>E-Mail Unsubscribe Cases</h1>
<p>
Last run: {{ last_at }}{% if last %} –
{{ last.get('new', 0) }} new, {{ last.get('unsubscribed', 0) }} unsubscribed,
{{ last.get('moved', 0) }} moved, {{ last.get('cleaned', 0) }} cleaned up.
{{ last.get('without_link', 0) }} mail(s) without unsubscribe link.
{{ last.get('ignored', 0) }} mail(s) from senders that are not allowed.{% endif %}
</p>
{% if last.get('error') %}<p class="error">Last run: {{ last['error'] }}</p>{% endif %}
{% if not agent %}<p class="error">The agent is switched off: cases are listed, nothing is unsubscribed automatically.</p>{% endif %}
<form action="{{ url_for('run_now') }}" method="post">
<button type="submit" class="btn-primary">Run now</button>
</form>
<form action="{{ url_for('pause') }}" method="post">
<input type="hidden" name="paused" value="{{ '0' if paused else '1' }}">
<button type="submit" class="btn-delete">{{ 'Resume agent' if paused else 'Pause agent' }}</button>
</form>
<table>
<tr>
<th>Subject</th>
<th>State</th>
<th>Reason</th>
<th>Unsubscribe Link</th>
<th>Actions</th>
</tr>
{% for case in cases %}
<tr>
<td>{{ case.subject }}</td>
<td>{{ case.label }}</td>
<td>
{{ case.reason }}
{% if case.evidence %}<br><small>“…{{ case.evidence }}” ({{ case.evidence_url }})</small>{% endif %}
</td>
<td>
{% for link in case.links %}
<a href="{{ link }}" target="_blank" rel="noopener noreferrer">{{ link[:30] + '...' }}</a><br>
{% endfor %}
</td>
<td>
{% if case.state in ('not_possible', 'unclear') %}
<form action="{{ url_for('retry') }}" method="post">
<input type="hidden" name="key" value="{{ case.key }}">
<button type="submit" class="btn-primary">Try again</button>
</form>
{% endif %}
{% if case.uid %}
<form action="{{ url_for('move_to_trash') }}" method="post">
<input type="hidden" name="email_id" value="{{ case.uid }}">
<button type="submit" class="btn-delete">Move to Trash</button>
</form>
{% endif %}
</td>
</tr>
{% endfor %}
</table>
<form action="/" method="get">
<button type="submit" class="btn-primary">Back to Home</button>
</form>
</div>
</body>
</html>
+126
View File
@@ -0,0 +1,126 @@
"""Case view and its actions (the store is real, the mailbox is faked)."""
import importlib
import json
import pytest
@pytest.fixture
def unsub(tmp_path, monkeypatch):
monkeypatch.setenv("DATA_DIR", str(tmp_path))
monkeypatch.setenv("AGENT_SCHEDULER", "0")
monkeypatch.setenv("AGENT_URL", "http://agent.test:8080")
monkeypatch.setenv("AGENT_ALLOWED_DOMAINS", "wagnersnetz.de, gmail.com")
monkeypatch.setenv("AGENT_ALLOWED_SENDERS", "Jens@wagnersnetz.de, philipp@wagnersnetz.de")
for name in ("OIDC_ISSUER", "OIDC_CLIENT_ID", "TOKEN_KEY"):
monkeypatch.delenv(name, raising=False)
import app
module = importlib.reload(app)
yield module
monkeypatch.delenv("DATA_DIR")
monkeypatch.delenv("AGENT_URL")
importlib.reload(app) # the other test modules expect the app without a store
def test_settings_from_environment(unsub):
assert unsub.STORE is not None and unsub.WORKER is not None
assert unsub.WORKER.allowed_domains == ("wagnersnetz.de", "gmail.com")
assert unsub.WORKER.allowed_senders == ("jens@wagnersnetz.de", "philipp@wagnersnetz.de")
assert unsub.WORKER.agent is unsub.call_agent
def test_result_shows_cases_with_state_reason_evidence_and_links(unsub, monkeypatch):
s = unsub.STORE
s.add("<ok@x>", "Fwd: Shop <b>News</b>", ["https://shop.example/u/1"], "p@wagnersnetz.de")
s.claim("<ok@x>")
s.finish("<ok@x>", "unsubscribed", "page confirms", "Sie wurden abgemeldet", "https://shop.example/done")
s.add("<no@x>", "Fwd: Bank", ["https://bank.example/u"], "")
s.claim("<no@x>")
s.finish("<no@x>", "not_possible", "login")
s.set_meta("last_run", json.dumps({"new": 2, "handled": 2, "unsubscribed": 1, "moved": 1, "cleaned": 0,
"without_link": 3, "ignored": 2, "error": ""}))
s.set_meta("last_run_at", "1800000000")
monkeypatch.setattr(unsub, "scan_inbox", lambda: {"<no@x>": b"42"})
html = unsub.app.test_client().get("/result").get_data(as_text=True)
assert "Fwd: Shop &lt;b&gt;News&lt;/b&gt;" in html and "<b>News</b>" not in html
assert "unsubscribed" in html and "Sie wurden abgemeldet" in html and "https://shop.example/done" in html
assert "not possible" in html and "login" in html and 'href="https://bank.example/u"' in html
assert 'name="email_id" value="42"' in html # trash button only for mails still in the inbox
assert html.count('action="/retry"') == 1 # only the failed case can be retried
assert "3 mail(s) without unsubscribe link" in html
assert "2 mail(s) from senders that are not allowed" in html
def test_retry_resets_the_case(unsub):
s = unsub.STORE
s.add("<no@x>", "s", ["https://bank.example/u"], "")
s.claim("<no@x>")
s.finish("<no@x>", "unclear", "x")
r = unsub.app.test_client().post("/retry", data={"key": "<no@x>"})
assert r.status_code == 302 and s.get("<no@x>")["state"] == "new" and s.get("<no@x>")["attempts"] == 0
def test_pause_and_resume(unsub):
c = unsub.app.test_client()
c.post("/pause", data={"paused": "1"})
assert unsub.STORE.meta("paused") == "1"
c.post("/pause", data={"paused": "0"})
assert unsub.STORE.meta("paused") == "0"
def test_run_now_starts_a_background_run_and_returns_at_once(unsub, monkeypatch):
calls, threads = [], []
real_start_run = unsub.start_run
def keep_thread():
threads.append(real_start_run())
return threads[-1]
monkeypatch.setattr(unsub, "start_run", keep_thread)
monkeypatch.setattr(unsub.WORKER, "run_once", lambda: calls.append(1) or {"error": ""})
assert unsub.app.test_client().post("/run-now").status_code == 302
threads[0].join(timeout=5)
assert calls == [1]
def boom():
raise RuntimeError("secret mail content")
monkeypatch.setattr(unsub.WORKER, "run_once", boom)
assert unsub.app.test_client().post("/run-now").status_code == 302
threads[1].join(timeout=5)
assert json.loads(unsub.STORE.meta("last_run"))["error"] == "RuntimeError"
def test_call_agent_maps_failures_to_unavailable(unsub, monkeypatch):
import httpx
def boom(*a, **kw):
raise httpx.ConnectError("refused")
monkeypatch.setattr(unsub.httpx, "post", boom)
with pytest.raises(unsub.AgentUnavailable):
unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de")
monkeypatch.setattr(unsub.httpx, "post", lambda *a, **kw: httpx.Response(500, text="x", request=httpx.Request("POST", a[0])))
with pytest.raises(unsub.AgentUnavailable):
unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de")
def test_call_agent_sends_only_url_and_address(unsub, monkeypatch):
import httpx
seen = {}
def post(url, json=None, timeout=None):
seen.update(url=url, json=json, timeout=timeout)
return httpx.Response(200, json={"result": "unclear", "reason": "x"}, request=httpx.Request("POST", url))
monkeypatch.setattr(unsub.httpx, "post", post)
assert unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de")["result"] == "unclear"
assert seen["url"] == "http://agent.test:8080/run"
assert seen["json"] == {"url": "https://shop.example/u", "address": "p@wagnersnetz.de"}
assert seen["timeout"] > 600 # two links, 5 minutes each
def test_without_data_dir_the_old_list_is_shown():
import app
assert app.STORE is None and app.WORKER is None