Fall-Ansicht, Zeitlauf und Übergabe an den Agenten verdrahtet

This commit is contained in:
Rsclub2_2 committed 2026-11-15 00:47:15 +01:00
1 parent 249293f382
commit 6df39d6b93
6 files changed
+330 -8

No files matched your search

+7 -1
View File
@@ -1,3 +1,9 @@
EMAIL_USERNAME=example@example.com
EMAIL_PASSWORD=example
EMAIL_SERVER=example.com
EMAIL_SERVER=example.com
# Unsubscribe agent (optional)
# DATA_DIR=/data
# AGENT_URL=http://unsubscriber-agent:8080
# AGENT_ALLOWED_DOMAINS=example.org,gmail.com
# AGENT_ALLOWED_SENDERS=jens@example.org
+1 -1
View File
@@ -5,7 +5,7 @@ WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY app.py oauth.py index.html result.html ./
COPY app.py oauth.py cases.py forwarded.py worker.py index.html result.html cases.html ./
COPY static ./static
RUN useradd -m -u 1000 app
+19
View File
@@ -34,3 +34,22 @@ Sammelpostfachs am SSO anmelden; das Refresh-Token liegt verschlüsselt in `DATA
Der Container startet gunicorn auf Port 5000 (kein Flask-Debugger) und gehört hinter einen
Reverse Proxy mit Anmeldung (bei Wagnersnetz Caddy `forward_auth` über Authelia).
Tests: `pytest tests`.
### Unsubscribe agent
With `DATA_DIR` set the app keeps one case per mail in `DATA_DIR/cases.db` and shows them under `/result`.
With `AGENT_URL` (the agent container from `agent/`, `http://<name>:8080`) a background run every
`AGENT_INTERVAL_MINUTES` (10) hands each new case to the agent: it opens the unsubscribe link in a sandboxed
browser, first checks for a confirmation without a model and otherwise lets a model operate the page with five
actions (click, select, check, insert the address, done). Successes are moved to the IMAP folder `DONE_FOLDER`
(`Erledigt`) and deleted after `RETENTION_DAYS` (30); everything else stays in the inbox with a reason.
"Run now" starts a run in the background and returns at once; the result page shows its outcome afterwards.
Further settings: `AGENT_CASE_TIMEOUT` (300), `AGENT_MAX_CASES` (10), `AGENT_ALLOWED_DOMAINS` (comma separated;
only addresses of these domains are ever typed into a page), `AGENT_SCHEDULER=0` (no background run).
`AGENT_ALLOWED_SENDERS` (comma separated) lists the forwarders whose mails become cases; mails from other
senders are ignored. With the agent on and an empty list nothing is processed.
The agent container needs `LLM_BASE_URL` (Open WebUI, `/api/chat/completions` is appended), `LLM_MODEL`,
`LLM_API_KEY`, optionally `MAX_STEPS` (20) and `CASE_TIMEOUT` (300). It must run in a network from which private
addresses are not reachable; the URL guard in `agent/guard.py` is only the first line of defence.
Tests: `pytest tests`, `cd agent && pytest tests` and `sh agent/tests/run-in-docker.sh` (browser tests).
+108 -6
View File
@@ -6,18 +6,24 @@ handled mails to the trash. Runs behind the reverse proxy (Caddy forward_auth /
mailbox credentials come from the environment.
"""
import datetime
import email
import imaplib
import json
import logging
import os
import re
import threading
from email.header import decode_header, make_header
import httpx
from bs4 import BeautifulSoup
from dotenv import load_dotenv
from flask import Flask, abort, redirect, render_template, request, url_for
from cases import CaseStore
from oauth import MailboxOAuth, OAuthError, xoauth2_string
from worker import AgentUnavailable, Worker, _tick, start_scheduler
load_dotenv() # optional .env for local runs
@@ -37,6 +43,12 @@ app = Flask(__name__, template_folder=template_dir)
# OAuth (SSO) for the collection mailbox if OIDC_* and TOKEN_KEY are set; otherwise EMAIL_PASSWORD
OAUTH = MailboxOAuth.from_env()
# Unsubscribe agent: cases live in DATA_DIR/cases.db; without AGENT_URL they are only recorded and shown
DATA_DIR = os.environ.get("DATA_DIR", "")
AGENT_URL = os.environ.get("AGENT_URL", "").rstrip("/")
AGENT_CASE_TIMEOUT = float(os.environ.get("AGENT_CASE_TIMEOUT", "300"))
DONE_FOLDER = os.environ.get("DONE_FOLDER", "Erledigt")
UNSUBSCRIBE_PATTERN = re.compile(
r"\babbestellen\b|\bunsubscribe\b|\bdeabonnieren\b|\babbestellung\b|\babmelden\b", re.IGNORECASE
)
@@ -126,6 +138,59 @@ def move_emails_to_trash(uids: list[str]) -> int:
return len(uids)
def parse_mail(message) -> tuple[str, list[str], str]:
text = get_text_from_email(message)
return get_subject(message), find_unsubscribe_links(text) if text else [], text
def call_agent(url: str, address: str) -> dict:
"""Hand one link to the agent container. It gets the link and the address, nothing else."""
try:
# the agent takes up to AGENT_CASE_TIMEOUT per link; leave room for browser start and shutdown
r = httpx.post(AGENT_URL + "/run", json={"url": url, "address": address},
timeout=2 * AGENT_CASE_TIMEOUT + 60)
r.raise_for_status()
return r.json()
except (httpx.HTTPError, ValueError) as e:
raise AgentUnavailable(f"agent not reachable: {type(e).__name__}") from None
def _env_list(name: str) -> list[str]:
return [v.strip().lower() for v in os.environ.get(name, "").split(",") if v.strip()]
STORE = CaseStore(os.path.join(DATA_DIR, "cases.db")) if DATA_DIR else None
WORKER = None
if STORE:
STORE.recover()
WORKER = Worker(
STORE, lambda: connect(), parse_mail, call_agent if AGENT_URL else None,
inbox=EMAIL_FOLDER, done_folder=DONE_FOLDER,
retention_days=int(os.environ.get("RETENTION_DAYS", "30")),
max_cases=int(os.environ.get("AGENT_MAX_CASES", "10")),
allowed_domains=_env_list("AGENT_ALLOWED_DOMAINS"),
allowed_senders=_env_list("AGENT_ALLOWED_SENDERS"),
)
if os.environ.get("AGENT_SCHEDULER", "1") == "1":
start_scheduler(WORKER, 60 * int(os.environ.get("AGENT_INTERVAL_MINUTES", "10")))
def scan_inbox() -> dict:
"""{case key: uid} of the mails currently in the inbox; creates cases for new mails."""
mail = connect()
try:
return WORKER.scan(mail)[0]
finally:
mail.logout()
def start_run() -> threading.Thread:
"""One run in the background (it can take many minutes); _tick notes any error instead of raising."""
thread = threading.Thread(target=_tick, args=(WORKER,), name="unsubscribe-run-now", daemon=True)
thread.start()
return thread
@app.route("/", methods=["GET"])
def index():
return render_template(
@@ -182,14 +247,51 @@ def oauth_error(e):
return redirect(url_for("index", error="Mailbox not connected or token expired – connect again"))
STATE_LABELS = {"new": "new", "running": "running", "unsubscribed": "unsubscribed",
"not_possible": "not possible", "unclear": "unclear"}
@app.route("/result")
def result():
mail = connect()
try:
links_data = collect_links(mail)
finally:
mail.logout()
return render_template("result.html", links_data=links_data)
if WORKER is None:
mail = connect()
try:
links_data = collect_links(mail)
finally:
mail.logout()
return render_template("result.html", links_data=links_data)
uids = scan_inbox()
cases = [dict(c, uid=uids[c["key"]].decode() if c["key"] in uids else "", label=STATE_LABELS[c["state"]])
for c in STORE.all()]
last = json.loads(STORE.meta("last_run", "{}"))
at = STORE.meta("last_run_at")
return render_template(
"cases.html", cases=cases, last=last, agent=bool(AGENT_URL), paused=STORE.meta("paused") == "1",
last_at=datetime.datetime.fromtimestamp(int(at)).strftime("%Y-%m-%d %H:%M") if at else "never")
@app.route("/run-now", methods=["POST"])
def run_now():
if WORKER is None:
abort(404)
start_run()
return redirect(url_for("result"))
@app.route("/retry", methods=["POST"])
def retry():
if STORE is None:
abort(404)
STORE.retry(request.form.get("key", ""))
return redirect(url_for("result"))
@app.route("/pause", methods=["POST"])
def pause():
if STORE is None:
abort(404)
STORE.set_meta("paused", "1" if request.form.get("paused") == "1" else "0")
return redirect(url_for("result"))
@app.route("/move-to-trash", methods=["POST"])
+69
View File
@@ -0,0 +1,69 @@
<!DOCTYPE html>
<html>
<head>
<title>E-Mail Unsubscribe Cases</title>
<link rel="stylesheet" href="{{ url_for('static', filename='styles.css') }}">
</head>
<body>
<div class="container">
<h1>E-Mail Unsubscribe Cases</h1>
<p>
Last run: {{ last_at }}{% if last %} –
{{ last.get('new', 0) }} new, {{ last.get('unsubscribed', 0) }} unsubscribed,
{{ last.get('moved', 0) }} moved, {{ last.get('cleaned', 0) }} cleaned up.
{{ last.get('without_link', 0) }} mail(s) without unsubscribe link.
{{ last.get('ignored', 0) }} mail(s) from senders that are not allowed.{% endif %}
</p>
{% if last.get('error') %}<p class="error">Last run: {{ last['error'] }}</p>{% endif %}
{% if not agent %}<p class="error">The agent is switched off: cases are listed, nothing is unsubscribed automatically.</p>{% endif %}
<form action="{{ url_for('run_now') }}" method="post">
<button type="submit" class="btn-primary">Run now</button>
</form>
<form action="{{ url_for('pause') }}" method="post">
<input type="hidden" name="paused" value="{{ '0' if paused else '1' }}">
<button type="submit" class="btn-delete">{{ 'Resume agent' if paused else 'Pause agent' }}</button>
</form>
<table>
<tr>
<th>Subject</th>
<th>State</th>
<th>Reason</th>
<th>Unsubscribe Link</th>
<th>Actions</th>
</tr>
{% for case in cases %}
<tr>
<td>{{ case.subject }}</td>
<td>{{ case.label }}</td>
<td>
{{ case.reason }}
{% if case.evidence %}<br><small>“…{{ case.evidence }}” ({{ case.evidence_url }})</small>{% endif %}
</td>
<td>
{% for link in case.links %}
<a href="{{ link }}" target="_blank" rel="noopener noreferrer">{{ link[:30] + '...' }}</a><br>
{% endfor %}
</td>
<td>
{% if case.state in ('not_possible', 'unclear') %}
<form action="{{ url_for('retry') }}" method="post">
<input type="hidden" name="key" value="{{ case.key }}">
<button type="submit" class="btn-primary">Try again</button>
</form>
{% endif %}
{% if case.uid %}
<form action="{{ url_for('move_to_trash') }}" method="post">
<input type="hidden" name="email_id" value="{{ case.uid }}">
<button type="submit" class="btn-delete">Move to Trash</button>
</form>
{% endif %}
</td>
</tr>
{% endfor %}
</table>
<form action="/" method="get">
<button type="submit" class="btn-primary">Back to Home</button>
</form>
</div>
</body>
</html>
+126
View File
@@ -0,0 +1,126 @@
"""Case view and its actions (the store is real, the mailbox is faked)."""
import importlib
import json
import pytest
@pytest.fixture
def unsub(tmp_path, monkeypatch):
monkeypatch.setenv("DATA_DIR", str(tmp_path))
monkeypatch.setenv("AGENT_SCHEDULER", "0")
monkeypatch.setenv("AGENT_URL", "http://agent.test:8080")
monkeypatch.setenv("AGENT_ALLOWED_DOMAINS", "wagnersnetz.de, gmail.com")
monkeypatch.setenv("AGENT_ALLOWED_SENDERS", "Jens@wagnersnetz.de, philipp@wagnersnetz.de")
for name in ("OIDC_ISSUER", "OIDC_CLIENT_ID", "TOKEN_KEY"):
monkeypatch.delenv(name, raising=False)
import app
module = importlib.reload(app)
yield module
monkeypatch.delenv("DATA_DIR")
monkeypatch.delenv("AGENT_URL")
importlib.reload(app) # the other test modules expect the app without a store
def test_settings_from_environment(unsub):
assert unsub.STORE is not None and unsub.WORKER is not None
assert unsub.WORKER.allowed_domains == ("wagnersnetz.de", "gmail.com")
assert unsub.WORKER.allowed_senders == ("jens@wagnersnetz.de", "philipp@wagnersnetz.de")
assert unsub.WORKER.agent is unsub.call_agent
def test_result_shows_cases_with_state_reason_evidence_and_links(unsub, monkeypatch):
s = unsub.STORE
s.add("<ok@x>", "Fwd: Shop <b>News</b>", ["https://shop.example/u/1"], "p@wagnersnetz.de")
s.claim("<ok@x>")
s.finish("<ok@x>", "unsubscribed", "page confirms", "Sie wurden abgemeldet", "https://shop.example/done")
s.add("<no@x>", "Fwd: Bank", ["https://bank.example/u"], "")
s.claim("<no@x>")
s.finish("<no@x>", "not_possible", "login")
s.set_meta("last_run", json.dumps({"new": 2, "handled": 2, "unsubscribed": 1, "moved": 1, "cleaned": 0,
"without_link": 3, "ignored": 2, "error": ""}))
s.set_meta("last_run_at", "1800000000")
monkeypatch.setattr(unsub, "scan_inbox", lambda: {"<no@x>": b"42"})
html = unsub.app.test_client().get("/result").get_data(as_text=True)
assert "Fwd: Shop &lt;b&gt;News&lt;/b&gt;" in html and "<b>News</b>" not in html
assert "unsubscribed" in html and "Sie wurden abgemeldet" in html and "https://shop.example/done" in html
assert "not possible" in html and "login" in html and 'href="https://bank.example/u"' in html
assert 'name="email_id" value="42"' in html # trash button only for mails still in the inbox
assert html.count('action="/retry"') == 1 # only the failed case can be retried
assert "3 mail(s) without unsubscribe link" in html
assert "2 mail(s) from senders that are not allowed" in html
def test_retry_resets_the_case(unsub):
s = unsub.STORE
s.add("<no@x>", "s", ["https://bank.example/u"], "")
s.claim("<no@x>")
s.finish("<no@x>", "unclear", "x")
r = unsub.app.test_client().post("/retry", data={"key": "<no@x>"})
assert r.status_code == 302 and s.get("<no@x>")["state"] == "new" and s.get("<no@x>")["attempts"] == 0
def test_pause_and_resume(unsub):
c = unsub.app.test_client()
c.post("/pause", data={"paused": "1"})
assert unsub.STORE.meta("paused") == "1"
c.post("/pause", data={"paused": "0"})
assert unsub.STORE.meta("paused") == "0"
def test_run_now_starts_a_background_run_and_returns_at_once(unsub, monkeypatch):
calls, threads = [], []
real_start_run = unsub.start_run
def keep_thread():
threads.append(real_start_run())
return threads[-1]
monkeypatch.setattr(unsub, "start_run", keep_thread)
monkeypatch.setattr(unsub.WORKER, "run_once", lambda: calls.append(1) or {"error": ""})
assert unsub.app.test_client().post("/run-now").status_code == 302
threads[0].join(timeout=5)
assert calls == [1]
def boom():
raise RuntimeError("secret mail content")
monkeypatch.setattr(unsub.WORKER, "run_once", boom)
assert unsub.app.test_client().post("/run-now").status_code == 302
threads[1].join(timeout=5)
assert json.loads(unsub.STORE.meta("last_run"))["error"] == "RuntimeError"
def test_call_agent_maps_failures_to_unavailable(unsub, monkeypatch):
import httpx
def boom(*a, **kw):
raise httpx.ConnectError("refused")
monkeypatch.setattr(unsub.httpx, "post", boom)
with pytest.raises(unsub.AgentUnavailable):
unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de")
monkeypatch.setattr(unsub.httpx, "post", lambda *a, **kw: httpx.Response(500, text="x", request=httpx.Request("POST", a[0])))
with pytest.raises(unsub.AgentUnavailable):
unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de")
def test_call_agent_sends_only_url_and_address(unsub, monkeypatch):
import httpx
seen = {}
def post(url, json=None, timeout=None):
seen.update(url=url, json=json, timeout=timeout)
return httpx.Response(200, json={"result": "unclear", "reason": "x"}, request=httpx.Request("POST", url))
monkeypatch.setattr(unsub.httpx, "post", post)
assert unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de")["result"] == "unclear"
assert seen["url"] == "http://agent.test:8080/run"
assert seen["json"] == {"url": "https://shop.example/u", "address": "p@wagnersnetz.de"}
assert seen["timeout"] > 600 # two links, 5 minutes each
def test_without_data_dir_the_old_list_is_shown():
import app
assert app.STORE is None and app.WORKER is None