Abmelde-Agent: Fälle, Zeitlauf und Agent-Dienst (Vikunja 97) #3
No files matched your search
+7
-1
@@ -1,3 +1,9 @@
|
||||
EMAIL_USERNAME=example@example.com
|
||||
EMAIL_PASSWORD=example
|
||||
EMAIL_SERVER=example.com
|
||||
EMAIL_SERVER=example.com
|
||||
|
||||
# Unsubscribe agent (optional)
|
||||
# DATA_DIR=/data
|
||||
# AGENT_URL=http://unsubscriber-agent:8080
|
||||
# AGENT_ALLOWED_DOMAINS=example.org,gmail.com
|
||||
# AGENT_ALLOWED_SENDERS=jens@example.org
|
||||
+1
-1
@@ -5,7 +5,7 @@ WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY app.py oauth.py index.html result.html ./
|
||||
COPY app.py oauth.py cases.py forwarded.py worker.py index.html result.html cases.html ./
|
||||
COPY static ./static
|
||||
|
||||
RUN useradd -m -u 1000 app
|
||||
|
||||
@@ -34,3 +34,22 @@ Sammelpostfachs am SSO anmelden; das Refresh-Token liegt verschlüsselt in `DATA
|
||||
Der Container startet gunicorn auf Port 5000 (kein Flask-Debugger) und gehört hinter einen
|
||||
Reverse Proxy mit Anmeldung (bei Wagnersnetz Caddy `forward_auth` über Authelia).
|
||||
Tests: `pytest tests`.
|
||||
|
||||
### Unsubscribe agent
|
||||
|
||||
With `DATA_DIR` set the app keeps one case per mail in `DATA_DIR/cases.db` and shows them under `/result`.
|
||||
With `AGENT_URL` (the agent container from `agent/`, `http://<name>:8080`) a background run every
|
||||
`AGENT_INTERVAL_MINUTES` (10) hands each new case to the agent: it opens the unsubscribe link in a sandboxed
|
||||
browser, first checks for a confirmation without a model and otherwise lets a model operate the page with five
|
||||
actions (click, select, check, insert the address, done). Successes are moved to the IMAP folder `DONE_FOLDER`
|
||||
(`Erledigt`) and deleted after `RETENTION_DAYS` (30); everything else stays in the inbox with a reason.
|
||||
"Run now" starts a run in the background and returns at once; the result page shows its outcome afterwards.
|
||||
Further settings: `AGENT_CASE_TIMEOUT` (300), `AGENT_MAX_CASES` (10), `AGENT_ALLOWED_DOMAINS` (comma separated;
|
||||
only addresses of these domains are ever typed into a page), `AGENT_SCHEDULER=0` (no background run).
|
||||
`AGENT_ALLOWED_SENDERS` (comma separated) lists the forwarders whose mails become cases; mails from other
|
||||
senders are ignored. With the agent on and an empty list nothing is processed.
|
||||
|
||||
The agent container needs `LLM_BASE_URL` (Open WebUI, `/api/chat/completions` is appended), `LLM_MODEL`,
|
||||
`LLM_API_KEY`, optionally `MAX_STEPS` (20) and `CASE_TIMEOUT` (300). It must run in a network from which private
|
||||
addresses are not reachable; the URL guard in `agent/guard.py` is only the first line of defence.
|
||||
Tests: `pytest tests`, `cd agent && pytest tests` and `sh agent/tests/run-in-docker.sh` (browser tests).
|
||||
@@ -6,18 +6,24 @@ handled mails to the trash. Runs behind the reverse proxy (Caddy forward_auth /
|
||||
mailbox credentials come from the environment.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import email
|
||||
import imaplib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
from email.header import decode_header, make_header
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
from dotenv import load_dotenv
|
||||
from flask import Flask, abort, redirect, render_template, request, url_for
|
||||
|
||||
from cases import CaseStore
|
||||
from oauth import MailboxOAuth, OAuthError, xoauth2_string
|
||||
from worker import AgentUnavailable, Worker, _tick, start_scheduler
|
||||
|
||||
load_dotenv() # optional .env for local runs
|
||||
|
||||
@@ -37,6 +43,12 @@ app = Flask(__name__, template_folder=template_dir)
|
||||
# OAuth (SSO) for the collection mailbox if OIDC_* and TOKEN_KEY are set; otherwise EMAIL_PASSWORD
|
||||
OAUTH = MailboxOAuth.from_env()
|
||||
|
||||
# Unsubscribe agent: cases live in DATA_DIR/cases.db; without AGENT_URL they are only recorded and shown
|
||||
DATA_DIR = os.environ.get("DATA_DIR", "")
|
||||
AGENT_URL = os.environ.get("AGENT_URL", "").rstrip("/")
|
||||
AGENT_CASE_TIMEOUT = float(os.environ.get("AGENT_CASE_TIMEOUT", "300"))
|
||||
DONE_FOLDER = os.environ.get("DONE_FOLDER", "Erledigt")
|
||||
|
||||
UNSUBSCRIBE_PATTERN = re.compile(
|
||||
r"\babbestellen\b|\bunsubscribe\b|\bdeabonnieren\b|\babbestellung\b|\babmelden\b", re.IGNORECASE
|
||||
)
|
||||
@@ -126,6 +138,59 @@ def move_emails_to_trash(uids: list[str]) -> int:
|
||||
return len(uids)
|
||||
|
||||
|
||||
def parse_mail(message) -> tuple[str, list[str], str]:
|
||||
text = get_text_from_email(message)
|
||||
return get_subject(message), find_unsubscribe_links(text) if text else [], text
|
||||
|
||||
|
||||
def call_agent(url: str, address: str) -> dict:
|
||||
"""Hand one link to the agent container. It gets the link and the address, nothing else."""
|
||||
try:
|
||||
# the agent takes up to AGENT_CASE_TIMEOUT per link; leave room for browser start and shutdown
|
||||
r = httpx.post(AGENT_URL + "/run", json={"url": url, "address": address},
|
||||
timeout=2 * AGENT_CASE_TIMEOUT + 60)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
except (httpx.HTTPError, ValueError) as e:
|
||||
raise AgentUnavailable(f"agent not reachable: {type(e).__name__}") from None
|
||||
|
||||
|
||||
def _env_list(name: str) -> list[str]:
|
||||
return [v.strip().lower() for v in os.environ.get(name, "").split(",") if v.strip()]
|
||||
|
||||
|
||||
STORE = CaseStore(os.path.join(DATA_DIR, "cases.db")) if DATA_DIR else None
|
||||
WORKER = None
|
||||
if STORE:
|
||||
STORE.recover()
|
||||
WORKER = Worker(
|
||||
STORE, lambda: connect(), parse_mail, call_agent if AGENT_URL else None,
|
||||
inbox=EMAIL_FOLDER, done_folder=DONE_FOLDER,
|
||||
retention_days=int(os.environ.get("RETENTION_DAYS", "30")),
|
||||
max_cases=int(os.environ.get("AGENT_MAX_CASES", "10")),
|
||||
allowed_domains=_env_list("AGENT_ALLOWED_DOMAINS"),
|
||||
allowed_senders=_env_list("AGENT_ALLOWED_SENDERS"),
|
||||
)
|
||||
if os.environ.get("AGENT_SCHEDULER", "1") == "1":
|
||||
start_scheduler(WORKER, 60 * int(os.environ.get("AGENT_INTERVAL_MINUTES", "10")))
|
||||
|
||||
|
||||
def scan_inbox() -> dict:
|
||||
"""{case key: uid} of the mails currently in the inbox; creates cases for new mails."""
|
||||
mail = connect()
|
||||
try:
|
||||
return WORKER.scan(mail)[0]
|
||||
finally:
|
||||
mail.logout()
|
||||
|
||||
|
||||
def start_run() -> threading.Thread:
|
||||
"""One run in the background (it can take many minutes); _tick notes any error instead of raising."""
|
||||
thread = threading.Thread(target=_tick, args=(WORKER,), name="unsubscribe-run-now", daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
|
||||
|
||||
@app.route("/", methods=["GET"])
|
||||
def index():
|
||||
return render_template(
|
||||
@@ -182,14 +247,51 @@ def oauth_error(e):
|
||||
return redirect(url_for("index", error="Mailbox not connected or token expired – connect again"))
|
||||
|
||||
|
||||
STATE_LABELS = {"new": "new", "running": "running", "unsubscribed": "unsubscribed",
|
||||
"not_possible": "not possible", "unclear": "unclear"}
|
||||
|
||||
|
||||
@app.route("/result")
|
||||
def result():
|
||||
mail = connect()
|
||||
try:
|
||||
links_data = collect_links(mail)
|
||||
finally:
|
||||
mail.logout()
|
||||
return render_template("result.html", links_data=links_data)
|
||||
if WORKER is None:
|
||||
mail = connect()
|
||||
try:
|
||||
links_data = collect_links(mail)
|
||||
finally:
|
||||
mail.logout()
|
||||
return render_template("result.html", links_data=links_data)
|
||||
uids = scan_inbox()
|
||||
cases = [dict(c, uid=uids[c["key"]].decode() if c["key"] in uids else "", label=STATE_LABELS[c["state"]])
|
||||
for c in STORE.all()]
|
||||
last = json.loads(STORE.meta("last_run", "{}"))
|
||||
at = STORE.meta("last_run_at")
|
||||
return render_template(
|
||||
"cases.html", cases=cases, last=last, agent=bool(AGENT_URL), paused=STORE.meta("paused") == "1",
|
||||
last_at=datetime.datetime.fromtimestamp(int(at)).strftime("%Y-%m-%d %H:%M") if at else "never")
|
||||
|
||||
|
||||
@app.route("/run-now", methods=["POST"])
|
||||
def run_now():
|
||||
if WORKER is None:
|
||||
abort(404)
|
||||
start_run()
|
||||
return redirect(url_for("result"))
|
||||
|
||||
|
||||
@app.route("/retry", methods=["POST"])
|
||||
def retry():
|
||||
if STORE is None:
|
||||
abort(404)
|
||||
STORE.retry(request.form.get("key", ""))
|
||||
return redirect(url_for("result"))
|
||||
|
||||
|
||||
@app.route("/pause", methods=["POST"])
|
||||
def pause():
|
||||
if STORE is None:
|
||||
abort(404)
|
||||
STORE.set_meta("paused", "1" if request.form.get("paused") == "1" else "0")
|
||||
return redirect(url_for("result"))
|
||||
|
||||
|
||||
@app.route("/move-to-trash", methods=["POST"])
|
||||
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>E-Mail Unsubscribe Cases</title>
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='styles.css') }}">
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>E-Mail Unsubscribe Cases</h1>
|
||||
<p>
|
||||
Last run: {{ last_at }}{% if last %} –
|
||||
{{ last.get('new', 0) }} new, {{ last.get('unsubscribed', 0) }} unsubscribed,
|
||||
{{ last.get('moved', 0) }} moved, {{ last.get('cleaned', 0) }} cleaned up.
|
||||
{{ last.get('without_link', 0) }} mail(s) without unsubscribe link.
|
||||
{{ last.get('ignored', 0) }} mail(s) from senders that are not allowed.{% endif %}
|
||||
</p>
|
||||
{% if last.get('error') %}<p class="error">Last run: {{ last['error'] }}</p>{% endif %}
|
||||
{% if not agent %}<p class="error">The agent is switched off: cases are listed, nothing is unsubscribed automatically.</p>{% endif %}
|
||||
<form action="{{ url_for('run_now') }}" method="post">
|
||||
<button type="submit" class="btn-primary">Run now</button>
|
||||
</form>
|
||||
<form action="{{ url_for('pause') }}" method="post">
|
||||
<input type="hidden" name="paused" value="{{ '0' if paused else '1' }}">
|
||||
<button type="submit" class="btn-delete">{{ 'Resume agent' if paused else 'Pause agent' }}</button>
|
||||
</form>
|
||||
<table>
|
||||
<tr>
|
||||
<th>Subject</th>
|
||||
<th>State</th>
|
||||
<th>Reason</th>
|
||||
<th>Unsubscribe Link</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
{% for case in cases %}
|
||||
<tr>
|
||||
<td>{{ case.subject }}</td>
|
||||
<td>{{ case.label }}</td>
|
||||
<td>
|
||||
{{ case.reason }}
|
||||
{% if case.evidence %}<br><small>“…{{ case.evidence }}” ({{ case.evidence_url }})</small>{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% for link in case.links %}
|
||||
<a href="{{ link }}" target="_blank" rel="noopener noreferrer">{{ link[:30] + '...' }}</a><br>
|
||||
{% endfor %}
|
||||
</td>
|
||||
<td>
|
||||
{% if case.state in ('not_possible', 'unclear') %}
|
||||
<form action="{{ url_for('retry') }}" method="post">
|
||||
<input type="hidden" name="key" value="{{ case.key }}">
|
||||
<button type="submit" class="btn-primary">Try again</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
{% if case.uid %}
|
||||
<form action="{{ url_for('move_to_trash') }}" method="post">
|
||||
<input type="hidden" name="email_id" value="{{ case.uid }}">
|
||||
<button type="submit" class="btn-delete">Move to Trash</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
<form action="/" method="get">
|
||||
<button type="submit" class="btn-primary">Back to Home</button>
|
||||
</form>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Case view and its actions (the store is real, the mailbox is faked)."""
|
||||
|
||||
import importlib
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsub(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATA_DIR", str(tmp_path))
|
||||
monkeypatch.setenv("AGENT_SCHEDULER", "0")
|
||||
monkeypatch.setenv("AGENT_URL", "http://agent.test:8080")
|
||||
monkeypatch.setenv("AGENT_ALLOWED_DOMAINS", "wagnersnetz.de, gmail.com")
|
||||
monkeypatch.setenv("AGENT_ALLOWED_SENDERS", "Jens@wagnersnetz.de, philipp@wagnersnetz.de")
|
||||
for name in ("OIDC_ISSUER", "OIDC_CLIENT_ID", "TOKEN_KEY"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
import app
|
||||
module = importlib.reload(app)
|
||||
yield module
|
||||
monkeypatch.delenv("DATA_DIR")
|
||||
monkeypatch.delenv("AGENT_URL")
|
||||
importlib.reload(app) # the other test modules expect the app without a store
|
||||
|
||||
|
||||
def test_settings_from_environment(unsub):
|
||||
assert unsub.STORE is not None and unsub.WORKER is not None
|
||||
assert unsub.WORKER.allowed_domains == ("wagnersnetz.de", "gmail.com")
|
||||
assert unsub.WORKER.allowed_senders == ("jens@wagnersnetz.de", "philipp@wagnersnetz.de")
|
||||
assert unsub.WORKER.agent is unsub.call_agent
|
||||
|
||||
|
||||
def test_result_shows_cases_with_state_reason_evidence_and_links(unsub, monkeypatch):
|
||||
s = unsub.STORE
|
||||
s.add("<ok@x>", "Fwd: Shop <b>News</b>", ["https://shop.example/u/1"], "p@wagnersnetz.de")
|
||||
s.claim("<ok@x>")
|
||||
s.finish("<ok@x>", "unsubscribed", "page confirms", "Sie wurden abgemeldet", "https://shop.example/done")
|
||||
s.add("<no@x>", "Fwd: Bank", ["https://bank.example/u"], "")
|
||||
s.claim("<no@x>")
|
||||
s.finish("<no@x>", "not_possible", "login")
|
||||
s.set_meta("last_run", json.dumps({"new": 2, "handled": 2, "unsubscribed": 1, "moved": 1, "cleaned": 0,
|
||||
"without_link": 3, "ignored": 2, "error": ""}))
|
||||
s.set_meta("last_run_at", "1800000000")
|
||||
monkeypatch.setattr(unsub, "scan_inbox", lambda: {"<no@x>": b"42"})
|
||||
html = unsub.app.test_client().get("/result").get_data(as_text=True)
|
||||
assert "Fwd: Shop <b>News</b>" in html and "<b>News</b>" not in html
|
||||
assert "unsubscribed" in html and "Sie wurden abgemeldet" in html and "https://shop.example/done" in html
|
||||
assert "not possible" in html and "login" in html and 'href="https://bank.example/u"' in html
|
||||
assert 'name="email_id" value="42"' in html # trash button only for mails still in the inbox
|
||||
assert html.count('action="/retry"') == 1 # only the failed case can be retried
|
||||
assert "3 mail(s) without unsubscribe link" in html
|
||||
assert "2 mail(s) from senders that are not allowed" in html
|
||||
|
||||
|
||||
def test_retry_resets_the_case(unsub):
|
||||
s = unsub.STORE
|
||||
s.add("<no@x>", "s", ["https://bank.example/u"], "")
|
||||
s.claim("<no@x>")
|
||||
s.finish("<no@x>", "unclear", "x")
|
||||
r = unsub.app.test_client().post("/retry", data={"key": "<no@x>"})
|
||||
assert r.status_code == 302 and s.get("<no@x>")["state"] == "new" and s.get("<no@x>")["attempts"] == 0
|
||||
|
||||
|
||||
def test_pause_and_resume(unsub):
|
||||
c = unsub.app.test_client()
|
||||
c.post("/pause", data={"paused": "1"})
|
||||
assert unsub.STORE.meta("paused") == "1"
|
||||
c.post("/pause", data={"paused": "0"})
|
||||
assert unsub.STORE.meta("paused") == "0"
|
||||
|
||||
|
||||
def test_run_now_starts_a_background_run_and_returns_at_once(unsub, monkeypatch):
|
||||
calls, threads = [], []
|
||||
real_start_run = unsub.start_run
|
||||
|
||||
def keep_thread():
|
||||
threads.append(real_start_run())
|
||||
return threads[-1]
|
||||
|
||||
monkeypatch.setattr(unsub, "start_run", keep_thread)
|
||||
monkeypatch.setattr(unsub.WORKER, "run_once", lambda: calls.append(1) or {"error": ""})
|
||||
assert unsub.app.test_client().post("/run-now").status_code == 302
|
||||
threads[0].join(timeout=5)
|
||||
assert calls == [1]
|
||||
|
||||
def boom():
|
||||
raise RuntimeError("secret mail content")
|
||||
|
||||
monkeypatch.setattr(unsub.WORKER, "run_once", boom)
|
||||
assert unsub.app.test_client().post("/run-now").status_code == 302
|
||||
threads[1].join(timeout=5)
|
||||
assert json.loads(unsub.STORE.meta("last_run"))["error"] == "RuntimeError"
|
||||
|
||||
|
||||
def test_call_agent_maps_failures_to_unavailable(unsub, monkeypatch):
|
||||
import httpx
|
||||
|
||||
def boom(*a, **kw):
|
||||
raise httpx.ConnectError("refused")
|
||||
|
||||
monkeypatch.setattr(unsub.httpx, "post", boom)
|
||||
with pytest.raises(unsub.AgentUnavailable):
|
||||
unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de")
|
||||
monkeypatch.setattr(unsub.httpx, "post", lambda *a, **kw: httpx.Response(500, text="x", request=httpx.Request("POST", a[0])))
|
||||
with pytest.raises(unsub.AgentUnavailable):
|
||||
unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de")
|
||||
|
||||
|
||||
def test_call_agent_sends_only_url_and_address(unsub, monkeypatch):
|
||||
import httpx
|
||||
seen = {}
|
||||
|
||||
def post(url, json=None, timeout=None):
|
||||
seen.update(url=url, json=json, timeout=timeout)
|
||||
return httpx.Response(200, json={"result": "unclear", "reason": "x"}, request=httpx.Request("POST", url))
|
||||
|
||||
monkeypatch.setattr(unsub.httpx, "post", post)
|
||||
assert unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de")["result"] == "unclear"
|
||||
assert seen["url"] == "http://agent.test:8080/run"
|
||||
assert seen["json"] == {"url": "https://shop.example/u", "address": "p@wagnersnetz.de"}
|
||||
assert seen["timeout"] > 600 # two links, 5 minutes each
|
||||
|
||||
|
||||
def test_without_data_dir_the_old_list_is_shown():
|
||||
import app
|
||||
assert app.STORE is None and app.WORKER is None
|
||||
Reference in new issue
Block a user