Abmelde-Agent: Fälle, Zeitlauf und Agent-Dienst (Vikunja 97) (#3)

This commit was merged in pull request #3.
This commit is contained in:
Rsclub2_2 committed 2026-11-16 07:41:08 +01:00
1 parent a9ba08bfe0
commit e0e8cad4de
36 files changed
+4385 -8

No files matched your search

+20
View File
@@ -24,6 +24,26 @@ def test_finds_anchor_and_plain_links_without_duplicates():
assert unsub.find_unsubscribe_links(html) == ["https://ex.org/u?id=1", "https://ex.org/unsubscribe/2"]
def test_escaped_ampersand_in_an_href_does_not_give_a_second_mangled_link():
html = ('<a href="https://ex.org/unsubscribe?u=1&amp;t=abc">Abmelden</a>'
' <a href="https://ex.org/unsubscribe-all?u=1">Von allen abmelden</a>')
assert unsub.find_unsubscribe_links(html) == ["https://ex.org/unsubscribe?u=1&t=abc",
"https://ex.org/unsubscribe-all?u=1"]
def test_plain_text_link_keeps_its_place_behind_an_escaped_duplicate():
html = ('<a href="https://ex.org/unsubscribe?u=1&amp;t=abc">Abmelden</a>'
' oder hier: https://ex.org/unsubscribe/2.')
assert unsub.find_unsubscribe_links(html) == ["https://ex.org/unsubscribe?u=1&t=abc",
"https://ex.org/unsubscribe/2"]
@pytest.mark.parametrize("tail", [".", ",", ";", ":", "!", "?", ")", "”", "’", "»", ").", "!”"])
def test_punctuation_after_a_plain_text_link_is_not_part_of_it(tail):
text = "Abmelden (siehe https://ex.org/unsubscribe/2" + tail + " danach)"
assert unsub.find_unsubscribe_links(text) == ["https://ex.org/unsubscribe/2"]
@pytest.mark.parametrize("href", ["javascript:alert(1)", "data:text/html,<script>x</script>", "JavaScript:x"])
def test_dangerous_links_are_dropped(href):
assert unsub.find_unsubscribe_links(f'<a href="{href}">Unsubscribe</a>') == []
+173
View File
@@ -0,0 +1,173 @@
"""Case store: one case per forwarded mail, keyed by Message-ID."""
from email.message import EmailMessage
import pytest
from cases import CaseStore, case_key
@pytest.fixture
def store(tmp_path):
return CaseStore(str(tmp_path / "cases.db"))
def _msg(mid="<a@x>", subject="News"):
m = EmailMessage()
if mid:
m["Message-ID"] = mid
m["Subject"] = subject
m["Date"] = "Sat, 14 Nov 2026 10:00:00 +0100"
m["From"] = "philipp@wagnersnetz.de"
return m
def test_key_is_message_id_or_stable_hash():
assert case_key(_msg("<a@x>")) == "<a@x>"
k = case_key(_msg(""))
assert k.startswith("sha256:") and k == case_key(_msg("")) and k != case_key(_msg("", "Other"))
def test_add_is_idempotent_and_keeps_links(store):
assert store.add("<a@x>", "News", ["https://e.org/u1", "https://e.org/u2"], "p@wagnersnetz.de") is True
assert store.add("<a@x>", "changed", [], "") is False
c = store.get("<a@x>")
assert (c["subject"], c["links"], c["address"], c["state"], c["attempts"], c["moved"]) == (
"News", ["https://e.org/u1", "https://e.org/u2"], "p@wagnersnetz.de", "new", 0, 0)
def test_pending_only_untried_new_cases_oldest_first(store):
for k in ("<1>", "<2>", "<3>"):
store.add(k, k, ["https://e.org"], "")
store.claim("<1>")
store.finish("<1>", "unclear", "x")
assert [c["key"] for c in store.pending(10)] == ["<2>", "<3>"]
assert [c["key"] for c in store.pending(1)] == ["<2>"]
def test_claim_counts_attempt_and_release_gives_it_back(store):
store.add("<1>", "s", ["https://e.org"], "")
store.claim("<1>")
assert (store.get("<1>")["state"], store.get("<1>")["attempts"]) == ("running", 1)
store.release("<1>")
assert (store.get("<1>")["state"], store.get("<1>")["attempts"]) == ("new", 0)
assert [c["key"] for c in store.pending(10)] == ["<1>"]
def test_finish_stores_reason_and_evidence(store):
store.add("<1>", "s", ["https://e.org"], "")
store.claim("<1>")
store.finish("<1>", "unsubscribed", "page confirms", "You have been unsubscribed", "https://e.org/done")
c = store.get("<1>")
assert (c["state"], c["reason"], c["evidence"], c["evidence_url"]) == (
"unsubscribed", "page confirms", "You have been unsubscribed", "https://e.org/done")
with pytest.raises(ValueError):
store.finish("<1>", "great", "")
def test_to_move_and_mark_moved(store):
store.add("<1>", "s", ["https://e.org"], "")
store.claim("<1>")
store.finish("<1>", "unsubscribed")
assert [c["key"] for c in store.to_move()] == ["<1>"]
store.mark_moved("<1>")
assert store.to_move() == []
def test_retry_makes_a_failed_case_pending_again(store):
store.add("<1>", "s", ["https://e.org"], "")
store.claim("<1>")
store.finish("<1>", "not_possible", "login")
store.retry("<1>")
c = store.get("<1>")
assert (c["state"], c["attempts"], c["reason"]) == ("new", 0, "")
def test_defer_gives_the_attempt_back_and_counts_deferrals(store):
store.add("<1>", "s", ["https://e.org"], "")
store.claim("<1>")
assert store.defer("<1>") == 1
c = store.get("<1>")
assert (c["state"], c["attempts"], c["deferrals"]) == ("new", 0, 1)
store.claim("<1>")
assert store.defer("<1>") == 2
def test_pending_puts_deferred_cases_behind_the_others(store):
for k in ("<1>", "<2>"):
store.add(k, k, ["https://e.org"], "")
store.claim("<1>")
store.defer("<1>")
store.add("<3>", "<3>", ["https://e.org"], "")
assert [c["key"] for c in store.pending(10)] == ["<2>", "<3>", "<1>"]
def test_retry_resets_the_deferrals(store):
store.add("<1>", "s", ["https://e.org"], "")
store.claim("<1>")
store.defer("<1>")
store.claim("<1>")
store.finish("<1>", "unclear", "model unavailable for this case")
store.retry("<1>")
assert (store.get("<1>")["state"], store.get("<1>")["deferrals"]) == ("new", 0)
def test_database_without_the_deferrals_column_is_migrated(tmp_path):
import sqlite3
path = str(tmp_path / "old.db")
db = sqlite3.connect(path)
db.execute(
"CREATE TABLE cases ("
"key TEXT PRIMARY KEY, subject TEXT NOT NULL, links TEXT NOT NULL, address TEXT NOT NULL, "
"state TEXT NOT NULL, reason TEXT NOT NULL DEFAULT '', evidence TEXT NOT NULL DEFAULT '', "
"evidence_url TEXT NOT NULL DEFAULT '', attempts INTEGER NOT NULL DEFAULT 0, "
"moved INTEGER NOT NULL DEFAULT 0, created_at REAL NOT NULL, updated_at REAL NOT NULL)")
db.execute("INSERT INTO cases (key, subject, links, address, state, created_at, updated_at) "
"VALUES ('<old>', 's', '[]', '', 'new', 1.0, 1.0)")
db.commit()
db.close()
store = CaseStore(path)
assert store.get("<old>")["deferrals"] == 0
store.claim("<old>")
assert store.defer("<old>") == 1
CaseStore(path) # opening a migrated database again changes nothing
assert store.get("<old>")["deferrals"] == 1
def test_expired_only_moved_successes_older_than_days(store):
for k in ("<old>", "<young>", "<failed>"):
store.add(k, "s", ["https://e.org"], "")
store.claim(k)
store.finish("<old>", "unsubscribed")
store.finish("<young>", "unsubscribed")
store.finish("<failed>", "unclear")
store.mark_moved("<old>")
store.mark_moved("<young>")
now = store.get("<old>")["updated_at"]
assert [c["key"] for c in store.expired(30, now=now + 31 * 86400)] == ["<old>", "<young>"]
assert store.expired(30, now=now + 29 * 86400) == []
store.delete("<old>")
assert store.get("<old>") is None
def test_recover_turns_running_into_unclear(store):
store.add("<1>", "s", ["https://e.org"], "")
store.claim("<1>")
assert store.recover() == 1
c = store.get("<1>")
assert (c["state"], c["reason"], c["attempts"]) == ("unclear", "interrupted by restart", 1)
def test_meta_roundtrip(store):
assert store.meta("paused", "0") == "0"
store.set_meta("paused", "1")
assert store.meta("paused") == "1"
def test_key_copes_with_a_non_ascii_message_id():
import email
raw = "Message-ID: <näh@x>\r\nSubject: s\r\n\r\nbody".encode("utf-8")
message = email.message_from_bytes(raw)
key = case_key(message)
assert isinstance(key, str) and key
assert case_key(message) == key
+183
View File
@@ -0,0 +1,183 @@
"""Case view and its actions (the store is real, the mailbox is faked)."""
import importlib
import json
import pytest
@pytest.fixture
def unsub(tmp_path, monkeypatch):
monkeypatch.setenv("DATA_DIR", str(tmp_path))
monkeypatch.setenv("AGENT_SCHEDULER", "0")
monkeypatch.setenv("AGENT_URL", "http://agent.test:8080")
monkeypatch.setenv("AGENT_ALLOWED_DOMAINS", "wagnersnetz.de, gmail.com")
monkeypatch.setenv("AGENT_ALLOWED_SENDERS", "Jens@wagnersnetz.de, philipp@wagnersnetz.de")
for name in ("OIDC_ISSUER", "OIDC_CLIENT_ID", "TOKEN_KEY"):
monkeypatch.delenv(name, raising=False)
import app
module = importlib.reload(app)
yield module
monkeypatch.delenv("DATA_DIR")
monkeypatch.delenv("AGENT_URL")
importlib.reload(app) # the other test modules expect the app without a store
def test_settings_from_environment(unsub):
assert unsub.STORE is not None and unsub.WORKER is not None
assert unsub.WORKER.allowed_domains == ("wagnersnetz.de", "gmail.com")
assert unsub.WORKER.allowed_senders == ("jens@wagnersnetz.de", "philipp@wagnersnetz.de")
assert unsub.WORKER.agent is unsub.call_agent
def test_result_shows_cases_with_state_reason_evidence_and_links(unsub, monkeypatch):
s = unsub.STORE
s.add("<ok@x>", "Fwd: Shop <b>News</b>", ["https://shop.example/u/1"], "p@wagnersnetz.de")
s.claim("<ok@x>")
s.finish("<ok@x>", "unsubscribed", "page confirms", "Sie wurden abgemeldet", "https://shop.example/done")
s.add("<no@x>", "Fwd: Bank", ["https://bank.example/u"], "")
s.claim("<no@x>")
s.finish("<no@x>", "not_possible", "login")
s.set_meta("last_run", json.dumps({"new": 2, "handled": 2, "unsubscribed": 1, "moved": 1, "cleaned": 0,
"without_link": 3, "ignored": 2, "error": ""}))
s.set_meta("last_run_at", "1800000000")
monkeypatch.setattr(unsub, "scan_inbox", lambda: {"<no@x>": b"42"})
html = unsub.app.test_client().get("/result").get_data(as_text=True)
assert "Fwd: Shop &lt;b&gt;News&lt;/b&gt;" in html and "<b>News</b>" not in html
assert "unsubscribed" in html and "Sie wurden abgemeldet" in html and "https://shop.example/done" in html
assert "not possible" in html and "login" in html and 'href="https://bank.example/u"' in html
assert 'name="email_id" value="42"' in html # trash button only for mails still in the inbox
assert html.count('action="/retry"') == 1 # only the failed case can be retried
assert "3 mail(s) without unsubscribe link" in html
assert "2 mail(s) from senders that are not allowed" in html
def test_result_still_lists_cases_when_the_inbox_scan_fails(unsub, monkeypatch):
from worker import ScanFailed
s = unsub.STORE
s.add("<no@x>", "Fwd: Bank", ["https://bank.example/u"], "")
s.claim("<no@x>")
s.finish("<no@x>", "not_possible", "login")
def failing():
raise ScanFailed("inbox scan failed")
monkeypatch.setattr(unsub, "scan_inbox", failing)
response = unsub.app.test_client().get("/result")
html = response.get_data(as_text=True)
assert response.status_code == 200
assert "Fwd: Bank" in html and "The inbox could not be read" in html
assert 'name="email_id"' not in html and 'action="/retry"' not in html # nothing is known about the mails
def test_max_deferrals_default_and_from_environment(unsub, monkeypatch):
import app
assert unsub.WORKER.max_deferrals == 12
monkeypatch.setenv("AGENT_MAX_DEFERRALS", "3")
assert importlib.reload(app).WORKER.max_deferrals == 3
def test_result_shows_cases_waiting_for_the_model(unsub, monkeypatch):
unsub.STORE.set_meta("last_run", json.dumps({"new": 0, "deferred": 2, "error": ""}))
monkeypatch.setattr(unsub, "scan_inbox", lambda: {})
html = unsub.app.test_client().get("/result").get_data(as_text=True)
assert "2 case(s) waiting for the model." in html
def test_retry_only_for_a_case_with_its_mail_in_the_inbox(unsub, monkeypatch):
s = unsub.STORE
for key in ("<here@x>", "<gone@x>"):
s.add(key, key, ["https://bank.example/u"], "")
s.claim(key)
s.finish(key, "not_possible", "login")
monkeypatch.setattr(unsub, "scan_inbox", lambda: {"<here@x>": b"42"})
html = unsub.app.test_client().get("/result").get_data(as_text=True)
assert html.count('action="/retry"') == 1 and 'name="key" value="&lt;here@x&gt;"' in html
assert 'name="key" value="&lt;gone@x&gt;"' not in html
def test_unmoved_success_shows_not_moved_yet(unsub, monkeypatch):
s = unsub.STORE
for key in ("<unmoved@x>", "<moved@x>"):
s.add(key, key, ["https://shop.example/u"], "")
s.claim(key)
s.finish(key, "unsubscribed", "page confirms")
s.mark_moved("<moved@x>")
monkeypatch.setattr(unsub, "scan_inbox", lambda: {"<unmoved@x>": b"7"})
html = unsub.app.test_client().get("/result").get_data(as_text=True)
assert html.count("not moved yet") == 1
row = html[html.index("&lt;unmoved@x&gt;"):]
assert row.index("not moved yet") < row.index("</tr>")
def test_retry_resets_the_case(unsub):
s = unsub.STORE
s.add("<no@x>", "s", ["https://bank.example/u"], "")
s.claim("<no@x>")
s.finish("<no@x>", "unclear", "x")
r = unsub.app.test_client().post("/retry", data={"key": "<no@x>"})
assert r.status_code == 302 and s.get("<no@x>")["state"] == "new" and s.get("<no@x>")["attempts"] == 0
def test_pause_and_resume(unsub):
c = unsub.app.test_client()
c.post("/pause", data={"paused": "1"})
assert unsub.STORE.meta("paused") == "1"
c.post("/pause", data={"paused": "0"})
assert unsub.STORE.meta("paused") == "0"
def test_run_now_starts_a_background_run_and_returns_at_once(unsub, monkeypatch):
calls, threads = [], []
real_start_run = unsub.start_run
def keep_thread():
threads.append(real_start_run())
return threads[-1]
monkeypatch.setattr(unsub, "start_run", keep_thread)
monkeypatch.setattr(unsub.WORKER, "run_once", lambda: calls.append(1) or {"error": ""})
assert unsub.app.test_client().post("/run-now").status_code == 302
threads[0].join(timeout=5)
assert calls == [1]
def boom():
raise RuntimeError("secret mail content")
monkeypatch.setattr(unsub.WORKER, "run_once", boom)
assert unsub.app.test_client().post("/run-now").status_code == 302
threads[1].join(timeout=5)
assert json.loads(unsub.STORE.meta("last_run"))["error"] == "RuntimeError"
def test_call_agent_maps_failures_to_unavailable(unsub, monkeypatch):
import httpx
def boom(*a, **kw):
raise httpx.ConnectError("refused")
monkeypatch.setattr(unsub.httpx, "post", boom)
with pytest.raises(unsub.AgentUnavailable):
unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de")
monkeypatch.setattr(unsub.httpx, "post", lambda *a, **kw: httpx.Response(500, text="x", request=httpx.Request("POST", a[0])))
with pytest.raises(unsub.AgentUnavailable):
unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de")
def test_call_agent_sends_only_url_and_address(unsub, monkeypatch):
import httpx
seen = {}
def post(url, json=None, timeout=None):
seen.update(url=url, json=json, timeout=timeout)
return httpx.Response(200, json={"result": "unclear", "reason": "x"}, request=httpx.Request("POST", url))
monkeypatch.setattr(unsub.httpx, "post", post)
assert unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de")["result"] == "unclear"
assert seen["url"] == "http://agent.test:8080/run"
assert seen["json"] == {"url": "https://shop.example/u", "address": "p@wagnersnetz.de"}
assert seen["timeout"] > 600 # two links, 5 minutes each
def test_without_data_dir_the_old_list_is_shown():
import app
assert app.STORE is None and app.WORKER is None
+74
View File
@@ -0,0 +1,74 @@
"""Which address did the newsletter go to? Taken from the quoted header of a forwarded mail."""
import pytest
from forwarded import original_recipient
ALLOWED = ("wagnersnetz.de", "gmail.com")
def test_german_forward_header():
text = "---------- Weitergeleitete Nachricht ---------\nVon: News <news@shop.example>\nAn: Philipp <philipp@wagnersnetz.de>\nBetreff: Angebot\n"
assert original_recipient(text, "jens@wagnersnetz.de", ALLOWED) == "philipp@wagnersnetz.de"
def test_english_forward_header_in_html():
html = "<div>From: news@shop.example<br>To: <a href='mailto:Some.One@Gmail.com'>Some.One@Gmail.com</a><br>Subject: x</div>"
assert original_recipient(html, "philipp@wagnersnetz.de", ALLOWED) == "some.one@gmail.com"
def test_quoted_prefix_and_several_recipients_first_allowed_wins():
text = "> To: other@elsewhere.example, anne@wagnersnetz.de\n"
assert original_recipient(text, "", ALLOWED) == "anne@wagnersnetz.de"
def test_falls_back_to_forwarder():
assert original_recipient("no header here", "Jens <jens@wagnersnetz.de>", ALLOWED) == "jens@wagnersnetz.de"
def test_foreign_domain_is_never_used():
text = "To: victim@elsewhere.example\n"
assert original_recipient(text, "someone@elsewhere.example", ALLOWED) == ""
@pytest.mark.parametrize("domain", ["evilwagnersnetz.de", "wagnersnetz.de.evil.example"])
def test_domain_must_match_exactly(domain):
assert original_recipient(f"To: x@{domain}\n", "", ALLOWED) == ""
def test_subdomain_is_not_allowed_implicitly():
assert original_recipient("To: x@mail.wagnersnetz.de\n", "", ALLOWED) == ""
def test_malformed_forwarder_is_not_used():
assert original_recipient("no header", "a@b@wagnersnetz.de", ALLOWED) == ""
assert original_recipient("no header", "<x y@wagnersnetz.de>", ALLOWED) == ""
def test_full_address_entry_allows_only_that_address():
allowed = ("wagnersnetz.de", "Some.One@Gmail.com")
assert original_recipient("To: some.one@gmail.com\n", "", allowed) == "some.one@gmail.com"
assert original_recipient("To: SOME.ONE@GMAIL.COM\n", "", allowed) == "some.one@gmail.com"
assert original_recipient("To: someone.else@gmail.com\n", "", allowed) == ""
def test_body_line_with_a_third_persons_public_address_is_refused():
text = ("Von: news@shop.example\nAn: philipp@gmail.com\n\nWrite to us!\nTo: someone@gmail.com\n")
allowed = ("wagnersnetz.de", "philipp@gmail.com")
assert original_recipient("To: someone@gmail.com\n", "philipp@gmail.com", allowed) == "philipp@gmail.com"
assert original_recipient(text, "", allowed) == "philipp@gmail.com"
def test_mixed_list_of_domains_and_addresses():
allowed = ("wagnersnetz.de", "anne@gmail.com")
assert original_recipient("To: x@elsewhere.example, jens@wagnersnetz.de\n", "", allowed) == "jens@wagnersnetz.de"
assert original_recipient("To: bob@gmail.com, anne@gmail.com\n", "", allowed) == "anne@gmail.com"
assert original_recipient("no header", "anne@gmail.com", allowed) == "anne@gmail.com"
assert original_recipient("no header", "bob@gmail.com", allowed) == ""
def test_address_entry_does_not_allow_its_domain_or_a_lookalike():
allowed = ("anne@gmail.com",)
assert original_recipient("To: gmail.com@gmail.com\n", "", allowed) == ""
assert original_recipient("To: xanne@gmail.com\n", "", allowed) == ""
assert original_recipient("To: anne@gmail.com.evil.example\n", "", allowed) == ""
+519
View File
@@ -0,0 +1,519 @@
"""Scheduler run: find cases, hand them to the agent, move successes, clean up (fake IMAP, fake agent)."""
import json
import time
from email.message import EmailMessage
import pytest
from cases import CaseStore
from cases import case_key
from worker import AgentUnavailable, Worker, _tick
def mail_bytes(mid, link="https://shop.example/unsubscribe/1", to="philipp@wagnersnetz.de", links=None,
sender="Jens <jens@wagnersnetz.de>"):
m = EmailMessage()
m["Message-ID"] = mid
m["Subject"] = "Fwd: News " + mid
m["From"] = sender
urls = links if links is not None else ([link] if link else [])
m.set_content("Von: news@shop.example\nAn: " + to + "\n\n" + "\n".join(urls))
return m.as_bytes()
class FakeIMAP:
"""Two folders; uid commands as far as the worker uses them."""
def __init__(self, inbox):
self.folders = {"INBOX": dict(inbox)}
self.selected = "INBOX"
self.deleted = set()
self.fail_copy = False
self.logged_out = False
self.fail_store_once = False
self.search_status = "OK"
self.fail_fetch = set()
self.calls = []
def select(self, folder):
if folder not in self.folders:
return "NO", [b"no such folder"]
self.selected = folder
return "OK", [b"1"]
def create(self, folder):
if folder in self.folders:
return "NO", [b"exists"]
self.folders[folder] = {}
return "OK", [b"created"]
def uid(self, command, *args):
self.calls.append((command, args))
box = self.folders[self.selected]
if command == "search":
if len(args) >= 4 and args[1] == "HEADER":
wanted = args[3].strip('"')
hits = [u for u, raw in box.items() if ("Message-ID: " + wanted).encode() in raw]
return "OK", [b" ".join(hits)]
if self.search_status != "OK":
return self.search_status, [b"search failed"]
return "OK", [b" ".join(box)]
if command == "fetch":
if args[0] in self.fail_fetch:
return "NO", [None]
return "OK", [(b"1 (BODY[] {n}", box[args[0]]), b")"]
if command == "copy":
if self.fail_copy or args[0] not in box:
return "NO", [b"copy failed"]
self.folders[args[1]][b"9" + args[0]] = box[args[0]]
return "OK", [b""]
if command == "store":
if self.fail_store_once:
self.fail_store_once = False
return "NO", [b"store failed"]
self.deleted.add((self.selected, args[0]))
return "OK", [b""]
raise AssertionError(command)
def expunge(self):
for folder, uid in list(self.deleted):
self.folders[folder].pop(uid, None)
self.deleted.clear()
return "OK", [b""]
def logout(self):
self.logged_out = True
def parse(message):
import app as unsub
text = unsub.get_text_from_email(message)
return unsub.get_subject(message), unsub.find_unsubscribe_links(text), text
class Agent:
def __init__(self, *results):
self.results = list(results)
self.calls = []
def __call__(self, url, address):
self.calls.append((url, address))
r = self.results.pop(0) if self.results else {"result": "unclear", "reason": "no more results"}
if isinstance(r, Exception):
raise r
return {"evidence": "", "evidence_url": "", "steps": 1, "stage": 2, **r}
OK = {"result": "unsubscribed", "reason": "page confirms", "evidence": "abgemeldet", "evidence_url": "https://shop.example/done"}
@pytest.fixture
def store(tmp_path):
return CaseStore(str(tmp_path / "cases.db"))
def worker(store, imap, agent, **kw):
kw.setdefault("allowed_senders", ("jens@wagnersnetz.de",))
return Worker(store, lambda: imap, parse, agent, allowed_domains=("wagnersnetz.de",), **kw)
def test_success_is_moved_to_the_done_folder(store):
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
agent = Agent(OK)
summary = worker(store, imap, agent).run_once()
assert agent.calls == [("https://shop.example/unsubscribe/1", "philipp@wagnersnetz.de")]
assert (summary["new"], summary["handled"], summary["unsubscribed"], summary["moved"], summary["error"]) == (1, 1, 1, 1, "")
assert imap.folders["INBOX"] == {} and len(imap.folders["Erledigt"]) == 1 and imap.logged_out
c = store.get("<a@x>")
assert (c["state"], c["moved"], c["evidence"]) == ("unsubscribed", 1, "abgemeldet")
assert json.loads(store.meta("last_run"))["moved"] == 1 and store.meta("last_run_at")
def test_failure_stays_in_the_inbox_and_is_not_tried_again(store):
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
agent = Agent({"result": "not_possible", "reason": "captcha"})
w = worker(store, imap, agent)
w.run_once()
w.run_once()
assert len(agent.calls) == 1 and b"7" in imap.folders["INBOX"]
assert (store.get("<a@x>")["state"], store.get("<a@x>")["reason"]) == ("not_possible", "captcha")
def test_second_link_is_tried_but_not_a_third_and_not_after_login(store):
links = ["https://shop.example/unsubscribe/1", "https://shop.example/unsubscribe/2", "https://shop.example/unsubscribe/3"]
imap = FakeIMAP({b"1": mail_bytes("<two@x>", links=links), b"2": mail_bytes("<login@x>", links=links)})
agent = Agent({"result": "unclear", "reason": "x"}, OK, {"result": "not_possible", "reason": "login"})
worker(store, imap, agent).run_once()
assert [u for u, _ in agent.calls] == [links[0], links[1], links[0]]
assert store.get("<two@x>")["state"] == "unsubscribed"
assert (store.get("<login@x>")["state"], store.get("<login@x>")["reason"]) == ("not_possible", "login")
def test_agent_error_does_not_burn_the_attempt(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
agent = Agent(AgentUnavailable("agent not reachable"))
w = worker(store, imap, agent)
summary = w.run_once()
assert summary["error"] == "agent not reachable" and summary["handled"] == 0 and len(agent.calls) == 1
assert [(c["state"], c["attempts"]) for c in store.all()] == [("new", 0), ("new", 0)]
agent.results = [OK, OK]
assert w.run_once()["unsubscribed"] == 2
def test_model_unavailable_result_counts_like_an_unavailable_agent(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
summary = worker(store, imap, Agent({"result": "error", "reason": "llm_unavailable"})).run_once()
assert summary["error"] == "llm_unavailable" and store.get("<a@x>")["attempts"] == 0
ERR = {"result": "error", "reason": "llm_unavailable"}
def test_model_error_defers_the_case_and_the_run_goes_on(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", link="https://shop.example/unsubscribe/a"),
b"2": mail_bytes("<b@x>", link="https://shop.example/unsubscribe/b")})
agent = Agent(ERR, OK)
summary = worker(store, imap, agent).run_once()
first, second = store.get("<a@x>"), store.get("<b@x>")
assert second["state"] == "unsubscribed"
assert (first["state"], first["attempts"], first["deferrals"]) == ("new", 0, 1)
assert summary["error"] == "" and summary["deferred"] == 1 and summary["handled"] == 1
def test_deferred_case_is_tried_after_newer_ones(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", link="https://shop.example/unsubscribe/a")})
agent = Agent(ERR)
w = worker(store, imap, agent)
assert w.run_once()["deferred"] == 1
imap.folders["INBOX"][b"2"] = mail_bytes("<b@x>", link="https://shop.example/unsubscribe/b")
agent.calls, agent.results = [], [OK, OK]
w.run_once()
assert [u for u, _ in agent.calls] == ["https://shop.example/unsubscribe/b", "https://shop.example/unsubscribe/a"]
def test_case_deferred_max_deferrals_times_ends_unclear(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
agent = Agent(ERR, ERR, ERR, OK)
w = worker(store, imap, agent, max_deferrals=3)
summaries = [w.run_once() for _ in range(4)]
c = store.get("<a@x>")
assert (c["state"], c["reason"]) == ("unclear", "model unavailable for this case")
assert len(agent.calls) == 3 and summaries[3]["handled"] == 0
assert [s["deferred"] for s in summaries[:2]] == [1, 1]
def test_transport_failure_still_stops_the_run_and_leaves_all_cases_untouched(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
agent = Agent(AgentUnavailable("agent not reachable"), OK)
summary = worker(store, imap, agent).run_once()
assert summary["error"] == "agent not reachable" and summary["deferred"] == 0 and len(agent.calls) == 1
assert [(c["state"], c["attempts"], c["deferrals"]) for c in store.all()] == [("new", 0, 0), ("new", 0, 0)]
def test_without_agent_cases_are_only_recorded(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<nolink@x>", link="")})
summary = worker(store, imap, None).run_once()
assert (summary["new"], summary["handled"], summary["without_link"]) == (1, 0, 1)
assert store.get("<a@x>")["state"] == "new" and store.get("<nolink@x>") is None
def test_paused_run_does_nothing(store):
store.set_meta("paused", "1")
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
agent = Agent(OK)
summary = worker(store, imap, agent).run_once()
assert summary["error"] == "paused" and agent.calls == [] and store.all() == []
def test_pause_during_a_run_stops_before_the_next_case(store):
imap = FakeIMAP({str(i).encode(): mail_bytes(f"<{i}@x>") for i in range(1, 4)})
class Pausing(Agent):
def __call__(self, url, address):
store.set_meta("paused", "1") # "Pause agent" pressed while the first case runs
return super().__call__(url, address)
agent = Pausing(OK, OK, OK)
summary = worker(store, imap, agent).run_once()
assert len(agent.calls) == 1 and summary["error"] == "paused"
assert summary["handled"] == 1 and summary["moved"] == 1 # what is finished is still moved
assert [c["state"] for c in store.all()].count("new") == 2
def test_at_most_max_cases_per_run(store):
imap = FakeIMAP({str(i).encode(): mail_bytes(f"<{i}@x>") for i in range(1, 5)})
agent = Agent(OK, OK, OK, OK)
assert worker(store, imap, agent, max_cases=3).run_once()["handled"] == 3
assert len(agent.calls) == 3
def test_address_of_a_foreign_domain_is_not_handed_over(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", to="victim@elsewhere.example")})
agent = Agent(OK)
worker(store, imap, agent).run_once()
assert agent.calls == [("https://shop.example/unsubscribe/1", "jens@wagnersnetz.de")] # the forwarder
def test_move_failure_keeps_case(store):
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
imap.fail_copy = True
w = worker(store, imap, Agent(OK))
summary = w.run_once()
assert summary["moved"] == 0 and b"7" in imap.folders["INBOX"]
assert (store.get("<a@x>")["state"], store.get("<a@x>")["moved"]) == ("unsubscribed", 0)
imap.fail_copy = False
assert w.run_once()["moved"] == 1 and imap.folders["INBOX"] == {}
def test_cleanup_deletes_old_successes_from_the_done_folder(store):
imap = FakeIMAP({b"7": mail_bytes("<old@x>"), b"8": mail_bytes("<young@x>")})
now = [time.time()] # store stamps real time; a fixed start date would make the test date-dependent
w = worker(store, imap, Agent(OK, OK), clock=lambda: now[0], retention_days=30)
w.run_once()
assert len(imap.folders["Erledigt"]) == 2
young_time = store.get("<young@x>")["updated_at"]
store._run("UPDATE cases SET updated_at = ? WHERE key = '<old@x>'", (young_time - 31 * 86400,))
now[0] = young_time + 60
summary = w.run_once()
assert summary["cleaned"] == 1
assert store.get("<old@x>") is None and store.get("<young@x>") is not None
assert len(imap.folders["Erledigt"]) == 1 and imap.selected == "INBOX"
def test_run_survives_a_broken_mail(store):
imap = FakeIMAP({b"1": b"\xff\xfe not a mail", b"2": mail_bytes("<b@x>")})
assert worker(store, imap, Agent(OK)).run_once()["unsubscribed"] == 1
def test_mail_from_unknown_sender_is_ignored(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender="Jens <JENS@wagnersnetz.de>"),
b"2": mail_bytes("<evil@x>", sender="Mallory <mallory@evil.example>")})
agent = Agent(OK)
summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once()
assert len(agent.calls) == 1 and summary["ignored"] == 1
assert [c["key"] for c in store.all()] == ["<a@x>"]
assert store.get("<evil@x>") is None and b"2" in imap.folders["INBOX"]
def test_no_allowed_senders_with_agent_means_nobody(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
agent = Agent(OK)
summary = worker(store, imap, agent, allowed_senders=()).run_once()
assert agent.calls == [] and store.all() == [] and summary["ignored"] == 1
def test_no_allowed_senders_without_agent_lists_everything(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
summary = worker(store, imap, None, allowed_senders=()).run_once()
assert store.get("<a@x>")["state"] == "new" and summary["ignored"] == 0
def test_spoofed_display_name_does_not_help(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender='"jens@wagnersnetz.de" <mallory@evil.example>')})
agent = Agent(OK)
summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once()
assert agent.calls == [] and store.all() == [] and summary["ignored"] == 1
def old_case(store, key, address="jens@wagnersnetz.de"):
store.add(key, "Fwd: old", ["https://shop.example/unsubscribe/1"], address)
def test_old_case_of_a_sender_no_longer_allowed_is_not_handled(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender="Mallory <mallory@evil.example>")})
worker(store, imap, None, allowed_senders=()).run_once()
assert store.get("<a@x>")["state"] == "new"
agent = Agent(OK)
summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once()
assert agent.calls == [] and summary["ignored"] == 1 and summary["handled"] == 0
assert (store.get("<a@x>")["state"], store.get("<a@x>")["attempts"]) == ("new", 0)
def test_case_whose_mail_left_the_inbox_is_not_handled(store):
old_case(store, "<gone@x>")
agent = Agent(OK)
worker(store, FakeIMAP({}), agent).run_once()
assert agent.calls == []
def test_failed_case_whose_mail_was_moved_to_the_trash_disappears(store):
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
w = worker(store, imap, Agent({"result": "not_possible", "reason": "captcha"}))
w.run_once()
assert store.get("<a@x>")["state"] == "not_possible"
imap.folders["INBOX"].pop(b"7") # "Move to Trash" on the page
summary = w.run_once()
assert store.get("<a@x>") is None and summary["gone"] == 1
def test_new_case_whose_mail_is_gone_is_deleted_but_a_running_one_is_left_alone(store):
old_case(store, "<new@x>")
old_case(store, "<running@x>")
store.claim("<running@x>")
summary = worker(store, FakeIMAP({}), Agent()).run_once()
assert store.get("<new@x>") is None and store.get("<running@x>")["state"] == "running"
assert summary["gone"] == 1
def test_unmoved_success_whose_mail_is_gone_counts_as_moved_and_expires(store):
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
imap.fail_copy = True
now = [time.time()]
w = worker(store, imap, Agent(OK), clock=lambda: now[0], retention_days=30)
w.run_once()
assert (store.get("<a@x>")["state"], store.get("<a@x>")["moved"]) == ("unsubscribed", 0)
imap.folders["INBOX"].pop(b"7") # moved by hand
summary = w.run_once()
assert store.get("<a@x>")["moved"] == 1 and summary["gone"] == 1
imap.folders["Erledigt"] = {}
now[0] = store.get("<a@x>")["updated_at"] + 31 * 86400
assert w.run_once()["cleaned"] == 1 and store.get("<a@x>") is None
@pytest.mark.parametrize("status", ["NO", "BAD"])
def test_failed_inbox_search_ends_the_run_with_an_error_and_deletes_nothing(store, status):
old_case(store, "<a@x>")
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
imap.search_status = status
agent = Agent(OK)
summary = worker(store, imap, agent).run_once()
assert summary["error"] == "inbox scan failed" and summary["gone"] == 0 and agent.calls == []
assert store.get("<a@x>")["state"] == "new" and imap.logged_out
assert json.loads(store.meta("last_run"))["error"] == "inbox scan failed"
def test_case_added_by_the_page_during_the_scan_is_not_tidied_away(store):
def parse_while_the_page_scans(message):
store.add("<arrived@x>", "s", ["https://shop.example/unsubscribe/9"], "") # page scan saw a newer mail
return parse(message)
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
w = Worker(store, lambda: imap, parse_while_the_page_scans, None, allowed_domains=("wagnersnetz.de",))
assert w.run_once()["gone"] == 0 and store.get("<arrived@x>") is not None
def test_unreadable_mail_in_the_inbox_means_nothing_is_tidied(store):
old_case(store, "<a@x>")
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
imap.fail_fetch = {b"1"} # could be the mail of <a@x>: its key is unknown
summary = worker(store, imap, Agent(OK, OK)).run_once()
assert store.get("<a@x>") is not None and summary["gone"] == 0
assert store.get("<b@x>")["state"] == "unsubscribed"
def test_case_of_a_mail_from_a_sender_no_longer_allowed_is_kept(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender="Mallory <mallory@evil.example>")})
worker(store, imap, None, allowed_senders=()).run_once()
summary = worker(store, imap, Agent(OK), allowed_senders=("jens@wagnersnetz.de",)).run_once()
assert store.get("<a@x>")["state"] == "new" and summary["gone"] == 0
def test_skipped_cases_do_not_eat_the_budget(store):
for i in range(3):
old_case(store, f"<gone{i}@x>")
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
agent = Agent(OK, OK, OK)
assert worker(store, imap, agent, max_cases=2).run_once()["handled"] == 2
assert len(agent.calls) == 2 and store.get("<a@x>")["state"] == "unsubscribed"
def test_agent_bug_ends_the_case_unclear_and_the_run_goes_on(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
summary = worker(store, imap, Agent(RuntimeError("boom"), OK)).run_once()
first, second = store.get("<a@x>"), store.get("<b@x>")
assert (first["state"], first["reason"], first["attempts"]) == ("unclear", "agent error", 1)
assert second["state"] == "unsubscribed"
assert summary["handled"] == 2 and summary["error"] == ""
class RawAgent:
def __init__(self, result):
self.result = result
def __call__(self, url, address):
return self.result
@pytest.mark.parametrize("result", [{}, {"result": "bought"}, None, "text"])
def test_malformed_agent_result_ends_the_case_unclear(store, result):
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
summary = worker(store, imap, RawAgent(result)).run_once()
c = store.get("<a@x>")
assert (c["state"], c["reason"], c["attempts"]) == ("unclear", "agent error", 1)
assert summary["handled"] == 1 and summary["error"] == ""
def test_exception_while_handling_one_mail_does_not_stop_the_run(store):
raw = ("Message-ID: <broken@x>\nSubject: broken\nFrom: jens@wagnersnetz.de\n\n"
"Von: news@shop.example\nAn: philipp@wagnersnetz.de\n\nhttps://shop.example/unsubscribe/9\n").encode("utf-8")
def parse_failing_on_one(message):
result = parse(message)
if result[0] == "broken":
raise ValueError("cannot handle this mail")
return result
imap = FakeIMAP({b"1": raw, b"2": mail_bytes("<b@x>")})
w = Worker(store, lambda: imap, parse_failing_on_one, Agent(OK), allowed_domains=("wagnersnetz.de",),
allowed_senders=("jens@wagnersnetz.de",))
summary = w.run_once()
assert store.get("<broken@x>") is None
assert store.get("<b@x>")["state"] == "unsubscribed" and summary["error"] == ""
@pytest.mark.parametrize("key", ["<a\r\nb@x>", "<a@x>\\", '<a"b@x>', "<n\u00e4h@x>"])
def test_cleanup_never_sends_a_hostile_message_id(store, key):
old_case(store, key)
store.claim(key)
store.finish(key, "unsubscribed")
store.mark_moved(key)
store._run("UPDATE cases SET updated_at = ? WHERE key = ?", (1.0, key))
imap = FakeIMAP({})
imap.folders["Erledigt"] = {}
summary = worker(store, imap, Agent()).run_once()
assert summary["cleaned"] == 1 and store.get(key) is None
assert not any(cmd == "search" and key in " ".join(map(str, args)) for cmd, args in imap.calls)
def test_scan_from_a_request_does_not_change_a_runs_counters(store):
w = worker(store, FakeIMAP({}), Agent())
result = w.scan(FakeIMAP({b"1": mail_bytes("<a@x>")}))
assert len(result) == 2 and not hasattr(w, "_created") and not hasattr(w, "_ignored")
def test_failed_delete_flag_keeps_the_case_unmoved(store):
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
imap.fail_store_once = True
w = worker(store, imap, Agent(OK))
assert w.run_once()["moved"] == 0
assert store.get("<a@x>")["moved"] == 0 and b"7" in imap.folders["INBOX"]
assert w.run_once()["moved"] == 1 and store.get("<a@x>")["moved"] == 1
ALL_KEYS = {"new", "handled", "unsubscribed", "moved", "cleaned", "without_link", "ignored", "error", "deferred", "gone"}
class Boom:
def __init__(self, store, set_meta_fails=False):
self.store, self.fail = store, set_meta_fails
self.clock = lambda: 1.0
def run_once(self):
raise ValueError("secret subject line")
class BrokenStore:
def set_meta(self, k, v):
raise OSError("disk")
def test_tick_records_only_the_exception_type(store):
_tick(Boom(store))
last = json.loads(store.meta("last_run"))
assert set(last) == ALL_KEYS and last["error"] == "ValueError"
assert "secret subject line" not in store.meta("last_run")
def test_tick_survives_a_failing_store():
_tick(Boom(BrokenStore()))