Abmelde-Agent: Fälle, Zeitlauf und Agent-Dienst (Vikunja 97) (#3)
This commit was merged in pull request #3.
This commit is contained in:
36 files changed
+4385
-8
No files matched your search
@@ -24,6 +24,26 @@ def test_finds_anchor_and_plain_links_without_duplicates():
|
||||
assert unsub.find_unsubscribe_links(html) == ["https://ex.org/u?id=1", "https://ex.org/unsubscribe/2"]
|
||||
|
||||
|
||||
def test_escaped_ampersand_in_an_href_does_not_give_a_second_mangled_link():
|
||||
html = ('<a href="https://ex.org/unsubscribe?u=1&t=abc">Abmelden</a>'
|
||||
' <a href="https://ex.org/unsubscribe-all?u=1">Von allen abmelden</a>')
|
||||
assert unsub.find_unsubscribe_links(html) == ["https://ex.org/unsubscribe?u=1&t=abc",
|
||||
"https://ex.org/unsubscribe-all?u=1"]
|
||||
|
||||
|
||||
def test_plain_text_link_keeps_its_place_behind_an_escaped_duplicate():
|
||||
html = ('<a href="https://ex.org/unsubscribe?u=1&t=abc">Abmelden</a>'
|
||||
' oder hier: https://ex.org/unsubscribe/2.')
|
||||
assert unsub.find_unsubscribe_links(html) == ["https://ex.org/unsubscribe?u=1&t=abc",
|
||||
"https://ex.org/unsubscribe/2"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tail", [".", ",", ";", ":", "!", "?", ")", "”", "’", "»", ").", "!”"])
|
||||
def test_punctuation_after_a_plain_text_link_is_not_part_of_it(tail):
|
||||
text = "Abmelden (siehe https://ex.org/unsubscribe/2" + tail + " danach)"
|
||||
assert unsub.find_unsubscribe_links(text) == ["https://ex.org/unsubscribe/2"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("href", ["javascript:alert(1)", "data:text/html,<script>x</script>", "JavaScript:x"])
|
||||
def test_dangerous_links_are_dropped(href):
|
||||
assert unsub.find_unsubscribe_links(f'<a href="{href}">Unsubscribe</a>') == []
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Case store: one case per forwarded mail, keyed by Message-ID."""
|
||||
|
||||
from email.message import EmailMessage
|
||||
|
||||
import pytest
|
||||
|
||||
from cases import CaseStore, case_key
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path):
|
||||
return CaseStore(str(tmp_path / "cases.db"))
|
||||
|
||||
|
||||
def _msg(mid="<a@x>", subject="News"):
|
||||
m = EmailMessage()
|
||||
if mid:
|
||||
m["Message-ID"] = mid
|
||||
m["Subject"] = subject
|
||||
m["Date"] = "Sat, 14 Nov 2026 10:00:00 +0100"
|
||||
m["From"] = "philipp@wagnersnetz.de"
|
||||
return m
|
||||
|
||||
|
||||
def test_key_is_message_id_or_stable_hash():
|
||||
assert case_key(_msg("<a@x>")) == "<a@x>"
|
||||
k = case_key(_msg(""))
|
||||
assert k.startswith("sha256:") and k == case_key(_msg("")) and k != case_key(_msg("", "Other"))
|
||||
|
||||
|
||||
def test_add_is_idempotent_and_keeps_links(store):
|
||||
assert store.add("<a@x>", "News", ["https://e.org/u1", "https://e.org/u2"], "p@wagnersnetz.de") is True
|
||||
assert store.add("<a@x>", "changed", [], "") is False
|
||||
c = store.get("<a@x>")
|
||||
assert (c["subject"], c["links"], c["address"], c["state"], c["attempts"], c["moved"]) == (
|
||||
"News", ["https://e.org/u1", "https://e.org/u2"], "p@wagnersnetz.de", "new", 0, 0)
|
||||
|
||||
|
||||
def test_pending_only_untried_new_cases_oldest_first(store):
|
||||
for k in ("<1>", "<2>", "<3>"):
|
||||
store.add(k, k, ["https://e.org"], "")
|
||||
store.claim("<1>")
|
||||
store.finish("<1>", "unclear", "x")
|
||||
assert [c["key"] for c in store.pending(10)] == ["<2>", "<3>"]
|
||||
assert [c["key"] for c in store.pending(1)] == ["<2>"]
|
||||
|
||||
|
||||
def test_claim_counts_attempt_and_release_gives_it_back(store):
|
||||
store.add("<1>", "s", ["https://e.org"], "")
|
||||
store.claim("<1>")
|
||||
assert (store.get("<1>")["state"], store.get("<1>")["attempts"]) == ("running", 1)
|
||||
store.release("<1>")
|
||||
assert (store.get("<1>")["state"], store.get("<1>")["attempts"]) == ("new", 0)
|
||||
assert [c["key"] for c in store.pending(10)] == ["<1>"]
|
||||
|
||||
|
||||
def test_finish_stores_reason_and_evidence(store):
|
||||
store.add("<1>", "s", ["https://e.org"], "")
|
||||
store.claim("<1>")
|
||||
store.finish("<1>", "unsubscribed", "page confirms", "You have been unsubscribed", "https://e.org/done")
|
||||
c = store.get("<1>")
|
||||
assert (c["state"], c["reason"], c["evidence"], c["evidence_url"]) == (
|
||||
"unsubscribed", "page confirms", "You have been unsubscribed", "https://e.org/done")
|
||||
with pytest.raises(ValueError):
|
||||
store.finish("<1>", "great", "")
|
||||
|
||||
|
||||
def test_to_move_and_mark_moved(store):
|
||||
store.add("<1>", "s", ["https://e.org"], "")
|
||||
store.claim("<1>")
|
||||
store.finish("<1>", "unsubscribed")
|
||||
assert [c["key"] for c in store.to_move()] == ["<1>"]
|
||||
store.mark_moved("<1>")
|
||||
assert store.to_move() == []
|
||||
|
||||
|
||||
def test_retry_makes_a_failed_case_pending_again(store):
|
||||
store.add("<1>", "s", ["https://e.org"], "")
|
||||
store.claim("<1>")
|
||||
store.finish("<1>", "not_possible", "login")
|
||||
store.retry("<1>")
|
||||
c = store.get("<1>")
|
||||
assert (c["state"], c["attempts"], c["reason"]) == ("new", 0, "")
|
||||
|
||||
|
||||
def test_defer_gives_the_attempt_back_and_counts_deferrals(store):
|
||||
store.add("<1>", "s", ["https://e.org"], "")
|
||||
store.claim("<1>")
|
||||
assert store.defer("<1>") == 1
|
||||
c = store.get("<1>")
|
||||
assert (c["state"], c["attempts"], c["deferrals"]) == ("new", 0, 1)
|
||||
store.claim("<1>")
|
||||
assert store.defer("<1>") == 2
|
||||
|
||||
|
||||
def test_pending_puts_deferred_cases_behind_the_others(store):
|
||||
for k in ("<1>", "<2>"):
|
||||
store.add(k, k, ["https://e.org"], "")
|
||||
store.claim("<1>")
|
||||
store.defer("<1>")
|
||||
store.add("<3>", "<3>", ["https://e.org"], "")
|
||||
assert [c["key"] for c in store.pending(10)] == ["<2>", "<3>", "<1>"]
|
||||
|
||||
|
||||
def test_retry_resets_the_deferrals(store):
|
||||
store.add("<1>", "s", ["https://e.org"], "")
|
||||
store.claim("<1>")
|
||||
store.defer("<1>")
|
||||
store.claim("<1>")
|
||||
store.finish("<1>", "unclear", "model unavailable for this case")
|
||||
store.retry("<1>")
|
||||
assert (store.get("<1>")["state"], store.get("<1>")["deferrals"]) == ("new", 0)
|
||||
|
||||
|
||||
def test_database_without_the_deferrals_column_is_migrated(tmp_path):
|
||||
import sqlite3
|
||||
path = str(tmp_path / "old.db")
|
||||
db = sqlite3.connect(path)
|
||||
db.execute(
|
||||
"CREATE TABLE cases ("
|
||||
"key TEXT PRIMARY KEY, subject TEXT NOT NULL, links TEXT NOT NULL, address TEXT NOT NULL, "
|
||||
"state TEXT NOT NULL, reason TEXT NOT NULL DEFAULT '', evidence TEXT NOT NULL DEFAULT '', "
|
||||
"evidence_url TEXT NOT NULL DEFAULT '', attempts INTEGER NOT NULL DEFAULT 0, "
|
||||
"moved INTEGER NOT NULL DEFAULT 0, created_at REAL NOT NULL, updated_at REAL NOT NULL)")
|
||||
db.execute("INSERT INTO cases (key, subject, links, address, state, created_at, updated_at) "
|
||||
"VALUES ('<old>', 's', '[]', '', 'new', 1.0, 1.0)")
|
||||
db.commit()
|
||||
db.close()
|
||||
store = CaseStore(path)
|
||||
assert store.get("<old>")["deferrals"] == 0
|
||||
store.claim("<old>")
|
||||
assert store.defer("<old>") == 1
|
||||
CaseStore(path) # opening a migrated database again changes nothing
|
||||
assert store.get("<old>")["deferrals"] == 1
|
||||
|
||||
|
||||
def test_expired_only_moved_successes_older_than_days(store):
|
||||
for k in ("<old>", "<young>", "<failed>"):
|
||||
store.add(k, "s", ["https://e.org"], "")
|
||||
store.claim(k)
|
||||
store.finish("<old>", "unsubscribed")
|
||||
store.finish("<young>", "unsubscribed")
|
||||
store.finish("<failed>", "unclear")
|
||||
store.mark_moved("<old>")
|
||||
store.mark_moved("<young>")
|
||||
now = store.get("<old>")["updated_at"]
|
||||
assert [c["key"] for c in store.expired(30, now=now + 31 * 86400)] == ["<old>", "<young>"]
|
||||
assert store.expired(30, now=now + 29 * 86400) == []
|
||||
store.delete("<old>")
|
||||
assert store.get("<old>") is None
|
||||
|
||||
|
||||
def test_recover_turns_running_into_unclear(store):
|
||||
store.add("<1>", "s", ["https://e.org"], "")
|
||||
store.claim("<1>")
|
||||
assert store.recover() == 1
|
||||
c = store.get("<1>")
|
||||
assert (c["state"], c["reason"], c["attempts"]) == ("unclear", "interrupted by restart", 1)
|
||||
|
||||
|
||||
def test_meta_roundtrip(store):
|
||||
assert store.meta("paused", "0") == "0"
|
||||
store.set_meta("paused", "1")
|
||||
assert store.meta("paused") == "1"
|
||||
|
||||
|
||||
def test_key_copes_with_a_non_ascii_message_id():
|
||||
import email
|
||||
raw = "Message-ID: <näh@x>\r\nSubject: s\r\n\r\nbody".encode("utf-8")
|
||||
message = email.message_from_bytes(raw)
|
||||
key = case_key(message)
|
||||
assert isinstance(key, str) and key
|
||||
assert case_key(message) == key
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Case view and its actions (the store is real, the mailbox is faked)."""
|
||||
|
||||
import importlib
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsub(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATA_DIR", str(tmp_path))
|
||||
monkeypatch.setenv("AGENT_SCHEDULER", "0")
|
||||
monkeypatch.setenv("AGENT_URL", "http://agent.test:8080")
|
||||
monkeypatch.setenv("AGENT_ALLOWED_DOMAINS", "wagnersnetz.de, gmail.com")
|
||||
monkeypatch.setenv("AGENT_ALLOWED_SENDERS", "Jens@wagnersnetz.de, philipp@wagnersnetz.de")
|
||||
for name in ("OIDC_ISSUER", "OIDC_CLIENT_ID", "TOKEN_KEY"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
import app
|
||||
module = importlib.reload(app)
|
||||
yield module
|
||||
monkeypatch.delenv("DATA_DIR")
|
||||
monkeypatch.delenv("AGENT_URL")
|
||||
importlib.reload(app) # the other test modules expect the app without a store
|
||||
|
||||
|
||||
def test_settings_from_environment(unsub):
|
||||
assert unsub.STORE is not None and unsub.WORKER is not None
|
||||
assert unsub.WORKER.allowed_domains == ("wagnersnetz.de", "gmail.com")
|
||||
assert unsub.WORKER.allowed_senders == ("jens@wagnersnetz.de", "philipp@wagnersnetz.de")
|
||||
assert unsub.WORKER.agent is unsub.call_agent
|
||||
|
||||
|
||||
def test_result_shows_cases_with_state_reason_evidence_and_links(unsub, monkeypatch):
|
||||
s = unsub.STORE
|
||||
s.add("<ok@x>", "Fwd: Shop <b>News</b>", ["https://shop.example/u/1"], "p@wagnersnetz.de")
|
||||
s.claim("<ok@x>")
|
||||
s.finish("<ok@x>", "unsubscribed", "page confirms", "Sie wurden abgemeldet", "https://shop.example/done")
|
||||
s.add("<no@x>", "Fwd: Bank", ["https://bank.example/u"], "")
|
||||
s.claim("<no@x>")
|
||||
s.finish("<no@x>", "not_possible", "login")
|
||||
s.set_meta("last_run", json.dumps({"new": 2, "handled": 2, "unsubscribed": 1, "moved": 1, "cleaned": 0,
|
||||
"without_link": 3, "ignored": 2, "error": ""}))
|
||||
s.set_meta("last_run_at", "1800000000")
|
||||
monkeypatch.setattr(unsub, "scan_inbox", lambda: {"<no@x>": b"42"})
|
||||
html = unsub.app.test_client().get("/result").get_data(as_text=True)
|
||||
assert "Fwd: Shop <b>News</b>" in html and "<b>News</b>" not in html
|
||||
assert "unsubscribed" in html and "Sie wurden abgemeldet" in html and "https://shop.example/done" in html
|
||||
assert "not possible" in html and "login" in html and 'href="https://bank.example/u"' in html
|
||||
assert 'name="email_id" value="42"' in html # trash button only for mails still in the inbox
|
||||
assert html.count('action="/retry"') == 1 # only the failed case can be retried
|
||||
assert "3 mail(s) without unsubscribe link" in html
|
||||
assert "2 mail(s) from senders that are not allowed" in html
|
||||
|
||||
|
||||
def test_result_still_lists_cases_when_the_inbox_scan_fails(unsub, monkeypatch):
|
||||
from worker import ScanFailed
|
||||
s = unsub.STORE
|
||||
s.add("<no@x>", "Fwd: Bank", ["https://bank.example/u"], "")
|
||||
s.claim("<no@x>")
|
||||
s.finish("<no@x>", "not_possible", "login")
|
||||
|
||||
def failing():
|
||||
raise ScanFailed("inbox scan failed")
|
||||
monkeypatch.setattr(unsub, "scan_inbox", failing)
|
||||
response = unsub.app.test_client().get("/result")
|
||||
html = response.get_data(as_text=True)
|
||||
assert response.status_code == 200
|
||||
assert "Fwd: Bank" in html and "The inbox could not be read" in html
|
||||
assert 'name="email_id"' not in html and 'action="/retry"' not in html # nothing is known about the mails
|
||||
|
||||
|
||||
def test_max_deferrals_default_and_from_environment(unsub, monkeypatch):
|
||||
import app
|
||||
assert unsub.WORKER.max_deferrals == 12
|
||||
monkeypatch.setenv("AGENT_MAX_DEFERRALS", "3")
|
||||
assert importlib.reload(app).WORKER.max_deferrals == 3
|
||||
|
||||
|
||||
def test_result_shows_cases_waiting_for_the_model(unsub, monkeypatch):
|
||||
unsub.STORE.set_meta("last_run", json.dumps({"new": 0, "deferred": 2, "error": ""}))
|
||||
monkeypatch.setattr(unsub, "scan_inbox", lambda: {})
|
||||
html = unsub.app.test_client().get("/result").get_data(as_text=True)
|
||||
assert "2 case(s) waiting for the model." in html
|
||||
|
||||
|
||||
def test_retry_only_for_a_case_with_its_mail_in_the_inbox(unsub, monkeypatch):
|
||||
s = unsub.STORE
|
||||
for key in ("<here@x>", "<gone@x>"):
|
||||
s.add(key, key, ["https://bank.example/u"], "")
|
||||
s.claim(key)
|
||||
s.finish(key, "not_possible", "login")
|
||||
monkeypatch.setattr(unsub, "scan_inbox", lambda: {"<here@x>": b"42"})
|
||||
html = unsub.app.test_client().get("/result").get_data(as_text=True)
|
||||
assert html.count('action="/retry"') == 1 and 'name="key" value="<here@x>"' in html
|
||||
assert 'name="key" value="<gone@x>"' not in html
|
||||
|
||||
|
||||
def test_unmoved_success_shows_not_moved_yet(unsub, monkeypatch):
|
||||
s = unsub.STORE
|
||||
for key in ("<unmoved@x>", "<moved@x>"):
|
||||
s.add(key, key, ["https://shop.example/u"], "")
|
||||
s.claim(key)
|
||||
s.finish(key, "unsubscribed", "page confirms")
|
||||
s.mark_moved("<moved@x>")
|
||||
monkeypatch.setattr(unsub, "scan_inbox", lambda: {"<unmoved@x>": b"7"})
|
||||
html = unsub.app.test_client().get("/result").get_data(as_text=True)
|
||||
assert html.count("not moved yet") == 1
|
||||
row = html[html.index("<unmoved@x>"):]
|
||||
assert row.index("not moved yet") < row.index("</tr>")
|
||||
|
||||
|
||||
def test_retry_resets_the_case(unsub):
|
||||
s = unsub.STORE
|
||||
s.add("<no@x>", "s", ["https://bank.example/u"], "")
|
||||
s.claim("<no@x>")
|
||||
s.finish("<no@x>", "unclear", "x")
|
||||
r = unsub.app.test_client().post("/retry", data={"key": "<no@x>"})
|
||||
assert r.status_code == 302 and s.get("<no@x>")["state"] == "new" and s.get("<no@x>")["attempts"] == 0
|
||||
|
||||
|
||||
def test_pause_and_resume(unsub):
|
||||
c = unsub.app.test_client()
|
||||
c.post("/pause", data={"paused": "1"})
|
||||
assert unsub.STORE.meta("paused") == "1"
|
||||
c.post("/pause", data={"paused": "0"})
|
||||
assert unsub.STORE.meta("paused") == "0"
|
||||
|
||||
|
||||
def test_run_now_starts_a_background_run_and_returns_at_once(unsub, monkeypatch):
|
||||
calls, threads = [], []
|
||||
real_start_run = unsub.start_run
|
||||
|
||||
def keep_thread():
|
||||
threads.append(real_start_run())
|
||||
return threads[-1]
|
||||
|
||||
monkeypatch.setattr(unsub, "start_run", keep_thread)
|
||||
monkeypatch.setattr(unsub.WORKER, "run_once", lambda: calls.append(1) or {"error": ""})
|
||||
assert unsub.app.test_client().post("/run-now").status_code == 302
|
||||
threads[0].join(timeout=5)
|
||||
assert calls == [1]
|
||||
|
||||
def boom():
|
||||
raise RuntimeError("secret mail content")
|
||||
|
||||
monkeypatch.setattr(unsub.WORKER, "run_once", boom)
|
||||
assert unsub.app.test_client().post("/run-now").status_code == 302
|
||||
threads[1].join(timeout=5)
|
||||
assert json.loads(unsub.STORE.meta("last_run"))["error"] == "RuntimeError"
|
||||
|
||||
|
||||
def test_call_agent_maps_failures_to_unavailable(unsub, monkeypatch):
|
||||
import httpx
|
||||
|
||||
def boom(*a, **kw):
|
||||
raise httpx.ConnectError("refused")
|
||||
|
||||
monkeypatch.setattr(unsub.httpx, "post", boom)
|
||||
with pytest.raises(unsub.AgentUnavailable):
|
||||
unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de")
|
||||
monkeypatch.setattr(unsub.httpx, "post", lambda *a, **kw: httpx.Response(500, text="x", request=httpx.Request("POST", a[0])))
|
||||
with pytest.raises(unsub.AgentUnavailable):
|
||||
unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de")
|
||||
|
||||
|
||||
def test_call_agent_sends_only_url_and_address(unsub, monkeypatch):
|
||||
import httpx
|
||||
seen = {}
|
||||
|
||||
def post(url, json=None, timeout=None):
|
||||
seen.update(url=url, json=json, timeout=timeout)
|
||||
return httpx.Response(200, json={"result": "unclear", "reason": "x"}, request=httpx.Request("POST", url))
|
||||
|
||||
monkeypatch.setattr(unsub.httpx, "post", post)
|
||||
assert unsub.call_agent("https://shop.example/u", "p@wagnersnetz.de")["result"] == "unclear"
|
||||
assert seen["url"] == "http://agent.test:8080/run"
|
||||
assert seen["json"] == {"url": "https://shop.example/u", "address": "p@wagnersnetz.de"}
|
||||
assert seen["timeout"] > 600 # two links, 5 minutes each
|
||||
|
||||
|
||||
def test_without_data_dir_the_old_list_is_shown():
|
||||
import app
|
||||
assert app.STORE is None and app.WORKER is None
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Which address did the newsletter go to? Taken from the quoted header of a forwarded mail."""
|
||||
|
||||
import pytest
|
||||
|
||||
from forwarded import original_recipient
|
||||
|
||||
ALLOWED = ("wagnersnetz.de", "gmail.com")
|
||||
|
||||
|
||||
def test_german_forward_header():
|
||||
text = "---------- Weitergeleitete Nachricht ---------\nVon: News <news@shop.example>\nAn: Philipp <philipp@wagnersnetz.de>\nBetreff: Angebot\n"
|
||||
assert original_recipient(text, "jens@wagnersnetz.de", ALLOWED) == "philipp@wagnersnetz.de"
|
||||
|
||||
|
||||
def test_english_forward_header_in_html():
|
||||
html = "<div>From: news@shop.example<br>To: <a href='mailto:Some.One@Gmail.com'>Some.One@Gmail.com</a><br>Subject: x</div>"
|
||||
assert original_recipient(html, "philipp@wagnersnetz.de", ALLOWED) == "some.one@gmail.com"
|
||||
|
||||
|
||||
def test_quoted_prefix_and_several_recipients_first_allowed_wins():
|
||||
text = "> To: other@elsewhere.example, anne@wagnersnetz.de\n"
|
||||
assert original_recipient(text, "", ALLOWED) == "anne@wagnersnetz.de"
|
||||
|
||||
|
||||
def test_falls_back_to_forwarder():
|
||||
assert original_recipient("no header here", "Jens <jens@wagnersnetz.de>", ALLOWED) == "jens@wagnersnetz.de"
|
||||
|
||||
|
||||
def test_foreign_domain_is_never_used():
|
||||
text = "To: victim@elsewhere.example\n"
|
||||
assert original_recipient(text, "someone@elsewhere.example", ALLOWED) == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("domain", ["evilwagnersnetz.de", "wagnersnetz.de.evil.example"])
|
||||
def test_domain_must_match_exactly(domain):
|
||||
assert original_recipient(f"To: x@{domain}\n", "", ALLOWED) == ""
|
||||
|
||||
|
||||
def test_subdomain_is_not_allowed_implicitly():
|
||||
assert original_recipient("To: x@mail.wagnersnetz.de\n", "", ALLOWED) == ""
|
||||
|
||||
|
||||
def test_malformed_forwarder_is_not_used():
|
||||
assert original_recipient("no header", "a@b@wagnersnetz.de", ALLOWED) == ""
|
||||
assert original_recipient("no header", "<x y@wagnersnetz.de>", ALLOWED) == ""
|
||||
|
||||
|
||||
def test_full_address_entry_allows_only_that_address():
|
||||
allowed = ("wagnersnetz.de", "Some.One@Gmail.com")
|
||||
assert original_recipient("To: some.one@gmail.com\n", "", allowed) == "some.one@gmail.com"
|
||||
assert original_recipient("To: SOME.ONE@GMAIL.COM\n", "", allowed) == "some.one@gmail.com"
|
||||
assert original_recipient("To: someone.else@gmail.com\n", "", allowed) == ""
|
||||
|
||||
|
||||
def test_body_line_with_a_third_persons_public_address_is_refused():
|
||||
text = ("Von: news@shop.example\nAn: philipp@gmail.com\n\nWrite to us!\nTo: someone@gmail.com\n")
|
||||
allowed = ("wagnersnetz.de", "philipp@gmail.com")
|
||||
assert original_recipient("To: someone@gmail.com\n", "philipp@gmail.com", allowed) == "philipp@gmail.com"
|
||||
assert original_recipient(text, "", allowed) == "philipp@gmail.com"
|
||||
|
||||
|
||||
def test_mixed_list_of_domains_and_addresses():
|
||||
allowed = ("wagnersnetz.de", "anne@gmail.com")
|
||||
assert original_recipient("To: x@elsewhere.example, jens@wagnersnetz.de\n", "", allowed) == "jens@wagnersnetz.de"
|
||||
assert original_recipient("To: bob@gmail.com, anne@gmail.com\n", "", allowed) == "anne@gmail.com"
|
||||
assert original_recipient("no header", "anne@gmail.com", allowed) == "anne@gmail.com"
|
||||
assert original_recipient("no header", "bob@gmail.com", allowed) == ""
|
||||
|
||||
|
||||
def test_address_entry_does_not_allow_its_domain_or_a_lookalike():
|
||||
allowed = ("anne@gmail.com",)
|
||||
assert original_recipient("To: gmail.com@gmail.com\n", "", allowed) == ""
|
||||
assert original_recipient("To: xanne@gmail.com\n", "", allowed) == ""
|
||||
assert original_recipient("To: anne@gmail.com.evil.example\n", "", allowed) == ""
|
||||
@@ -0,0 +1,519 @@
|
||||
"""Scheduler run: find cases, hand them to the agent, move successes, clean up (fake IMAP, fake agent)."""
|
||||
|
||||
import json
|
||||
import time
|
||||
from email.message import EmailMessage
|
||||
|
||||
import pytest
|
||||
|
||||
from cases import CaseStore
|
||||
from cases import case_key
|
||||
from worker import AgentUnavailable, Worker, _tick
|
||||
|
||||
|
||||
def mail_bytes(mid, link="https://shop.example/unsubscribe/1", to="philipp@wagnersnetz.de", links=None,
|
||||
sender="Jens <jens@wagnersnetz.de>"):
|
||||
m = EmailMessage()
|
||||
m["Message-ID"] = mid
|
||||
m["Subject"] = "Fwd: News " + mid
|
||||
m["From"] = sender
|
||||
urls = links if links is not None else ([link] if link else [])
|
||||
m.set_content("Von: news@shop.example\nAn: " + to + "\n\n" + "\n".join(urls))
|
||||
return m.as_bytes()
|
||||
|
||||
|
||||
class FakeIMAP:
|
||||
"""Two folders; uid commands as far as the worker uses them."""
|
||||
|
||||
def __init__(self, inbox):
|
||||
self.folders = {"INBOX": dict(inbox)}
|
||||
self.selected = "INBOX"
|
||||
self.deleted = set()
|
||||
self.fail_copy = False
|
||||
self.logged_out = False
|
||||
self.fail_store_once = False
|
||||
self.search_status = "OK"
|
||||
self.fail_fetch = set()
|
||||
self.calls = []
|
||||
|
||||
def select(self, folder):
|
||||
if folder not in self.folders:
|
||||
return "NO", [b"no such folder"]
|
||||
self.selected = folder
|
||||
return "OK", [b"1"]
|
||||
|
||||
def create(self, folder):
|
||||
if folder in self.folders:
|
||||
return "NO", [b"exists"]
|
||||
self.folders[folder] = {}
|
||||
return "OK", [b"created"]
|
||||
|
||||
def uid(self, command, *args):
|
||||
self.calls.append((command, args))
|
||||
box = self.folders[self.selected]
|
||||
if command == "search":
|
||||
if len(args) >= 4 and args[1] == "HEADER":
|
||||
wanted = args[3].strip('"')
|
||||
hits = [u for u, raw in box.items() if ("Message-ID: " + wanted).encode() in raw]
|
||||
return "OK", [b" ".join(hits)]
|
||||
if self.search_status != "OK":
|
||||
return self.search_status, [b"search failed"]
|
||||
return "OK", [b" ".join(box)]
|
||||
if command == "fetch":
|
||||
if args[0] in self.fail_fetch:
|
||||
return "NO", [None]
|
||||
return "OK", [(b"1 (BODY[] {n}", box[args[0]]), b")"]
|
||||
if command == "copy":
|
||||
if self.fail_copy or args[0] not in box:
|
||||
return "NO", [b"copy failed"]
|
||||
self.folders[args[1]][b"9" + args[0]] = box[args[0]]
|
||||
return "OK", [b""]
|
||||
if command == "store":
|
||||
if self.fail_store_once:
|
||||
self.fail_store_once = False
|
||||
return "NO", [b"store failed"]
|
||||
self.deleted.add((self.selected, args[0]))
|
||||
return "OK", [b""]
|
||||
raise AssertionError(command)
|
||||
|
||||
def expunge(self):
|
||||
for folder, uid in list(self.deleted):
|
||||
self.folders[folder].pop(uid, None)
|
||||
self.deleted.clear()
|
||||
return "OK", [b""]
|
||||
|
||||
def logout(self):
|
||||
self.logged_out = True
|
||||
|
||||
|
||||
def parse(message):
|
||||
import app as unsub
|
||||
text = unsub.get_text_from_email(message)
|
||||
return unsub.get_subject(message), unsub.find_unsubscribe_links(text), text
|
||||
|
||||
|
||||
class Agent:
|
||||
def __init__(self, *results):
|
||||
self.results = list(results)
|
||||
self.calls = []
|
||||
|
||||
def __call__(self, url, address):
|
||||
self.calls.append((url, address))
|
||||
r = self.results.pop(0) if self.results else {"result": "unclear", "reason": "no more results"}
|
||||
if isinstance(r, Exception):
|
||||
raise r
|
||||
return {"evidence": "", "evidence_url": "", "steps": 1, "stage": 2, **r}
|
||||
|
||||
|
||||
OK = {"result": "unsubscribed", "reason": "page confirms", "evidence": "abgemeldet", "evidence_url": "https://shop.example/done"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path):
|
||||
return CaseStore(str(tmp_path / "cases.db"))
|
||||
|
||||
|
||||
def worker(store, imap, agent, **kw):
|
||||
kw.setdefault("allowed_senders", ("jens@wagnersnetz.de",))
|
||||
return Worker(store, lambda: imap, parse, agent, allowed_domains=("wagnersnetz.de",), **kw)
|
||||
|
||||
|
||||
def test_success_is_moved_to_the_done_folder(store):
|
||||
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
|
||||
agent = Agent(OK)
|
||||
summary = worker(store, imap, agent).run_once()
|
||||
assert agent.calls == [("https://shop.example/unsubscribe/1", "philipp@wagnersnetz.de")]
|
||||
assert (summary["new"], summary["handled"], summary["unsubscribed"], summary["moved"], summary["error"]) == (1, 1, 1, 1, "")
|
||||
assert imap.folders["INBOX"] == {} and len(imap.folders["Erledigt"]) == 1 and imap.logged_out
|
||||
c = store.get("<a@x>")
|
||||
assert (c["state"], c["moved"], c["evidence"]) == ("unsubscribed", 1, "abgemeldet")
|
||||
assert json.loads(store.meta("last_run"))["moved"] == 1 and store.meta("last_run_at")
|
||||
|
||||
|
||||
def test_failure_stays_in_the_inbox_and_is_not_tried_again(store):
|
||||
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
|
||||
agent = Agent({"result": "not_possible", "reason": "captcha"})
|
||||
w = worker(store, imap, agent)
|
||||
w.run_once()
|
||||
w.run_once()
|
||||
assert len(agent.calls) == 1 and b"7" in imap.folders["INBOX"]
|
||||
assert (store.get("<a@x>")["state"], store.get("<a@x>")["reason"]) == ("not_possible", "captcha")
|
||||
|
||||
|
||||
def test_second_link_is_tried_but_not_a_third_and_not_after_login(store):
|
||||
links = ["https://shop.example/unsubscribe/1", "https://shop.example/unsubscribe/2", "https://shop.example/unsubscribe/3"]
|
||||
imap = FakeIMAP({b"1": mail_bytes("<two@x>", links=links), b"2": mail_bytes("<login@x>", links=links)})
|
||||
agent = Agent({"result": "unclear", "reason": "x"}, OK, {"result": "not_possible", "reason": "login"})
|
||||
worker(store, imap, agent).run_once()
|
||||
assert [u for u, _ in agent.calls] == [links[0], links[1], links[0]]
|
||||
assert store.get("<two@x>")["state"] == "unsubscribed"
|
||||
assert (store.get("<login@x>")["state"], store.get("<login@x>")["reason"]) == ("not_possible", "login")
|
||||
|
||||
|
||||
def test_agent_error_does_not_burn_the_attempt(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
|
||||
agent = Agent(AgentUnavailable("agent not reachable"))
|
||||
w = worker(store, imap, agent)
|
||||
summary = w.run_once()
|
||||
assert summary["error"] == "agent not reachable" and summary["handled"] == 0 and len(agent.calls) == 1
|
||||
assert [(c["state"], c["attempts"]) for c in store.all()] == [("new", 0), ("new", 0)]
|
||||
agent.results = [OK, OK]
|
||||
assert w.run_once()["unsubscribed"] == 2
|
||||
|
||||
|
||||
def test_model_unavailable_result_counts_like_an_unavailable_agent(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
|
||||
summary = worker(store, imap, Agent({"result": "error", "reason": "llm_unavailable"})).run_once()
|
||||
assert summary["error"] == "llm_unavailable" and store.get("<a@x>")["attempts"] == 0
|
||||
|
||||
|
||||
ERR = {"result": "error", "reason": "llm_unavailable"}
|
||||
|
||||
|
||||
def test_model_error_defers_the_case_and_the_run_goes_on(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>", link="https://shop.example/unsubscribe/a"),
|
||||
b"2": mail_bytes("<b@x>", link="https://shop.example/unsubscribe/b")})
|
||||
agent = Agent(ERR, OK)
|
||||
summary = worker(store, imap, agent).run_once()
|
||||
first, second = store.get("<a@x>"), store.get("<b@x>")
|
||||
assert second["state"] == "unsubscribed"
|
||||
assert (first["state"], first["attempts"], first["deferrals"]) == ("new", 0, 1)
|
||||
assert summary["error"] == "" and summary["deferred"] == 1 and summary["handled"] == 1
|
||||
|
||||
|
||||
def test_deferred_case_is_tried_after_newer_ones(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>", link="https://shop.example/unsubscribe/a")})
|
||||
agent = Agent(ERR)
|
||||
w = worker(store, imap, agent)
|
||||
assert w.run_once()["deferred"] == 1
|
||||
imap.folders["INBOX"][b"2"] = mail_bytes("<b@x>", link="https://shop.example/unsubscribe/b")
|
||||
agent.calls, agent.results = [], [OK, OK]
|
||||
w.run_once()
|
||||
assert [u for u, _ in agent.calls] == ["https://shop.example/unsubscribe/b", "https://shop.example/unsubscribe/a"]
|
||||
|
||||
|
||||
def test_case_deferred_max_deferrals_times_ends_unclear(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
|
||||
agent = Agent(ERR, ERR, ERR, OK)
|
||||
w = worker(store, imap, agent, max_deferrals=3)
|
||||
summaries = [w.run_once() for _ in range(4)]
|
||||
c = store.get("<a@x>")
|
||||
assert (c["state"], c["reason"]) == ("unclear", "model unavailable for this case")
|
||||
assert len(agent.calls) == 3 and summaries[3]["handled"] == 0
|
||||
assert [s["deferred"] for s in summaries[:2]] == [1, 1]
|
||||
|
||||
|
||||
def test_transport_failure_still_stops_the_run_and_leaves_all_cases_untouched(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
|
||||
agent = Agent(AgentUnavailable("agent not reachable"), OK)
|
||||
summary = worker(store, imap, agent).run_once()
|
||||
assert summary["error"] == "agent not reachable" and summary["deferred"] == 0 and len(agent.calls) == 1
|
||||
assert [(c["state"], c["attempts"], c["deferrals"]) for c in store.all()] == [("new", 0, 0), ("new", 0, 0)]
|
||||
|
||||
|
||||
def test_without_agent_cases_are_only_recorded(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<nolink@x>", link="")})
|
||||
summary = worker(store, imap, None).run_once()
|
||||
assert (summary["new"], summary["handled"], summary["without_link"]) == (1, 0, 1)
|
||||
assert store.get("<a@x>")["state"] == "new" and store.get("<nolink@x>") is None
|
||||
|
||||
|
||||
def test_paused_run_does_nothing(store):
|
||||
store.set_meta("paused", "1")
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
|
||||
agent = Agent(OK)
|
||||
summary = worker(store, imap, agent).run_once()
|
||||
assert summary["error"] == "paused" and agent.calls == [] and store.all() == []
|
||||
|
||||
|
||||
def test_pause_during_a_run_stops_before_the_next_case(store):
|
||||
imap = FakeIMAP({str(i).encode(): mail_bytes(f"<{i}@x>") for i in range(1, 4)})
|
||||
|
||||
class Pausing(Agent):
|
||||
def __call__(self, url, address):
|
||||
store.set_meta("paused", "1") # "Pause agent" pressed while the first case runs
|
||||
return super().__call__(url, address)
|
||||
|
||||
agent = Pausing(OK, OK, OK)
|
||||
summary = worker(store, imap, agent).run_once()
|
||||
assert len(agent.calls) == 1 and summary["error"] == "paused"
|
||||
assert summary["handled"] == 1 and summary["moved"] == 1 # what is finished is still moved
|
||||
assert [c["state"] for c in store.all()].count("new") == 2
|
||||
|
||||
|
||||
def test_at_most_max_cases_per_run(store):
|
||||
imap = FakeIMAP({str(i).encode(): mail_bytes(f"<{i}@x>") for i in range(1, 5)})
|
||||
agent = Agent(OK, OK, OK, OK)
|
||||
assert worker(store, imap, agent, max_cases=3).run_once()["handled"] == 3
|
||||
assert len(agent.calls) == 3
|
||||
|
||||
|
||||
def test_address_of_a_foreign_domain_is_not_handed_over(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>", to="victim@elsewhere.example")})
|
||||
agent = Agent(OK)
|
||||
worker(store, imap, agent).run_once()
|
||||
assert agent.calls == [("https://shop.example/unsubscribe/1", "jens@wagnersnetz.de")] # the forwarder
|
||||
|
||||
|
||||
def test_move_failure_keeps_case(store):
|
||||
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
|
||||
imap.fail_copy = True
|
||||
w = worker(store, imap, Agent(OK))
|
||||
summary = w.run_once()
|
||||
assert summary["moved"] == 0 and b"7" in imap.folders["INBOX"]
|
||||
assert (store.get("<a@x>")["state"], store.get("<a@x>")["moved"]) == ("unsubscribed", 0)
|
||||
imap.fail_copy = False
|
||||
assert w.run_once()["moved"] == 1 and imap.folders["INBOX"] == {}
|
||||
|
||||
|
||||
def test_cleanup_deletes_old_successes_from_the_done_folder(store):
|
||||
imap = FakeIMAP({b"7": mail_bytes("<old@x>"), b"8": mail_bytes("<young@x>")})
|
||||
now = [time.time()] # store stamps real time; a fixed start date would make the test date-dependent
|
||||
w = worker(store, imap, Agent(OK, OK), clock=lambda: now[0], retention_days=30)
|
||||
w.run_once()
|
||||
assert len(imap.folders["Erledigt"]) == 2
|
||||
young_time = store.get("<young@x>")["updated_at"]
|
||||
store._run("UPDATE cases SET updated_at = ? WHERE key = '<old@x>'", (young_time - 31 * 86400,))
|
||||
now[0] = young_time + 60
|
||||
summary = w.run_once()
|
||||
assert summary["cleaned"] == 1
|
||||
assert store.get("<old@x>") is None and store.get("<young@x>") is not None
|
||||
assert len(imap.folders["Erledigt"]) == 1 and imap.selected == "INBOX"
|
||||
|
||||
|
||||
def test_run_survives_a_broken_mail(store):
|
||||
imap = FakeIMAP({b"1": b"\xff\xfe not a mail", b"2": mail_bytes("<b@x>")})
|
||||
assert worker(store, imap, Agent(OK)).run_once()["unsubscribed"] == 1
|
||||
|
||||
|
||||
def test_mail_from_unknown_sender_is_ignored(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender="Jens <JENS@wagnersnetz.de>"),
|
||||
b"2": mail_bytes("<evil@x>", sender="Mallory <mallory@evil.example>")})
|
||||
agent = Agent(OK)
|
||||
summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once()
|
||||
assert len(agent.calls) == 1 and summary["ignored"] == 1
|
||||
assert [c["key"] for c in store.all()] == ["<a@x>"]
|
||||
assert store.get("<evil@x>") is None and b"2" in imap.folders["INBOX"]
|
||||
|
||||
|
||||
def test_no_allowed_senders_with_agent_means_nobody(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
|
||||
agent = Agent(OK)
|
||||
summary = worker(store, imap, agent, allowed_senders=()).run_once()
|
||||
assert agent.calls == [] and store.all() == [] and summary["ignored"] == 1
|
||||
|
||||
|
||||
def test_no_allowed_senders_without_agent_lists_everything(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
|
||||
summary = worker(store, imap, None, allowed_senders=()).run_once()
|
||||
assert store.get("<a@x>")["state"] == "new" and summary["ignored"] == 0
|
||||
|
||||
|
||||
def test_spoofed_display_name_does_not_help(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender='"jens@wagnersnetz.de" <mallory@evil.example>')})
|
||||
agent = Agent(OK)
|
||||
summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once()
|
||||
assert agent.calls == [] and store.all() == [] and summary["ignored"] == 1
|
||||
|
||||
|
||||
def old_case(store, key, address="jens@wagnersnetz.de"):
|
||||
store.add(key, "Fwd: old", ["https://shop.example/unsubscribe/1"], address)
|
||||
|
||||
|
||||
def test_old_case_of_a_sender_no_longer_allowed_is_not_handled(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender="Mallory <mallory@evil.example>")})
|
||||
worker(store, imap, None, allowed_senders=()).run_once()
|
||||
assert store.get("<a@x>")["state"] == "new"
|
||||
agent = Agent(OK)
|
||||
summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once()
|
||||
assert agent.calls == [] and summary["ignored"] == 1 and summary["handled"] == 0
|
||||
assert (store.get("<a@x>")["state"], store.get("<a@x>")["attempts"]) == ("new", 0)
|
||||
|
||||
|
||||
def test_case_whose_mail_left_the_inbox_is_not_handled(store):
|
||||
old_case(store, "<gone@x>")
|
||||
agent = Agent(OK)
|
||||
worker(store, FakeIMAP({}), agent).run_once()
|
||||
assert agent.calls == []
|
||||
|
||||
|
||||
def test_failed_case_whose_mail_was_moved_to_the_trash_disappears(store):
|
||||
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
|
||||
w = worker(store, imap, Agent({"result": "not_possible", "reason": "captcha"}))
|
||||
w.run_once()
|
||||
assert store.get("<a@x>")["state"] == "not_possible"
|
||||
imap.folders["INBOX"].pop(b"7") # "Move to Trash" on the page
|
||||
summary = w.run_once()
|
||||
assert store.get("<a@x>") is None and summary["gone"] == 1
|
||||
|
||||
|
||||
def test_new_case_whose_mail_is_gone_is_deleted_but_a_running_one_is_left_alone(store):
|
||||
old_case(store, "<new@x>")
|
||||
old_case(store, "<running@x>")
|
||||
store.claim("<running@x>")
|
||||
summary = worker(store, FakeIMAP({}), Agent()).run_once()
|
||||
assert store.get("<new@x>") is None and store.get("<running@x>")["state"] == "running"
|
||||
assert summary["gone"] == 1
|
||||
|
||||
|
||||
def test_unmoved_success_whose_mail_is_gone_counts_as_moved_and_expires(store):
|
||||
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
|
||||
imap.fail_copy = True
|
||||
now = [time.time()]
|
||||
w = worker(store, imap, Agent(OK), clock=lambda: now[0], retention_days=30)
|
||||
w.run_once()
|
||||
assert (store.get("<a@x>")["state"], store.get("<a@x>")["moved"]) == ("unsubscribed", 0)
|
||||
imap.folders["INBOX"].pop(b"7") # moved by hand
|
||||
summary = w.run_once()
|
||||
assert store.get("<a@x>")["moved"] == 1 and summary["gone"] == 1
|
||||
imap.folders["Erledigt"] = {}
|
||||
now[0] = store.get("<a@x>")["updated_at"] + 31 * 86400
|
||||
assert w.run_once()["cleaned"] == 1 and store.get("<a@x>") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["NO", "BAD"])
|
||||
def test_failed_inbox_search_ends_the_run_with_an_error_and_deletes_nothing(store, status):
|
||||
old_case(store, "<a@x>")
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
|
||||
imap.search_status = status
|
||||
agent = Agent(OK)
|
||||
summary = worker(store, imap, agent).run_once()
|
||||
assert summary["error"] == "inbox scan failed" and summary["gone"] == 0 and agent.calls == []
|
||||
assert store.get("<a@x>")["state"] == "new" and imap.logged_out
|
||||
assert json.loads(store.meta("last_run"))["error"] == "inbox scan failed"
|
||||
|
||||
|
||||
def test_case_added_by_the_page_during_the_scan_is_not_tidied_away(store):
|
||||
def parse_while_the_page_scans(message):
|
||||
store.add("<arrived@x>", "s", ["https://shop.example/unsubscribe/9"], "") # page scan saw a newer mail
|
||||
return parse(message)
|
||||
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
|
||||
w = Worker(store, lambda: imap, parse_while_the_page_scans, None, allowed_domains=("wagnersnetz.de",))
|
||||
assert w.run_once()["gone"] == 0 and store.get("<arrived@x>") is not None
|
||||
|
||||
|
||||
def test_unreadable_mail_in_the_inbox_means_nothing_is_tidied(store):
|
||||
old_case(store, "<a@x>")
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
|
||||
imap.fail_fetch = {b"1"} # could be the mail of <a@x>: its key is unknown
|
||||
summary = worker(store, imap, Agent(OK, OK)).run_once()
|
||||
assert store.get("<a@x>") is not None and summary["gone"] == 0
|
||||
assert store.get("<b@x>")["state"] == "unsubscribed"
|
||||
|
||||
|
||||
def test_case_of_a_mail_from_a_sender_no_longer_allowed_is_kept(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender="Mallory <mallory@evil.example>")})
|
||||
worker(store, imap, None, allowed_senders=()).run_once()
|
||||
summary = worker(store, imap, Agent(OK), allowed_senders=("jens@wagnersnetz.de",)).run_once()
|
||||
assert store.get("<a@x>")["state"] == "new" and summary["gone"] == 0
|
||||
|
||||
|
||||
def test_skipped_cases_do_not_eat_the_budget(store):
|
||||
for i in range(3):
|
||||
old_case(store, f"<gone{i}@x>")
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
|
||||
agent = Agent(OK, OK, OK)
|
||||
assert worker(store, imap, agent, max_cases=2).run_once()["handled"] == 2
|
||||
assert len(agent.calls) == 2 and store.get("<a@x>")["state"] == "unsubscribed"
|
||||
|
||||
|
||||
def test_agent_bug_ends_the_case_unclear_and_the_run_goes_on(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
|
||||
summary = worker(store, imap, Agent(RuntimeError("boom"), OK)).run_once()
|
||||
first, second = store.get("<a@x>"), store.get("<b@x>")
|
||||
assert (first["state"], first["reason"], first["attempts"]) == ("unclear", "agent error", 1)
|
||||
assert second["state"] == "unsubscribed"
|
||||
assert summary["handled"] == 2 and summary["error"] == ""
|
||||
|
||||
|
||||
class RawAgent:
|
||||
def __init__(self, result):
|
||||
self.result = result
|
||||
|
||||
def __call__(self, url, address):
|
||||
return self.result
|
||||
|
||||
|
||||
@pytest.mark.parametrize("result", [{}, {"result": "bought"}, None, "text"])
|
||||
def test_malformed_agent_result_ends_the_case_unclear(store, result):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
|
||||
summary = worker(store, imap, RawAgent(result)).run_once()
|
||||
c = store.get("<a@x>")
|
||||
assert (c["state"], c["reason"], c["attempts"]) == ("unclear", "agent error", 1)
|
||||
assert summary["handled"] == 1 and summary["error"] == ""
|
||||
|
||||
|
||||
def test_exception_while_handling_one_mail_does_not_stop_the_run(store):
|
||||
raw = ("Message-ID: <broken@x>\nSubject: broken\nFrom: jens@wagnersnetz.de\n\n"
|
||||
"Von: news@shop.example\nAn: philipp@wagnersnetz.de\n\nhttps://shop.example/unsubscribe/9\n").encode("utf-8")
|
||||
|
||||
def parse_failing_on_one(message):
|
||||
result = parse(message)
|
||||
if result[0] == "broken":
|
||||
raise ValueError("cannot handle this mail")
|
||||
return result
|
||||
|
||||
imap = FakeIMAP({b"1": raw, b"2": mail_bytes("<b@x>")})
|
||||
w = Worker(store, lambda: imap, parse_failing_on_one, Agent(OK), allowed_domains=("wagnersnetz.de",),
|
||||
allowed_senders=("jens@wagnersnetz.de",))
|
||||
summary = w.run_once()
|
||||
assert store.get("<broken@x>") is None
|
||||
assert store.get("<b@x>")["state"] == "unsubscribed" and summary["error"] == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", ["<a\r\nb@x>", "<a@x>\\", '<a"b@x>', "<n\u00e4h@x>"])
|
||||
def test_cleanup_never_sends_a_hostile_message_id(store, key):
|
||||
old_case(store, key)
|
||||
store.claim(key)
|
||||
store.finish(key, "unsubscribed")
|
||||
store.mark_moved(key)
|
||||
store._run("UPDATE cases SET updated_at = ? WHERE key = ?", (1.0, key))
|
||||
imap = FakeIMAP({})
|
||||
imap.folders["Erledigt"] = {}
|
||||
summary = worker(store, imap, Agent()).run_once()
|
||||
assert summary["cleaned"] == 1 and store.get(key) is None
|
||||
assert not any(cmd == "search" and key in " ".join(map(str, args)) for cmd, args in imap.calls)
|
||||
|
||||
|
||||
def test_scan_from_a_request_does_not_change_a_runs_counters(store):
|
||||
w = worker(store, FakeIMAP({}), Agent())
|
||||
result = w.scan(FakeIMAP({b"1": mail_bytes("<a@x>")}))
|
||||
assert len(result) == 2 and not hasattr(w, "_created") and not hasattr(w, "_ignored")
|
||||
|
||||
|
||||
def test_failed_delete_flag_keeps_the_case_unmoved(store):
|
||||
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
|
||||
imap.fail_store_once = True
|
||||
w = worker(store, imap, Agent(OK))
|
||||
assert w.run_once()["moved"] == 0
|
||||
assert store.get("<a@x>")["moved"] == 0 and b"7" in imap.folders["INBOX"]
|
||||
assert w.run_once()["moved"] == 1 and store.get("<a@x>")["moved"] == 1
|
||||
|
||||
|
||||
ALL_KEYS = {"new", "handled", "unsubscribed", "moved", "cleaned", "without_link", "ignored", "error", "deferred", "gone"}
|
||||
|
||||
|
||||
class Boom:
|
||||
def __init__(self, store, set_meta_fails=False):
|
||||
self.store, self.fail = store, set_meta_fails
|
||||
self.clock = lambda: 1.0
|
||||
|
||||
def run_once(self):
|
||||
raise ValueError("secret subject line")
|
||||
|
||||
|
||||
class BrokenStore:
|
||||
def set_meta(self, k, v):
|
||||
raise OSError("disk")
|
||||
|
||||
|
||||
def test_tick_records_only_the_exception_type(store):
|
||||
_tick(Boom(store))
|
||||
last = json.loads(store.meta("last_run"))
|
||||
assert set(last) == ALL_KEYS and last["error"] == "ValueError"
|
||||
assert "secret subject line" not in store.meta("last_run")
|
||||
|
||||
|
||||
def test_tick_survives_a_failing_store():
|
||||
_tick(Boom(BrokenStore()))
|
||||
Reference in new issue
Block a user