Zeitlauf: Befunde aus dem Review behoben
This commit is contained in:
2 files changed
+193
-27
No files matched your search
+132
-1
@@ -7,7 +7,8 @@ from email.message import EmailMessage
|
||||
import pytest
|
||||
|
||||
from cases import CaseStore
|
||||
from worker import AgentUnavailable, Worker
|
||||
from cases import case_key
|
||||
from worker import AgentUnavailable, Worker, _tick
|
||||
|
||||
|
||||
def mail_bytes(mid, link="https://shop.example/unsubscribe/1", to="philipp@wagnersnetz.de", links=None,
|
||||
@@ -30,6 +31,8 @@ class FakeIMAP:
|
||||
self.deleted = set()
|
||||
self.fail_copy = False
|
||||
self.logged_out = False
|
||||
self.fail_store_once = False
|
||||
self.calls = []
|
||||
|
||||
def select(self, folder):
|
||||
if folder not in self.folders:
|
||||
@@ -44,6 +47,7 @@ class FakeIMAP:
|
||||
return "OK", [b"created"]
|
||||
|
||||
def uid(self, command, *args):
|
||||
self.calls.append((command, args))
|
||||
box = self.folders[self.selected]
|
||||
if command == "search":
|
||||
if len(args) >= 4 and args[1] == "HEADER":
|
||||
@@ -59,6 +63,9 @@ class FakeIMAP:
|
||||
self.folders[args[1]][b"9" + args[0]] = box[args[0]]
|
||||
return "OK", [b""]
|
||||
if command == "store":
|
||||
if self.fail_store_once:
|
||||
self.fail_store_once = False
|
||||
return "NO", [b"store failed"]
|
||||
self.deleted.add((self.selected, args[0]))
|
||||
return "OK", [b""]
|
||||
raise AssertionError(command)
|
||||
@@ -242,3 +249,127 @@ def test_spoofed_display_name_does_not_help(store):
|
||||
agent = Agent(OK)
|
||||
summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once()
|
||||
assert agent.calls == [] and store.all() == [] and summary["ignored"] == 1
|
||||
|
||||
|
||||
def old_case(store, key, address="jens@wagnersnetz.de"):
|
||||
store.add(key, "Fwd: old", ["https://shop.example/unsubscribe/1"], address)
|
||||
|
||||
|
||||
def test_old_case_of_a_sender_no_longer_allowed_is_not_handled(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender="Mallory <mallory@evil.example>")})
|
||||
worker(store, imap, None, allowed_senders=()).run_once()
|
||||
assert store.get("<a@x>")["state"] == "new"
|
||||
agent = Agent(OK)
|
||||
summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once()
|
||||
assert agent.calls == [] and summary["ignored"] == 1 and summary["handled"] == 0
|
||||
assert (store.get("<a@x>")["state"], store.get("<a@x>")["attempts"]) == ("new", 0)
|
||||
|
||||
|
||||
def test_case_whose_mail_left_the_inbox_is_not_handled(store):
|
||||
old_case(store, "<gone@x>")
|
||||
agent = Agent(OK)
|
||||
worker(store, FakeIMAP({}), agent).run_once()
|
||||
assert agent.calls == []
|
||||
|
||||
|
||||
def test_skipped_cases_do_not_eat_the_budget(store):
|
||||
for i in range(3):
|
||||
old_case(store, f"<gone{i}@x>")
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
|
||||
agent = Agent(OK, OK, OK)
|
||||
assert worker(store, imap, agent, max_cases=2).run_once()["handled"] == 2
|
||||
assert len(agent.calls) == 2 and store.get("<a@x>")["state"] == "unsubscribed"
|
||||
|
||||
|
||||
def test_agent_bug_ends_the_case_unclear_and_the_run_goes_on(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
|
||||
summary = worker(store, imap, Agent(RuntimeError("boom"), OK)).run_once()
|
||||
first, second = store.get("<a@x>"), store.get("<b@x>")
|
||||
assert (first["state"], first["reason"], first["attempts"]) == ("unclear", "agent error", 1)
|
||||
assert second["state"] == "unsubscribed"
|
||||
assert summary["handled"] == 2 and summary["error"] == ""
|
||||
|
||||
|
||||
class RawAgent:
|
||||
def __init__(self, result):
|
||||
self.result = result
|
||||
|
||||
def __call__(self, url, address):
|
||||
return self.result
|
||||
|
||||
|
||||
@pytest.mark.parametrize("result", [{}, {"result": "bought"}, None, "text"])
|
||||
def test_malformed_agent_result_ends_the_case_unclear(store, result):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
|
||||
summary = worker(store, imap, RawAgent(result)).run_once()
|
||||
c = store.get("<a@x>")
|
||||
assert (c["state"], c["reason"], c["attempts"]) == ("unclear", "agent error", 1)
|
||||
assert summary["handled"] == 1 and summary["error"] == ""
|
||||
|
||||
|
||||
def test_mail_with_non_ascii_message_id_does_not_stop_the_run(store):
|
||||
raw = ("Message-ID: <näh@x>\nSubject: Fwd: x\nFrom: jens@wagnersnetz.de\n\n"
|
||||
"Von: news@shop.example\nAn: philipp@wagnersnetz.de\n\nhttps://shop.example/unsubscribe/9\n").encode("utf-8")
|
||||
import email
|
||||
with pytest.raises(Exception):
|
||||
case_key(email.message_from_bytes(raw)) # proves the mail really is a poison pill
|
||||
imap = FakeIMAP({b"1": raw, b"2": mail_bytes("<b@x>")})
|
||||
summary = worker(store, imap, Agent(OK, OK)).run_once()
|
||||
assert store.get("<b@x>")["state"] == "unsubscribed" and summary["error"] == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", ["<a\r\nb@x>", "<a@x>\\", '<a"b@x>', "<n\u00e4h@x>"])
|
||||
def test_cleanup_never_sends_a_hostile_message_id(store, key):
|
||||
old_case(store, key)
|
||||
store.claim(key)
|
||||
store.finish(key, "unsubscribed")
|
||||
store.mark_moved(key)
|
||||
store._run("UPDATE cases SET updated_at = ? WHERE key = ?", (1.0, key))
|
||||
imap = FakeIMAP({})
|
||||
imap.folders["Erledigt"] = {}
|
||||
summary = worker(store, imap, Agent()).run_once()
|
||||
assert summary["cleaned"] == 1 and store.get(key) is None
|
||||
assert not any(cmd == "search" and key in " ".join(map(str, args)) for cmd, args in imap.calls)
|
||||
|
||||
|
||||
def test_scan_from_a_request_does_not_change_a_runs_counters(store):
|
||||
w = worker(store, FakeIMAP({}), Agent())
|
||||
result = w.scan(FakeIMAP({b"1": mail_bytes("<a@x>")}))
|
||||
assert len(result) == 2 and not hasattr(w, "_created") and not hasattr(w, "_ignored")
|
||||
|
||||
|
||||
def test_failed_delete_flag_keeps_the_case_unmoved(store):
|
||||
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
|
||||
imap.fail_store_once = True
|
||||
w = worker(store, imap, Agent(OK))
|
||||
assert w.run_once()["moved"] == 0
|
||||
assert store.get("<a@x>")["moved"] == 0 and b"7" in imap.folders["INBOX"]
|
||||
assert w.run_once()["moved"] == 1 and store.get("<a@x>")["moved"] == 1
|
||||
|
||||
|
||||
ALL_KEYS = {"new", "handled", "unsubscribed", "moved", "cleaned", "without_link", "ignored", "error"}
|
||||
|
||||
|
||||
class Boom:
|
||||
def __init__(self, store, set_meta_fails=False):
|
||||
self.store, self.fail = store, set_meta_fails
|
||||
self.clock = lambda: 1.0
|
||||
|
||||
def run_once(self):
|
||||
raise ValueError("secret subject line")
|
||||
|
||||
|
||||
class BrokenStore:
|
||||
def set_meta(self, k, v):
|
||||
raise OSError("disk")
|
||||
|
||||
|
||||
def test_tick_records_only_the_exception_type(store):
|
||||
_tick(Boom(store))
|
||||
last = json.loads(store.meta("last_run"))
|
||||
assert set(last) == ALL_KEYS and last["error"] == "ValueError"
|
||||
assert "secret subject line" not in store.meta("last_run")
|
||||
|
||||
|
||||
def test_tick_survives_a_failing_store():
|
||||
_tick(Boom(BrokenStore()))
|
||||
@@ -13,6 +13,13 @@ from forwarded import original_recipient
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_LINKS = 2
|
||||
FINAL_RESULTS = ("unsubscribed", "not_possible", "unclear", "error")
|
||||
SUMMARY_KEYS = ("new", "handled", "unsubscribed", "moved", "cleaned", "without_link", "ignored", "error")
|
||||
|
||||
|
||||
def _searchable(key: str) -> bool:
|
||||
"""Only plain printable ASCII without quote and backslash may go into an IMAP SEARCH command."""
|
||||
return bool(key) and all(0x20 <= ord(c) <= 0x7E and c not in '"\\' for c in key)
|
||||
|
||||
|
||||
class AgentUnavailable(Exception):
|
||||
@@ -31,30 +38,34 @@ class Worker:
|
||||
|
||||
def scan(self, mail) -> tuple[dict, int]:
|
||||
"""Create a case for every mail with unsubscribe links. Returns ({key: uid}, mails without link)."""
|
||||
return self._scan(mail)[:2]
|
||||
|
||||
def _scan(self, mail) -> tuple[dict, int, int, int]:
|
||||
"""Like scan, plus the number of created cases and of ignored mails (no instance state)."""
|
||||
_, data = mail.uid("search", None, "ALL")
|
||||
uids, without, self._created, self._ignored = {}, 0, 0, 0
|
||||
uids, without, created, ignored = {}, 0, 0, 0
|
||||
for uid in data[0].split():
|
||||
try:
|
||||
_, fetched = mail.uid("fetch", uid, "(BODY.PEEK[])")
|
||||
message = email.message_from_bytes(fetched[0][1])
|
||||
subject, links, text = self.parse(message)
|
||||
if not links:
|
||||
without += 1
|
||||
continue
|
||||
forwarder = parseaddr(message.get("From", ""))[1]
|
||||
if (self.allowed_senders and forwarder.lower() not in self.allowed_senders) or (
|
||||
not self.allowed_senders and self.agent is not None): # fail closed
|
||||
ignored += 1
|
||||
continue
|
||||
key = case_key(message)
|
||||
recipient = original_recipient(text, forwarder, self.allowed_domains)
|
||||
except Exception as e: # one broken mail must not stop the run
|
||||
logger.warning("mail %s skipped: %s", uid, type(e).__name__)
|
||||
continue
|
||||
if not links:
|
||||
without += 1
|
||||
continue
|
||||
forwarder = parseaddr(message.get("From", ""))[1]
|
||||
sender = forwarder.lower()
|
||||
if (self.allowed_senders and sender not in self.allowed_senders) or (
|
||||
not self.allowed_senders and self.agent is not None): # fail closed
|
||||
self._ignored += 1
|
||||
continue
|
||||
key = case_key(message)
|
||||
uids[key] = uid
|
||||
if self.store.add(key, subject, links, original_recipient(text, forwarder, self.allowed_domains)):
|
||||
self._created += 1
|
||||
return uids, without
|
||||
if self.store.add(key, subject, links, recipient):
|
||||
created += 1
|
||||
return uids, without, created, ignored
|
||||
|
||||
def _handle(self, case: dict) -> str:
|
||||
"""Run one case through the agent. Returns the final state; raises AgentUnavailable."""
|
||||
@@ -63,6 +74,8 @@ class Worker:
|
||||
try:
|
||||
for link in case["links"][:MAX_LINKS]:
|
||||
result = self.agent(link, case["address"])
|
||||
if result["result"] not in FINAL_RESULTS:
|
||||
raise ValueError("malformed agent result")
|
||||
if result["result"] == "error":
|
||||
raise AgentUnavailable(result.get("reason", "agent error"))
|
||||
if result["result"] == "unsubscribed" or result.get("reason") == "login":
|
||||
@@ -70,6 +83,10 @@ class Worker:
|
||||
except AgentUnavailable:
|
||||
self.store.release(case["key"])
|
||||
raise
|
||||
except Exception as e: # agent bug or malformed result: the attempt counts, the run goes on
|
||||
logger.warning("agent failed on a case: %s", type(e).__name__)
|
||||
self.store.finish(case["key"], "unclear", "agent error")
|
||||
return "unclear"
|
||||
self.store.finish(case["key"], result["result"], result.get("reason", ""), result.get("evidence", ""),
|
||||
result.get("evidence_url", ""))
|
||||
return result["result"]
|
||||
@@ -86,7 +103,10 @@ class Worker:
|
||||
if status != "OK":
|
||||
logger.warning("could not move a mail to %s", self.done_folder)
|
||||
continue
|
||||
mail.uid("store", uid, "+FLAGS", "(\\Deleted)")
|
||||
status, _ = mail.uid("store", uid, "+FLAGS", "(\\Deleted)")
|
||||
if status != "OK":
|
||||
logger.warning("could not flag a copied mail as deleted")
|
||||
continue
|
||||
self.store.mark_moved(case["key"])
|
||||
moved += 1
|
||||
mail.expunge()
|
||||
@@ -101,8 +121,9 @@ class Worker:
|
||||
return 0
|
||||
try:
|
||||
for case in expired:
|
||||
if not case["key"].startswith("sha256:"): # mails without Message-ID cannot be found again
|
||||
_, data = mail.uid("search", None, "HEADER", "Message-ID", '"' + case["key"].replace('"', "") + '"')
|
||||
key = case["key"]
|
||||
if not key.startswith("sha256:") and _searchable(key): # others cannot be found (or not safely)
|
||||
_, data = mail.uid("search", None, "HEADER", "Message-ID", '"' + key + '"')
|
||||
for uid in data[0].split():
|
||||
mail.uid("store", uid, "+FLAGS", "(\\Deleted)")
|
||||
self.store.delete(case["key"])
|
||||
@@ -122,10 +143,13 @@ class Worker:
|
||||
return self._note(summary)
|
||||
mail = self.connect()
|
||||
try:
|
||||
uids, summary["without_link"] = self.scan(mail)
|
||||
summary["new"], summary["ignored"] = self._created, self._ignored
|
||||
uids, summary["without_link"], summary["new"], summary["ignored"] = self._scan(mail)
|
||||
if self.agent is not None:
|
||||
for case in self.store.pending(self.max_cases):
|
||||
for case in self.store.pending(10_000):
|
||||
if summary["handled"] >= self.max_cases:
|
||||
break
|
||||
if case["key"] not in uids: # only cases this scan vouched for (mail there, sender allowed)
|
||||
continue
|
||||
try:
|
||||
state = self._handle(case)
|
||||
except AgentUnavailable as e:
|
||||
@@ -147,17 +171,28 @@ class Worker:
|
||||
return summary
|
||||
|
||||
|
||||
def _tick(worker: Worker) -> None:
|
||||
"""One scheduled run. Never raises; notes only the exception type (messages can contain mail content)."""
|
||||
try:
|
||||
worker.run_once()
|
||||
except Exception as e:
|
||||
name = type(e).__name__
|
||||
logger.warning("scheduled run failed: %s", name)
|
||||
try:
|
||||
summary = {k: 0 for k in SUMMARY_KEYS}
|
||||
summary["error"] = name
|
||||
worker.store.set_meta("last_run", json.dumps(summary))
|
||||
worker.store.set_meta("last_run_at", str(int(worker.clock())))
|
||||
except Exception as e2:
|
||||
logger.warning("could not note the failed run: %s", type(e2).__name__)
|
||||
|
||||
|
||||
def start_scheduler(worker: Worker, interval_seconds: int) -> threading.Thread:
|
||||
"""Background thread: one run every interval. Errors (mailbox not connected, IMAP down) are noted, not raised."""
|
||||
|
||||
def loop():
|
||||
while True:
|
||||
try:
|
||||
worker.run_once()
|
||||
except Exception as e:
|
||||
logger.warning("scheduled run failed: %s: %s", type(e).__name__, e)
|
||||
worker.store.set_meta("last_run", json.dumps({"error": f"{type(e).__name__}: {e}"[:200]}))
|
||||
worker.store.set_meta("last_run_at", str(int(worker.clock())))
|
||||
_tick(worker)
|
||||
time.sleep(interval_seconds)
|
||||
|
||||
thread = threading.Thread(target=loop, name="unsubscribe-scheduler", daemon=True)
|
||||
|
||||
Reference in new issue
Block a user