diff --git a/tests/test_worker.py b/tests/test_worker.py index 0aaf28d..9b72e48 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -7,7 +7,8 @@ from email.message import EmailMessage import pytest from cases import CaseStore -from worker import AgentUnavailable, Worker +from cases import case_key +from worker import AgentUnavailable, Worker, _tick def mail_bytes(mid, link="https://shop.example/unsubscribe/1", to="philipp@wagnersnetz.de", links=None, @@ -30,6 +31,8 @@ class FakeIMAP: self.deleted = set() self.fail_copy = False self.logged_out = False + self.fail_store_once = False + self.calls = [] def select(self, folder): if folder not in self.folders: @@ -44,6 +47,7 @@ class FakeIMAP: return "OK", [b"created"] def uid(self, command, *args): + self.calls.append((command, args)) box = self.folders[self.selected] if command == "search": if len(args) >= 4 and args[1] == "HEADER": @@ -59,6 +63,9 @@ class FakeIMAP: self.folders[args[1]][b"9" + args[0]] = box[args[0]] return "OK", [b""] if command == "store": + if self.fail_store_once: + self.fail_store_once = False + return "NO", [b"store failed"] self.deleted.add((self.selected, args[0])) return "OK", [b""] raise AssertionError(command) @@ -242,3 +249,127 @@ def test_spoofed_display_name_does_not_help(store): agent = Agent(OK) summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once() assert agent.calls == [] and store.all() == [] and summary["ignored"] == 1 + + +def old_case(store, key, address="jens@wagnersnetz.de"): + store.add(key, "Fwd: old", ["https://shop.example/unsubscribe/1"], address) + + +def test_old_case_of_a_sender_no_longer_allowed_is_not_handled(store): + imap = FakeIMAP({b"1": mail_bytes("", sender="Mallory ")}) + worker(store, imap, None, allowed_senders=()).run_once() + assert store.get("")["state"] == "new" + agent = Agent(OK) + summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once() + assert agent.calls == [] and summary["ignored"] == 1 and summary["handled"] == 0 + assert (store.get("")["state"], store.get("")["attempts"]) == ("new", 0) + + +def test_case_whose_mail_left_the_inbox_is_not_handled(store): + old_case(store, "") + agent = Agent(OK) + worker(store, FakeIMAP({}), agent).run_once() + assert agent.calls == [] + + +def test_skipped_cases_do_not_eat_the_budget(store): + for i in range(3): + old_case(store, f"") + imap = FakeIMAP({b"1": mail_bytes(""), b"2": mail_bytes("")}) + agent = Agent(OK, OK, OK) + assert worker(store, imap, agent, max_cases=2).run_once()["handled"] == 2 + assert len(agent.calls) == 2 and store.get("")["state"] == "unsubscribed" + + +def test_agent_bug_ends_the_case_unclear_and_the_run_goes_on(store): + imap = FakeIMAP({b"1": mail_bytes(""), b"2": mail_bytes("")}) + summary = worker(store, imap, Agent(RuntimeError("boom"), OK)).run_once() + first, second = store.get(""), store.get("") + assert (first["state"], first["reason"], first["attempts"]) == ("unclear", "agent error", 1) + assert second["state"] == "unsubscribed" + assert summary["handled"] == 2 and summary["error"] == "" + + +class RawAgent: + def __init__(self, result): + self.result = result + + def __call__(self, url, address): + return self.result + + +@pytest.mark.parametrize("result", [{}, {"result": "bought"}, None, "text"]) +def test_malformed_agent_result_ends_the_case_unclear(store, result): + imap = FakeIMAP({b"1": mail_bytes("")}) + summary = worker(store, imap, RawAgent(result)).run_once() + c = store.get("") + assert (c["state"], c["reason"], c["attempts"]) == ("unclear", "agent error", 1) + assert summary["handled"] == 1 and summary["error"] == "" + + +def test_mail_with_non_ascii_message_id_does_not_stop_the_run(store): + raw = ("Message-ID: \nSubject: Fwd: x\nFrom: jens@wagnersnetz.de\n\n" + "Von: news@shop.example\nAn: philipp@wagnersnetz.de\n\nhttps://shop.example/unsubscribe/9\n").encode("utf-8") + import email + with pytest.raises(Exception): + case_key(email.message_from_bytes(raw)) # proves the mail really is a poison pill + imap = FakeIMAP({b"1": raw, b"2": mail_bytes("")}) + summary = worker(store, imap, Agent(OK, OK)).run_once() + assert store.get("")["state"] == "unsubscribed" and summary["error"] == "" + + +@pytest.mark.parametrize("key", ["", "\\", '', ""]) +def test_cleanup_never_sends_a_hostile_message_id(store, key): + old_case(store, key) + store.claim(key) + store.finish(key, "unsubscribed") + store.mark_moved(key) + store._run("UPDATE cases SET updated_at = ? WHERE key = ?", (1.0, key)) + imap = FakeIMAP({}) + imap.folders["Erledigt"] = {} + summary = worker(store, imap, Agent()).run_once() + assert summary["cleaned"] == 1 and store.get(key) is None + assert not any(cmd == "search" and key in " ".join(map(str, args)) for cmd, args in imap.calls) + + +def test_scan_from_a_request_does_not_change_a_runs_counters(store): + w = worker(store, FakeIMAP({}), Agent()) + result = w.scan(FakeIMAP({b"1": mail_bytes("")})) + assert len(result) == 2 and not hasattr(w, "_created") and not hasattr(w, "_ignored") + + +def test_failed_delete_flag_keeps_the_case_unmoved(store): + imap = FakeIMAP({b"7": mail_bytes("")}) + imap.fail_store_once = True + w = worker(store, imap, Agent(OK)) + assert w.run_once()["moved"] == 0 + assert store.get("")["moved"] == 0 and b"7" in imap.folders["INBOX"] + assert w.run_once()["moved"] == 1 and store.get("")["moved"] == 1 + + +ALL_KEYS = {"new", "handled", "unsubscribed", "moved", "cleaned", "without_link", "ignored", "error"} + + +class Boom: + def __init__(self, store, set_meta_fails=False): + self.store, self.fail = store, set_meta_fails + self.clock = lambda: 1.0 + + def run_once(self): + raise ValueError("secret subject line") + + +class BrokenStore: + def set_meta(self, k, v): + raise OSError("disk") + + +def test_tick_records_only_the_exception_type(store): + _tick(Boom(store)) + last = json.loads(store.meta("last_run")) + assert set(last) == ALL_KEYS and last["error"] == "ValueError" + assert "secret subject line" not in store.meta("last_run") + + +def test_tick_survives_a_failing_store(): + _tick(Boom(BrokenStore())) diff --git a/worker.py b/worker.py index eae0dd7..5e9601d 100644 --- a/worker.py +++ b/worker.py @@ -13,6 +13,13 @@ from forwarded import original_recipient logger = logging.getLogger(__name__) MAX_LINKS = 2 +FINAL_RESULTS = ("unsubscribed", "not_possible", "unclear", "error") +SUMMARY_KEYS = ("new", "handled", "unsubscribed", "moved", "cleaned", "without_link", "ignored", "error") + + +def _searchable(key: str) -> bool: + """Only plain printable ASCII without quote and backslash may go into an IMAP SEARCH command.""" + return bool(key) and all(0x20 <= ord(c) <= 0x7E and c not in '"\\' for c in key) class AgentUnavailable(Exception): @@ -31,30 +38,34 @@ class Worker: def scan(self, mail) -> tuple[dict, int]: """Create a case for every mail with unsubscribe links. Returns ({key: uid}, mails without link).""" + return self._scan(mail)[:2] + + def _scan(self, mail) -> tuple[dict, int, int, int]: + """Like scan, plus the number of created cases and of ignored mails (no instance state).""" _, data = mail.uid("search", None, "ALL") - uids, without, self._created, self._ignored = {}, 0, 0, 0 + uids, without, created, ignored = {}, 0, 0, 0 for uid in data[0].split(): try: _, fetched = mail.uid("fetch", uid, "(BODY.PEEK[])") message = email.message_from_bytes(fetched[0][1]) subject, links, text = self.parse(message) + if not links: + without += 1 + continue + forwarder = parseaddr(message.get("From", ""))[1] + if (self.allowed_senders and forwarder.lower() not in self.allowed_senders) or ( + not self.allowed_senders and self.agent is not None): # fail closed + ignored += 1 + continue + key = case_key(message) + recipient = original_recipient(text, forwarder, self.allowed_domains) except Exception as e: # one broken mail must not stop the run logger.warning("mail %s skipped: %s", uid, type(e).__name__) continue - if not links: - without += 1 - continue - forwarder = parseaddr(message.get("From", ""))[1] - sender = forwarder.lower() - if (self.allowed_senders and sender not in self.allowed_senders) or ( - not self.allowed_senders and self.agent is not None): # fail closed - self._ignored += 1 - continue - key = case_key(message) uids[key] = uid - if self.store.add(key, subject, links, original_recipient(text, forwarder, self.allowed_domains)): - self._created += 1 - return uids, without + if self.store.add(key, subject, links, recipient): + created += 1 + return uids, without, created, ignored def _handle(self, case: dict) -> str: """Run one case through the agent. Returns the final state; raises AgentUnavailable.""" @@ -63,6 +74,8 @@ class Worker: try: for link in case["links"][:MAX_LINKS]: result = self.agent(link, case["address"]) + if result["result"] not in FINAL_RESULTS: + raise ValueError("malformed agent result") if result["result"] == "error": raise AgentUnavailable(result.get("reason", "agent error")) if result["result"] == "unsubscribed" or result.get("reason") == "login": @@ -70,6 +83,10 @@ class Worker: except AgentUnavailable: self.store.release(case["key"]) raise + except Exception as e: # agent bug or malformed result: the attempt counts, the run goes on + logger.warning("agent failed on a case: %s", type(e).__name__) + self.store.finish(case["key"], "unclear", "agent error") + return "unclear" self.store.finish(case["key"], result["result"], result.get("reason", ""), result.get("evidence", ""), result.get("evidence_url", "")) return result["result"] @@ -86,7 +103,10 @@ class Worker: if status != "OK": logger.warning("could not move a mail to %s", self.done_folder) continue - mail.uid("store", uid, "+FLAGS", "(\\Deleted)") + status, _ = mail.uid("store", uid, "+FLAGS", "(\\Deleted)") + if status != "OK": + logger.warning("could not flag a copied mail as deleted") + continue self.store.mark_moved(case["key"]) moved += 1 mail.expunge() @@ -101,8 +121,9 @@ class Worker: return 0 try: for case in expired: - if not case["key"].startswith("sha256:"): # mails without Message-ID cannot be found again - _, data = mail.uid("search", None, "HEADER", "Message-ID", '"' + case["key"].replace('"', "") + '"') + key = case["key"] + if not key.startswith("sha256:") and _searchable(key): # others cannot be found (or not safely) + _, data = mail.uid("search", None, "HEADER", "Message-ID", '"' + key + '"') for uid in data[0].split(): mail.uid("store", uid, "+FLAGS", "(\\Deleted)") self.store.delete(case["key"]) @@ -122,10 +143,13 @@ class Worker: return self._note(summary) mail = self.connect() try: - uids, summary["without_link"] = self.scan(mail) - summary["new"], summary["ignored"] = self._created, self._ignored + uids, summary["without_link"], summary["new"], summary["ignored"] = self._scan(mail) if self.agent is not None: - for case in self.store.pending(self.max_cases): + for case in self.store.pending(10_000): + if summary["handled"] >= self.max_cases: + break + if case["key"] not in uids: # only cases this scan vouched for (mail there, sender allowed) + continue try: state = self._handle(case) except AgentUnavailable as e: @@ -147,17 +171,28 @@ class Worker: return summary +def _tick(worker: Worker) -> None: + """One scheduled run. Never raises; notes only the exception type (messages can contain mail content).""" + try: + worker.run_once() + except Exception as e: + name = type(e).__name__ + logger.warning("scheduled run failed: %s", name) + try: + summary = {k: 0 for k in SUMMARY_KEYS} + summary["error"] = name + worker.store.set_meta("last_run", json.dumps(summary)) + worker.store.set_meta("last_run_at", str(int(worker.clock()))) + except Exception as e2: + logger.warning("could not note the failed run: %s", type(e2).__name__) + + def start_scheduler(worker: Worker, interval_seconds: int) -> threading.Thread: """Background thread: one run every interval. Errors (mailbox not connected, IMAP down) are noted, not raised.""" def loop(): while True: - try: - worker.run_once() - except Exception as e: - logger.warning("scheduled run failed: %s: %s", type(e).__name__, e) - worker.store.set_meta("last_run", json.dumps({"error": f"{type(e).__name__}: {e}"[:200]})) - worker.store.set_meta("last_run_at", str(int(worker.clock()))) + _tick(worker) time.sleep(interval_seconds) thread = threading.Thread(target=loop, name="unsubscribe-scheduler", daemon=True)