From 65c09d96e87fb660e4d459321fce3c366c55d94d Mon Sep 17 00:00:00 2001 From: Philipp Wagner Date: Sun, 15 Nov 2026 00:40:20 +0100 Subject: [PATCH] =?UTF-8?q?Zeitlauf:=20F=C3=A4lle=20finden,=20an=20den=20A?= =?UTF-8?q?genten=20geben,=20verschieben,=20aufr=C3=A4umen?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_worker.py | 244 +++++++++++++++++++++++++++++++++++++++++++ worker.py | 165 +++++++++++++++++++++++++++++ 2 files changed, 409 insertions(+) create mode 100644 tests/test_worker.py create mode 100644 worker.py diff --git a/tests/test_worker.py b/tests/test_worker.py new file mode 100644 index 0000000..0aaf28d --- /dev/null +++ b/tests/test_worker.py @@ -0,0 +1,244 @@ +"""Scheduler run: find cases, hand them to the agent, move successes, clean up (fake IMAP, fake agent).""" + +import json +import time +from email.message import EmailMessage + +import pytest + +from cases import CaseStore +from worker import AgentUnavailable, Worker + + +def mail_bytes(mid, link="https://shop.example/unsubscribe/1", to="philipp@wagnersnetz.de", links=None, + sender="Jens "): + m = EmailMessage() + m["Message-ID"] = mid + m["Subject"] = "Fwd: News " + mid + m["From"] = sender + urls = links if links is not None else ([link] if link else []) + m.set_content("Von: news@shop.example\nAn: " + to + "\n\n" + "\n".join(urls)) + return m.as_bytes() + + +class FakeIMAP: + """Two folders; uid commands as far as the worker uses them.""" + + def __init__(self, inbox): + self.folders = {"INBOX": dict(inbox)} + self.selected = "INBOX" + self.deleted = set() + self.fail_copy = False + self.logged_out = False + + def select(self, folder): + if folder not in self.folders: + return "NO", [b"no such folder"] + self.selected = folder + return "OK", [b"1"] + + def create(self, folder): + if folder in self.folders: + return "NO", [b"exists"] + self.folders[folder] = {} + return "OK", [b"created"] + + def uid(self, command, *args): + box = self.folders[self.selected] + if command == "search": + if len(args) >= 4 and args[1] == "HEADER": + wanted = args[3].strip('"') + hits = [u for u, raw in box.items() if ("Message-ID: " + wanted).encode() in raw] + return "OK", [b" ".join(hits)] + return "OK", [b" ".join(box)] + if command == "fetch": + return "OK", [(b"1 (BODY[] {n}", box[args[0]]), b")"] + if command == "copy": + if self.fail_copy or args[0] not in box: + return "NO", [b"copy failed"] + self.folders[args[1]][b"9" + args[0]] = box[args[0]] + return "OK", [b""] + if command == "store": + self.deleted.add((self.selected, args[0])) + return "OK", [b""] + raise AssertionError(command) + + def expunge(self): + for folder, uid in list(self.deleted): + self.folders[folder].pop(uid, None) + self.deleted.clear() + return "OK", [b""] + + def logout(self): + self.logged_out = True + + +def parse(message): + import app as unsub + text = unsub.get_text_from_email(message) + return unsub.get_subject(message), unsub.find_unsubscribe_links(text), text + + +class Agent: + def __init__(self, *results): + self.results = list(results) + self.calls = [] + + def __call__(self, url, address): + self.calls.append((url, address)) + r = self.results.pop(0) if self.results else {"result": "unclear", "reason": "no more results"} + if isinstance(r, Exception): + raise r + return {"evidence": "", "evidence_url": "", "steps": 1, "stage": 2, **r} + + +OK = {"result": "unsubscribed", "reason": "page confirms", "evidence": "abgemeldet", "evidence_url": "https://shop.example/done"} + + +@pytest.fixture +def store(tmp_path): + return CaseStore(str(tmp_path / "cases.db")) + + +def worker(store, imap, agent, **kw): + kw.setdefault("allowed_senders", ("jens@wagnersnetz.de",)) + return Worker(store, lambda: imap, parse, agent, allowed_domains=("wagnersnetz.de",), **kw) + + +def test_success_is_moved_to_the_done_folder(store): + imap = FakeIMAP({b"7": mail_bytes("")}) + agent = Agent(OK) + summary = worker(store, imap, agent).run_once() + assert agent.calls == [("https://shop.example/unsubscribe/1", "philipp@wagnersnetz.de")] + assert (summary["new"], summary["handled"], summary["unsubscribed"], summary["moved"], summary["error"]) == (1, 1, 1, 1, "") + assert imap.folders["INBOX"] == {} and len(imap.folders["Erledigt"]) == 1 and imap.logged_out + c = store.get("") + assert (c["state"], c["moved"], c["evidence"]) == ("unsubscribed", 1, "abgemeldet") + assert json.loads(store.meta("last_run"))["moved"] == 1 and store.meta("last_run_at") + + +def test_failure_stays_in_the_inbox_and_is_not_tried_again(store): + imap = FakeIMAP({b"7": mail_bytes("")}) + agent = Agent({"result": "not_possible", "reason": "captcha"}) + w = worker(store, imap, agent) + w.run_once() + w.run_once() + assert len(agent.calls) == 1 and b"7" in imap.folders["INBOX"] + assert (store.get("")["state"], store.get("")["reason"]) == ("not_possible", "captcha") + + +def test_second_link_is_tried_but_not_a_third_and_not_after_login(store): + links = ["https://shop.example/unsubscribe/1", "https://shop.example/unsubscribe/2", "https://shop.example/unsubscribe/3"] + imap = FakeIMAP({b"1": mail_bytes("", links=links), b"2": mail_bytes("", links=links)}) + agent = Agent({"result": "unclear", "reason": "x"}, OK, {"result": "not_possible", "reason": "login"}) + worker(store, imap, agent).run_once() + assert [u for u, _ in agent.calls] == [links[0], links[1], links[0]] + assert store.get("")["state"] == "unsubscribed" + assert (store.get("")["state"], store.get("")["reason"]) == ("not_possible", "login") + + +def test_agent_error_does_not_burn_the_attempt(store): + imap = FakeIMAP({b"1": mail_bytes(""), b"2": mail_bytes("")}) + agent = Agent(AgentUnavailable("agent not reachable")) + w = worker(store, imap, agent) + summary = w.run_once() + assert summary["error"] == "agent not reachable" and summary["handled"] == 0 and len(agent.calls) == 1 + assert [(c["state"], c["attempts"]) for c in store.all()] == [("new", 0), ("new", 0)] + agent.results = [OK, OK] + assert w.run_once()["unsubscribed"] == 2 + + +def test_model_unavailable_result_counts_like_an_unavailable_agent(store): + imap = FakeIMAP({b"1": mail_bytes("")}) + summary = worker(store, imap, Agent({"result": "error", "reason": "llm_unavailable"})).run_once() + assert summary["error"] == "llm_unavailable" and store.get("")["attempts"] == 0 + + +def test_without_agent_cases_are_only_recorded(store): + imap = FakeIMAP({b"1": mail_bytes(""), b"2": mail_bytes("", link="")}) + summary = worker(store, imap, None).run_once() + assert (summary["new"], summary["handled"], summary["without_link"]) == (1, 0, 1) + assert store.get("")["state"] == "new" and store.get("") is None + + +def test_paused_run_does_nothing(store): + store.set_meta("paused", "1") + imap = FakeIMAP({b"1": mail_bytes("")}) + agent = Agent(OK) + summary = worker(store, imap, agent).run_once() + assert summary["error"] == "paused" and agent.calls == [] and store.all() == [] + + +def test_at_most_max_cases_per_run(store): + imap = FakeIMAP({str(i).encode(): mail_bytes(f"<{i}@x>") for i in range(1, 5)}) + agent = Agent(OK, OK, OK, OK) + assert worker(store, imap, agent, max_cases=3).run_once()["handled"] == 3 + assert len(agent.calls) == 3 + + +def test_address_of_a_foreign_domain_is_not_handed_over(store): + imap = FakeIMAP({b"1": mail_bytes("", to="victim@elsewhere.example")}) + agent = Agent(OK) + worker(store, imap, agent).run_once() + assert agent.calls == [("https://shop.example/unsubscribe/1", "jens@wagnersnetz.de")] # the forwarder + + +def test_move_failure_keeps_case(store): + imap = FakeIMAP({b"7": mail_bytes("")}) + imap.fail_copy = True + w = worker(store, imap, Agent(OK)) + summary = w.run_once() + assert summary["moved"] == 0 and b"7" in imap.folders["INBOX"] + assert (store.get("")["state"], store.get("")["moved"]) == ("unsubscribed", 0) + imap.fail_copy = False + assert w.run_once()["moved"] == 1 and imap.folders["INBOX"] == {} + + +def test_cleanup_deletes_old_successes_from_the_done_folder(store): + imap = FakeIMAP({b"7": mail_bytes(""), b"8": mail_bytes("")}) + now = [time.time()] # store stamps real time; a fixed start date would make the test date-dependent + w = worker(store, imap, Agent(OK, OK), clock=lambda: now[0], retention_days=30) + w.run_once() + assert len(imap.folders["Erledigt"]) == 2 + young_time = store.get("")["updated_at"] + store._run("UPDATE cases SET updated_at = ? WHERE key = ''", (young_time - 31 * 86400,)) + now[0] = young_time + 60 + summary = w.run_once() + assert summary["cleaned"] == 1 + assert store.get("") is None and store.get("") is not None + assert len(imap.folders["Erledigt"]) == 1 and imap.selected == "INBOX" + + +def test_run_survives_a_broken_mail(store): + imap = FakeIMAP({b"1": b"\xff\xfe not a mail", b"2": mail_bytes("")}) + assert worker(store, imap, Agent(OK)).run_once()["unsubscribed"] == 1 + + +def test_mail_from_unknown_sender_is_ignored(store): + imap = FakeIMAP({b"1": mail_bytes("", sender="Jens "), + b"2": mail_bytes("", sender="Mallory ")}) + agent = Agent(OK) + summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once() + assert len(agent.calls) == 1 and summary["ignored"] == 1 + assert [c["key"] for c in store.all()] == [""] + assert store.get("") is None and b"2" in imap.folders["INBOX"] + + +def test_no_allowed_senders_with_agent_means_nobody(store): + imap = FakeIMAP({b"1": mail_bytes("")}) + agent = Agent(OK) + summary = worker(store, imap, agent, allowed_senders=()).run_once() + assert agent.calls == [] and store.all() == [] and summary["ignored"] == 1 + + +def test_no_allowed_senders_without_agent_lists_everything(store): + imap = FakeIMAP({b"1": mail_bytes("")}) + summary = worker(store, imap, None, allowed_senders=()).run_once() + assert store.get("")["state"] == "new" and summary["ignored"] == 0 + + +def test_spoofed_display_name_does_not_help(store): + imap = FakeIMAP({b"1": mail_bytes("", sender='"jens@wagnersnetz.de" ')}) + agent = Agent(OK) + summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once() + assert agent.calls == [] and store.all() == [] and summary["ignored"] == 1 diff --git a/worker.py b/worker.py new file mode 100644 index 0000000..eae0dd7 --- /dev/null +++ b/worker.py @@ -0,0 +1,165 @@ +"""Scheduler of the unsubscribe agent: find cases in the inbox, hand them to the agent, tidy up.""" + +import email +import json +import logging +import threading +import time +from email.utils import parseaddr + +from cases import case_key +from forwarded import original_recipient + +logger = logging.getLogger(__name__) + +MAX_LINKS = 2 + + +class AgentUnavailable(Exception): + """Agent container or model not reachable: the case keeps its attempt.""" + + +class Worker: + def __init__(self, store, connect, parse, agent, *, inbox="INBOX", done_folder="Erledigt", retention_days=30, + max_cases=10, allowed_domains=(), allowed_senders=(), clock=time.time): + self.store, self.connect, self.parse, self.agent = store, connect, parse, agent + self.inbox, self.done_folder = inbox, done_folder + self.retention_days, self.max_cases = retention_days, max_cases + self.allowed_domains, self.clock = tuple(allowed_domains), clock + self.allowed_senders = tuple(a.lower() for a in allowed_senders) + self._lock = threading.Lock() + + def scan(self, mail) -> tuple[dict, int]: + """Create a case for every mail with unsubscribe links. Returns ({key: uid}, mails without link).""" + _, data = mail.uid("search", None, "ALL") + uids, without, self._created, self._ignored = {}, 0, 0, 0 + for uid in data[0].split(): + try: + _, fetched = mail.uid("fetch", uid, "(BODY.PEEK[])") + message = email.message_from_bytes(fetched[0][1]) + subject, links, text = self.parse(message) + except Exception as e: # one broken mail must not stop the run + logger.warning("mail %s skipped: %s", uid, type(e).__name__) + continue + if not links: + without += 1 + continue + forwarder = parseaddr(message.get("From", ""))[1] + sender = forwarder.lower() + if (self.allowed_senders and sender not in self.allowed_senders) or ( + not self.allowed_senders and self.agent is not None): # fail closed + self._ignored += 1 + continue + key = case_key(message) + uids[key] = uid + if self.store.add(key, subject, links, original_recipient(text, forwarder, self.allowed_domains)): + self._created += 1 + return uids, without + + def _handle(self, case: dict) -> str: + """Run one case through the agent. Returns the final state; raises AgentUnavailable.""" + self.store.claim(case["key"]) + result = {"result": "unclear", "reason": "no link"} + try: + for link in case["links"][:MAX_LINKS]: + result = self.agent(link, case["address"]) + if result["result"] == "error": + raise AgentUnavailable(result.get("reason", "agent error")) + if result["result"] == "unsubscribed" or result.get("reason") == "login": + break + except AgentUnavailable: + self.store.release(case["key"]) + raise + self.store.finish(case["key"], result["result"], result.get("reason", ""), result.get("evidence", ""), + result.get("evidence_url", "")) + return result["result"] + + def _move(self, mail, uids: dict) -> int: + moved = 0 + todo = [c for c in self.store.to_move() if c["key"] in uids] + if not todo: + return 0 + mail.create(self.done_folder) # answers NO if it exists already + for case in todo: + uid = uids[case["key"]] + status, _ = mail.uid("copy", uid, self.done_folder) + if status != "OK": + logger.warning("could not move a mail to %s", self.done_folder) + continue + mail.uid("store", uid, "+FLAGS", "(\\Deleted)") + self.store.mark_moved(case["key"]) + moved += 1 + mail.expunge() + return moved + + def _cleanup(self, mail) -> int: + expired = self.store.expired(self.retention_days, now=self.clock()) + if not expired: + return 0 + status, _ = mail.select(self.done_folder) + if status != "OK": + return 0 + try: + for case in expired: + if not case["key"].startswith("sha256:"): # mails without Message-ID cannot be found again + _, data = mail.uid("search", None, "HEADER", "Message-ID", '"' + case["key"].replace('"', "") + '"') + for uid in data[0].split(): + mail.uid("store", uid, "+FLAGS", "(\\Deleted)") + self.store.delete(case["key"]) + mail.expunge() + finally: + mail.select(self.inbox) + return len(expired) + + def run_once(self) -> dict: + summary = {"new": 0, "handled": 0, "unsubscribed": 0, "moved": 0, "cleaned": 0, "without_link": 0, "ignored": 0, "error": ""} + if not self._lock.acquire(blocking=False): + summary["error"] = "already running" + return summary + try: + if self.store.meta("paused") == "1": + summary["error"] = "paused" + return self._note(summary) + mail = self.connect() + try: + uids, summary["without_link"] = self.scan(mail) + summary["new"], summary["ignored"] = self._created, self._ignored + if self.agent is not None: + for case in self.store.pending(self.max_cases): + try: + state = self._handle(case) + except AgentUnavailable as e: + summary["error"] = str(e) + break + summary["handled"] += 1 + summary["unsubscribed"] += state == "unsubscribed" + summary["moved"] = self._move(mail, uids) + summary["cleaned"] = self._cleanup(mail) + finally: + mail.logout() + return self._note(summary) + finally: + self._lock.release() + + def _note(self, summary: dict) -> dict: + self.store.set_meta("last_run", json.dumps(summary)) + self.store.set_meta("last_run_at", str(int(self.clock()))) + return summary + + +def start_scheduler(worker: Worker, interval_seconds: int) -> threading.Thread: + """Background thread: one run every interval. Errors (mailbox not connected, IMAP down) are noted, not raised.""" + + def loop(): + while True: + try: + worker.run_once() + except Exception as e: + logger.warning("scheduled run failed: %s: %s", type(e).__name__, e) + worker.store.set_meta("last_run", json.dumps({"error": f"{type(e).__name__}: {e}"[:200]})) + worker.store.set_meta("last_run_at", str(int(worker.clock()))) + time.sleep(interval_seconds) + + thread = threading.Thread(target=loop, name="unsubscribe-scheduler", daemon=True) + thread.start() + return thread