Zeitlauf: Fälle finden, an den Agenten geben, verschieben, aufräumen

This commit is contained in:
Rsclub2_2 committed 2026-11-15 00:40:20 +01:00
1 parent c7b054a3bb
commit 65c09d96e8
2 files changed
+409

No files matched your search

+244
View File
@@ -0,0 +1,244 @@
"""Scheduler run: find cases, hand them to the agent, move successes, clean up (fake IMAP, fake agent)."""
import json
import time
from email.message import EmailMessage
import pytest
from cases import CaseStore
from worker import AgentUnavailable, Worker
def mail_bytes(mid, link="https://shop.example/unsubscribe/1", to="philipp@wagnersnetz.de", links=None,
sender="Jens <jens@wagnersnetz.de>"):
m = EmailMessage()
m["Message-ID"] = mid
m["Subject"] = "Fwd: News " + mid
m["From"] = sender
urls = links if links is not None else ([link] if link else [])
m.set_content("Von: news@shop.example\nAn: " + to + "\n\n" + "\n".join(urls))
return m.as_bytes()
class FakeIMAP:
"""Two folders; uid commands as far as the worker uses them."""
def __init__(self, inbox):
self.folders = {"INBOX": dict(inbox)}
self.selected = "INBOX"
self.deleted = set()
self.fail_copy = False
self.logged_out = False
def select(self, folder):
if folder not in self.folders:
return "NO", [b"no such folder"]
self.selected = folder
return "OK", [b"1"]
def create(self, folder):
if folder in self.folders:
return "NO", [b"exists"]
self.folders[folder] = {}
return "OK", [b"created"]
def uid(self, command, *args):
box = self.folders[self.selected]
if command == "search":
if len(args) >= 4 and args[1] == "HEADER":
wanted = args[3].strip('"')
hits = [u for u, raw in box.items() if ("Message-ID: " + wanted).encode() in raw]
return "OK", [b" ".join(hits)]
return "OK", [b" ".join(box)]
if command == "fetch":
return "OK", [(b"1 (BODY[] {n}", box[args[0]]), b")"]
if command == "copy":
if self.fail_copy or args[0] not in box:
return "NO", [b"copy failed"]
self.folders[args[1]][b"9" + args[0]] = box[args[0]]
return "OK", [b""]
if command == "store":
self.deleted.add((self.selected, args[0]))
return "OK", [b""]
raise AssertionError(command)
def expunge(self):
for folder, uid in list(self.deleted):
self.folders[folder].pop(uid, None)
self.deleted.clear()
return "OK", [b""]
def logout(self):
self.logged_out = True
def parse(message):
import app as unsub
text = unsub.get_text_from_email(message)
return unsub.get_subject(message), unsub.find_unsubscribe_links(text), text
class Agent:
def __init__(self, *results):
self.results = list(results)
self.calls = []
def __call__(self, url, address):
self.calls.append((url, address))
r = self.results.pop(0) if self.results else {"result": "unclear", "reason": "no more results"}
if isinstance(r, Exception):
raise r
return {"evidence": "", "evidence_url": "", "steps": 1, "stage": 2, **r}
OK = {"result": "unsubscribed", "reason": "page confirms", "evidence": "abgemeldet", "evidence_url": "https://shop.example/done"}
@pytest.fixture
def store(tmp_path):
return CaseStore(str(tmp_path / "cases.db"))
def worker(store, imap, agent, **kw):
kw.setdefault("allowed_senders", ("jens@wagnersnetz.de",))
return Worker(store, lambda: imap, parse, agent, allowed_domains=("wagnersnetz.de",), **kw)
def test_success_is_moved_to_the_done_folder(store):
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
agent = Agent(OK)
summary = worker(store, imap, agent).run_once()
assert agent.calls == [("https://shop.example/unsubscribe/1", "philipp@wagnersnetz.de")]
assert (summary["new"], summary["handled"], summary["unsubscribed"], summary["moved"], summary["error"]) == (1, 1, 1, 1, "")
assert imap.folders["INBOX"] == {} and len(imap.folders["Erledigt"]) == 1 and imap.logged_out
c = store.get("<a@x>")
assert (c["state"], c["moved"], c["evidence"]) == ("unsubscribed", 1, "abgemeldet")
assert json.loads(store.meta("last_run"))["moved"] == 1 and store.meta("last_run_at")
def test_failure_stays_in_the_inbox_and_is_not_tried_again(store):
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
agent = Agent({"result": "not_possible", "reason": "captcha"})
w = worker(store, imap, agent)
w.run_once()
w.run_once()
assert len(agent.calls) == 1 and b"7" in imap.folders["INBOX"]
assert (store.get("<a@x>")["state"], store.get("<a@x>")["reason"]) == ("not_possible", "captcha")
def test_second_link_is_tried_but_not_a_third_and_not_after_login(store):
links = ["https://shop.example/unsubscribe/1", "https://shop.example/unsubscribe/2", "https://shop.example/unsubscribe/3"]
imap = FakeIMAP({b"1": mail_bytes("<two@x>", links=links), b"2": mail_bytes("<login@x>", links=links)})
agent = Agent({"result": "unclear", "reason": "x"}, OK, {"result": "not_possible", "reason": "login"})
worker(store, imap, agent).run_once()
assert [u for u, _ in agent.calls] == [links[0], links[1], links[0]]
assert store.get("<two@x>")["state"] == "unsubscribed"
assert (store.get("<login@x>")["state"], store.get("<login@x>")["reason"]) == ("not_possible", "login")
def test_agent_error_does_not_burn_the_attempt(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
agent = Agent(AgentUnavailable("agent not reachable"))
w = worker(store, imap, agent)
summary = w.run_once()
assert summary["error"] == "agent not reachable" and summary["handled"] == 0 and len(agent.calls) == 1
assert [(c["state"], c["attempts"]) for c in store.all()] == [("new", 0), ("new", 0)]
agent.results = [OK, OK]
assert w.run_once()["unsubscribed"] == 2
def test_model_unavailable_result_counts_like_an_unavailable_agent(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
summary = worker(store, imap, Agent({"result": "error", "reason": "llm_unavailable"})).run_once()
assert summary["error"] == "llm_unavailable" and store.get("<a@x>")["attempts"] == 0
def test_without_agent_cases_are_only_recorded(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<nolink@x>", link="")})
summary = worker(store, imap, None).run_once()
assert (summary["new"], summary["handled"], summary["without_link"]) == (1, 0, 1)
assert store.get("<a@x>")["state"] == "new" and store.get("<nolink@x>") is None
def test_paused_run_does_nothing(store):
store.set_meta("paused", "1")
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
agent = Agent(OK)
summary = worker(store, imap, agent).run_once()
assert summary["error"] == "paused" and agent.calls == [] and store.all() == []
def test_at_most_max_cases_per_run(store):
imap = FakeIMAP({str(i).encode(): mail_bytes(f"<{i}@x>") for i in range(1, 5)})
agent = Agent(OK, OK, OK, OK)
assert worker(store, imap, agent, max_cases=3).run_once()["handled"] == 3
assert len(agent.calls) == 3
def test_address_of_a_foreign_domain_is_not_handed_over(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", to="victim@elsewhere.example")})
agent = Agent(OK)
worker(store, imap, agent).run_once()
assert agent.calls == [("https://shop.example/unsubscribe/1", "jens@wagnersnetz.de")] # the forwarder
def test_move_failure_keeps_case(store):
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
imap.fail_copy = True
w = worker(store, imap, Agent(OK))
summary = w.run_once()
assert summary["moved"] == 0 and b"7" in imap.folders["INBOX"]
assert (store.get("<a@x>")["state"], store.get("<a@x>")["moved"]) == ("unsubscribed", 0)
imap.fail_copy = False
assert w.run_once()["moved"] == 1 and imap.folders["INBOX"] == {}
def test_cleanup_deletes_old_successes_from_the_done_folder(store):
imap = FakeIMAP({b"7": mail_bytes("<old@x>"), b"8": mail_bytes("<young@x>")})
now = [time.time()] # store stamps real time; a fixed start date would make the test date-dependent
w = worker(store, imap, Agent(OK, OK), clock=lambda: now[0], retention_days=30)
w.run_once()
assert len(imap.folders["Erledigt"]) == 2
young_time = store.get("<young@x>")["updated_at"]
store._run("UPDATE cases SET updated_at = ? WHERE key = '<old@x>'", (young_time - 31 * 86400,))
now[0] = young_time + 60
summary = w.run_once()
assert summary["cleaned"] == 1
assert store.get("<old@x>") is None and store.get("<young@x>") is not None
assert len(imap.folders["Erledigt"]) == 1 and imap.selected == "INBOX"
def test_run_survives_a_broken_mail(store):
imap = FakeIMAP({b"1": b"\xff\xfe not a mail", b"2": mail_bytes("<b@x>")})
assert worker(store, imap, Agent(OK)).run_once()["unsubscribed"] == 1
def test_mail_from_unknown_sender_is_ignored(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender="Jens <JENS@wagnersnetz.de>"),
b"2": mail_bytes("<evil@x>", sender="Mallory <mallory@evil.example>")})
agent = Agent(OK)
summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once()
assert len(agent.calls) == 1 and summary["ignored"] == 1
assert [c["key"] for c in store.all()] == ["<a@x>"]
assert store.get("<evil@x>") is None and b"2" in imap.folders["INBOX"]
def test_no_allowed_senders_with_agent_means_nobody(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
agent = Agent(OK)
summary = worker(store, imap, agent, allowed_senders=()).run_once()
assert agent.calls == [] and store.all() == [] and summary["ignored"] == 1
def test_no_allowed_senders_without_agent_lists_everything(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
summary = worker(store, imap, None, allowed_senders=()).run_once()
assert store.get("<a@x>")["state"] == "new" and summary["ignored"] == 0
def test_spoofed_display_name_does_not_help(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender='"jens@wagnersnetz.de" <mallory@evil.example>')})
agent = Agent(OK)
summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once()
assert agent.calls == [] and store.all() == [] and summary["ignored"] == 1
+165
View File
@@ -0,0 +1,165 @@
"""Scheduler of the unsubscribe agent: find cases in the inbox, hand them to the agent, tidy up."""
import email
import json
import logging
import threading
import time
from email.utils import parseaddr
from cases import case_key
from forwarded import original_recipient
logger = logging.getLogger(__name__)
MAX_LINKS = 2
class AgentUnavailable(Exception):
"""Agent container or model not reachable: the case keeps its attempt."""
class Worker:
def __init__(self, store, connect, parse, agent, *, inbox="INBOX", done_folder="Erledigt", retention_days=30,
max_cases=10, allowed_domains=(), allowed_senders=(), clock=time.time):
self.store, self.connect, self.parse, self.agent = store, connect, parse, agent
self.inbox, self.done_folder = inbox, done_folder
self.retention_days, self.max_cases = retention_days, max_cases
self.allowed_domains, self.clock = tuple(allowed_domains), clock
self.allowed_senders = tuple(a.lower() for a in allowed_senders)
self._lock = threading.Lock()
def scan(self, mail) -> tuple[dict, int]:
"""Create a case for every mail with unsubscribe links. Returns ({key: uid}, mails without link)."""
_, data = mail.uid("search", None, "ALL")
uids, without, self._created, self._ignored = {}, 0, 0, 0
for uid in data[0].split():
try:
_, fetched = mail.uid("fetch", uid, "(BODY.PEEK[])")
message = email.message_from_bytes(fetched[0][1])
subject, links, text = self.parse(message)
except Exception as e: # one broken mail must not stop the run
logger.warning("mail %s skipped: %s", uid, type(e).__name__)
continue
if not links:
without += 1
continue
forwarder = parseaddr(message.get("From", ""))[1]
sender = forwarder.lower()
if (self.allowed_senders and sender not in self.allowed_senders) or (
not self.allowed_senders and self.agent is not None): # fail closed
self._ignored += 1
continue
key = case_key(message)
uids[key] = uid
if self.store.add(key, subject, links, original_recipient(text, forwarder, self.allowed_domains)):
self._created += 1
return uids, without
def _handle(self, case: dict) -> str:
"""Run one case through the agent. Returns the final state; raises AgentUnavailable."""
self.store.claim(case["key"])
result = {"result": "unclear", "reason": "no link"}
try:
for link in case["links"][:MAX_LINKS]:
result = self.agent(link, case["address"])
if result["result"] == "error":
raise AgentUnavailable(result.get("reason", "agent error"))
if result["result"] == "unsubscribed" or result.get("reason") == "login":
break
except AgentUnavailable:
self.store.release(case["key"])
raise
self.store.finish(case["key"], result["result"], result.get("reason", ""), result.get("evidence", ""),
result.get("evidence_url", ""))
return result["result"]
def _move(self, mail, uids: dict) -> int:
moved = 0
todo = [c for c in self.store.to_move() if c["key"] in uids]
if not todo:
return 0
mail.create(self.done_folder) # answers NO if it exists already
for case in todo:
uid = uids[case["key"]]
status, _ = mail.uid("copy", uid, self.done_folder)
if status != "OK":
logger.warning("could not move a mail to %s", self.done_folder)
continue
mail.uid("store", uid, "+FLAGS", "(\\Deleted)")
self.store.mark_moved(case["key"])
moved += 1
mail.expunge()
return moved
def _cleanup(self, mail) -> int:
expired = self.store.expired(self.retention_days, now=self.clock())
if not expired:
return 0
status, _ = mail.select(self.done_folder)
if status != "OK":
return 0
try:
for case in expired:
if not case["key"].startswith("sha256:"): # mails without Message-ID cannot be found again
_, data = mail.uid("search", None, "HEADER", "Message-ID", '"' + case["key"].replace('"', "") + '"')
for uid in data[0].split():
mail.uid("store", uid, "+FLAGS", "(\\Deleted)")
self.store.delete(case["key"])
mail.expunge()
finally:
mail.select(self.inbox)
return len(expired)
def run_once(self) -> dict:
summary = {"new": 0, "handled": 0, "unsubscribed": 0, "moved": 0, "cleaned": 0, "without_link": 0, "ignored": 0, "error": ""}
if not self._lock.acquire(blocking=False):
summary["error"] = "already running"
return summary
try:
if self.store.meta("paused") == "1":
summary["error"] = "paused"
return self._note(summary)
mail = self.connect()
try:
uids, summary["without_link"] = self.scan(mail)
summary["new"], summary["ignored"] = self._created, self._ignored
if self.agent is not None:
for case in self.store.pending(self.max_cases):
try:
state = self._handle(case)
except AgentUnavailable as e:
summary["error"] = str(e)
break
summary["handled"] += 1
summary["unsubscribed"] += state == "unsubscribed"
summary["moved"] = self._move(mail, uids)
summary["cleaned"] = self._cleanup(mail)
finally:
mail.logout()
return self._note(summary)
finally:
self._lock.release()
def _note(self, summary: dict) -> dict:
self.store.set_meta("last_run", json.dumps(summary))
self.store.set_meta("last_run_at", str(int(self.clock())))
return summary
def start_scheduler(worker: Worker, interval_seconds: int) -> threading.Thread:
"""Background thread: one run every interval. Errors (mailbox not connected, IMAP down) are noted, not raised."""
def loop():
while True:
try:
worker.run_once()
except Exception as e:
logger.warning("scheduled run failed: %s: %s", type(e).__name__, e)
worker.store.set_meta("last_run", json.dumps({"error": f"{type(e).__name__}: {e}"[:200]}))
worker.store.set_meta("last_run_at", str(int(worker.clock())))
time.sleep(interval_seconds)
thread = threading.Thread(target=loop, name="unsubscribe-scheduler", daemon=True)
thread.start()
return thread