Zeitlauf: Fälle finden, an den Agenten geben, verschieben, aufräumen
This commit is contained in:
2 files changed
+409
No files matched your search
@@ -0,0 +1,244 @@
|
||||
"""Scheduler run: find cases, hand them to the agent, move successes, clean up (fake IMAP, fake agent)."""
|
||||
|
||||
import json
|
||||
import time
|
||||
from email.message import EmailMessage
|
||||
|
||||
import pytest
|
||||
|
||||
from cases import CaseStore
|
||||
from worker import AgentUnavailable, Worker
|
||||
|
||||
|
||||
def mail_bytes(mid, link="https://shop.example/unsubscribe/1", to="philipp@wagnersnetz.de", links=None,
|
||||
sender="Jens <jens@wagnersnetz.de>"):
|
||||
m = EmailMessage()
|
||||
m["Message-ID"] = mid
|
||||
m["Subject"] = "Fwd: News " + mid
|
||||
m["From"] = sender
|
||||
urls = links if links is not None else ([link] if link else [])
|
||||
m.set_content("Von: news@shop.example\nAn: " + to + "\n\n" + "\n".join(urls))
|
||||
return m.as_bytes()
|
||||
|
||||
|
||||
class FakeIMAP:
|
||||
"""Two folders; uid commands as far as the worker uses them."""
|
||||
|
||||
def __init__(self, inbox):
|
||||
self.folders = {"INBOX": dict(inbox)}
|
||||
self.selected = "INBOX"
|
||||
self.deleted = set()
|
||||
self.fail_copy = False
|
||||
self.logged_out = False
|
||||
|
||||
def select(self, folder):
|
||||
if folder not in self.folders:
|
||||
return "NO", [b"no such folder"]
|
||||
self.selected = folder
|
||||
return "OK", [b"1"]
|
||||
|
||||
def create(self, folder):
|
||||
if folder in self.folders:
|
||||
return "NO", [b"exists"]
|
||||
self.folders[folder] = {}
|
||||
return "OK", [b"created"]
|
||||
|
||||
def uid(self, command, *args):
|
||||
box = self.folders[self.selected]
|
||||
if command == "search":
|
||||
if len(args) >= 4 and args[1] == "HEADER":
|
||||
wanted = args[3].strip('"')
|
||||
hits = [u for u, raw in box.items() if ("Message-ID: " + wanted).encode() in raw]
|
||||
return "OK", [b" ".join(hits)]
|
||||
return "OK", [b" ".join(box)]
|
||||
if command == "fetch":
|
||||
return "OK", [(b"1 (BODY[] {n}", box[args[0]]), b")"]
|
||||
if command == "copy":
|
||||
if self.fail_copy or args[0] not in box:
|
||||
return "NO", [b"copy failed"]
|
||||
self.folders[args[1]][b"9" + args[0]] = box[args[0]]
|
||||
return "OK", [b""]
|
||||
if command == "store":
|
||||
self.deleted.add((self.selected, args[0]))
|
||||
return "OK", [b""]
|
||||
raise AssertionError(command)
|
||||
|
||||
def expunge(self):
|
||||
for folder, uid in list(self.deleted):
|
||||
self.folders[folder].pop(uid, None)
|
||||
self.deleted.clear()
|
||||
return "OK", [b""]
|
||||
|
||||
def logout(self):
|
||||
self.logged_out = True
|
||||
|
||||
|
||||
def parse(message):
|
||||
import app as unsub
|
||||
text = unsub.get_text_from_email(message)
|
||||
return unsub.get_subject(message), unsub.find_unsubscribe_links(text), text
|
||||
|
||||
|
||||
class Agent:
|
||||
def __init__(self, *results):
|
||||
self.results = list(results)
|
||||
self.calls = []
|
||||
|
||||
def __call__(self, url, address):
|
||||
self.calls.append((url, address))
|
||||
r = self.results.pop(0) if self.results else {"result": "unclear", "reason": "no more results"}
|
||||
if isinstance(r, Exception):
|
||||
raise r
|
||||
return {"evidence": "", "evidence_url": "", "steps": 1, "stage": 2, **r}
|
||||
|
||||
|
||||
OK = {"result": "unsubscribed", "reason": "page confirms", "evidence": "abgemeldet", "evidence_url": "https://shop.example/done"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path):
|
||||
return CaseStore(str(tmp_path / "cases.db"))
|
||||
|
||||
|
||||
def worker(store, imap, agent, **kw):
|
||||
kw.setdefault("allowed_senders", ("jens@wagnersnetz.de",))
|
||||
return Worker(store, lambda: imap, parse, agent, allowed_domains=("wagnersnetz.de",), **kw)
|
||||
|
||||
|
||||
def test_success_is_moved_to_the_done_folder(store):
|
||||
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
|
||||
agent = Agent(OK)
|
||||
summary = worker(store, imap, agent).run_once()
|
||||
assert agent.calls == [("https://shop.example/unsubscribe/1", "philipp@wagnersnetz.de")]
|
||||
assert (summary["new"], summary["handled"], summary["unsubscribed"], summary["moved"], summary["error"]) == (1, 1, 1, 1, "")
|
||||
assert imap.folders["INBOX"] == {} and len(imap.folders["Erledigt"]) == 1 and imap.logged_out
|
||||
c = store.get("<a@x>")
|
||||
assert (c["state"], c["moved"], c["evidence"]) == ("unsubscribed", 1, "abgemeldet")
|
||||
assert json.loads(store.meta("last_run"))["moved"] == 1 and store.meta("last_run_at")
|
||||
|
||||
|
||||
def test_failure_stays_in_the_inbox_and_is_not_tried_again(store):
|
||||
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
|
||||
agent = Agent({"result": "not_possible", "reason": "captcha"})
|
||||
w = worker(store, imap, agent)
|
||||
w.run_once()
|
||||
w.run_once()
|
||||
assert len(agent.calls) == 1 and b"7" in imap.folders["INBOX"]
|
||||
assert (store.get("<a@x>")["state"], store.get("<a@x>")["reason"]) == ("not_possible", "captcha")
|
||||
|
||||
|
||||
def test_second_link_is_tried_but_not_a_third_and_not_after_login(store):
|
||||
links = ["https://shop.example/unsubscribe/1", "https://shop.example/unsubscribe/2", "https://shop.example/unsubscribe/3"]
|
||||
imap = FakeIMAP({b"1": mail_bytes("<two@x>", links=links), b"2": mail_bytes("<login@x>", links=links)})
|
||||
agent = Agent({"result": "unclear", "reason": "x"}, OK, {"result": "not_possible", "reason": "login"})
|
||||
worker(store, imap, agent).run_once()
|
||||
assert [u for u, _ in agent.calls] == [links[0], links[1], links[0]]
|
||||
assert store.get("<two@x>")["state"] == "unsubscribed"
|
||||
assert (store.get("<login@x>")["state"], store.get("<login@x>")["reason"]) == ("not_possible", "login")
|
||||
|
||||
|
||||
def test_agent_error_does_not_burn_the_attempt(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
|
||||
agent = Agent(AgentUnavailable("agent not reachable"))
|
||||
w = worker(store, imap, agent)
|
||||
summary = w.run_once()
|
||||
assert summary["error"] == "agent not reachable" and summary["handled"] == 0 and len(agent.calls) == 1
|
||||
assert [(c["state"], c["attempts"]) for c in store.all()] == [("new", 0), ("new", 0)]
|
||||
agent.results = [OK, OK]
|
||||
assert w.run_once()["unsubscribed"] == 2
|
||||
|
||||
|
||||
def test_model_unavailable_result_counts_like_an_unavailable_agent(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
|
||||
summary = worker(store, imap, Agent({"result": "error", "reason": "llm_unavailable"})).run_once()
|
||||
assert summary["error"] == "llm_unavailable" and store.get("<a@x>")["attempts"] == 0
|
||||
|
||||
|
||||
def test_without_agent_cases_are_only_recorded(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<nolink@x>", link="")})
|
||||
summary = worker(store, imap, None).run_once()
|
||||
assert (summary["new"], summary["handled"], summary["without_link"]) == (1, 0, 1)
|
||||
assert store.get("<a@x>")["state"] == "new" and store.get("<nolink@x>") is None
|
||||
|
||||
|
||||
def test_paused_run_does_nothing(store):
|
||||
store.set_meta("paused", "1")
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
|
||||
agent = Agent(OK)
|
||||
summary = worker(store, imap, agent).run_once()
|
||||
assert summary["error"] == "paused" and agent.calls == [] and store.all() == []
|
||||
|
||||
|
||||
def test_at_most_max_cases_per_run(store):
|
||||
imap = FakeIMAP({str(i).encode(): mail_bytes(f"<{i}@x>") for i in range(1, 5)})
|
||||
agent = Agent(OK, OK, OK, OK)
|
||||
assert worker(store, imap, agent, max_cases=3).run_once()["handled"] == 3
|
||||
assert len(agent.calls) == 3
|
||||
|
||||
|
||||
def test_address_of_a_foreign_domain_is_not_handed_over(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>", to="victim@elsewhere.example")})
|
||||
agent = Agent(OK)
|
||||
worker(store, imap, agent).run_once()
|
||||
assert agent.calls == [("https://shop.example/unsubscribe/1", "jens@wagnersnetz.de")] # the forwarder
|
||||
|
||||
|
||||
def test_move_failure_keeps_case(store):
|
||||
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
|
||||
imap.fail_copy = True
|
||||
w = worker(store, imap, Agent(OK))
|
||||
summary = w.run_once()
|
||||
assert summary["moved"] == 0 and b"7" in imap.folders["INBOX"]
|
||||
assert (store.get("<a@x>")["state"], store.get("<a@x>")["moved"]) == ("unsubscribed", 0)
|
||||
imap.fail_copy = False
|
||||
assert w.run_once()["moved"] == 1 and imap.folders["INBOX"] == {}
|
||||
|
||||
|
||||
def test_cleanup_deletes_old_successes_from_the_done_folder(store):
|
||||
imap = FakeIMAP({b"7": mail_bytes("<old@x>"), b"8": mail_bytes("<young@x>")})
|
||||
now = [time.time()] # store stamps real time; a fixed start date would make the test date-dependent
|
||||
w = worker(store, imap, Agent(OK, OK), clock=lambda: now[0], retention_days=30)
|
||||
w.run_once()
|
||||
assert len(imap.folders["Erledigt"]) == 2
|
||||
young_time = store.get("<young@x>")["updated_at"]
|
||||
store._run("UPDATE cases SET updated_at = ? WHERE key = '<old@x>'", (young_time - 31 * 86400,))
|
||||
now[0] = young_time + 60
|
||||
summary = w.run_once()
|
||||
assert summary["cleaned"] == 1
|
||||
assert store.get("<old@x>") is None and store.get("<young@x>") is not None
|
||||
assert len(imap.folders["Erledigt"]) == 1 and imap.selected == "INBOX"
|
||||
|
||||
|
||||
def test_run_survives_a_broken_mail(store):
|
||||
imap = FakeIMAP({b"1": b"\xff\xfe not a mail", b"2": mail_bytes("<b@x>")})
|
||||
assert worker(store, imap, Agent(OK)).run_once()["unsubscribed"] == 1
|
||||
|
||||
|
||||
def test_mail_from_unknown_sender_is_ignored(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender="Jens <JENS@wagnersnetz.de>"),
|
||||
b"2": mail_bytes("<evil@x>", sender="Mallory <mallory@evil.example>")})
|
||||
agent = Agent(OK)
|
||||
summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once()
|
||||
assert len(agent.calls) == 1 and summary["ignored"] == 1
|
||||
assert [c["key"] for c in store.all()] == ["<a@x>"]
|
||||
assert store.get("<evil@x>") is None and b"2" in imap.folders["INBOX"]
|
||||
|
||||
|
||||
def test_no_allowed_senders_with_agent_means_nobody(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
|
||||
agent = Agent(OK)
|
||||
summary = worker(store, imap, agent, allowed_senders=()).run_once()
|
||||
assert agent.calls == [] and store.all() == [] and summary["ignored"] == 1
|
||||
|
||||
|
||||
def test_no_allowed_senders_without_agent_lists_everything(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
|
||||
summary = worker(store, imap, None, allowed_senders=()).run_once()
|
||||
assert store.get("<a@x>")["state"] == "new" and summary["ignored"] == 0
|
||||
|
||||
|
||||
def test_spoofed_display_name_does_not_help(store):
|
||||
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender='"jens@wagnersnetz.de" <mallory@evil.example>')})
|
||||
agent = Agent(OK)
|
||||
summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once()
|
||||
assert agent.calls == [] and store.all() == [] and summary["ignored"] == 1
|
||||
@@ -0,0 +1,165 @@
|
||||
"""Scheduler of the unsubscribe agent: find cases in the inbox, hand them to the agent, tidy up."""
|
||||
|
||||
import email
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from email.utils import parseaddr
|
||||
|
||||
from cases import case_key
|
||||
from forwarded import original_recipient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_LINKS = 2
|
||||
|
||||
|
||||
class AgentUnavailable(Exception):
|
||||
"""Agent container or model not reachable: the case keeps its attempt."""
|
||||
|
||||
|
||||
class Worker:
|
||||
def __init__(self, store, connect, parse, agent, *, inbox="INBOX", done_folder="Erledigt", retention_days=30,
|
||||
max_cases=10, allowed_domains=(), allowed_senders=(), clock=time.time):
|
||||
self.store, self.connect, self.parse, self.agent = store, connect, parse, agent
|
||||
self.inbox, self.done_folder = inbox, done_folder
|
||||
self.retention_days, self.max_cases = retention_days, max_cases
|
||||
self.allowed_domains, self.clock = tuple(allowed_domains), clock
|
||||
self.allowed_senders = tuple(a.lower() for a in allowed_senders)
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def scan(self, mail) -> tuple[dict, int]:
|
||||
"""Create a case for every mail with unsubscribe links. Returns ({key: uid}, mails without link)."""
|
||||
_, data = mail.uid("search", None, "ALL")
|
||||
uids, without, self._created, self._ignored = {}, 0, 0, 0
|
||||
for uid in data[0].split():
|
||||
try:
|
||||
_, fetched = mail.uid("fetch", uid, "(BODY.PEEK[])")
|
||||
message = email.message_from_bytes(fetched[0][1])
|
||||
subject, links, text = self.parse(message)
|
||||
except Exception as e: # one broken mail must not stop the run
|
||||
logger.warning("mail %s skipped: %s", uid, type(e).__name__)
|
||||
continue
|
||||
if not links:
|
||||
without += 1
|
||||
continue
|
||||
forwarder = parseaddr(message.get("From", ""))[1]
|
||||
sender = forwarder.lower()
|
||||
if (self.allowed_senders and sender not in self.allowed_senders) or (
|
||||
not self.allowed_senders and self.agent is not None): # fail closed
|
||||
self._ignored += 1
|
||||
continue
|
||||
key = case_key(message)
|
||||
uids[key] = uid
|
||||
if self.store.add(key, subject, links, original_recipient(text, forwarder, self.allowed_domains)):
|
||||
self._created += 1
|
||||
return uids, without
|
||||
|
||||
def _handle(self, case: dict) -> str:
|
||||
"""Run one case through the agent. Returns the final state; raises AgentUnavailable."""
|
||||
self.store.claim(case["key"])
|
||||
result = {"result": "unclear", "reason": "no link"}
|
||||
try:
|
||||
for link in case["links"][:MAX_LINKS]:
|
||||
result = self.agent(link, case["address"])
|
||||
if result["result"] == "error":
|
||||
raise AgentUnavailable(result.get("reason", "agent error"))
|
||||
if result["result"] == "unsubscribed" or result.get("reason") == "login":
|
||||
break
|
||||
except AgentUnavailable:
|
||||
self.store.release(case["key"])
|
||||
raise
|
||||
self.store.finish(case["key"], result["result"], result.get("reason", ""), result.get("evidence", ""),
|
||||
result.get("evidence_url", ""))
|
||||
return result["result"]
|
||||
|
||||
def _move(self, mail, uids: dict) -> int:
|
||||
moved = 0
|
||||
todo = [c for c in self.store.to_move() if c["key"] in uids]
|
||||
if not todo:
|
||||
return 0
|
||||
mail.create(self.done_folder) # answers NO if it exists already
|
||||
for case in todo:
|
||||
uid = uids[case["key"]]
|
||||
status, _ = mail.uid("copy", uid, self.done_folder)
|
||||
if status != "OK":
|
||||
logger.warning("could not move a mail to %s", self.done_folder)
|
||||
continue
|
||||
mail.uid("store", uid, "+FLAGS", "(\\Deleted)")
|
||||
self.store.mark_moved(case["key"])
|
||||
moved += 1
|
||||
mail.expunge()
|
||||
return moved
|
||||
|
||||
def _cleanup(self, mail) -> int:
|
||||
expired = self.store.expired(self.retention_days, now=self.clock())
|
||||
if not expired:
|
||||
return 0
|
||||
status, _ = mail.select(self.done_folder)
|
||||
if status != "OK":
|
||||
return 0
|
||||
try:
|
||||
for case in expired:
|
||||
if not case["key"].startswith("sha256:"): # mails without Message-ID cannot be found again
|
||||
_, data = mail.uid("search", None, "HEADER", "Message-ID", '"' + case["key"].replace('"', "") + '"')
|
||||
for uid in data[0].split():
|
||||
mail.uid("store", uid, "+FLAGS", "(\\Deleted)")
|
||||
self.store.delete(case["key"])
|
||||
mail.expunge()
|
||||
finally:
|
||||
mail.select(self.inbox)
|
||||
return len(expired)
|
||||
|
||||
def run_once(self) -> dict:
|
||||
summary = {"new": 0, "handled": 0, "unsubscribed": 0, "moved": 0, "cleaned": 0, "without_link": 0, "ignored": 0, "error": ""}
|
||||
if not self._lock.acquire(blocking=False):
|
||||
summary["error"] = "already running"
|
||||
return summary
|
||||
try:
|
||||
if self.store.meta("paused") == "1":
|
||||
summary["error"] = "paused"
|
||||
return self._note(summary)
|
||||
mail = self.connect()
|
||||
try:
|
||||
uids, summary["without_link"] = self.scan(mail)
|
||||
summary["new"], summary["ignored"] = self._created, self._ignored
|
||||
if self.agent is not None:
|
||||
for case in self.store.pending(self.max_cases):
|
||||
try:
|
||||
state = self._handle(case)
|
||||
except AgentUnavailable as e:
|
||||
summary["error"] = str(e)
|
||||
break
|
||||
summary["handled"] += 1
|
||||
summary["unsubscribed"] += state == "unsubscribed"
|
||||
summary["moved"] = self._move(mail, uids)
|
||||
summary["cleaned"] = self._cleanup(mail)
|
||||
finally:
|
||||
mail.logout()
|
||||
return self._note(summary)
|
||||
finally:
|
||||
self._lock.release()
|
||||
|
||||
def _note(self, summary: dict) -> dict:
|
||||
self.store.set_meta("last_run", json.dumps(summary))
|
||||
self.store.set_meta("last_run_at", str(int(self.clock())))
|
||||
return summary
|
||||
|
||||
|
||||
def start_scheduler(worker: Worker, interval_seconds: int) -> threading.Thread:
|
||||
"""Background thread: one run every interval. Errors (mailbox not connected, IMAP down) are noted, not raised."""
|
||||
|
||||
def loop():
|
||||
while True:
|
||||
try:
|
||||
worker.run_once()
|
||||
except Exception as e:
|
||||
logger.warning("scheduled run failed: %s: %s", type(e).__name__, e)
|
||||
worker.store.set_meta("last_run", json.dumps({"error": f"{type(e).__name__}: {e}"[:200]}))
|
||||
worker.store.set_meta("last_run_at", str(int(worker.clock())))
|
||||
time.sleep(interval_seconds)
|
||||
|
||||
thread = threading.Thread(target=loop, name="unsubscribe-scheduler", daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
Reference in new issue
Block a user