Ein Eintrag ohne @ erlaubt jeden Absender dieser Domain (nicht ihrer Subdomains), eine ganze Adresse genau diesen. Was keine einfache Adresse ist, wird abgewiesen.
265 lines
12 KiB
Python
265 lines
12 KiB
Python
"""Scheduler of the unsubscribe agent: find cases in the inbox, hand them to the agent, tidy up."""
|
|
|
|
import email
|
|
import json
|
|
import logging
|
|
import threading
|
|
import time
|
|
from email.utils import parseaddr
|
|
|
|
from cases import case_key
|
|
from forwarded import original_recipient, sender_allowed
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
MAX_LINKS = 2
|
|
FINAL_RESULTS = ("unsubscribed", "not_possible", "unclear", "error")
|
|
SUMMARY_KEYS = ("new", "handled", "unsubscribed", "moved", "cleaned", "without_link", "ignored", "error", "deferred",
|
|
"gone")
|
|
MODEL_UNAVAILABLE = "model unavailable for this case"
|
|
|
|
|
|
def _searchable(key: str) -> bool:
|
|
"""Only plain printable ASCII without quote and backslash may go into an IMAP SEARCH command."""
|
|
return bool(key) and all(0x20 <= ord(c) <= 0x7E and c not in '"\\' for c in key)
|
|
|
|
|
|
class AgentUnavailable(Exception):
|
|
"""Agent container not reachable (transport failure): the case keeps its attempt and the run stops."""
|
|
|
|
|
|
class ScanFailed(Exception):
|
|
"""The inbox SEARCH did not answer OK: nothing about the inbox is known."""
|
|
|
|
|
|
class _Deferred(Exception):
|
|
"""The agent answered error (model unavailable for this page): the case waits, the run goes on."""
|
|
|
|
def __init__(self, reason: str, final: bool):
|
|
super().__init__(reason)
|
|
self.final = final
|
|
|
|
|
|
class Worker:
|
|
def __init__(self, store, connect, parse, agent, *, inbox="INBOX", done_folder="Erledigt", retention_days=30,
|
|
max_cases=10, allowed_domains=(), allowed_senders=(), max_deferrals=12, clock=time.time):
|
|
self.store, self.connect, self.parse, self.agent = store, connect, parse, agent
|
|
self.inbox, self.done_folder = inbox, done_folder
|
|
self.retention_days, self.max_cases, self.max_deferrals = retention_days, max_cases, max_deferrals
|
|
self.allowed_domains, self.clock = tuple(allowed_domains), clock
|
|
self.allowed_senders = tuple(a.lower() for a in allowed_senders)
|
|
self._lock = threading.Lock()
|
|
|
|
def scan(self, mail) -> tuple[dict, int]:
|
|
"""Create a case for every mail with unsubscribe links. Returns ({key: uid}, mails without link)."""
|
|
return self._scan(mail)[:2]
|
|
|
|
def _scan(self, mail) -> tuple[dict, int, int, int, set | None]:
|
|
"""Like scan, plus the number of created cases, of ignored mails and the keys of all mails in the inbox
|
|
(None if a mail could not be read, so its key is unknown). No instance state. Raises ScanFailed."""
|
|
status, data = mail.uid("search", None, "ALL")
|
|
if status != "OK":
|
|
raise ScanFailed("inbox scan failed")
|
|
uids, without, created, ignored, present, complete = {}, 0, 0, 0, set(), True
|
|
for uid in data[0].split():
|
|
key = None
|
|
try:
|
|
_, fetched = mail.uid("fetch", uid, "(BODY.PEEK[])")
|
|
message = email.message_from_bytes(fetched[0][1])
|
|
key = case_key(message)
|
|
present.add(key)
|
|
subject, links, text = self.parse(message)
|
|
if not links:
|
|
without += 1
|
|
continue
|
|
forwarder = parseaddr(message.get("From", ""))[1]
|
|
if (self.allowed_senders and not sender_allowed(forwarder, self.allowed_senders)) or (
|
|
not self.allowed_senders and self.agent is not None): # fail closed
|
|
ignored += 1
|
|
continue
|
|
recipient = original_recipient(text, forwarder, self.allowed_domains)
|
|
except Exception as e: # one broken mail must not stop the run
|
|
logger.warning("mail %s skipped: %s", uid, type(e).__name__)
|
|
if key is None:
|
|
complete = False # not even its key is known
|
|
continue
|
|
uids[key] = uid
|
|
if self.store.add(key, subject, links, recipient):
|
|
created += 1
|
|
return uids, without, created, ignored, present if complete else None
|
|
|
|
def _tidy(self, present: set, since: float) -> int:
|
|
"""Cases whose mail is no longer in the inbox: nothing left to do or to show, except an unmoved success,
|
|
which now counts as moved so that it expires. Running cases, and cases created after the scan started (by
|
|
the page's own scan, for a mail this scan did not see), are left alone."""
|
|
gone = 0
|
|
for case in self.store.all():
|
|
if case["key"] in present or case["created_at"] >= since:
|
|
continue
|
|
if case["state"] in ("new", "not_possible", "unclear"):
|
|
self.store.delete(case["key"])
|
|
elif case["state"] == "unsubscribed" and not case["moved"]:
|
|
self.store.mark_moved(case["key"])
|
|
else:
|
|
continue
|
|
gone += 1
|
|
return gone
|
|
|
|
def _handle(self, case: dict) -> str:
|
|
"""Run one case through the agent. Returns the final state; raises AgentUnavailable or _Deferred."""
|
|
self.store.claim(case["key"])
|
|
result = {"result": "unclear", "reason": "no link"}
|
|
try:
|
|
for link in case["links"][:MAX_LINKS]:
|
|
result = self.agent(link, case["address"])
|
|
if result["result"] not in FINAL_RESULTS:
|
|
raise ValueError("malformed agent result")
|
|
if result["result"] == "error":
|
|
raise _Deferred(str(result.get("reason") or "agent error"), False)
|
|
if result["result"] == "unsubscribed" or result.get("reason") == "login":
|
|
break
|
|
except AgentUnavailable:
|
|
self.store.release(case["key"])
|
|
raise
|
|
except _Deferred as e:
|
|
if self.store.defer(case["key"]) >= self.max_deferrals:
|
|
self.store.finish(case["key"], "unclear", MODEL_UNAVAILABLE)
|
|
e.final = True
|
|
raise
|
|
except Exception as e: # agent bug or malformed result: the attempt counts, the run goes on
|
|
logger.warning("agent failed on a case: %s", type(e).__name__)
|
|
self.store.finish(case["key"], "unclear", "agent error")
|
|
return "unclear"
|
|
self.store.finish(case["key"], result["result"], result.get("reason", ""), result.get("evidence", ""),
|
|
result.get("evidence_url", ""))
|
|
return result["result"]
|
|
|
|
def _move(self, mail, uids: dict) -> int:
|
|
moved = 0
|
|
todo = [c for c in self.store.to_move() if c["key"] in uids]
|
|
if not todo:
|
|
return 0
|
|
mail.create(self.done_folder) # answers NO if it exists already
|
|
for case in todo:
|
|
uid = uids[case["key"]]
|
|
status, _ = mail.uid("copy", uid, self.done_folder)
|
|
if status != "OK":
|
|
logger.warning("could not move a mail to %s", self.done_folder)
|
|
continue
|
|
status, _ = mail.uid("store", uid, "+FLAGS", "(\\Deleted)")
|
|
if status != "OK":
|
|
logger.warning("could not flag a copied mail as deleted")
|
|
continue
|
|
self.store.mark_moved(case["key"])
|
|
moved += 1
|
|
mail.expunge()
|
|
return moved
|
|
|
|
def _cleanup(self, mail) -> int:
|
|
expired = self.store.expired(self.retention_days, now=self.clock())
|
|
if not expired:
|
|
return 0
|
|
status, _ = mail.select(self.done_folder)
|
|
if status != "OK":
|
|
return 0
|
|
try:
|
|
for case in expired:
|
|
key = case["key"]
|
|
if not key.startswith("sha256:") and _searchable(key): # others cannot be found (or not safely)
|
|
_, data = mail.uid("search", None, "HEADER", "Message-ID", '"' + key + '"')
|
|
for uid in data[0].split():
|
|
mail.uid("store", uid, "+FLAGS", "(\\Deleted)")
|
|
self.store.delete(case["key"])
|
|
mail.expunge()
|
|
finally:
|
|
mail.select(self.inbox)
|
|
return len(expired)
|
|
|
|
def run_once(self) -> dict:
|
|
summary = {k: 0 for k in SUMMARY_KEYS}
|
|
summary["error"] = ""
|
|
if not self._lock.acquire(blocking=False):
|
|
summary["error"] = "already running"
|
|
return summary
|
|
try:
|
|
if self.store.meta("paused") == "1":
|
|
summary["error"] = "paused"
|
|
return self._note(summary)
|
|
mail = self.connect()
|
|
try:
|
|
started = time.time() # the store's clock, not self.clock
|
|
try:
|
|
uids, summary["without_link"], summary["new"], summary["ignored"], present = self._scan(mail)
|
|
except ScanFailed as e: # nothing is known about the inbox: change nothing
|
|
summary["error"] = str(e)
|
|
return self._note(summary)
|
|
if present is not None: # only a complete scan may tidy up
|
|
summary["gone"] = self._tidy(present, started)
|
|
if self.agent is not None:
|
|
deferral = ""
|
|
for case in self.store.pending(10_000):
|
|
if summary["handled"] + summary["deferred"] >= self.max_cases: # every agent call counts
|
|
break
|
|
if case["key"] not in uids: # only cases this scan vouched for (mail there, sender allowed)
|
|
continue
|
|
if self.store.meta("paused") == "1": # "Pause agent" also stops a run in progress
|
|
summary["error"] = "paused"
|
|
break
|
|
try:
|
|
state = self._handle(case)
|
|
except AgentUnavailable as e:
|
|
summary["error"] = str(e)
|
|
break
|
|
except _Deferred as e:
|
|
if e.final: # deferred too often: finished as unclear
|
|
summary["handled"] += 1
|
|
else:
|
|
summary["deferred"] += 1
|
|
deferral = str(e)
|
|
continue
|
|
summary["handled"] += 1
|
|
summary["unsubscribed"] += state == "unsubscribed"
|
|
if deferral and not summary["handled"] and not summary["error"]:
|
|
summary["error"] = deferral # no case could be handled at all
|
|
summary["moved"] = self._move(mail, uids)
|
|
summary["cleaned"] = self._cleanup(mail)
|
|
finally:
|
|
mail.logout()
|
|
return self._note(summary)
|
|
finally:
|
|
self._lock.release()
|
|
|
|
def _note(self, summary: dict) -> dict:
|
|
self.store.set_meta("last_run", json.dumps(summary))
|
|
self.store.set_meta("last_run_at", str(int(self.clock())))
|
|
return summary
|
|
|
|
|
|
def _tick(worker: Worker) -> None:
|
|
"""One scheduled run. Never raises; notes only the exception type (messages can contain mail content)."""
|
|
try:
|
|
worker.run_once()
|
|
except Exception as e:
|
|
name = type(e).__name__
|
|
logger.warning("scheduled run failed: %s", name)
|
|
try:
|
|
summary = {k: 0 for k in SUMMARY_KEYS}
|
|
summary["error"] = name
|
|
worker.store.set_meta("last_run", json.dumps(summary))
|
|
worker.store.set_meta("last_run_at", str(int(worker.clock())))
|
|
except Exception as e2:
|
|
logger.warning("could not note the failed run: %s", type(e2).__name__)
|
|
|
|
|
|
def start_scheduler(worker: Worker, interval_seconds: int) -> threading.Thread:
|
|
"""Background thread: one run every interval. Errors (mailbox not connected, IMAP down) are noted, not raised."""
|
|
|
|
def loop():
|
|
while True:
|
|
_tick(worker)
|
|
time.sleep(interval_seconds)
|
|
|
|
thread = threading.Thread(target=loop, name="unsubscribe-scheduler", daemon=True)
|
|
thread.start()
|
|
return thread
|