Zeitlauf: Befunde aus dem Review behoben

This commit is contained in:
Rsclub2_2 committed 2026-11-15 00:43:51 +01:00
1 parent 65c09d96e8
commit 57b93cefff
2 files changed
+193 -27

No files matched your search

+132 -1
View File
@@ -7,7 +7,8 @@ from email.message import EmailMessage
import pytest
from cases import CaseStore
from worker import AgentUnavailable, Worker
from cases import case_key
from worker import AgentUnavailable, Worker, _tick
def mail_bytes(mid, link="https://shop.example/unsubscribe/1", to="philipp@wagnersnetz.de", links=None,
@@ -30,6 +31,8 @@ class FakeIMAP:
self.deleted = set()
self.fail_copy = False
self.logged_out = False
self.fail_store_once = False
self.calls = []
def select(self, folder):
if folder not in self.folders:
@@ -44,6 +47,7 @@ class FakeIMAP:
return "OK", [b"created"]
def uid(self, command, *args):
self.calls.append((command, args))
box = self.folders[self.selected]
if command == "search":
if len(args) >= 4 and args[1] == "HEADER":
@@ -59,6 +63,9 @@ class FakeIMAP:
self.folders[args[1]][b"9" + args[0]] = box[args[0]]
return "OK", [b""]
if command == "store":
if self.fail_store_once:
self.fail_store_once = False
return "NO", [b"store failed"]
self.deleted.add((self.selected, args[0]))
return "OK", [b""]
raise AssertionError(command)
@@ -242,3 +249,127 @@ def test_spoofed_display_name_does_not_help(store):
agent = Agent(OK)
summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once()
assert agent.calls == [] and store.all() == [] and summary["ignored"] == 1
def old_case(store, key, address="jens@wagnersnetz.de"):
store.add(key, "Fwd: old", ["https://shop.example/unsubscribe/1"], address)
def test_old_case_of_a_sender_no_longer_allowed_is_not_handled(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender="Mallory <mallory@evil.example>")})
worker(store, imap, None, allowed_senders=()).run_once()
assert store.get("<a@x>")["state"] == "new"
agent = Agent(OK)
summary = worker(store, imap, agent, allowed_senders=("jens@wagnersnetz.de",)).run_once()
assert agent.calls == [] and summary["ignored"] == 1 and summary["handled"] == 0
assert (store.get("<a@x>")["state"], store.get("<a@x>")["attempts"]) == ("new", 0)
def test_case_whose_mail_left_the_inbox_is_not_handled(store):
old_case(store, "<gone@x>")
agent = Agent(OK)
worker(store, FakeIMAP({}), agent).run_once()
assert agent.calls == []
def test_skipped_cases_do_not_eat_the_budget(store):
for i in range(3):
old_case(store, f"<gone{i}@x>")
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
agent = Agent(OK, OK, OK)
assert worker(store, imap, agent, max_cases=2).run_once()["handled"] == 2
assert len(agent.calls) == 2 and store.get("<a@x>")["state"] == "unsubscribed"
def test_agent_bug_ends_the_case_unclear_and_the_run_goes_on(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
summary = worker(store, imap, Agent(RuntimeError("boom"), OK)).run_once()
first, second = store.get("<a@x>"), store.get("<b@x>")
assert (first["state"], first["reason"], first["attempts"]) == ("unclear", "agent error", 1)
assert second["state"] == "unsubscribed"
assert summary["handled"] == 2 and summary["error"] == ""
class RawAgent:
def __init__(self, result):
self.result = result
def __call__(self, url, address):
return self.result
@pytest.mark.parametrize("result", [{}, {"result": "bought"}, None, "text"])
def test_malformed_agent_result_ends_the_case_unclear(store, result):
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
summary = worker(store, imap, RawAgent(result)).run_once()
c = store.get("<a@x>")
assert (c["state"], c["reason"], c["attempts"]) == ("unclear", "agent error", 1)
assert summary["handled"] == 1 and summary["error"] == ""
def test_mail_with_non_ascii_message_id_does_not_stop_the_run(store):
raw = ("Message-ID: <näh@x>\nSubject: Fwd: x\nFrom: jens@wagnersnetz.de\n\n"
"Von: news@shop.example\nAn: philipp@wagnersnetz.de\n\nhttps://shop.example/unsubscribe/9\n").encode("utf-8")
import email
with pytest.raises(Exception):
case_key(email.message_from_bytes(raw)) # proves the mail really is a poison pill
imap = FakeIMAP({b"1": raw, b"2": mail_bytes("<b@x>")})
summary = worker(store, imap, Agent(OK, OK)).run_once()
assert store.get("<b@x>")["state"] == "unsubscribed" and summary["error"] == ""
@pytest.mark.parametrize("key", ["<a\r\nb@x>", "<a@x>\\", '<a"b@x>', "<n\u00e4h@x>"])
def test_cleanup_never_sends_a_hostile_message_id(store, key):
old_case(store, key)
store.claim(key)
store.finish(key, "unsubscribed")
store.mark_moved(key)
store._run("UPDATE cases SET updated_at = ? WHERE key = ?", (1.0, key))
imap = FakeIMAP({})
imap.folders["Erledigt"] = {}
summary = worker(store, imap, Agent()).run_once()
assert summary["cleaned"] == 1 and store.get(key) is None
assert not any(cmd == "search" and key in " ".join(map(str, args)) for cmd, args in imap.calls)
def test_scan_from_a_request_does_not_change_a_runs_counters(store):
w = worker(store, FakeIMAP({}), Agent())
result = w.scan(FakeIMAP({b"1": mail_bytes("<a@x>")}))
assert len(result) == 2 and not hasattr(w, "_created") and not hasattr(w, "_ignored")
def test_failed_delete_flag_keeps_the_case_unmoved(store):
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
imap.fail_store_once = True
w = worker(store, imap, Agent(OK))
assert w.run_once()["moved"] == 0
assert store.get("<a@x>")["moved"] == 0 and b"7" in imap.folders["INBOX"]
assert w.run_once()["moved"] == 1 and store.get("<a@x>")["moved"] == 1
ALL_KEYS = {"new", "handled", "unsubscribed", "moved", "cleaned", "without_link", "ignored", "error"}
class Boom:
def __init__(self, store, set_meta_fails=False):
self.store, self.fail = store, set_meta_fails
self.clock = lambda: 1.0
def run_once(self):
raise ValueError("secret subject line")
class BrokenStore:
def set_meta(self, k, v):
raise OSError("disk")
def test_tick_records_only_the_exception_type(store):
_tick(Boom(store))
last = json.loads(store.meta("last_run"))
assert set(last) == ALL_KEYS and last["error"] == "ValueError"
assert "secret subject line" not in store.meta("last_run")
def test_tick_survives_a_failing_store():
_tick(Boom(BrokenStore()))
+61 -26
View File
@@ -13,6 +13,13 @@ from forwarded import original_recipient
logger = logging.getLogger(__name__)
MAX_LINKS = 2
FINAL_RESULTS = ("unsubscribed", "not_possible", "unclear", "error")
SUMMARY_KEYS = ("new", "handled", "unsubscribed", "moved", "cleaned", "without_link", "ignored", "error")
def _searchable(key: str) -> bool:
"""Only plain printable ASCII without quote and backslash may go into an IMAP SEARCH command."""
return bool(key) and all(0x20 <= ord(c) <= 0x7E and c not in '"\\' for c in key)
class AgentUnavailable(Exception):
@@ -31,30 +38,34 @@ class Worker:
def scan(self, mail) -> tuple[dict, int]:
"""Create a case for every mail with unsubscribe links. Returns ({key: uid}, mails without link)."""
return self._scan(mail)[:2]
def _scan(self, mail) -> tuple[dict, int, int, int]:
"""Like scan, plus the number of created cases and of ignored mails (no instance state)."""
_, data = mail.uid("search", None, "ALL")
uids, without, self._created, self._ignored = {}, 0, 0, 0
uids, without, created, ignored = {}, 0, 0, 0
for uid in data[0].split():
try:
_, fetched = mail.uid("fetch", uid, "(BODY.PEEK[])")
message = email.message_from_bytes(fetched[0][1])
subject, links, text = self.parse(message)
if not links:
without += 1
continue
forwarder = parseaddr(message.get("From", ""))[1]
if (self.allowed_senders and forwarder.lower() not in self.allowed_senders) or (
not self.allowed_senders and self.agent is not None): # fail closed
ignored += 1
continue
key = case_key(message)
recipient = original_recipient(text, forwarder, self.allowed_domains)
except Exception as e: # one broken mail must not stop the run
logger.warning("mail %s skipped: %s", uid, type(e).__name__)
continue
if not links:
without += 1
continue
forwarder = parseaddr(message.get("From", ""))[1]
sender = forwarder.lower()
if (self.allowed_senders and sender not in self.allowed_senders) or (
not self.allowed_senders and self.agent is not None): # fail closed
self._ignored += 1
continue
key = case_key(message)
uids[key] = uid
if self.store.add(key, subject, links, original_recipient(text, forwarder, self.allowed_domains)):
self._created += 1
return uids, without
if self.store.add(key, subject, links, recipient):
created += 1
return uids, without, created, ignored
def _handle(self, case: dict) -> str:
"""Run one case through the agent. Returns the final state; raises AgentUnavailable."""
@@ -63,6 +74,8 @@ class Worker:
try:
for link in case["links"][:MAX_LINKS]:
result = self.agent(link, case["address"])
if result["result"] not in FINAL_RESULTS:
raise ValueError("malformed agent result")
if result["result"] == "error":
raise AgentUnavailable(result.get("reason", "agent error"))
if result["result"] == "unsubscribed" or result.get("reason") == "login":
@@ -70,6 +83,10 @@ class Worker:
except AgentUnavailable:
self.store.release(case["key"])
raise
except Exception as e: # agent bug or malformed result: the attempt counts, the run goes on
logger.warning("agent failed on a case: %s", type(e).__name__)
self.store.finish(case["key"], "unclear", "agent error")
return "unclear"
self.store.finish(case["key"], result["result"], result.get("reason", ""), result.get("evidence", ""),
result.get("evidence_url", ""))
return result["result"]
@@ -86,7 +103,10 @@ class Worker:
if status != "OK":
logger.warning("could not move a mail to %s", self.done_folder)
continue
mail.uid("store", uid, "+FLAGS", "(\\Deleted)")
status, _ = mail.uid("store", uid, "+FLAGS", "(\\Deleted)")
if status != "OK":
logger.warning("could not flag a copied mail as deleted")
continue
self.store.mark_moved(case["key"])
moved += 1
mail.expunge()
@@ -101,8 +121,9 @@ class Worker:
return 0
try:
for case in expired:
if not case["key"].startswith("sha256:"): # mails without Message-ID cannot be found again
_, data = mail.uid("search", None, "HEADER", "Message-ID", '"' + case["key"].replace('"', "") + '"')
key = case["key"]
if not key.startswith("sha256:") and _searchable(key): # others cannot be found (or not safely)
_, data = mail.uid("search", None, "HEADER", "Message-ID", '"' + key + '"')
for uid in data[0].split():
mail.uid("store", uid, "+FLAGS", "(\\Deleted)")
self.store.delete(case["key"])
@@ -122,10 +143,13 @@ class Worker:
return self._note(summary)
mail = self.connect()
try:
uids, summary["without_link"] = self.scan(mail)
summary["new"], summary["ignored"] = self._created, self._ignored
uids, summary["without_link"], summary["new"], summary["ignored"] = self._scan(mail)
if self.agent is not None:
for case in self.store.pending(self.max_cases):
for case in self.store.pending(10_000):
if summary["handled"] >= self.max_cases:
break
if case["key"] not in uids: # only cases this scan vouched for (mail there, sender allowed)
continue
try:
state = self._handle(case)
except AgentUnavailable as e:
@@ -147,17 +171,28 @@ class Worker:
return summary
def _tick(worker: Worker) -> None:
"""One scheduled run. Never raises; notes only the exception type (messages can contain mail content)."""
try:
worker.run_once()
except Exception as e:
name = type(e).__name__
logger.warning("scheduled run failed: %s", name)
try:
summary = {k: 0 for k in SUMMARY_KEYS}
summary["error"] = name
worker.store.set_meta("last_run", json.dumps(summary))
worker.store.set_meta("last_run_at", str(int(worker.clock())))
except Exception as e2:
logger.warning("could not note the failed run: %s", type(e2).__name__)
def start_scheduler(worker: Worker, interval_seconds: int) -> threading.Thread:
"""Background thread: one run every interval. Errors (mailbox not connected, IMAP down) are noted, not raised."""
def loop():
while True:
try:
worker.run_once()
except Exception as e:
logger.warning("scheduled run failed: %s: %s", type(e).__name__, e)
worker.store.set_meta("last_run", json.dumps({"error": f"{type(e).__name__}: {e}"[:200]}))
worker.store.set_meta("last_run_at", str(int(worker.clock())))
_tick(worker)
time.sleep(interval_seconds)
thread = threading.Thread(target=loop, name="unsubscribe-scheduler", daemon=True)