Befunde der Abschlussprüfung behoben

This commit is contained in:
Rsclub2_2 committed 2026-11-15 03:02:45 +01:00
1 parent 7bd3498501
commit 95a23c0e45
23 files changed
+651 -44

No files matched your search

+20
View File
@@ -24,6 +24,26 @@ def test_finds_anchor_and_plain_links_without_duplicates():
assert unsub.find_unsubscribe_links(html) == ["https://ex.org/u?id=1", "https://ex.org/unsubscribe/2"]
def test_escaped_ampersand_in_an_href_does_not_give_a_second_mangled_link():
html = ('<a href="https://ex.org/unsubscribe?u=1&amp;t=abc">Abmelden</a>'
' <a href="https://ex.org/unsubscribe-all?u=1">Von allen abmelden</a>')
assert unsub.find_unsubscribe_links(html) == ["https://ex.org/unsubscribe?u=1&t=abc",
"https://ex.org/unsubscribe-all?u=1"]
def test_plain_text_link_keeps_its_place_behind_an_escaped_duplicate():
html = ('<a href="https://ex.org/unsubscribe?u=1&amp;t=abc">Abmelden</a>'
' oder hier: https://ex.org/unsubscribe/2.')
assert unsub.find_unsubscribe_links(html) == ["https://ex.org/unsubscribe?u=1&t=abc",
"https://ex.org/unsubscribe/2"]
@pytest.mark.parametrize("tail", [".", ",", ";", ":", "!", "?", ")", "”", "’", "»", ").", "!”"])
def test_punctuation_after_a_plain_text_link_is_not_part_of_it(tail):
text = "Abmelden (siehe https://ex.org/unsubscribe/2" + tail + " danach)"
assert unsub.find_unsubscribe_links(text) == ["https://ex.org/unsubscribe/2"]
@pytest.mark.parametrize("href", ["javascript:alert(1)", "data:text/html,<script>x</script>", "JavaScript:x"])
def test_dangerous_links_are_dropped(href):
assert unsub.find_unsubscribe_links(f'<a href="{href}">Unsubscribe</a>') == []
+51
View File
@@ -83,6 +83,57 @@ def test_retry_makes_a_failed_case_pending_again(store):
assert (c["state"], c["attempts"], c["reason"]) == ("new", 0, "")
def test_defer_gives_the_attempt_back_and_counts_deferrals(store):
store.add("<1>", "s", ["https://e.org"], "")
store.claim("<1>")
assert store.defer("<1>") == 1
c = store.get("<1>")
assert (c["state"], c["attempts"], c["deferrals"]) == ("new", 0, 1)
store.claim("<1>")
assert store.defer("<1>") == 2
def test_pending_puts_deferred_cases_behind_the_others(store):
for k in ("<1>", "<2>"):
store.add(k, k, ["https://e.org"], "")
store.claim("<1>")
store.defer("<1>")
store.add("<3>", "<3>", ["https://e.org"], "")
assert [c["key"] for c in store.pending(10)] == ["<2>", "<3>", "<1>"]
def test_retry_resets_the_deferrals(store):
store.add("<1>", "s", ["https://e.org"], "")
store.claim("<1>")
store.defer("<1>")
store.claim("<1>")
store.finish("<1>", "unclear", "model unavailable for this case")
store.retry("<1>")
assert (store.get("<1>")["state"], store.get("<1>")["deferrals"]) == ("new", 0)
def test_database_without_the_deferrals_column_is_migrated(tmp_path):
import sqlite3
path = str(tmp_path / "old.db")
db = sqlite3.connect(path)
db.execute(
"CREATE TABLE cases ("
"key TEXT PRIMARY KEY, subject TEXT NOT NULL, links TEXT NOT NULL, address TEXT NOT NULL, "
"state TEXT NOT NULL, reason TEXT NOT NULL DEFAULT '', evidence TEXT NOT NULL DEFAULT '', "
"evidence_url TEXT NOT NULL DEFAULT '', attempts INTEGER NOT NULL DEFAULT 0, "
"moved INTEGER NOT NULL DEFAULT 0, created_at REAL NOT NULL, updated_at REAL NOT NULL)")
db.execute("INSERT INTO cases (key, subject, links, address, state, created_at, updated_at) "
"VALUES ('<old>', 's', '[]', '', 'new', 1.0, 1.0)")
db.commit()
db.close()
store = CaseStore(path)
assert store.get("<old>")["deferrals"] == 0
store.claim("<old>")
assert store.defer("<old>") == 1
CaseStore(path) # opening a migrated database again changes nothing
assert store.get("<old>")["deferrals"] == 1
def test_expired_only_moved_successes_older_than_days(store):
for k in ("<old>", "<young>", "<failed>"):
store.add(k, "s", ["https://e.org"], "")
+40
View File
@@ -52,6 +52,46 @@ def test_result_shows_cases_with_state_reason_evidence_and_links(unsub, monkeypa
assert "2 mail(s) from senders that are not allowed" in html
def test_max_deferrals_default_and_from_environment(unsub, monkeypatch):
import app
assert unsub.WORKER.max_deferrals == 12
monkeypatch.setenv("AGENT_MAX_DEFERRALS", "3")
assert importlib.reload(app).WORKER.max_deferrals == 3
def test_result_shows_cases_waiting_for_the_model(unsub, monkeypatch):
unsub.STORE.set_meta("last_run", json.dumps({"new": 0, "deferred": 2, "error": ""}))
monkeypatch.setattr(unsub, "scan_inbox", lambda: {})
html = unsub.app.test_client().get("/result").get_data(as_text=True)
assert "2 case(s) waiting for the model." in html
def test_retry_only_for_a_case_with_its_mail_in_the_inbox(unsub, monkeypatch):
s = unsub.STORE
for key in ("<here@x>", "<gone@x>"):
s.add(key, key, ["https://bank.example/u"], "")
s.claim(key)
s.finish(key, "not_possible", "login")
monkeypatch.setattr(unsub, "scan_inbox", lambda: {"<here@x>": b"42"})
html = unsub.app.test_client().get("/result").get_data(as_text=True)
assert html.count('action="/retry"') == 1 and 'name="key" value="&lt;here@x&gt;"' in html
assert 'name="key" value="&lt;gone@x&gt;"' not in html
def test_unmoved_success_shows_not_moved_yet(unsub, monkeypatch):
s = unsub.STORE
for key in ("<unmoved@x>", "<moved@x>"):
s.add(key, key, ["https://shop.example/u"], "")
s.claim(key)
s.finish(key, "unsubscribed", "page confirms")
s.mark_moved("<moved@x>")
monkeypatch.setattr(unsub, "scan_inbox", lambda: {"<unmoved@x>": b"7"})
html = unsub.app.test_client().get("/result").get_data(as_text=True)
assert html.count("not moved yet") == 1
row = html[html.index("&lt;unmoved@x&gt;"):]
assert row.index("not moved yet") < row.index("</tr>")
def test_retry_resets_the_case(unsub):
s = unsub.STORE
s.add("<no@x>", "s", ["https://bank.example/u"], "")
+29
View File
@@ -43,3 +43,32 @@ def test_subdomain_is_not_allowed_implicitly():
def test_malformed_forwarder_is_not_used():
assert original_recipient("no header", "a@b@wagnersnetz.de", ALLOWED) == ""
assert original_recipient("no header", "<x y@wagnersnetz.de>", ALLOWED) == ""
def test_full_address_entry_allows_only_that_address():
allowed = ("wagnersnetz.de", "Some.One@Gmail.com")
assert original_recipient("To: some.one@gmail.com\n", "", allowed) == "some.one@gmail.com"
assert original_recipient("To: SOME.ONE@GMAIL.COM\n", "", allowed) == "some.one@gmail.com"
assert original_recipient("To: someone.else@gmail.com\n", "", allowed) == ""
def test_body_line_with_a_third_persons_public_address_is_refused():
text = ("Von: news@shop.example\nAn: philipp@gmail.com\n\nWrite to us!\nTo: someone@gmail.com\n")
allowed = ("wagnersnetz.de", "philipp@gmail.com")
assert original_recipient("To: someone@gmail.com\n", "philipp@gmail.com", allowed) == "philipp@gmail.com"
assert original_recipient(text, "", allowed) == "philipp@gmail.com"
def test_mixed_list_of_domains_and_addresses():
allowed = ("wagnersnetz.de", "anne@gmail.com")
assert original_recipient("To: x@elsewhere.example, jens@wagnersnetz.de\n", "", allowed) == "jens@wagnersnetz.de"
assert original_recipient("To: bob@gmail.com, anne@gmail.com\n", "", allowed) == "anne@gmail.com"
assert original_recipient("no header", "anne@gmail.com", allowed) == "anne@gmail.com"
assert original_recipient("no header", "bob@gmail.com", allowed) == ""
def test_address_entry_does_not_allow_its_domain_or_a_lookalike():
allowed = ("anne@gmail.com",)
assert original_recipient("To: gmail.com@gmail.com\n", "", allowed) == ""
assert original_recipient("To: xanne@gmail.com\n", "", allowed) == ""
assert original_recipient("To: anne@gmail.com.evil.example\n", "", allowed) == ""
+138 -1
View File
@@ -32,6 +32,8 @@ class FakeIMAP:
self.fail_copy = False
self.logged_out = False
self.fail_store_once = False
self.search_status = "OK"
self.fail_fetch = set()
self.calls = []
def select(self, folder):
@@ -54,8 +56,12 @@ class FakeIMAP:
wanted = args[3].strip('"')
hits = [u for u, raw in box.items() if ("Message-ID: " + wanted).encode() in raw]
return "OK", [b" ".join(hits)]
if self.search_status != "OK":
return self.search_status, [b"search failed"]
return "OK", [b" ".join(box)]
if command == "fetch":
if args[0] in self.fail_fetch:
return "NO", [None]
return "OK", [(b"1 (BODY[] {n}", box[args[0]]), b")"]
if command == "copy":
if self.fail_copy or args[0] not in box:
@@ -161,6 +167,50 @@ def test_model_unavailable_result_counts_like_an_unavailable_agent(store):
assert summary["error"] == "llm_unavailable" and store.get("<a@x>")["attempts"] == 0
ERR = {"result": "error", "reason": "llm_unavailable"}
def test_model_error_defers_the_case_and_the_run_goes_on(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", link="https://shop.example/unsubscribe/a"),
b"2": mail_bytes("<b@x>", link="https://shop.example/unsubscribe/b")})
agent = Agent(ERR, OK)
summary = worker(store, imap, agent).run_once()
first, second = store.get("<a@x>"), store.get("<b@x>")
assert second["state"] == "unsubscribed"
assert (first["state"], first["attempts"], first["deferrals"]) == ("new", 0, 1)
assert summary["error"] == "" and summary["deferred"] == 1 and summary["handled"] == 1
def test_deferred_case_is_tried_after_newer_ones(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", link="https://shop.example/unsubscribe/a")})
agent = Agent(ERR)
w = worker(store, imap, agent)
assert w.run_once()["deferred"] == 1
imap.folders["INBOX"][b"2"] = mail_bytes("<b@x>", link="https://shop.example/unsubscribe/b")
agent.calls, agent.results = [], [OK, OK]
w.run_once()
assert [u for u, _ in agent.calls] == ["https://shop.example/unsubscribe/b", "https://shop.example/unsubscribe/a"]
def test_case_deferred_max_deferrals_times_ends_unclear(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
agent = Agent(ERR, ERR, ERR, OK)
w = worker(store, imap, agent, max_deferrals=3)
summaries = [w.run_once() for _ in range(4)]
c = store.get("<a@x>")
assert (c["state"], c["reason"]) == ("unclear", "model unavailable for this case")
assert len(agent.calls) == 3 and summaries[3]["handled"] == 0
assert [s["deferred"] for s in summaries[:2]] == [1, 1]
def test_transport_failure_still_stops_the_run_and_leaves_all_cases_untouched(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
agent = Agent(AgentUnavailable("agent not reachable"), OK)
summary = worker(store, imap, agent).run_once()
assert summary["error"] == "agent not reachable" and summary["deferred"] == 0 and len(agent.calls) == 1
assert [(c["state"], c["attempts"], c["deferrals"]) for c in store.all()] == [("new", 0, 0), ("new", 0, 0)]
def test_without_agent_cases_are_only_recorded(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<nolink@x>", link="")})
summary = worker(store, imap, None).run_once()
@@ -176,6 +226,21 @@ def test_paused_run_does_nothing(store):
assert summary["error"] == "paused" and agent.calls == [] and store.all() == []
def test_pause_during_a_run_stops_before_the_next_case(store):
imap = FakeIMAP({str(i).encode(): mail_bytes(f"<{i}@x>") for i in range(1, 4)})
class Pausing(Agent):
def __call__(self, url, address):
store.set_meta("paused", "1") # "Pause agent" pressed while the first case runs
return super().__call__(url, address)
agent = Pausing(OK, OK, OK)
summary = worker(store, imap, agent).run_once()
assert len(agent.calls) == 1 and summary["error"] == "paused"
assert summary["handled"] == 1 and summary["moved"] == 1 # what is finished is still moved
assert [c["state"] for c in store.all()].count("new") == 2
def test_at_most_max_cases_per_run(store):
imap = FakeIMAP({str(i).encode(): mail_bytes(f"<{i}@x>") for i in range(1, 5)})
agent = Agent(OK, OK, OK, OK)
@@ -272,6 +337,78 @@ def test_case_whose_mail_left_the_inbox_is_not_handled(store):
assert agent.calls == []
def test_failed_case_whose_mail_was_moved_to_the_trash_disappears(store):
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
w = worker(store, imap, Agent({"result": "not_possible", "reason": "captcha"}))
w.run_once()
assert store.get("<a@x>")["state"] == "not_possible"
imap.folders["INBOX"].pop(b"7") # "Move to Trash" on the page
summary = w.run_once()
assert store.get("<a@x>") is None and summary["gone"] == 1
def test_new_case_whose_mail_is_gone_is_deleted_but_a_running_one_is_left_alone(store):
old_case(store, "<new@x>")
old_case(store, "<running@x>")
store.claim("<running@x>")
summary = worker(store, FakeIMAP({}), Agent()).run_once()
assert store.get("<new@x>") is None and store.get("<running@x>")["state"] == "running"
assert summary["gone"] == 1
def test_unmoved_success_whose_mail_is_gone_counts_as_moved_and_expires(store):
imap = FakeIMAP({b"7": mail_bytes("<a@x>")})
imap.fail_copy = True
now = [time.time()]
w = worker(store, imap, Agent(OK), clock=lambda: now[0], retention_days=30)
w.run_once()
assert (store.get("<a@x>")["state"], store.get("<a@x>")["moved"]) == ("unsubscribed", 0)
imap.folders["INBOX"].pop(b"7") # moved by hand
summary = w.run_once()
assert store.get("<a@x>")["moved"] == 1 and summary["gone"] == 1
imap.folders["Erledigt"] = {}
now[0] = store.get("<a@x>")["updated_at"] + 31 * 86400
assert w.run_once()["cleaned"] == 1 and store.get("<a@x>") is None
@pytest.mark.parametrize("status", ["NO", "BAD"])
def test_failed_inbox_search_ends_the_run_with_an_error_and_deletes_nothing(store, status):
old_case(store, "<a@x>")
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
imap.search_status = status
agent = Agent(OK)
summary = worker(store, imap, agent).run_once()
assert summary["error"] == "inbox scan failed" and summary["gone"] == 0 and agent.calls == []
assert store.get("<a@x>")["state"] == "new" and imap.logged_out
assert json.loads(store.meta("last_run"))["error"] == "inbox scan failed"
def test_case_added_by_the_page_during_the_scan_is_not_tidied_away(store):
def parse_while_the_page_scans(message):
store.add("<arrived@x>", "s", ["https://shop.example/unsubscribe/9"], "") # page scan saw a newer mail
return parse(message)
imap = FakeIMAP({b"1": mail_bytes("<a@x>")})
w = Worker(store, lambda: imap, parse_while_the_page_scans, None, allowed_domains=("wagnersnetz.de",))
assert w.run_once()["gone"] == 0 and store.get("<arrived@x>") is not None
def test_unreadable_mail_in_the_inbox_means_nothing_is_tidied(store):
old_case(store, "<a@x>")
imap = FakeIMAP({b"1": mail_bytes("<a@x>"), b"2": mail_bytes("<b@x>")})
imap.fail_fetch = {b"1"} # could be the mail of <a@x>: its key is unknown
summary = worker(store, imap, Agent(OK, OK)).run_once()
assert store.get("<a@x>") is not None and summary["gone"] == 0
assert store.get("<b@x>")["state"] == "unsubscribed"
def test_case_of_a_mail_from_a_sender_no_longer_allowed_is_kept(store):
imap = FakeIMAP({b"1": mail_bytes("<a@x>", sender="Mallory <mallory@evil.example>")})
worker(store, imap, None, allowed_senders=()).run_once()
summary = worker(store, imap, Agent(OK), allowed_senders=("jens@wagnersnetz.de",)).run_once()
assert store.get("<a@x>")["state"] == "new" and summary["gone"] == 0
def test_skipped_cases_do_not_eat_the_budget(store):
for i in range(3):
old_case(store, f"<gone{i}@x>")
@@ -354,7 +491,7 @@ def test_failed_delete_flag_keeps_the_case_unmoved(store):
assert w.run_once()["moved"] == 1 and store.get("<a@x>")["moved"] == 1
ALL_KEYS = {"new", "handled", "unsubscribed", "moved", "cleaned", "without_link", "ignored", "error"}
ALL_KEYS = {"new", "handled", "unsubscribed", "moved", "cleaned", "without_link", "ignored", "error", "deferred", "gone"}
class Boom: