Befunde der Abschlussprüfung behoben
This commit is contained in:
23 files changed
+651
-44
No files matched your search
@@ -14,7 +14,9 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_LINKS = 2
|
||||
FINAL_RESULTS = ("unsubscribed", "not_possible", "unclear", "error")
|
||||
SUMMARY_KEYS = ("new", "handled", "unsubscribed", "moved", "cleaned", "without_link", "ignored", "error")
|
||||
SUMMARY_KEYS = ("new", "handled", "unsubscribed", "moved", "cleaned", "without_link", "ignored", "error", "deferred",
|
||||
"gone")
|
||||
MODEL_UNAVAILABLE = "model unavailable for this case"
|
||||
|
||||
|
||||
def _searchable(key: str) -> bool:
|
||||
@@ -23,15 +25,27 @@ def _searchable(key: str) -> bool:
|
||||
|
||||
|
||||
class AgentUnavailable(Exception):
|
||||
"""Agent container or model not reachable: the case keeps its attempt."""
|
||||
"""Agent container not reachable (transport failure): the case keeps its attempt and the run stops."""
|
||||
|
||||
|
||||
class ScanFailed(Exception):
|
||||
"""The inbox SEARCH did not answer OK: nothing about the inbox is known."""
|
||||
|
||||
|
||||
class _Deferred(Exception):
|
||||
"""The agent answered error (model unavailable for this page): the case waits, the run goes on."""
|
||||
|
||||
def __init__(self, reason: str, final: bool):
|
||||
super().__init__(reason)
|
||||
self.final = final
|
||||
|
||||
|
||||
class Worker:
|
||||
def __init__(self, store, connect, parse, agent, *, inbox="INBOX", done_folder="Erledigt", retention_days=30,
|
||||
max_cases=10, allowed_domains=(), allowed_senders=(), clock=time.time):
|
||||
max_cases=10, allowed_domains=(), allowed_senders=(), max_deferrals=12, clock=time.time):
|
||||
self.store, self.connect, self.parse, self.agent = store, connect, parse, agent
|
||||
self.inbox, self.done_folder = inbox, done_folder
|
||||
self.retention_days, self.max_cases = retention_days, max_cases
|
||||
self.retention_days, self.max_cases, self.max_deferrals = retention_days, max_cases, max_deferrals
|
||||
self.allowed_domains, self.clock = tuple(allowed_domains), clock
|
||||
self.allowed_senders = tuple(a.lower() for a in allowed_senders)
|
||||
self._lock = threading.Lock()
|
||||
@@ -40,14 +54,20 @@ class Worker:
|
||||
"""Create a case for every mail with unsubscribe links. Returns ({key: uid}, mails without link)."""
|
||||
return self._scan(mail)[:2]
|
||||
|
||||
def _scan(self, mail) -> tuple[dict, int, int, int]:
|
||||
"""Like scan, plus the number of created cases and of ignored mails (no instance state)."""
|
||||
_, data = mail.uid("search", None, "ALL")
|
||||
uids, without, created, ignored = {}, 0, 0, 0
|
||||
def _scan(self, mail) -> tuple[dict, int, int, int, set | None]:
|
||||
"""Like scan, plus the number of created cases, of ignored mails and the keys of all mails in the inbox
|
||||
(None if a mail could not be read, so its key is unknown). No instance state. Raises ScanFailed."""
|
||||
status, data = mail.uid("search", None, "ALL")
|
||||
if status != "OK":
|
||||
raise ScanFailed("inbox scan failed")
|
||||
uids, without, created, ignored, present, complete = {}, 0, 0, 0, set(), True
|
||||
for uid in data[0].split():
|
||||
key = None
|
||||
try:
|
||||
_, fetched = mail.uid("fetch", uid, "(BODY.PEEK[])")
|
||||
message = email.message_from_bytes(fetched[0][1])
|
||||
key = case_key(message)
|
||||
present.add(key)
|
||||
subject, links, text = self.parse(message)
|
||||
if not links:
|
||||
without += 1
|
||||
@@ -57,18 +77,36 @@ class Worker:
|
||||
not self.allowed_senders and self.agent is not None): # fail closed
|
||||
ignored += 1
|
||||
continue
|
||||
key = case_key(message)
|
||||
recipient = original_recipient(text, forwarder, self.allowed_domains)
|
||||
except Exception as e: # one broken mail must not stop the run
|
||||
logger.warning("mail %s skipped: %s", uid, type(e).__name__)
|
||||
if key is None:
|
||||
complete = False # not even its key is known
|
||||
continue
|
||||
uids[key] = uid
|
||||
if self.store.add(key, subject, links, recipient):
|
||||
created += 1
|
||||
return uids, without, created, ignored
|
||||
return uids, without, created, ignored, present if complete else None
|
||||
|
||||
def _tidy(self, present: set, since: float) -> int:
|
||||
"""Cases whose mail is no longer in the inbox: nothing left to do or to show, except an unmoved success,
|
||||
which now counts as moved so that it expires. Running cases, and cases created after the scan started (by
|
||||
the page's own scan, for a mail this scan did not see), are left alone."""
|
||||
gone = 0
|
||||
for case in self.store.all():
|
||||
if case["key"] in present or case["created_at"] >= since:
|
||||
continue
|
||||
if case["state"] in ("new", "not_possible", "unclear"):
|
||||
self.store.delete(case["key"])
|
||||
elif case["state"] == "unsubscribed" and not case["moved"]:
|
||||
self.store.mark_moved(case["key"])
|
||||
else:
|
||||
continue
|
||||
gone += 1
|
||||
return gone
|
||||
|
||||
def _handle(self, case: dict) -> str:
|
||||
"""Run one case through the agent. Returns the final state; raises AgentUnavailable."""
|
||||
"""Run one case through the agent. Returns the final state; raises AgentUnavailable or _Deferred."""
|
||||
self.store.claim(case["key"])
|
||||
result = {"result": "unclear", "reason": "no link"}
|
||||
try:
|
||||
@@ -77,12 +115,17 @@ class Worker:
|
||||
if result["result"] not in FINAL_RESULTS:
|
||||
raise ValueError("malformed agent result")
|
||||
if result["result"] == "error":
|
||||
raise AgentUnavailable(result.get("reason", "agent error"))
|
||||
raise _Deferred(str(result.get("reason") or "agent error"), False)
|
||||
if result["result"] == "unsubscribed" or result.get("reason") == "login":
|
||||
break
|
||||
except AgentUnavailable:
|
||||
self.store.release(case["key"])
|
||||
raise
|
||||
except _Deferred as e:
|
||||
if self.store.defer(case["key"]) >= self.max_deferrals:
|
||||
self.store.finish(case["key"], "unclear", MODEL_UNAVAILABLE)
|
||||
e.final = True
|
||||
raise
|
||||
except Exception as e: # agent bug or malformed result: the attempt counts, the run goes on
|
||||
logger.warning("agent failed on a case: %s", type(e).__name__)
|
||||
self.store.finish(case["key"], "unclear", "agent error")
|
||||
@@ -133,7 +176,8 @@ class Worker:
|
||||
return len(expired)
|
||||
|
||||
def run_once(self) -> dict:
|
||||
summary = {"new": 0, "handled": 0, "unsubscribed": 0, "moved": 0, "cleaned": 0, "without_link": 0, "ignored": 0, "error": ""}
|
||||
summary = {k: 0 for k in SUMMARY_KEYS}
|
||||
summary["error"] = ""
|
||||
if not self._lock.acquire(blocking=False):
|
||||
summary["error"] = "already running"
|
||||
return summary
|
||||
@@ -143,20 +187,40 @@ class Worker:
|
||||
return self._note(summary)
|
||||
mail = self.connect()
|
||||
try:
|
||||
uids, summary["without_link"], summary["new"], summary["ignored"] = self._scan(mail)
|
||||
started = time.time() # the store's clock, not self.clock
|
||||
try:
|
||||
uids, summary["without_link"], summary["new"], summary["ignored"], present = self._scan(mail)
|
||||
except ScanFailed as e: # nothing is known about the inbox: change nothing
|
||||
summary["error"] = str(e)
|
||||
return self._note(summary)
|
||||
if present is not None: # only a complete scan may tidy up
|
||||
summary["gone"] = self._tidy(present, started)
|
||||
if self.agent is not None:
|
||||
deferral = ""
|
||||
for case in self.store.pending(10_000):
|
||||
if summary["handled"] >= self.max_cases:
|
||||
if summary["handled"] + summary["deferred"] >= self.max_cases: # every agent call counts
|
||||
break
|
||||
if case["key"] not in uids: # only cases this scan vouched for (mail there, sender allowed)
|
||||
continue
|
||||
if self.store.meta("paused") == "1": # "Pause agent" also stops a run in progress
|
||||
summary["error"] = "paused"
|
||||
break
|
||||
try:
|
||||
state = self._handle(case)
|
||||
except AgentUnavailable as e:
|
||||
summary["error"] = str(e)
|
||||
break
|
||||
except _Deferred as e:
|
||||
if e.final: # deferred too often: finished as unclear
|
||||
summary["handled"] += 1
|
||||
else:
|
||||
summary["deferred"] += 1
|
||||
deferral = str(e)
|
||||
continue
|
||||
summary["handled"] += 1
|
||||
summary["unsubscribed"] += state == "unsubscribed"
|
||||
if deferral and not summary["handled"] and not summary["error"]:
|
||||
summary["error"] = deferral # no case could be handled at all
|
||||
summary["moved"] = self._move(mail, uids)
|
||||
summary["cleaned"] = self._cleanup(mail)
|
||||
finally:
|
||||
|
||||
Reference in new issue
Block a user