Bugfixes and added notify ability to CLI

pyquarantine/__init__.py

@@ -112,7 +112,7 @@ class QuarantineMilter(Milter.Base):
                 for quarantine in self.config:
                     if len(self.recipients_quarantines) == len(self.recipients):
                         # every recipient matched a quarantine already
-                        if max([q["index"] for q in self.recipients_quarantines.values()]) <= quarantine["index"]:
+                        if min([q["index"] for q in self.recipients_quarantines.values()]) <= quarantine["index"]:
                              # every recipient matched a quarantine with at least the same precedence already, skip checks against quarantines with lower precedence
                              self.logger.debug("{}: {}: skip further checks of this header".format(self.queueid, quarantine["name"]))
                              break
@@ -135,7 +135,6 @@ class QuarantineMilter(Milter.Base):
 
                         # iterate recipients
                         for recipient in recipients_to_check.copy():
-
                             if recipient in whitelisted_recipients:
                                 # recipient is whitelisted in this quarantine
                                 self.logger.debug("{}: {}: recipient '{}' is whitelisted".format(self.queueid, quarantine["name"], recipient))
@@ -199,8 +198,9 @@ class QuarantineMilter(Milter.Base):
         try:
             # processing recipients grouped by quarantines
             quarantines = []
-            keyfunc = lambda x: self.recipients_quarantines[x]
+            for quarantine, recipients in groupby(
-            for quarantine, recipients in groupby(sorted(self.recipients_quarantines, key=keyfunc), keyfunc):
+                    sorted(self.recipients_quarantines, key=lambda x: self.recipients_quarantines[x]["index"])
+                    , lambda x: self.recipients_quarantines[x]):
                 quarantines.append((quarantine, list(recipients)))
 
             # iterate quarantines sorted by index
@@ -212,7 +212,7 @@ class QuarantineMilter(Milter.Base):
                     # add email to quarantine
                     self.logger.info("{}: adding to quarantine '{}' for: {}".format(self.queueid, quarantine["name"], ", ".join(recipients)))
                     try:
-                        quarantine_id = quarantine["quarantine_obj"].add(self.queueid, self.mailfrom, recipients, fp=self.fp)
+                        quarantine_id = quarantine["quarantine_obj"].add(self.queueid, self.mailfrom, recipients, self.subject, fp=self.fp)
                     except RuntimeError as e:
                         self.logger.error("{}: unable to add to quarantine '{}': {}".format(self.queueid, quarantine["name"], e))
                         return Milter.TEMPFAIL

pyquarantine/cli.py

@@ -98,11 +98,26 @@ def list_quarantine_emails(config, args):
         raise RuntimeError("quarantine type is set to None, unable to list emails")
 
     # find emails and transform some metadata values to strings
+    rows = []
     emails = quarantine.find(mailfrom=args.mailfrom, recipients=args.recipients, older_than=args.older_than)
     for quarantine_id, metadata in emails.items():
-        emails[quarantine_id]["quarantine_id"] = quarantine_id
+        row = emails[quarantine_id]
-        emails[quarantine_id]["recipient_str"] = ", ".join(metadata["recipients"])
+        row["quarantine_id"] = quarantine_id
-        emails[quarantine_id]["date_str"] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(metadata["date"]))
+        row["date"] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(metadata["date"]))
+        row["mailfrom"] = metadata["mailfrom"]
+        row["recipient"] = metadata["recipients"].pop(0)
+        row["subject"] = emails[quarantine_id]["subject"][:60]
+        rows.append(row)
+
+        if metadata["recipients"]:
+            row = {
+                "quarantine_id": "",
+                "date": "",
+                "mailfrom": "",
+                "recipient": metadata["recipients"].pop(0),
+                "subject": ""
+            }
+            rows.append(row)
 
     if args.batch:
         # batch mode, print quarantine IDs, each on a new line
@@ -111,8 +126,8 @@ def list_quarantine_emails(config, args):
 
     if not emails: logger.info("quarantine '{}' is empty".format(args.quarantine))
     print_table(
-        [("Quarantine-ID", "quarantine_id"), ("From", "from"), ("Recipient(s)", "recipient_str"), ("Date", "date_str")],
+        [("Quarantine-ID", "quarantine_id"), ("Date", "date"), ("From", "mailfrom"), ("Recipient(s)", "recipient"), ("Subject", "subject")],
-        emails.values()
+        rows
     )
 
 
@@ -179,7 +194,7 @@ def add_whitelist_entry(config, args):
 
     # add entry to whitelist
     whitelist.add(args.mailfrom, args.recipient, args.comment, args.permanent)
-    logger.info("successfully added whitelist entry")
+    logger.info("whitelist entry added successfully")
 
 
 def delete_whitelist_entry(config, args):
@@ -190,7 +205,17 @@ def delete_whitelist_entry(config, args):
         raise RuntimeError("whitelist type is set to None, unable to delete entries")
 
     whitelist.delete(args.whitelist_id)
-    logger.info("successfully deleted whitelist entry")
+    logger.info("whitelist entry deleted successfully")
+
+
+def notify_email(config, args):
+    logger = logging.getLogger(__name__)
+
+    quarantine = _get_quarantine_obj(config, args.quarantine)
+    if quarantine == None:
+        raise RuntimeError("quarantine type is set to None, unable to send notification")
+    quarantine.notify(args.quarantine_id, args.recipient)
+    logger.info("sent notification successfully")
 
 
 def release_email(config, args):
@@ -201,10 +226,7 @@ def release_email(config, args):
         raise RuntimeError("quarantine type is set to None, unable to release email")
 
     quarantine.release(args.quarantine_id, args.recipient)
-    if args.recipient:
+    logger.info("quarantined email released successfully")
-        logger.info("successfully released quarantined email '{}' to '{}' from quarantine '{}'".format(args.quarantine_id, args.recipient, args.quarantine))
-    else:
-        logger.info("successfully released quarantined email '{}' from quarantine '{}'".format(args.quarantine_id, args.quarantine))
 
 
 def delete_email(config, args):
@@ -215,10 +237,7 @@ def delete_email(config, args):
         raise RuntimeError("quarantine type is set to None, unable to delete email")
 
     quarantine.delete(args.quarantine_id, args.recipient)
-    if args.recipient:
+    logger.info("quarantined email deleted successfully")
-        logger.info("successfully deleted email [quarantine-id: {}] to '{}' from quarantine '{}'".format(args.quarantine_id, args.recipient, args.quarantine))
-    else:
-        logger.info("successfully deleted email [quarantine-id: {}] from quarantine '{}'".format(args.quarantine_id, args.quarantine))
 
 
 class StdErrFilter(logging.Filter):
@@ -240,7 +259,8 @@ def main():
             default=pyquarantine.QuarantineMilter.get_configfiles())
     parser.add_argument("-d", "--debug", help="Log debugging messages.", action="store_true")
     parser.set_defaults(syslog=False)
-    subparsers = parser.add_subparsers()
+    subparsers = parser.add_subparsers(dest="command", title="Commands")
+    subparsers.required = True
 
     # list command
     list_parser = subparsers.add_parser("list", help="List available quarantines.", formatter_class=formatter_class)
@@ -250,7 +270,8 @@ def main():
     # quarantine command group
     quarantine_parser = subparsers.add_parser("quarantine", description="Manage quarantines.", help="Manage quarantines.", formatter_class=formatter_class)
     quarantine_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
-    quarantine_subparsers = quarantine_parser.add_subparsers()
+    quarantine_subparsers = quarantine_parser.add_subparsers(dest="command", title="Quarantine commands")
+    quarantine_subparsers.required = True
     # quarantine list command
     quarantine_list_parser = quarantine_subparsers.add_parser("list", description="List emails in quarantines.", help="List emails in quarantine.", formatter_class=formatter_class)
     quarantine_list_parser.add_argument("-f", "--from", dest="mailfrom", help="Filter emails by from address.", default=None, nargs="+")
@@ -258,6 +279,13 @@ def main():
     quarantine_list_parser.add_argument("-o", "--older-than", dest="older_than", help="Filter emails by age (days).", default=None, type=float)
     quarantine_list_parser.add_argument("-b", "--batch", help="Print results using only email quarantine IDs, each on a new line.", action="store_true")
     quarantine_list_parser.set_defaults(func=list_quarantine_emails)
+    # quarantine notify command
+    quarantine_notify_parser = quarantine_subparsers.add_parser("notify", description="Notify recipient about email in quarantine.", help="Notify recipient about email in quarantine.", formatter_class=formatter_class)
+    quarantine_notify_parser.add_argument("quarantine_id", metavar="ID", help="Quarantine ID.")
+    quarantine_notify_parser_group = quarantine_notify_parser.add_mutually_exclusive_group(required=True)
+    quarantine_notify_parser_group.add_argument("-t", "--to", dest="recipient", help="Release email for one recipient address.")
+    quarantine_notify_parser_group.add_argument("-a", "--all", help="Release email for all recipients.", action="store_true")
+    quarantine_notify_parser.set_defaults(func=notify_email)
     # quarantine release command
     quarantine_release_parser = quarantine_subparsers.add_parser("release", description="Release email from quarantine.", help="Release email from quarantine.", formatter_class=formatter_class)
     quarantine_release_parser.add_argument("quarantine_id", metavar="ID", help="Quarantine ID.")
@@ -278,7 +306,8 @@ def main():
     # whitelist command group
     whitelist_parser = subparsers.add_parser("whitelist", description="Manage whitelists.", help="Manage whitelists.", formatter_class=formatter_class)
     whitelist_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
-    whitelist_subparsers = whitelist_parser.add_subparsers()
+    whitelist_subparsers = whitelist_parser.add_subparsers(dest="command", title="Whitelist commands")
+    whitelist_subparsers.required = True
     # whitelist list command
     whitelist_list_parser = whitelist_subparsers.add_parser("list", description="List whitelist entries.", help="List whitelist entries.", formatter_class=formatter_class)
     whitelist_list_parser.add_argument("-f", "--from", dest="mailfrom", help="Filter entries by from address.", default=None, nargs="+")

pyquarantine/notifications.py

@@ -32,7 +32,7 @@ class BaseNotification(object):
         self.config = config
         self.logger = logging.getLogger(__name__)
 
-    def notify(self, queueid, quarantine_id, subject, mailfrom, recipients, fp):
+    def notify(self, queueid, quarantine_id, subject, mailfrom, recipients, fp, synchronous=False):
         fp.seek(0)
         pass
 
@@ -143,7 +143,7 @@ class EMailNotification(BaseNotification):
         else:
             self.logger.debug("{}: content mimetype is {}".format(queueid, mimetype))
 
-        return BeautifulSoup(text, "lxml", from_encoding=part.get_content_charset())
+        return BeautifulSoup(text, "lxml")
 
     def get_text_multipart(self, queueid, msg, preferred=_html_text):
         "Get the mail text of a multipart email in html form."
@@ -205,9 +205,9 @@ class EMailNotification(BaseNotification):
 
         return soup
 
-    def notify(self, queueid, quarantine_id, subject, mailfrom, recipients, fp):
+    def notify(self, queueid, quarantine_id, subject, mailfrom, recipients, fp, synchronous=False):
         "Notify recipients via email."
-        super(EMailNotification, self).notify(queueid, quarantine_id, subject, mailfrom, recipients, fp)
+        super(EMailNotification, self).notify(queueid, quarantine_id, subject, mailfrom, recipients, fp, synchronous)
 
         # extract html text from email
         self.logger.debug("{}: extraction email text from original email".format(queueid))
@@ -252,7 +252,13 @@ class EMailNotification(BaseNotification):
                 msg.attach(self.replacement_img)
 
             self.logger.debug("{}: sending notification email to: {}".format(queueid, recipient))
-            mailer.sendmail(self.smtp_host, self.smtp_port, queueid, self.mailfrom, recipient, msg.as_string(), "notification email")
+            if synchronous:
+                try:
+                    mailer.smtp_send(self.smtp_host, self.smtp_port, self.mailfrom, recipient, msg.as_string())
+                except Exception as e:
+                    raise RuntimeError("error while sending email to '{}': {}".format(recipient, e))
+            else:
+                mailer.sendmail(self.smtp_host, self.smtp_port, queueid, self.mailfrom, recipient, msg.as_string(), "notification email")
 
 
 # list of notification types and their related notification classes

pyquarantine/quarantines.py

@@ -33,7 +33,7 @@ class BaseQuarantine(object):
         self.config = config
         self.logger = logging.getLogger(__name__)
 
-    def add(self, queueid, mailfrom, recipients, fp):
+    def add(self, queueid, mailfrom, recipients, subject, fp):
         "Add email to quarantine."
         fp.seek(0)
         return ""
@@ -50,6 +50,12 @@ class BaseQuarantine(object):
         "Delete email from quarantine."
         return
 
+    def notify(self, quarantine_id, recipient=None):
+        "Notify recipient about email in quarantine."
+        if not self.config["notification_obj"]:
+            raise RuntimeError("notification type is set to None, unable to send notifications")
+        return
+
     def release(self, quarantine_id, recipient=None):
         "Release email from quarantine."
         return
@@ -103,9 +109,9 @@ class FileQuarantine(BaseQuarantine):
         except IOError as e:
             raise RuntimeError("unable to remove data file: {}".format(e))
 
-    def add(self, queueid, mailfrom, recipients, fp):
+    def add(self, queueid, mailfrom, recipients, subject, fp):
         "Add email to file quarantine and return quarantine-id."
-        super(FileQuarantine, self).add(queueid, mailfrom, recipients, fp)
+        super(FileQuarantine, self).add(queueid, mailfrom, recipients, subject, fp)
         quarantine_id = "{}_{}".format(datetime.now().strftime("%Y%m%d%H%M%S"), queueid)
 
         # save mail
@@ -113,8 +119,9 @@ class FileQuarantine(BaseQuarantine):
 
         # save metadata
         metadata = {
-            "from": mailfrom,
+            "mailfrom": mailfrom,
             "recipients": recipients,
+            "subject": subject,
             "date": timegm(gmtime()),
             "queue_id": queueid
         }
@@ -163,7 +170,7 @@ class FileQuarantine(BaseQuarantine):
                     continue
 
             if mailfrom != None:
-                if metadata["from"] not in mailfrom:
+                if metadata["mailfrom"] not in mailfrom:
                     continue
 
             if recipients != None:
@@ -197,6 +204,30 @@ class FileQuarantine(BaseQuarantine):
             else:
                 self._save_metafile(quarantine_id, metadata)
 
+    def notify(self, quarantine_id, recipient=None):
+        "Notify recipient about email in quarantine."
+        super(FileQuarantine, self).notify(quarantine_id, recipient)
+
+        try:
+            metadata = self.get_metadata(quarantine_id)
+        except RuntimeError as e:
+            raise RuntimeError("unable to release email: {}".format(e))
+
+        if recipient != None:
+            if recipient not in metadata["recipients"]:
+                raise RuntimeError("invalid recipient '{}'".format(recipient))
+            recipients = [recipient]
+        else:
+            recipients = metadata["recipients"]
+
+        datafile = os.path.join(self.directory, quarantine_id)
+        try:
+            with open(datafile, "rb") as fp:
+                self.config["notification_obj"].notify(metadata["queue_id"], quarantine_id, metadata["subject"], metadata["mailfrom"], recipients, fp, synchronous=True)
+        except IOError as e:
+            raise(RuntimeError("unable to read data file: {}".format(e)))
+
+
     def release(self, quarantine_id, recipient=None):
         "Release email from quarantine."
         super(FileQuarantine, self).release(quarantine_id, recipient)
@@ -206,7 +237,6 @@ class FileQuarantine(BaseQuarantine):
         except RuntimeError as e:
             raise RuntimeError("unable to release email: {}".format(e))
 
-        datafile = os.path.join(self.directory, quarantine_id)
         if recipient != None:
             if recipient not in metadata["recipients"]:
                 raise RuntimeError("invalid recipient '{}'".format(recipient))
@@ -214,6 +244,7 @@ class FileQuarantine(BaseQuarantine):
         else:
             recipients = metadata["recipients"]
 
+        datafile = os.path.join(self.directory, quarantine_id)
         try:
             with open(datafile, "rb") as f:
                 mail = f.read()
@@ -222,7 +253,7 @@ class FileQuarantine(BaseQuarantine):
 
         for recipient in recipients:
             try:
-                mailer.smtp_send(self.config["smtp_host"], self.config["smtp_port"], metadata["from"], recipient, mail)
+                mailer.smtp_send(self.config["smtp_host"], self.config["smtp_port"], metadata["mailfrom"], recipient, mail)
             except Exception as e:
                 raise RuntimeError("error while sending email to '{}': {}".format(recipient, e))
 

pyquarantine/whitelists.py

@@ -247,7 +247,7 @@ class WhitelistCache(object):
 
     def get_whitelisted_recipients(self, whitelist, mailfrom, recipients):
         self.load(whitelist, mailfrom, recipients)
-        return filter(lambda x: self.cache[whitelist][x], self.cache[whitelist].keys())
+        return list(filter(lambda x: self.cache[whitelist][x], self.cache[whitelist].keys()))
 
 
 # list of whitelist types and their related whitelist classes