Improve email body extraction and decoding

README.md

@@ -45,7 +45,7 @@ The following configuration options are mandatory in each quarantine section:
 * **whitelist_type**  
   One of the whitelist types described below.
 * **smtp_host**  
-  SMTP host to inject original e-mails. This is needed if not all recipients of an e-mail are whitelisted
+  SMTP host used to release original e-mails from the quarantine.
 * **smtp_port**  
   SMTP port
 
@@ -87,7 +87,7 @@ The following configuration options are optional in each quarantine section:
   * **{EMAIL_QUARANTINE_ID}**  
     Quarantine-ID of the original e-mail if available, empty otherwise.
   * **{EMAIL_HTML_TEXT}**  
-    Sanitized version of the e-mail text part of the original e-mail. Only harmless HTML tags and attributes are included. Images are replaced with the image set by notification_email_replacement_img option.
+    Sanitized version of the e-mail text part of the original e-mail. Only harmless HTML tags and attributes are included. Images are optionally stripped or replaced with the image set by notification_email_replacement_img option.
 
   Some template variables are only available if the regex of the matching quarantine contains subgroups or named subgroups (python syntax). This is useful to include information (e.g. virus names, spam points, ...) of the matching header within the notification.  
   The following dynamic template variables are available:
@@ -97,20 +97,31 @@ The following configuration options are optional in each quarantine section:
     Content of a named subgroup, 'subgroup_name' will be replaced by its name.
 
   The following configuration options are mandatory for this notification type:
+  * **notification_email_smtp_host**  
+    SMTP host used to send notification e-mails.
+  * **notification_email_smtp_port**  
+    SMTP port.
   * **notification_email_envelope_from**  
     Notification e-mail envelope from-address.
   * **notification_email_from**  
-    Value of the notification e-mail from header. Every e-mail template variable described above is usable.
+    Value of the notification e-mail from header. Optionally, you may use the EMAIL_FROM template variable described above.
   * **notification_email_subject**  
-    Notification e-mail subject. All e-mail template variable described above is usable.
+    Notification e-mail subject. Optionally, you may use the EMAIL_SUBJECT template variable described above.
   * **notification_email_template**  
     Path to the notification e-mail template. It is hold in memory during runtime.
-  * **notification_email_replacement_img**  
-    Path to the image to replace images in e-mails. It is hold in memory during runtime. Leave it empty to disable.
   * **notification_email_embedded_imgs**  
     Comma-separated list of images to embed into the notification e-mail. The Content-ID of each image will be set to the filename, so you can reference it from the e-mail template. All images are hold in memory during runtime.  
     Leave empty to disable.
 
+  The following configuration options are optional for this notification type:
+  * **notification_email_strip_images**  
+    Enable to strip images from e-mails. This option superseeds notification_email_replacement_img.
+  * **notification_email_replacement_img**  
+    Path to an image to replace images in e-mails. It is hold in memory during runtime.
+  * **notification_email_parser_lib**  
+    HTML parser library used to parse text part of emails.
+
+
 ### Actions
 Every quarantine responds with a milter-action if an e-mail header matches the configured regular expression. Please think carefully what you set here or your MTA will do something you do not want.  
 The following actions are available:

docs/pyquarantine.conf.example

@@ -88,6 +88,18 @@ reject_reason = Message rejected
 #
 notification_type = email
 
+# Option: notification_email_smtp_host
+# Notes:  Set the SMTP host. It will be used to send notification e-mails.
+# Values: [ HOSTNAME | IP_ADDRESS ]
+#
+notification_email_smtp_host = 127.0.0.1
+
+# Option: notification_email_smtp_port
+# Notes:  Set the SMTP port.
+# Values: [ PORT ]
+#
+notification_email_smtp_port = 25
+
 # Option: notification_email_envelope_from
 # Notes:  Set the envelope-from address used when sending notification emails.
 #         This option is needed by notification type 'email'.
@@ -117,10 +129,15 @@ notification_email_subject = Spam Quarantine Notification
 #
 notification_email_template = templates/notification.template
 
+# Option: notification_email_strip_images
+# Notes:  Optionally enable this option to strip img tags from emails.
+# Values: [ TRUE | ON | YES | FALSE | OFF | NO ]
+#
+notification_email_strip_images = False
+
 # Option: notification_email_replacement_img
-# Notes:  Set the path to the replacement image for img tags within emails.
+# Notes:  Optionally set the path to a replacement image for img tags within emails.
 #         A relative path to this config file can be used.
-#         This option is needed by notification type 'email'.
 # Values: [ IMAGE_PATH ]
 #
 notification_email_replacement_img = templates/removed.png
@@ -133,6 +150,13 @@ notification_email_replacement_img = templates/removed.png
 #
 notification_email_embedded_imgs = templates/logo.png
 
+# Option: notification_email_parser_lib
+# Notes:  Optionally set the parser library used to parse
+#         the text part of emails.
+# Values: [ lxml | html.parser ]
+#
+notification_email_parser_lib = lxml
+
 # Option: whitelist_type
 # Notes:  Set the whitelist type.
 # Values: [ db | none ]

pyquarantine/__init__.py

@@ -12,7 +12,18 @@
 # along with PyQuarantineMilter.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-__all__ = ["QuarantineMilter", "generate_milter_config", "reload_config", "mailer", "notifications", "run", "quarantines", "whitelists"]
+__all__ = [
+    "QuarantineMilter",
+    "generate_milter_config",
+    "reload_config",
+    "cli",
+    "mailer",
+    "notifications",
+    "quarantines",
+    "run",
+    "version",
+    "whitelists"]
+
 name = "pyquarantine"
 
 import Milter
@@ -42,9 +53,15 @@ class QuarantineMilter(Milter.Base):
     global_config = None
 
     # list of default config files
-    _config_files = ["/etc/pyquarantine/pyquarantine.conf", os.path.expanduser('~/pyquarantine.conf'), "pyquarantine.conf"]
+    _config_files = [
+        "/etc/pyquarantine/pyquarantine.conf",
+        os.path.expanduser('~/pyquarantine.conf'),
+        "pyquarantine.conf"]
     # list of possible actions
-    _actions = {"ACCEPT": Milter.ACCEPT, "REJECT": Milter.REJECT, "DISCARD": Milter.DISCARD}
+    _actions = {
+        "ACCEPT": Milter.ACCEPT,
+        "REJECT": Milter.REJECT,
+        "DISCARD": Milter.DISCARD}
 
     def __init__(self):
         self.logger = logging.getLogger(__name__)
@@ -53,11 +70,17 @@ class QuarantineMilter(Milter.Base):
         self.config = QuarantineMilter.config
 
     def _get_preferred_quarantine(self):
-        matching_quarantines = [q for q in self.recipients_quarantines.values() if q]
+        matching_quarantines = [
+            q for q in self.recipients_quarantines.values() if q]
         if self.global_config["preferred_quarantine_action"] == "first":
-            quarantine = sorted(matching_quarantines, key=lambda x: x["index"])[0]
+            quarantine = sorted(
+                matching_quarantines,
+                key=lambda x: x["index"])[0]
         else:
-            quarantine = sorted(matching_quarantines, key=lambda x: x["index"], reverse=True)[0]
+            quarantine = sorted(
+                matching_quarantines,
+                key=lambda x: x["index"],
+                reverse=True)[0]
         return quarantine
 
     @staticmethod
@@ -73,16 +96,23 @@ class QuarantineMilter(Milter.Base):
         QuarantineMilter._config_files = config_files
 
     def connect(self, IPname, family, hostaddr):
-        self.logger.debug("accepted milter connection from {} port {}".format(*hostaddr))
+        self.logger.debug(
+            "accepted milter connection from {} port {}".format(
+                *hostaddr))
         ip = IPAddress(hostaddr[0])
         for quarantine in self.config.copy():
             for ignore in quarantine["ignore_hosts_list"]:
                 if ip in ignore:
-                    self.logger.debug("host {} is ignored by quarantine {}".format(hostaddr[0], quarantine["name"]))
+                    self.logger.debug(
+                        "host {} is ignored by quarantine {}".format(
+                            hostaddr[0], quarantine["name"]))
                     self.config.remove(quarantine)
                     break
             if not self.config:
-                self.logger.debug("host {} is ignored by all quarantines, skip further processing", hostaddr[0])
+                self.logger.debug(
+                    "host {} is ignored by all quarantines, "
+                    "skip further processing",
+                    hostaddr[0])
                 return Milter.ACCEPT
         return Milter.CONTINUE
 
@@ -100,7 +130,8 @@ class QuarantineMilter(Milter.Base):
     @Milter.noreply
     def data(self):
         self.queueid = self.getsymval('i')
-        self.logger.debug("{}: received queue-id from MTA".format(self.queueid))
+        self.logger.debug(
+            "{}: received queue-id from MTA".format(self.queueid))
         self.recipients = list(self.recipients)
         self.headers = []
         return Milter.CONTINUE
@@ -122,28 +153,43 @@ class QuarantineMilter(Milter.Base):
             recipients_to_check = self.recipients.copy()
             for name, value in self.headers:
                 header = "{}: {}".format(name, value)
-                self.logger.debug("{}: checking header against configured quarantines: {}".format(self.queueid, header))
+                self.logger.debug(
+                    "{}: checking header against configured quarantines: {}".format(
+                        self.queueid, header))
                 # iterate quarantines
                 for quarantine in self.config:
-                    if len(self.recipients_quarantines) == len(self.recipients):
+                    if len(self.recipients_quarantines) == len(
+                            self.recipients):
                         # every recipient matched a quarantine already
-                        if quarantine["index"] >= max([q["index"] for q in self.recipients_quarantines.values()]):
-                             # all recipients matched a quarantine with at least the same precedence already, skip checks against quarantines with lower precedence
-                             self.logger.debug("{}: {}: skip further checks of this header".format(self.queueid, quarantine["name"]))
+                        if quarantine["index"] >= max(
+                                [q["index"] for q in self.recipients_quarantines.values()]):
+                            # all recipients matched a quarantine with at least
+                            # the same precedence already, skip checks against
+                            # quarantines with lower precedence
+                            self.logger.debug(
+                                "{}: {}: skip further checks of this header".format(
+                                    self.queueid, quarantine["name"]))
                             break
 
                     # check email header against quarantine regex
-                    self.logger.debug("{}: {}: checking header against regex '{}'".format(self.queueid, quarantine["name"], quarantine["regex"]))
+                    self.logger.debug(
+                        "{}: {}: checking header against regex '{}'".format(
+                            self.queueid, quarantine["name"], quarantine["regex"]))
                     match = quarantine["regex_compiled"].search(header)
                     if match:
-                        self.logger.debug("{}: {}: header matched regex".format(self.queueid, quarantine["name"]))
+                        self.logger.debug(
+                            "{}: {}: header matched regex".format(
+                                self.queueid, quarantine["name"]))
                         # check for whitelisted recipients
                         whitelist = quarantine["whitelist_obj"]
-                        if whitelist != None:
+                        if whitelist is not None:
                             try:
-                                whitelisted_recipients = self.whitelist_cache.get_whitelisted_recipients(whitelist, self.mailfrom, recipients_to_check)
+                                whitelisted_recipients = self.whitelist_cache.get_whitelisted_recipients(
+                                    whitelist, self.mailfrom, recipients_to_check)
                             except RuntimeError as e:
-                                self.logger.error("{}: {}: unable to query whitelist: {}".format(self.queueid, quarantine["name"], e))
+                                self.logger.error(
+                                    "{}: {}: unable to query whitelist: {}".format(
+                                        self.queueid, quarantine["name"], e))
                                 return Milter.TEMPFAIL
                         else:
                             whitelisted_recipients = {}
@@ -152,28 +198,44 @@ class QuarantineMilter(Milter.Base):
                         for recipient in recipients_to_check.copy():
                             if recipient in whitelisted_recipients:
                                 # recipient is whitelisted in this quarantine
-                                self.logger.debug("{}: {}: recipient '{}' is whitelisted".format(self.queueid, quarantine["name"], recipient))
+                                self.logger.debug(
+                                    "{}: {}: recipient '{}' is whitelisted".format(
+                                        self.queueid, quarantine["name"], recipient))
                                 continue
 
-                            if recipient not in self.recipients_quarantines.keys() or self.recipients_quarantines[recipient]["index"] > quarantine["index"]:
-                                self.logger.debug("{}: {}: set quarantine for recipient '{}'".format(self.queueid, quarantine["name"], recipient))
-                                # save match for later use as template variables
+                            if recipient not in self.recipients_quarantines.keys() or \
+                                    self.recipients_quarantines[recipient]["index"] > quarantine["index"]:
+                                self.logger.debug(
+                                    "{}: {}: set quarantine for recipient '{}'".format(
+                                        self.queueid, quarantine["name"], recipient))
+                                # save match for later use as template
+                                # variables
                                 self.quarantines_matches[quarantine["name"]] = match
                                 self.recipients_quarantines[recipient] = quarantine
                                 if quarantine["index"] == 0:
-                                    # we do not need to check recipients which matched the quarantine with the highest precedence already
+                                    # we do not need to check recipients which
+                                    # matched the quarantine with the highest
+                                    # precedence already
                                     recipients_to_check.remove(recipient)
                             else:
-                                self.logger.debug("{}: {}: a quarantine with the same or higher precedence matched already for recipient '{}'".format(self.queueid, quarantine["name"], recipient))
+                                self.logger.debug(
+                                    "{}: {}: a quarantine with same or higher precedence "
+                                    "matched already for recipient '{}'".format(
+                                        self.queueid, quarantine["name"], recipient))
 
                 if not recipients_to_check:
-                    self.logger.debug("{}: all recipients matched the first quarantine, skipping all remaining header checks".format(self.queueid))
+                    self.logger.debug(
+                        "{}: all recipients matched the first quarantine, "
+                        "skipping all remaining header checks".format(
+                            self.queueid))
                     break
 
             # check if no quarantine has matched for all recipients
             if not self.recipients_quarantines:
                 # accept email
-                self.logger.info("{}: passed clean for all recipients".format(self.queueid))
+                self.logger.info(
+                    "{}: passed clean for all recipients".format(
+                        self.queueid))
                 return Milter.ACCEPT
 
             # check if the email body is needed
@@ -184,7 +246,9 @@ class QuarantineMilter(Milter.Base):
                     break
 
             if keep_body:
-                self.logger.debug("{}: initializing memory buffer to save email data".format(self.queueid))
+                self.logger.debug(
+                    "{}: initializing memory buffer to save email data".format(
+                        self.queueid))
                 # initialize memory buffer to save email data
                 self.fp = BytesIO()
                 # write email headers to memory buffer
@@ -192,10 +256,15 @@ class QuarantineMilter(Milter.Base):
                     self.fp.write("{}: {}\n".format(name, value).encode())
                 self.fp.write("\n".encode())
             else:
-                # quarantine and notification are disabled on all matching quarantines, return configured action
+                # quarantine and notification are disabled on all matching
+                # quarantines, return configured action
                 quarantine = self._get_preferred_quarantine()
-                self.logger.info("{}: {} matching quarantine is '{}', performing milter action {}".format(self.queueid, self.global_config["preferred_quarantine_action"],
-                        quarantine["name"], quarantine["action"].upper()))
+                self.logger.info(
+                    "{}: {} matching quarantine is '{}', performing milter action {}".format(
+                        self.queueid,
+                        self.global_config["preferred_quarantine_action"],
+                        quarantine["name"],
+                        quarantine["action"].upper()))
                 if quarantine["action"] == "reject":
                     self.setreply("554", "5.7.0", quarantine["reject_reason"])
                 return quarantine["milter_action"]
@@ -203,7 +272,8 @@ class QuarantineMilter(Milter.Base):
             return Milter.CONTINUE
 
         except Exception as e:
-            self.logger.exception("an exception occured in eoh function: {}".format(e))
+            self.logger.exception(
+                "an exception occured in eoh function: {}".format(e))
             return Milter.TEMPFAIL
 
     def body(self, chunk):
@@ -211,7 +281,8 @@ class QuarantineMilter(Milter.Base):
             # save received body chunk
             self.fp.write(chunk)
         except Exception as e:
-            self.logger.exception("an exception occured in body function: {}".format(e))
+            self.logger.exception(
+                "an exception occured in body function: {}".format(e))
             return Milter.TEMPFAIL
         return Milter.CONTINUE
 
@@ -220,39 +291,53 @@ class QuarantineMilter(Milter.Base):
             # processing recipients grouped by quarantines
             quarantines = []
             for quarantine, recipients in groupby(
-                    sorted(self.recipients_quarantines, key=lambda x: self.recipients_quarantines[x]["index"])
-                    , lambda x: self.recipients_quarantines[x]):
+                    sorted(self.recipients_quarantines,
+                           key=lambda x: self.recipients_quarantines[x]["index"]),
+                    lambda x: self.recipients_quarantines[x]):
                 quarantines.append((quarantine, list(recipients)))
 
             # iterate quarantines sorted by index
-            for quarantine, recipients in sorted(quarantines, key=lambda x: x[0]["index"]):
+            for quarantine, recipients in sorted(
+                    quarantines, key=lambda x: x[0]["index"]):
                 quarantine_id = ""
                 headers = defaultdict(str)
                 for name, value in self.headers:
                     headers[name.lower()] = value
-                subgroups = self.quarantines_matches[quarantine["name"]].groups(default="")
-                named_subgroups = self.quarantines_matches[quarantine["name"]].groupdict(default="")
+                subgroups = self.quarantines_matches[quarantine["name"]].groups(
+                    default="")
+                named_subgroups = self.quarantines_matches[quarantine["name"]].groupdict(
+                    default="")
 
                 # check if a quarantine is configured
-                if quarantine["quarantine_obj"] != None:
+                if quarantine["quarantine_obj"] is not None:
                     # add email to quarantine
-                    self.logger.info("{}: adding to quarantine '{}' for: {}".format(self.queueid, quarantine["name"], ", ".join(recipients)))
+                    self.logger.info("{}: adding to quarantine '{}' for: {}".format(
+                        self.queueid, quarantine["name"], ", ".join(recipients)))
                     try:
-                        quarantine_id = quarantine["quarantine_obj"].add(self.queueid, self.mailfrom, recipients, headers, self.fp,
+                        quarantine_id = quarantine["quarantine_obj"].add(
+                            self.queueid, self.mailfrom, recipients, headers, self.fp,
                             subgroups, named_subgroups)
                     except RuntimeError as e:
-                        self.logger.error("{}: unable to add to quarantine '{}': {}".format(self.queueid, quarantine["name"], e))
+                        self.logger.error(
+                            "{}: unable to add to quarantine '{}': {}".format(
+                                self.queueid, quarantine["name"], e))
                         return Milter.TEMPFAIL
 
                 # check if a notification is configured
-                if quarantine["notification_obj"] != None:
+                if quarantine["notification_obj"] is not None:
                     # notify
-                    self.logger.info("{}: sending notification for quarantine '{}' to: {}".format(self.queueid, quarantine["name"], ", ".join(recipients)))
+                    self.logger.info(
+                        "{}: sending notification for quarantine '{}' to: {}".format(
+                            self.queueid, quarantine["name"], ", ".join(recipients)))
                     try:
-                        quarantine["notification_obj"].notify(self.queueid, quarantine_id, self.mailfrom, recipients, headers, self.fp,
+                        quarantine["notification_obj"].notify(
+                            self.queueid, quarantine_id,
+                            self.mailfrom, recipients, headers, self.fp,
                             subgroups, named_subgroups)
                     except RuntimeError as e:
-                        self.logger.error("{}: unable to send notification for quarantine '{}': {}".format(self.queueid, quarantine["name"], e))
+                        self.logger.error(
+                            "{}: unable to send notification for quarantine '{}': {}".format(
+                                self.queueid, quarantine["name"], e))
                         return Milter.TEMPFAIL
 
                 # remove processed recipient
@@ -264,19 +349,27 @@ class QuarantineMilter(Milter.Base):
 
             # email passed clean for at least one recipient, accepting email
             if self.recipients:
-                self.logger.info("{}: passed clean for: {}".format(self.queueid, ", ".join(self.recipients)))
+                self.logger.info(
+                    "{}: passed clean for: {}".format(
+                        self.queueid, ", ".join(
+                            self.recipients)))
                 return Milter.ACCEPT
 
-            ## return configured action
+            # return configured action
             quarantine = self._get_preferred_quarantine()
-            self.logger.info("{}: {} matching quarantine is '{}', performing milter action {}".format(self.queueid, self.global_config["preferred_quarantine_action"],
-                    quarantine["name"], quarantine["action"].upper()))
+            self.logger.info(
+                "{}: {} matching quarantine is '{}', performing milter action {}".format(
+                    self.queueid,
+                    self.global_config["preferred_quarantine_action"],
+                    quarantine["name"],
+                    quarantine["action"].upper()))
             if quarantine["action"] == "reject":
                 self.setreply("554", "5.7.0", quarantine["reject_reason"])
             return quarantine["milter_action"]
 
         except Exception as e:
-            self.logger.exception("an exception occured in eom function: {}".format(e))
+            self.logger.exception(
+                "an exception occured in eom function: {}".format(e))
             return Milter.TEMPFAIL
 
 
@@ -298,24 +391,30 @@ def generate_milter_config(configtest=False, config_files=[]):
 
     # check if mandatory config options in global section are present
     if "global" not in parser.sections():
-        raise RuntimeError("mandatory section 'global' not present in config file")
+        raise RuntimeError(
+            "mandatory section 'global' not present in config file")
     for option in ["quarantines", "preferred_quarantine_action"]:
         if not parser.has_option("global", option):
-            raise RuntimeError("mandatory option '{}' not present in config section 'global'".format(option))
+            raise RuntimeError(
+                "mandatory option '{}' not present in config section 'global'".format(option))
 
     # read global config section
     global_config = dict(parser.items("global"))
     global_config["preferred_quarantine_action"] = global_config["preferred_quarantine_action"].lower()
     if global_config["preferred_quarantine_action"] not in ["first", "last"]:
-        raise RuntimeError("option preferred_quarantine_action has illegal value")
+        raise RuntimeError(
+            "option preferred_quarantine_action has illegal value")
 
     # read active quarantine names
-    quarantine_names = [ q.strip() for q in global_config["quarantines"].split(",") ]
+    quarantine_names = [
+        q.strip() for q in global_config["quarantines"].split(",")]
     if len(quarantine_names) != len(set(quarantine_names)):
-        raise RuntimeError("at least one quarantine is specified multiple times in quarantines option")
+        raise RuntimeError(
+            "at least one quarantine is specified multiple times in quarantines option")
     if "global" in quarantine_names:
         quarantine_names.remove("global")
-        logger.warning("removed illegal quarantine name 'global' from list of active quarantines")
+        logger.warning(
+            "removed illegal quarantine name 'global' from list of active quarantines")
     if not quarantine_names:
         raise RuntimeError("no quarantines configured")
 
@@ -327,16 +426,20 @@ def generate_milter_config(configtest=False, config_files=[]):
 
         # check if config section for current quarantine exists
         if quarantine_name not in parser.sections():
-            raise RuntimeError("config section '{}' does not exist".format(quarantine_name))
+            raise RuntimeError(
+                "config section '{}' does not exist".format(quarantine_name))
         config = dict(parser.items(quarantine_name))
 
         # check if mandatory config options are present in config
-        for option in ["regex", "quarantine_type", "notification_type", "action", "whitelist_type", "smtp_host", "smtp_port"]:
+        for option in ["regex", "quarantine_type", "notification_type",
+                       "action", "whitelist_type", "smtp_host", "smtp_port"]:
             if option not in config.keys() and \
                     option in global_config.keys():
                 config[option] = global_config[option]
             if option not in config.keys():
-                raise RuntimeError("mandatory option '{}' not present in config section '{}' or 'global'".format(option, quarantine_name))
+                raise RuntimeError(
+                    "mandatory option '{}' not present in config section '{}' or 'global'".format(
+                        option, quarantine_name))
 
         # check if optional config options are present in config
         defaults = {
@@ -357,45 +460,67 @@ def generate_milter_config(configtest=False, config_files=[]):
         config["index"] = index
 
         # pre-compile regex
-        logger.debug("{}: compiling regex '{}'".format(quarantine_name, config["regex"]))
-        config["regex_compiled"] = re.compile(config["regex"], re.MULTILINE + re.DOTALL + re.IGNORECASE)
+        logger.debug(
+            "{}: compiling regex '{}'".format(
+                quarantine_name,
+                config["regex"]))
+        config["regex_compiled"] = re.compile(
+            config["regex"], re.MULTILINE + re.DOTALL + re.IGNORECASE)
 
         # create quarantine instance
         quarantine_type = config["quarantine_type"].lower()
         if quarantine_type in quarantines.TYPES.keys():
-            logger.debug("{}: initializing quarantine type '{}'".format(quarantine_name, quarantine_type.upper()))
-            quarantine = quarantines.TYPES[quarantine_type](global_config, config, configtest)
+            logger.debug(
+                "{}: initializing quarantine type '{}'".format(
+                    quarantine_name,
+                    quarantine_type.upper()))
+            quarantine = quarantines.TYPES[quarantine_type](
+                global_config, config, configtest)
         elif quarantine_type == "none":
             logger.debug("{}: quarantine is NONE".format(quarantine_name))
             quarantine = None
         else:
-            raise RuntimeError("{}: unknown quarantine type '{}'".format(quarantine_name, quarantine_type))
+            raise RuntimeError(
+                "{}: unknown quarantine type '{}'".format(
+                    quarantine_name, quarantine_type))
 
         config["quarantine_obj"] = quarantine
 
         # create whitelist instance
         whitelist_type = config["whitelist_type"].lower()
         if whitelist_type in whitelists.TYPES.keys():
-            logger.debug("{}: initializing whitelist type '{}'".format(quarantine_name, whitelist_type.upper()))
-            whitelist = whitelists.TYPES[whitelist_type](global_config, config, configtest)
+            logger.debug(
+                "{}: initializing whitelist type '{}'".format(
+                    quarantine_name,
+                    whitelist_type.upper()))
+            whitelist = whitelists.TYPES[whitelist_type](
+                global_config, config, configtest)
         elif whitelist_type == "none":
             logger.debug("{}: whitelist is NONE".format(quarantine_name))
             whitelist = None
         else:
-            raise RuntimeError("{}: unknown whitelist type '{}'".format(quarantine_name, whitelist_type))
+            raise RuntimeError(
+                "{}: unknown whitelist type '{}'".format(
+                    quarantine_name, whitelist_type))
 
         config["whitelist_obj"] = whitelist
 
         # create notification instance
         notification_type = config["notification_type"].lower()
         if notification_type in notifications.TYPES.keys():
-            logger.debug("{}: initializing notification type '{}'".format(quarantine_name, notification_type.upper()))
-            notification = notifications.TYPES[notification_type](global_config, config, configtest)
+            logger.debug(
+                "{}: initializing notification type '{}'".format(
+                    quarantine_name,
+                    notification_type.upper()))
+            notification = notifications.TYPES[notification_type](
+                global_config, config, configtest)
         elif notification_type == "none":
             logger.debug("{}: notification is NONE".format(quarantine_name))
             notification = None
         else:
-            raise RuntimeError("{}: unknown notification type '{}'".format(quarantine_name, notification_type))
+            raise RuntimeError(
+                "{}: unknown notification type '{}'".format(
+                    quarantine_name, notification_type))
 
         config["notification_obj"] = notification
 
@@ -405,11 +530,14 @@ def generate_milter_config(configtest=False, config_files=[]):
             logger.debug("{}: action is {}".format(quarantine_name, action))
             config["milter_action"] = QuarantineMilter.get_actions()[action]
         else:
-            raise RuntimeError("{}: unknown action '{}'".format(quarantine_name, action))
+            raise RuntimeError(
+                "{}: unknown action '{}'".format(
+                    quarantine_name, action))
 
         # create host/network whitelist
         config["ignore_hosts_list"] = []
-        ignored = set([ p.strip() for p in config["ignore_hosts"].split(",") if p])
+        ignored = set([p.strip()
+                       for p in config["ignore_hosts"].split(",") if p])
         for ignore in ignored:
             if not ignore:
                 continue
@@ -421,7 +549,10 @@ def generate_milter_config(configtest=False, config_files=[]):
             else:
                 config["ignore_hosts_list"].append(net)
         if config["ignore_hosts_list"]:
-            logger.debug("{}: ignore hosts: {}".format(quarantine_name, ", ".join(ignored)))
+            logger.debug(
+                "{}: ignore hosts: {}".format(
+                    quarantine_name,
+                    ", ".join(ignored)))
 
         milter_config.append(config)
 

pyquarantine/cli.py

@@ -22,10 +22,12 @@ import time
 
 import pyquarantine
 
+from pyquarantine.version import __version__ as version
 
 def _get_quarantine_obj(config, quarantine):
     try:
-        quarantine_obj = next((q["quarantine_obj"] for q in config if q["name"] == quarantine))
+        quarantine_obj = next((q["quarantine_obj"]
+                               for q in config if q["name"] == quarantine))
     except StopIteration:
         raise RuntimeError("invalid quarantine '{}'".format(quarantine))
     return quarantine_obj
@@ -33,7 +35,8 @@ def _get_quarantine_obj(config, quarantine):
 
 def _get_whitelist_obj(config, quarantine):
     try:
-        whitelist_obj = next((q["whitelist_obj"] for q in config if q["name"] == quarantine))
+        whitelist_obj = next((q["whitelist_obj"]
+                              for q in config if q["name"] == quarantine))
     except StopIteration:
         raise RuntimeError("invalid quarantine '{}'".format(quarantine))
     return whitelist_obj
@@ -51,7 +54,8 @@ def print_table(columns, rows):
         # get the length of the header string
         lengths = [len(header)]
         # get the length of the longest value
-        lengths.append(len(str(max(rows, key=lambda x: len(str(x[key])))[key])))
+        lengths.append(
+            len(str(max(rows, key=lambda x: len(str(x[key])))[key])))
         # use the the longer one
         length = max(lengths)
         column_lengths.append(length)
@@ -70,7 +74,7 @@ def print_table(columns, rows):
     print(row_format.format(*[column[0] for column in columns]))
     print(separator)
 
-    keys = [ entry[1] for entry in columns ]
+    keys = [entry[1] for entry in columns]
     # print rows
     for entry in rows:
         row = []
@@ -81,10 +85,11 @@ def print_table(columns, rows):
 
 def list_quarantines(config, args):
     if args.batch:
-        print("\n".join([ quarantine["name"] for quarantine in config ]))
+        print("\n".join([quarantine["name"] for quarantine in config]))
     else:
         print_table(
-            [("Name", "name"), ("Quarantine", "quarantine_type"), ("Notification", "notification_type"), ("Action", "action")],
+            [("Name", "name"), ("Quarantine", "quarantine_type"),
+             ("Notification", "notification_type"), ("Action", "action")],
             config
         )
 
@@ -94,16 +99,23 @@ def list_quarantine_emails(config, args):
 
     # get quarantine object
     quarantine = _get_quarantine_obj(config, args.quarantine)
-    if quarantine == None:
-        raise RuntimeError("quarantine type is set to None, unable to list emails")
+    if quarantine is None:
+        raise RuntimeError(
+            "quarantine type is set to None, unable to list emails")
 
     # find emails and transform some metadata values to strings
     rows = []
-    emails = quarantine.find(mailfrom=args.mailfrom, recipients=args.recipients, older_than=args.older_than)
+    emails = quarantine.find(
+        mailfrom=args.mailfrom,
+        recipients=args.recipients,
+        older_than=args.older_than)
     for quarantine_id, metadata in emails.items():
         row = emails[quarantine_id]
         row["quarantine_id"] = quarantine_id
-        row["date"] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(metadata["date"]))
+        row["date"] = time.strftime(
+            '%Y-%m-%d %H:%M:%S',
+            time.localtime(
+                metadata["date"]))
         row["mailfrom"] = metadata["mailfrom"]
         row["recipient"] = metadata["recipients"].pop(0)
         row["subject"] = emails[quarantine_id]["headers"]["subject"][:60]
@@ -124,9 +136,12 @@ def list_quarantine_emails(config, args):
         print("\n".join(emails.keys()))
         return
 
-    if not emails: logger.info("quarantine '{}' is empty".format(args.quarantine))
+    if not emails:
+        logger.info("quarantine '{}' is empty".format(args.quarantine))
     print_table(
-        [("Quarantine-ID", "quarantine_id"), ("Date", "date"), ("From", "mailfrom"), ("Recipient(s)", "recipient"), ("Subject", "subject")],
+        [("Quarantine-ID", "quarantine_id"), ("Date", "date"),
+         ("From", "mailfrom"), ("Recipient(s)", "recipient"),
+         ("Subject", "subject")],
         rows
     )
 
@@ -136,25 +151,34 @@ def list_whitelist(config, args):
 
     # get whitelist object
     whitelist = _get_whitelist_obj(config, args.quarantine)
-    if whitelist == None:
-        raise RuntimeError("whitelist type is set to None, unable to list entries")
+    if whitelist is None:
+        raise RuntimeError(
+            "whitelist type is set to None, unable to list entries")
 
     # find whitelist entries
-    entries = whitelist.find(mailfrom=args.mailfrom, recipients=args.recipients, older_than=args.older_than)
+    entries = whitelist.find(
+        mailfrom=args.mailfrom,
+        recipients=args.recipients,
+        older_than=args.older_than)
     if not entries:
-        logger.info("whitelist of quarantine '{}' is empty".format(args.quarantine))
+        logger.info(
+            "whitelist of quarantine '{}' is empty".format(
+                args.quarantine))
         return
 
     # transform some values to strings
     for entry_id, entry in entries.items():
         entries[entry_id]["permanent_str"] = str(entry["permanent"])
-        entries[entry_id]["created_str"] = entry["created"].strftime('%Y-%m-%d %H:%M:%S')
-        entries[entry_id]["last_used_str"] = entry["last_used"].strftime('%Y-%m-%d %H:%M:%S')
+        entries[entry_id]["created_str"] = entry["created"].strftime(
+            '%Y-%m-%d %H:%M:%S')
+        entries[entry_id]["last_used_str"] = entry["last_used"].strftime(
+            '%Y-%m-%d %H:%M:%S')
 
     print_table(
         [
-            ("ID", "id"), ("From", "mailfrom"), ("To", "recipient"), ("Created", "created_str"),
-            ("Last used", "last_used_str"), ("Comment", "comment"), ("Permanent", "permanent_str")
+            ("ID", "id"), ("From", "mailfrom"), ("To", "recipient"),
+            ("Created", "created_str"), ("Last used", "last_used_str"),
+            ("Comment", "comment"), ("Permanent", "permanent_str")
         ],
         entries.values()
     )
@@ -165,32 +189,40 @@ def add_whitelist_entry(config, args):
 
     # get whitelist object
     whitelist = _get_whitelist_obj(config, args.quarantine)
-    if whitelist == None:
-        raise RuntimeError("whitelist type is set to None, unable to add entries")
+    if whitelist is None:
+        raise RuntimeError(
+            "whitelist type is set to None, unable to add entries")
 
     # check existing entries
     entries = whitelist.check(args.mailfrom, args.recipient)
     if entries:
         # check if the exact entry exists already
         for entry in entries.values():
-            if entry["mailfrom"] == args.mailfrom and entry["recipient"] == args.recipient:
-                raise RuntimeError("an entry with this from/to combination already exists")
+            if entry["mailfrom"] == args.mailfrom and \
+                    entry["recipient"] == args.recipient:
+                raise RuntimeError(
+                    "an entry with this from/to combination already exists")
 
         if not args.force:
             # the entry is already covered by others
             for entry_id, entry in entries.items():
                 entries[entry_id]["permanent_str"] = str(entry["permanent"])
-                entries[entry_id]["created_str"] = entry["created"].strftime('%Y-%m-%d %H:%M:%S')
-                entries[entry_id]["last_used_str"] = entry["last_used"].strftime('%Y-%m-%d %H:%M:%S')
+                entries[entry_id]["created_str"] = entry["created"].strftime(
+                    '%Y-%m-%d %H:%M:%S')
+                entries[entry_id]["last_used_str"] = entry["last_used"].strftime(
+                    '%Y-%m-%d %H:%M:%S')
             print_table(
                 [
-                    ("ID", "id"), ("From", "mailfrom"), ("To", "recipient"), ("Created", "created_str"),
-                    ("Last used", "last_used_str"), ("Comment", "comment"), ("Permanent", "permanent_str")
+                    ("ID", "id"), ("From", "mailfrom"), ("To", "recipient"),
+                    ("Created", "created_str"), ("Last used", "last_used_str"),
+                    ("Comment", "comment"), ("Permanent", "permanent_str")
                 ],
                 entries.values()
             )
             print("")
-            raise RuntimeError("from/to combination is already covered by the entries above, use --force to override.")
+            raise RuntimeError(
+                "from/to combination is already covered by the entries above, "
+                "use --force to override.")
 
     # add entry to whitelist
     whitelist.add(args.mailfrom, args.recipient, args.comment, args.permanent)
@@ -201,8 +233,9 @@ def delete_whitelist_entry(config, args):
     logger = logging.getLogger(__name__)
 
     whitelist = _get_whitelist_obj(config, args.quarantine)
-    if whitelist == None:
-        raise RuntimeError("whitelist type is set to None, unable to delete entries")
+    if whitelist is None:
+        raise RuntimeError(
+            "whitelist type is set to None, unable to delete entries")
 
     whitelist.delete(args.whitelist_id)
     logger.info("whitelist entry deleted successfully")
@@ -212,8 +245,9 @@ def notify_email(config, args):
     logger = logging.getLogger(__name__)
 
     quarantine = _get_quarantine_obj(config, args.quarantine)
-    if quarantine == None:
-        raise RuntimeError("quarantine type is set to None, unable to send notification")
+    if quarantine is None:
+        raise RuntimeError(
+            "quarantine type is set to None, unable to send notification")
     quarantine.notify(args.quarantine_id, args.recipient)
     logger.info("sent notification successfully")
 
@@ -222,8 +256,9 @@ def release_email(config, args):
     logger = logging.getLogger(__name__)
 
     quarantine = _get_quarantine_obj(config, args.quarantine)
-    if quarantine == None:
-        raise RuntimeError("quarantine type is set to None, unable to release email")
+    if quarantine is None:
+        raise RuntimeError(
+            "quarantine type is set to None, unable to release email")
 
     quarantine.release(args.quarantine_id, args.recipient)
     logger.info("quarantined email released successfully")
@@ -233,8 +268,9 @@ def delete_email(config, args):
     logger = logging.getLogger(__name__)
 
     quarantine = _get_quarantine_obj(config, args.quarantine)
-    if quarantine == None:
-        raise RuntimeError("quarantine type is set to None, unable to delete email")
+    if quarantine is None:
+        raise RuntimeError(
+            "quarantine type is set to None, unable to delete email")
 
     quarantine.delete(args.quarantine_id, args.recipient)
     logger.info("quarantined email deleted successfully")
@@ -253,78 +289,238 @@ class StdOutFilter(logging.Filter):
 def main():
     "PyQuarantine command-line interface."
     # parse command line
-    formatter_class = lambda prog: argparse.HelpFormatter(prog, max_help_position=50, width=140)
-    parser = argparse.ArgumentParser(description="PyQuarantine CLI", formatter_class=formatter_class)
-    parser.add_argument("-c", "--config", help="Config files to read.", nargs="+", metavar="CFG",
+    def formatter_class(prog): return argparse.HelpFormatter(
+        prog, max_help_position=50, width=140)
+    parser = argparse.ArgumentParser(
+        description="PyQuarantine CLI",
+        formatter_class=formatter_class)
+    parser.add_argument(
+        "-c", "--config",
+        help="Config files to read.",
+        nargs="+", metavar="CFG",
         default=pyquarantine.QuarantineMilter.get_configfiles())
-    parser.add_argument("-d", "--debug", help="Log debugging messages.", action="store_true")
+    parser.add_argument(
+        "-d", "--debug",
+        help="Log debugging messages.",
+        action="store_true")
+    parser.add_argument(
+        "-v", "--version",
+        help="Print version.",
+        action="version",
+        version="%(prog)s ({})".format(version))
     parser.set_defaults(syslog=False)
-    subparsers = parser.add_subparsers(dest="command", title="Commands")
+    subparsers = parser.add_subparsers(
+        dest="command",
+        title="Commands")
     subparsers.required = True
 
     # list command
-    list_parser = subparsers.add_parser("list", help="List available quarantines.", formatter_class=formatter_class)
-    list_parser.add_argument("-b", "--batch", help="Print results using only quarantine names, each on a new line.", action="store_true")
+    list_parser = subparsers.add_parser(
+        "list",
+        help="List available quarantines.",
+        formatter_class=formatter_class)
+    list_parser.add_argument(
+        "-b", "--batch",
+        help="Print results using only quarantine names, each on a new line.",
+        action="store_true")
     list_parser.set_defaults(func=list_quarantines)
 
     # quarantine command group
-    quarantine_parser = subparsers.add_parser("quarantine", description="Manage quarantines.", help="Manage quarantines.", formatter_class=formatter_class)
-    quarantine_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
-    quarantine_subparsers = quarantine_parser.add_subparsers(dest="command", title="Quarantine commands")
+    quarantine_parser = subparsers.add_parser(
+        "quarantine",
+        description="Manage quarantines.",
+        help="Manage quarantines.",
+        formatter_class=formatter_class)
+    quarantine_parser.add_argument(
+        "quarantine",
+        metavar="QUARANTINE",
+        help="Quarantine name.")
+    quarantine_subparsers = quarantine_parser.add_subparsers(
+        dest="command",
+        title="Quarantine commands")
     quarantine_subparsers.required = True
     # quarantine list command
-    quarantine_list_parser = quarantine_subparsers.add_parser("list", description="List emails in quarantines.", help="List emails in quarantine.", formatter_class=formatter_class)
-    quarantine_list_parser.add_argument("-f", "--from", dest="mailfrom", help="Filter emails by from address.", default=None, nargs="+")
-    quarantine_list_parser.add_argument("-t", "--to", dest="recipients", help="Filter emails by recipient address.", default=None, nargs="+")
-    quarantine_list_parser.add_argument("-o", "--older-than", dest="older_than", help="Filter emails by age (days).", default=None, type=float)
-    quarantine_list_parser.add_argument("-b", "--batch", help="Print results using only email quarantine IDs, each on a new line.", action="store_true")
+    quarantine_list_parser = quarantine_subparsers.add_parser(
+        "list",
+        description="List emails in quarantines.",
+        help="List emails in quarantine.",
+        formatter_class=formatter_class)
+    quarantine_list_parser.add_argument(
+        "-f", "--from",
+        dest="mailfrom",
+        help="Filter emails by from address.",
+        default=None,
+        nargs="+")
+    quarantine_list_parser.add_argument(
+        "-t", "--to",
+        dest="recipients",
+        help="Filter emails by recipient address.",
+        default=None,
+        nargs="+")
+    quarantine_list_parser.add_argument(
+        "-o", "--older-than",
+        dest="older_than",
+        help="Filter emails by age (days).",
+        default=None,
+        type=float)
+    quarantine_list_parser.add_argument(
+        "-b", "--batch",
+        help="Print results using only email quarantine IDs, each on a new line.",
+        action="store_true")
     quarantine_list_parser.set_defaults(func=list_quarantine_emails)
     # quarantine notify command
-    quarantine_notify_parser = quarantine_subparsers.add_parser("notify", description="Notify recipient about email in quarantine.", help="Notify recipient about email in quarantine.", formatter_class=formatter_class)
-    quarantine_notify_parser.add_argument("quarantine_id", metavar="ID", help="Quarantine ID.")
-    quarantine_notify_parser_group = quarantine_notify_parser.add_mutually_exclusive_group(required=True)
-    quarantine_notify_parser_group.add_argument("-t", "--to", dest="recipient", help="Release email for one recipient address.")
-    quarantine_notify_parser_group.add_argument("-a", "--all", help="Release email for all recipients.", action="store_true")
+    quarantine_notify_parser = quarantine_subparsers.add_parser(
+        "notify",
+        description="Notify recipient about email in quarantine.",
+        help="Notify recipient about email in quarantine.",
+        formatter_class=formatter_class)
+    quarantine_notify_parser.add_argument(
+        "quarantine_id",
+        metavar="ID",
+        help="Quarantine ID.")
+    quarantine_notify_parser_group = quarantine_notify_parser.add_mutually_exclusive_group(
+        required=True)
+    quarantine_notify_parser_group.add_argument(
+        "-t", "--to",
+        dest="recipient",
+        help="Release email for one recipient address.")
+    quarantine_notify_parser_group.add_argument(
+        "-a", "--all",
+        help="Release email for all recipients.",
+        action="store_true")
     quarantine_notify_parser.set_defaults(func=notify_email)
     # quarantine release command
-    quarantine_release_parser = quarantine_subparsers.add_parser("release", description="Release email from quarantine.", help="Release email from quarantine.", formatter_class=formatter_class)
-    quarantine_release_parser.add_argument("quarantine_id", metavar="ID", help="Quarantine ID.")
-    quarantine_release_parser.add_argument("-n", "--disable-syslog", dest="syslog", help="Disable syslog messages.", action="store_false")
-    quarantine_release_parser_group = quarantine_release_parser.add_mutually_exclusive_group(required=True)
-    quarantine_release_parser_group.add_argument("-t", "--to", dest="recipient", help="Release email for one recipient address.")
-    quarantine_release_parser_group.add_argument("-a", "--all", help="Release email for all recipients.", action="store_true")
+    quarantine_release_parser = quarantine_subparsers.add_parser(
+        "release",
+        description="Release email from quarantine.",
+        help="Release email from quarantine.",
+        formatter_class=formatter_class)
+    quarantine_release_parser.add_argument(
+        "quarantine_id",
+        metavar="ID",
+        help="Quarantine ID.")
+    quarantine_release_parser.add_argument(
+        "-n",
+        "--disable-syslog",
+        dest="syslog",
+        help="Disable syslog messages.",
+        action="store_false")
+    quarantine_release_parser_group = quarantine_release_parser.add_mutually_exclusive_group(
+        required=True)
+    quarantine_release_parser_group.add_argument(
+        "-t", "--to",
+        dest="recipient",
+        help="Release email for one recipient address.")
+    quarantine_release_parser_group.add_argument(
+        "-a", "--all",
+        help="Release email for all recipients.",
+        action="store_true")
     quarantine_release_parser.set_defaults(func=release_email)
     # quarantine delete command
-    quarantine_delete_parser = quarantine_subparsers.add_parser("delete", description="Delete email from quarantine.", help="Delete email from quarantine.", formatter_class=formatter_class)
-    quarantine_delete_parser.add_argument("quarantine_id", metavar="ID", help="Quarantine ID.")
-    quarantine_delete_parser.add_argument("-n", "--disable-syslog", dest="syslog", help="Disable syslog messages.", action="store_false")
-    quarantine_delete_parser_group = quarantine_delete_parser.add_mutually_exclusive_group(required=True)
-    quarantine_delete_parser_group.add_argument("-t", "--to", dest="recipient", help="Delete email for one recipient address.")
-    quarantine_delete_parser_group.add_argument("-a", "--all", help="Delete email for all recipients.", action="store_true")
+    quarantine_delete_parser = quarantine_subparsers.add_parser(
+        "delete",
+        description="Delete email from quarantine.",
+        help="Delete email from quarantine.",
+        formatter_class=formatter_class)
+    quarantine_delete_parser.add_argument(
+        "quarantine_id",
+        metavar="ID",
+        help="Quarantine ID.")
+    quarantine_delete_parser.add_argument(
+        "-n", "--disable-syslog",
+        dest="syslog",
+        help="Disable syslog messages.",
+        action="store_false")
+    quarantine_delete_parser_group = quarantine_delete_parser.add_mutually_exclusive_group(
+        required=True)
+    quarantine_delete_parser_group.add_argument(
+        "-t", "--to",
+        dest="recipient",
+        help="Delete email for one recipient address.")
+    quarantine_delete_parser_group.add_argument(
+        "-a", "--all",
+        help="Delete email for all recipients.",
+        action="store_true")
     quarantine_delete_parser.set_defaults(func=delete_email)
 
     # whitelist command group
-    whitelist_parser = subparsers.add_parser("whitelist", description="Manage whitelists.", help="Manage whitelists.", formatter_class=formatter_class)
-    whitelist_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
-    whitelist_subparsers = whitelist_parser.add_subparsers(dest="command", title="Whitelist commands")
+    whitelist_parser = subparsers.add_parser(
+        "whitelist",
+        description="Manage whitelists.",
+        help="Manage whitelists.",
+        formatter_class=formatter_class)
+    whitelist_parser.add_argument(
+        "quarantine",
+        metavar="QUARANTINE",
+        help="Quarantine name.")
+    whitelist_subparsers = whitelist_parser.add_subparsers(
+        dest="command",
+        title="Whitelist commands")
     whitelist_subparsers.required = True
     # whitelist list command
-    whitelist_list_parser = whitelist_subparsers.add_parser("list", description="List whitelist entries.", help="List whitelist entries.", formatter_class=formatter_class)
-    whitelist_list_parser.add_argument("-f", "--from", dest="mailfrom", help="Filter entries by from address.", default=None, nargs="+")
-    whitelist_list_parser.add_argument("-t", "--to", dest="recipients", help="Filter entries by recipient address.", default=None, nargs="+")
-    whitelist_list_parser.add_argument("-o", "--older-than", dest="older_than", help="Filter emails by last used date (days).", default=None, type=float)
+    whitelist_list_parser = whitelist_subparsers.add_parser(
+        "list",
+        description="List whitelist entries.",
+        help="List whitelist entries.",
+        formatter_class=formatter_class)
+    whitelist_list_parser.add_argument(
+        "-f", "--from",
+        dest="mailfrom",
+        help="Filter entries by from address.",
+        default=None,
+        nargs="+")
+    whitelist_list_parser.add_argument(
+        "-t", "--to",
+        dest="recipients",
+        help="Filter entries by recipient address.",
+        default=None,
+        nargs="+")
+    whitelist_list_parser.add_argument(
+        "-o", "--older-than",
+        dest="older_than",
+        help="Filter emails by last used date (days).",
+        default=None,
+        type=float)
     whitelist_list_parser.set_defaults(func=list_whitelist)
     # whitelist add command
-    whitelist_add_parser = whitelist_subparsers.add_parser("add", description="Add whitelist entry.", help="Add whitelist entry.", formatter_class=formatter_class)
-    whitelist_add_parser.add_argument("-f", "--from", dest="mailfrom", help="From address.", required=True)
-    whitelist_add_parser.add_argument("-t", "--to", dest="recipient", help="Recipient address.", required=True)
-    whitelist_add_parser.add_argument("-c", "--comment", help="Comment.", default="added by CLI")
-    whitelist_add_parser.add_argument("-p", "--permanent", help="Add a permanent entry.", action="store_true")
-    whitelist_add_parser.add_argument("--force", help="Force adding an entry, even if already covered by another entry.", action="store_true")
+    whitelist_add_parser = whitelist_subparsers.add_parser(
+        "add",
+        description="Add whitelist entry.",
+        help="Add whitelist entry.",
+        formatter_class=formatter_class)
+    whitelist_add_parser.add_argument(
+        "-f", "--from",
+        dest="mailfrom",
+        help="From address.",
+        required=True)
+    whitelist_add_parser.add_argument(
+        "-t", "--to",
+        dest="recipient",
+        help="Recipient address.",
+        required=True)
+    whitelist_add_parser.add_argument(
+        "-c", "--comment",
+        help="Comment.",
+        default="added by CLI")
+    whitelist_add_parser.add_argument(
+        "-p", "--permanent",
+        help="Add a permanent entry.",
+        action="store_true")
+    whitelist_add_parser.add_argument(
+        "--force",
+        help="Force adding an entry, even if already covered by another entry.",
+        action="store_true")
     whitelist_add_parser.set_defaults(func=add_whitelist_entry)
     # whitelist delete command
-    whitelist_delete_parser = whitelist_subparsers.add_parser("delete", description="Delete whitelist entry.", help="Delete whitelist entry.", formatter_class=formatter_class)
-    whitelist_delete_parser.add_argument("whitelist_id", metavar="ID", help="Whitelist ID.")
+    whitelist_delete_parser = whitelist_subparsers.add_parser(
+        "delete",
+        description="Delete whitelist entry.",
+        help="Delete whitelist entry.",
+        formatter_class=formatter_class)
+    whitelist_delete_parser.add_argument(
+        "whitelist_id",
+        metavar="ID",
+        help="Whitelist ID.")
     whitelist_delete_parser.set_defaults(func=delete_whitelist_entry)
 
     args = parser.parse_args()
@@ -336,7 +532,8 @@ def main():
 
     # setup console log
     if args.debug:
-        formatter = logging.Formatter("%(levelname)s: [%(name)s] - %(message)s")
+        formatter = logging.Formatter(
+            "%(levelname)s: [%(name)s] - %(message)s")
     else:
         formatter = logging.Formatter("%(levelname)s: %(message)s")
     # stdout
@@ -355,17 +552,21 @@ def main():
 
     # try to generate milter configs
     try:
-        global_config, config = pyquarantine.generate_milter_config(config_files=args.config, configtest=True)
+        global_config, config = pyquarantine.generate_milter_config(
+            config_files=args.config, configtest=True)
     except RuntimeError as e:
         logger.error(e)
         sys.exit(255)
 
     if args.syslog:
         # setup syslog
-        sysloghandler = logging.handlers.SysLogHandler(address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL)
+        sysloghandler = logging.handlers.SysLogHandler(
+            address="/dev/log",
+            facility=logging.handlers.SysLogHandler.LOG_MAIL)
         sysloghandler.setLevel(loglevel)
         if args.debug:
-            formatter = logging.Formatter("pyquarantine: [%(name)s] [%(levelname)s] %(message)s")
+            formatter = logging.Formatter(
+                "pyquarantine: [%(name)s] [%(levelname)s] %(message)s")
         else:
             formatter = logging.Formatter("pyquarantine: %(message)s")
         sysloghandler.setFormatter(formatter)

pyquarantine/mailer.py

@@ -26,6 +26,10 @@ process = None
 
 def smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail):
     s = smtplib.SMTP(host=smtp_host, port=smtp_port)
+    s.ehlo()
+    if s.has_extn("STARTTLS"):
+        s.starttls()
+        s.ehlo()
     s.sendmail(mailfrom, [recipient], mail)
     s.quit()
 
@@ -38,31 +42,37 @@ def mailprocess():
     try:
         while True:
             m = queue.get()
-            if not m: break
+            if not m:
+                break
 
             smtp_host, smtp_port, queueid, mailfrom, recipient, mail, emailtype = m
             try:
                 smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail)
             except Exception as e:
-                logger.error("{}: error while sending {} to '{}': {}".format(queueid, emailtype, recipient, e))
+                logger.error(
+                    "{}: error while sending {} to '{}': {}".format(
+                        queueid, emailtype, recipient, e))
             else:
-                logger.info("{}: successfully sent {} to: {}".format(queueid, emailtype, recipient))
+                logger.info(
+                    "{}: successfully sent {} to: {}".format(
+                        queueid, emailtype, recipient))
     except KeyboardInterrupt:
         pass
     logger.debug("mailer process terminated")
 
 
-def sendmail(smtp_host, smtp_port, queueid, mailfrom, recipients, mail, emailtype="email"):
+def sendmail(smtp_host, smtp_port, queueid, mailfrom, recipients, mail,
+             emailtype="email"):
     "Send an email."
     global logger
     global process
     global queue
 
-    if type(recipients) == str:
+    if isinstance(recipients, str):
         recipients = [recipients]
 
     # start mailprocess if it is not started yet
-    if process == None:
+    if process is None:
         process = Process(target=mailprocess)
         process.daemon = True
         logger.debug("starting mailer process")
@@ -70,6 +80,9 @@ def sendmail(smtp_host, smtp_port, queueid, mailfrom, recipients, mail, emailtyp
 
     for recipient in recipients:
         try:
-            queue.put((smtp_host, smtp_port, queueid, mailfrom, recipient, mail, emailtype), timeout=30)
+            queue.put(
+                (smtp_host, smtp_port, queueid, mailfrom, recipient, mail,
+                 emailtype),
+                timeout=30)
         except Queue.Full as e:
             raise RuntimeError("email queue is full")

pyquarantine/notifications.py

@@ -26,15 +26,18 @@ from os.path import basename
 
 from pyquarantine import mailer
 
+
 class BaseNotification(object):
     "Notification base class"
+
     def __init__(self, global_config, config, configtest=False):
         self.quarantine_name = config["name"]
         self.global_config = global_config
         self.config = config
         self.logger = logging.getLogger(__name__)
 
-    def notify(self, queueid, quarantine_id, mailfrom, recipients, headers, fp, subgroups=None, named_subgroups=None, synchronous=False):
+    def notify(self, queueid, quarantine_id, mailfrom, recipients, headers,
+               fp, subgroups=None, named_subgroups=None, synchronous=False):
         fp.seek(0)
         pass
 
@@ -68,6 +71,8 @@ class EMailNotification(BaseNotification):
         "i",
         "img",
         "li",
+        "p",
+        "pre",
         "span",
         "table",
         "td",
@@ -77,13 +82,14 @@ class EMailNotification(BaseNotification):
         "u",
         "ul"
     ]
-    good_attributes = [
+    _good_attributes = [
         "align",
         "alt",
         "bgcolor",
         "border",
         "cellpadding",
         "cellspacing",
+        "class",
         "color",
         "colspan",
         "dir",
@@ -104,17 +110,40 @@ class EMailNotification(BaseNotification):
     ]
 
     def __init__(self, global_config, config, configtest=False):
-        super(EMailNotification, self).__init__(global_config, config, configtest)
+        super(EMailNotification, self).__init__(
+            global_config, config, configtest)
 
         # check if mandatory options are present in config
-        for option in ["smtp_host", "smtp_port", "notification_email_envelope_from", "notification_email_from", "notification_email_subject", "notification_email_template", "notification_email_replacement_img", "notification_email_embedded_imgs"]:
+        for option in [
+            "notification_email_smtp_host",
+            "notification_email_smtp_port",
+            "notification_email_envelope_from",
+            "notification_email_from",
+            "notification_email_subject",
+            "notification_email_template",
+                "notification_email_embedded_imgs"]:
             if option not in self.config.keys() and option in self.global_config.keys():
                 self.config[option] = self.global_config[option]
             if option not in self.config.keys():
-                raise RuntimeError("mandatory option '{}' not present in config section '{}' or 'global'".format(option, self.quarantine_name))
+                raise RuntimeError(
+                    "mandatory option '{}' not present in config section '{}' or 'global'".format(
+                        option, self.quarantine_name))
 
-        self.smtp_host = self.config["smtp_host"]
-        self.smtp_port = self.config["smtp_port"]
+        # check if optional config options are present in config
+        defaults = {
+            "notification_email_replacement_img": "",
+            "notification_email_strip_images": "false",
+            "notification_email_parser_lib": "lxml"
+        }
+        for option in defaults.keys():
+            if option not in config.keys() and \
+                    option in global_config.keys():
+                config[option] = global_config[option]
+            if option not in config.keys():
+                config[option] = defaults[option]
+
+        self.smtp_host = self.config["notification_email_smtp_host"]
+        self.smtp_port = self.config["notification_email_smtp_port"]
         self.mailfrom = self.config["notification_email_envelope_from"]
         self.from_header = self.config["notification_email_from"]
         self.subject = self.config["notification_email_subject"]
@@ -125,37 +154,56 @@ class EMailNotification(BaseNotification):
         try:
             self.from_header.format_map(testvars)
         except ValueError as e:
-            raise RuntimeError("error parsing notification_email_from: {}".format(e))
+            raise RuntimeError(
+                "error parsing notification_email_from: {}".format(e))
 
         # test-parse subject
         try:
             self.subject.format_map(testvars)
         except ValueError as e:
-            raise RuntimeError("error parsing notification_email_subject: {}".format(e))
+            raise RuntimeError(
+                "error parsing notification_email_subject: {}".format(e))
 
         # read and parse email notification template
         try:
-            self.template = open(self.config["notification_email_template"], "r").read()
+            self.template = open(
+                self.config["notification_email_template"], "r").read()
             self.template.format_map(testvars)
         except IOError as e:
             raise RuntimeError("error reading template: {}".format(e))
         except ValueError as e:
             raise RuntimeError("error parsing template: {}".format(e))
 
-        # read email replacement image if specified
-        replacement_img_path = self.config["notification_email_replacement_img"].strip()
-        if replacement_img_path:
-            try:
-                self.replacement_img = MIMEImage(open(replacement_img_path, "rb").read())
-            except IOError as e:
-                raise RuntimeError("error reading replacement image: {}".format(e))
+        strip_images = self.config["notification_email_strip_images"].strip().upper()
+        if strip_images in ["TRUE", "ON", "YES"]:
+            self.strip_images = True
+        elif strip_images in ["FALSE", "OFF", "NO"]:
+            self.strip_images = False
         else:
-                self.replacement_img.add_header("Content-ID", "<removed_for_security_reasons>")
+            raise RuntimeError("error parsing notification_email_strip_images: unknown value")
+
+        self.parser_lib = self.config["notification_email_parser_lib"].strip()
+        if self.parser_lib not in ["lxml", "html.parser"]:
+            raise RuntimeError("error parsing notification_email_parser_lib: unknown value")
+
+        # read email replacement image if specified
+        replacement_img = self.config["notification_email_replacement_img"].strip()
+        if not self.strip_images and replacement_img:
+            try:
+                self.replacement_img = MIMEImage(
+                    open(replacement_img, "rb").read())
+            except IOError as e:
+                raise RuntimeError(
+                    "error reading replacement image: {}".format(e))
+            else:
+                self.replacement_img.add_header(
+                    "Content-ID", "<removed_for_security_reasons>")
         else:
             self.replacement_img = None
 
         # read images to embed if specified
-        embedded_img_paths = [ p.strip() for p in self.config["notification_email_embedded_imgs"].split(",") if p]
+        embedded_img_paths = [
+            p.strip() for p in self.config["notification_email_embedded_imgs"].split(",") if p]
         self.embedded_imgs = []
         for img_path in embedded_img_paths:
             # read image
@@ -167,36 +215,47 @@ class EMailNotification(BaseNotification):
                 img.add_header("Content-ID", "<{}>".format(basename(img_path)))
                 self.embedded_imgs.append(img)
 
-
-    def get_text(self, queueid, part):
-        "Get the mail text in html form from email part."
-        mimetype = part.get_content_type()
-
-        self.logger.debug("{}: extracting content of email text part".format(queueid))
-        text = part.get_payload(decode=True)
-
-        if mimetype == EMailNotification._plain_text:
-            self.logger.debug("{}: content mimetype is {}, converting to {}".format(queueid, mimetype, self._html_text))
-            text = re.sub(r"^(.*)$", r"\1<br/>\n", text.decode(), flags=re.MULTILINE)
-        else:
-            self.logger.debug("{}: content mimetype is {}".format(queueid, mimetype))
-
-        return BeautifulSoup(text, "lxml")
-
-    def get_text_multipart(self, queueid, msg, preferred=_html_text):
-        "Get the mail text of a multipart email in html form."
-        soup = None
-
-        for part in msg.get_payload():
-            mimetype = part.get_content_type()
-            if mimetype in [EMailNotification._plain_text, EMailNotification._html_text]:
-                soup = self.get_text(queueid, part)
-            elif mimetype.startswith("multipart"):
-                soup = self.get_text_multipart(queueid, part, preferred)
-
-            if soup != None and mimetype == preferred:
+    def get_decoded_email_body(self, queueid, msg, preferred=_html_text):
+        "Find and decode email body."
+        # try to find the body part
+        self.logger.debug("{}: trying to find email body".format(queueid))
+        body = None
+        for part in msg.walk():
+            content_type = part.get_content_type()
+            if content_type in [EMailNotification._plain_text,
+                                EMailNotification._html_text]:
+                body = part
+                if content_type == preferred:
                     break
-        return soup
+
+        if body is not None:
+            # get the character set, fallback to utf-8 if not defined in header
+            charset = body.get_content_charset()
+            if charset is None:
+                charset = "utf-8"
+
+            # decode content
+            content = body.get_payload(decode=True).decode(
+                encoding=charset, errors="replace")
+
+            content_type = body.get_content_type()
+            if content_type == EMailNotification._plain_text:
+                # convert text/plain to text/html
+                self.logger.debug(
+                    "{}: content type is {}, converting to {}".format(
+                        queueid, content_type, EMailNotification._html_text))
+                content = re.sub(r"^(.*)$", r"\1<br/>",
+                                 escape(content), flags=re.MULTILINE)
+            else:
+                self.logger.debug(
+                    "{}: content type is {}".format(
+                        queueid, content_type))
+        else:
+            self.logger.error(
+                "{}: unable to find email body".format(queueid))
+            content = "ERROR: unable to find email body"
+
+        return content
 
     def sanitize(self, queueid, soup):
         "Sanitize mail html text."
@@ -204,59 +263,83 @@ class EMailNotification(BaseNotification):
 
         # completly remove bad elements
         for element in soup(EMailNotification._bad_tags):
-            self.logger.debug("{}: removing dangerous tag '{}' and its content".format(queueid, element.name))
+            self.logger.debug(
+                "{}: removing dangerous tag '{}' and its content".format(
+                    queueid, element.name))
             element.extract()
 
         # remove not whitelisted elements, but keep their content
         for element in soup.find_all(True):
             if element.name not in EMailNotification._good_tags:
-                self.logger.debug("{}: removing tag '{}', keep its content".format(queueid, element.name))
+                self.logger.debug(
+                    "{}: removing tag '{}', keep its content".format(
+                        queueid, element.name))
                 element.replaceWithChildren()
 
         # remove not whitelisted attributes
         for element in soup.find_all(True):
-            for attribute in element.attrs.keys():
-                if attribute not in EMailNotification.good_attributes:
+            for attribute in list(element.attrs.keys()):
+                if attribute not in EMailNotification._good_attributes:
                     if element.name == "a" and attribute == "href":
-                        self.logger.debug("{}: setting attribute href to '#' on tag '{}'".format(queueid, element.name))
+                        self.logger.debug(
+                            "{}: setting attribute href to '#' on tag '{}'".format(
+                                queueid, element.name))
                         element["href"] = "#"
                     else:
-                        self.logger.debug("{}: removing attribute '{}' from tag '{}'".format(queueid, attribute, element.name))
+                        self.logger.debug(
+                            "{}: removing attribute '{}' from tag '{}'".format(
+                                queueid, attribute, element.name))
                         del(element.attrs[attribute])
         return soup
 
-    def get_html_text_part(self, queueid, msg):
-        "Get the mail text of an email in html form."
-        soup = None
-        mimetype = msg.get_content_type()
-
-        self.logger.debug("{}: trying to find text part of email".format(queueid))
-        if mimetype in [EMailNotification._plain_text, EMailNotification._html_text]:
-            soup = self.get_text(queueid, msg)
-        elif mimetype.startswith("multipart"):
-            soup = self.get_text_multipart(queueid, msg)
-
-        if soup == None:
-            self.logger.error("{}: unable to extract text part of email".format(queueid))
-            text = "ERROR: unable to extract text from email body"
-            soup = BeautifulSoup(text, "lxml", "UTF-8")
-
-        return soup
-
-    def notify(self, queueid, quarantine_id, mailfrom, recipients, headers, fp, subgroups=None, named_subgroups=None, synchronous=False):
+    def notify(self, queueid, quarantine_id, mailfrom, recipients, headers, fp,
+               subgroups=None, named_subgroups=None, synchronous=False):
         "Notify recipients via email."
-        super(EMailNotification, self).notify(queueid, quarantine_id, mailfrom, recipients, headers, fp, subgroups, named_subgroups, synchronous)
+        super(
+            EMailNotification,
+            self).notify(
+            queueid,
+            quarantine_id,
+            mailfrom,
+            recipients,
+            headers,
+            fp,
+            subgroups,
+            named_subgroups,
+            synchronous)
 
-        # extract html text from email
-        self.logger.debug("{}: extraction email text from original email".format(queueid))
-        soup = self.get_html_text_part(queueid, email.message_from_binary_file(fp))
+        # extract body from email
+        content = self.get_decoded_email_body(
+            queueid, email.message_from_binary_file(fp))
+
+        # create BeautifulSoup object
+        self.logger.debug(
+            "{}: trying to create BeatufilSoup object with parser lib {}, "
+            "text length is {} bytes".format(
+                queueid, self.parser_lib, len(content)))
+        soup = BeautifulSoup(content, self.parser_lib)
+        self.logger.debug(
+            "{}: sucessfully created BeautifulSoup object".format(queueid))
 
         # replace picture sources
         image_replaced = False
-        if self.replacement_img:
+        if self.strip_images:
+            self.logger.debug(
+                "{}: looking for images to strip".format(queueid))
             for element in soup("img"):
                 if "src" in element.attrs.keys():
-                    self.logger.debug("{}: replacing image: {}".format(queueid, element["src"]))
+                    self.logger.debug(
+                        "{}: strip image: {}".format(
+                            queueid, element["src"]))
+                element.extract()
+        elif self.replacement_img:
+            self.logger.debug(
+                "{}: looking for images to replace".format(queueid))
+            for element in soup("img"):
+                if "src" in element.attrs.keys():
+                    self.logger.debug(
+                        "{}: replacing image: {}".format(
+                            queueid, element["src"]))
                 element["src"] = "cid:removed_for_security_reasons"
                 image_replaced = True
 
@@ -266,7 +349,9 @@ class EMailNotification(BaseNotification):
 
         # sending email notifications
         for recipient in recipients:
-            self.logger.debug("{}: generating notification email for '{}'".format(queueid, recipient))
+            self.logger.debug(
+                "{}: generating notification email for '{}'".format(
+                    queueid, recipient))
             self.logger.debug("{}: parsing email template".format(queueid))
 
             # generate dict containing all template variables
@@ -283,35 +368,46 @@ class EMailNotification(BaseNotification):
                 for subgroup in subgroups:
                     variables["SUBGROUP_{}".format(number)] = escape(subgroup)
             if named_subgroups:
-                for key, value in named_subgroups.items(): named_subgroups[key] = escape(value)
+                for key, value in named_subgroups.items():
+                    named_subgroups[key] = escape(value)
                 variables.update(named_subgroups)
 
             # parse template
             htmltext = self.template.format_map(variables)
 
             msg = MIMEMultipart('related')
-            msg["Subject"] = self.subject.format_map(variables)
-            msg["From"] = "<{}>".format(self.from_header.format_map(variables))
-            msg["To"] = "<{}>".format(recipient)
+            msg["From"] = self.from_header.format_map(
+                defaultdict(str, EMAIL_FROM=headers["from"]))
+            msg["To"] = headers["to"]
+            msg["Subject"] = self.subject.format_map(
+                defaultdict(str, EMAIL_SUBJECT=headers["subject"]))
             msg["Date"] = email.utils.formatdate()
             msg.attach(MIMEText(htmltext, "html", 'UTF-8'))
 
             if image_replaced:
-                self.logger.debug("{}: attaching notification_replacement_img".format(queueid))
+                self.logger.debug(
+                    "{}: attaching notification_replacement_img".format(queueid))
                 msg.attach(self.replacement_img)
 
             for img in self.embedded_imgs:
                 self.logger.debug("{}: attaching imgage".format(queueid))
                 msg.attach(img)
 
-            self.logger.debug("{}: sending notification email to: {}".format(queueid, recipient))
+            self.logger.debug(
+                "{}: sending notification email to: {}".format(
+                    queueid, recipient))
             if synchronous:
                 try:
-                    mailer.smtp_send(self.smtp_host, self.smtp_port, self.mailfrom, recipient, msg.as_string())
+                    mailer.smtp_send(self.smtp_host, self.smtp_port,
+                                     self.mailfrom, recipient, msg.as_string())
                 except Exception as e:
-                    raise RuntimeError("error while sending email to '{}': {}".format(recipient, e))
+                    raise RuntimeError(
+                        "error while sending email to '{}': {}".format(
+                            recipient, e))
             else:
-                mailer.sendmail(self.smtp_host, self.smtp_port, queueid, self.mailfrom, recipient, msg.as_string(), "notification email")
+                mailer.sendmail(self.smtp_host, self.smtp_port, queueid,
+                                self.mailfrom, recipient, msg.as_string(),
+                                "notification email")
 
 
 # list of notification types and their related notification classes

pyquarantine/quarantines.py

@@ -27,13 +27,15 @@ from pyquarantine import mailer
 
 class BaseQuarantine(object):
     "Quarantine base class"
+
     def __init__(self, global_config, config, configtest=False):
         self.name = config["name"]
         self.global_config = global_config
         self.config = config
         self.logger = logging.getLogger(__name__)
 
-    def add(self, queueid, mailfrom, recipients, headers, fp, subgroups=None, named_subgroups=None):
+    def add(self, queueid, mailfrom, recipients, headers,
+            fp, subgroups=None, named_subgroups=None):
         "Add email to quarantine."
         fp.seek(0)
         return ""
@@ -53,7 +55,8 @@ class BaseQuarantine(object):
     def notify(self, quarantine_id, recipient=None):
         "Notify recipient about email in quarantine."
         if not self.config["notification_obj"]:
-            raise RuntimeError("notification type is set to None, unable to send notifications")
+            raise RuntimeError(
+                "notification type is set to None, unable to send notifications")
         return
 
     def release(self, quarantine_id, recipient=None):
@@ -63,6 +66,7 @@ class BaseQuarantine(object):
 
 class FileQuarantine(BaseQuarantine):
     "Quarantine class to store mails on filesystem."
+
     def __init__(self, global_config, config, configtest=False):
         super(FileQuarantine, self).__init__(global_config, config, configtest)
 
@@ -71,12 +75,17 @@ class FileQuarantine(BaseQuarantine):
             if option not in self.config.keys() and option in self.global_config.keys():
                 self.config[option] = self.global_config[option]
             if option not in self.config.keys():
-                raise RuntimeError("mandatory option '{}' not present in config section '{}' or 'global'".format(option, self.name))
+                raise RuntimeError(
+                    "mandatory option '{}' not present in config section '{}' or 'global'".format(
+                        option, self.name))
         self.directory = self.config["quarantine_directory"]
 
         # check if quarantine directory exists and is writable
-        if not os.path.isdir(self.directory) or not os.access(self.directory, os.W_OK):
-            raise RuntimeError("file quarantine directory '{}' does not exist or is not writable".format(self.directory))
+        if not os.path.isdir(self.directory) or not os.access(
+                self.directory, os.W_OK):
+            raise RuntimeError(
+                "file quarantine directory '{}' does not exist or is not writable".format(
+                    self.directory))
         self._metadata_suffix = ".metadata"
 
     def _save_datafile(self, quarantine_id, fp):
@@ -88,7 +97,9 @@ class FileQuarantine(BaseQuarantine):
             raise RuntimeError("unable save data file: {}".format(e))
 
     def _save_metafile(self, quarantine_id, metadata):
-        metafile = os.path.join(self.directory, "{}{}".format(quarantine_id, self._metadata_suffix))
+        metafile = os.path.join(
+            self.directory, "{}{}".format(
+                quarantine_id, self._metadata_suffix))
         try:
             with open(metafile, "w") as f:
                 json.dump(metadata, f, indent=2)
@@ -109,10 +120,21 @@ class FileQuarantine(BaseQuarantine):
         except IOError as e:
             raise RuntimeError("unable to remove data file: {}".format(e))
 
-    def add(self, queueid, mailfrom, recipients, headers, fp, subgroups=None, named_subgroups=None):
+    def add(self, queueid, mailfrom, recipients, headers,
+            fp, subgroups=None, named_subgroups=None):
         "Add email to file quarantine and return quarantine-id."
-        super(FileQuarantine, self).add(queueid, mailfrom, recipients, headers, fp, subgroups, named_subgroups)
-        quarantine_id = "{}_{}".format(datetime.now().strftime("%Y%m%d%H%M%S"), queueid)
+        super(
+            FileQuarantine,
+            self).add(
+            queueid,
+            mailfrom,
+            recipients,
+            headers,
+            fp,
+            subgroups,
+            named_subgroups)
+        quarantine_id = "{}_{}".format(
+            datetime.now().strftime("%Y%m%d%H%M%S"), queueid)
 
         # save mail
         self._save_datafile(quarantine_id, fp)
@@ -140,9 +162,12 @@ class FileQuarantine(BaseQuarantine):
         "Return metadata of quarantined email."
         super(FileQuarantine, self).get_metadata(quarantine_id)
 
-        metafile = os.path.join(self.directory, "{}{}".format(quarantine_id, self._metadata_suffix))
+        metafile = os.path.join(
+            self.directory, "{}{}".format(
+                quarantine_id, self._metadata_suffix))
         if not os.path.isfile(metafile):
-            raise RuntimeError("invalid quarantine id '{}'".format(quarantine_id))
+            raise RuntimeError(
+                "invalid quarantine id '{}'".format(quarantine_id))
 
         try:
             with open(metafile, "r") as f:
@@ -150,33 +175,41 @@ class FileQuarantine(BaseQuarantine):
         except IOError as e:
             raise RuntimeError("unable to read metadata file: {}".format(e))
         except json.JSONDecodeError as e:
-            raise RuntimeError("invalid meta file '{}': {}".format(metafile, e))
+            raise RuntimeError(
+                "invalid meta file '{}': {}".format(
+                    metafile, e))
 
         return metadata
 
     def find(self, mailfrom=None, recipients=None, older_than=None):
         "Find emails in quarantine."
         super(FileQuarantine, self).find(mailfrom, recipients, older_than)
-        if type(mailfrom) == str: mailfrom = [mailfrom]
-        if type(recipients) == str: recipients = [recipients]
+        if isinstance(mailfrom, str):
+            mailfrom = [mailfrom]
+        if isinstance(recipients, str):
+            recipients = [recipients]
 
         emails = {}
-        metafiles = glob(os.path.join(self.directory, "*{}".format(self._metadata_suffix)))
+        metafiles = glob(os.path.join(
+            self.directory, "*{}".format(self._metadata_suffix)))
         for metafile in metafiles:
-            if not os.path.isfile(metafile): continue
-
-            quarantine_id = os.path.basename(metafile[:-len(self._metadata_suffix)])
-            metadata = self.get_metadata(quarantine_id)
-            if older_than != None:
-                if timegm(gmtime()) - metadata["date"] < (older_than * 24 * 3600):
+            if not os.path.isfile(metafile):
                 continue
 
-            if mailfrom != None:
+            quarantine_id = os.path.basename(
+                metafile[:-len(self._metadata_suffix)])
+            metadata = self.get_metadata(quarantine_id)
+            if older_than is not None:
+                if timegm(gmtime()) - metadata["date"] < (older_than * 86400):
+                    continue
+
+            if mailfrom is not None:
                 if metadata["mailfrom"] not in mailfrom:
                     continue
 
-            if recipients != None:
-                if len(recipients) == 1 and recipients[0] not in metadata["recipients"]:
+            if recipients is not None:
+                if len(recipients) == 1 and \
+                        recipients[0] not in metadata["recipients"]:
                     continue
                 elif len(set(recipients + metadata["recipients"])) == len(recipients + metadata["recipients"]):
                     continue
@@ -194,7 +227,7 @@ class FileQuarantine(BaseQuarantine):
         except RuntimeError as e:
             raise RuntimeError("unable to delete email: {}".format(e))
 
-        if recipient == None:
+        if recipient is None:
             self._remove(quarantine_id)
         else:
             if recipient not in metadata["recipients"]:
@@ -215,7 +248,7 @@ class FileQuarantine(BaseQuarantine):
         except RuntimeError as e:
             raise RuntimeError("unable to release email: {}".format(e))
 
-        if recipient != None:
+        if recipient is not None:
             if recipient not in metadata["recipients"]:
                 raise RuntimeError("invalid recipient '{}'".format(recipient))
             recipients = [recipient]
@@ -225,11 +258,13 @@ class FileQuarantine(BaseQuarantine):
         datafile = os.path.join(self.directory, quarantine_id)
         try:
             with open(datafile, "rb") as fp:
-                self.config["notification_obj"].notify(metadata["queue_id"], quarantine_id, metadata["mailfrom"], recipients, metadata["headers"], fp,
-                        metadata["subgroups"], metadata["named_subgroups"], synchronous=True)
+                self.config["notification_obj"].notify(
+                    metadata["queue_id"], quarantine_id, metadata["mailfrom"],
+                    recipients, metadata["headers"], fp,
+                    metadata["subgroups"], metadata["named_subgroups"],
+                    synchronous=True)
         except IOError as e:
-            raise(RuntimeError("unable to read data file: {}".format(e)))
-
+            raise RuntimeError
 
     def release(self, quarantine_id, recipient=None):
         "Release email from quarantine."
@@ -240,7 +275,7 @@ class FileQuarantine(BaseQuarantine):
         except RuntimeError as e:
             raise RuntimeError("unable to release email: {}".format(e))
 
-        if recipient != None:
+        if recipient is not None:
             if recipient not in metadata["recipients"]:
                 raise RuntimeError("invalid recipient '{}'".format(recipient))
             recipients = [recipient]
@@ -256,9 +291,16 @@ class FileQuarantine(BaseQuarantine):
 
         for recipient in recipients:
             try:
-                mailer.smtp_send(self.config["smtp_host"], self.config["smtp_port"], metadata["mailfrom"], recipient, mail)
+                mailer.smtp_send(
+                    self.config["smtp_host"],
+                    self.config["smtp_port"],
+                    metadata["mailfrom"],
+                    recipient,
+                    mail)
             except Exception as e:
-                raise RuntimeError("error while sending email to '{}': {}".format(recipient, e))
+                raise RuntimeError(
+                    "error while sending email to '{}': {}".format(
+                        recipient, e))
 
             self.delete(quarantine_id, recipient)
 

pyquarantine/run.py

@@ -22,17 +22,37 @@ import sys
 
 import pyquarantine
 
+from pyquarantine.version import __version__ as version
 
 def main():
     "Run PyQuarantine-Milter."
     # parse command line
-    parser = argparse.ArgumentParser(description="PyQuarantine milter daemon",
-            formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=45, width=140))
-    parser.add_argument("-c", "--config", help="List of config files to read.", nargs="+",
+    parser = argparse.ArgumentParser(
+        description="PyQuarantine milter daemon",
+        formatter_class=lambda prog: argparse.HelpFormatter(
+            prog, max_help_position=45, width=140))
+    parser.add_argument(
+        "-c", "--config",
+        help="List of config files to read.",
+        nargs="+",
         default=pyquarantine.QuarantineMilter.get_configfiles())
-    parser.add_argument("-s", "--socket", help="Socket used to communicate with the MTA.", required=True)
-    parser.add_argument("-d", "--debug", help="Log debugging messages.", action="store_true")
-    parser.add_argument("-t", "--test", help="Check configuration.", action="store_true")
+    parser.add_argument(
+        "-s", "--socket",
+        help="Socket used to communicate with the MTA.",
+        required=True)
+    parser.add_argument(
+        "-d", "--debug",
+        help="Log debugging messages.",
+        action="store_true")
+    parser.add_argument(
+        "-t", "--test",
+        help="Check configuration.",
+        action="store_true")
+    parser.add_argument(
+        "-v", "--version",
+        help="Print version.",
+        action="version",
+        version="%(prog)s ({})".format(version))
     args = parser.parse_args()
 
     # setup logging
@@ -65,11 +85,14 @@ def main():
             sys.exit(255)
         else:
             sys.exit(0)
-    formatter = logging.Formatter("%(asctime)s {}: [%(levelname)s] %(message)s".format(logname), datefmt="%Y-%m-%d %H:%M:%S")
+    formatter = logging.Formatter(
+        "%(asctime)s {}: [%(levelname)s] %(message)s".format(logname),
+        datefmt="%Y-%m-%d %H:%M:%S")
     stdouthandler.setFormatter(formatter)
 
     # setup syslog
-    sysloghandler = logging.handlers.SysLogHandler(address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL)
+    sysloghandler = logging.handlers.SysLogHandler(
+        address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL)
     sysloghandler.setLevel(loglevel)
     formatter = logging.Formatter("{}: %(message)s".format(syslog_name))
     sysloghandler.setFormatter(formatter)
@@ -89,12 +112,13 @@ def main():
     # register to have the Milter factory create instances of your class:
     Milter.factory = pyquarantine.QuarantineMilter
     Milter.set_exception_policy(Milter.TEMPFAIL)
-    #Milter.set_flags(0)       # tell sendmail which features we use
+    # Milter.set_flags(0)       # tell sendmail which features we use
 
     # run milter
     rc = 0
     try:
-        Milter.runmilter("pyquarantine-milter", socketname=args.socket, timeout=300)
+        Milter.runmilter("pyquarantine-milter", socketname=args.socket,
+                         timeout=300)
     except Milter.milter.error as e:
         logger.error(e)
         rc = 255

pyquarantine/version.py

@@ -0,0 +1 @@
+__version__ = "0.0.5"

pyquarantine/whitelists.py

@@ -12,24 +12,26 @@
 # along with PyQuarantineMilter.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-import datetime
 import logging
 import peewee
 import re
 import sys
 
+from datetime import datetime
 from playhouse.db_url import connect
 
 
 class WhitelistBase(object):
     "Whitelist base class"
+
     def __init__(self, global_config, config, configtest=False):
         self.global_config = global_config
         self.config = config
         self.configtest = configtest
         self.name = config["name"]
         self.logger = logging.getLogger(__name__)
-        self.valid_entry_regex = re.compile(r"^[a-zA-Z0-9_.+-]*?(@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)?$")
+        self.valid_entry_regex = re.compile(
+            r"^[a-zA-Z0-9_.+-]*?(@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)?$")
 
     def check(self, mailfrom, recipient):
         "Check if mailfrom/recipient combination is whitelisted."
@@ -56,15 +58,16 @@ class WhitelistBase(object):
 class WhitelistModel(peewee.Model):
     mailfrom = peewee.CharField()
     recipient = peewee.CharField()
-    created = peewee.DateTimeField(default=datetime.datetime.now)
-    last_used = peewee.DateTimeField(default=datetime.datetime.now)
+    created = peewee.DateTimeField(default=datetime.now)
+    last_used = peewee.DateTimeField(default=datetime.now)
     comment = peewee.TextField(default="")
     permanent = peewee.BooleanField(default=False)
 
 
 class Meta(object):
     indexes = (
-        (('mailfrom', 'recipient'), True), # trailing comma is mandatory if only one index should be created
+        # trailing comma is mandatory if only one index should be created
+        (('mailfrom', 'recipient'), True),
     )
 
 
@@ -74,14 +77,21 @@ class DatabaseWhitelist(WhitelistBase):
     _db_tables = {}
 
     def __init__(self, global_config, config, configtest=False):
-        super(DatabaseWhitelist, self).__init__(global_config, config, configtest)
+        super(
+            DatabaseWhitelist,
+            self).__init__(
+            global_config,
+            config,
+            configtest)
 
         # check if mandatory options are present in config
         for option in ["whitelist_db_connection", "whitelist_db_table"]:
             if option not in self.config.keys() and option in self.global_config.keys():
                 self.config[option] = self.global_config[option]
             if option not in self.config.keys():
-                raise RuntimeError("mandatory option '{}' not present in config section '{}' or 'global'".format(option, self.name))
+                raise RuntimeError(
+                    "mandatory option '{}' not present in config section '{}' or 'global'".format(
+                        option, self.name))
 
         tablename = self.config["whitelist_db_table"]
         connection_string = self.config["whitelist_db_connection"]
@@ -91,10 +101,16 @@ class DatabaseWhitelist(WhitelistBase):
         else:
             try:
                 # connect to database
-                self.logger.debug("connecting to database '{}'".format(re.sub(r"(.*?://.*?):.*?(@.*)", r"\1:<PASSWORD>\2", connection_string)))
+                self.logger.debug(
+                    "connecting to database '{}'".format(
+                        re.sub(
+                            r"(.*?://.*?):.*?(@.*)",
+                            r"\1:<PASSWORD>\2",
+                            connection_string)))
                 db = connect(connection_string)
             except Exception as e:
-                raise RuntimeError("unable to connect to database: {}".format(e))
+                raise RuntimeError(
+                    "unable to connect to database: {}".format(e))
 
             DatabaseWhitelist._db_connections[connection_string] = db
 
@@ -115,7 +131,9 @@ class DatabaseWhitelist(WhitelistBase):
                 try:
                     db.create_tables([self.model])
                 except Exception as e:
-                    raise RuntimeError("unable to initialize table '{}': {}".format(tablename, e))
+                    raise RuntimeError(
+                        "unable to initialize table '{}': {}".format(
+                            tablename, e))
 
     def _entry_to_dict(self, entry):
         result = {}
@@ -144,7 +162,9 @@ class DatabaseWhitelist(WhitelistBase):
         super(DatabaseWhitelist, self).check(mailfrom, recipient)
 
         # generate list of possible mailfroms
-        self.logger.debug("query database for whitelist entries from <{}> to <{}>".format(mailfrom, recipient))
+        self.logger.debug(
+            "query database for whitelist entries from <{}> to <{}>".format(
+                mailfrom, recipient))
         mailfroms = [""]
         if "@" in mailfrom and not mailfrom.startswith("@"):
             mailfroms.append("@{}".format(mailfrom.split("@")[1]))
@@ -158,7 +178,10 @@ class DatabaseWhitelist(WhitelistBase):
 
         # query the database
         try:
-            entries = list(self.model.select().where(self.model.mailfrom.in_(mailfroms), self.model.recipient.in_(recipients)))
+            entries = list(
+                self.model.select().where(
+                    self.model.mailfrom.in_(mailfroms),
+                    self.model.recipient.in_(recipients)))
         except Exception as e:
             raise RuntimeError("unable to query database: {}".format(e))
 
@@ -171,7 +194,7 @@ class DatabaseWhitelist(WhitelistBase):
 
         # use entry with the highest weight
         entry = entries[0]
-        entry.last_used = datetime.datetime.now()
+        entry.last_used = datetime.now()
         entry.save()
         result = {}
         for entry in entries:
@@ -183,21 +206,23 @@ class DatabaseWhitelist(WhitelistBase):
         "Find whitelist entries."
         super(DatabaseWhitelist, self).find(mailfrom, recipients, older_than)
 
-        if type(mailfrom) == str: mailfrom = [mailfrom]
-        if type(recipients) == str: recipients = [recipients]
+        if isinstance(mailfrom, str):
+            mailfrom = [mailfrom]
+        if isinstance(recipients, str):
+            recipients = [recipients]
 
         entries = {}
         try:
             for entry in list(self.model.select()):
-                if older_than != None: 
-                    if (datetime.datetime.now() - entry.last_used).total_seconds() < (older_than * 24 * 3600):
+                if older_than is not None:
+                    if (datetime.now() - entry.last_used).total_seconds() < (older_than * 86400):
                         continue
 
-                if mailfrom != None:
+                if mailfrom is not None:
                     if entry.mailfrom not in mailfrom:
                         continue
 
-                if recipients != None:
+                if recipients is not None:
                     if entry.recipient not in recipients:
                         continue
 
@@ -209,10 +234,20 @@ class DatabaseWhitelist(WhitelistBase):
 
     def add(self, mailfrom, recipient, comment, permanent):
         "Add entry to whitelist."
-        super(DatabaseWhitelist, self).add(mailfrom, recipient, comment, permanent)
+        super(
+            DatabaseWhitelist,
+            self).add(
+            mailfrom,
+            recipient,
+            comment,
+            permanent)
 
         try:
-            self.model.create(mailfrom=mailfrom, recipient=recipient, comment=comment, permanent=permanent)
+            self.model.create(
+                mailfrom=mailfrom,
+                recipient=recipient,
+                comment=comment,
+                permanent=permanent)
         except Exception as e:
             raise RuntimeError("unable to add entry to database: {}".format(e))
 
@@ -224,7 +259,8 @@ class DatabaseWhitelist(WhitelistBase):
             query = self.model.delete().where(self.model.id == whitelist_id)
             deleted = query.execute()
         except Exception as e:
-            raise RuntimeError("unable to delete entry from database: {}".format(e))
+            raise RuntimeError(
+                "unable to delete entry from database: {}".format(e))
 
         if deleted == 0:
             raise RuntimeError("invalid whitelist id")
@@ -239,15 +275,19 @@ class WhitelistCache(object):
             self.check(whitelist, mailfrom, recipient)
 
     def check(self, whitelist, mailfrom, recipient):
-        if whitelist not in self.cache.keys(): self.cache[whitelist] = {}
-        if recipient not in self.cache[whitelist].keys(): self.cache[whitelist][recipient] = None
-        if self.cache[whitelist][recipient] == None:
-            self.cache[whitelist][recipient] = whitelist.check(mailfrom, recipient)
+        if whitelist not in self.cache.keys():
+            self.cache[whitelist] = {}
+        if recipient not in self.cache[whitelist].keys():
+            self.cache[whitelist][recipient] = None
+        if self.cache[whitelist][recipient] is None:
+            self.cache[whitelist][recipient] = whitelist.check(
+                mailfrom, recipient)
         return self.cache[whitelist][recipient]
 
     def get_whitelisted_recipients(self, whitelist, mailfrom, recipients):
         self.load(whitelist, mailfrom, recipients)
-        return list(filter(lambda x: self.cache[whitelist][x], self.cache[whitelist].keys()))
+        return list(
+            filter(lambda x: self.cache[whitelist][x], self.cache[whitelist].keys()))
 
 
 # list of whitelist types and their related whitelist classes

setup.py

@@ -4,8 +4,11 @@ def read_file(fname):
     with open(fname, 'r') as f:
         return f.read()
 
+version = {}
+exec(read_file("pyquarantine/version.py"), version)
+
 setup(name = "pyquarantine",
-    version = "0.0.1",
+    version = version["__version__"],
     author = "Thomas Oettli",
     author_email = "spacefreak@noop.ch",
     description = "A pymilter based sendmail/postfix pre-queue filter.",
@@ -32,6 +35,6 @@ setup(name = "pyquarantine",
             "pyquarantine=pyquarantine.cli:main"
         ]
     },
-    install_requires = ["pymilter", "netaddr", "beautifulsoup4", "peewee"],
+    install_requires = ["pymilter", "netaddr", "beautifulsoup4[lxml]", "peewee"],
     python_requires = ">=3"
 )