Improve email body extraction and decoding

README.md

@@ -45,7 +45,7 @@ The following configuration options are mandatory in each quarantine section:
 * **whitelist_type**  
   One of the whitelist types described below.
 * **smtp_host**  
-  SMTP host to inject original e-mails. This is needed if not all recipients of an e-mail are whitelisted
+  SMTP host used to release original e-mails from the quarantine.
 * **smtp_port**  
   SMTP port
 
@@ -97,6 +97,10 @@ The following configuration options are optional in each quarantine section:
     Content of a named subgroup, 'subgroup_name' will be replaced by its name.
 
   The following configuration options are mandatory for this notification type:
+  * **notification_email_smtp_host**  
+    SMTP host used to send notification e-mails.
+  * **notification_email_smtp_port**  
+    SMTP port.
   * **notification_email_envelope_from**  
     Notification e-mail envelope from-address.
   * **notification_email_from**  

docs/pyquarantine.conf.example

@@ -88,6 +88,18 @@ reject_reason = Message rejected
 #
 notification_type = email
 
+# Option: notification_email_smtp_host
+# Notes:  Set the SMTP host. It will be used to send notification e-mails.
+# Values: [ HOSTNAME | IP_ADDRESS ]
+#
+notification_email_smtp_host = 127.0.0.1
+
+# Option: notification_email_smtp_port
+# Notes:  Set the SMTP port.
+# Values: [ PORT ]
+#
+notification_email_smtp_port = 25
+
 # Option: notification_email_envelope_from
 # Notes:  Set the envelope-from address used when sending notification emails.
 #         This option is needed by notification type 'email'.

pyquarantine/mailer.py

@@ -26,6 +26,10 @@ process = None
 
 def smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail):
     s = smtplib.SMTP(host=smtp_host, port=smtp_port)
+    s.ehlo()
+    if s.has_extn("STARTTLS"):
+        s.starttls()
+        s.ehlo()
     s.sendmail(mailfrom, [recipient], mail)
     s.quit()
 

pyquarantine/notifications.py

@@ -115,8 +115,8 @@ class EMailNotification(BaseNotification):
 
         # check if mandatory options are present in config
         for option in [
-            "smtp_host",
-            "smtp_port",
+            "notification_email_smtp_host",
+            "notification_email_smtp_port",
             "notification_email_envelope_from",
             "notification_email_from",
             "notification_email_subject",
@@ -142,8 +142,8 @@ class EMailNotification(BaseNotification):
             if option not in config.keys():
                 config[option] = defaults[option]
 
-        self.smtp_host = self.config["smtp_host"]
-        self.smtp_port = self.config["smtp_port"]
+        self.smtp_host = self.config["notification_email_smtp_host"]
+        self.smtp_port = self.config["notification_email_smtp_port"]
         self.mailfrom = self.config["notification_email_envelope_from"]
         self.from_header = self.config["notification_email_from"]
         self.subject = self.config["notification_email_subject"]
@@ -215,48 +215,47 @@ class EMailNotification(BaseNotification):
                 img.add_header("Content-ID", "<{}>".format(basename(img_path)))
                 self.embedded_imgs.append(img)
 
-    def get_text(self, queueid, part):
-        "Get the mail text in html form from email part."
-        mimetype = part.get_content_type()
+    def get_decoded_email_body(self, queueid, msg, preferred=_html_text):
+        "Find and decode email body."
+        # try to find the body part
+        self.logger.debug("{}: trying to find email body".format(queueid))
+        body = None
+        for part in msg.walk():
+            content_type = part.get_content_type()
+            if content_type in [EMailNotification._plain_text,
+                                EMailNotification._html_text]:
+                body = part
+                if content_type == preferred:
+                    break
 
-        self.logger.debug(
-            "{}: extracting content of email text part".format(queueid))
-        text = part.get_payload(decode=True)
+        if body is not None:
+            # get the character set, fallback to utf-8 if not defined in header
+            charset = body.get_content_charset()
+            if charset is None:
+                charset = "utf-8"
 
-        if mimetype == EMailNotification._plain_text:
+            # decode content
+            content = body.get_payload(decode=True).decode(
+                encoding=charset, errors="replace")
+
+            content_type = body.get_content_type()
+            if content_type == EMailNotification._plain_text:
+                # convert text/plain to text/html
                 self.logger.debug(
-                "{}: content mimetype is {}, converting to {}".format(
-                    queueid, mimetype, self._html_text))
-            text = re.sub(r"^(.*)$", r"\1<br/>",
-                          escape(text.decode()), flags=re.MULTILINE)
+                    "{}: content type is {}, converting to {}".format(
+                        queueid, content_type, EMailNotification._html_text))
+                content = re.sub(r"^(.*)$", r"\1<br/>",
+                                 escape(content), flags=re.MULTILINE)
             else:
                 self.logger.debug(
-                "{}: content mimetype is {}".format(
-                    queueid, mimetype))
-        self.logger.debug(
-            "{}: trying to create BeatufilSoup object with parser lib {}, "
-            "text length is {} bytes".format(
-                queueid, self.parser_lib, len(text)))
-        soup = BeautifulSoup(text, self.parser_lib)
-        self.logger.debug(
-            "{}: sucessfully created BeautifulSoup object".format(queueid))
-        return soup
+                    "{}: content type is {}".format(
+                        queueid, content_type))
+        else:
+            self.logger.error(
+                "{}: unable to find email body".format(queueid))
+            content = "ERROR: unable to find email body"
 
-    def get_text_multipart(self, queueid, msg, preferred=_html_text):
-        "Get the mail text of a multipart email in html form."
-        soup = None
-
-        for part in msg.get_payload():
-            mimetype = part.get_content_type()
-            if mimetype in [EMailNotification._plain_text,
-                            EMailNotification._html_text]:
-                soup = self.get_text(queueid, part)
-            elif mimetype.startswith("multipart"):
-                soup = self.get_text_multipart(queueid, part, preferred)
-
-            if soup is not None and mimetype == preferred:
-                break
-        return soup
+        return content
 
     def sanitize(self, queueid, soup):
         "Sanitize mail html text."
@@ -293,27 +292,6 @@ class EMailNotification(BaseNotification):
                         del(element.attrs[attribute])
         return soup
 
-    def get_html_text_part(self, queueid, msg):
-        "Get the mail text of an email in html form."
-        soup = None
-        mimetype = msg.get_content_type()
-
-        self.logger.debug(
-            "{}: trying to find text part of email".format(queueid))
-        if mimetype in [EMailNotification._plain_text,
-                        EMailNotification._html_text]:
-            soup = self.get_text(queueid, msg)
-        elif mimetype.startswith("multipart"):
-            soup = self.get_text_multipart(queueid, msg)
-
-        if soup is None:
-            self.logger.error(
-                "{}: unable to extract text part of email".format(queueid))
-            text = "ERROR: unable to extract text from email body"
-            soup = BeautifulSoup(text, "lxml", "UTF-8")
-
-        return soup
-
     def notify(self, queueid, quarantine_id, mailfrom, recipients, headers, fp,
                subgroups=None, named_subgroups=None, synchronous=False):
         "Notify recipients via email."
@@ -330,12 +308,19 @@ class EMailNotification(BaseNotification):
             named_subgroups,
             synchronous)
 
-        # extract html text from email
-        self.logger.debug(
-            "{}: extraction email text from original email".format(queueid))
-        soup = self.get_html_text_part(
+        # extract body from email
+        content = self.get_decoded_email_body(
             queueid, email.message_from_binary_file(fp))
 
+        # create BeautifulSoup object
+        self.logger.debug(
+            "{}: trying to create BeatufilSoup object with parser lib {}, "
+            "text length is {} bytes".format(
+                queueid, self.parser_lib, len(content)))
+        soup = BeautifulSoup(content, self.parser_lib)
+        self.logger.debug(
+            "{}: sucessfully created BeautifulSoup object".format(queueid))
+
         # replace picture sources
         image_replaced = False
         if self.strip_images:

pyquarantine/version.py

@@ -1 +1 @@
-__version__ = "0.0.2"
+__version__ = "0.0.5"