Change version to 0.0.9

Signed-off-by: Thomas Oettli <spacefreak@noop.ch>

README.md

@@ -77,11 +77,17 @@ The following configuration options are optional in each quarantine section:
 
   The following template variables are available:
   * **{EMAIL_ENVELOPE_FROM}**  
-    E-mail from-address received by the milter.
+    E-mail sender address received by the milter.
+  * **{EMAIL_ENVELOPE_FROM_URL}**  
+    Like EMAIL_ENVELOPE_FROM, but URL encoded
   * **{EMAIL_FROM}**  
-    Value of the from header of the original e-mail.
+    Value of the FROM header of the original e-mail.
-  * **{EMAIL_TO}**  
+  * **{EMAIL_ENVELOPE_TO}**  
     E-mail recipient address of this notification.
+  * **{EMAIL_ENVELOPE_TO_URL}**  
+    Like EMAIL_ENVELOPE_TO, but URL encoded
+  * **{EMAIL_TO}**  
+    Value of the TO header of the original e-mail.
   * **{EMAIL_SUBJECT}**  
     Configured e-mail subject.
   * **{EMAIL_QUARANTINE_ID}**  

docs/templates/notification.template

@@ -10,6 +10,10 @@
     <td><b>From:</b></td>
     <td>{EMAIL_FROM}</td>
    </tr>
+   <tr>
+    <td><b>Envelope-To:</b></td>
+    <td>{EMAIL_ENVELOPE_TO}</td>
+   </tr>
    <tr>
     <td><b>To:</b></td>
     <td>{EMAIL_TO}</td>

pyquarantine/__init__.py

@@ -95,7 +95,8 @@ class QuarantineMilter(Milter.Base):
     def set_configfiles(config_files):
         QuarantineMilter._config_files = config_files
 
-    def connect(self, IPname, family, hostaddr):
+    def connect(self, hostname, family, hostaddr):
+        self.hostaddr = hostaddr
         self.logger.debug(
             "accepted milter connection from {} port {}".format(
                 *hostaddr))
@@ -134,15 +135,33 @@ class QuarantineMilter(Milter.Base):
             "{}: received queue-id from MTA".format(self.queueid))
         self.recipients = list(self.recipients)
         self.headers = []
+        self.logger.debug(
+            "{}: initializing memory buffer to save email data".format(
+                self.queueid))
+        # initialize memory buffer to save email data
+        self.fp = BytesIO()
         return Milter.CONTINUE
 
     @Milter.noreply
     def header(self, name, value):
-        self.headers.append((name, value))
+        try:
+            # write email header to memory buffer
+            self.fp.write("{}: {}\r\n".format(name, value).encode(
+                encoding="ascii", errors="surrogateescape"))
+            # keep copy of header without surrogates for later use
+            self.headers.append((
+                name.encode(errors="surrogateescape").decode(errors="replace"),
+                value.encode(errors="surrogateescape").decode(errors="replace")))
+        except Exception as e:
+            self.logger.exception(
+                "an exception occured in header function: {}".format(e))
+            return Milter.TEMPFAIL
+
         return Milter.CONTINUE
 
     def eoh(self):
         try:
+            self.fp.write("\r\n".encode(encoding="ascii"))
             self.whitelist_cache = whitelists.WhitelistCache()
 
             # initialize dicts to set quaranines per recipient and keep matches
@@ -238,26 +257,14 @@ class QuarantineMilter(Milter.Base):
                         self.queueid))
                 return Milter.ACCEPT
 
-            # check if the email body is needed
+            # check if the mail body is needed
-            keep_body = False
             for recipient, quarantine in self.recipients_quarantines.items():
                 if quarantine["quarantine_obj"] or quarantine["notification_obj"]:
-                    keep_body = True
+                    # mail body is needed, continue processing
-                    break
+                    return Milter.CONTINUE
 
-            if keep_body:
-                self.logger.debug(
-                    "{}: initializing memory buffer to save email data".format(
-                        self.queueid))
-                # initialize memory buffer to save email data
-                self.fp = BytesIO()
-                # write email headers to memory buffer
-                for name, value in self.headers:
-                    self.fp.write("{}: {}\n".format(name, value).encode())
-                self.fp.write("\n".encode())
-            else:
             # quarantine and notification are disabled on all matching
-                # quarantines, return configured action
+            # quarantines, just return configured action
             quarantine = self._get_preferred_quarantine()
             self.logger.info(
                 "{}: {} matching quarantine is '{}', performing milter action {}".format(
@@ -269,8 +276,6 @@ class QuarantineMilter(Milter.Base):
                 self.setreply("554", "5.7.0", quarantine["reject_reason"])
             return quarantine["milter_action"]
 
-            return Milter.CONTINUE
-
         except Exception as e:
             self.logger.exception(
                 "an exception occured in eoh function: {}".format(e))
@@ -372,6 +377,11 @@ class QuarantineMilter(Milter.Base):
                 "an exception occured in eom function: {}".format(e))
             return Milter.TEMPFAIL
 
+    def close(self):
+        self.logger.debug(
+            "disconnect from {} port {}".format(
+                *self.hostaddr))
+
 
 def generate_milter_config(configtest=False, config_files=[]):
     "Generate the configuration for QuarantineMilter class."

pyquarantine/cli.py

@@ -20,6 +20,8 @@ import logging.handlers
 import sys
 import time
 
+from email.header import decode_header, make_header
+
 import pyquarantine
 
 from pyquarantine.version import __version__ as version
@@ -56,7 +58,7 @@ def print_table(columns, rows):
         # get the length of the longest value
         lengths.append(
             len(str(max(rows, key=lambda x: len(str(x[key])))[key])))
-        # use the the longer one
+        # use the longer one
         length = max(lengths)
         column_lengths.append(length)
         column_formats.append("{{:<{}}}".format(length))
@@ -118,7 +120,11 @@ def list_quarantine_emails(config, args):
                 metadata["date"]))
         row["mailfrom"] = metadata["mailfrom"]
         row["recipient"] = metadata["recipients"].pop(0)
-        row["subject"] = emails[quarantine_id]["headers"]["subject"][:60]
+        if "subject" not in emails[quarantine_id]["headers"].keys():
+            emails[quarantine_id]["headers"]["subject"] = ""
+        row["subject"] = str(make_header(decode_header(
+            emails[quarantine_id]["headers"]["subject"])))[:60].replace(
+                "\r", "").replace("\n", "").strip()
         rows.append(row)
 
         if metadata["recipients"]:

pyquarantine/mailer.py

@@ -26,6 +26,10 @@ process = None
 
 def smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail):
     s = smtplib.SMTP(host=smtp_host, port=smtp_port)
+    s.ehlo()
+    if s.has_extn("STARTTLS"):
+        s.starttls()
+        s.ehlo()
     s.sendmail(mailfrom, [recipient], mail)
     s.quit()
 

pyquarantine/notifications.py

@@ -19,10 +19,12 @@ import re
 from bs4 import BeautifulSoup
 from cgi import escape
 from collections import defaultdict
+from email.header import decode_header, make_header
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 from email.mime.image import MIMEImage
 from os.path import basename
+from urllib.parse import quote
 
 from pyquarantine import mailer
 
@@ -215,48 +217,44 @@ class EMailNotification(BaseNotification):
                 img.add_header("Content-ID", "<{}>".format(basename(img_path)))
                 self.embedded_imgs.append(img)
 
-    def get_text(self, queueid, part):
+    def get_decoded_email_body(self, queueid, msg, preferred=_html_text):
-        "Get the mail text in html form from email part."
+        "Find and decode email body."
-        mimetype = part.get_content_type()
+        # try to find the body part
+        self.logger.debug("{}: trying to find email body".format(queueid))
+        body = None
+        for part in msg.walk():
+            content_type = part.get_content_type()
+            if content_type in [EMailNotification._plain_text,
+                                EMailNotification._html_text]:
+                body = part
+                if content_type == preferred:
+                    break
 
+        if body is not None:
+            charset = body.get_content_charset() or "utf-8"
+            content = body.get_payload(decode=True)
+            try:
+                content = content.decode(encoding=charset, errors="replace")
+            except LookupError:
+                content = content.decode("utf-8", errors="replace")
+            content_type = body.get_content_type()
+            if content_type == EMailNotification._plain_text:
+                # convert text/plain to text/html
                 self.logger.debug(
-            "{}: extracting content of email text part".format(queueid))
+                    "{}: content type is {}, converting to {}".format(
-        text = part.get_payload(decode=True)
+                        queueid, content_type, EMailNotification._html_text))
-
+                content = re.sub(r"^(.*)$", r"\1<br/>",
-        if mimetype == EMailNotification._plain_text:
+                                 escape(content), flags=re.MULTILINE)
-            self.logger.debug(
-                "{}: content mimetype is {}, converting to {}".format(
-                    queueid, mimetype, self._html_text))
-            text = re.sub(r"^(.*)$", r"\1<br/>",
-                          escape(text.decode()), flags=re.MULTILINE)
             else:
                 self.logger.debug(
-                "{}: content mimetype is {}".format(
+                    "{}: content type is {}".format(
-                    queueid, mimetype))
+                        queueid, content_type))
-        self.logger.debug(
+        else:
-            "{}: trying to create BeatufilSoup object with parser lib {}, "
+            self.logger.error(
-            "text length is {} bytes".format(
+                "{}: unable to find email body".format(queueid))
-                queueid, self.parser_lib, len(text)))
+            content = "ERROR: unable to find email body"
-        soup = BeautifulSoup(text, self.parser_lib)
-        self.logger.debug(
-            "{}: sucessfully created BeautifulSoup object".format(queueid))
-        return soup
 
-    def get_text_multipart(self, queueid, msg, preferred=_html_text):
+        return content
-        "Get the mail text of a multipart email in html form."
-        soup = None
-
-        for part in msg.get_payload():
-            mimetype = part.get_content_type()
-            if mimetype in [EMailNotification._plain_text,
-                            EMailNotification._html_text]:
-                soup = self.get_text(queueid, part)
-            elif mimetype.startswith("multipart"):
-                soup = self.get_text_multipart(queueid, part, preferred)
-
-            if soup is not None and mimetype == preferred:
-                break
-        return soup
 
     def sanitize(self, queueid, soup):
         "Sanitize mail html text."
@@ -293,27 +291,6 @@ class EMailNotification(BaseNotification):
                         del(element.attrs[attribute])
         return soup
 
-    def get_html_text_part(self, queueid, msg):
-        "Get the mail text of an email in html form."
-        soup = None
-        mimetype = msg.get_content_type()
-
-        self.logger.debug(
-            "{}: trying to find text part of email".format(queueid))
-        if mimetype in [EMailNotification._plain_text,
-                        EMailNotification._html_text]:
-            soup = self.get_text(queueid, msg)
-        elif mimetype.startswith("multipart"):
-            soup = self.get_text_multipart(queueid, msg)
-
-        if soup is None:
-            self.logger.error(
-                "{}: unable to extract text part of email".format(queueid))
-            text = "ERROR: unable to extract text from email body"
-            soup = BeautifulSoup(text, "lxml", "UTF-8")
-
-        return soup
-
     def notify(self, queueid, quarantine_id, mailfrom, recipients, headers, fp,
                subgroups=None, named_subgroups=None, synchronous=False):
         "Notify recipients via email."
@@ -330,12 +307,19 @@ class EMailNotification(BaseNotification):
             named_subgroups,
             synchronous)
 
-        # extract html text from email
+        # extract body from email
-        self.logger.debug(
+        content = self.get_decoded_email_body(
-            "{}: extraction email text from original email".format(queueid))
-        soup = self.get_html_text_part(
             queueid, email.message_from_binary_file(fp))
 
+        # create BeautifulSoup object
+        self.logger.debug(
+            "{}: trying to create BeatufilSoup object with parser lib {}, "
+            "text length is {} bytes".format(
+                queueid, self.parser_lib, len(content)))
+        soup = BeautifulSoup(content, self.parser_lib)
+        self.logger.debug(
+            "{}: sucessfully created BeautifulSoup object".format(queueid))
+
         # replace picture sources
         image_replaced = False
         if self.strip_images:
@@ -368,14 +352,26 @@ class EMailNotification(BaseNotification):
                 "{}: generating notification email for '{}'".format(
                     queueid, recipient))
             self.logger.debug("{}: parsing email template".format(queueid))
+            # decode some headers
+            decoded_headers = {}
+            for header in ["from", "to", "subject"]:
+                if header in headers:
+                    decoded_headers[header] = str(
+                        make_header(decode_header(headers[header])))
+                else:
+                    headers[header] = ""
+                    decoded_headers[header] = ""
 
             # generate dict containing all template variables
             variables = defaultdict(str,
                                     EMAIL_HTML_TEXT=sanitized_text,
-                                    EMAIL_FROM=escape(headers["from"]),
+                                    EMAIL_FROM=escape(decoded_headers["from"]),
                                     EMAIL_ENVELOPE_FROM=escape(mailfrom),
-                                    EMAIL_TO=escape(recipient),
+                                    EMAIL_ENVELOPE_FROM_URL=escape(quote(mailfrom)),
-                                    EMAIL_SUBJECT=escape(headers["subject"]),
+                                    EMAIL_TO=escape(decoded_headers["to"]),
+                                    EMAIL_ENVELOPE_TO=escape(recipient),
+                                    EMAIL_ENVELOPE_TO_URL=escape(quote(recipient)),
+                                    EMAIL_SUBJECT=escape(decoded_headers["subject"]),
                                     EMAIL_QUARANTINE_ID=quarantine_id)
 
             if subgroups:

pyquarantine/version.py

@@ -1 +1 @@
-__version__ = "0.0.3"
+__version__ = "0.0.9"