fix header decoding

Signed-off-by: Thomas Oettli <spacefreak@noop.ch>

README.md

@@ -120,7 +120,7 @@ Config options for conditions objects:
 * **headers** (optional)  
   Matches e-mails for which all regular expressions in the given list are matching at least one e-mail header.
 * **whitelist** (optional)  
-  Matches e-mails for which the whitelist has no entry for the envelope-from and envelope-to address combination, see section [Whitelist](#Whitelist).
+  Matches e-mails for which the whitelist has no entry for the envelope-from and envelope-to address combination, see section [Whitelist](#Whitelist) for details.
 * **var** (optional)  
   Matches e-mails for which a previous action or condition has set the given metavariable.
 * **metavar** (optional)  
@@ -210,14 +210,14 @@ Options:
 Quarantine e-mail.  
 Options:
 * **store**  
-  Options for e-mail storage, see action **store** in section [Action types](#Action-types).  
+  Options for e-mail storage, see action [store](#store).  
   If the option **metadata** is not specificall set for this storage, it will be set to true.
 * **smtp_host**  
   SMTP host used to release e-mails from quarantine.
 * **smtp_port**  
   SMTP port used to release e-mails from quarantine.
 * **notify** (optional)  
-  Options for e-mail notifications, see action **notify** in section [Action types](#Action-types).
+  Options for e-mail notifications, see action [notify](#notify).
 * **milter_action** (optional)  
   Milter action to perform. If set, no further rules or actions will be processed.  
   Please think carefully what you set here or your MTA may do something you do not want it to do.  
@@ -231,7 +231,7 @@ Options:
 * **reject_reason** (optional, default: "Message rejected")  
   Reject message sent to MTA if milter_action is set to reject.
 * **whitelist** (optional)  
-  Options for a whitelist, see **whitelist** in section [Conditions](#Conditions).
+  Options for a whitelist object, see section [Whitelist](#Whitelist).
 
 ### Storage types
 Available storage types:

distribution/gentoo/mail-filter/pyquarantine-milter/pyquarantine-milter-9999.ebuild

@@ -2,7 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 
 EAPI=7
-PYTHON_COMPAT=( python3_{9,10} )
+PYTHON_COMPAT=( python3_{9..10} )
 DISTUTILS_USE_SETUPTOOLS=rdepend
 
 SCM=""
@@ -37,7 +37,7 @@ RDEPEND="
 	lxml? ( dev-python/lxml[${PYTHON_USEDEP}] )
 	dev-python/netaddr[${PYTHON_USEDEP}]
 	dev-python/peewee[${PYTHON_USEDEP}]
-	dev-python/pymilter[${PYTHON_USEDEP}]"
+	>=dev-python/pymilter-1.5[${PYTHON_USEDEP}]"
 
 python_install_all() {
 	distutils-r1_python_install_all

distribution/pypi/build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -e
-
+set -x
 PYTHON=$(which python)
 
 script_dir=$(dirname "$(readlink -f -- "$BASH_SOURCE")")

pyquarantine/__init__.py

@@ -27,7 +27,7 @@ __all__ = [
     "whitelist",
     "QuarantineMilter"]
 
-__version__ = "2.0.0"
+__version__ = "2.0.7"
 
 from pyquarantine import _runtime_patches
 
@@ -208,6 +208,7 @@ class QuarantineMilter(Milter.Base):
 
         return Milter.CONTINUE
 
+    @Milter.decode("replace")
     def envfrom(self, mailfrom, *str):
         try:
             self.mailfrom = "@".join(parse_addr(mailfrom)).lower()
@@ -219,6 +220,7 @@ class QuarantineMilter(Milter.Base):
 
         return Milter.CONTINUE
 
+    @Milter.decode("replace")
     def envrcpt(self, to, *str):
         try:
             self.rcpts.add("@".join(parse_addr(to)).lower())
@@ -243,6 +245,7 @@ class QuarantineMilter(Milter.Base):
 
         return Milter.CONTINUE
 
+    @Milter.decode("replace")
     def header(self, field, value):
         try:
             # remove CR and LF from address fields, otherwise pythons

pyquarantine/_install.py

@@ -21,14 +21,18 @@ import shutil
 import sys
 
 
-SYSTEMD_PATH = "/lib/systemd/system"
+SYSTEMD_PATHS = ["/lib/systemd/system", "/usr/lib/systemd/system"]
 OPENRC = "/sbin/openrc"
 
 
 def _systemd_files(pkg_dir, name):
+    for path in SYSTEMD_PATHS:
+        if os.path.isdir(path):
+            break
+
     return [
         (f"{pkg_dir}/misc/systemd/{name}-milter.service",
-            f"{SYSTEMD_PATH}/{name}-milter.service", True)]
+            f"{path}/{name}-milter.service", True)]
 
 
 def _openrc_files(pkg_dir, name):
@@ -117,7 +121,11 @@ def _check_root():
 
 
 def _check_systemd():
-    systemd = os.path.isdir(SYSTEMD_PATH)
+    for path in SYSTEMD_PATHS:
+        systemd = os.path.isdir(path)
+        if systemd:
+            break
+
     if systemd:
         logging.info("systemd detected")
 

pyquarantine/_runtime_patches.py

@@ -12,6 +12,7 @@
 # along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from sys import version_info
 import encodings
 
 
@@ -150,27 +151,45 @@ def get_obs_local_part(value):
 setattr(email._header_value_parser, "get_obs_local_part", get_obs_local_part)
 
 
-# https://bugs.python.org/issue30681
+#######################################
+#  add charset alias for windows-874  #
+#######################################
 #
-# fix: https://github.com/python/cpython/pull/22090
+# https://bugs.python.org/issue17254
+#
+# fix: https://github.com/python/cpython/pull/10237
 
-import email.errors
-from email.errors import HeaderDefect
+aliases = encodings.aliases.aliases
+
+for alias in ["windows-874", "windows_874"]:
+    if alias not in aliases:
+        aliases[alias] = "cp874"
+
+setattr(encodings.aliases, "aliases", aliases)
 
 
-class InvalidDateDefect(HeaderDefect):
+if version_info.major == 3 and version_info.minor < 10:
+    # https://bugs.python.org/issue30681
+    #
+    # fix: https://github.com/python/cpython/pull/22090
+
+    import email.errors
+    from email.errors import HeaderDefect
+
+
+    class InvalidDateDefect(HeaderDefect):
         """Header has unparseable or invalid date"""
 
 
-setattr(email.errors, "InvalidDateDefect", InvalidDateDefect)
+    setattr(email.errors, "InvalidDateDefect", InvalidDateDefect)
 
 
-import email.utils
-from email.utils import _parsedate_tz
-import datetime
+    import email.utils
+    from email.utils import _parsedate_tz
+    import datetime
 
 
-def parsedate_to_datetime(data):
+    def parsedate_to_datetime(data):
         parsed_date_tz = _parsedate_tz(data)
         if parsed_date_tz is None:
             raise ValueError('Invalid date value or format "%s"' % str(data))
@@ -181,14 +200,14 @@ def parsedate_to_datetime(data):
                 tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
 
 
-setattr(email.utils, "parsedate_to_datetime", parsedate_to_datetime)
+    setattr(email.utils, "parsedate_to_datetime", parsedate_to_datetime)
 
 
-import email.headerregistry
-from email import utils, _header_value_parser as parser
+    import email.headerregistry
+    from email import utils, _header_value_parser as parser
 
-@classmethod
-def parse(cls, value, kwds):
+    @classmethod
+    def parse(cls, value, kwds):
         if not value:
             kwds['defects'].append(errors.HeaderMissingRequiredValue())
             kwds['datetime'] = None
@@ -209,21 +228,4 @@ def parse(cls, value, kwds):
         kwds['parse_tree'] = cls.value_parser(kwds['decoded'])
 
 
-setattr(email.headerregistry.DateHeader, "parse", parse)
-
-
-#######################################
-#  add charset alias for windows-874  #
-#######################################
-#
-# https://bugs.python.org/issue17254
-#
-# fix: https://github.com/python/cpython/pull/10237
-
-aliases = encodings.aliases.aliases
-
-for alias in ["windows-874", "windows_874"]:
-    if alias not in aliases:
-        aliases[alias] = "cp874"
-
-setattr(encodings.aliases, "aliases", aliases)
+    setattr(email.headerregistry.DateHeader, "parse", parse)

pyquarantine/base.py

@@ -65,6 +65,134 @@ class MilterMessage(MIMEPart):
 
         self._headers = newheaders
 
+    def _find_body_parent(self, part, preferencelist, parent=None):
+        if part.is_attachment():
+            return
+        maintype, subtype = part.get_content_type().split("/")
+        if maintype == "text":
+            if subtype in preferencelist:
+                yield(preferencelist.index(subtype), parent)
+            return
+        if maintype != "multipart" or not self.is_multipart():
+            return
+        if subtype != "related":
+            for subpart in part.iter_parts():
+                yield from self._find_body_parent(
+                    subpart, preferencelist, part)
+            return
+        if 'related' in preferencelist:
+            yield(preferencelist.index('related'), parent)
+        candidate = None
+        start = part.get_param('start')
+        if start:
+            for subpart in part.iter_parts():
+                if subpart['content-id'] == start:
+                    candidate = subpart
+                    break
+        if candidate is None:
+            subparts = part.get_payload()
+            candidate = subparts[0] if subparts else None
+        if candidate is not None:
+            yield from self._find_body_parent(candidate, preferencelist, part)
+
+    def get_body_parent(self, preferencelist=("related", "html", "plain")):
+        best_prio = len(preferencelist)
+        body_parent = None
+        for prio, parent in self._find_body_parent(self, preferencelist):
+            if prio < best_prio:
+                best_prio = prio
+                body_parent = parent
+                if prio == 0:
+                    break
+        return body_parent
+
+    def get_body_content(self, pref):
+        part = None
+        content = None
+        if not self.is_multipart() and \
+                self.get_content_type() == f"text/{pref}":
+            part = self
+        else:
+            part = self.get_body(preferencelist=(pref))
+
+        if part is not None:
+            content = part.get_content()
+
+        return (part, content)
+
+    def set_body(self, text_content=None, html_content=None):
+        parent = self.get_body_parent() or self
+        if "Content-Type" not in parent:
+            # set Content-Type header if not present, otherwise
+            # make_alternative and make_mixed skip the payload
+            parent["Content-Type"] = parent.get_content_type()
+
+        maintype, subtype = parent.get_content_type().split("/")
+        if not parent.is_multipart() or maintype != "multipart":
+            if maintype == "text" and subtype in ("html", "plain"):
+                parent.make_alternative()
+                maintype, subtype = ("multipart", "alternative")
+            else:
+                parent.make_mixed()
+                maintype, subtype = ("multipart", "mixed")
+
+        text_body = parent.get_body(preferencelist=("plain"))
+        html_body = parent.get_body(preferencelist=("html"))
+
+        if text_content is not None:
+            if text_body:
+                text_body.set_content(text_content)
+            else:
+                if not html_body or subtype == "alternative":
+                    inject_body_part(parent, text_content)
+                else:
+                    html_body.add_alternative(text_content)
+                text_body = parent.get_body(preferencelist=("plain"))
+
+        if html_content is not None:
+            if html_body:
+                html_body.set_content(html_content, subtype="html")
+            else:
+                if not text_body or subtype == "alternative":
+                    inject_body_part(parent, html_content, subtype="html")
+                else:
+                    text_body.add_alternative(html_content, subtype="html")
+
+
+def inject_body_part(part, content, subtype="plain"):
+    parts = []
+    text_body = None
+    text_content = None
+    if subtype == "html":
+        text_body, text_content = part.get_body_content("plain")
+
+    for p in part.iter_parts():
+        if text_body and p == text_body:
+            continue
+        parts.append(p)
+
+    boundary = part.get_boundary()
+    p_subtype = part.get_content_subtype()
+    part.clear_content()
+    if text_content != None:
+        part.set_content(text_content)
+        part.add_alternative(content, subtype=subtype)
+    else:
+        part.set_content(content, subtype=subtype)
+
+    if part.get_content_subtype() != p_subtype:
+        if p_subtype == "alternative":
+            part.make_alternative()
+        elif p_subtype == "related":
+            part.make_related()
+        else:
+            part.make_mixed()
+
+    if boundary:
+        part.set_boundary(boundary)
+    for p in parts:
+        part.attach(p)
+
 
 def replace_illegal_chars(string):
     """Remove illegal characters from header values."""

pyquarantine/modify.py

@@ -183,26 +183,6 @@ def _has_content_before_body_tag(soup):
     return False
 
 
-def _inject_body(milter, text_content="", html_content=None):
-    if not milter.msg.is_multipart():
-        milter.msg.make_mixed()
-
-    attachments = []
-    for attachment in milter.msg.iter_attachments():
-        if "content-disposition" not in attachment:
-            attachment["Content-Disposition"] = "attachment"
-        attachments.append(attachment)
-
-    milter.msg.clear_content()
-    milter.msg.set_content(text_content)
-    if html_content is not None:
-        milter.msg.add_alternative(html_content, subtype="html")
-    milter.msg.make_mixed()
-
-    for attachment in attachments:
-        milter.msg.attach(attachment)
-
-
 def _wrap_message(milter):
     attachment = MIMEPart(policy=SMTP)
     attachment.set_content(milter.msg_as_bytes(),
@@ -248,29 +228,28 @@ class AddDisclaimer:
         self.pretend = pretend
 
     def patch_message_body(self, milter, logger):
-        text_body, text_content = _get_body_content(milter.msg, "plain")
-        html_body, html_content = _get_body_content(milter.msg, "html")
+        text_body, text_content = milter.msg.get_body_content("plain")
+        html_body, html_content = milter.msg.get_body_content("html")
 
         if text_content is None and html_content is None:
-            logger.info("message contains only attachment(s), "
-                        "inject empty body")
+            logger.info("message contains no body, inject it")
             if self.add_html_body:
-                _inject_body(milter, "", "")
-                html_body, html_content = _get_body_content(milter.msg, "html")
+                milter.msg.set_body("", "")
+                html_body, html_content = milter.msg.get_body_content("html")
             else:
-                _inject_body(milter, "")
-            text_body, text_content = _get_body_content(milter.msg, "plain")
+                milter.msg.set_body("")
+            text_body, text_content = milter.msg.get_body_content("plain")
 
         if html_content is None and self.add_html_body:
-            logger.info("inject html body part generated from plain body")
+            logger.info("inject html body based on plain body")
             header = '<meta http-equiv="Content-Type" content="text/html; ' \
                      'charset=utf-8">'
             html_text = re.sub(r"^(.*)$", r"\1<br/>",
                                escape(text_content, quote=False),
                                flags=re.MULTILINE)
-            _inject_body(milter, text_content, f"{header}{html_text}")
-            text_body, text_content = _get_body_content(milter.msg, "plain")
-            html_body, html_content = _get_body_content(milter.msg, "html")
+            milter.msg.set_body(None, f"{header}{html_text}")
+            text_body, text_content = milter.msg.get_body_content("plain")
+            html_body, html_content = milter.msg.get_body_content("html")
 
         variables = defaultdict(str, milter.msginfo["vars"])
         variables["ENVELOPE_FROM"] = escape(

pyquarantine/run.py

@@ -135,7 +135,7 @@ def main():
     sysloghandler = logging.handlers.SysLogHandler(
         address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL)
     sysloghandler.setFormatter(
-        logging.Formatter("pyquarantine: %(message)s"))
+        logging.Formatter(f"{name}[%(process)d]: %(message)s"))
     root_logger.addHandler(sysloghandler)
 
     logger.info("milter starting")
@@ -149,7 +149,7 @@ def main():
 
     # increase the recursion limit so that BeautifulSoup can
     # parse larger html content
-    sys.setrecursionlimit(2000)
+    sys.setrecursionlimit(4000)
 
     rc = 0
     try:

pyquarantine/storage.py

@@ -549,7 +549,7 @@ class Quarantine:
             if not rcpts:
                 # all recipients whitelisted
                 return
-            milter.msginfo["rcpts"] = rcpts
+            milter.msginfo["rcpts"] = rcpts.copy()
 
         if self._milter_action in ["REJECT", "DISCARD"]:
             logger.info(f"quarantine message for {rcpts}")
@@ -563,6 +563,5 @@ class Quarantine:
 
         if self._milter_action is not None:
             milter.delrcpt(rcpts)
-            if self._milter_action in ["ACCEPT", "REJECT"] and \
-                    not milter.msginfo["rcpts"]:
+            if not milter.msginfo["rcpts"]:
                 return (self._milter_action, self._reason)

setup.py

@@ -18,7 +18,7 @@ setup(name = "pyquarantine",
         #   3 - Alpha
         #   4 - Beta
         #   5 - Production/Stable
-        "Development Status :: 4 - Beta",
+        "Development Status :: 5 - Production/Stable",
         "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
         "Operating System :: OS Independent",
         "Programming Language :: Python",
@@ -48,6 +48,6 @@ setup(name = "pyquarantine",
             ]
         )
     ],
-    install_requires = ["pymilter", "jsonschema", "netaddr", "beautifulsoup4[lxml]", "peewee"],
-    python_requires = ">=3.8"
+    install_requires = ["pymilter >= 1.5", "jsonschema", "netaddr", "beautifulsoup4[lxml]", "peewee"],
+    python_requires = ">=3.9"
 )