fix encoding issues

pymodmilter/__init__.py

@@ -34,7 +34,7 @@ from Milter.utils import parse_addr
 from collections import defaultdict
 from email import message_from_binary_file
 from email.header import Header
-from email.policy import default as default_policy, SMTP
+from email.policy import SMTPUTF8
 from io import BytesIO
 from netaddr import IPNetwork, AddrFormatError
 
@@ -297,7 +297,7 @@ class ModifyMilter(Milter.Base):
         try:
             self.fp.seek(0)
             self.msg = message_from_binary_file(
-                self.fp, _class=MilterMessage, policy=default_policy)
+                self.fp, _class=MilterMessage, policy=SMTPUTF8)
 
             self.msg_info = defaultdict(str)
             self.msg_info["ip"] = self.IP
@@ -316,7 +316,7 @@ class ModifyMilter(Milter.Base):
                     break
 
             if self._replacebody:
-                data = self.msg.as_bytes(policy=SMTP)
+                data = self.msg.as_bytes()
                 body_pos = data.find(b"\r\n\r\n") + 4
                 self.logger.debug("milter: replacebody")
                 super().replacebody(data[body_pos:])

pymodmilter/_runtime_patches.py

@@ -90,6 +90,66 @@ setattr(email._header_value_parser, "DisplayName", DisplayName)
 setattr(email._header_value_parser, "get_name_addr", get_name_addr)
 
 
+# https://bugs.python.org/issue42484
+#
+# fix: https://github.com/python/cpython/pull/24669
+
+from email._header_value_parser import DOT, ObsLocalPart, ValueTerminal, get_word
+
+
+def get_obs_local_part(value):
+    """ obs-local-part = word *("." word)
+    """
+    obs_local_part = ObsLocalPart()
+    last_non_ws_was_dot = False
+    while value and (value[0]=='\\' or value[0] not in PHRASE_ENDS):
+        if value[0] == '.':
+            if last_non_ws_was_dot:
+                obs_local_part.defects.append(errors.InvalidHeaderDefect(
+                    "invalid repeated '.'"))
+            obs_local_part.append(DOT)
+            last_non_ws_was_dot = True
+            value = value[1:]
+            continue
+        elif value[0]=='\\':
+            obs_local_part.append(ValueTerminal(value[0],
+                                                'misplaced-special'))
+            value = value[1:]
+            obs_local_part.defects.append(errors.InvalidHeaderDefect(
+                "'\\' character outside of quoted-string/ccontent"))
+            last_non_ws_was_dot = False
+            continue
+        if obs_local_part and obs_local_part[-1].token_type != 'dot':
+            obs_local_part.defects.append(errors.InvalidHeaderDefect(
+                "missing '.' between words"))
+        try:
+            token, value = get_word(value)
+            last_non_ws_was_dot = False
+        except errors.HeaderParseError:
+            if value[0] not in CFWS_LEADER:
+                raise
+            token, value = get_cfws(value)
+        obs_local_part.append(token)
+    if not obs_local_part:
+        return obs_local_part, value
+    if (obs_local_part[0].token_type == 'dot' or
+            obs_local_part[0].token_type=='cfws' and
+            obs_local_part[1].token_type=='dot'):
+        obs_local_part.defects.append(errors.InvalidHeaderDefect(
+            "Invalid leading '.' in local part"))
+    if (obs_local_part[-1].token_type == 'dot' or
+            obs_local_part[-1].token_type=='cfws' and
+            obs_local_part[-2].token_type=='dot'):
+        obs_local_part.defects.append(errors.InvalidHeaderDefect(
+            "Invalid trailing '.' in local part"))
+    if obs_local_part.defects:
+        obs_local_part.token_type = 'invalid-obs-local-part'
+    return obs_local_part, value
+
+
+setattr(email._header_value_parser, "get_obs_local_part", get_obs_local_part)
+
+
 # https://bugs.python.org/issue30681
 #
 # fix: https://github.com/python/cpython/pull/22090

pymodmilter/actions.py

@@ -32,7 +32,6 @@ from collections import defaultdict
 from copy import copy
 from datetime import datetime
 from email.message import MIMEPart
-from email.policy import SMTP
 
 from pymodmilter import CustomLogger, BaseConfig
 from pymodmilter.conditions import ConditionsConfig, Conditions
@@ -205,7 +204,7 @@ def _patch_message_body(milter, action, text_template, html_template, logger):
 
 def _wrap_message(milter, logger):
     attachment = MIMEPart()
-    attachment.set_content(milter.msg.as_bytes(policy=SMTP),
+    attachment.set_content(milter.msg.as_bytes(),
                            maintype="plain", subtype="text",
                            disposition="attachment",
                            filename=f"{milter.qid}.eml",
@@ -331,7 +330,7 @@ def store(milter, directory, original=False, pretend=False,
                 milter.fp.seek(0)
                 fp.write(milter.fp.read())
             else:
-                fp.write(milter.msg.as_bytes(policy=SMTP))
+                fp.write(milter.msg.as_bytes())
     except IOError as e:
         raise RuntimeError(f"unable to store message: {e}")