switch from cgi.escape to html.escape

MANIFEST.in

@@ -1,3 +1,3 @@
 include LICENSE README.md
-recursive-include docs *
-recursive-include misc *
+recursive-include pyquarantine/docs *
+recursive-include pyquarantine/misc *

pyquarantine/__init__.py

@@ -17,11 +17,12 @@ import configparser
 import logging
 import os
 import re
+import encodings
 
 from Milter.utils import parse_addr
 from collections import defaultdict
-from email.policy import default as default_policy
-from email.parser import BytesHeaderParser
+from email.charset import Charset
+from email.header import Header, decode_header
 from io import BytesIO
 from itertools import groupby
 from netaddr import IPAddress, IPNetwork, AddrFormatError
@@ -31,6 +32,7 @@ from pyquarantine import storages
 from pyquarantine import whitelists
 
 __all__ = [
+    "make_header",
     "Quarantine",
     "QuarantineMilter",
     "setup_milter",
@@ -40,9 +42,47 @@ __all__ = [
     "notifications",
     "storages",
     "run",
-    "version",
     "whitelists"]
 
+__version__ = "1.0.8"
+
+
+################################################
+#  add charset alias for windows-874 encoding  #
+################################################
+
+aliases = encodings.aliases.aliases
+
+for alias in ["windows-874", "windows_874"]:
+    if alias not in aliases:
+        aliases[alias] = "cp874"
+
+setattr(encodings.aliases, "aliases", aliases)
+
+################################################
+
+
+def make_header(decoded_seq, maxlinelen=None, header_name=None,
+                continuation_ws=' ', errors='strict'):
+    """Create a Header from a sequence of pairs as returned by decode_header()
+
+    decode_header() takes a header value string and returns a sequence of
+    pairs of the format (decoded_string, charset) where charset is the string
+    name of the character set.
+
+    This function takes one of those sequence of pairs and returns a Header
+    instance.  Optional maxlinelen, header_name, and continuation_ws are as in
+    the Header constructor.
+    """
+    h = Header(maxlinelen=maxlinelen, header_name=header_name,
+               continuation_ws=continuation_ws)
+    for s, charset in decoded_seq:
+        # None means us-ascii but we can simply pass it on to h.append()
+        if charset is not None and not isinstance(charset, Charset):
+            charset = Charset(charset)
+        h.append(s, charset, errors=errors)
+    return h
+
 
 class Quarantine(object):
     """Quarantine class suitable for QuarantineMilter
@@ -64,8 +104,9 @@ class Quarantine(object):
         self.logger = logging.getLogger(__name__)
         self.name = name
         self.index = index
+        self.regex = regex
         if regex:
-            self.regex = re.compile(
+            self.re = re.compile(
                 regex, re.MULTILINE + re.DOTALL + re.IGNORECASE)
         self.storage = storage
         self.whitelist = whitelist
@@ -102,7 +143,7 @@ class Quarantine(object):
         # pre-compile regex
         self.logger.debug(
             f"{self.name}: compiling regex '{cfg['regex']}'")
-        self.regex = re.compile(
+        self.re = re.compile(
             cfg["regex"], re.MULTILINE + re.DOTALL + re.IGNORECASE)
 
         self.smtp_host = cfg["smtp_host"]
@@ -264,7 +305,7 @@ class Quarantine(object):
         return False
 
     def match(self, header):
-        return self.regex.search(header)
+        return self.re.search(header)
 
 
 class QuarantineMilter(Milter.Base):
@@ -319,7 +360,7 @@ class QuarantineMilter(Milter.Base):
             if quarantine.host_in_whitelist(hostaddr):
                 self.logger.debug(
                     f"host {hostaddr[0]} is in whitelist of "
-                    f"quarantine {quarantine['name']}")
+                    f"quarantine {quarantine.name}")
                 self.quarantines.remove(quarantine)
                 if not self.quarantines:
                     self.logger.debug(
@@ -349,27 +390,32 @@ class QuarantineMilter(Milter.Base):
             f"{self.qid}: initializing memory buffer to save email data")
         # initialize memory buffer to save email data
         self.fp = BytesIO()
+        self.headers = []
         return Milter.CONTINUE
 
-    @Milter.noreply
     def header(self, name, value):
         try:
+            # remove surrogates from value
+            value = value.encode(
+                errors="surrogateescape").decode(errors="replace")
+            self.logger.debug(f"{self.qid}: received header: {name}: {value}")
             # write email header to memory buffer
             self.fp.write(f"{name}: {value}\r\n".encode(
                 encoding="ascii", errors="replace"))
+            header = make_header(decode_header(value), errors="replace")
+            value = str(header).replace("\x00", "")
+            self.logger.debug(
+                f"{self.qid}: decoded header: {name}: {value}")
+            self.headers.append((name, value))
+            return Milter.CONTINUE
         except Exception as e:
             self.logger.exception(
                 f"an exception occured in header function: {e}")
             return Milter.TEMPFAIL
 
-        return Milter.CONTINUE
-
     def eoh(self):
         try:
             self.fp.write("\r\n".encode(encoding="ascii"))
-            self.fp.seek(0)
-            self.headers = BytesHeaderParser(
-                policy=default_policy).parse(self.fp).items()
             self.wl_cache = whitelists.WhitelistCache()
 
             # initialize dicts to set quaranines per recipient and keep matches
@@ -402,7 +448,7 @@ class QuarantineMilter(Milter.Base):
                     # check email header against quarantine regex
                     self.logger.debug(
                         f"{self.qid}: {quarantine.name}: checking header "
-                        f"against regex '{quarantine.regex}'")
+                        f"against regex '{str(quarantine.regex)}'")
                     match = quarantine.match(header)
                     if match:
                         self.logger.debug(

pyquarantine/cli.py

@@ -21,14 +21,14 @@ import sys
 import time
 
 from pyquarantine import QuarantineMilter, setup_milter
-from pyquarantine.version import __version__ as version
+from pyquarantine import __version__ as version
 
 
 def _get_quarantine(quarantines, name):
     try:
         quarantine = next((q for q in quarantines if q.name == name))
     except StopIteration:
-        raise RuntimeError(f"invalid quarantine 'name'")
+        raise RuntimeError("invalid quarantine 'name'")
     return quarantine
 
 

pyquarantine/notifications.py

@@ -17,12 +17,12 @@ import logging
 import re
 
 from bs4 import BeautifulSoup
-from cgi import escape
 from collections import defaultdict
-from email import policy
+from email.policy import default as default_policy
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 from email.mime.image import MIMEImage
+from html import escape
 from os.path import basename
 from urllib.parse import quote
 
@@ -212,7 +212,7 @@ class EMailNotification(BaseNotification):
                 raise RuntimeError(f"error reading image: {e}")
             else:
                 filename = basename(img_path)
-                img.add_header(f"Content-ID", f"<{filename}>")
+                img.add_header("Content-ID", f"<{filename}>")
                 self.embedded_imgs.append(img)
 
     def get_email_body_soup(self, qid, msg):
@@ -244,7 +244,8 @@ class EMailNotification(BaseNotification):
                     f"{qid}: content type is {content_type}, "
                     f"converting to text/html")
                 content = re.sub(r"^(.*)$", r"\1<br/>",
-                                 escape(content), flags=re.MULTILINE)
+                                 escape(content, quote=False),
+                                 flags=re.MULTILINE)
             else:
                 self.logger.debug(
                     f"{qid}: content type is {content_type}")
@@ -317,7 +318,7 @@ class EMailNotification(BaseNotification):
 
         # extract body from email
         soup = self.get_email_body_soup(
-            qid, email.message_from_binary_file(fp, policy=policy.default))
+            qid, email.message_from_binary_file(fp, policy=default_policy))
 
         # replace picture sources
         image_replaced = False
@@ -353,22 +354,24 @@ class EMailNotification(BaseNotification):
             variables = defaultdict(
                     str,
                     EMAIL_HTML_TEXT=sanitized_text,
-                    EMAIL_FROM=escape(headers["from"]),
-                    EMAIL_ENVELOPE_FROM=escape(mailfrom),
-                    EMAIL_ENVELOPE_FROM_URL=escape(quote(mailfrom)),
-                    EMAIL_TO=escape(headers["to"]),
-                    EMAIL_ENVELOPE_TO=escape(recipient),
+                    EMAIL_FROM=escape(headers["from"], quote=False),
+                    EMAIL_ENVELOPE_FROM=escape(mailfrom, quote=False),
+                    EMAIL_ENVELOPE_FROM_URL=escape(quote(mailfrom),
+                                                   quote=False),
+                    EMAIL_TO=escape(headers["to"], quote=False),
+                    EMAIL_ENVELOPE_TO=escape(recipient, quote=False),
                     EMAIL_ENVELOPE_TO_URL=escape(quote(recipient)),
-                    EMAIL_SUBJECT=escape(headers["subject"]),
+                    EMAIL_SUBJECT=escape(headers["subject"], quote=False),
                     EMAIL_QUARANTINE_ID=storage_id)
 
             if subgroups:
                 number = 0
                 for subgroup in subgroups:
-                    variables[f"SUBGROUP_{number}"] = escape(subgroup)
+                    variables[f"SUBGROUP_{number}"] = escape(subgroup,
+                                                             quote=False)
             if named_subgroups:
                 for key, value in named_subgroups.items():
-                    named_subgroups[key] = escape(value)
+                    named_subgroups[key] = escape(value, quote=False)
                 variables.update(named_subgroups)
 
             # parse template

pyquarantine/run.py

@@ -22,7 +22,7 @@ import sys
 
 import pyquarantine
 
-from pyquarantine.version import __version__ as version
+from pyquarantine import __version__ as version
 
 
 def main():

pyquarantine/version.py

@@ -1 +0,0 @@
-__version__ = "1.0.3"

setup.cfg

@@ -0,0 +1,2 @@
+[metadata]
+version = attr: pyquarantine.__version__

setup.py

@@ -4,11 +4,8 @@ def read_file(fname):
     with open(fname, 'r') as f:
         return f.read()
 
-version = {}
-exec(read_file("pyquarantine/version.py"), version)
 
 setup(name = "pyquarantine",
-    version = version["__version__"],
     author = "Thomas Oettli",
     author_email = "spacefreak@noop.ch",
     description = "A pymilter based sendmail/postfix pre-queue filter.",