Change version to 1.0.5

pyquarantine/__init__.py

@@ -12,7 +12,26 @@
 # along with PyQuarantineMilter.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+import Milter
+import configparser
+import logging
+import os
+import re
+
+from Milter.utils import parse_addr
+from collections import defaultdict
+from email.charset import Charset
+from email.header import Header, decode_header
+from io import BytesIO
+from itertools import groupby
+from netaddr import IPAddress, IPNetwork, AddrFormatError
+from pyquarantine import mailer
+from pyquarantine import notifications
+from pyquarantine import storages
+from pyquarantine import whitelists
+
 __all__ = [
+    "make_header",
     "Quarantine",
     "QuarantineMilter",
     "setup_milter",
@@ -25,32 +44,34 @@ __all__ = [
     "version",
     "whitelists"]
 
-name = "pyquarantine"
 
-import Milter
-import configparser
-import logging
-import os
-import re
-import sys
+def make_header(decoded_seq, maxlinelen=None, header_name=None,
+                continuation_ws=' ', errors='strict'):
+    """Create a Header from a sequence of pairs as returned by decode_header()
 
-from Milter.utils import parse_addr
-from collections import defaultdict
-from email.policy import default as default_policy
-from email.parser import BytesHeaderParser
-from io import BytesIO
-from itertools import groupby
-from netaddr import IPAddress, IPNetwork
-from pyquarantine import mailer
-from pyquarantine import notifications
-from pyquarantine import storages
-from pyquarantine import whitelists
+    decode_header() takes a header value string and returns a sequence of
+    pairs of the format (decoded_string, charset) where charset is the string
+    name of the character set.
+
+    This function takes one of those sequence of pairs and returns a Header
+    instance.  Optional maxlinelen, header_name, and continuation_ws are as in
+    the Header constructor.
+    """
+    h = Header(maxlinelen=maxlinelen, header_name=header_name,
+               continuation_ws=continuation_ws)
+    for s, charset in decoded_seq:
+        # None means us-ascii but we can simply pass it on to h.append()
+        if charset is not None and not isinstance(charset, Charset):
+            charset = Charset(charset)
+        h.append(s, charset, errors=errors)
+    return h
 
 
 class Quarantine(object):
     """Quarantine class suitable for QuarantineMilter
 
-    The class holds all the objects and functions needed for QuarantineMilter quarantine.
+    The class holds all the objects and functions needed
+    for QuarantineMilter quarantine.
 
     """
 
@@ -66,8 +87,9 @@ class Quarantine(object):
         self.logger = logging.getLogger(__name__)
         self.name = name
         self.index = index
+        self.regex = regex
         if regex:
-            self.regex = re.compile(
+            self.re = re.compile(
                 regex, re.MULTILINE + re.DOTALL + re.IGNORECASE)
         self.storage = storage
         self.whitelist = whitelist
@@ -98,12 +120,13 @@ class Quarantine(object):
                 cfg[opt] = defaults[opt]
             else:
                 raise RuntimeError(
-                    f"mandatory option '{opt}' not present in config section '{self.name}' or 'global'")
+                    f"mandatory option '{opt}' not present in "
+                    f"config section '{self.name}' or 'global'")
 
         # pre-compile regex
         self.logger.debug(
             f"{self.name}: compiling regex '{cfg['regex']}'")
-        self.regex = re.compile(
+        self.re = re.compile(
             cfg["regex"], re.MULTILINE + re.DOTALL + re.IGNORECASE)
 
         self.smtp_host = cfg["smtp_host"]
@@ -113,7 +136,8 @@ class Quarantine(object):
         storage_type = cfg["storage_type"].lower()
         if storage_type in storages.TYPES:
             self.logger.debug(
-                f"{self.name}: initializing storage type '{storage_type.upper()}'")
+                f"{self.name}: initializing storage "
+                f"type '{storage_type.upper()}'")
             self.storage = storages.TYPES[storage_type](
                 self.name, global_cfg, cfg, test)
         elif storage_type == "none":
@@ -127,11 +151,12 @@ class Quarantine(object):
         whitelist_type = cfg["whitelist_type"].lower()
         if whitelist_type in whitelists.TYPES:
             self.logger.debug(
-                f"{self.name}: initializing whitelist type '{whitelist_type.upper()}'")
+                f"{self.name}: initializing whitelist "
+                f"type '{whitelist_type.upper()}'")
             self.whitelist = whitelists.TYPES[whitelist_type](
                self.name, global_cfg, cfg, test)
         elif whitelist_type == "none":
-            logger.debug(f"{self.name}: whitelist is NONE")
+            self.logger.debug(f"{self.name}: whitelist is NONE")
             self.whitelist = None
         else:
             raise RuntimeError(
@@ -141,7 +166,8 @@ class Quarantine(object):
         notification_type = cfg["notification_type"].lower()
         if notification_type in notifications.TYPES:
             self.logger.debug(
-                f"{self.name}: initializing notification type '{notification_type.upper()}'")
+                f"{self.name}: initializing notification "
+                f"type '{notification_type.upper()}'")
             self.notification = notifications.TYPES[notification_type](
                 self.name, global_cfg, cfg, test)
         elif notification_type == "none":
@@ -149,7 +175,8 @@ class Quarantine(object):
             self.notification = None
         else:
             raise RuntimeError(
-                f"{self.name}: unknown notification type '{notification_type}'")
+                f"{self.name}: unknown notification "
+                f"type '{notification_type}'")
 
         # determining milter action for this quarantine
         action = cfg["action"].upper()
@@ -174,7 +201,8 @@ class Quarantine(object):
             try:
                 net = IPNetwork(host)
             except AddrFormatError as e:
-                raise RuntimeError(f"{self.name}: error parsing host_whitelist: {e}")
+                raise RuntimeError(
+                    f"{self.name}: error parsing host_whitelist: {e}")
             else:
                 self.host_whitelist.append(net)
         if self.host_whitelist:
@@ -190,7 +218,8 @@ class Quarantine(object):
 
         if not self.notification:
             raise RuntimeError(
-                "notification type is set to None, unable to send notification")
+                "notification type is set to None, "
+                "unable to send notification")
 
         fp, metadata = self.storage.get_mail(storage_id)
 
@@ -255,21 +284,22 @@ class Quarantine(object):
         ip = IPAddress(hostaddr[0])
         for entry in self.host_whitelist:
             if ip in entry:
-                return true
+                return True
         return False
 
     def match(self, header):
-        return self.regex.search(header)
+        return self.re.search(header)
 
 
 class QuarantineMilter(Milter.Base):
     """QuarantineMilter based on Milter.Base to implement milter communication
 
-    The class variable quarantines needs to be filled by runng the setup_milter function.
+    The class variable quarantines needs to be filled by
+    runng the setup_milter function.
 
     """
     quarantines = []
-    preferred_quarantine_action = "first"
+    preferred_action = "first"
 
     # list of default config files
     _cfg_files = [
@@ -284,8 +314,8 @@ class QuarantineMilter(Milter.Base):
 
     def _get_preferred_quarantine(self):
         matching_quarantines = [
-            q for q in self.recipients_quarantines.values() if q]
-        if self.preferred_quarantine_action == "first":
+            q for q in self.rcpts_quarantines.values() if q]
+        if self.preferred_action == "first":
             quarantine = sorted(
                 matching_quarantines,
                 key=lambda q: q.index)[0]
@@ -307,16 +337,18 @@ class QuarantineMilter(Milter.Base):
     def connect(self, hostname, family, hostaddr):
         self.hostaddr = hostaddr
         self.logger.debug(
-            f"accepted milter connection from {hostaddr[0]} port {hostaddr[1]}")
+            f"accepted milter connection from {hostaddr[0]} "
+            f"port {hostaddr[1]}")
         for quarantine in self.quarantines.copy():
             if quarantine.host_in_whitelist(hostaddr):
                 self.logger.debug(
-                    f"host {hostaddr[0]} is in whitelist of quarantine {quarantine['name']}")
+                    f"host {hostaddr[0]} is in whitelist of "
+                    f"quarantine {quarantine['name']}")
                 self.quarantines.remove(quarantine)
                 if not self.quarantines:
                     self.logger.debug(
-                        f"host {hostaddr[0]} is in whitelist of all quarantines, "
-                        f"skip further processing")
+                        f"host {hostaddr[0]} is in whitelist of all "
+                        f"quarantines, skip further processing")
                     return Milter.ACCEPT
         return Milter.CONTINUE
 
@@ -341,114 +373,132 @@ class QuarantineMilter(Milter.Base):
             f"{self.qid}: initializing memory buffer to save email data")
         # initialize memory buffer to save email data
         self.fp = BytesIO()
+        self.headers = []
         return Milter.CONTINUE
 
-    @Milter.noreply
     def header(self, name, value):
         try:
+            # remove surrogates from value
+            value = value.encode(
+                errors="surrogateescape").decode(errors="replace")
+            self.logger.debug(f"{self.qid}: received header: {name}: {value}")
             # write email header to memory buffer
             self.fp.write(f"{name}: {value}\r\n".encode(
                 encoding="ascii", errors="replace"))
+            header = make_header(decode_header(value), errors="replace")
+            value = str(header).replace("\x00", "")
+            self.logger.debug(
+                f"{self.qid}: decoded header: {name}: {value}")
+            self.headers.append((name, value))
+            return Milter.CONTINUE
         except Exception as e:
             self.logger.exception(
                 f"an exception occured in header function: {e}")
             return Milter.TEMPFAIL
 
-        return Milter.CONTINUE
-
     def eoh(self):
         try:
             self.fp.write("\r\n".encode(encoding="ascii"))
-            self.fp.seek(0)
-            self.headers = BytesHeaderParser(
-                policy=default_policy).parse(self.fp).items()
-            self.whitelist_cache = whitelists.WhitelistCache()
+            self.wl_cache = whitelists.WhitelistCache()
 
             # initialize dicts to set quaranines per recipient and keep matches
-            self.recipients_quarantines = {}
-            self.quarantines_matches = {}
+            self.rcpts_quarantines = {}
+            self.matches = {}
 
             # iterate email headers
-            recipients_to_check = self.recipients.copy()
+            rcpts_to_check = self.recipients.copy()
             for name, value in self.headers:
                 header = f"{name}: {value}"
                 self.logger.debug(
-                    f"{self.qid}: checking header against configured quarantines: {header}")
+                    f"{self.qid}: checking header against configured "
+                    f"quarantines: {header}")
                 # iterate quarantines
                 for quarantine in self.quarantines:
-                    if len(self.recipients_quarantines) == len(
+                    if len(self.rcpts_quarantines) == len(
                             self.recipients):
                         # every recipient matched a quarantine already
                         if quarantine.index >= max(
-                                [q.index for q in self.recipients_quarantines.values()]):
+                                [q.index for q in
+                                    self.rcpts_quarantines.values()]):
                             # all recipients matched a quarantine with at least
                             # the same precedence already, skip checks against
                             # quarantines with lower precedence
                             self.logger.debug(
-                                f"{self.qid}: {quarantine.name}: skip further checks of this header")
+                                f"{self.qid}: {quarantine.name}: skip further "
+                                f"checks of this header")
                             break
 
                     # check email header against quarantine regex
                     self.logger.debug(
-                        f"{self.qid}: {quarantine.name}: checking header against regex '{quarantine.regex}'")
+                        f"{self.qid}: {quarantine.name}: checking header "
+                        f"against regex '{str(quarantine.regex)}'")
                     match = quarantine.match(header)
                     if match:
                         self.logger.debug(
-                            f"{self.qid}: {quarantine.name}: header matched regex")
+                            f"{self.qid}: {quarantine.name}: "
+                            f"header matched regex")
                         # check for whitelisted recipients
                         whitelist = quarantine.get_whitelist()
                         if whitelist:
                             try:
-                                whitelisted_recipients = self.whitelist_cache.get_whitelisted_recipients(
-                                    whitelist, self.mailfrom, recipients_to_check)
+                                wl_recipients = self.wl_cache.get_recipients(
+                                        whitelist,
+                                        self.mailfrom,
+                                        rcpts_to_check)
                             except RuntimeError as e:
                                 self.logger.error(
-                                    f"{self.qid}: {quarantine.name}: unable to query whitelist: {e}")
+                                    f"{self.qid}: {quarantine.name}: unable "
+                                    f"to query whitelist: {e}")
                                 return Milter.TEMPFAIL
                         else:
-                            whitelisted_recipients = {}
+                            wl_recipients = {}
 
                         # iterate recipients
-                        for recipient in recipients_to_check.copy():
-                            if recipient in whitelisted_recipients:
+                        for rcpt in rcpts_to_check.copy():
+                            if rcpt in wl_recipients:
                                 # recipient is whitelisted in this quarantine
                                 self.logger.debug(
-                                    f"{self.qid}: {quarantine.name}: recipient '{recipient}' is whitelisted")
+                                    f"{self.qid}: {quarantine.name}: "
+                                    f"recipient '{rcpt}' is whitelisted")
                                 continue
 
-                            if recipient not in self.recipients_quarantines.keys() or \
-                                    self.recipients_quarantines[recipient].index > quarantine.index:
+                            if rcpt not in self.rcpts_quarantines or \
+                                    self.rcpts_quarantines[rcpt].index > \
+                                    quarantine.index:
                                 self.logger.debug(
-                                    f"{self.qid}: {quarantine.name}: set quarantine for recipient '{recipient}'")
+                                    f"{self.qid}: {quarantine.name}: set "
+                                    f"quarantine for recipient '{rcpt}'")
                                 # save match for later use as template
                                 # variables
-                                self.quarantines_matches[quarantine.name] = match
-                                self.recipients_quarantines[recipient] = quarantine
+                                self.matches[quarantine.name] = match
+                                self.rcpts_quarantines[rcpt] = quarantine
                                 if quarantine.index == 0:
                                     # we do not need to check recipients which
                                     # matched the quarantine with the highest
                                     # precedence already
-                                    recipients_to_check.remove(recipient)
+                                    rcpts_to_check.remove(rcpt)
                             else:
                                 self.logger.debug(
-                                    f"{self.qid}: {quarantine.name}: a quarantine with same or higher "
-                                    f"precedence matched already for recipient '{recipient}'")
+                                    f"{self.qid}: {quarantine.name}: a "
+                                    f"quarantine with same or higher "
+                                    f"precedence matched already for "
+                                    f"recipient '{rcpt}'")
 
-                if not recipients_to_check:
+                if not rcpts_to_check:
                     self.logger.debug(
-                        f"{self.qid}: all recipients matched the first quarantine, "
-                        f"skipping all remaining header checks")
+                        f"{self.qid}: all recipients matched the first "
+                        f"quarantine, skipping all remaining header checks")
                     break
 
             # check if no quarantine has matched for all recipients
-            if not self.recipients_quarantines:
+            if not self.rcpts_quarantines:
                 # accept email
                 self.logger.info(
                     f"{self.qid}: passed clean for all recipients")
                 return Milter.ACCEPT
 
             # check if the mail body is needed
-            for recipient, quarantine in self.recipients_quarantines.items():
+            for recipient, quarantine in self.rcpts_quarantines.items():
                 if quarantine.get_storage() or quarantine.get_notification():
                     # mail body is needed, continue processing
                     return Milter.CONTINUE
@@ -457,7 +507,7 @@ class QuarantineMilter(Milter.Base):
             # quarantines, just return configured action
             quarantine = self._get_preferred_quarantine()
             self.logger.info(
-                f"{self.qid}: {self.preferred_quarantine_action} "
+                f"{self.qid}: {self.preferred_action} "
                 f"matching quarantine is '{quarantine.name}', performing "
                 f"milter action {quarantine.action}")
             if quarantine.action == "REJECT":
@@ -484,9 +534,9 @@ class QuarantineMilter(Milter.Base):
             # processing recipients grouped by quarantines
             quarantines = []
             for quarantine, recipients in groupby(
-                    sorted(self.recipients_quarantines,
-                           key=lambda x: self.recipients_quarantines[x].index),
-                    lambda x: self.recipients_quarantines[x]):
+                    sorted(self.rcpts_quarantines,
+                           key=lambda x: self.rcpts_quarantines[x].index),
+                    lambda x: self.rcpts_quarantines[x]):
                 quarantines.append((quarantine, list(recipients)))
 
             # iterate quarantines sorted by index
@@ -495,9 +545,9 @@ class QuarantineMilter(Milter.Base):
                 headers = defaultdict(str)
                 for name, value in self.headers:
                     headers[name.lower()] = value
-                subgroups = self.quarantines_matches[quarantine.name].groups(
+                subgroups = self.matches[quarantine.name].groups(
                     default="")
-                named_subgroups = self.quarantines_matches[quarantine.name].groupdict(
+                named_subgroups = self.matches[quarantine.name].groupdict(
                     default="")
 
                 rcpts = ", ".join(recipients)
@@ -508,14 +558,16 @@ class QuarantineMilter(Milter.Base):
                 if storage:
                     # add email to quarantine
                     self.logger.info(
-                        f"{self.qid}: adding to quarantine '{quarantine.name}' for: {rcpts}")
+                        f"{self.qid}: adding to quarantine "
+                        f"'{quarantine.name}' for: {rcpts}")
                     try:
                         storage_id = storage.add(
-                            self.qid, self.mailfrom, recipients, headers, self.fp,
-                            subgroups, named_subgroups)
+                            self.qid, self.mailfrom, recipients, headers,
+                            self.fp, subgroups, named_subgroups)
                     except RuntimeError as e:
                         self.logger.error(
-                            f"{self.qid}: unable to add to quarantine '{quarantine.name}': {e}")
+                            f"{self.qid}: unable to add to quarantine "
+                            f"'{quarantine.name}': {e}")
                         return Milter.TEMPFAIL
 
                 # check if a notification is configured
@@ -551,7 +603,7 @@ class QuarantineMilter(Milter.Base):
             # return configured action
             quarantine = self._get_preferred_quarantine()
             self.logger.info(
-                f"{self.qid}: {self.preferred_quarantine_action} matching "
+                f"{self.qid}: {self.preferred_action} matching "
                 f"quarantine is '{quarantine.name}', performing milter "
                 f"action {quarantine.action}")
             if quarantine.action == "REJECT":
@@ -592,12 +644,13 @@ def setup_milter(test=False, cfg_files=[]):
     for option in ["quarantines", "preferred_quarantine_action"]:
         if not parser.has_option("global", option):
             raise RuntimeError(
-                f"mandatory option '{option}' not present in config section 'global'")
+                f"mandatory option '{option}' not present in config "
+                f"section 'global'")
 
     # read global config section
     global_cfg = dict(parser.items("global"))
-    preferred_quarantine_action = global_cfg["preferred_quarantine_action"].lower()
-    if preferred_quarantine_action not in ["first", "last"]:
+    preferred_action = global_cfg["preferred_quarantine_action"].lower()
+    if preferred_action not in ["first", "last"]:
         raise RuntimeError(
             "option preferred_quarantine_action has illegal value")
 
@@ -606,11 +659,13 @@ def setup_milter(test=False, cfg_files=[]):
         q.strip() for q in global_cfg["quarantines"].split(",")]
     if len(quarantines) != len(set(quarantines)):
         raise RuntimeError(
-            "at least one quarantine is specified multiple times in quarantines option")
+            "at least one quarantine is specified multiple times "
+            "in quarantines option")
     if "global" in quarantines:
         quarantines.remove("global")
         logger.warning(
-            "removed illegal quarantine name 'global' from list of active quarantines")
+            "removed illegal quarantine name 'global' from list of "
+            "active quarantines")
     if not quarantines:
         raise RuntimeError("no quarantines configured")
 
@@ -628,7 +683,7 @@ def setup_milter(test=False, cfg_files=[]):
         quarantine.setup_from_cfg(global_cfg, cfg, test)
         milter_quarantines.append(quarantine)
 
-    QuarantineMilter.preferred_quarantine_action = preferred_quarantine_action
+    QuarantineMilter.preferred_action = preferred_action
     QuarantineMilter.quarantines = milter_quarantines
 
 

pyquarantine/cli.py

@@ -20,11 +20,10 @@ import logging.handlers
 import sys
 import time
 
-from email.header import decode_header
-
 from pyquarantine import QuarantineMilter, setup_milter
 from pyquarantine.version import __version__ as version
 
+
 def _get_quarantine(quarantines, name):
     try:
         quarantine = next((q for q in quarantines if q.name == name))
@@ -32,6 +31,7 @@ def _get_quarantine(quarantines, name):
         raise RuntimeError(f"invalid quarantine 'name'")
     return quarantine
 
+
 def _get_storage(quarantines, name):
     quarantine = _get_quarantine(quarantines, name)
     storage = quarantine.get_storage()
@@ -40,6 +40,7 @@ def _get_storage(quarantines, name):
                 "storage type is set to NONE")
     return storage
 
+
 def _get_notification(quarantines, name):
     quarantine = _get_quarantine(quarantines, name)
     notification = quarantine.get_notification()
@@ -48,6 +49,7 @@ def _get_notification(quarantines, name):
                 "notification type is set to NONE")
     return notification
 
+
 def _get_whitelist(quarantines, name):
     quarantine = _get_quarantine(quarantines, name)
     whitelist = quarantine.get_whitelist()
@@ -56,6 +58,7 @@ def _get_whitelist(quarantines, name):
                 "whitelist type is set to NONE")
     return whitelist
 
+
 def print_table(columns, rows):
     if not rows:
         return

pyquarantine/mailer.py

@@ -14,7 +14,6 @@
 
 import logging
 import smtplib
-import sys
 
 from multiprocessing import Process, Queue
 
@@ -50,7 +49,8 @@ def mailprocess():
                 smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail)
             except Exception as e:
                 logger.error(
-                    f"{qid}: error while sending {emailtype} to '{recipient}': {e}")
+                    f"{qid}: error while sending {emailtype} "
+                    f"to '{recipient}': {e}")
             else:
                 logger.info(
                     f"{qid}: successfully sent {emailtype} to: {recipient}")
@@ -82,5 +82,5 @@ def sendmail(smtp_host, smtp_port, qid, mailfrom, recipients, mail,
                 (smtp_host, smtp_port, qid, mailfrom, recipient, mail,
                  emailtype),
                 timeout=30)
-        except Queue.Full as e:
+        except Queue.Full:
             raise RuntimeError("email queue is full")

pyquarantine/notifications.py

@@ -19,8 +19,7 @@ import re
 from bs4 import BeautifulSoup
 from cgi import escape
 from collections import defaultdict
-from email import policy
-from email.header import decode_header
+from email.policy import default as default_policy
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 from email.mime.image import MIMEImage
@@ -136,7 +135,8 @@ class EMailNotification(BaseNotification):
                 cfg[opt] = defaults[opt]
             else:
                 raise RuntimeError(
-                    f"mandatory option '{opt}' not present in config section '{self.name}' or 'global'")
+                    f"mandatory option '{opt}' not present in config "
+                    f"section '{self.name}' or 'global'")
 
         self.smtp_host = cfg["notification_email_smtp_host"]
         self.smtp_port = cfg["notification_email_smtp_port"]
@@ -176,11 +176,13 @@ class EMailNotification(BaseNotification):
         elif strip_images in ["FALSE", "OFF", "NO"]:
             self.strip_images = False
         else:
-            raise RuntimeError("error parsing notification_email_strip_images: unknown value")
+            raise RuntimeError(
+                "error parsing notification_email_strip_images: unknown value")
 
         self.parser_lib = cfg["notification_email_parser_lib"].strip()
         if self.parser_lib not in ["lxml", "html.parser"]:
-            raise RuntimeError("error parsing notification_email_parser_lib: unknown value")
+            raise RuntimeError(
+                "error parsing notification_email_parser_lib: unknown value")
 
         # read email replacement image if specified
         replacement_img = cfg["notification_email_replacement_img"].strip()
@@ -199,7 +201,8 @@ class EMailNotification(BaseNotification):
 
         # read images to embed if specified
         embedded_img_paths = [
-            p.strip() for p in cfg["notification_email_embedded_imgs"].split(",") if p]
+            p.strip() for p in cfg["notification_email_embedded_imgs"].split(
+                ",") if p]
         self.embedded_imgs = []
         for img_path in embedded_img_paths:
             # read image
@@ -219,7 +222,9 @@ class EMailNotification(BaseNotification):
         try:
             body = msg.get_body(preferencelist=("html", "plain"))
         except Exception as e:
-            self.logger.error(f"{qid}: an error occured in email.message.EmailMessage.get_body: {e}")
+            self.logger.error(
+                f"{qid}: an error occured in "
+                f"email.message.EmailMessage.get_body: {e}")
             body = None
 
         if body:
@@ -228,13 +233,16 @@ class EMailNotification(BaseNotification):
             try:
                 content = content.decode(encoding=charset, errors="replace")
             except LookupError:
-                self.logger.info(f"{qid}: unknown encoding '{charset}', falling back to UTF-8")
+                self.logger.info(
+                    f"{qid}: unknown encoding '{charset}', "
+                    f"falling back to UTF-8")
                 content = content.decode("utf-8", errors="replace")
             content_type = body.get_content_type()
             if content_type == "text/plain":
                 # convert text/plain to text/html
                 self.logger.debug(
-                    f"{qid}: content type is {content_type}, converting to text/html")
+                    f"{qid}: content type is {content_type}, "
+                    f"converting to text/html")
                 content = re.sub(r"^(.*)$", r"\1<br/>",
                                  escape(content), flags=re.MULTILINE)
             else:
@@ -248,7 +256,8 @@ class EMailNotification(BaseNotification):
         # create BeautifulSoup object
         length = len(content)
         self.logger.debug(
-            f"{qid}: trying to create BeatufilSoup object with parser lib {self.parser_lib}, "
+            f"{qid}: trying to create BeatufilSoup object with "
+            f"parser lib {self.parser_lib}, "
             f"text length is {length} bytes")
         soup = BeautifulSoup(content, self.parser_lib)
         self.logger.debug(
@@ -263,7 +272,8 @@ class EMailNotification(BaseNotification):
         # completly remove bad elements
         for element in soup(EMailNotification._bad_tags):
             self.logger.debug(
-                f"{qid}: removing dangerous tag '{element.name}' and its content")
+                f"{qid}: removing dangerous tag '{element.name}' "
+                f"and its content")
             element.extract()
 
         # remove not whitelisted elements, but keep their content
@@ -279,11 +289,13 @@ class EMailNotification(BaseNotification):
                 if attribute not in EMailNotification._good_attributes:
                     if element.name == "a" and attribute == "href":
                         self.logger.debug(
-                            f"{qid}: setting attribute href to '#' on tag '{element.name}'")
+                            f"{qid}: setting attribute href to '#' "
+                            f"on tag '{element.name}'")
                         element["href"] = "#"
                     else:
                         self.logger.debug(
-                            f"{qid}: removing attribute '{attribute}' from tag '{element.name}'")
+                            f"{qid}: removing attribute '{attribute}' "
+                            f"from tag '{element.name}'")
                         del(element.attrs[attribute])
         return soup
 
@@ -305,7 +317,7 @@ class EMailNotification(BaseNotification):
 
         # extract body from email
         soup = self.get_email_body_soup(
-            qid, email.message_from_binary_file(fp, policy=policy.default))
+            qid, email.message_from_binary_file(fp, policy=default_policy))
 
         # replace picture sources
         image_replaced = False
@@ -338,7 +350,8 @@ class EMailNotification(BaseNotification):
             self.logger.debug(f"{qid}: parsing email template")
 
             # generate dict containing all template variables
-            variables = defaultdict(str,
+            variables = defaultdict(
+                    str,
                     EMAIL_HTML_TEXT=sanitized_text,
                     EMAIL_FROM=escape(headers["from"]),
                     EMAIL_ENVELOPE_FROM=escape(mailfrom),

pyquarantine/run.py

@@ -24,6 +24,7 @@ import pyquarantine
 
 from pyquarantine.version import __version__ as version
 
+
 def main():
     "Run PyQuarantine-Milter."
     # parse command line

pyquarantine/storages.py

@@ -73,14 +73,16 @@ class FileMailStorage(BaseMailStorage):
                 cfg[opt] = defaults[opt]
             else:
                 raise RuntimeError(
-                    f"mandatory option '{opt}' not present in config section '{self.name}' or 'global'")
+                    f"mandatory option '{opt}' not present in config "
+                    f"section '{self.name}' or 'global'")
         self.directory = cfg["storage_directory"]
 
         # check if quarantine directory exists and is writable
         if not os.path.isdir(self.directory) or not os.access(
                 self.directory, os.W_OK):
             raise RuntimeError(
-                f"file quarantine directory '{self.directory}' does not exist or is not writable")
+                f"file quarantine directory '{self.directory}' does "
+                f"not exist or is not writable")
         self._metadata_suffix = ".metadata"
 
     def _save_datafile(self, storage_id, fp):
@@ -203,7 +205,8 @@ class FileMailStorage(BaseMailStorage):
                 if len(recipients) == 1 and \
                         recipients[0] not in metadata["recipients"]:
                     continue
-                elif len(set(recipients + metadata["recipients"])) == len(recipients + metadata["recipients"]):
+                elif len(set(recipients + metadata["recipients"])) == \
+                        len(recipients + metadata["recipients"]):
                     continue
 
             emails[storage_id] = metadata

pyquarantine/version.py

@@ -1 +1 @@
-__version__ = "1.0.2"
+__version__ = "1.0.5"

pyquarantine/whitelists.py

@@ -15,7 +15,6 @@
 import logging
 import peewee
 import re
-import sys
 
 from datetime import datetime
 from playhouse.db_url import connect
@@ -88,7 +87,8 @@ class DatabaseWhitelist(WhitelistBase):
         defaults = {}
 
         # check config
-        for opt in ["whitelist_db_connection", "whitelist_db_table"] + list(defaults.keys()):
+        for opt in ["whitelist_db_connection",
+                    "whitelist_db_table"] + list(defaults.keys()):
             if opt in cfg:
                 continue
             if opt in global_cfg:
@@ -97,7 +97,8 @@ class DatabaseWhitelist(WhitelistBase):
                 cfg[opt] = defaults[opt]
             else:
                 raise RuntimeError(
-                    f"mandatory option '{opt}' not present in config section '{self.name}' or 'global'")
+                    f"mandatory option '{opt}' not present in config "
+                    f"section '{self.name}' or 'global'")
 
         tablename = cfg["whitelist_db_table"]
         connection_string = cfg["whitelist_db_connection"]
@@ -168,7 +169,8 @@ class DatabaseWhitelist(WhitelistBase):
 
         # generate list of possible mailfroms
         self.logger.debug(
-            f"query database for whitelist entries from <{mailfrom}> to <{recipient}>")
+            f"query database for whitelist entries from <{mailfrom}> "
+            f"to <{recipient}>")
         mailfroms = [""]
         if "@" in mailfrom and not mailfrom.startswith("@"):
             domain = mailfrom.split("@")[1]
@@ -221,7 +223,8 @@ class DatabaseWhitelist(WhitelistBase):
         try:
             for entry in list(self.model.select()):
                 if older_than is not None:
-                    if (datetime.now() - entry.last_used).total_seconds() < (older_than * 86400):
+                    delta = (datetime.now() - entry.last_used).total_seconds()
+                    if delta < (older_than * 86400):
                         continue
 
                 if mailfrom is not None:
@@ -290,10 +293,11 @@ class WhitelistCache(object):
                 mailfrom, recipient)
         return self.cache[whitelist][recipient]
 
-    def get_whitelisted_recipients(self, whitelist, mailfrom, recipients):
+    def get_recipients(self, whitelist, mailfrom, recipients):
         self.load(whitelist, mailfrom, recipients)
-        return list(
-            filter(lambda x: self.cache[whitelist][x], self.cache[whitelist].keys()))
+        return list(filter(
+            lambda x: self.cache[whitelist][x],
+            self.cache[whitelist].keys()))
 
 
 # list of whitelist types and their related whitelist classes