Change version to 1.0.2

README.md

@@ -8,7 +8,7 @@ Each quarantine can be configured with a quarantine type, notification type, whi
 
 Addionally, pyquarantine-milter provides a sanitized, harmless version of the text parts of e-mails, which can be embedded in e-mail notifications. This makes it easier for users to decide, if a match is a false-positive or not. If a matching quarantine provides a quarantine ID of the original e-mail, it is also available as a template variable. This is useful if you want to add links to a webservice to notification e-mails, to give your users the ability to release e-mails or whitelist the from-address for example. The webservice then releases the e-mail from the quarantine.
 
-The project is currently in alpha status, but will soon be used in a productive enterprise environment and possibly existing bugs will be fixed.
+The project is currently in beta status, but it is already used in a productive enterprise environment which processes about a million e-mails per month.
 
 ## Requirements
 * pymilter <https://pythonhosted.org/pymilter/>
@@ -36,8 +36,8 @@ The following configuration options are mandatory in the global section:
 The following configuration options are mandatory in each quarantine section:
 * **regex**  
   Case insensitive regular expression to filter e-mail headers.
-* **quarantine_type**  
+* **storage_type**  
-  One of the quarantine-types described below.
+  One of the storage types described below.
 * **action**  
   One of the actions described below.
 * **notification_type**  
@@ -50,13 +50,13 @@ The following configuration options are mandatory in each quarantine section:
   SMTP port
 
 The following configuration options are optional in each quarantine section:
-* **ignore_hosts**  
+* **host_whitelist**  
   Comma-separated list of host and network addresses to be ignored by this quarantine.
 * **reject_reason**  
   Reason to return to the client if action is set to reject.
 
 
-### Quarantine types
+### Storage types
 * **NONE**  
   Original e-mails scrapped, sent to nirvana, black-holed or however you want to call it.
 
@@ -64,7 +64,7 @@ The following configuration options are optional in each quarantine section:
   Original e-mails are stored on the filesystem with a unique filename. The filename is available as a
   template variable used in notifiaction templates.  
   The following configuration options are mandatory for this quarantine type:
-  * **quarantine_directory**  
+  * **storage_directory**  
     The directory in which quarantined e-mails are stored.
 
 

docs/pyquarantine.conf.example

@@ -30,12 +30,12 @@ preferred_quarantine_action = last
 
 
 [spam]
-# Option: ignore_hosts
+# Option: host_whitelist
 # Notes:  Set a list of host and network addresses to be ignored by this quarantine.
 #         All the common host/network notations are supported, including IPv6.
 # Value:  [ HOST ]
 #
-ignore_hosts = 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
+host_whitelist = 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
 
 # Option: regex
 # Notes:  Set the case insensitive regular expression to match against email headers.
@@ -57,18 +57,18 @@ smtp_host = 127.0.0.1
 #
 smtp_port = 25
 
-# Option: quarantine_type
+# Option: storage_type
-# Notes:  Set the quarantine type.
+# Notes:  Set the storage type.
 # Values: [ file | none ]
 #
-quarantine_type = file
+storage_type = file
 
-# Option: quarantine_directory
+# Option: storage_directory
 # Notes:  Set the directory to store quarantined emails.
 #         This option is needed by quarantine type 'file'.
 # Values: [ DIRECTORY ]
 #
-quarantine_directory = /var/lib/pyquarantine/spam
+storage_directory = /var/lib/pyquarantine/spam
 
 # Option: action
 # Notes:  Set the milter action to perform if email is processed by this quarantine.

pyquarantine/__init__.py

@@ -13,13 +13,14 @@
 #
 
 __all__ = [
+    "Quarantine",
     "QuarantineMilter",
-    "generate_milter_config",
+    "setup_milter",
     "reload_config",
     "cli",
     "mailer",
     "notifications",
-    "quarantines",
+    "storages",
     "run",
     "version",
     "whitelists"]
@@ -35,85 +36,287 @@ import sys
 
 from Milter.utils import parse_addr
 from collections import defaultdict
+from email.policy import default as default_policy
+from email.parser import BytesHeaderParser
 from io import BytesIO
 from itertools import groupby
 from netaddr import IPAddress, IPNetwork
-from pyquarantine import quarantines
+from pyquarantine import mailer
 from pyquarantine import notifications
+from pyquarantine import storages
 from pyquarantine import whitelists
 
 
-class QuarantineMilter(Milter.Base):
+class Quarantine(object):
-    """QuarantineMilter based on Milter.Base to implement milter communication
+    """Quarantine class suitable for QuarantineMilter
 
-    The class variable config needs to be filled with the result of the generate_milter_config function.
+    The class holds all the objects and functions needed for QuarantineMilter quarantine.
 
     """
-    config = None
-    global_config = None
 
-    # list of default config files
-    _config_files = [
-        "/etc/pyquarantine/pyquarantine.conf",
-        os.path.expanduser('~/pyquarantine.conf'),
-        "pyquarantine.conf"]
     # list of possible actions
     _actions = {
         "ACCEPT": Milter.ACCEPT,
         "REJECT": Milter.REJECT,
         "DISCARD": Milter.DISCARD}
 
+    def __init__(self, name, index=0, regex=None, storage=None, whitelist=None,
+            host_whitelist=[], notification=None, action="ACCEPT",
+            reject_reason=None):
+        self.logger = logging.getLogger(__name__)
+        self.name = name
+        self.index = index
+        if regex:
+            self.regex = re.compile(
+                regex, re.MULTILINE + re.DOTALL + re.IGNORECASE)
+        self.storage = storage
+        self.whitelist = whitelist
+        self.host_whitelist = host_whitelist
+        self.notification = notification
+        action = action.upper()
+        assert action in self._actions
+        self.action = action
+        self.milter_action = self._actions[action]
+        self.reject_reason = reject_reason
+
+    def setup_from_cfg(self, global_cfg, cfg, test=False):
+        defaults = {
+            "action": "accept",
+            "reject_reason": "Message rejected",
+            "storage_type": "none",
+            "notification_type": "none",
+            "whitelist_type": "none",
+            "host_whitelist": ""
+        }
+        # check config
+        for opt in ["regex", "smtp_host", "smtp_port"] + list(defaults.keys()):
+            if opt in cfg:
+                continue
+            if opt in global_cfg:
+                cfg[opt] = global_cfg[opt]
+            elif opt in defaults:
+                cfg[opt] = defaults[opt]
+            else:
+                raise RuntimeError(
+                    f"mandatory option '{opt}' not present in config section '{self.name}' or 'global'")
+
+        # pre-compile regex
+        self.logger.debug(
+            f"{self.name}: compiling regex '{cfg['regex']}'")
+        self.regex = re.compile(
+            cfg["regex"], re.MULTILINE + re.DOTALL + re.IGNORECASE)
+
+        self.smtp_host = cfg["smtp_host"]
+        self.smtp_port = cfg["smtp_port"]
+
+        # create storage instance
+        storage_type = cfg["storage_type"].lower()
+        if storage_type in storages.TYPES:
+            self.logger.debug(
+                f"{self.name}: initializing storage type '{storage_type.upper()}'")
+            self.storage = storages.TYPES[storage_type](
+                self.name, global_cfg, cfg, test)
+        elif storage_type == "none":
+            self.logger.debug(f"{self.name}: storage is NONE")
+            self.storage = None
+        else:
+            raise RuntimeError(
+                f"{self.name}: unknown storage type '{storage_type}'")
+
+        # create whitelist instance
+        whitelist_type = cfg["whitelist_type"].lower()
+        if whitelist_type in whitelists.TYPES:
+            self.logger.debug(
+                f"{self.name}: initializing whitelist type '{whitelist_type.upper()}'")
+            self.whitelist = whitelists.TYPES[whitelist_type](
+               self.name, global_cfg, cfg, test)
+        elif whitelist_type == "none":
+            logger.debug(f"{self.name}: whitelist is NONE")
+            self.whitelist = None
+        else:
+            raise RuntimeError(
+                f"{self.name}: unknown whitelist type '{whitelist_type}'")
+
+        # create notification instance
+        notification_type = cfg["notification_type"].lower()
+        if notification_type in notifications.TYPES:
+            self.logger.debug(
+                f"{self.name}: initializing notification type '{notification_type.upper()}'")
+            self.notification = notifications.TYPES[notification_type](
+                self.name, global_cfg, cfg, test)
+        elif notification_type == "none":
+            self.logger.debug(f"{self.name}: notification is NONE")
+            self.notification = None
+        else:
+            raise RuntimeError(
+                f"{self.name}: unknown notification type '{notification_type}'")
+
+        # determining milter action for this quarantine
+        action = cfg["action"].upper()
+        if action in self._actions:
+            self.logger.debug(f"{self.name}: action is {action}")
+            self.action = action
+            self.milter_action = self._actions[action]
+        else:
+            raise RuntimeError(
+                f"{self.name}: unknown action '{action}'")
+
+        self.reject_reason = cfg["reject_reason"]
+
+        # create host/network whitelist
+        self.host_whitelist = []
+        host_whitelist = set([p.strip()
+                       for p in cfg["host_whitelist"].split(",") if p])
+        for host in host_whitelist:
+            if not host:
+                continue
+            # parse network notation
+            try:
+                net = IPNetwork(host)
+            except AddrFormatError as e:
+                raise RuntimeError(f"{self.name}: error parsing host_whitelist: {e}")
+            else:
+                self.host_whitelist.append(net)
+        if self.host_whitelist:
+            whitelist = ", ".join([str(ip) for ip in host_whitelist])
+            self.logger.debug(
+                f"{self.name}: host whitelist: {whitelist}")
+
+    def notify(self, storage_id, recipient=None, synchronous=True):
+        "Notify recipient about email in storage."
+        if not self.storage:
+            raise RuntimeError(
+                "storage type is set to None, unable to send notification")
+
+        if not self.notification:
+            raise RuntimeError(
+                "notification type is set to None, unable to send notification")
+
+        fp, metadata = self.storage.get_mail(storage_id)
+
+        if recipient is not None:
+            if recipient not in metadata["recipients"]:
+                raise RuntimeError(f"invalid recipient '{recipient}'")
+            recipients = [recipient]
+        else:
+            recipients = metadata["recipients"]
+
+        self.notification.notify(
+            metadata["queue_id"], storage_id, metadata["mailfrom"],
+            recipients, metadata["headers"], fp,
+            metadata["subgroups"], metadata["named_subgroups"],
+            synchronous)
+        fp.close()
+
+    def release(self, storage_id, recipients=None):
+        "Release email from storage."
+        if not self.storage:
+            raise RuntimeError(
+                "storage type is set to None, unable to release email")
+
+        fp, metadata = self.storage.get_mail(storage_id)
+        try:
+            mail = fp.read()
+            fp.close()
+        except IOError as e:
+            raise RuntimeError(f"unable to read data file: {e}")
+
+        if recipients and type(recipients) == str:
+            recipients = [recipients]
+        else:
+            recipients = metadata["recipients"]
+
+        for recipient in recipients:
+            if recipient not in metadata["recipients"]:
+                raise RuntimeError(f"invalid recipient '{recipient}'")
+
+            try:
+                mailer.smtp_send(
+                    self.smtp_host,
+                    self.smtp_port,
+                    metadata["mailfrom"],
+                    recipient,
+                    mail)
+            except Exception as e:
+                raise RuntimeError(
+                    f"error while sending email to '{recipient}': {e}")
+            self.storage.delete(storage_id, recipient)
+
+    def get_storage(self):
+        return self.storage
+
+    def get_notification(self):
+        return self.notification
+
+    def get_whitelist(self):
+        return self.whitelist
+
+    def host_in_whitelist(self, hostaddr):
+        ip = IPAddress(hostaddr[0])
+        for entry in self.host_whitelist:
+            if ip in entry:
+                return true
+        return False
+
+    def match(self, header):
+        return self.regex.search(header)
+
+
+class QuarantineMilter(Milter.Base):
+    """QuarantineMilter based on Milter.Base to implement milter communication
+
+    The class variable quarantines needs to be filled by runng the setup_milter function.
+
+    """
+    quarantines = []
+    preferred_quarantine_action = "first"
+
+    # list of default config files
+    _cfg_files = [
+        "/etc/pyquarantine/pyquarantine.conf",
+        os.path.expanduser('~/pyquarantine.conf'),
+        "pyquarantine.conf"]
+
     def __init__(self):
         self.logger = logging.getLogger(__name__)
-        # save config, it must not change during runtime
+        # save runtime config, it must not change during runtime
-        self.global_config = QuarantineMilter.global_config
+        self.quarantines = QuarantineMilter.quarantines
-        self.config = QuarantineMilter.config
 
     def _get_preferred_quarantine(self):
         matching_quarantines = [
             q for q in self.recipients_quarantines.values() if q]
-        if self.global_config["preferred_quarantine_action"] == "first":
+        if self.preferred_quarantine_action == "first":
             quarantine = sorted(
                 matching_quarantines,
-                key=lambda x: x["index"])[0]
+                key=lambda q: q.index)[0]
         else:
             quarantine = sorted(
                 matching_quarantines,
-                key=lambda x: x["index"],
+                key=lambda q: q.index,
                 reverse=True)[0]
         return quarantine
 
     @staticmethod
-    def get_configfiles():
+    def get_cfg_files():
-        return QuarantineMilter._config_files
+        return QuarantineMilter._cfg_files
 
     @staticmethod
-    def get_actions():
+    def set_cfg_files(cfg_files):
-        return QuarantineMilter._actions
+        QuarantineMilter._cfg_files = cfg_files
-
-    @staticmethod
-    def set_configfiles(config_files):
-        QuarantineMilter._config_files = config_files
 
     def connect(self, hostname, family, hostaddr):
         self.hostaddr = hostaddr
         self.logger.debug(
-            "accepted milter connection from {} port {}".format(
+            f"accepted milter connection from {hostaddr[0]} port {hostaddr[1]}")
-                *hostaddr))
+        for quarantine in self.quarantines.copy():
-        ip = IPAddress(hostaddr[0])
+            if quarantine.host_in_whitelist(hostaddr):
-        for quarantine in self.config.copy():
-            for ignore in quarantine["ignore_hosts_list"]:
-                if ip in ignore:
                 self.logger.debug(
-                        "host {} is ignored by quarantine {}".format(
+                    f"host {hostaddr[0]} is in whitelist of quarantine {quarantine['name']}")
-                            hostaddr[0], quarantine["name"]))
+                self.quarantines.remove(quarantine)
-                    self.config.remove(quarantine)
+                if not self.quarantines:
-                    break
-            if not self.config:
                     self.logger.debug(
-                    "host {} is ignored by all quarantines, "
+                        f"host {hostaddr[0]} is in whitelist of all quarantines, "
-                    "skip further processing",
+                        f"skip further processing")
-                    hostaddr[0])
                     return Milter.ACCEPT
         return Milter.CONTINUE
 
@@ -130,14 +333,12 @@ class QuarantineMilter(Milter.Base):
 
     @Milter.noreply
     def data(self):
-        self.queueid = self.getsymval('i')
+        self.qid = self.getsymval('i')
         self.logger.debug(
-            "{}: received queue-id from MTA".format(self.queueid))
+            f"{self.qid}: received queue-id from MTA")
         self.recipients = list(self.recipients)
-        self.headers = []
         self.logger.debug(
-            "{}: initializing memory buffer to save email data".format(
+            f"{self.qid}: initializing memory buffer to save email data")
-                self.queueid))
         # initialize memory buffer to save email data
         self.fp = BytesIO()
         return Milter.CONTINUE
@@ -146,15 +347,11 @@ class QuarantineMilter(Milter.Base):
     def header(self, name, value):
         try:
             # write email header to memory buffer
-            self.fp.write("{}: {}\r\n".format(name, value).encode(
+            self.fp.write(f"{name}: {value}\r\n".encode(
-                encoding="ascii", errors="surrogateescape"))
+                encoding="ascii", errors="replace"))
-            # keep copy of header without surrogates for later use
-            self.headers.append((
-                name.encode(errors="surrogateescape").decode(errors="replace"),
-                value.encode(errors="surrogateescape").decode(errors="replace")))
         except Exception as e:
             self.logger.exception(
-                "an exception occured in header function: {}".format(e))
+                f"an exception occured in header function: {e}")
             return Milter.TEMPFAIL
 
         return Milter.CONTINUE
@@ -162,6 +359,9 @@ class QuarantineMilter(Milter.Base):
     def eoh(self):
         try:
             self.fp.write("\r\n".encode(encoding="ascii"))
+            self.fp.seek(0)
+            self.headers = BytesHeaderParser(
+                policy=default_policy).parse(self.fp).items()
             self.whitelist_cache = whitelists.WhitelistCache()
 
             # initialize dicts to set quaranines per recipient and keep matches
@@ -171,44 +371,39 @@ class QuarantineMilter(Milter.Base):
             # iterate email headers
             recipients_to_check = self.recipients.copy()
             for name, value in self.headers:
-                header = "{}: {}".format(name, value)
+                header = f"{name}: {value}"
                 self.logger.debug(
-                    "{}: checking header against configured quarantines: {}".format(
+                    f"{self.qid}: checking header against configured quarantines: {header}")
-                        self.queueid, header))
                 # iterate quarantines
-                for quarantine in self.config:
+                for quarantine in self.quarantines:
                     if len(self.recipients_quarantines) == len(
                             self.recipients):
                         # every recipient matched a quarantine already
-                        if quarantine["index"] >= max(
+                        if quarantine.index >= max(
-                                [q["index"] for q in self.recipients_quarantines.values()]):
+                                [q.index for q in self.recipients_quarantines.values()]):
                             # all recipients matched a quarantine with at least
                             # the same precedence already, skip checks against
                             # quarantines with lower precedence
                             self.logger.debug(
-                                "{}: {}: skip further checks of this header".format(
+                                f"{self.qid}: {quarantine.name}: skip further checks of this header")
-                                    self.queueid, quarantine["name"]))
                             break
 
                     # check email header against quarantine regex
                     self.logger.debug(
-                        "{}: {}: checking header against regex '{}'".format(
+                        f"{self.qid}: {quarantine.name}: checking header against regex '{quarantine.regex}'")
-                            self.queueid, quarantine["name"], quarantine["regex"]))
+                    match = quarantine.match(header)
-                    match = quarantine["regex_compiled"].search(header)
                     if match:
                         self.logger.debug(
-                            "{}: {}: header matched regex".format(
+                            f"{self.qid}: {quarantine.name}: header matched regex")
-                                self.queueid, quarantine["name"]))
                         # check for whitelisted recipients
-                        whitelist = quarantine["whitelist_obj"]
+                        whitelist = quarantine.get_whitelist()
-                        if whitelist is not None:
+                        if whitelist:
                             try:
                                 whitelisted_recipients = self.whitelist_cache.get_whitelisted_recipients(
                                     whitelist, self.mailfrom, recipients_to_check)
                             except RuntimeError as e:
                                 self.logger.error(
-                                    "{}: {}: unable to query whitelist: {}".format(
+                                    f"{self.qid}: {quarantine.name}: unable to query whitelist: {e}")
-                                        self.queueid, quarantine["name"], e))
                                 return Milter.TEMPFAIL
                         else:
                             whitelisted_recipients = {}
@@ -218,48 +413,43 @@ class QuarantineMilter(Milter.Base):
                             if recipient in whitelisted_recipients:
                                 # recipient is whitelisted in this quarantine
                                 self.logger.debug(
-                                    "{}: {}: recipient '{}' is whitelisted".format(
+                                    f"{self.qid}: {quarantine.name}: recipient '{recipient}' is whitelisted")
-                                        self.queueid, quarantine["name"], recipient))
                                 continue
 
                             if recipient not in self.recipients_quarantines.keys() or \
-                                    self.recipients_quarantines[recipient]["index"] > quarantine["index"]:
+                                    self.recipients_quarantines[recipient].index > quarantine.index:
                                 self.logger.debug(
-                                    "{}: {}: set quarantine for recipient '{}'".format(
+                                    f"{self.qid}: {quarantine.name}: set quarantine for recipient '{recipient}'")
-                                        self.queueid, quarantine["name"], recipient))
                                 # save match for later use as template
                                 # variables
-                                self.quarantines_matches[quarantine["name"]] = match
+                                self.quarantines_matches[quarantine.name] = match
                                 self.recipients_quarantines[recipient] = quarantine
-                                if quarantine["index"] == 0:
+                                if quarantine.index == 0:
                                     # we do not need to check recipients which
                                     # matched the quarantine with the highest
                                     # precedence already
                                     recipients_to_check.remove(recipient)
                             else:
                                 self.logger.debug(
-                                    "{}: {}: a quarantine with same or higher precedence "
+                                    f"{self.qid}: {quarantine.name}: a quarantine with same or higher "
-                                    "matched already for recipient '{}'".format(
+                                    f"precedence matched already for recipient '{recipient}'")
-                                        self.queueid, quarantine["name"], recipient))
 
                 if not recipients_to_check:
                     self.logger.debug(
-                        "{}: all recipients matched the first quarantine, "
+                        f"{self.qid}: all recipients matched the first quarantine, "
-                        "skipping all remaining header checks".format(
+                        f"skipping all remaining header checks")
-                            self.queueid))
                     break
 
             # check if no quarantine has matched for all recipients
             if not self.recipients_quarantines:
                 # accept email
                 self.logger.info(
-                    "{}: passed clean for all recipients".format(
+                    f"{self.qid}: passed clean for all recipients")
-                        self.queueid))
                 return Milter.ACCEPT
 
             # check if the mail body is needed
             for recipient, quarantine in self.recipients_quarantines.items():
-                if quarantine["quarantine_obj"] or quarantine["notification_obj"]:
+                if quarantine.get_storage() or quarantine.get_notification():
                     # mail body is needed, continue processing
                     return Milter.CONTINUE
 
@@ -267,18 +457,16 @@ class QuarantineMilter(Milter.Base):
             # quarantines, just return configured action
             quarantine = self._get_preferred_quarantine()
             self.logger.info(
-                "{}: {} matching quarantine is '{}', performing milter action {}".format(
+                f"{self.qid}: {self.preferred_quarantine_action} "
-                    self.queueid,
+                f"matching quarantine is '{quarantine.name}', performing "
-                    self.global_config["preferred_quarantine_action"],
+                f"milter action {quarantine.action}")
-                    quarantine["name"],
+            if quarantine.action == "REJECT":
-                    quarantine["action"].upper()))
+                self.setreply("554", "5.7.0", quarantine.reject_reason)
-            if quarantine["action"] == "reject":
+            return quarantine.milter_action
-                self.setreply("554", "5.7.0", quarantine["reject_reason"])
-            return quarantine["milter_action"]
 
         except Exception as e:
             self.logger.exception(
-                "an exception occured in eoh function: {}".format(e))
+                f"an exception occured in eoh function: {e}")
             return Milter.TEMPFAIL
 
     def body(self, chunk):
@@ -287,7 +475,7 @@ class QuarantineMilter(Milter.Base):
             self.fp.write(chunk)
         except Exception as e:
             self.logger.exception(
-                "an exception occured in body function: {}".format(e))
+                f"an exception occured in body function: {e}")
             return Milter.TEMPFAIL
         return Milter.CONTINUE
 
@@ -297,52 +485,53 @@ class QuarantineMilter(Milter.Base):
             quarantines = []
             for quarantine, recipients in groupby(
                     sorted(self.recipients_quarantines,
-                           key=lambda x: self.recipients_quarantines[x]["index"]),
+                           key=lambda x: self.recipients_quarantines[x].index),
                     lambda x: self.recipients_quarantines[x]):
                 quarantines.append((quarantine, list(recipients)))
 
             # iterate quarantines sorted by index
             for quarantine, recipients in sorted(
-                    quarantines, key=lambda x: x[0]["index"]):
+                    quarantines, key=lambda x: x[0].index):
-                quarantine_id = ""
                 headers = defaultdict(str)
                 for name, value in self.headers:
                     headers[name.lower()] = value
-                subgroups = self.quarantines_matches[quarantine["name"]].groups(
+                subgroups = self.quarantines_matches[quarantine.name].groups(
                     default="")
-                named_subgroups = self.quarantines_matches[quarantine["name"]].groupdict(
+                named_subgroups = self.quarantines_matches[quarantine.name].groupdict(
                     default="")
 
-                # check if a quarantine is configured
+                rcpts = ", ".join(recipients)
-                if quarantine["quarantine_obj"] is not None:
+
+                # check if a storage is configured
+                storage_id = ""
+                storage = quarantine.get_storage()
+                if storage:
                     # add email to quarantine
-                    self.logger.info("{}: adding to quarantine '{}' for: {}".format(
+                    self.logger.info(
-                        self.queueid, quarantine["name"], ", ".join(recipients)))
+                        f"{self.qid}: adding to quarantine '{quarantine.name}' for: {rcpts}")
                     try:
-                        quarantine_id = quarantine["quarantine_obj"].add(
+                        storage_id = storage.add(
-                            self.queueid, self.mailfrom, recipients, headers, self.fp,
+                            self.qid, self.mailfrom, recipients, headers, self.fp,
                             subgroups, named_subgroups)
                     except RuntimeError as e:
                         self.logger.error(
-                            "{}: unable to add to quarantine '{}': {}".format(
+                            f"{self.qid}: unable to add to quarantine '{quarantine.name}': {e}")
-                                self.queueid, quarantine["name"], e))
                         return Milter.TEMPFAIL
 
                 # check if a notification is configured
-                if quarantine["notification_obj"] is not None:
+                notification = quarantine.get_notification()
+                if notification:
                     # notify
                     self.logger.info(
-                        "{}: sending notification for quarantine '{}' to: {}".format(
+                        f"{self.qid}: sending notification to: {rcpts}")
-                            self.queueid, quarantine["name"], ", ".join(recipients)))
                     try:
-                        quarantine["notification_obj"].notify(
+                        notification.notify(
-                            self.queueid, quarantine_id,
+                            self.qid, storage_id,
                             self.mailfrom, recipients, headers, self.fp,
                             subgroups, named_subgroups)
                     except RuntimeError as e:
                         self.logger.error(
-                            "{}: unable to send notification for quarantine '{}': {}".format(
+                            f"{self.qid}: unable to send notification: {e}")
-                                self.queueid, quarantine["name"], e))
                         return Milter.TEMPFAIL
 
                 # remove processed recipient
@@ -354,50 +543,47 @@ class QuarantineMilter(Milter.Base):
 
             # email passed clean for at least one recipient, accepting email
             if self.recipients:
+                rcpts = ", ".join(recipients)
                 self.logger.info(
-                    "{}: passed clean for: {}".format(
+                    f"{self.qid}: passed clean for: {rcpts}")
-                        self.queueid, ", ".join(
-                            self.recipients)))
                 return Milter.ACCEPT
 
             # return configured action
             quarantine = self._get_preferred_quarantine()
             self.logger.info(
-                "{}: {} matching quarantine is '{}', performing milter action {}".format(
+                f"{self.qid}: {self.preferred_quarantine_action} matching "
-                    self.queueid,
+                f"quarantine is '{quarantine.name}', performing milter "
-                    self.global_config["preferred_quarantine_action"],
+                f"action {quarantine.action}")
-                    quarantine["name"],
+            if quarantine.action == "REJECT":
-                    quarantine["action"].upper()))
+                self.setreply("554", "5.7.0", quarantine.reject_reason)
-            if quarantine["action"] == "reject":
+            return quarantine.milter_action
-                self.setreply("554", "5.7.0", quarantine["reject_reason"])
-            return quarantine["milter_action"]
 
         except Exception as e:
             self.logger.exception(
-                "an exception occured in eom function: {}".format(e))
+                f"an exception occured in eom function: {e}")
             return Milter.TEMPFAIL
 
     def close(self):
         self.logger.debug(
-            "disconnect from {} port {}".format(
+            f"disconnect from {self.hostaddr[0]} port {self.hostaddr[1]}")
-                *self.hostaddr))
+        return Milter.CONTINUE
 
 
-def generate_milter_config(configtest=False, config_files=[]):
+def setup_milter(test=False, cfg_files=[]):
     "Generate the configuration for QuarantineMilter class."
     logger = logging.getLogger(__name__)
 
     # read config file
     parser = configparser.ConfigParser()
-    if not config_files:
+    if not cfg_files:
-        config_files = parser.read(QuarantineMilter.get_configfiles())
+        cfg_files = parser.read(QuarantineMilter.get_cfg_files())
     else:
-        config_files = parser.read(config_files)
+        cfg_files = parser.read(cfg_files)
-    if not config_files:
+    if not cfg_files:
         raise RuntimeError("config file not found")
 
-    QuarantineMilter.set_configfiles(config_files)
+    QuarantineMilter.set_cfg_files(cfg_files)
-    os.chdir(os.path.dirname(config_files[0]))
+    os.chdir(os.path.dirname(cfg_files[0]))
 
     # check if mandatory config options in global section are present
     if "global" not in parser.sections():
@@ -406,167 +592,44 @@ def generate_milter_config(configtest=False, config_files=[]):
     for option in ["quarantines", "preferred_quarantine_action"]:
         if not parser.has_option("global", option):
             raise RuntimeError(
-                "mandatory option '{}' not present in config section 'global'".format(option))
+                f"mandatory option '{option}' not present in config section 'global'")
 
     # read global config section
-    global_config = dict(parser.items("global"))
+    global_cfg = dict(parser.items("global"))
-    global_config["preferred_quarantine_action"] = global_config["preferred_quarantine_action"].lower()
+    preferred_quarantine_action = global_cfg["preferred_quarantine_action"].lower()
-    if global_config["preferred_quarantine_action"] not in ["first", "last"]:
+    if preferred_quarantine_action not in ["first", "last"]:
         raise RuntimeError(
             "option preferred_quarantine_action has illegal value")
 
     # read active quarantine names
-    quarantine_names = [
+    quarantines = [
-        q.strip() for q in global_config["quarantines"].split(",")]
+        q.strip() for q in global_cfg["quarantines"].split(",")]
-    if len(quarantine_names) != len(set(quarantine_names)):
+    if len(quarantines) != len(set(quarantines)):
         raise RuntimeError(
             "at least one quarantine is specified multiple times in quarantines option")
-    if "global" in quarantine_names:
+    if "global" in quarantines:
-        quarantine_names.remove("global")
+        quarantines.remove("global")
         logger.warning(
             "removed illegal quarantine name 'global' from list of active quarantines")
-    if not quarantine_names:
+    if not quarantines:
         raise RuntimeError("no quarantines configured")
 
-    milter_config = []
+    milter_quarantines = []
-
     logger.debug("preparing milter configuration ...")
     # iterate quarantine names
-    for index, quarantine_name in enumerate(quarantine_names):
+    for index, name in enumerate(quarantines):
-
         # check if config section for current quarantine exists
-        if quarantine_name not in parser.sections():
+        if name not in parser.sections():
             raise RuntimeError(
-                "config section '{}' does not exist".format(quarantine_name))
+                f"config section '{name}' does not exist")
-        config = dict(parser.items(quarantine_name))
 
-        # check if mandatory config options are present in config
+        cfg = dict(parser.items(name))
-        for option in ["regex", "quarantine_type", "notification_type",
+        quarantine = Quarantine(name, index)
-                       "action", "whitelist_type", "smtp_host", "smtp_port"]:
+        quarantine.setup_from_cfg(global_cfg, cfg, test)
-            if option not in config.keys() and \
+        milter_quarantines.append(quarantine)
-                    option in global_config.keys():
-                config[option] = global_config[option]
-            if option not in config.keys():
-                raise RuntimeError(
-                    "mandatory option '{}' not present in config section '{}' or 'global'".format(
-                        option, quarantine_name))
 
-        # check if optional config options are present in config
+    QuarantineMilter.preferred_quarantine_action = preferred_quarantine_action
-        defaults = {
+    QuarantineMilter.quarantines = milter_quarantines
-            "reject_reason": "Message rejected",
-            "ignore_hosts": ""
-        }
-        for option in defaults.keys():
-            if option not in config.keys() and \
-                    option in global_config.keys():
-                config[option] = global_config[option]
-            if option not in config.keys():
-                config[option] = defaults[option]
-
-        # set quarantine name
-        config["name"] = quarantine_name
-
-        # set the index
-        config["index"] = index
-
-        # pre-compile regex
-        logger.debug(
-            "{}: compiling regex '{}'".format(
-                quarantine_name,
-                config["regex"]))
-        config["regex_compiled"] = re.compile(
-            config["regex"], re.MULTILINE + re.DOTALL + re.IGNORECASE)
-
-        # create quarantine instance
-        quarantine_type = config["quarantine_type"].lower()
-        if quarantine_type in quarantines.TYPES.keys():
-            logger.debug(
-                "{}: initializing quarantine type '{}'".format(
-                    quarantine_name,
-                    quarantine_type.upper()))
-            quarantine = quarantines.TYPES[quarantine_type](
-                global_config, config, configtest)
-        elif quarantine_type == "none":
-            logger.debug("{}: quarantine is NONE".format(quarantine_name))
-            quarantine = None
-        else:
-            raise RuntimeError(
-                "{}: unknown quarantine type '{}'".format(
-                    quarantine_name, quarantine_type))
-
-        config["quarantine_obj"] = quarantine
-
-        # create whitelist instance
-        whitelist_type = config["whitelist_type"].lower()
-        if whitelist_type in whitelists.TYPES.keys():
-            logger.debug(
-                "{}: initializing whitelist type '{}'".format(
-                    quarantine_name,
-                    whitelist_type.upper()))
-            whitelist = whitelists.TYPES[whitelist_type](
-                global_config, config, configtest)
-        elif whitelist_type == "none":
-            logger.debug("{}: whitelist is NONE".format(quarantine_name))
-            whitelist = None
-        else:
-            raise RuntimeError(
-                "{}: unknown whitelist type '{}'".format(
-                    quarantine_name, whitelist_type))
-
-        config["whitelist_obj"] = whitelist
-
-        # create notification instance
-        notification_type = config["notification_type"].lower()
-        if notification_type in notifications.TYPES.keys():
-            logger.debug(
-                "{}: initializing notification type '{}'".format(
-                    quarantine_name,
-                    notification_type.upper()))
-            notification = notifications.TYPES[notification_type](
-                global_config, config, configtest)
-        elif notification_type == "none":
-            logger.debug("{}: notification is NONE".format(quarantine_name))
-            notification = None
-        else:
-            raise RuntimeError(
-                "{}: unknown notification type '{}'".format(
-                    quarantine_name, notification_type))
-
-        config["notification_obj"] = notification
-
-        # determining milter action for this quarantine
-        action = config["action"].upper()
-        if action in QuarantineMilter.get_actions().keys():
-            logger.debug("{}: action is {}".format(quarantine_name, action))
-            config["milter_action"] = QuarantineMilter.get_actions()[action]
-        else:
-            raise RuntimeError(
-                "{}: unknown action '{}'".format(
-                    quarantine_name, action))
-
-        # create host/network whitelist
-        config["ignore_hosts_list"] = []
-        ignored = set([p.strip()
-                       for p in config["ignore_hosts"].split(",") if p])
-        for ignore in ignored:
-            if not ignore:
-                continue
-            # parse network notation
-            try:
-                net = IPNetwork(ignore)
-            except AddrFormatError as e:
-                raise RuntimeError("error parsing ignore_hosts: {}".format(e))
-            else:
-                config["ignore_hosts_list"].append(net)
-        if config["ignore_hosts_list"]:
-            logger.debug(
-                "{}: ignore hosts: {}".format(
-                    quarantine_name,
-                    ", ".join(ignored)))
-
-        milter_config.append(config)
-
-    return global_config, milter_config
 
 
 def reload_config():
@@ -574,11 +637,9 @@ def reload_config():
     logger = logging.getLogger(__name__)
 
     try:
-        global_config, config = generate_milter_config()
+        setup_milter()
     except RuntimeError as e:
         logger.info(e)
         logger.info("daemon is still running with previous configuration")
     else:
-        logger.info("reloading configuration")
+        logger.info("reloaded configuration")
-        QuarantineMilter.global_config = global_config
-        QuarantineMilter.config = config

pyquarantine/cli.py

@@ -20,29 +20,41 @@ import logging.handlers
 import sys
 import time
 
-from email.header import decode_header, make_header
+from email.header import decode_header
-
-import pyquarantine
 
+from pyquarantine import QuarantineMilter, setup_milter
 from pyquarantine.version import __version__ as version
 
-def _get_quarantine_obj(config, quarantine):
+def _get_quarantine(quarantines, name):
     try:
-        quarantine_obj = next((q["quarantine_obj"]
+        quarantine = next((q for q in quarantines if q.name == name))
-                               for q in config if q["name"] == quarantine))
     except StopIteration:
-        raise RuntimeError("invalid quarantine '{}'".format(quarantine))
+        raise RuntimeError(f"invalid quarantine 'name'")
-    return quarantine_obj
+    return quarantine
 
+def _get_storage(quarantines, name):
+    quarantine = _get_quarantine(quarantines, name)
+    storage = quarantine.get_storage()
+    if not storage:
+        raise RuntimeError(
+                "storage type is set to NONE")
+    return storage
 
-def _get_whitelist_obj(config, quarantine):
+def _get_notification(quarantines, name):
-    try:
+    quarantine = _get_quarantine(quarantines, name)
-        whitelist_obj = next((q["whitelist_obj"]
+    notification = quarantine.get_notification()
-                              for q in config if q["name"] == quarantine))
+    if not notification:
-    except StopIteration:
+        raise RuntimeError(
-        raise RuntimeError("invalid quarantine '{}'".format(quarantine))
+                "notification type is set to NONE")
-    return whitelist_obj
+    return notification
 
+def _get_whitelist(quarantines, name):
+    quarantine = _get_quarantine(quarantines, name)
+    whitelist = quarantine.get_whitelist()
+    if not whitelist:
+        raise RuntimeError(
+                "whitelist type is set to NONE")
+    return whitelist
 
 def print_table(columns, rows):
     if not rows:
@@ -61,7 +73,7 @@ def print_table(columns, rows):
         # use the longer one
         length = max(lengths)
         column_lengths.append(length)
-        column_formats.append("{{:<{}}}".format(length))
+        column_formats.append(f"{{:<{length}}}")
 
     # define row format
     row_format = " | ".join(column_formats)
@@ -85,51 +97,70 @@ def print_table(columns, rows):
         print(row_format.format(*row))
 
 
-def list_quarantines(config, args):
+def list_quarantines(quarantines, args):
     if args.batch:
-        print("\n".join([quarantine["name"] for quarantine in config]))
+        print("\n".join([q.name for q in quarantines]))
     else:
+        qlist = []
+        for q in quarantines:
+            storage = q.get_storage()
+            if storage:
+                storage_type = q.get_storage().storage_type
+            else:
+                storage_type = "NONE"
+
+            notification = q.get_notification()
+            if notification:
+                notification_type = q.get_notification().notification_type
+            else:
+                notification_type = "NONE"
+
+            whitelist = q.get_whitelist()
+            if whitelist:
+                whitelist_type = q.get_whitelist().whitelist_type
+            else:
+                whitelist_type = "NONE"
+
+            qlist.append({
+                "name": q.name,
+                "storage": storage_type,
+                "notification": notification_type,
+                "whitelist": whitelist_type,
+                "action": q.action})
         print_table(
-            [("Name", "name"), ("Quarantine", "quarantine_type"),
+            [("Name", "name"),
-             ("Notification", "notification_type"), ("Action", "action")],
+             ("Storage", "storage"),
-            config
+             ("Notification", "notification"),
+             ("Whitelist", "whitelist"),
+             ("Action", "action")],
+            qlist
         )
 
 
-def list_quarantine_emails(config, args):
+def list_quarantine_emails(quarantines, args):
     logger = logging.getLogger(__name__)
-
+    storage = _get_storage(quarantines, args.quarantine)
-    # get quarantine object
-    quarantine = _get_quarantine_obj(config, args.quarantine)
-    if quarantine is None:
-        raise RuntimeError(
-            "quarantine type is set to None, unable to list emails")
-
     # find emails and transform some metadata values to strings
     rows = []
-    emails = quarantine.find(
+    emails = storage.find(
-        mailfrom=args.mailfrom,
+        args.mailfrom, args.recipients, args.older_than)
-        recipients=args.recipients,
+    for storage_id, metadata in emails.items():
-        older_than=args.older_than)
+        row = emails[storage_id]
-    for quarantine_id, metadata in emails.items():
+        row["storage_id"] = storage_id
-        row = emails[quarantine_id]
-        row["quarantine_id"] = quarantine_id
         row["date"] = time.strftime(
             '%Y-%m-%d %H:%M:%S',
             time.localtime(
                 metadata["date"]))
         row["mailfrom"] = metadata["mailfrom"]
         row["recipient"] = metadata["recipients"].pop(0)
-        if "subject" not in emails[quarantine_id]["headers"].keys():
+        if "subject" not in emails[storage_id]["headers"].keys():
-            emails[quarantine_id]["headers"]["subject"] = ""
+            emails[storage_id]["headers"]["subject"] = ""
-        row["subject"] = str(make_header(decode_header(
+        row["subject"] = emails[storage_id]["headers"]["subject"][:60].strip()
-            emails[quarantine_id]["headers"]["subject"])))[:60].replace(
-                "\r", "").replace("\n", "").strip()
         rows.append(row)
 
         if metadata["recipients"]:
             row = {
-                "quarantine_id": "",
+                "storage_id": "",
                 "date": "",
                 "mailfrom": "",
                 "recipient": metadata["recipients"].pop(0),
@@ -143,23 +174,18 @@ def list_quarantine_emails(config, args):
         return
 
     if not emails:
-        logger.info("quarantine '{}' is empty".format(args.quarantine))
+        logger.info(f"quarantine '{args.quarantine}' is empty")
     print_table(
-        [("Quarantine-ID", "quarantine_id"), ("Date", "date"),
+        [("Quarantine-ID", "storage_id"), ("Date", "date"),
          ("From", "mailfrom"), ("Recipient(s)", "recipient"),
          ("Subject", "subject")],
         rows
     )
 
 
-def list_whitelist(config, args):
+def list_whitelist(quarantines, args):
     logger = logging.getLogger(__name__)
-
+    whitelist = _get_whitelist(quarantines, args.quarantine)
-    # get whitelist object
-    whitelist = _get_whitelist_obj(config, args.quarantine)
-    if whitelist is None:
-        raise RuntimeError(
-            "whitelist type is set to None, unable to list entries")
 
     # find whitelist entries
     entries = whitelist.find(
@@ -168,8 +194,7 @@ def list_whitelist(config, args):
         older_than=args.older_than)
     if not entries:
         logger.info(
-            "whitelist of quarantine '{}' is empty".format(
+            f"whitelist of quarantine '{args.quarantine}' is empty")
-                args.quarantine))
         return
 
     # transform some values to strings
@@ -190,14 +215,9 @@ def list_whitelist(config, args):
     )
 
 
-def add_whitelist_entry(config, args):
+def add_whitelist_entry(quarantines, args):
     logger = logging.getLogger(__name__)
-
+    whitelist = _get_whitelist(quarantines, args.quarantine)
-    # get whitelist object
-    whitelist = _get_whitelist_obj(config, args.quarantine)
-    if whitelist is None:
-        raise RuntimeError(
-            "whitelist type is set to None, unable to add entries")
 
     # check existing entries
     entries = whitelist.check(args.mailfrom, args.recipient)
@@ -235,50 +255,31 @@ def add_whitelist_entry(config, args):
     logger.info("whitelist entry added successfully")
 
 
-def delete_whitelist_entry(config, args):
+def delete_whitelist_entry(quarantines, args):
     logger = logging.getLogger(__name__)
-
+    whitelist = _get_whitelist(quarantines, args.quarantine)
-    whitelist = _get_whitelist_obj(config, args.quarantine)
-    if whitelist is None:
-        raise RuntimeError(
-            "whitelist type is set to None, unable to delete entries")
-
     whitelist.delete(args.whitelist_id)
     logger.info("whitelist entry deleted successfully")
 
 
-def notify_email(config, args):
+def notify(quarantines, args):
     logger = logging.getLogger(__name__)
-
+    quarantine = _get_quarantine(quarantines, args.quarantine)
-    quarantine = _get_quarantine_obj(config, args.quarantine)
-    if quarantine is None:
-        raise RuntimeError(
-            "quarantine type is set to None, unable to send notification")
     quarantine.notify(args.quarantine_id, args.recipient)
-    logger.info("sent notification successfully")
+    logger.info("notification sent successfully")
 
 
-def release_email(config, args):
+def release(quarantines, args):
     logger = logging.getLogger(__name__)
-
+    quarantine = _get_quarantine(quarantines, args.quarantine)
-    quarantine = _get_quarantine_obj(config, args.quarantine)
-    if quarantine is None:
-        raise RuntimeError(
-            "quarantine type is set to None, unable to release email")
-
     quarantine.release(args.quarantine_id, args.recipient)
     logger.info("quarantined email released successfully")
 
 
-def delete_email(config, args):
+def delete(quarantines, args):
     logger = logging.getLogger(__name__)
-
+    storage = _get_storage(quarantines, args.quarantine)
-    quarantine = _get_quarantine_obj(config, args.quarantine)
+    storage.delete(args.quarantine_id, args.recipient)
-    if quarantine is None:
-        raise RuntimeError(
-            "quarantine type is set to None, unable to delete email")
-
-    quarantine.delete(args.quarantine_id, args.recipient)
     logger.info("quarantined email deleted successfully")
 
 
@@ -304,7 +305,7 @@ def main():
         "-c", "--config",
         help="Config files to read.",
         nargs="+", metavar="CFG",
-        default=pyquarantine.QuarantineMilter.get_configfiles())
+        default=QuarantineMilter.get_cfg_files())
     parser.add_argument(
         "-d", "--debug",
         help="Log debugging messages.",
@@ -313,7 +314,7 @@ def main():
         "-v", "--version",
         help="Print version.",
         action="version",
-        version="%(prog)s ({})".format(version))
+        version=f"%(prog)s ({version})")
     parser.set_defaults(syslog=False)
     subparsers = parser.add_subparsers(
         dest="command",
@@ -394,7 +395,7 @@ def main():
         "-a", "--all",
         help="Release email for all recipients.",
         action="store_true")
-    quarantine_notify_parser.set_defaults(func=notify_email)
+    quarantine_notify_parser.set_defaults(func=notify)
     # quarantine release command
     quarantine_release_parser = quarantine_subparsers.add_parser(
         "release",
@@ -421,7 +422,7 @@ def main():
         "-a", "--all",
         help="Release email for all recipients.",
         action="store_true")
-    quarantine_release_parser.set_defaults(func=release_email)
+    quarantine_release_parser.set_defaults(func=release)
     # quarantine delete command
     quarantine_delete_parser = quarantine_subparsers.add_parser(
         "delete",
@@ -447,7 +448,7 @@ def main():
         "-a", "--all",
         help="Delete email for all recipients.",
         action="store_true")
-    quarantine_delete_parser.set_defaults(func=delete_email)
+    quarantine_delete_parser.set_defaults(func=delete)
 
     # whitelist command group
     whitelist_parser = subparsers.add_parser(
@@ -558,8 +559,8 @@ def main():
 
     # try to generate milter configs
     try:
-        global_config, config = pyquarantine.generate_milter_config(
+        setup_milter(
-            config_files=args.config, configtest=True)
+            cfg_files=args.config, test=True)
     except RuntimeError as e:
         logger.error(e)
         sys.exit(255)
@@ -580,7 +581,7 @@ def main():
 
     # call the commands function
     try:
-        args.func(config, args)
+        args.func(QuarantineMilter.quarantines, args)
     except RuntimeError as e:
         logger.error(e)
         sys.exit(1)

pyquarantine/mailer.py

@@ -45,23 +45,21 @@ def mailprocess():
             if not m:
                 break
 
-            smtp_host, smtp_port, queueid, mailfrom, recipient, mail, emailtype = m
+            smtp_host, smtp_port, qid, mailfrom, recipient, mail, emailtype = m
             try:
                 smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail)
             except Exception as e:
                 logger.error(
-                    "{}: error while sending {} to '{}': {}".format(
+                    f"{qid}: error while sending {emailtype} to '{recipient}': {e}")
-                        queueid, emailtype, recipient, e))
             else:
                 logger.info(
-                    "{}: successfully sent {} to: {}".format(
+                    f"{qid}: successfully sent {emailtype} to: {recipient}")
-                        queueid, emailtype, recipient))
     except KeyboardInterrupt:
         pass
     logger.debug("mailer process terminated")
 
 
-def sendmail(smtp_host, smtp_port, queueid, mailfrom, recipients, mail,
+def sendmail(smtp_host, smtp_port, qid, mailfrom, recipients, mail,
              emailtype="email"):
     "Send an email."
     global logger
@@ -81,7 +79,7 @@ def sendmail(smtp_host, smtp_port, queueid, mailfrom, recipients, mail,
     for recipient in recipients:
         try:
             queue.put(
-                (smtp_host, smtp_port, queueid, mailfrom, recipient, mail,
+                (smtp_host, smtp_port, qid, mailfrom, recipient, mail,
                  emailtype),
                 timeout=30)
         except Queue.Full as e:

pyquarantine/notifications.py

@@ -19,7 +19,8 @@ import re
 from bs4 import BeautifulSoup
 from cgi import escape
 from collections import defaultdict
-from email.header import decode_header, make_header
+from email import policy
+from email.header import decode_header
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 from email.mime.image import MIMEImage
@@ -31,14 +32,13 @@ from pyquarantine import mailer
 
 class BaseNotification(object):
     "Notification base class"
+    notification_type = "base"
 
-    def __init__(self, global_config, config, configtest=False):
+    def __init__(self, name, global_cfg, cfg, test=False):
-        self.quarantine_name = config["name"]
+        self.name = name
-        self.global_config = global_config
-        self.config = config
         self.logger = logging.getLogger(__name__)
 
-    def notify(self, queueid, quarantine_id, mailfrom, recipients, headers,
+    def notify(self, qid, storage_id, mailfrom, recipients, headers,
                fp, subgroups=None, named_subgroups=None, synchronous=False):
         fp.seek(0)
         pass
@@ -46,8 +46,7 @@ class BaseNotification(object):
 
 class EMailNotification(BaseNotification):
     "Notification class to send notifications via mail."
-    _html_text = "text/html"
+    notification_type = "email"
-    _plain_text = "text/plain"
     _bad_tags = [
         "applet",
         "embed",
@@ -111,44 +110,39 @@ class EMailNotification(BaseNotification):
         "width"
     ]
 
-    def __init__(self, global_config, config, configtest=False):
+    def __init__(self, name, global_cfg, cfg, test=False):
         super(EMailNotification, self).__init__(
-            global_config, config, configtest)
+            name, global_cfg, cfg, test)
 
-        # check if mandatory options are present in config
+        defaults = {
-        for option in [
+            "notification_email_replacement_img": "",
+            "notification_email_strip_images": "false",
+            "notification_email_parser_lib": "lxml"
+        }
+        # check config
+        for opt in [
             "notification_email_smtp_host",
             "notification_email_smtp_port",
             "notification_email_envelope_from",
             "notification_email_from",
             "notification_email_subject",
             "notification_email_template",
-                "notification_email_embedded_imgs"]:
+                "notification_email_embedded_imgs"] + list(defaults.keys()):
-            if option not in self.config.keys() and option in self.global_config.keys():
+            if opt in cfg:
-                self.config[option] = self.global_config[option]
+                continue
-            if option not in self.config.keys():
+            if opt in global_cfg:
+                cfg[opt] = global_cfg[opt]
+            elif opt in defaults:
+                cfg[opt] = defaults[opt]
+            else:
                 raise RuntimeError(
-                    "mandatory option '{}' not present in config section '{}' or 'global'".format(
+                    f"mandatory option '{opt}' not present in config section '{self.name}' or 'global'")
-                        option, self.quarantine_name))
 
-        # check if optional config options are present in config
+        self.smtp_host = cfg["notification_email_smtp_host"]
-        defaults = {
+        self.smtp_port = cfg["notification_email_smtp_port"]
-            "notification_email_replacement_img": "",
+        self.mailfrom = cfg["notification_email_envelope_from"]
-            "notification_email_strip_images": "false",
+        self.from_header = cfg["notification_email_from"]
-            "notification_email_parser_lib": "lxml"
+        self.subject = cfg["notification_email_subject"]
-        }
-        for option in defaults.keys():
-            if option not in config.keys() and \
-                    option in global_config.keys():
-                config[option] = global_config[option]
-            if option not in config.keys():
-                config[option] = defaults[option]
-
-        self.smtp_host = self.config["notification_email_smtp_host"]
-        self.smtp_port = self.config["notification_email_smtp_port"]
-        self.mailfrom = self.config["notification_email_envelope_from"]
-        self.from_header = self.config["notification_email_from"]
-        self.subject = self.config["notification_email_subject"]
 
         testvars = defaultdict(str, test="TEST")
 
@@ -157,26 +151,26 @@ class EMailNotification(BaseNotification):
             self.from_header.format_map(testvars)
         except ValueError as e:
             raise RuntimeError(
-                "error parsing notification_email_from: {}".format(e))
+                f"error parsing notification_email_from: {e}")
 
         # test-parse subject
         try:
             self.subject.format_map(testvars)
         except ValueError as e:
             raise RuntimeError(
-                "error parsing notification_email_subject: {}".format(e))
+                f"error parsing notification_email_subject: {e}")
 
         # read and parse email notification template
         try:
             self.template = open(
-                self.config["notification_email_template"], "r").read()
+                cfg["notification_email_template"], "r").read()
             self.template.format_map(testvars)
         except IOError as e:
-            raise RuntimeError("error reading template: {}".format(e))
+            raise RuntimeError(f"error reading template: {e}")
         except ValueError as e:
-            raise RuntimeError("error parsing template: {}".format(e))
+            raise RuntimeError(f"error parsing template: {e}")
 
-        strip_images = self.config["notification_email_strip_images"].strip().upper()
+        strip_images = cfg["notification_email_strip_images"].strip().upper()
         if strip_images in ["TRUE", "ON", "YES"]:
             self.strip_images = True
         elif strip_images in ["FALSE", "OFF", "NO"]:
@@ -184,19 +178,19 @@ class EMailNotification(BaseNotification):
         else:
             raise RuntimeError("error parsing notification_email_strip_images: unknown value")
 
-        self.parser_lib = self.config["notification_email_parser_lib"].strip()
+        self.parser_lib = cfg["notification_email_parser_lib"].strip()
         if self.parser_lib not in ["lxml", "html.parser"]:
             raise RuntimeError("error parsing notification_email_parser_lib: unknown value")
 
         # read email replacement image if specified
-        replacement_img = self.config["notification_email_replacement_img"].strip()
+        replacement_img = cfg["notification_email_replacement_img"].strip()
         if not self.strip_images and replacement_img:
             try:
                 self.replacement_img = MIMEImage(
                     open(replacement_img, "rb").read())
             except IOError as e:
                 raise RuntimeError(
-                    "error reading replacement image: {}".format(e))
+                    f"error reading replacement image: {e}")
             else:
                 self.replacement_img.add_header(
                     "Content-ID", "<removed_for_security_reasons>")
@@ -205,74 +199,78 @@ class EMailNotification(BaseNotification):
 
         # read images to embed if specified
         embedded_img_paths = [
-            p.strip() for p in self.config["notification_email_embedded_imgs"].split(",") if p]
+            p.strip() for p in cfg["notification_email_embedded_imgs"].split(",") if p]
         self.embedded_imgs = []
         for img_path in embedded_img_paths:
             # read image
             try:
                 img = MIMEImage(open(img_path, "rb").read())
             except IOError as e:
-                raise RuntimeError("error reading image: {}".format(e))
+                raise RuntimeError(f"error reading image: {e}")
             else:
-                img.add_header("Content-ID", "<{}>".format(basename(img_path)))
+                filename = basename(img_path)
+                img.add_header(f"Content-ID", f"<{filename}>")
                 self.embedded_imgs.append(img)
 
-    def get_decoded_email_body(self, queueid, msg, preferred=_html_text):
+    def get_email_body_soup(self, qid, msg):
-        "Find and decode email body."
+        "Extract and decode email body and return it as BeautifulSoup object."
         # try to find the body part
-        self.logger.debug("{}: trying to find email body".format(queueid))
+        self.logger.debug(f"{qid}: trying to find email body")
+        try:
+            body = msg.get_body(preferencelist=("html", "plain"))
+        except Exception as e:
+            self.logger.error(f"{qid}: an error occured in email.message.EmailMessage.get_body: {e}")
             body = None
-        for part in msg.walk():
-            content_type = part.get_content_type()
-            if content_type in [EMailNotification._plain_text,
-                                EMailNotification._html_text]:
-                body = part
-                if content_type == preferred:
-                    break
 
-        if body is not None:
+        if body:
             charset = body.get_content_charset() or "utf-8"
             content = body.get_payload(decode=True)
             try:
                 content = content.decode(encoding=charset, errors="replace")
             except LookupError:
+                self.logger.info(f"{qid}: unknown encoding '{charset}', falling back to UTF-8")
                 content = content.decode("utf-8", errors="replace")
             content_type = body.get_content_type()
-            if content_type == EMailNotification._plain_text:
+            if content_type == "text/plain":
                 # convert text/plain to text/html
                 self.logger.debug(
-                    "{}: content type is {}, converting to {}".format(
+                    f"{qid}: content type is {content_type}, converting to text/html")
-                        queueid, content_type, EMailNotification._html_text))
                 content = re.sub(r"^(.*)$", r"\1<br/>",
                                  escape(content), flags=re.MULTILINE)
             else:
                 self.logger.debug(
-                    "{}: content type is {}".format(
+                    f"{qid}: content type is {content_type}")
-                        queueid, content_type))
         else:
             self.logger.error(
-                "{}: unable to find email body".format(queueid))
+                f"{qid}: unable to find email body")
             content = "ERROR: unable to find email body"
 
-        return content
+        # create BeautifulSoup object
+        length = len(content)
+        self.logger.debug(
+            f"{qid}: trying to create BeatufilSoup object with parser lib {self.parser_lib}, "
+            f"text length is {length} bytes")
+        soup = BeautifulSoup(content, self.parser_lib)
+        self.logger.debug(
+            f"{qid}: sucessfully created BeautifulSoup object")
 
-    def sanitize(self, queueid, soup):
+        return soup
+
+    def sanitize(self, qid, soup):
         "Sanitize mail html text."
-        self.logger.debug("{}: sanitizing email text".format(queueid))
+        self.logger.debug(f"{qid}: sanitizing email text")
 
         # completly remove bad elements
         for element in soup(EMailNotification._bad_tags):
             self.logger.debug(
-                "{}: removing dangerous tag '{}' and its content".format(
+                f"{qid}: removing dangerous tag '{element.name}' and its content")
-                    queueid, element.name))
             element.extract()
 
         # remove not whitelisted elements, but keep their content
         for element in soup.find_all(True):
             if element.name not in EMailNotification._good_tags:
                 self.logger.debug(
-                    "{}: removing tag '{}', keep its content".format(
+                    f"{qid}: removing tag '{element.name}', keep its content")
-                        queueid, element.name))
                 element.replaceWithChildren()
 
         # remove not whitelisted attributes
@@ -281,24 +279,22 @@ class EMailNotification(BaseNotification):
                 if attribute not in EMailNotification._good_attributes:
                     if element.name == "a" and attribute == "href":
                         self.logger.debug(
-                            "{}: setting attribute href to '#' on tag '{}'".format(
+                            f"{qid}: setting attribute href to '#' on tag '{element.name}'")
-                                queueid, element.name))
                         element["href"] = "#"
                     else:
                         self.logger.debug(
-                            "{}: removing attribute '{}' from tag '{}'".format(
+                            f"{qid}: removing attribute '{attribute}' from tag '{element.name}'")
-                                queueid, attribute, element.name))
                         del(element.attrs[attribute])
         return soup
 
-    def notify(self, queueid, quarantine_id, mailfrom, recipients, headers, fp,
+    def notify(self, qid, storage_id, mailfrom, recipients, headers, fp,
                subgroups=None, named_subgroups=None, synchronous=False):
         "Notify recipients via email."
         super(
             EMailNotification,
             self).notify(
-            queueid,
+            qid,
-            quarantine_id,
+            storage_id,
             mailfrom,
             recipients,
             headers,
@@ -308,76 +304,55 @@ class EMailNotification(BaseNotification):
             synchronous)
 
         # extract body from email
-        content = self.get_decoded_email_body(
+        soup = self.get_email_body_soup(
-            queueid, email.message_from_binary_file(fp))
+            qid, email.message_from_binary_file(fp, policy=policy.default))
-
-        # create BeautifulSoup object
-        self.logger.debug(
-            "{}: trying to create BeatufilSoup object with parser lib {}, "
-            "text length is {} bytes".format(
-                queueid, self.parser_lib, len(content)))
-        soup = BeautifulSoup(content, self.parser_lib)
-        self.logger.debug(
-            "{}: sucessfully created BeautifulSoup object".format(queueid))
 
         # replace picture sources
         image_replaced = False
         if self.strip_images:
             self.logger.debug(
-                "{}: looking for images to strip".format(queueid))
+                f"{qid}: looking for images to strip")
             for element in soup("img"):
                 if "src" in element.attrs.keys():
                     self.logger.debug(
-                        "{}: strip image: {}".format(
+                        f"{qid}: strip image: {element['src']}")
-                            queueid, element["src"]))
                 element.extract()
         elif self.replacement_img:
             self.logger.debug(
-                "{}: looking for images to replace".format(queueid))
+                f"{qid}: looking for images to replace")
             for element in soup("img"):
                 if "src" in element.attrs.keys():
                     self.logger.debug(
-                        "{}: replacing image: {}".format(
+                        f"{qid}: replacing image: {element['src']}")
-                            queueid, element["src"]))
                 element["src"] = "cid:removed_for_security_reasons"
                 image_replaced = True
 
         # sanitizing email text of original email
-        sanitized_text = self.sanitize(queueid, soup)
+        sanitized_text = self.sanitize(qid, soup)
         del soup
 
         # sending email notifications
         for recipient in recipients:
             self.logger.debug(
-                "{}: generating notification email for '{}'".format(
+                f"{qid}: generating notification email for '{recipient}'")
-                    queueid, recipient))
+            self.logger.debug(f"{qid}: parsing email template")
-            self.logger.debug("{}: parsing email template".format(queueid))
-            # decode some headers
-            decoded_headers = {}
-            for header in ["from", "to", "subject"]:
-                if header in headers:
-                    decoded_headers[header] = str(
-                        make_header(decode_header(headers[header])))
-                else:
-                    headers[header] = ""
-                    decoded_headers[header] = ""
 
             # generate dict containing all template variables
             variables = defaultdict(str,
                                     EMAIL_HTML_TEXT=sanitized_text,
-                                    EMAIL_FROM=escape(decoded_headers["from"]),
+                                    EMAIL_FROM=escape(headers["from"]),
                                     EMAIL_ENVELOPE_FROM=escape(mailfrom),
                                     EMAIL_ENVELOPE_FROM_URL=escape(quote(mailfrom)),
-                                    EMAIL_TO=escape(decoded_headers["to"]),
+                                    EMAIL_TO=escape(headers["to"]),
                                     EMAIL_ENVELOPE_TO=escape(recipient),
                                     EMAIL_ENVELOPE_TO_URL=escape(quote(recipient)),
-                                    EMAIL_SUBJECT=escape(decoded_headers["subject"]),
+                                    EMAIL_SUBJECT=escape(headers["subject"]),
-                                    EMAIL_QUARANTINE_ID=quarantine_id)
+                                    EMAIL_QUARANTINE_ID=storage_id)
 
             if subgroups:
                 number = 0
                 for subgroup in subgroups:
-                    variables["SUBGROUP_{}".format(number)] = escape(subgroup)
+                    variables[f"SUBGROUP_{number}"] = escape(subgroup)
             if named_subgroups:
                 for key, value in named_subgroups.items():
                     named_subgroups[key] = escape(value)
@@ -397,26 +372,24 @@ class EMailNotification(BaseNotification):
 
             if image_replaced:
                 self.logger.debug(
-                    "{}: attaching notification_replacement_img".format(queueid))
+                    f"{qid}: attaching notification_replacement_img")
                 msg.attach(self.replacement_img)
 
             for img in self.embedded_imgs:
-                self.logger.debug("{}: attaching imgage".format(queueid))
+                self.logger.debug(f"{qid}: attaching imgage")
                 msg.attach(img)
 
             self.logger.debug(
-                "{}: sending notification email to: {}".format(
+                f"{qid}: sending notification email to: {recipient}")
-                    queueid, recipient))
             if synchronous:
                 try:
                     mailer.smtp_send(self.smtp_host, self.smtp_port,
                                      self.mailfrom, recipient, msg.as_string())
                 except Exception as e:
                     raise RuntimeError(
-                        "error while sending email to '{}': {}".format(
+                        f"error while sending email to '{recipient}': {e}")
-                            recipient, e))
             else:
-                mailer.sendmail(self.smtp_host, self.smtp_port, queueid,
+                mailer.sendmail(self.smtp_host, self.smtp_port, qid,
                                 self.mailfrom, recipient, msg.as_string(),
                                 "notification email")
 

pyquarantine/quarantines.py

@@ -1,309 +0,0 @@
-# PyQuarantine-Milter is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# PyQuarantine-Milter is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with PyQuarantineMilter.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-import json
-import logging
-import os
-
-from calendar import timegm
-from datetime import datetime
-from glob import glob
-from shutil import copyfileobj
-from time import gmtime
-
-from pyquarantine import mailer
-
-
-class BaseQuarantine(object):
-    "Quarantine base class"
-
-    def __init__(self, global_config, config, configtest=False):
-        self.name = config["name"]
-        self.global_config = global_config
-        self.config = config
-        self.logger = logging.getLogger(__name__)
-
-    def add(self, queueid, mailfrom, recipients, headers,
-            fp, subgroups=None, named_subgroups=None):
-        "Add email to quarantine."
-        fp.seek(0)
-        return ""
-
-    def find(self, mailfrom=None, recipients=None, older_than=None):
-        "Find emails in quarantine."
-        return
-
-    def get_metadata(self, quarantine_id):
-        "Return metadata of quarantined email."
-        return
-
-    def delete(self, quarantine_id, recipient=None):
-        "Delete email from quarantine."
-        return
-
-    def notify(self, quarantine_id, recipient=None):
-        "Notify recipient about email in quarantine."
-        if not self.config["notification_obj"]:
-            raise RuntimeError(
-                "notification type is set to None, unable to send notifications")
-        return
-
-    def release(self, quarantine_id, recipient=None):
-        "Release email from quarantine."
-        return
-
-
-class FileQuarantine(BaseQuarantine):
-    "Quarantine class to store mails on filesystem."
-
-    def __init__(self, global_config, config, configtest=False):
-        super(FileQuarantine, self).__init__(global_config, config, configtest)
-
-        # check if mandatory options are present in config
-        for option in ["quarantine_directory"]:
-            if option not in self.config.keys() and option in self.global_config.keys():
-                self.config[option] = self.global_config[option]
-            if option not in self.config.keys():
-                raise RuntimeError(
-                    "mandatory option '{}' not present in config section '{}' or 'global'".format(
-                        option, self.name))
-        self.directory = self.config["quarantine_directory"]
-
-        # check if quarantine directory exists and is writable
-        if not os.path.isdir(self.directory) or not os.access(
-                self.directory, os.W_OK):
-            raise RuntimeError(
-                "file quarantine directory '{}' does not exist or is not writable".format(
-                    self.directory))
-        self._metadata_suffix = ".metadata"
-
-    def _save_datafile(self, quarantine_id, fp):
-        datafile = os.path.join(self.directory, quarantine_id)
-        try:
-            with open(datafile, "wb") as f:
-                copyfileobj(fp, f)
-        except IOError as e:
-            raise RuntimeError("unable save data file: {}".format(e))
-
-    def _save_metafile(self, quarantine_id, metadata):
-        metafile = os.path.join(
-            self.directory, "{}{}".format(
-                quarantine_id, self._metadata_suffix))
-        try:
-            with open(metafile, "w") as f:
-                json.dump(metadata, f, indent=2)
-        except IOError as e:
-            raise RuntimeError("unable to save metadata file: {}".format(e))
-
-    def _remove(self, quarantine_id):
-        datafile = os.path.join(self.directory, quarantine_id)
-        metafile = "{}{}".format(datafile, self._metadata_suffix)
-
-        try:
-            os.remove(metafile)
-        except IOError as e:
-            raise RuntimeError("unable to remove metadata file: {}".format(e))
-
-        try:
-            os.remove(datafile)
-        except IOError as e:
-            raise RuntimeError("unable to remove data file: {}".format(e))
-
-    def add(self, queueid, mailfrom, recipients, headers,
-            fp, subgroups=None, named_subgroups=None):
-        "Add email to file quarantine and return quarantine-id."
-        super(
-            FileQuarantine,
-            self).add(
-            queueid,
-            mailfrom,
-            recipients,
-            headers,
-            fp,
-            subgroups,
-            named_subgroups)
-        quarantine_id = "{}_{}".format(
-            datetime.now().strftime("%Y%m%d%H%M%S"), queueid)
-
-        # save mail
-        self._save_datafile(quarantine_id, fp)
-
-        # save metadata
-        metadata = {
-            "mailfrom": mailfrom,
-            "recipients": recipients,
-            "headers": headers,
-            "date": timegm(gmtime()),
-            "queue_id": queueid,
-            "subgroups": subgroups,
-            "named_subgroups": named_subgroups
-        }
-        try:
-            self._save_metafile(quarantine_id, metadata)
-        except RuntimeError as e:
-            datafile = os.path.join(self.directory, quarantine_id)
-            os.remove(datafile)
-            raise e
-
-        return quarantine_id
-
-    def get_metadata(self, quarantine_id):
-        "Return metadata of quarantined email."
-        super(FileQuarantine, self).get_metadata(quarantine_id)
-
-        metafile = os.path.join(
-            self.directory, "{}{}".format(
-                quarantine_id, self._metadata_suffix))
-        if not os.path.isfile(metafile):
-            raise RuntimeError(
-                "invalid quarantine id '{}'".format(quarantine_id))
-
-        try:
-            with open(metafile, "r") as f:
-                metadata = json.load(f)
-        except IOError as e:
-            raise RuntimeError("unable to read metadata file: {}".format(e))
-        except json.JSONDecodeError as e:
-            raise RuntimeError(
-                "invalid meta file '{}': {}".format(
-                    metafile, e))
-
-        return metadata
-
-    def find(self, mailfrom=None, recipients=None, older_than=None):
-        "Find emails in quarantine."
-        super(FileQuarantine, self).find(mailfrom, recipients, older_than)
-        if isinstance(mailfrom, str):
-            mailfrom = [mailfrom]
-        if isinstance(recipients, str):
-            recipients = [recipients]
-
-        emails = {}
-        metafiles = glob(os.path.join(
-            self.directory, "*{}".format(self._metadata_suffix)))
-        for metafile in metafiles:
-            if not os.path.isfile(metafile):
-                continue
-
-            quarantine_id = os.path.basename(
-                metafile[:-len(self._metadata_suffix)])
-            metadata = self.get_metadata(quarantine_id)
-            if older_than is not None:
-                if timegm(gmtime()) - metadata["date"] < (older_than * 86400):
-                    continue
-
-            if mailfrom is not None:
-                if metadata["mailfrom"] not in mailfrom:
-                    continue
-
-            if recipients is not None:
-                if len(recipients) == 1 and \
-                        recipients[0] not in metadata["recipients"]:
-                    continue
-                elif len(set(recipients + metadata["recipients"])) == len(recipients + metadata["recipients"]):
-                    continue
-
-            emails[quarantine_id] = metadata
-
-        return emails
-
-    def delete(self, quarantine_id, recipient=None):
-        "Delete email in quarantine."
-        super(FileQuarantine, self).delete(quarantine_id, recipient)
-
-        try:
-            metadata = self.get_metadata(quarantine_id)
-        except RuntimeError as e:
-            raise RuntimeError("unable to delete email: {}".format(e))
-
-        if recipient is None:
-            self._remove(quarantine_id)
-        else:
-            if recipient not in metadata["recipients"]:
-                raise RuntimeError("invalid recipient '{}'".format(recipient))
-
-            metadata["recipients"].remove(recipient)
-            if not metadata["recipients"]:
-                self._remove(quarantine_id)
-            else:
-                self._save_metafile(quarantine_id, metadata)
-
-    def notify(self, quarantine_id, recipient=None):
-        "Notify recipient about email in quarantine."
-        super(FileQuarantine, self).notify(quarantine_id, recipient)
-
-        try:
-            metadata = self.get_metadata(quarantine_id)
-        except RuntimeError as e:
-            raise RuntimeError("unable to release email: {}".format(e))
-
-        if recipient is not None:
-            if recipient not in metadata["recipients"]:
-                raise RuntimeError("invalid recipient '{}'".format(recipient))
-            recipients = [recipient]
-        else:
-            recipients = metadata["recipients"]
-
-        datafile = os.path.join(self.directory, quarantine_id)
-        try:
-            with open(datafile, "rb") as fp:
-                self.config["notification_obj"].notify(
-                    metadata["queue_id"], quarantine_id, metadata["mailfrom"],
-                    recipients, metadata["headers"], fp,
-                    metadata["subgroups"], metadata["named_subgroups"],
-                    synchronous=True)
-        except IOError as e:
-            raise RuntimeError
-
-    def release(self, quarantine_id, recipient=None):
-        "Release email from quarantine."
-        super(FileQuarantine, self).release(quarantine_id, recipient)
-
-        try:
-            metadata = self.get_metadata(quarantine_id)
-        except RuntimeError as e:
-            raise RuntimeError("unable to release email: {}".format(e))
-
-        if recipient is not None:
-            if recipient not in metadata["recipients"]:
-                raise RuntimeError("invalid recipient '{}'".format(recipient))
-            recipients = [recipient]
-        else:
-            recipients = metadata["recipients"]
-
-        datafile = os.path.join(self.directory, quarantine_id)
-        try:
-            with open(datafile, "rb") as f:
-                mail = f.read()
-        except IOError as e:
-            raise RuntimeError("unable to read data file: {}".format(e))
-
-        for recipient in recipients:
-            try:
-                mailer.smtp_send(
-                    self.config["smtp_host"],
-                    self.config["smtp_port"],
-                    metadata["mailfrom"],
-                    recipient,
-                    mail)
-            except Exception as e:
-                raise RuntimeError(
-                    "error while sending email to '{}': {}".format(
-                        recipient, e))
-
-            self.delete(quarantine_id, recipient)
-
-
-# list of quarantine types and their related quarantine classes
-TYPES = {"file": FileQuarantine}

pyquarantine/run.py

@@ -35,11 +35,11 @@ def main():
         "-c", "--config",
         help="List of config files to read.",
         nargs="+",
-        default=pyquarantine.QuarantineMilter.get_configfiles())
+        default=pyquarantine.QuarantineMilter.get_cfg_files())
     parser.add_argument(
         "-s", "--socket",
         help="Socket used to communicate with the MTA.",
-        required=True)
+        default="inet:8899@127.0.0.1")
     parser.add_argument(
         "-d", "--debug",
         help="Log debugging messages.",
@@ -52,7 +52,7 @@ def main():
         "-v", "--version",
         help="Print version.",
         action="version",
-        version="%(prog)s ({})".format(version))
+        version=f"%(prog)s ({version})")
     args = parser.parse_args()
 
     # setup logging
@@ -61,24 +61,24 @@ def main():
     syslog_name = logname
     if args.debug:
         loglevel = logging.DEBUG
-        logname = "{}[%(name)s]".format(logname)
+        logname = f"{logname}[%(name)s]"
-        syslog_name = "{}: [%(name)s] %(levelname)s".format(syslog_name)
+        syslog_name = f"{syslog_name}: [%(name)s] %(levelname)s"
 
     # set config files for milter class
-    pyquarantine.QuarantineMilter.set_configfiles(args.config)
+    pyquarantine.QuarantineMilter.set_cfg_files(args.config)
     root_logger = logging.getLogger()
     root_logger.setLevel(loglevel)
 
     # setup console log
     stdouthandler = logging.StreamHandler(sys.stdout)
     stdouthandler.setLevel(logging.DEBUG)
-    formatter = logging.Formatter("%(message)s".format(logname))
+    formatter = logging.Formatter("%(message)s")
     stdouthandler.setFormatter(formatter)
     root_logger.addHandler(stdouthandler)
     logger = logging.getLogger(__name__)
     if args.test:
         try:
-            pyquarantine.generate_milter_config(args.test)
+            pyquarantine.setup_milter(test=args.test)
             print("Configuration ok")
         except RuntimeError as e:
             logger.error(e)
@@ -86,7 +86,7 @@ def main():
         else:
             sys.exit(0)
     formatter = logging.Formatter(
-        "%(asctime)s {}: [%(levelname)s] %(message)s".format(logname),
+        f"%(asctime)s {logname}: [%(levelname)s] %(message)s",
         datefmt="%Y-%m-%d %H:%M:%S")
     stdouthandler.setFormatter(formatter)
 
@@ -94,25 +94,21 @@ def main():
     sysloghandler = logging.handlers.SysLogHandler(
         address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL)
     sysloghandler.setLevel(loglevel)
-    formatter = logging.Formatter("{}: %(message)s".format(syslog_name))
+    formatter = logging.Formatter(f"{syslog_name}: %(message)s")
     sysloghandler.setFormatter(formatter)
     root_logger.addHandler(sysloghandler)
 
     logger.info("PyQuarantine-Milter starting")
     try:
         # generate milter config
-        global_config, config = pyquarantine.generate_milter_config()
+        pyquarantine.setup_milter()
     except RuntimeError as e:
         logger.error(e)
         sys.exit(255)
 
-    pyquarantine.QuarantineMilter.global_config = global_config
-    pyquarantine.QuarantineMilter.config = config
-
     # register to have the Milter factory create instances of your class:
     Milter.factory = pyquarantine.QuarantineMilter
     Milter.set_exception_policy(Milter.TEMPFAIL)
-    # Milter.set_flags(0)       # tell sendmail which features we use
 
     # run milter
     rc = 0
@@ -122,6 +118,7 @@ def main():
     except Milter.milter.error as e:
         logger.error(e)
         rc = 255
+
     pyquarantine.mailer.queue.put(None)
     logger.info("PyQuarantine-Milter terminated")
     sys.exit(rc)

pyquarantine/storages.py

@@ -0,0 +1,249 @@
+# PyQuarantine-Milter is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# PyQuarantine-Milter is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PyQuarantineMilter.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import json
+import logging
+import os
+
+from calendar import timegm
+from datetime import datetime
+from glob import glob
+from shutil import copyfileobj
+from time import gmtime
+
+
+class BaseMailStorage(object):
+    "Mail storage base class"
+    storage_type = "base"
+
+    def __init__(self, name, global_cfg, cfg, test=False):
+        self.name = name
+        self.logger = logging.getLogger(__name__)
+
+    def add(self, qid, mailfrom, recipients, headers,
+            fp, subgroups=None, named_subgroups=None):
+        "Add email to storage."
+        fp.seek(0)
+        return ""
+
+    def find(self, mailfrom=None, recipients=None, older_than=None):
+        "Find emails in storage."
+        return
+
+    def get_metadata(self, storage_id):
+        "Return metadata of email in storage."
+        return
+
+    def delete(self, storage_id, recipients=None):
+        "Delete email from storage."
+        return
+
+    def get_mail(self, storage_id):
+        "Return a file pointer to the email and metadata."
+        return
+
+
+class FileMailStorage(BaseMailStorage):
+    "Storage class to store mails on filesystem."
+    storage_type = "file"
+
+    def __init__(self, name, global_cfg, cfg, test=False):
+        super(FileMailStorage, self).__init__(name, global_cfg, cfg, test)
+
+        defaults = {}
+        # check config
+
+        for opt in ["storage_directory"] + list(defaults.keys()):
+            if opt in cfg:
+                continue
+            if opt in global_cfg:
+                cfg[opt] = global_cfg[opt]
+            elif opt in defaults:
+                cfg[opt] = defaults[opt]
+            else:
+                raise RuntimeError(
+                    f"mandatory option '{opt}' not present in config section '{self.name}' or 'global'")
+        self.directory = cfg["storage_directory"]
+
+        # check if quarantine directory exists and is writable
+        if not os.path.isdir(self.directory) or not os.access(
+                self.directory, os.W_OK):
+            raise RuntimeError(
+                f"file quarantine directory '{self.directory}' does not exist or is not writable")
+        self._metadata_suffix = ".metadata"
+
+    def _save_datafile(self, storage_id, fp):
+        datafile = os.path.join(self.directory, storage_id)
+        try:
+            with open(datafile, "wb") as f:
+                copyfileobj(fp, f)
+        except IOError as e:
+            raise RuntimeError(f"unable save data file: {e}")
+
+    def _save_metafile(self, storage_id, metadata):
+        metafile = os.path.join(
+            self.directory, f"{storage_id}{self._metadata_suffix}")
+        try:
+            with open(metafile, "w") as f:
+                json.dump(metadata, f, indent=2)
+        except IOError as e:
+            raise RuntimeError(f"unable to save metadata file: {e}")
+
+    def _remove(self, storage_id):
+        datafile = os.path.join(self.directory, storage_id)
+        metafile = f"{datafile}{self._metadata_suffix}"
+
+        try:
+            os.remove(metafile)
+        except IOError as e:
+            raise RuntimeError(f"unable to remove metadata file: {e}")
+
+        try:
+            os.remove(datafile)
+        except IOError as e:
+            raise RuntimeError(f"unable to remove data file: {e}")
+
+    def add(self, qid, mailfrom, recipients, headers,
+            fp, subgroups=None, named_subgroups=None):
+        "Add email to file storage and return storage id."
+        super(
+            FileMailStorage,
+            self).add(
+            qid,
+            mailfrom,
+            recipients,
+            headers,
+            fp,
+            subgroups,
+            named_subgroups)
+        timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
+        storage_id = f"{timestamp}_{qid}"
+
+        # save mail
+        self._save_datafile(storage_id, fp)
+
+        # save metadata
+        metadata = {
+            "mailfrom": mailfrom,
+            "recipients": recipients,
+            "headers": headers,
+            "date": timegm(gmtime()),
+            "queue_id": qid,
+            "subgroups": subgroups,
+            "named_subgroups": named_subgroups
+        }
+        try:
+            self._save_metafile(storage_id, metadata)
+        except RuntimeError as e:
+            datafile = os.path.join(self.directory, storage_id)
+            os.remove(datafile)
+            raise e
+
+        return storage_id
+
+    def get_metadata(self, storage_id):
+        "Return metadata of email in storage."
+        super(FileMailStorage, self).get_metadata(storage_id)
+
+        metafile = os.path.join(
+            self.directory, f"{storage_id}{self._metadata_suffix}")
+        if not os.path.isfile(metafile):
+            raise RuntimeError(
+                f"invalid storage id '{storage_id}'")
+
+        try:
+            with open(metafile, "r") as f:
+                metadata = json.load(f)
+        except IOError as e:
+            raise RuntimeError(f"unable to read metadata file: {e}")
+        except json.JSONDecodeError as e:
+            raise RuntimeError(
+                f"invalid metafile '{metafile}': {e}")
+
+        return metadata
+
+    def find(self, mailfrom=None, recipients=None, older_than=None):
+        "Find emails in storage."
+        super(FileMailStorage, self).find(mailfrom, recipients, older_than)
+        if isinstance(mailfrom, str):
+            mailfrom = [mailfrom]
+        if isinstance(recipients, str):
+            recipients = [recipients]
+
+        emails = {}
+        metafiles = glob(os.path.join(
+            self.directory, f"*{self._metadata_suffix}"))
+        for metafile in metafiles:
+            if not os.path.isfile(metafile):
+                continue
+
+            storage_id = os.path.basename(
+                metafile[:-len(self._metadata_suffix)])
+            metadata = self.get_metadata(storage_id)
+            if older_than is not None:
+                if timegm(gmtime()) - metadata["date"] < (older_than * 86400):
+                    continue
+
+            if mailfrom is not None:
+                if metadata["mailfrom"] not in mailfrom:
+                    continue
+
+            if recipients is not None:
+                if len(recipients) == 1 and \
+                        recipients[0] not in metadata["recipients"]:
+                    continue
+                elif len(set(recipients + metadata["recipients"])) == len(recipients + metadata["recipients"]):
+                    continue
+
+            emails[storage_id] = metadata
+
+        return emails
+
+    def delete(self, storage_id, recipients=None):
+        "Delete email from storage."
+        super(FileMailStorage, self).delete(storage_id, recipients)
+
+        try:
+            metadata = self.get_metadata(storage_id)
+        except RuntimeError as e:
+            raise RuntimeError(f"unable to delete email: {e}")
+
+        if not recipients:
+            self._remove(storage_id)
+        else:
+            if type(recipients) == str:
+                recipients = [recipients]
+            for recipient in recipients:
+                if recipient not in metadata["recipients"]:
+                    raise RuntimeError(f"invalid recipient '{recipient}'")
+                metadata["recipients"].remove(recipient)
+                if not metadata["recipients"]:
+                    self._remove(storage_id)
+                else:
+                    self._save_metafile(storage_id, metadata)
+
+    def get_mail(self, storage_id):
+        super(FileMailStorage, self).get_mail(storage_id)
+
+        metadata = self.get_metadata(storage_id)
+        datafile = os.path.join(self.directory, storage_id)
+        try:
+            fp = open(datafile, "rb")
+        except IOError as e:
+            raise RuntimeError(f"unable to open email data file: {e}")
+        return (fp, metadata)
+
+
+# list of storage types and their related storage classes
+TYPES = {"file": FileMailStorage}

pyquarantine/version.py

@@ -1 +1 @@
-__version__ = "0.0.9"
+__version__ = "1.0.2"

pyquarantine/whitelists.py

@@ -24,14 +24,14 @@ from playhouse.db_url import connect
 class WhitelistBase(object):
     "Whitelist base class"
 
-    def __init__(self, global_config, config, configtest=False):
+    whitelist_type = "base"
-        self.global_config = global_config
+
-        self.config = config
+    def __init__(self, name, global_cfg, cfg, test=False):
-        self.configtest = configtest
+        self.name = name
-        self.name = config["name"]
+        self.test = test
         self.logger = logging.getLogger(__name__)
         self.valid_entry_regex = re.compile(
-            r"^[a-zA-Z0-9_.+-]*?(@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)?$")
+            r"^[a-zA-Z0-9_.=+-]*?(@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)?$")
 
     def check(self, mailfrom, recipient):
         "Check if mailfrom/recipient combination is whitelisted."
@@ -73,44 +73,50 @@ class Meta(object):
 
 class DatabaseWhitelist(WhitelistBase):
     "Whitelist class to store whitelist in a database"
+    whitelist_type = "db"
     _db_connections = {}
     _db_tables = {}
 
-    def __init__(self, global_config, config, configtest=False):
+    def __init__(self, name, global_cfg, cfg, test=False):
         super(
             DatabaseWhitelist,
             self).__init__(
-            global_config,
+            global_cfg,
-            config,
+            cfg,
-            configtest)
+            test)
 
-        # check if mandatory options are present in config
+        defaults = {}
-        for option in ["whitelist_db_connection", "whitelist_db_table"]:
+
-            if option not in self.config.keys() and option in self.global_config.keys():
+        # check config
-                self.config[option] = self.global_config[option]
+        for opt in ["whitelist_db_connection", "whitelist_db_table"] + list(defaults.keys()):
-            if option not in self.config.keys():
+            if opt in cfg:
+                continue
+            if opt in global_cfg:
+                cfg[opt] = global_cfg[opt]
+            elif opt in defaults:
+                cfg[opt] = defaults[opt]
+            else:
                 raise RuntimeError(
-                    "mandatory option '{}' not present in config section '{}' or 'global'".format(
+                    f"mandatory option '{opt}' not present in config section '{self.name}' or 'global'")
-                        option, self.name))
 
-        tablename = self.config["whitelist_db_table"]
+        tablename = cfg["whitelist_db_table"]
-        connection_string = self.config["whitelist_db_connection"]
+        connection_string = cfg["whitelist_db_connection"]
 
         if connection_string in DatabaseWhitelist._db_connections.keys():
             db = DatabaseWhitelist._db_connections[connection_string]
         else:
             try:
                 # connect to database
-                self.logger.debug(
+                conn = re.sub(
-                    "connecting to database '{}'".format(
-                        re.sub(
                     r"(.*?://.*?):.*?(@.*)",
                     r"\1:<PASSWORD>\2",
-                            connection_string)))
+                    connection_string)
+                self.logger.debug(
+                    f"connecting to database '{conn}'")
                 db = connect(connection_string)
             except Exception as e:
                 raise RuntimeError(
-                    "unable to connect to database: {}".format(e))
+                    f"unable to connect to database: {e}")
 
             DatabaseWhitelist._db_connections[connection_string] = db
 
@@ -118,7 +124,7 @@ class DatabaseWhitelist(WhitelistBase):
         self.meta = Meta
         self.meta.database = db
         self.meta.table_name = tablename
-        self.model = type("WhitelistModel_{}".format(self.name), (WhitelistModel,), {
+        self.model = type(f"WhitelistModel_{self.name}", (WhitelistModel,), {
             "Meta": self.meta
         })
 
@@ -127,13 +133,12 @@ class DatabaseWhitelist(WhitelistBase):
 
         if tablename not in DatabaseWhitelist._db_tables[connection_string]:
             DatabaseWhitelist._db_tables[connection_string].append(tablename)
-            if not self.configtest:
+            if not self.test:
                 try:
                     db.create_tables([self.model])
                 except Exception as e:
                     raise RuntimeError(
-                        "unable to initialize table '{}': {}".format(
+                        f"unable to initialize table '{tablename}': {e}")
-                            tablename, e))
 
     def _entry_to_dict(self, entry):
         result = {}
@@ -163,17 +168,18 @@ class DatabaseWhitelist(WhitelistBase):
 
         # generate list of possible mailfroms
         self.logger.debug(
-            "query database for whitelist entries from <{}> to <{}>".format(
+            f"query database for whitelist entries from <{mailfrom}> to <{recipient}>")
-                mailfrom, recipient))
         mailfroms = [""]
         if "@" in mailfrom and not mailfrom.startswith("@"):
-            mailfroms.append("@{}".format(mailfrom.split("@")[1]))
+            domain = mailfrom.split("@")[1]
+            mailfroms.append(f"@{domain}")
         mailfroms.append(mailfrom)
 
         # generate list of possible recipients
         recipients = [""]
         if "@" in recipient and not recipient.startswith("@"):
-            recipients.append("@{}".format(recipient.split("@")[1]))
+            domain = recipient.split("@")[1]
+            recipients.append(f"@{domain}")
         recipients.append(recipient)
 
         # query the database
@@ -183,7 +189,7 @@ class DatabaseWhitelist(WhitelistBase):
                     self.model.mailfrom.in_(mailfroms),
                     self.model.recipient.in_(recipients)))
         except Exception as e:
-            raise RuntimeError("unable to query database: {}".format(e))
+            raise RuntimeError(f"unable to query database: {e}")
 
         if not entries:
             # no whitelist entry found
@@ -228,7 +234,7 @@ class DatabaseWhitelist(WhitelistBase):
 
                 entries.update(self._entry_to_dict(entry))
         except Exception as e:
-            raise RuntimeError("unable to query database: {}".format(e))
+            raise RuntimeError(f"unable to query database: {e}")
 
         return entries
 
@@ -249,7 +255,7 @@ class DatabaseWhitelist(WhitelistBase):
                 comment=comment,
                 permanent=permanent)
         except Exception as e:
-            raise RuntimeError("unable to add entry to database: {}".format(e))
+            raise RuntimeError(f"unable to add entry to database: {e}")
 
     def delete(self, whitelist_id):
         "Delete entry from whitelist."
@@ -260,7 +266,7 @@ class DatabaseWhitelist(WhitelistBase):
             deleted = query.execute()
         except Exception as e:
             raise RuntimeError(
-                "unable to delete entry from database: {}".format(e))
+                f"unable to delete entry from database: {e}")
 
         if deleted == 0:
             raise RuntimeError("invalid whitelist id")

setup.py

@@ -22,7 +22,7 @@ setup(name = "pyquarantine",
         #   3 - Alpha
         #   4 - Beta
         #   5 - Production/Stable
-        "Development Status :: 3 - Alpha",
+        "Development Status :: 4 - Beta",
         "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
         "Operating System :: OS Independent",
         "Programming Language :: Python",
@@ -36,5 +36,5 @@ setup(name = "pyquarantine",
         ]
     },
     install_requires = ["pymilter", "netaddr", "beautifulsoup4[lxml]", "peewee"],
-    python_requires = ">=3"
+    python_requires = ">=3.6"
 )