Change README.md

README.md

@@ -8,7 +8,7 @@ Each quarantine can be configured with a quarantine type, notification type, whi
 
 Addionally, pyquarantine-milter provides a sanitized, harmless version of the text parts of e-mails, which can be embedded in e-mail notifications. This makes it easier for users to decide, if a match is a false-positive or not. If a matching quarantine provides a quarantine ID of the original e-mail, it is also available as a template variable. This is useful if you want to add links to a webservice to notification e-mails, to give your users the ability to release e-mails or whitelist the from-address for example. The webservice then releases the e-mail from the quarantine.
 
-The project is currently in alpha status, but will soon be used in a productive enterprise environment and possibly existing bugs will be fixed.
+The project is currently in beta status, but it is already used in a productive enterprise environment which processes about a million e-mails per month.
 
 ## Requirements
 * pymilter <https://pythonhosted.org/pymilter/>

pyquarantine/__init__.py

@@ -36,6 +36,8 @@ import sys
 
 from Milter.utils import parse_addr
 from collections import defaultdict
+from email.policy import default as default_policy
+from email.parser import BytesHeaderParser
 from io import BytesIO
 from itertools import groupby
 from netaddr import IPAddress, IPNetwork
@@ -96,13 +98,11 @@ class Quarantine(object):
                 cfg[opt] = defaults[opt]
             else:
                 raise RuntimeError(
-                    "mandatory option '{}' not present in config section '{}' or 'global'".format(
+                    f"mandatory option '{opt}' not present in config section '{self.name}' or 'global'")
-                        opt, self.name))
 
         # pre-compile regex
         self.logger.debug(
-            "{}: compiling regex '{}'".format(
+            f"{self.name}: compiling regex '{cfg['regex']}'")
-                self.name, cfg["regex"]))
         self.regex = re.compile(
             cfg["regex"], re.MULTILINE + re.DOTALL + re.IGNORECASE)
 
@@ -113,62 +113,53 @@ class Quarantine(object):
         storage_type = cfg["storage_type"].lower()
         if storage_type in storages.TYPES:
             self.logger.debug(
-                "{}: initializing storage type '{}'".format(
+                f"{self.name}: initializing storage type '{storage_type.upper()}'")
-                    self.name,
-                    storage_type.upper()))
             self.storage = storages.TYPES[storage_type](
                 self.name, global_cfg, cfg, test)
         elif storage_type == "none":
-            self.logger.debug("{}: storage is NONE".format(self.name))
+            self.logger.debug(f"{self.name}: storage is NONE")
             self.storage = None
         else:
             raise RuntimeError(
-                "{}: unknown storage type '{}'".format(
+                f"{self.name}: unknown storage type '{storage_type}'")
-                    self.name, storage_type))
 
         # create whitelist instance
         whitelist_type = cfg["whitelist_type"].lower()
         if whitelist_type in whitelists.TYPES:
             self.logger.debug(
-                "{}: initializing whitelist type '{}'".format(
+                f"{self.name}: initializing whitelist type '{whitelist_type.upper()}'")
-                    self.name,
-                    whitelist_type.upper()))
             self.whitelist = whitelists.TYPES[whitelist_type](
                self.name, global_cfg, cfg, test)
         elif whitelist_type == "none":
-            logger.debug("{}: whitelist is NONE".format(self.name))
+            logger.debug(f"{self.name}: whitelist is NONE")
             self.whitelist = None
         else:
             raise RuntimeError(
-                "{}: unknown whitelist type '{}'".format(
+                f"{self.name}: unknown whitelist type '{whitelist_type}'")
-                    self.name, whitelist_type))
 
         # create notification instance
         notification_type = cfg["notification_type"].lower()
         if notification_type in notifications.TYPES:
             self.logger.debug(
-                "{}: initializing notification type '{}'".format(
+                f"{self.name}: initializing notification type '{notification_type.upper()}'")
-                    self.name,
-                    notification_type.upper()))
             self.notification = notifications.TYPES[notification_type](
                 self.name, global_cfg, cfg, test)
         elif notification_type == "none":
-            self.logger.debug("{}: notification is NONE".format(self.name))
+            self.logger.debug(f"{self.name}: notification is NONE")
             self.notification = None
         else:
             raise RuntimeError(
-                "{}: unknown notification type '{}'".format(
+                f"{self.name}: unknown notification type '{notification_type}'")
-                    self.name, notification_type))
 
         # determining milter action for this quarantine
         action = cfg["action"].upper()
         if action in self._actions:
-            self.logger.debug("{}: action is {}".format(self.name, action))
+            self.logger.debug(f"{self.name}: action is {action}")
             self.action = action
             self.milter_action = self._actions[action]
         else:
             raise RuntimeError(
-                "{}: unknown action '{}'".format(self._name, action))
+                f"{self.name}: unknown action '{action}'")
 
         self.reject_reason = cfg["reject_reason"]
 
@@ -183,15 +174,13 @@ class Quarantine(object):
             try:
                 net = IPNetwork(host)
             except AddrFormatError as e:
-                raise RuntimeError("{}: error parsing host_whitelist: {}".format(
+                raise RuntimeError(f"{self.name}: error parsing host_whitelist: {e}")
-                    self.name, e))
             else:
                 self.host_whitelist.append(net)
         if self.host_whitelist:
+            whitelist = ", ".join([str(ip) for ip in host_whitelist])
             self.logger.debug(
-                "{}: host whitelist: {}".format(
+                f"{self.name}: host whitelist: {whitelist}")
-                    self.name,
-                    ", ".join([str(ip) for ip in host_whitelist])))
 
     def notify(self, storage_id, recipient=None, synchronous=True):
         "Notify recipient about email in storage."
@@ -207,7 +196,7 @@ class Quarantine(object):
 
         if recipient is not None:
             if recipient not in metadata["recipients"]:
-                raise RuntimeError("invalid recipient '{}'".format(recipient))
+                raise RuntimeError(f"invalid recipient '{recipient}'")
             recipients = [recipient]
         else:
             recipients = metadata["recipients"]
@@ -230,7 +219,7 @@ class Quarantine(object):
             mail = fp.read()
             fp.close()
         except IOError as e:
-            raise RuntimeError("unable to read data file: {}".format(e))
+            raise RuntimeError(f"unable to read data file: {e}")
 
         if recipients and type(recipients) == str:
             recipients = [recipients]
@@ -239,7 +228,7 @@ class Quarantine(object):
 
         for recipient in recipients:
             if recipient not in metadata["recipients"]:
-                raise RuntimeError("invalid recipient '{}'".format(recipient))
+                raise RuntimeError(f"invalid recipient '{recipient}'")
 
             try:
                 mailer.smtp_send(
@@ -250,8 +239,7 @@ class Quarantine(object):
                     mail)
             except Exception as e:
                 raise RuntimeError(
-                    "error while sending email to '{}': {}".format(
+                    f"error while sending email to '{recipient}': {e}")
-                        recipient, e))
             self.storage.delete(storage_id, recipient)
 
     def get_storage(self):
@@ -319,19 +307,16 @@ class QuarantineMilter(Milter.Base):
     def connect(self, hostname, family, hostaddr):
         self.hostaddr = hostaddr
         self.logger.debug(
-            "accepted milter connection from {} port {}".format(
+            f"accepted milter connection from {hostaddr[0]} port {hostaddr[1]}")
-                *hostaddr))
         for quarantine in self.quarantines.copy():
             if quarantine.host_in_whitelist(hostaddr):
                 self.logger.debug(
-                    "host {} is in whitelist of quarantine {}".format(
+                    f"host {hostaddr[0]} is in whitelist of quarantine {quarantine['name']}")
-                        hostaddr[0], quarantine["name"]))
                 self.quarantines.remove(quarantine)
                 if not self.quarantines:
                     self.logger.debug(
-                        "host {} is in whitelist of all quarantines, "
+                        f"host {hostaddr[0]} is in whitelist of all quarantines, "
-                        "skip further processing",
+                        f"skip further processing")
-                        hostaddr[0])
                     return Milter.ACCEPT
         return Milter.CONTINUE
 
@@ -348,14 +333,12 @@ class QuarantineMilter(Milter.Base):
 
     @Milter.noreply
     def data(self):
-        self.queueid = self.getsymval('i')
+        self.qid = self.getsymval('i')
         self.logger.debug(
-            "{}: received queue-id from MTA".format(self.queueid))
+            f"{self.qid}: received queue-id from MTA")
         self.recipients = list(self.recipients)
-        self.headers = []
         self.logger.debug(
-            "{}: initializing memory buffer to save email data".format(
+            f"{self.qid}: initializing memory buffer to save email data")
-                self.queueid))
         # initialize memory buffer to save email data
         self.fp = BytesIO()
         return Milter.CONTINUE
@@ -364,15 +347,11 @@ class QuarantineMilter(Milter.Base):
     def header(self, name, value):
         try:
             # write email header to memory buffer
-            self.fp.write("{}: {}\r\n".format(name, value).encode(
+            self.fp.write(f"{name}: {value}\r\n".encode(
-                encoding="ascii", errors="surrogateescape"))
+                encoding="ascii", errors="replace"))
-            # keep copy of header without surrogates for later use
-            self.headers.append((
-                name.encode(errors="surrogateescape").decode(errors="replace"),
-                value.encode(errors="surrogateescape").decode(errors="replace")))
         except Exception as e:
             self.logger.exception(
-                "an exception occured in header function: {}".format(e))
+                f"an exception occured in header function: {e}")
             return Milter.TEMPFAIL
 
         return Milter.CONTINUE
@@ -380,6 +359,9 @@ class QuarantineMilter(Milter.Base):
     def eoh(self):
         try:
             self.fp.write("\r\n".encode(encoding="ascii"))
+            self.fp.seek(0)
+            self.headers = BytesHeaderParser(
+                policy=default_policy).parse(self.fp).items()
             self.whitelist_cache = whitelists.WhitelistCache()
 
             # initialize dicts to set quaranines per recipient and keep matches
@@ -389,10 +371,9 @@ class QuarantineMilter(Milter.Base):
             # iterate email headers
             recipients_to_check = self.recipients.copy()
             for name, value in self.headers:
-                header = "{}: {}".format(name, value)
+                header = f"{name}: {value}"
                 self.logger.debug(
-                    "{}: checking header against configured quarantines: {}".format(
+                    f"{self.qid}: checking header against configured quarantines: {header}")
-                        self.queueid, header))
                 # iterate quarantines
                 for quarantine in self.quarantines:
                     if len(self.recipients_quarantines) == len(
@@ -404,19 +385,16 @@ class QuarantineMilter(Milter.Base):
                             # the same precedence already, skip checks against
                             # quarantines with lower precedence
                             self.logger.debug(
-                                "{}: {}: skip further checks of this header".format(
+                                f"{self.qid}: {quarantine.name}: skip further checks of this header")
-                                    self.queueid, quarantine.name))
                             break
 
                     # check email header against quarantine regex
                     self.logger.debug(
-                        "{}: {}: checking header against regex '{}'".format(
+                        f"{self.qid}: {quarantine.name}: checking header against regex '{quarantine.regex}'")
-                            self.queueid, quarantine.name, quarantine.regex))
                     match = quarantine.match(header)
                     if match:
                         self.logger.debug(
-                            "{}: {}: header matched regex".format(
+                            f"{self.qid}: {quarantine.name}: header matched regex")
-                                self.queueid, quarantine.name))
                         # check for whitelisted recipients
                         whitelist = quarantine.get_whitelist()
                         if whitelist:
@@ -425,8 +403,7 @@ class QuarantineMilter(Milter.Base):
                                     whitelist, self.mailfrom, recipients_to_check)
                             except RuntimeError as e:
                                 self.logger.error(
-                                    "{}: {}: unable to query whitelist: {}".format(
+                                    f"{self.qid}: {quarantine.name}: unable to query whitelist: {e}")
-                                        self.queueid, quarantine.name, e))
                                 return Milter.TEMPFAIL
                         else:
                             whitelisted_recipients = {}
@@ -436,15 +413,13 @@ class QuarantineMilter(Milter.Base):
                             if recipient in whitelisted_recipients:
                                 # recipient is whitelisted in this quarantine
                                 self.logger.debug(
-                                    "{}: {}: recipient '{}' is whitelisted".format(
+                                    f"{self.qid}: {quarantine.name}: recipient '{recipient}' is whitelisted")
-                                        self.queueid, quarantine.name, recipient))
                                 continue
 
                             if recipient not in self.recipients_quarantines.keys() or \
                                     self.recipients_quarantines[recipient].index > quarantine.index:
                                 self.logger.debug(
-                                    "{}: {}: set quarantine for recipient '{}'".format(
+                                    f"{self.qid}: {quarantine.name}: set quarantine for recipient '{recipient}'")
-                                        self.queueid, quarantine.name, recipient))
                                 # save match for later use as template
                                 # variables
                                 self.quarantines_matches[quarantine.name] = match
@@ -456,23 +431,20 @@ class QuarantineMilter(Milter.Base):
                                     recipients_to_check.remove(recipient)
                             else:
                                 self.logger.debug(
-                                    "{}: {}: a quarantine with same or higher precedence "
+                                    f"{self.qid}: {quarantine.name}: a quarantine with same or higher "
-                                    "matched already for recipient '{}'".format(
+                                    f"precedence matched already for recipient '{recipient}'")
-                                        self.queueid, quarantine.name, recipient))
 
                 if not recipients_to_check:
                     self.logger.debug(
-                        "{}: all recipients matched the first quarantine, "
+                        f"{self.qid}: all recipients matched the first quarantine, "
-                        "skipping all remaining header checks".format(
+                        f"skipping all remaining header checks")
-                            self.queueid))
                     break
 
             # check if no quarantine has matched for all recipients
             if not self.recipients_quarantines:
                 # accept email
                 self.logger.info(
-                    "{}: passed clean for all recipients".format(
+                    f"{self.qid}: passed clean for all recipients")
-                        self.queueid))
                 return Milter.ACCEPT
 
             # check if the mail body is needed
@@ -485,18 +457,16 @@ class QuarantineMilter(Milter.Base):
             # quarantines, just return configured action
             quarantine = self._get_preferred_quarantine()
             self.logger.info(
-                "{}: {} matching quarantine is '{}', performing milter action {}".format(
+                f"{self.qid}: {self.preferred_quarantine_action} "
-                    self.queueid,
+                f"matching quarantine is '{quarantine.name}', performing "
-                    self.preferred_quarantine_action,
+                f"milter action {quarantine.action}")
-                    quarantine.name,
-                    quarantine.action))
             if quarantine.action == "REJECT":
                 self.setreply("554", "5.7.0", quarantine.reject_reason)
             return quarantine.milter_action
 
         except Exception as e:
             self.logger.exception(
-                "an exception occured in eoh function: {}".format(e))
+                f"an exception occured in eoh function: {e}")
             return Milter.TEMPFAIL
 
     def body(self, chunk):
@@ -505,7 +475,7 @@ class QuarantineMilter(Milter.Base):
             self.fp.write(chunk)
         except Exception as e:
             self.logger.exception(
-                "an exception occured in body function: {}".format(e))
+                f"an exception occured in body function: {e}")
             return Milter.TEMPFAIL
         return Milter.CONTINUE
 
@@ -530,21 +500,22 @@ class QuarantineMilter(Milter.Base):
                 named_subgroups = self.quarantines_matches[quarantine.name].groupdict(
                     default="")
 
+                rcpts = ", ".join(recipients)
+
                 # check if a storage is configured
                 storage_id = ""
                 storage = quarantine.get_storage()
                 if storage:
                     # add email to quarantine
-                    self.logger.info("{}: adding to quarantine '{}' for: {}".format(
+                    self.logger.info(
-                        self.queueid, quarantine.name, ", ".join(recipients)))
+                        f"{self.qid}: adding to quarantine '{quarantine.name}' for: {rcpts}")
                     try:
                         storage_id = storage.add(
-                            self.queueid, self.mailfrom, recipients, headers, self.fp,
+                            self.qid, self.mailfrom, recipients, headers, self.fp,
                             subgroups, named_subgroups)
                     except RuntimeError as e:
                         self.logger.error(
-                            "{}: unable to add to quarantine '{}': {}".format(
+                            f"{self.qid}: unable to add to quarantine '{quarantine.name}': {e}")
-                                self.queueid, quarantine.name, e))
                         return Milter.TEMPFAIL
 
                 # check if a notification is configured
@@ -552,17 +523,15 @@ class QuarantineMilter(Milter.Base):
                 if notification:
                     # notify
                     self.logger.info(
-                        "{}: sending notification for quarantine '{}' to: {}".format(
+                        f"{self.qid}: sending notification to: {rcpts}")
-                            self.queueid, quarantine.name, ", ".join(recipients)))
                     try:
                         notification.notify(
-                            self.queueid, storage_id,
+                            self.qid, storage_id,
                             self.mailfrom, recipients, headers, self.fp,
                             subgroups, named_subgroups)
                     except RuntimeError as e:
                         self.logger.error(
-                            "{}: unable to send notification for quarantine '{}': {}".format(
+                            f"{self.qid}: unable to send notification: {e}")
-                                self.queueid, quarantine.name, e))
                         return Milter.TEMPFAIL
 
                 # remove processed recipient
@@ -574,33 +543,29 @@ class QuarantineMilter(Milter.Base):
 
             # email passed clean for at least one recipient, accepting email
             if self.recipients:
+                rcpts = ", ".join(recipients)
                 self.logger.info(
-                    "{}: passed clean for: {}".format(
+                    f"{self.qid}: passed clean for: {rcpts}")
-                        self.queueid, ", ".join(
-                            self.recipients)))
                 return Milter.ACCEPT
 
             # return configured action
             quarantine = self._get_preferred_quarantine()
             self.logger.info(
-                "{}: {} matching quarantine is '{}', performing milter action {}".format(
+                f"{self.qid}: {self.preferred_quarantine_action} matching "
-                    self.queueid,
+                f"quarantine is '{quarantine.name}', performing milter "
-                    self.preferred_quarantine_action,
+                f"action {quarantine.action}")
-                    quarantine.name,
-                    quarantine.action))
             if quarantine.action == "REJECT":
                 self.setreply("554", "5.7.0", quarantine.reject_reason)
             return quarantine.milter_action
 
         except Exception as e:
             self.logger.exception(
-                "an exception occured in eom function: {}".format(e))
+                f"an exception occured in eom function: {e}")
             return Milter.TEMPFAIL
 
     def close(self):
         self.logger.debug(
-            "disconnect from {} port {}".format(
+            f"disconnect from {self.hostaddr[0]} port {self.hostaddr[1]}")
-                *self.hostaddr))
         return Milter.CONTINUE
 
 
@@ -627,7 +592,7 @@ def setup_milter(test=False, cfg_files=[]):
     for option in ["quarantines", "preferred_quarantine_action"]:
         if not parser.has_option("global", option):
             raise RuntimeError(
-                "mandatory option '{}' not present in config section 'global'".format(option))
+                f"mandatory option '{option}' not present in config section 'global'")
 
     # read global config section
     global_cfg = dict(parser.items("global"))
@@ -656,7 +621,7 @@ def setup_milter(test=False, cfg_files=[]):
         # check if config section for current quarantine exists
         if name not in parser.sections():
             raise RuntimeError(
-                "config section '{}' does not exist".format(name))
+                f"config section '{name}' does not exist")
 
         cfg = dict(parser.items(name))
         quarantine = Quarantine(name, index)

pyquarantine/cli.py

@@ -20,17 +20,16 @@ import logging.handlers
 import sys
 import time
 
-from email.header import decode_header, make_header
+from email.header import decode_header
-
-import pyquarantine
 
+from pyquarantine import QuarantineMilter, setup_milter
 from pyquarantine.version import __version__ as version
 
 def _get_quarantine(quarantines, name):
     try:
         quarantine = next((q for q in quarantines if q.name == name))
     except StopIteration:
-        raise RuntimeError("invalid quarantine '{}'".format(name))
+        raise RuntimeError(f"invalid quarantine 'name'")
     return quarantine
 
 def _get_storage(quarantines, name):
@@ -74,7 +73,7 @@ def print_table(columns, rows):
         # use the longer one
         length = max(lengths)
         column_lengths.append(length)
-        column_formats.append("{{:<{}}}".format(length))
+        column_formats.append(f"{{:<{length}}}")
 
     # define row format
     row_format = " | ".join(column_formats)
@@ -156,9 +155,7 @@ def list_quarantine_emails(quarantines, args):
         row["recipient"] = metadata["recipients"].pop(0)
         if "subject" not in emails[storage_id]["headers"].keys():
             emails[storage_id]["headers"]["subject"] = ""
-        row["subject"] = str(make_header(decode_header(
+        row["subject"] = emails[storage_id]["headers"]["subject"][:60].strip()
-            emails[storage_id]["headers"]["subject"])))[:60].replace(
-                "\r", "").replace("\n", "").strip()
         rows.append(row)
 
         if metadata["recipients"]:
@@ -177,7 +174,7 @@ def list_quarantine_emails(quarantines, args):
         return
 
     if not emails:
-        logger.info("quarantine '{}' is empty".format(args.quarantine))
+        logger.info(f"quarantine '{args.quarantine}' is empty")
     print_table(
         [("Quarantine-ID", "storage_id"), ("Date", "date"),
          ("From", "mailfrom"), ("Recipient(s)", "recipient"),
@@ -197,8 +194,7 @@ def list_whitelist(quarantines, args):
         older_than=args.older_than)
     if not entries:
         logger.info(
-            "whitelist of quarantine '{}' is empty".format(
+            f"whitelist of quarantine '{args.quarantine}' is empty")
-                args.quarantine))
         return
 
     # transform some values to strings
@@ -309,7 +305,7 @@ def main():
         "-c", "--config",
         help="Config files to read.",
         nargs="+", metavar="CFG",
-        default=pyquarantine.QuarantineMilter.get_cfg_files())
+        default=QuarantineMilter.get_cfg_files())
     parser.add_argument(
         "-d", "--debug",
         help="Log debugging messages.",
@@ -318,7 +314,7 @@ def main():
         "-v", "--version",
         help="Print version.",
         action="version",
-        version="%(prog)s ({})".format(version))
+        version=f"%(prog)s ({version})")
     parser.set_defaults(syslog=False)
     subparsers = parser.add_subparsers(
         dest="command",
@@ -563,7 +559,7 @@ def main():
 
     # try to generate milter configs
     try:
-        pyquarantine.setup_milter(
+        setup_milter(
             cfg_files=args.config, test=True)
     except RuntimeError as e:
         logger.error(e)
@@ -585,7 +581,7 @@ def main():
 
     # call the commands function
     try:
-        args.func(pyquarantine.QuarantineMilter.quarantines, args)
+        args.func(QuarantineMilter.quarantines, args)
     except RuntimeError as e:
         logger.error(e)
         sys.exit(1)

pyquarantine/mailer.py

@@ -45,23 +45,21 @@ def mailprocess():
             if not m:
                 break
 
-            smtp_host, smtp_port, queueid, mailfrom, recipient, mail, emailtype = m
+            smtp_host, smtp_port, qid, mailfrom, recipient, mail, emailtype = m
             try:
                 smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail)
             except Exception as e:
                 logger.error(
-                    "{}: error while sending {} to '{}': {}".format(
+                    f"{qid}: error while sending {emailtype} to '{recipient}': {e}")
-                        queueid, emailtype, recipient, e))
             else:
                 logger.info(
-                    "{}: successfully sent {} to: {}".format(
+                    f"{qid}: successfully sent {emailtype} to: {recipient}")
-                        queueid, emailtype, recipient))
     except KeyboardInterrupt:
         pass
     logger.debug("mailer process terminated")
 
 
-def sendmail(smtp_host, smtp_port, queueid, mailfrom, recipients, mail,
+def sendmail(smtp_host, smtp_port, qid, mailfrom, recipients, mail,
              emailtype="email"):
     "Send an email."
     global logger
@@ -81,7 +79,7 @@ def sendmail(smtp_host, smtp_port, queueid, mailfrom, recipients, mail,
     for recipient in recipients:
         try:
             queue.put(
-                (smtp_host, smtp_port, queueid, mailfrom, recipient, mail,
+                (smtp_host, smtp_port, qid, mailfrom, recipient, mail,
                  emailtype),
                 timeout=30)
         except Queue.Full as e:

pyquarantine/notifications.py

@@ -20,7 +20,7 @@ from bs4 import BeautifulSoup
 from cgi import escape
 from collections import defaultdict
 from email import policy
-from email.header import decode_header, make_header
+from email.header import decode_header
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 from email.mime.image import MIMEImage
@@ -38,7 +38,7 @@ class BaseNotification(object):
         self.name = name
         self.logger = logging.getLogger(__name__)
 
-    def notify(self, queueid, storage_id, mailfrom, recipients, headers,
+    def notify(self, qid, storage_id, mailfrom, recipients, headers,
                fp, subgroups=None, named_subgroups=None, synchronous=False):
         fp.seek(0)
         pass
@@ -136,8 +136,7 @@ class EMailNotification(BaseNotification):
                 cfg[opt] = defaults[opt]
             else:
                 raise RuntimeError(
-                    "mandatory option '{}' not present in config section '{}' or 'global'".format(
+                    f"mandatory option '{opt}' not present in config section '{self.name}' or 'global'")
-                        opt, self.name))
 
         self.smtp_host = cfg["notification_email_smtp_host"]
         self.smtp_port = cfg["notification_email_smtp_port"]
@@ -152,14 +151,14 @@ class EMailNotification(BaseNotification):
             self.from_header.format_map(testvars)
         except ValueError as e:
             raise RuntimeError(
-                "error parsing notification_email_from: {}".format(e))
+                f"error parsing notification_email_from: {e}")
 
         # test-parse subject
         try:
             self.subject.format_map(testvars)
         except ValueError as e:
             raise RuntimeError(
-                "error parsing notification_email_subject: {}".format(e))
+                f"error parsing notification_email_subject: {e}")
 
         # read and parse email notification template
         try:
@@ -167,9 +166,9 @@ class EMailNotification(BaseNotification):
                 cfg["notification_email_template"], "r").read()
             self.template.format_map(testvars)
         except IOError as e:
-            raise RuntimeError("error reading template: {}".format(e))
+            raise RuntimeError(f"error reading template: {e}")
         except ValueError as e:
-            raise RuntimeError("error parsing template: {}".format(e))
+            raise RuntimeError(f"error parsing template: {e}")
 
         strip_images = cfg["notification_email_strip_images"].strip().upper()
         if strip_images in ["TRUE", "ON", "YES"]:
@@ -191,7 +190,7 @@ class EMailNotification(BaseNotification):
                     open(replacement_img, "rb").read())
             except IOError as e:
                 raise RuntimeError(
-                    "error reading replacement image: {}".format(e))
+                    f"error reading replacement image: {e}")
             else:
                 self.replacement_img.add_header(
                     "Content-ID", "<removed_for_security_reasons>")
@@ -207,20 +206,20 @@ class EMailNotification(BaseNotification):
             try:
                 img = MIMEImage(open(img_path, "rb").read())
             except IOError as e:
-                raise RuntimeError("error reading image: {}".format(e))
+                raise RuntimeError(f"error reading image: {e}")
             else:
-                img.add_header("Content-ID", "<{}>".format(basename(img_path)))
+                filename = basename(img_path)
+                img.add_header(f"Content-ID", f"<{filename}>")
                 self.embedded_imgs.append(img)
 
-    def get_email_body_soup(self, queueid, msg):
+    def get_email_body_soup(self, qid, msg):
         "Extract and decode email body and return it as BeautifulSoup object."
         # try to find the body part
-        self.logger.debug("{}: trying to find email body".format(queueid))
+        self.logger.debug(f"{qid}: trying to find email body")
         try:
             body = msg.get_body(preferencelist=("html", "plain"))
         except Exception as e:
-            self.logger.error("{}: an error occured in email.message.EmailMessage.get_body: {}".format(
+            self.logger.error(f"{qid}: an error occured in email.message.EmailMessage.get_body: {e}")
-                queueid, e))
             body = None
 
         if body:
@@ -229,54 +228,49 @@ class EMailNotification(BaseNotification):
             try:
                 content = content.decode(encoding=charset, errors="replace")
             except LookupError:
-                self.logger.info("{}: unknown encoding '{}', falling back to UTF-8".format(
+                self.logger.info(f"{qid}: unknown encoding '{charset}', falling back to UTF-8")
-                    queueid, charset))
                 content = content.decode("utf-8", errors="replace")
             content_type = body.get_content_type()
             if content_type == "text/plain":
                 # convert text/plain to text/html
                 self.logger.debug(
-                    "{}: content type is {}, converting to text/html".format(
+                    f"{qid}: content type is {content_type}, converting to text/html")
-                        queueid, content_type))
                 content = re.sub(r"^(.*)$", r"\1<br/>",
                                  escape(content), flags=re.MULTILINE)
             else:
                 self.logger.debug(
-                    "{}: content type is {}".format(
+                    f"{qid}: content type is {content_type}")
-                        queueid, content_type))
         else:
             self.logger.error(
-                "{}: unable to find email body".format(queueid))
+                f"{qid}: unable to find email body")
             content = "ERROR: unable to find email body"
 
         # create BeautifulSoup object
+        length = len(content)
         self.logger.debug(
-            "{}: trying to create BeatufilSoup object with parser lib {}, "
+            f"{qid}: trying to create BeatufilSoup object with parser lib {self.parser_lib}, "
-            "text length is {} bytes".format(
+            f"text length is {length} bytes")
-                queueid, self.parser_lib, len(content)))
         soup = BeautifulSoup(content, self.parser_lib)
         self.logger.debug(
-            "{}: sucessfully created BeautifulSoup object".format(queueid))
+            f"{qid}: sucessfully created BeautifulSoup object")
 
         return soup
 
-    def sanitize(self, queueid, soup):
+    def sanitize(self, qid, soup):
         "Sanitize mail html text."
-        self.logger.debug("{}: sanitizing email text".format(queueid))
+        self.logger.debug(f"{qid}: sanitizing email text")
 
         # completly remove bad elements
         for element in soup(EMailNotification._bad_tags):
             self.logger.debug(
-                "{}: removing dangerous tag '{}' and its content".format(
+                f"{qid}: removing dangerous tag '{element_name}' and its content")
-                    queueid, element.name))
             element.extract()
 
         # remove not whitelisted elements, but keep their content
         for element in soup.find_all(True):
             if element.name not in EMailNotification._good_tags:
                 self.logger.debug(
-                    "{}: removing tag '{}', keep its content".format(
+                    f"{qid}: removing tag '{element.name}', keep its content")
-                        queueid, element.name))
                 element.replaceWithChildren()
 
         # remove not whitelisted attributes
@@ -285,23 +279,21 @@ class EMailNotification(BaseNotification):
                 if attribute not in EMailNotification._good_attributes:
                     if element.name == "a" and attribute == "href":
                         self.logger.debug(
-                            "{}: setting attribute href to '#' on tag '{}'".format(
+                            f"{qid}: setting attribute href to '#' on tag '{element.name}'")
-                                queueid, element.name))
                         element["href"] = "#"
                     else:
                         self.logger.debug(
-                            "{}: removing attribute '{}' from tag '{}'".format(
+                            f"{qid}: removing attribute '{attribute}' from tag '{element.name}'")
-                                queueid, attribute, element.name))
                         del(element.attrs[attribute])
         return soup
 
-    def notify(self, queueid, storage_id, mailfrom, recipients, headers, fp,
+    def notify(self, qid, storage_id, mailfrom, recipients, headers, fp,
                subgroups=None, named_subgroups=None, synchronous=False):
         "Notify recipients via email."
         super(
             EMailNotification,
             self).notify(
-            queueid,
+            qid,
             storage_id,
             mailfrom,
             recipients,
@@ -313,66 +305,54 @@ class EMailNotification(BaseNotification):
 
         # extract body from email
         soup = self.get_email_body_soup(
-            queueid, email.message_from_binary_file(fp, policy=policy.default))
+            qid, email.message_from_binary_file(fp, policy=policy.default))
 
         # replace picture sources
         image_replaced = False
         if self.strip_images:
             self.logger.debug(
-                "{}: looking for images to strip".format(queueid))
+                f"{qid}: looking for images to strip")
             for element in soup("img"):
                 if "src" in element.attrs.keys():
                     self.logger.debug(
-                        "{}: strip image: {}".format(
+                        f"{qid}: strip image: {element['src']}")
-                            queueid, element["src"]))
                 element.extract()
         elif self.replacement_img:
             self.logger.debug(
-                "{}: looking for images to replace".format(queueid))
+                f"{qid}: looking for images to replace")
             for element in soup("img"):
                 if "src" in element.attrs.keys():
                     self.logger.debug(
-                        "{}: replacing image: {}".format(
+                        f"{qid}: replacing image: {element['src']}")
-                            queueid, element["src"]))
                 element["src"] = "cid:removed_for_security_reasons"
                 image_replaced = True
 
         # sanitizing email text of original email
-        sanitized_text = self.sanitize(queueid, soup)
+        sanitized_text = self.sanitize(qid, soup)
         del soup
 
         # sending email notifications
         for recipient in recipients:
             self.logger.debug(
-                "{}: generating notification email for '{}'".format(
+                f"{qid}: generating notification email for '{recipient}'")
-                    queueid, recipient))
+            self.logger.debug(f"{qid}: parsing email template")
-            self.logger.debug("{}: parsing email template".format(queueid))
-            # decode some headers
-            decoded_headers = {}
-            for header in ["from", "to", "subject"]:
-                if header in headers:
-                    decoded_headers[header] = str(
-                        make_header(decode_header(headers[header])))
-                else:
-                    headers[header] = ""
-                    decoded_headers[header] = ""
 
             # generate dict containing all template variables
             variables = defaultdict(str,
                                     EMAIL_HTML_TEXT=sanitized_text,
-                                    EMAIL_FROM=escape(decoded_headers["from"]),
+                                    EMAIL_FROM=escape(headers["from"]),
                                     EMAIL_ENVELOPE_FROM=escape(mailfrom),
                                     EMAIL_ENVELOPE_FROM_URL=escape(quote(mailfrom)),
-                                    EMAIL_TO=escape(decoded_headers["to"]),
+                                    EMAIL_TO=escape(headers["to"]),
                                     EMAIL_ENVELOPE_TO=escape(recipient),
                                     EMAIL_ENVELOPE_TO_URL=escape(quote(recipient)),
-                                    EMAIL_SUBJECT=escape(decoded_headers["subject"]),
+                                    EMAIL_SUBJECT=escape(headers["subject"]),
                                     EMAIL_QUARANTINE_ID=storage_id)
 
             if subgroups:
                 number = 0
                 for subgroup in subgroups:
-                    variables["SUBGROUP_{}".format(number)] = escape(subgroup)
+                    variables[f"SUBGROUP_{number}"] = escape(subgroup)
             if named_subgroups:
                 for key, value in named_subgroups.items():
                     named_subgroups[key] = escape(value)
@@ -392,26 +372,24 @@ class EMailNotification(BaseNotification):
 
             if image_replaced:
                 self.logger.debug(
-                    "{}: attaching notification_replacement_img".format(queueid))
+                    f"{qid}: attaching notification_replacement_img")
                 msg.attach(self.replacement_img)
 
             for img in self.embedded_imgs:
-                self.logger.debug("{}: attaching imgage".format(queueid))
+                self.logger.debug(f"{qid}: attaching imgage")
                 msg.attach(img)
 
             self.logger.debug(
-                "{}: sending notification email to: {}".format(
+                f"{qid}: sending notification email to: {recipient}")
-                    queueid, recipient))
             if synchronous:
                 try:
                     mailer.smtp_send(self.smtp_host, self.smtp_port,
                                      self.mailfrom, recipient, msg.as_string())
                 except Exception as e:
                     raise RuntimeError(
-                        "error while sending email to '{}': {}".format(
+                        f"error while sending email to '{recipient}': {e}")
-                            recipient, e))
             else:
-                mailer.sendmail(self.smtp_host, self.smtp_port, queueid,
+                mailer.sendmail(self.smtp_host, self.smtp_port, qid,
                                 self.mailfrom, recipient, msg.as_string(),
                                 "notification email")
 

pyquarantine/run.py

@@ -52,7 +52,7 @@ def main():
         "-v", "--version",
         help="Print version.",
         action="version",
-        version="%(prog)s ({})".format(version))
+        version=f"%(prog)s ({version})")
     args = parser.parse_args()
 
     # setup logging
@@ -61,8 +61,8 @@ def main():
     syslog_name = logname
     if args.debug:
         loglevel = logging.DEBUG
-        logname = "{}[%(name)s]".format(logname)
+        logname = f"{logname}[%(name)s]"
-        syslog_name = "{}: [%(name)s] %(levelname)s".format(syslog_name)
+        syslog_name = f"{syslog_name}: [%(name)s] %(levelname)s"
 
     # set config files for milter class
     pyquarantine.QuarantineMilter.set_cfg_files(args.config)
@@ -72,7 +72,7 @@ def main():
     # setup console log
     stdouthandler = logging.StreamHandler(sys.stdout)
     stdouthandler.setLevel(logging.DEBUG)
-    formatter = logging.Formatter("%(message)s".format(logname))
+    formatter = logging.Formatter("%(message)s")
     stdouthandler.setFormatter(formatter)
     root_logger.addHandler(stdouthandler)
     logger = logging.getLogger(__name__)
@@ -86,7 +86,7 @@ def main():
         else:
             sys.exit(0)
     formatter = logging.Formatter(
-        "%(asctime)s {}: [%(levelname)s] %(message)s".format(logname),
+        f"%(asctime)s {logname}: [%(levelname)s] %(message)s",
         datefmt="%Y-%m-%d %H:%M:%S")
     stdouthandler.setFormatter(formatter)
 
@@ -94,7 +94,7 @@ def main():
     sysloghandler = logging.handlers.SysLogHandler(
         address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL)
     sysloghandler.setLevel(loglevel)
-    formatter = logging.Formatter("{}: %(message)s".format(syslog_name))
+    formatter = logging.Formatter(f"{syslog_name}: %(message)s")
     sysloghandler.setFormatter(formatter)
     root_logger.addHandler(sysloghandler)
 

pyquarantine/storages.py

@@ -31,7 +31,7 @@ class BaseMailStorage(object):
         self.name = name
         self.logger = logging.getLogger(__name__)
 
-    def add(self, queueid, mailfrom, recipients, headers,
+    def add(self, qid, mailfrom, recipients, headers,
             fp, subgroups=None, named_subgroups=None):
         "Add email to storage."
         fp.seek(0)
@@ -73,16 +73,14 @@ class FileMailStorage(BaseMailStorage):
                 cfg[opt] = defaults[opt]
             else:
                 raise RuntimeError(
-                    "mandatory option '{}' not present in config section '{}' or 'global'".format(
+                    f"mandatory option '{opt}' not present in config section '{self.name}' or 'global'")
-                        opt, self.name))
         self.directory = cfg["storage_directory"]
 
         # check if quarantine directory exists and is writable
         if not os.path.isdir(self.directory) or not os.access(
                 self.directory, os.W_OK):
             raise RuntimeError(
-                "file quarantine directory '{}' does not exist or is not writable".format(
+                f"file quarantine directory '{self.directory}' does not exist or is not writable")
-                    self.directory))
         self._metadata_suffix = ".metadata"
 
     def _save_datafile(self, storage_id, fp):
@@ -91,47 +89,46 @@ class FileMailStorage(BaseMailStorage):
             with open(datafile, "wb") as f:
                 copyfileobj(fp, f)
         except IOError as e:
-            raise RuntimeError("unable save data file: {}".format(e))
+            raise RuntimeError(f"unable save data file: {e}")
 
     def _save_metafile(self, storage_id, metadata):
         metafile = os.path.join(
-            self.directory, "{}{}".format(
+            self.directory, f"{storage_id}{self._metadata_suffix}")
-                storage_id, self._metadata_suffix))
         try:
             with open(metafile, "w") as f:
                 json.dump(metadata, f, indent=2)
         except IOError as e:
-            raise RuntimeError("unable to save metadata file: {}".format(e))
+            raise RuntimeError(f"unable to save metadata file: {e}")
 
     def _remove(self, storage_id):
         datafile = os.path.join(self.directory, storage_id)
-        metafile = "{}{}".format(datafile, self._metadata_suffix)
+        metafile = f"{datafile}{self._metadata_suffix}"
 
         try:
             os.remove(metafile)
         except IOError as e:
-            raise RuntimeError("unable to remove metadata file: {}".format(e))
+            raise RuntimeError(f"unable to remove metadata file: {e}")
 
         try:
             os.remove(datafile)
         except IOError as e:
-            raise RuntimeError("unable to remove data file: {}".format(e))
+            raise RuntimeError(f"unable to remove data file: {e}")
 
-    def add(self, queueid, mailfrom, recipients, headers,
+    def add(self, qid, mailfrom, recipients, headers,
             fp, subgroups=None, named_subgroups=None):
         "Add email to file storage and return storage id."
         super(
             FileMailStorage,
             self).add(
-            queueid,
+            qid,
             mailfrom,
             recipients,
             headers,
             fp,
             subgroups,
             named_subgroups)
-        storage_id = "{}_{}".format(
+        timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
-            datetime.now().strftime("%Y%m%d%H%M%S"), queueid)
+        storage_id = f"{timestamp}_{qid}"
 
         # save mail
         self._save_datafile(storage_id, fp)
@@ -142,7 +139,7 @@ class FileMailStorage(BaseMailStorage):
             "recipients": recipients,
             "headers": headers,
             "date": timegm(gmtime()),
-            "queue_id": queueid,
+            "queue_id": qid,
             "subgroups": subgroups,
             "named_subgroups": named_subgroups
         }
@@ -160,21 +157,19 @@ class FileMailStorage(BaseMailStorage):
         super(FileMailStorage, self).get_metadata(storage_id)
 
         metafile = os.path.join(
-            self.directory, "{}{}".format(
+            self.directory, f"{storage_id}{self._metadata_suffix}")
-                storage_id, self._metadata_suffix))
         if not os.path.isfile(metafile):
             raise RuntimeError(
-                "invalid storage id '{}'".format(storage_id))
+                f"invalid storage id '{storage_id}'")
 
         try:
             with open(metafile, "r") as f:
                 metadata = json.load(f)
         except IOError as e:
-            raise RuntimeError("unable to read metadata file: {}".format(e))
+            raise RuntimeError(f"unable to read metadata file: {e}")
         except json.JSONDecodeError as e:
             raise RuntimeError(
-                "invalid meta file '{}': {}".format(
+                f"invalid metafile '{metafile}': {e}")
-                    metafile, e))
 
         return metadata
 
@@ -188,7 +183,7 @@ class FileMailStorage(BaseMailStorage):
 
         emails = {}
         metafiles = glob(os.path.join(
-            self.directory, "*{}".format(self._metadata_suffix)))
+            self.directory, f"*{self._metadata_suffix}"))
         for metafile in metafiles:
             if not os.path.isfile(metafile):
                 continue
@@ -222,7 +217,7 @@ class FileMailStorage(BaseMailStorage):
         try:
             metadata = self.get_metadata(storage_id)
         except RuntimeError as e:
-            raise RuntimeError("unable to delete email: {}".format(e))
+            raise RuntimeError(f"unable to delete email: {e}")
 
         if not recipients:
             self._remove(storage_id)
@@ -231,7 +226,7 @@ class FileMailStorage(BaseMailStorage):
                 recipients = [recipients]
             for recipient in recipients:
                 if recipient not in metadata["recipients"]:
-                    raise RuntimeError("invalid recipient '{}'".format(recipient))
+                    raise RuntimeError(f"invalid recipient '{recipient}'")
                 metadata["recipients"].remove(recipient)
                 if not metadata["recipients"]:
                     self._remove(storage_id)
@@ -246,7 +241,7 @@ class FileMailStorage(BaseMailStorage):
         try:
             fp = open(datafile, "rb")
         except IOError as e:
-            raise RuntimeError("unable to open email data file: {}".format(e))
+            raise RuntimeError(f"unable to open email data file: {e}")
         return (fp, metadata)
 
 

pyquarantine/version.py

@@ -1 +1 @@
-__version__ = "1.0.0"
+__version__ = "1.0.1"

pyquarantine/whitelists.py

@@ -97,8 +97,7 @@ class DatabaseWhitelist(WhitelistBase):
                 cfg[opt] = defaults[opt]
             else:
                 raise RuntimeError(
-                    "mandatory option '{}' not present in config section '{}' or 'global'".format(
+                    f"mandatory option '{opt}' not present in config section '{self.name}' or 'global'")
-                        opt, self.name))
 
         tablename = cfg["whitelist_db_table"]
         connection_string = cfg["whitelist_db_connection"]
@@ -108,16 +107,16 @@ class DatabaseWhitelist(WhitelistBase):
         else:
             try:
                 # connect to database
+                conn = re.sub(
+                    r"(.*?://.*?):.*?(@.*)",
+                    r"\1:<PASSWORD>\2",
+                    connection_string)
                 self.logger.debug(
-                    "connecting to database '{}'".format(
+                    f"connecting to database '{conn}'")
-                        re.sub(
-                            r"(.*?://.*?):.*?(@.*)",
-                            r"\1:<PASSWORD>\2",
-                            connection_string)))
                 db = connect(connection_string)
             except Exception as e:
                 raise RuntimeError(
-                    "unable to connect to database: {}".format(e))
+                    f"unable to connect to database: {e}")
 
             DatabaseWhitelist._db_connections[connection_string] = db
 
@@ -125,7 +124,7 @@ class DatabaseWhitelist(WhitelistBase):
         self.meta = Meta
         self.meta.database = db
         self.meta.table_name = tablename
-        self.model = type("WhitelistModel_{}".format(self.name), (WhitelistModel,), {
+        self.model = type(f"WhitelistModel_{self.name}", (WhitelistModel,), {
             "Meta": self.meta
         })
 
@@ -139,8 +138,7 @@ class DatabaseWhitelist(WhitelistBase):
                     db.create_tables([self.model])
                 except Exception as e:
                     raise RuntimeError(
-                        "unable to initialize table '{}': {}".format(
+                        f"unable to initialize table '{tablename}': {e}")
-                            tablename, e))
 
     def _entry_to_dict(self, entry):
         result = {}
@@ -170,17 +168,18 @@ class DatabaseWhitelist(WhitelistBase):
 
         # generate list of possible mailfroms
         self.logger.debug(
-            "query database for whitelist entries from <{}> to <{}>".format(
+            f"query database for whitelist entries from <{mailfrom}> to <{recipient}>")
-                mailfrom, recipient))
         mailfroms = [""]
         if "@" in mailfrom and not mailfrom.startswith("@"):
-            mailfroms.append("@{}".format(mailfrom.split("@")[1]))
+            domain = mailfrom.split("@")[1]
+            mailfroms.append(f"@{domain}")
         mailfroms.append(mailfrom)
 
         # generate list of possible recipients
         recipients = [""]
         if "@" in recipient and not recipient.startswith("@"):
-            recipients.append("@{}".format(recipient.split("@")[1]))
+            domain = recipient.split("@")[1]
+            recipients.append(f"@{domain}")
         recipients.append(recipient)
 
         # query the database
@@ -190,7 +189,7 @@ class DatabaseWhitelist(WhitelistBase):
                     self.model.mailfrom.in_(mailfroms),
                     self.model.recipient.in_(recipients)))
         except Exception as e:
-            raise RuntimeError("unable to query database: {}".format(e))
+            raise RuntimeError(f"unable to query database: {e}")
 
         if not entries:
             # no whitelist entry found
@@ -235,7 +234,7 @@ class DatabaseWhitelist(WhitelistBase):
 
                 entries.update(self._entry_to_dict(entry))
         except Exception as e:
-            raise RuntimeError("unable to query database: {}".format(e))
+            raise RuntimeError(f"unable to query database: {e}")
 
         return entries
 
@@ -256,7 +255,7 @@ class DatabaseWhitelist(WhitelistBase):
                 comment=comment,
                 permanent=permanent)
         except Exception as e:
-            raise RuntimeError("unable to add entry to database: {}".format(e))
+            raise RuntimeError(f"unable to add entry to database: {e}")
 
     def delete(self, whitelist_id):
         "Delete entry from whitelist."
@@ -267,7 +266,7 @@ class DatabaseWhitelist(WhitelistBase):
             deleted = query.execute()
         except Exception as e:
             raise RuntimeError(
-                "unable to delete entry from database: {}".format(e))
+                f"unable to delete entry from database: {e}")
 
         if deleted == 0:
             raise RuntimeError("invalid whitelist id")