diff --git a/pyquarantine/__init__.py b/pyquarantine/__init__.py index 1471fc0..cebfe38 100644 --- a/pyquarantine/__init__.py +++ b/pyquarantine/__init__.py @@ -36,6 +36,8 @@ import sys from Milter.utils import parse_addr from collections import defaultdict +from email.policy import default as default_policy +from email.parser import BytesHeaderParser from io import BytesIO from itertools import groupby from netaddr import IPAddress, IPNetwork @@ -96,13 +98,11 @@ class Quarantine(object): cfg[opt] = defaults[opt] else: raise RuntimeError( - "mandatory option '{}' not present in config section '{}' or 'global'".format( - opt, self.name)) + f"mandatory option '{opt}' not present in config section '{self.name}' or 'global'") # pre-compile regex self.logger.debug( - "{}: compiling regex '{}'".format( - self.name, cfg["regex"])) + f"{self.name}: compiling regex '{cfg['regex']}'") self.regex = re.compile( cfg["regex"], re.MULTILINE + re.DOTALL + re.IGNORECASE) @@ -113,62 +113,53 @@ class Quarantine(object): storage_type = cfg["storage_type"].lower() if storage_type in storages.TYPES: self.logger.debug( - "{}: initializing storage type '{}'".format( - self.name, - storage_type.upper())) + f"{self.name}: initializing storage type '{storage_type.upper()}'") self.storage = storages.TYPES[storage_type]( self.name, global_cfg, cfg, test) elif storage_type == "none": - self.logger.debug("{}: storage is NONE".format(self.name)) + self.logger.debug(f"{self.name}: storage is NONE") self.storage = None else: raise RuntimeError( - "{}: unknown storage type '{}'".format( - self.name, storage_type)) + f"{self.name}: unknown storage type '{storage_type}'") # create whitelist instance whitelist_type = cfg["whitelist_type"].lower() if whitelist_type in whitelists.TYPES: self.logger.debug( - "{}: initializing whitelist type '{}'".format( - self.name, - whitelist_type.upper())) + f"{self.name}: initializing whitelist type '{whitelist_type.upper()}'") self.whitelist = whitelists.TYPES[whitelist_type]( self.name, global_cfg, cfg, test) elif whitelist_type == "none": - logger.debug("{}: whitelist is NONE".format(self.name)) + logger.debug(f"{self.name}: whitelist is NONE") self.whitelist = None else: raise RuntimeError( - "{}: unknown whitelist type '{}'".format( - self.name, whitelist_type)) + f"{self.name}: unknown whitelist type '{whitelist_type}'") # create notification instance notification_type = cfg["notification_type"].lower() if notification_type in notifications.TYPES: self.logger.debug( - "{}: initializing notification type '{}'".format( - self.name, - notification_type.upper())) + f"{self.name}: initializing notification type '{notification_type.upper()}'") self.notification = notifications.TYPES[notification_type]( self.name, global_cfg, cfg, test) elif notification_type == "none": - self.logger.debug("{}: notification is NONE".format(self.name)) + self.logger.debug(f"{self.name}: notification is NONE") self.notification = None else: raise RuntimeError( - "{}: unknown notification type '{}'".format( - self.name, notification_type)) + f"{self.name}: unknown notification type '{notification_type}'") # determining milter action for this quarantine action = cfg["action"].upper() if action in self._actions: - self.logger.debug("{}: action is {}".format(self.name, action)) + self.logger.debug(f"{self.name}: action is {action}") self.action = action self.milter_action = self._actions[action] else: raise RuntimeError( - "{}: unknown action '{}'".format(self._name, action)) + f"{self.name}: unknown action '{action}'") self.reject_reason = cfg["reject_reason"] @@ -183,15 +174,13 @@ class Quarantine(object): try: net = IPNetwork(host) except AddrFormatError as e: - raise RuntimeError("{}: error parsing host_whitelist: {}".format( - self.name, e)) + raise RuntimeError(f"{self.name}: error parsing host_whitelist: {e}") else: self.host_whitelist.append(net) if self.host_whitelist: + whitelist = ", ".join([str(ip) for ip in host_whitelist]) self.logger.debug( - "{}: host whitelist: {}".format( - self.name, - ", ".join([str(ip) for ip in host_whitelist]))) + f"{self.name}: host whitelist: {whitelist}") def notify(self, storage_id, recipient=None, synchronous=True): "Notify recipient about email in storage." @@ -207,7 +196,7 @@ class Quarantine(object): if recipient is not None: if recipient not in metadata["recipients"]: - raise RuntimeError("invalid recipient '{}'".format(recipient)) + raise RuntimeError(f"invalid recipient '{recipient}'") recipients = [recipient] else: recipients = metadata["recipients"] @@ -230,7 +219,7 @@ class Quarantine(object): mail = fp.read() fp.close() except IOError as e: - raise RuntimeError("unable to read data file: {}".format(e)) + raise RuntimeError(f"unable to read data file: {e}") if recipients and type(recipients) == str: recipients = [recipients] @@ -239,7 +228,7 @@ class Quarantine(object): for recipient in recipients: if recipient not in metadata["recipients"]: - raise RuntimeError("invalid recipient '{}'".format(recipient)) + raise RuntimeError(f"invalid recipient '{recipient}'") try: mailer.smtp_send( @@ -250,8 +239,7 @@ class Quarantine(object): mail) except Exception as e: raise RuntimeError( - "error while sending email to '{}': {}".format( - recipient, e)) + f"error while sending email to '{recipient}': {e}") self.storage.delete(storage_id, recipient) def get_storage(self): @@ -319,19 +307,16 @@ class QuarantineMilter(Milter.Base): def connect(self, hostname, family, hostaddr): self.hostaddr = hostaddr self.logger.debug( - "accepted milter connection from {} port {}".format( - *hostaddr)) + f"accepted milter connection from {hostaddr[0]} port {hostaddr[1]}") for quarantine in self.quarantines.copy(): if quarantine.host_in_whitelist(hostaddr): self.logger.debug( - "host {} is in whitelist of quarantine {}".format( - hostaddr[0], quarantine["name"])) + f"host {hostaddr[0]} is in whitelist of quarantine {quarantine['name']}") self.quarantines.remove(quarantine) if not self.quarantines: self.logger.debug( - "host {} is in whitelist of all quarantines, " - "skip further processing", - hostaddr[0]) + f"host {hostaddr[0]} is in whitelist of all quarantines, " + f"skip further processing") return Milter.ACCEPT return Milter.CONTINUE @@ -348,14 +333,12 @@ class QuarantineMilter(Milter.Base): @Milter.noreply def data(self): - self.queueid = self.getsymval('i') + self.qid = self.getsymval('i') self.logger.debug( - "{}: received queue-id from MTA".format(self.queueid)) + f"{self.qid}: received queue-id from MTA") self.recipients = list(self.recipients) - self.headers = [] self.logger.debug( - "{}: initializing memory buffer to save email data".format( - self.queueid)) + f"{self.qid}: initializing memory buffer to save email data") # initialize memory buffer to save email data self.fp = BytesIO() return Milter.CONTINUE @@ -364,15 +347,11 @@ class QuarantineMilter(Milter.Base): def header(self, name, value): try: # write email header to memory buffer - self.fp.write("{}: {}\r\n".format(name, value).encode( - encoding="ascii", errors="surrogateescape")) - # keep copy of header without surrogates for later use - self.headers.append(( - name.encode(errors="surrogateescape").decode(errors="replace"), - value.encode(errors="surrogateescape").decode(errors="replace"))) + self.fp.write(f"{name}: {value}\r\n".encode( + encoding="ascii", errors="replace")) except Exception as e: self.logger.exception( - "an exception occured in header function: {}".format(e)) + f"an exception occured in header function: {e}") return Milter.TEMPFAIL return Milter.CONTINUE @@ -380,6 +359,9 @@ class QuarantineMilter(Milter.Base): def eoh(self): try: self.fp.write("\r\n".encode(encoding="ascii")) + self.fp.seek(0) + self.headers = BytesHeaderParser( + policy=default_policy).parse(self.fp).items() self.whitelist_cache = whitelists.WhitelistCache() # initialize dicts to set quaranines per recipient and keep matches @@ -389,10 +371,9 @@ class QuarantineMilter(Milter.Base): # iterate email headers recipients_to_check = self.recipients.copy() for name, value in self.headers: - header = "{}: {}".format(name, value) + header = f"{name}: {value}" self.logger.debug( - "{}: checking header against configured quarantines: {}".format( - self.queueid, header)) + f"{self.qid}: checking header against configured quarantines: {header}") # iterate quarantines for quarantine in self.quarantines: if len(self.recipients_quarantines) == len( @@ -404,19 +385,16 @@ class QuarantineMilter(Milter.Base): # the same precedence already, skip checks against # quarantines with lower precedence self.logger.debug( - "{}: {}: skip further checks of this header".format( - self.queueid, quarantine.name)) + f"{self.qid}: {quarantine.name}: skip further checks of this header") break # check email header against quarantine regex self.logger.debug( - "{}: {}: checking header against regex '{}'".format( - self.queueid, quarantine.name, quarantine.regex)) + f"{self.qid}: {quarantine.name}: checking header against regex '{quarantine.regex}'") match = quarantine.match(header) if match: self.logger.debug( - "{}: {}: header matched regex".format( - self.queueid, quarantine.name)) + f"{self.qid}: {quarantine.name}: header matched regex") # check for whitelisted recipients whitelist = quarantine.get_whitelist() if whitelist: @@ -425,8 +403,7 @@ class QuarantineMilter(Milter.Base): whitelist, self.mailfrom, recipients_to_check) except RuntimeError as e: self.logger.error( - "{}: {}: unable to query whitelist: {}".format( - self.queueid, quarantine.name, e)) + f"{self.qid}: {quarantine.name}: unable to query whitelist: {e}") return Milter.TEMPFAIL else: whitelisted_recipients = {} @@ -436,15 +413,13 @@ class QuarantineMilter(Milter.Base): if recipient in whitelisted_recipients: # recipient is whitelisted in this quarantine self.logger.debug( - "{}: {}: recipient '{}' is whitelisted".format( - self.queueid, quarantine.name, recipient)) + f"{self.qid}: {quarantine.name}: recipient '{recipient}' is whitelisted") continue if recipient not in self.recipients_quarantines.keys() or \ self.recipients_quarantines[recipient].index > quarantine.index: self.logger.debug( - "{}: {}: set quarantine for recipient '{}'".format( - self.queueid, quarantine.name, recipient)) + f"{self.qid}: {quarantine.name}: set quarantine for recipient '{recipient}'") # save match for later use as template # variables self.quarantines_matches[quarantine.name] = match @@ -456,23 +431,20 @@ class QuarantineMilter(Milter.Base): recipients_to_check.remove(recipient) else: self.logger.debug( - "{}: {}: a quarantine with same or higher precedence " - "matched already for recipient '{}'".format( - self.queueid, quarantine.name, recipient)) + f"{self.qid}: {quarantine.name}: a quarantine with same or higher " + f"precedence matched already for recipient '{recipient}'") if not recipients_to_check: self.logger.debug( - "{}: all recipients matched the first quarantine, " - "skipping all remaining header checks".format( - self.queueid)) + f"{self.qid}: all recipients matched the first quarantine, " + f"skipping all remaining header checks") break # check if no quarantine has matched for all recipients if not self.recipients_quarantines: # accept email self.logger.info( - "{}: passed clean for all recipients".format( - self.queueid)) + f"{self.qid}: passed clean for all recipients") return Milter.ACCEPT # check if the mail body is needed @@ -485,18 +457,16 @@ class QuarantineMilter(Milter.Base): # quarantines, just return configured action quarantine = self._get_preferred_quarantine() self.logger.info( - "{}: {} matching quarantine is '{}', performing milter action {}".format( - self.queueid, - self.preferred_quarantine_action, - quarantine.name, - quarantine.action)) + f"{self.qid}: {self.preferred_quarantine_action} " + f"matching quarantine is '{quarantine.name}', performing " + f"milter action {quarantine.action}") if quarantine.action == "REJECT": self.setreply("554", "5.7.0", quarantine.reject_reason) return quarantine.milter_action except Exception as e: self.logger.exception( - "an exception occured in eoh function: {}".format(e)) + f"an exception occured in eoh function: {e}") return Milter.TEMPFAIL def body(self, chunk): @@ -505,7 +475,7 @@ class QuarantineMilter(Milter.Base): self.fp.write(chunk) except Exception as e: self.logger.exception( - "an exception occured in body function: {}".format(e)) + f"an exception occured in body function: {e}") return Milter.TEMPFAIL return Milter.CONTINUE @@ -530,21 +500,22 @@ class QuarantineMilter(Milter.Base): named_subgroups = self.quarantines_matches[quarantine.name].groupdict( default="") + rcpts = ", ".join(recipients) + # check if a storage is configured storage_id = "" storage = quarantine.get_storage() if storage: # add email to quarantine - self.logger.info("{}: adding to quarantine '{}' for: {}".format( - self.queueid, quarantine.name, ", ".join(recipients))) + self.logger.info( + f"{self.qid}: adding to quarantine '{quarantine.name}' for: {rcpts}") try: storage_id = storage.add( - self.queueid, self.mailfrom, recipients, headers, self.fp, + self.qid, self.mailfrom, recipients, headers, self.fp, subgroups, named_subgroups) except RuntimeError as e: self.logger.error( - "{}: unable to add to quarantine '{}': {}".format( - self.queueid, quarantine.name, e)) + f"{self.qid}: unable to add to quarantine '{quarantine.name}': {e}") return Milter.TEMPFAIL # check if a notification is configured @@ -552,17 +523,15 @@ class QuarantineMilter(Milter.Base): if notification: # notify self.logger.info( - "{}: sending notification for quarantine '{}' to: {}".format( - self.queueid, quarantine.name, ", ".join(recipients))) + f"{self.qid}: sending notification to: {rcpts}") try: notification.notify( - self.queueid, storage_id, + self.qid, storage_id, self.mailfrom, recipients, headers, self.fp, subgroups, named_subgroups) except RuntimeError as e: self.logger.error( - "{}: unable to send notification for quarantine '{}': {}".format( - self.queueid, quarantine.name, e)) + f"{self.qid}: unable to send notification: {e}") return Milter.TEMPFAIL # remove processed recipient @@ -574,33 +543,29 @@ class QuarantineMilter(Milter.Base): # email passed clean for at least one recipient, accepting email if self.recipients: + rcpts = ", ".join(recipients) self.logger.info( - "{}: passed clean for: {}".format( - self.queueid, ", ".join( - self.recipients))) + f"{self.qid}: passed clean for: {rcpts}") return Milter.ACCEPT # return configured action quarantine = self._get_preferred_quarantine() self.logger.info( - "{}: {} matching quarantine is '{}', performing milter action {}".format( - self.queueid, - self.preferred_quarantine_action, - quarantine.name, - quarantine.action)) + f"{self.qid}: {self.preferred_quarantine_action} matching " + f"quarantine is '{quarantine.name}', performing milter " + f"action {quarantine.action}") if quarantine.action == "REJECT": self.setreply("554", "5.7.0", quarantine.reject_reason) return quarantine.milter_action except Exception as e: self.logger.exception( - "an exception occured in eom function: {}".format(e)) + f"an exception occured in eom function: {e}") return Milter.TEMPFAIL def close(self): self.logger.debug( - "disconnect from {} port {}".format( - *self.hostaddr)) + f"disconnect from {self.hostaddr[0]} port {self.hostaddr[1]}") return Milter.CONTINUE @@ -627,7 +592,7 @@ def setup_milter(test=False, cfg_files=[]): for option in ["quarantines", "preferred_quarantine_action"]: if not parser.has_option("global", option): raise RuntimeError( - "mandatory option '{}' not present in config section 'global'".format(option)) + f"mandatory option '{option}' not present in config section 'global'") # read global config section global_cfg = dict(parser.items("global")) @@ -656,7 +621,7 @@ def setup_milter(test=False, cfg_files=[]): # check if config section for current quarantine exists if name not in parser.sections(): raise RuntimeError( - "config section '{}' does not exist".format(name)) + f"config section '{name}' does not exist") cfg = dict(parser.items(name)) quarantine = Quarantine(name, index) diff --git a/pyquarantine/cli.py b/pyquarantine/cli.py index 42e7bdf..edc8e1c 100644 --- a/pyquarantine/cli.py +++ b/pyquarantine/cli.py @@ -20,17 +20,16 @@ import logging.handlers import sys import time -from email.header import decode_header, make_header - -import pyquarantine +from email.header import decode_header +from pyquarantine import QuarantineMilter, setup_milter from pyquarantine.version import __version__ as version def _get_quarantine(quarantines, name): try: quarantine = next((q for q in quarantines if q.name == name)) except StopIteration: - raise RuntimeError("invalid quarantine '{}'".format(name)) + raise RuntimeError(f"invalid quarantine 'name'") return quarantine def _get_storage(quarantines, name): @@ -74,7 +73,7 @@ def print_table(columns, rows): # use the longer one length = max(lengths) column_lengths.append(length) - column_formats.append("{{:<{}}}".format(length)) + column_formats.append(f"{{:<{length}}}") # define row format row_format = " | ".join(column_formats) @@ -156,9 +155,7 @@ def list_quarantine_emails(quarantines, args): row["recipient"] = metadata["recipients"].pop(0) if "subject" not in emails[storage_id]["headers"].keys(): emails[storage_id]["headers"]["subject"] = "" - row["subject"] = str(make_header(decode_header( - emails[storage_id]["headers"]["subject"])))[:60].replace( - "\r", "").replace("\n", "").strip() + row["subject"] = emails[storage_id]["headers"]["subject"][:60].strip() rows.append(row) if metadata["recipients"]: @@ -177,7 +174,7 @@ def list_quarantine_emails(quarantines, args): return if not emails: - logger.info("quarantine '{}' is empty".format(args.quarantine)) + logger.info(f"quarantine '{args.quarantine}' is empty") print_table( [("Quarantine-ID", "storage_id"), ("Date", "date"), ("From", "mailfrom"), ("Recipient(s)", "recipient"), @@ -197,8 +194,7 @@ def list_whitelist(quarantines, args): older_than=args.older_than) if not entries: logger.info( - "whitelist of quarantine '{}' is empty".format( - args.quarantine)) + f"whitelist of quarantine '{args.quarantine}' is empty") return # transform some values to strings @@ -309,7 +305,7 @@ def main(): "-c", "--config", help="Config files to read.", nargs="+", metavar="CFG", - default=pyquarantine.QuarantineMilter.get_cfg_files()) + default=QuarantineMilter.get_cfg_files()) parser.add_argument( "-d", "--debug", help="Log debugging messages.", @@ -318,7 +314,7 @@ def main(): "-v", "--version", help="Print version.", action="version", - version="%(prog)s ({})".format(version)) + version=f"%(prog)s ({version})") parser.set_defaults(syslog=False) subparsers = parser.add_subparsers( dest="command", @@ -563,7 +559,7 @@ def main(): # try to generate milter configs try: - pyquarantine.setup_milter( + setup_milter( cfg_files=args.config, test=True) except RuntimeError as e: logger.error(e) @@ -585,7 +581,7 @@ def main(): # call the commands function try: - args.func(pyquarantine.QuarantineMilter.quarantines, args) + args.func(QuarantineMilter.quarantines, args) except RuntimeError as e: logger.error(e) sys.exit(1) diff --git a/pyquarantine/mailer.py b/pyquarantine/mailer.py index 2426445..83791ae 100644 --- a/pyquarantine/mailer.py +++ b/pyquarantine/mailer.py @@ -45,23 +45,21 @@ def mailprocess(): if not m: break - smtp_host, smtp_port, queueid, mailfrom, recipient, mail, emailtype = m + smtp_host, smtp_port, qid, mailfrom, recipient, mail, emailtype = m try: smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail) except Exception as e: logger.error( - "{}: error while sending {} to '{}': {}".format( - queueid, emailtype, recipient, e)) + f"{qid}: error while sending {emailtype} to '{recipient}': {e}") else: logger.info( - "{}: successfully sent {} to: {}".format( - queueid, emailtype, recipient)) + f"{qid}: successfully sent {emailtype} to: {recipient}") except KeyboardInterrupt: pass logger.debug("mailer process terminated") -def sendmail(smtp_host, smtp_port, queueid, mailfrom, recipients, mail, +def sendmail(smtp_host, smtp_port, qid, mailfrom, recipients, mail, emailtype="email"): "Send an email." global logger @@ -81,7 +79,7 @@ def sendmail(smtp_host, smtp_port, queueid, mailfrom, recipients, mail, for recipient in recipients: try: queue.put( - (smtp_host, smtp_port, queueid, mailfrom, recipient, mail, + (smtp_host, smtp_port, qid, mailfrom, recipient, mail, emailtype), timeout=30) except Queue.Full as e: diff --git a/pyquarantine/notifications.py b/pyquarantine/notifications.py index 1d4e4d7..638a339 100644 --- a/pyquarantine/notifications.py +++ b/pyquarantine/notifications.py @@ -20,7 +20,7 @@ from bs4 import BeautifulSoup from cgi import escape from collections import defaultdict from email import policy -from email.header import decode_header, make_header +from email.header import decode_header from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.mime.image import MIMEImage @@ -38,7 +38,7 @@ class BaseNotification(object): self.name = name self.logger = logging.getLogger(__name__) - def notify(self, queueid, storage_id, mailfrom, recipients, headers, + def notify(self, qid, storage_id, mailfrom, recipients, headers, fp, subgroups=None, named_subgroups=None, synchronous=False): fp.seek(0) pass @@ -136,8 +136,7 @@ class EMailNotification(BaseNotification): cfg[opt] = defaults[opt] else: raise RuntimeError( - "mandatory option '{}' not present in config section '{}' or 'global'".format( - opt, self.name)) + f"mandatory option '{opt}' not present in config section '{self.name}' or 'global'") self.smtp_host = cfg["notification_email_smtp_host"] self.smtp_port = cfg["notification_email_smtp_port"] @@ -152,14 +151,14 @@ class EMailNotification(BaseNotification): self.from_header.format_map(testvars) except ValueError as e: raise RuntimeError( - "error parsing notification_email_from: {}".format(e)) + f"error parsing notification_email_from: {e}") # test-parse subject try: self.subject.format_map(testvars) except ValueError as e: raise RuntimeError( - "error parsing notification_email_subject: {}".format(e)) + f"error parsing notification_email_subject: {e}") # read and parse email notification template try: @@ -167,9 +166,9 @@ class EMailNotification(BaseNotification): cfg["notification_email_template"], "r").read() self.template.format_map(testvars) except IOError as e: - raise RuntimeError("error reading template: {}".format(e)) + raise RuntimeError(f"error reading template: {e}") except ValueError as e: - raise RuntimeError("error parsing template: {}".format(e)) + raise RuntimeError(f"error parsing template: {e}") strip_images = cfg["notification_email_strip_images"].strip().upper() if strip_images in ["TRUE", "ON", "YES"]: @@ -191,7 +190,7 @@ class EMailNotification(BaseNotification): open(replacement_img, "rb").read()) except IOError as e: raise RuntimeError( - "error reading replacement image: {}".format(e)) + f"error reading replacement image: {e}") else: self.replacement_img.add_header( "Content-ID", "") @@ -207,20 +206,20 @@ class EMailNotification(BaseNotification): try: img = MIMEImage(open(img_path, "rb").read()) except IOError as e: - raise RuntimeError("error reading image: {}".format(e)) + raise RuntimeError(f"error reading image: {e}") else: - img.add_header("Content-ID", "<{}>".format(basename(img_path))) + filename = basename(img_path) + img.add_header(f"Content-ID", f"<{filename}>") self.embedded_imgs.append(img) - def get_email_body_soup(self, queueid, msg): + def get_email_body_soup(self, qid, msg): "Extract and decode email body and return it as BeautifulSoup object." # try to find the body part - self.logger.debug("{}: trying to find email body".format(queueid)) + self.logger.debug(f"{qid}: trying to find email body") try: body = msg.get_body(preferencelist=("html", "plain")) except Exception as e: - self.logger.error("{}: an error occured in email.message.EmailMessage.get_body: {}".format( - queueid, e)) + self.logger.error(f"{qid}: an error occured in email.message.EmailMessage.get_body: {e}") body = None if body: @@ -229,54 +228,49 @@ class EMailNotification(BaseNotification): try: content = content.decode(encoding=charset, errors="replace") except LookupError: - self.logger.info("{}: unknown encoding '{}', falling back to UTF-8".format( - queueid, charset)) + self.logger.info(f"{qid}: unknown encoding '{charset}', falling back to UTF-8") content = content.decode("utf-8", errors="replace") content_type = body.get_content_type() if content_type == "text/plain": # convert text/plain to text/html self.logger.debug( - "{}: content type is {}, converting to text/html".format( - queueid, content_type)) + f"{qid}: content type is {content_type}, converting to text/html") content = re.sub(r"^(.*)$", r"\1
", escape(content), flags=re.MULTILINE) else: self.logger.debug( - "{}: content type is {}".format( - queueid, content_type)) + f"{qid}: content type is {content_type}") else: self.logger.error( - "{}: unable to find email body".format(queueid)) + f"{qid}: unable to find email body") content = "ERROR: unable to find email body" # create BeautifulSoup object + length = len(content) self.logger.debug( - "{}: trying to create BeatufilSoup object with parser lib {}, " - "text length is {} bytes".format( - queueid, self.parser_lib, len(content))) + f"{qid}: trying to create BeatufilSoup object with parser lib {self.parser_lib}, " + f"text length is {length} bytes") soup = BeautifulSoup(content, self.parser_lib) self.logger.debug( - "{}: sucessfully created BeautifulSoup object".format(queueid)) + f"{qid}: sucessfully created BeautifulSoup object") return soup - def sanitize(self, queueid, soup): + def sanitize(self, qid, soup): "Sanitize mail html text." - self.logger.debug("{}: sanitizing email text".format(queueid)) + self.logger.debug(f"{qid}: sanitizing email text") # completly remove bad elements for element in soup(EMailNotification._bad_tags): self.logger.debug( - "{}: removing dangerous tag '{}' and its content".format( - queueid, element.name)) + f"{qid}: removing dangerous tag '{element_name}' and its content") element.extract() # remove not whitelisted elements, but keep their content for element in soup.find_all(True): if element.name not in EMailNotification._good_tags: self.logger.debug( - "{}: removing tag '{}', keep its content".format( - queueid, element.name)) + f"{qid}: removing tag '{element.name}', keep its content") element.replaceWithChildren() # remove not whitelisted attributes @@ -285,23 +279,21 @@ class EMailNotification(BaseNotification): if attribute not in EMailNotification._good_attributes: if element.name == "a" and attribute == "href": self.logger.debug( - "{}: setting attribute href to '#' on tag '{}'".format( - queueid, element.name)) + f"{qid}: setting attribute href to '#' on tag '{element.name}'") element["href"] = "#" else: self.logger.debug( - "{}: removing attribute '{}' from tag '{}'".format( - queueid, attribute, element.name)) + f"{qid}: removing attribute '{attribute}' from tag '{element.name}'") del(element.attrs[attribute]) return soup - def notify(self, queueid, storage_id, mailfrom, recipients, headers, fp, + def notify(self, qid, storage_id, mailfrom, recipients, headers, fp, subgroups=None, named_subgroups=None, synchronous=False): "Notify recipients via email." super( EMailNotification, self).notify( - queueid, + qid, storage_id, mailfrom, recipients, @@ -313,66 +305,54 @@ class EMailNotification(BaseNotification): # extract body from email soup = self.get_email_body_soup( - queueid, email.message_from_binary_file(fp, policy=policy.default)) + qid, email.message_from_binary_file(fp, policy=policy.default)) # replace picture sources image_replaced = False if self.strip_images: self.logger.debug( - "{}: looking for images to strip".format(queueid)) + f"{qid}: looking for images to strip") for element in soup("img"): if "src" in element.attrs.keys(): self.logger.debug( - "{}: strip image: {}".format( - queueid, element["src"])) + f"{qid}: strip image: {element['src']}") element.extract() elif self.replacement_img: self.logger.debug( - "{}: looking for images to replace".format(queueid)) + f"{qid}: looking for images to replace") for element in soup("img"): if "src" in element.attrs.keys(): self.logger.debug( - "{}: replacing image: {}".format( - queueid, element["src"])) + f"{qid}: replacing image: {element['src']}") element["src"] = "cid:removed_for_security_reasons" image_replaced = True # sanitizing email text of original email - sanitized_text = self.sanitize(queueid, soup) + sanitized_text = self.sanitize(qid, soup) del soup # sending email notifications for recipient in recipients: self.logger.debug( - "{}: generating notification email for '{}'".format( - queueid, recipient)) - self.logger.debug("{}: parsing email template".format(queueid)) - # decode some headers - decoded_headers = {} - for header in ["from", "to", "subject"]: - if header in headers: - decoded_headers[header] = str( - make_header(decode_header(headers[header]))) - else: - headers[header] = "" - decoded_headers[header] = "" + f"{qid}: generating notification email for '{recipient}'") + self.logger.debug(f"{qid}: parsing email template") # generate dict containing all template variables variables = defaultdict(str, EMAIL_HTML_TEXT=sanitized_text, - EMAIL_FROM=escape(decoded_headers["from"]), + EMAIL_FROM=escape(headers["from"]), EMAIL_ENVELOPE_FROM=escape(mailfrom), EMAIL_ENVELOPE_FROM_URL=escape(quote(mailfrom)), - EMAIL_TO=escape(decoded_headers["to"]), + EMAIL_TO=escape(headers["to"]), EMAIL_ENVELOPE_TO=escape(recipient), EMAIL_ENVELOPE_TO_URL=escape(quote(recipient)), - EMAIL_SUBJECT=escape(decoded_headers["subject"]), + EMAIL_SUBJECT=escape(headers["subject"]), EMAIL_QUARANTINE_ID=storage_id) if subgroups: number = 0 for subgroup in subgroups: - variables["SUBGROUP_{}".format(number)] = escape(subgroup) + variables[f"SUBGROUP_{number}"] = escape(subgroup) if named_subgroups: for key, value in named_subgroups.items(): named_subgroups[key] = escape(value) @@ -392,26 +372,24 @@ class EMailNotification(BaseNotification): if image_replaced: self.logger.debug( - "{}: attaching notification_replacement_img".format(queueid)) + f"{qid}: attaching notification_replacement_img") msg.attach(self.replacement_img) for img in self.embedded_imgs: - self.logger.debug("{}: attaching imgage".format(queueid)) + self.logger.debug(f"{qid}: attaching imgage") msg.attach(img) self.logger.debug( - "{}: sending notification email to: {}".format( - queueid, recipient)) + f"{qid}: sending notification email to: {recipient}") if synchronous: try: mailer.smtp_send(self.smtp_host, self.smtp_port, self.mailfrom, recipient, msg.as_string()) except Exception as e: raise RuntimeError( - "error while sending email to '{}': {}".format( - recipient, e)) + f"error while sending email to '{recipient}': {e}") else: - mailer.sendmail(self.smtp_host, self.smtp_port, queueid, + mailer.sendmail(self.smtp_host, self.smtp_port, qid, self.mailfrom, recipient, msg.as_string(), "notification email") diff --git a/pyquarantine/run.py b/pyquarantine/run.py index 487d736..876fac0 100644 --- a/pyquarantine/run.py +++ b/pyquarantine/run.py @@ -52,7 +52,7 @@ def main(): "-v", "--version", help="Print version.", action="version", - version="%(prog)s ({})".format(version)) + version=f"%(prog)s ({version})") args = parser.parse_args() # setup logging @@ -61,8 +61,8 @@ def main(): syslog_name = logname if args.debug: loglevel = logging.DEBUG - logname = "{}[%(name)s]".format(logname) - syslog_name = "{}: [%(name)s] %(levelname)s".format(syslog_name) + logname = f"{logname}[%(name)s]" + syslog_name = f"{syslog_name}: [%(name)s] %(levelname)s" # set config files for milter class pyquarantine.QuarantineMilter.set_cfg_files(args.config) @@ -72,7 +72,7 @@ def main(): # setup console log stdouthandler = logging.StreamHandler(sys.stdout) stdouthandler.setLevel(logging.DEBUG) - formatter = logging.Formatter("%(message)s".format(logname)) + formatter = logging.Formatter("%(message)s") stdouthandler.setFormatter(formatter) root_logger.addHandler(stdouthandler) logger = logging.getLogger(__name__) @@ -86,7 +86,7 @@ def main(): else: sys.exit(0) formatter = logging.Formatter( - "%(asctime)s {}: [%(levelname)s] %(message)s".format(logname), + f"%(asctime)s {logname}: [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S") stdouthandler.setFormatter(formatter) @@ -94,7 +94,7 @@ def main(): sysloghandler = logging.handlers.SysLogHandler( address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL) sysloghandler.setLevel(loglevel) - formatter = logging.Formatter("{}: %(message)s".format(syslog_name)) + formatter = logging.Formatter(f"{syslog_name}: %(message)s") sysloghandler.setFormatter(formatter) root_logger.addHandler(sysloghandler) diff --git a/pyquarantine/storages.py b/pyquarantine/storages.py index 864aaab..63f0861 100644 --- a/pyquarantine/storages.py +++ b/pyquarantine/storages.py @@ -31,7 +31,7 @@ class BaseMailStorage(object): self.name = name self.logger = logging.getLogger(__name__) - def add(self, queueid, mailfrom, recipients, headers, + def add(self, qid, mailfrom, recipients, headers, fp, subgroups=None, named_subgroups=None): "Add email to storage." fp.seek(0) @@ -73,16 +73,14 @@ class FileMailStorage(BaseMailStorage): cfg[opt] = defaults[opt] else: raise RuntimeError( - "mandatory option '{}' not present in config section '{}' or 'global'".format( - opt, self.name)) + f"mandatory option '{opt}' not present in config section '{self.name}' or 'global'") self.directory = cfg["storage_directory"] # check if quarantine directory exists and is writable if not os.path.isdir(self.directory) or not os.access( self.directory, os.W_OK): raise RuntimeError( - "file quarantine directory '{}' does not exist or is not writable".format( - self.directory)) + f"file quarantine directory '{self.directory}' does not exist or is not writable") self._metadata_suffix = ".metadata" def _save_datafile(self, storage_id, fp): @@ -91,47 +89,46 @@ class FileMailStorage(BaseMailStorage): with open(datafile, "wb") as f: copyfileobj(fp, f) except IOError as e: - raise RuntimeError("unable save data file: {}".format(e)) + raise RuntimeError(f"unable save data file: {e}") def _save_metafile(self, storage_id, metadata): metafile = os.path.join( - self.directory, "{}{}".format( - storage_id, self._metadata_suffix)) + self.directory, f"{storage_id}{self._metadata_suffix}") try: with open(metafile, "w") as f: json.dump(metadata, f, indent=2) except IOError as e: - raise RuntimeError("unable to save metadata file: {}".format(e)) + raise RuntimeError(f"unable to save metadata file: {e}") def _remove(self, storage_id): datafile = os.path.join(self.directory, storage_id) - metafile = "{}{}".format(datafile, self._metadata_suffix) + metafile = f"{datafile}{self._metadata_suffix}" try: os.remove(metafile) except IOError as e: - raise RuntimeError("unable to remove metadata file: {}".format(e)) + raise RuntimeError(f"unable to remove metadata file: {e}") try: os.remove(datafile) except IOError as e: - raise RuntimeError("unable to remove data file: {}".format(e)) + raise RuntimeError(f"unable to remove data file: {e}") - def add(self, queueid, mailfrom, recipients, headers, + def add(self, qid, mailfrom, recipients, headers, fp, subgroups=None, named_subgroups=None): "Add email to file storage and return storage id." super( FileMailStorage, self).add( - queueid, + qid, mailfrom, recipients, headers, fp, subgroups, named_subgroups) - storage_id = "{}_{}".format( - datetime.now().strftime("%Y%m%d%H%M%S"), queueid) + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + storage_id = f"{timestamp}_{qid}" # save mail self._save_datafile(storage_id, fp) @@ -142,7 +139,7 @@ class FileMailStorage(BaseMailStorage): "recipients": recipients, "headers": headers, "date": timegm(gmtime()), - "queue_id": queueid, + "queue_id": qid, "subgroups": subgroups, "named_subgroups": named_subgroups } @@ -160,21 +157,19 @@ class FileMailStorage(BaseMailStorage): super(FileMailStorage, self).get_metadata(storage_id) metafile = os.path.join( - self.directory, "{}{}".format( - storage_id, self._metadata_suffix)) + self.directory, f"{storage_id}{self._metadata_suffix}") if not os.path.isfile(metafile): raise RuntimeError( - "invalid storage id '{}'".format(storage_id)) + f"invalid storage id '{storage_id}'") try: with open(metafile, "r") as f: metadata = json.load(f) except IOError as e: - raise RuntimeError("unable to read metadata file: {}".format(e)) + raise RuntimeError(f"unable to read metadata file: {e}") except json.JSONDecodeError as e: raise RuntimeError( - "invalid meta file '{}': {}".format( - metafile, e)) + f"invalid metafile '{metafile}': {e}") return metadata @@ -188,7 +183,7 @@ class FileMailStorage(BaseMailStorage): emails = {} metafiles = glob(os.path.join( - self.directory, "*{}".format(self._metadata_suffix))) + self.directory, f"*{self._metadata_suffix}")) for metafile in metafiles: if not os.path.isfile(metafile): continue @@ -222,7 +217,7 @@ class FileMailStorage(BaseMailStorage): try: metadata = self.get_metadata(storage_id) except RuntimeError as e: - raise RuntimeError("unable to delete email: {}".format(e)) + raise RuntimeError(f"unable to delete email: {e}") if not recipients: self._remove(storage_id) @@ -231,7 +226,7 @@ class FileMailStorage(BaseMailStorage): recipients = [recipients] for recipient in recipients: if recipient not in metadata["recipients"]: - raise RuntimeError("invalid recipient '{}'".format(recipient)) + raise RuntimeError(f"invalid recipient '{recipient}'") metadata["recipients"].remove(recipient) if not metadata["recipients"]: self._remove(storage_id) @@ -246,7 +241,7 @@ class FileMailStorage(BaseMailStorage): try: fp = open(datafile, "rb") except IOError as e: - raise RuntimeError("unable to open email data file: {}".format(e)) + raise RuntimeError(f"unable to open email data file: {e}") return (fp, metadata) diff --git a/pyquarantine/version.py b/pyquarantine/version.py index 5becc17..5c4105c 100644 --- a/pyquarantine/version.py +++ b/pyquarantine/version.py @@ -1 +1 @@ -__version__ = "1.0.0" +__version__ = "1.0.1" diff --git a/pyquarantine/whitelists.py b/pyquarantine/whitelists.py index ad66cf0..e182bb4 100644 --- a/pyquarantine/whitelists.py +++ b/pyquarantine/whitelists.py @@ -97,8 +97,7 @@ class DatabaseWhitelist(WhitelistBase): cfg[opt] = defaults[opt] else: raise RuntimeError( - "mandatory option '{}' not present in config section '{}' or 'global'".format( - opt, self.name)) + f"mandatory option '{opt}' not present in config section '{self.name}' or 'global'") tablename = cfg["whitelist_db_table"] connection_string = cfg["whitelist_db_connection"] @@ -108,16 +107,16 @@ class DatabaseWhitelist(WhitelistBase): else: try: # connect to database + conn = re.sub( + r"(.*?://.*?):.*?(@.*)", + r"\1:\2", + connection_string) self.logger.debug( - "connecting to database '{}'".format( - re.sub( - r"(.*?://.*?):.*?(@.*)", - r"\1:\2", - connection_string))) + f"connecting to database '{conn}'") db = connect(connection_string) except Exception as e: raise RuntimeError( - "unable to connect to database: {}".format(e)) + f"unable to connect to database: {e}") DatabaseWhitelist._db_connections[connection_string] = db @@ -125,7 +124,7 @@ class DatabaseWhitelist(WhitelistBase): self.meta = Meta self.meta.database = db self.meta.table_name = tablename - self.model = type("WhitelistModel_{}".format(self.name), (WhitelistModel,), { + self.model = type(f"WhitelistModel_{self.name}", (WhitelistModel,), { "Meta": self.meta }) @@ -139,8 +138,7 @@ class DatabaseWhitelist(WhitelistBase): db.create_tables([self.model]) except Exception as e: raise RuntimeError( - "unable to initialize table '{}': {}".format( - tablename, e)) + f"unable to initialize table '{tablename}': {e}") def _entry_to_dict(self, entry): result = {} @@ -170,17 +168,18 @@ class DatabaseWhitelist(WhitelistBase): # generate list of possible mailfroms self.logger.debug( - "query database for whitelist entries from <{}> to <{}>".format( - mailfrom, recipient)) + f"query database for whitelist entries from <{mailfrom}> to <{recipient}>") mailfroms = [""] if "@" in mailfrom and not mailfrom.startswith("@"): - mailfroms.append("@{}".format(mailfrom.split("@")[1])) + domain = mailfrom.split("@")[1] + mailfroms.append(f"@{domain}") mailfroms.append(mailfrom) # generate list of possible recipients recipients = [""] if "@" in recipient and not recipient.startswith("@"): - recipients.append("@{}".format(recipient.split("@")[1])) + domain = recipient.split("@")[1] + recipients.append(f"@{domain}") recipients.append(recipient) # query the database @@ -190,7 +189,7 @@ class DatabaseWhitelist(WhitelistBase): self.model.mailfrom.in_(mailfroms), self.model.recipient.in_(recipients))) except Exception as e: - raise RuntimeError("unable to query database: {}".format(e)) + raise RuntimeError(f"unable to query database: {e}") if not entries: # no whitelist entry found @@ -235,7 +234,7 @@ class DatabaseWhitelist(WhitelistBase): entries.update(self._entry_to_dict(entry)) except Exception as e: - raise RuntimeError("unable to query database: {}".format(e)) + raise RuntimeError(f"unable to query database: {e}") return entries @@ -256,7 +255,7 @@ class DatabaseWhitelist(WhitelistBase): comment=comment, permanent=permanent) except Exception as e: - raise RuntimeError("unable to add entry to database: {}".format(e)) + raise RuntimeError(f"unable to add entry to database: {e}") def delete(self, whitelist_id): "Delete entry from whitelist." @@ -267,7 +266,7 @@ class DatabaseWhitelist(WhitelistBase): deleted = query.execute() except Exception as e: raise RuntimeError( - "unable to delete entry from database: {}".format(e)) + f"unable to delete entry from database: {e}") if deleted == 0: raise RuntimeError("invalid whitelist id")