From b41ae3033555367ad9277b8d42d1d78142576cf7 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 28 Aug 2019 16:07:26 +0200 Subject: [PATCH] improve handling of template variables --- README.md | 10 ++++-- docs/pyquarantine.conf.example | 15 +++++--- docs/templates/notification.template | 4 +++ pyquarantine/__init__.py | 20 ++++++----- pyquarantine/cli.py | 2 +- pyquarantine/notifications.py | 52 +++++++++++++++++++--------- pyquarantine/quarantines.py | 10 +++--- 7 files changed, 75 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index eb3dddc..8d9d125 100644 --- a/README.md +++ b/README.md @@ -73,8 +73,10 @@ The following configuration options are optional in each quarantine section: Quarantine e-mail notifications are sent to recipients. The SMTP host and port, E-mail template, from-address and the subject are configurable for each quarantine. The templates must contain the notification e-mail text in HTML form. The following template variables are available: + * **{EMAIL_ENVELOPE_FROM}** + E-mail from-address received by the milter. * **{EMAIL_FROM}** - E-mail from-address received by the milter (envelope-from). + Value of the from header of the original e-mail. * **{EMAIL_TO}** E-mail recipient address of this notification. * **{EMAIL_SUBJECT}** @@ -92,10 +94,12 @@ The following configuration options are optional in each quarantine section: Content of a named subgroup, 'subgroup_name' will be replaced by its name. The following configuration options are mandatory for this notification type: + * **notification_email_envelope_from** + Notification e-mail envelope from-address. * **notification_email_from** - Notification e-mail from-address. + Value of the notification e-mail from header. Every e-mail template variable described above is usable. * **notification_email_subject** - Notification e-mail subject. + Notification e-mail subject. All e-mail template variable described above is usable. * **notification_email_template** Path to the notification e-mail template. It is hold in memory during runtime. * **notification_email_replacement_img** diff --git a/docs/pyquarantine.conf.example b/docs/pyquarantine.conf.example index 84f4faa..2b03d4c 100644 --- a/docs/pyquarantine.conf.example +++ b/docs/pyquarantine.conf.example @@ -82,12 +82,19 @@ reject_reason = Message rejected # notification_type = email -# Option: notification_email_from -# Notes: Set the from address used when sending notification emails. +# Option: notification_email_envelope_from +# Notes: Set the envelope-from address used when sending notification emails. # This option is needed by notification type 'email'. -# Values: [ FROM_ADDRESS ] +# Values: [ ENVELOPE_FROM_ADDRESS ] # -notification_email_from = notification@domain.tld +notification_email_envelope_from = notification@domain.tld + +# Option: notification_email_from +# Notes: Set the from header used when sending notification emails. +# This option is needed by notification type 'email'. +# Values: [ FROM_HEADER ] +# +notification_email_from = Notification # Option: notification_email_usbject # Notes: Set the subject used when sending notification emails. diff --git a/docs/templates/notification.template b/docs/templates/notification.template index e1419ca..3c62ef2 100644 --- a/docs/templates/notification.template +++ b/docs/templates/notification.template @@ -2,6 +2,10 @@

Quarantine notification

+ + + + diff --git a/pyquarantine/__init__.py b/pyquarantine/__init__.py index e06889e..2739838 100644 --- a/pyquarantine/__init__.py +++ b/pyquarantine/__init__.py @@ -22,6 +22,7 @@ import re import sys from Milter.utils import parse_addr +from collections import defaultdict from io import BytesIO from itertools import groupby @@ -87,14 +88,11 @@ class QuarantineMilter(Milter.Base): self.logger.debug("{}: received queue-id from MTA".format(self.queueid)) self.recipients = list(self.recipients) self.headers = [] - self.subject = "" return Milter.CONTINUE @Milter.noreply def header(self, name, value): - self.headers.append("{}: {}".format(name, value)) - if name.lower() == "subject": - self.subject = value + self.headers.append((name, value)) return Milter.CONTINUE def eoh(self): @@ -107,7 +105,8 @@ class QuarantineMilter(Milter.Base): # iterate email headers recipients_to_check = self.recipients.copy() - for header in self.headers: + for name, value in self.headers: + header = "{}: {}".format(name, value) self.logger.debug("{}: checking header against configured quarantines: {}".format(self.queueid, header)) # iterate quarantines for quarantine in self.config: @@ -174,7 +173,9 @@ class QuarantineMilter(Milter.Base): # initialize memory buffer to save email data self.fp = BytesIO() # write email headers to memory buffer - self.fp.write("{}\n".format("\n".join(self.headers)).encode()) + for name, value in self.headers: + self.fp.write("{}: {}\n".format(name, value).encode()) + self.fp.write("\n".encode()) else: # quarantine and notification are disabled on all matching quarantines, return configured action quarantine = self._get_preferred_quarantine() @@ -211,6 +212,9 @@ class QuarantineMilter(Milter.Base): # iterate quarantines sorted by index for quarantine, recipients in sorted(quarantines, key=lambda x: x[0]["index"]): quarantine_id = "" + headers = defaultdict(str) + for name, value in self.headers: + headers[name.lower()] = value subgroups = self.quarantines_matches[quarantine["name"]].groups(default="") named_subgroups = self.quarantines_matches[quarantine["name"]].groupdict(default="") @@ -219,7 +223,7 @@ class QuarantineMilter(Milter.Base): # add email to quarantine self.logger.info("{}: adding to quarantine '{}' for: {}".format(self.queueid, quarantine["name"], ", ".join(recipients))) try: - quarantine_id = quarantine["quarantine_obj"].add(self.queueid, self.mailfrom, recipients, self.subject, self.fp, + quarantine_id = quarantine["quarantine_obj"].add(self.queueid, self.mailfrom, recipients, headers, self.fp, subgroups, named_subgroups) except RuntimeError as e: self.logger.error("{}: unable to add to quarantine '{}': {}".format(self.queueid, quarantine["name"], e)) @@ -230,7 +234,7 @@ class QuarantineMilter(Milter.Base): # notify self.logger.info("{}: sending notification for quarantine '{}' to: {}".format(self.queueid, quarantine["name"], ", ".join(recipients))) try: - quarantine["notification_obj"].notify(self.queueid, quarantine_id, self.subject, self.mailfrom, recipients, self.fp, + quarantine["notification_obj"].notify(self.queueid, quarantine_id, self.mailfrom, recipients, headers, self.fp, subgroups, named_subgroups) except RuntimeError as e: self.logger.error("{}: unable to send notification for quarantine '{}': {}".format(self.queueid, quarantine["name"], e)) diff --git a/pyquarantine/cli.py b/pyquarantine/cli.py index a80bf50..4bcc18e 100644 --- a/pyquarantine/cli.py +++ b/pyquarantine/cli.py @@ -106,7 +106,7 @@ def list_quarantine_emails(config, args): row["date"] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(metadata["date"])) row["mailfrom"] = metadata["mailfrom"] row["recipient"] = metadata["recipients"].pop(0) - row["subject"] = emails[quarantine_id]["subject"][:60] + row["subject"] = emails[quarantine_id]["headers"]["subject"][:60] rows.append(row) if metadata["recipients"]: diff --git a/pyquarantine/notifications.py b/pyquarantine/notifications.py index 1d2f732..278cc4f 100644 --- a/pyquarantine/notifications.py +++ b/pyquarantine/notifications.py @@ -18,6 +18,7 @@ import re from bs4 import BeautifulSoup from cgi import escape +from collections import defaultdict from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.mime.image import MIMEImage @@ -33,7 +34,7 @@ class BaseNotification(object): self.config = config self.logger = logging.getLogger(__name__) - def notify(self, queueid, quarantine_id, subject, mailfrom, recipients, fp, subgroups=None, named_subgroups=None, synchronous=False): + def notify(self, queueid, quarantine_id, mailfrom, recipients, headers, fp, subgroups=None, named_subgroups=None, synchronous=False): fp.seek(0) pass @@ -106,7 +107,7 @@ class EMailNotification(BaseNotification): super(EMailNotification, self).__init__(global_config, config, configtest) # check if mandatory options are present in config - for option in ["smtp_host", "smtp_port", "notification_email_from", "notification_email_subject", "notification_email_template", "notification_email_replacement_img", "notification_email_embedded_imgs"]: + for option in ["smtp_host", "smtp_port", "notification_email_envelope_from", "notification_email_from", "notification_email_subject", "notification_email_template", "notification_email_replacement_img", "notification_email_embedded_imgs"]: if option not in self.config.keys() and option in self.global_config.keys(): self.config[option] = self.global_config[option] if option not in self.config.keys(): @@ -114,16 +115,31 @@ class EMailNotification(BaseNotification): self.smtp_host = self.config["smtp_host"] self.smtp_port = self.config["smtp_port"] - self.mailfrom = self.config["notification_email_from"] + self.mailfrom = self.config["notification_email_envelope_from"] + self.from_header = self.config["notification_email_from"] self.subject = self.config["notification_email_subject"] + testvars = defaultdict(str, test="TEST") + + # test-parse from header + try: + self.from_header.format_map(testvars) + except ValueError as e: + raise RuntimeError("error parsing notification_email_from: {}".format(e)) + + # test-parse subject + try: + self.subject.format_map(testvars) + except ValueError as e: + raise RuntimeError("error parsing notification_email_subject: {}".format(e)) + # read and parse email notification template try: self.template = open(self.config["notification_email_template"], "r").read() - self.template.format(TEST="test") + self.template.format_map(testvars) except IOError as e: raise RuntimeError("error reading template: {}".format(e)) - except KeyError as e: + except ValueError as e: raise RuntimeError("error parsing template: {}".format(e)) # read email replacement image if specified @@ -227,9 +243,9 @@ class EMailNotification(BaseNotification): return soup - def notify(self, queueid, quarantine_id, subject, mailfrom, recipients, fp, subgroups=None, named_subgroups=None, synchronous=False): + def notify(self, queueid, quarantine_id, mailfrom, recipients, headers, fp, subgroups=None, named_subgroups=None, synchronous=False): "Notify recipients via email." - super(EMailNotification, self).notify(queueid, quarantine_id, subject, mailfrom, recipients, fp, subgroups, named_subgroups, synchronous) + super(EMailNotification, self).notify(queueid, quarantine_id, mailfrom, recipients, headers, fp, subgroups, named_subgroups, synchronous) # extract html text from email self.logger.debug("{}: extraction email text from original email".format(queueid)) @@ -246,6 +262,7 @@ class EMailNotification(BaseNotification): # sanitizing email text of original email sanitized_text = self.sanitize(queueid, soup) + del soup # sending email notifications for recipient in recipients: @@ -253,13 +270,14 @@ class EMailNotification(BaseNotification): self.logger.debug("{}: parsing email template".format(queueid)) # generate dict containing all template variables - variables = { - "EMAIL_HTML_TEXT": sanitized_text, - "EMAIL_FROM": escape(mailfrom), - "EMAIL_TO": escape(recipient), - "EMAIL_SUBJECT": escape(subject), - "EMAIL_QUARANTINE_ID": quarantine_id - } + variables = defaultdict(str, + EMAIL_HTML_TEXT=sanitized_text, + EMAIL_FROM=escape(headers["from"]), + EMAIL_ENVELOPE_FROM=escape(mailfrom), + EMAIL_TO=escape(recipient), + EMAIL_SUBJECT=escape(headers["subject"]), + EMAIL_QUARANTINE_ID=quarantine_id) + if subgroups: number = 0 for subgroup in subgroups: @@ -269,11 +287,11 @@ class EMailNotification(BaseNotification): variables.update(named_subgroups) # parse template - htmltext = self.template.format(**variables) + htmltext = self.template.format_map(variables) msg = MIMEMultipart('alternative') - msg["Subject"] = self.subject - msg["From"] = "<{}>".format(self.mailfrom) + msg["Subject"] = self.subject.format_map(variables) + msg["From"] = "<{}>".format(self.from_header.format_map(variables)) msg["To"] = "<{}>".format(recipient) msg["Date"] = email.utils.formatdate() msg.attach(MIMEText(htmltext, "html", 'UTF-8')) diff --git a/pyquarantine/quarantines.py b/pyquarantine/quarantines.py index c05941a..b6a326c 100644 --- a/pyquarantine/quarantines.py +++ b/pyquarantine/quarantines.py @@ -33,7 +33,7 @@ class BaseQuarantine(object): self.config = config self.logger = logging.getLogger(__name__) - def add(self, queueid, mailfrom, recipients, subject, fp, subgroups=None, named_subgroups=None): + def add(self, queueid, mailfrom, recipients, headers, fp, subgroups=None, named_subgroups=None): "Add email to quarantine." fp.seek(0) return "" @@ -109,9 +109,9 @@ class FileQuarantine(BaseQuarantine): except IOError as e: raise RuntimeError("unable to remove data file: {}".format(e)) - def add(self, queueid, mailfrom, recipients, subject, fp, subgroups=None, named_subgroups=None): + def add(self, queueid, mailfrom, recipients, headers, fp, subgroups=None, named_subgroups=None): "Add email to file quarantine and return quarantine-id." - super(FileQuarantine, self).add(queueid, mailfrom, recipients, subject, fp, subgroups, named_subgroups) + super(FileQuarantine, self).add(queueid, mailfrom, recipients, headers, fp, subgroups, named_subgroups) quarantine_id = "{}_{}".format(datetime.now().strftime("%Y%m%d%H%M%S"), queueid) # save mail @@ -121,7 +121,7 @@ class FileQuarantine(BaseQuarantine): metadata = { "mailfrom": mailfrom, "recipients": recipients, - "subject": subject, + "headers": headers, "date": timegm(gmtime()), "queue_id": queueid, "subgroups": subgroups, @@ -225,7 +225,7 @@ class FileQuarantine(BaseQuarantine): datafile = os.path.join(self.directory, quarantine_id) try: with open(datafile, "rb") as fp: - self.config["notification_obj"].notify(metadata["queue_id"], quarantine_id, metadata["subject"], metadata["mailfrom"], recipients, fp, + self.config["notification_obj"].notify(metadata["queue_id"], quarantine_id, metadata["mailfrom"], recipients, metadata["headers"], fp, metadata["subgroups"], metadata["named_subgroups"], synchronous=True) except IOError as e: raise(RuntimeError("unable to read data file: {}".format(e)))
Envelope-From:{EMAIL_ENVELOPE_FROM}
From: {EMAIL_FROM}