diff --git a/README.md b/README.md
index c9a2aaf..fddf458 100644
--- a/README.md
+++ b/README.md
@@ -24,10 +24,12 @@ Any available configuration option can be set in the global section as default i
The following configuration options are mandatory in the global section:
* **quarantines**
Comma-separated, ordered list of active quarantines. For each, there must be a section of the same name in the configuration.
-* **smtp_host**
- SMTP host to inject original e-mails. This is needed if not all recipients of an e-mail are whitelisted
-* **smtp_port**
- SMTP port
+* **preferred_quarantin_action**
+ Defines which quarantine action should be preferred if multiple quarantines are matching for multiple recipients.
+ If at least one recipient receives the original e-mail due to whitelisting, the action is always ACCEPT.
+ Possible values are:
+ * **first**
+ * **last**
### Quarantine sections
The following configuration options are mandatory in each quarantine section:
@@ -41,6 +43,10 @@ The following configuration options are mandatory in each quarantine section:
One of the notification types described below.
* **whitelist_type**
One of the whitelist types described below.
+* **smtp_host**
+ SMTP host to inject original e-mails. This is needed if not all recipients of an e-mail are whitelisted
+* **smtp_port**
+ SMTP port
### Quarantine types
* **NONE**
diff --git a/docs/pyquarantine.conf.example b/docs/pyquarantine.conf.example
index f89aa38..1853677 100644
--- a/docs/pyquarantine.conf.example
+++ b/docs/pyquarantine.conf.example
@@ -19,6 +19,15 @@
#
quarantines = spam
+# Option: quarantine_action_precedence
+# Notes: Set if the action of the first or the last matching quarantine should
+# be used if multiple recipients match multiple quarantines. If an original
+# email is delivered to at least one recipient due to whitelisting, the
+# email will always be accepted.
+# Values: [ first | last ]
+#
+preferred_quarantine_action = last
+
# Option: smtp_host
# Notes: Set the SMTP host. It will be used to (re-)inject emails.
# Values: [ HOSTNAME | IP_ADDRESS ]
diff --git a/docs/templates/notification.template b/docs/templates/notification.template
index 985ae5e..2dba60d 100644
--- a/docs/templates/notification.template
+++ b/docs/templates/notification.template
@@ -1,10 +1,25 @@
-This is a spam notification mail
-From: {EMAIL_FROM}
-To: {EMAIL_TO}
-Subject: {EMAIL_SUBJECT}
-Quarantine-ID: {EMAIL_QUARANTINE_ID}
-{EMAIL_HTML_TEXT}
+ Quarantine ntification
+
+
+ | From: |
+ {EMAIL_FROM} |
+
+
+ | To: |
+ {EMAIL_TO} |
+
+
+ | Subject: |
+ {EMAIL_SUBJECT} |
+
+
+ | Quarantine ID: |
+ {EMAIL_QUARANTINE_ID} |
+
+
+ Preview of the original e-mail
+ {EMAIL_HTML_TEXT}
diff --git a/pyquarantine/__init__.py b/pyquarantine/__init__.py
index 2f35e32..98c663c 100644
--- a/pyquarantine/__init__.py
+++ b/pyquarantine/__init__.py
@@ -30,7 +30,7 @@ import notifications
import whitelists
from Milter.utils import parse_addr
-
+from itertools import groupby
class QuarantineMilter(Milter.Base):
@@ -40,18 +40,27 @@ class QuarantineMilter(Milter.Base):
"""
config = None
+ global_config = None
# list of default config files
_config_files = ["/etc/pyquarantine/pyquarantine.conf", os.path.expanduser('~/pyquarantine.conf'), "pyquarantine.conf"]
# list of possible actions
_actions = {"ACCEPT": Milter.ACCEPT, "REJECT": Milter.REJECT, "DISCARD": Milter.DISCARD}
-
def __init__(self):
self.logger = logging.getLogger(__name__)
# save config, it must not change during runtime
+ self.global_config = QuarantineMilter.global_config
self.config = QuarantineMilter.config
+ def _get_preferred_quarantine(self):
+ matching_quarantines = [q for q in self.recipients_quarantines.values() if q]
+ if self.global_config["preferred_quarantine_action"] == "first":
+ quarantine = sorted(matching_quarantines, key=lambda x: x["index"])[0]
+ else:
+ quarantine = sorted(matching_quarantines, key=lambda x: x["index"], reverse=True)[0]
+ return quarantine
+
@staticmethod
def get_configfiles():
return QuarantineMilter._config_files
@@ -93,57 +102,89 @@ class QuarantineMilter(Milter.Base):
def eoh(self):
try:
- self.matched = None
self.whitelist_cache = whitelists.WhitelistCache()
+
+ # initialize a dict to set quaranines per recipient
+ self.recipients_quarantines = {}
+
# iterate email headers
+ recipients_to_check = self.recipients[:]
for header in self.headers:
- self.logger.debug("{}: checking header '{}' against regex of every configured quarantine".format(self.queueid, header))
+ self.logger.debug("{}: checking header against configured quarantines: {}".format(self.queueid, header))
# iterate quarantines
- for name, quarantine in self.config.items():
- if self.matched != None and quarantine["index"] == self.matched["index"]:
- # a quarantine with higher precedence already matched, skip checks of quarantines with lower precedence
- self.logger.debug("{}: quarantine '{}' matched already, skip further checks of this header".format(self.queueid, name))
- break
- self.logger.debug("{}: checking header against quarantine '{}'".format(self.queueid, name))
- # check if header matches regex
+ for quarantine in self.config:
+ if len(self.recipients_quarantines) == len(self.recipients):
+ # every recipient matched a quarantine already
+ if max([q["index"] for q in self.recipients_quarantines.values()]) <= quarantine["index"]:
+ # every recipient matched a quarantine with at least the same precedence already, skip checks against quarantines with lower precedence
+ self.logger.debug("{}: {}: skip further checks of this header".format(self.queueid, quarantine["name"]))
+ break
+
+ # check email header against quarantine regex
+ self.logger.debug("{}: {}: checking header against regex '{}'".format(self.queueid, quarantine["name"], quarantine["regex"]))
if quarantine["regex_compiled"].match(header):
+ self.logger.debug("{}: {}: header matched regex".format(self.queueid, quarantine["name"]))
+
+ # check for whitelisted recipients
whitelist = quarantine["whitelist_obj"]
if whitelist != None:
try:
- whitelisted_recipients = self.whitelist_cache.get_whitelisted_recipients(whitelist, self.mailfrom, self.recipients)
+ whitelisted_recipients = self.whitelist_cache.get_whitelisted_recipients(whitelist, self.mailfrom, recipients_to_check)
except RuntimeError as e:
- self.logger.error("{}: unable to query whitelist: {}".format(self.queueid, e))
+ self.logger.error("{}: {}: unable to query whitelist: {}".format(self.queueid, quarantine["name"], e))
return Milter.TEMPFAIL
- if len(whitelisted_recipients) == len(self.recipients):
- # all recipients are whitelisted, continue with header checks
- self.logger.debug("{}: header matched regex, but all recipients are whitelisted in quarantine '{}', continue checking this header".format(self.queueid, name))
+ else:
+ whitelisted_recipients = {}
+
+ # iterate recipients
+ for recipient in recipients_to_check[:]:
+
+ if recipient in whitelisted_recipients:
+ # recipient is whitelisted in this quarantine
+ self.logger.debug("{}: {}: recipient '{}' is whitelisted".format(self.queueid, quarantine["name"], recipient))
continue
- self.matched = quarantine
- # skip checks of this header with quarantines with lower precedence
- self.logger.debug("{}: header matched regex in quarantine '{}', further checks of this header will be skipped".format(self.queueid, name))
- break
- if self.matched != None and self.matched["index"] == 0:
- self.logger.debug("{}: skipping checks of remaining headers, the quarantine with highest precedence matched already".format(self.queueid))
+
+ if recipient not in self.recipients_quarantines.keys() or self.recipients_quarantines[recipient]["index"] > quarantine["index"]:
+ self.logger.debug("{}: {}: set quarantine for recipient '{}'".format(self.queueid, quarantine["name"], recipient))
+ self.recipients_quarantines[recipient] = quarantine
+ if quarantine["index"] == 0:
+ # we do not need to check recipients which matched the quarantine with the highest precedence already
+ recipients_to_check.remove(recipient)
+ else:
+ self.logger.debug("{}: {}: a quarantine with the same or higher precedence matched already for recipient '{}'".format(self.queueid, quarantine["name"], recipient))
+
+ if not recipients_to_check:
+ self.logger.debug("{}: all recipients matched the first quarantine, skipping all remaining header checks".format(self.queueid))
break
- if self.matched != None:
- self.logger.info("{}: email matched quarantine '{}'".format(self.queueid, self.matched["name"]))
- # one of the configured quarantines matched
- if self.matched["quarantine_obj"] != None or self.matched["notification_obj"] != None:
- self.logger.debug("{}: initializing memory buffer to save email data".format(self.queueid))
- # quarantine or notification configured, initialize memory buffer to save mail
- self.fp = StringIO.StringIO()
- # write email headers to memory buffer
- self.fp.write("{}\n".format("\n".join(self.headers)))
- else:
- # quarantine and notification disabled, return configured action
- self.logger.debug("{}: ".format(self.queueid))
- self.logger.info("{}: quarantine and notification disabled, responding with configured action: {}".format(self.queueid, self.matched["action"].upper()))
- return self.matched["milter_action"]
- else:
- # no quarantine matched, accept mail
- self.logger.info("{}: email passed clean".format(self.queueid))
+
+ # check if no quarantine has matched for all recipients
+ if not self.recipients_quarantines:
+ # accept email
+ self.logger.info("{}: passed clean for all recipients".format(self.queueid))
return Milter.ACCEPT
+
+ # check if the email body is needed
+ keep_body = False
+ for recipient, quarantine in self.recipients_quarantines.items():
+ if quarantine["quarantine_obj"] or quarantine["notification_obj"]:
+ keep_body = True
+ break
+
+ if keep_body:
+ self.logger.debug("{}: initializing memory buffer to save email data".format(self.queueid))
+ # initialize memory buffer to save email data
+ self.fp = StringIO.StringIO()
+ # write email headers to memory buffer
+ self.fp.write("{}\n".format("\n".join(self.headers)))
+ else:
+ # quarantine and notification are disabled on all matching quarantines, return configured action
+ quarantine = self._get_preferred_quarantine()
+ self.logger.info("{}: {} matching quarantine is '{}', performing milter action {}".format(self.queueid, self.global_config["preferred_quarantine_action"],
+ quarantine["name"], quarantine["action"].upper()))
+ return quarantine["milter_action"]
+
return Milter.CONTINUE
+
except Exception as e:
self.logger.exception("an exception occured in eoh function: {}".format(e))
return Milter.TEMPFAIL
@@ -159,46 +200,54 @@ class QuarantineMilter(Milter.Base):
def eom(self):
try:
- if self.matched["whitelist_obj"] != None:
- try:
- whitelisted_recipients = self.whitelist_cache.get_whitelisted_recipients(self.matched["whitelist_obj"], self.mailfrom, self.recipients)
- except RuntimeError as e:
- self.logger.error("{}: unable to query whitelist: {}".format(self.queueid, e))
- return Milter.TEMPFAIL
- if len(whitelisted_recipients) > 0:
- for recipient in whitelisted_recipients:
- self.recipients.remove(recipient)
- self.fp.seek(0)
- self.logger.info("{}: sending original email to whitelisted recipient(s): {}".format(self.queueid, "<{}>".format(">,<".join(whitelisted_recipients))))
- try:
- mailer.sendmail(self.matched["smtp_host"], self.matched["smtp_port"], self.queueid, self.mailfrom, whitelisted_recipients, self.fp.read())
- except RuntimeError as e:
- self.logger.error("{}: unable to send original email: {}".format(self.queueid, e))
- return Milter.TEMPFAIL
- if len(self.recipients) > 0:
+ # processing recipients grouped by quarantines
+ quarantines = []
+ keyfunc = lambda x: self.recipients_quarantines[x]
+ for quarantine, recipients in groupby(sorted(self.recipients_quarantines, key=keyfunc), keyfunc):
+ quarantines.append((quarantine, list(recipients)))
+
+ # iterate quarantines sorted by index
+ for quarantine, recipients in sorted(quarantines, key=lambda x: x[0]["index"]):
quarantine_id = ""
- if self.matched["quarantine_obj"] != None:
+
+ # check if a quarantine is configured
+ if quarantine["quarantine_obj"] != None:
# add email to quarantine
- self.fp.seek(0)
- self.logger.info("{}: adding email to quarantine of recipient(s): {}".format(self.queueid, "<{}>".format(">,<".join(self.recipients))))
+ self.logger.info("{}: adding to quarantine '{}' for: {}".format(self.queueid, quarantine["name"], ", ".join(recipients)))
try:
- quarantine_id = self.matched["quarantine_obj"].add(self.queueid, self.mailfrom, self.recipients, fp=self.fp)
+ quarantine_id = quarantine["quarantine_obj"].add(self.queueid, self.mailfrom, recipients, fp=self.fp)
except RuntimeError as e:
- self.logger.error("{}: unable to add email to quarantine: {}".format(self.queueid, e))
+ self.logger.error("{}: unable to add to quarantine '{}': {}".format(self.queueid, quarantine["name"], e))
return Milter.TEMPFAIL
- if self.matched["notification_obj"] != None:
+
+ # check if a notification is configured
+ if quarantine["notification_obj"] != None:
# notify
- self.fp.seek(0)
- self.logger.info("{}: sending notification(s) to: {}".format(self.queueid, "<{}>".format(">,<".join(self.recipients))))
+ self.logger.info("{}: sending notification for quarantine '{}' to: {}".format(self.queueid, quarantine["name"], ", ".join(recipients)))
try:
- self.matched["notification_obj"].notify(self.queueid, quarantine_id, self.subject, self.mailfrom, self.recipients, fp=self.fp)
+ quarantine["notification_obj"].notify(self.queueid, quarantine_id, self.subject, self.mailfrom, recipients, fp=self.fp)
except RuntimeError as e:
- self.logger.error("{}: unable to send notification(s): {}".format(self.queueid, e))
+ self.logger.error("{}: unable to send notification for quarantine '{}': {}".format(self.queueid, quarantine["name"], e))
return Milter.TEMPFAIL
+
+ # remove processed recipient
+ for recipient in recipients:
+ self.delrcpt(recipient)
+ self.recipients.remove(recipient)
+
self.fp.close()
- # return configured action
- self.logger.info("{}: responding with configured action: {}".format(self.queueid, self.matched["action"].upper()))
- return self.matched["milter_action"]
+
+ # email passed clean for at least one recipient, accepting email
+ if self.recipients:
+ self.logger.info("{}: passed clean for: {}".format(self.queueid, ", ".join(self.recipients)))
+ return Milter.ACCEPT
+
+ ## return configured action
+ quarantine = self._get_preferred_quarantine()
+ self.logger.info("{}: {} matching quarantine is '{}', performing milter action {}".format(self.queueid, self.global_config["preferred_quarantine_action"],
+ quarantine["name"], quarantine["action"].upper()))
+ return quarantine["milter_action"]
+
except Exception as e:
self.logger.exception("an exception occured in eom function: {}".format(e))
return Milter.TEMPFAIL
@@ -207,107 +256,131 @@ class QuarantineMilter(Milter.Base):
def generate_milter_config(configtest=False, config_files=[]):
"Generate the configuration for QuarantineMilter class."
logger = logging.getLogger(__name__)
+
# read config file
parser = ConfigParser.ConfigParser()
- if len(config_files) == 0:
+ if not config_files:
config_files = parser.read(QuarantineMilter.get_configfiles())
else:
config_files = parser.read(config_files)
- if len(config_files) == 0:
+ if not config_files:
raise RuntimeError("config file not found")
+
QuarantineMilter.set_configfiles(config_files)
os.chdir(os.path.dirname(config_files[0]))
+
# check if mandatory config options in global section are present
if "global" not in parser.sections():
raise RuntimeError("mandatory section 'global' not present in config file")
- for option in ["quarantines"]:
+ for option in ["quarantines", "preferred_quarantine_action"]:
if not parser.has_option("global", option):
raise RuntimeError("mandatory option '{}' not present in config section 'global'".format(option))
- config = {}
- config["global"] = dict(parser.items("global"))
- # iterate configured quarantines
- quarantine_names = list(set(map(str.strip, parser.get("global", "quarantines").split(","))))
+
+ # read global config section
+ global_config = dict(parser.items("global"))
+ global_config["preferred_quarantine_action"] = global_config["preferred_quarantine_action"].lower()
+ if global_config["preferred_quarantine_action"] not in ["first", "last"]:
+ raise RuntimeError("option preferred_quarantine_action has illegal value")
+
+ # read active quarantine names
+ quarantine_names = list(set(map(str.strip, global_config["quarantines"].split(","))))
if "global" in quarantine_names:
+ quarantine_names.remove("global")
logger.warning("removed illegal quarantine name 'global' from list of active quarantines")
- del(quarantine_names["global"])
- if len(quarantine_names) == 0:
+ if not quarantine_names:
raise RuntimeError("no quarantines configured")
- idx = 0
- for name in quarantine_names:
- name = name.strip()
- # check if config section for current quarantine is present
- if name not in parser.sections():
- raise RuntimeError("config section '{}' is not present".format(name))
- config[name] = dict(parser.items(name))
- config[name]["name"] = name
+
+ milter_config = []
+
+ logger.debug("preparing milter configuration ...")
+ # iterate quarantine names
+ for index, quarantine_name in enumerate(quarantine_names):
+
+ # check if config section for current quarantine exists
+ if quarantine_name not in parser.sections():
+ raise RuntimeError("config section '{}' does not exist".format(quarantine_name))
+ config = dict(parser.items(quarantine_name))
+
# check if mandatory config options are present in config
for option in ["regex", "quarantine_type", "notification_type", "action", "whitelist_type", "smtp_host", "smtp_port"]:
- if option not in config[name].keys() and \
- option in config["global"].keys():
- config[name][option] = config["global"][option]
- if option not in config[name].keys():
- raise RuntimeError("mandatory option '{}' not present in config section '{}' or 'global'".format(option, name))
- logger.debug("preparing configuration for quarantine '{}' ...".format(name))
- ## add the index
- config[name]["index"] = idx
- idx += 1
- # compile regex
- regex = config[name]["regex"]
- logger.debug("=> compiling regex '{}'".format(regex))
- config[name]["regex_compiled"] = re.compile(regex)
+ if option not in config.keys() and \
+ option in global_config.keys():
+ config[option] = global_config[option]
+ if option not in config.keys():
+ raise RuntimeError("mandatory option '{}' not present in config section '{}' or 'global'".format(option, quarantine_name))
+
+ # set quarantine name
+ config["name"] = quarantine_name
+
+ # set the index
+ config["index"] = index
+
+ # pre-compile regex
+ logger.debug("{}: compiling regex '{}'".format(quarantine_name, config["regex"]))
+ config["regex_compiled"] = re.compile(config["regex"])
+
# create quarantine instance
- quarantine_type = config[name]["quarantine_type"].lower()
- if quarantine_type in quarantines.quarantine_types.keys():
- logger.debug("=> initializing quarantine type '{}'".format(quarantine_type))
- quarantine = quarantines.quarantine_types[quarantine_type](name, config, configtest)
+ quarantine_type = config["quarantine_type"].lower()
+ if quarantine_type in quarantines.TYPES.keys():
+ logger.debug("{}: initializing quarantine type '{}'".format(quarantine_name, quarantine_type.upper()))
+ quarantine = quarantines.TYPES[quarantine_type](global_config, config, configtest)
elif quarantine_type == "none":
- logger.debug("=> setting quarantine to NONE")
+ logger.debug("{}: quarantine is NONE".format(quarantine_name))
quarantine = None
else:
- raise RuntimeError("unknown quarantine_type '{}'".format(quarantine_type))
- config[name]["quarantine_obj"] = quarantine
+ raise RuntimeError("{}: unknown quarantine type '{}'".format(quarantine_name, quarantine_type))
+
+ config["quarantine_obj"] = quarantine
+
# create whitelist instance
- whitelist_type = config[name]["whitelist_type"].lower()
- if whitelist_type in whitelists.whitelist_types.keys():
- logger.debug("=> initializing whitelist database")
- whitelist = whitelists.whitelist_types[whitelist_type](name, config, configtest)
- else:
- logger.debug("=> setting whitelist to NONE")
+ whitelist_type = config["whitelist_type"].lower()
+ if whitelist_type in whitelists.TYPES.keys():
+ logger.debug("{}: initializing whitelist type '{}'".format(quarantine_name, whitelist_type.upper()))
+ whitelist = whitelists.TYPES[whitelist_type](global_config, config, configtest)
+ elif whitelist_type == "none":
+ logger.debug("{}: whitelist is NONE".format(quarantine_name))
whitelist = None
- config[name]["whitelist_obj"] = whitelist
+ else:
+ raise RuntimeError("{}: unknown whitelist type '{}'".format(quarantine_name, whitelist_type))
+
+ config["whitelist_obj"] = whitelist
+
# create notification instance
- notification_type = config[name]["notification_type"].lower()
- if notification_type in notifications.notification_types.keys():
- logger.debug("=> initializing notification type '{}'".format(notification_type))
- notification = notifications.notification_types[notification_type](name, config, configtest)
+ notification_type = config["notification_type"].lower()
+ if notification_type in notifications.TYPES.keys():
+ logger.debug("{}: initializing notification type '{}'".format(quarantine_name, notification_type.upper()))
+ notification = notifications.TYPES[notification_type](global_config, config, configtest)
elif notification_type == "none":
- logger.debug("=> setting notification to NONE")
+ logger.debug("{}: notification is NONE".format(quarantine_name))
notification = None
else:
- raise RuntimeError("unknown notification type '{}'".format(notification_type))
- config[name]["notification_obj"] = notification
- # determining milter action for this quarantine
- action = config[name]["action"].upper()
- if action in QuarantineMilter.get_actions().keys():
- logger.debug("=> setting action to {}".format(action))
- config[name]["milter_action"] = QuarantineMilter.get_actions()[action]
- else:
- raise RuntimeError("unknown action '{}' configured for quarantine '{}'".format(action, name))
- # remove global section from config, every section should be a quarantine
- del(config["global"])
- return config
+ raise RuntimeError("{}: unknown notification type '{}'".format(quarantine_name, notification_type))
+ config["notification_obj"] = notification
+
+ # determining milter action for this quarantine
+ action = config["action"].upper()
+ if action in QuarantineMilter.get_actions().keys():
+ logger.debug("{}: action is {}".format(quarantine_name, action))
+ config["milter_action"] = QuarantineMilter.get_actions()[action]
+ else:
+ raise RuntimeError("{}: unknown action '{}'".format(quarantine_name, action))
+
+ milter_config.append(config)
+
+ return global_config, milter_config
def reload_config():
"Reload the configuration of QuarantineMilter class."
logger = logging.getLogger(__name__)
- logger.debug("received SIGUSR1")
+
try:
- config = generate_milter_config()
+ global_config, config = generate_milter_config()
except RuntimeError as e:
logger.info(e)
logger.info("daemon is still running with previous configuration")
else:
logger.info("reloading configuration")
+ QuarantineMilter.global_config = global_config
QuarantineMilter.config = config
diff --git a/pyquarantine/cli.py b/pyquarantine/cli.py
index 23b8748..66c9355 100644
--- a/pyquarantine/cli.py
+++ b/pyquarantine/cli.py
@@ -23,33 +23,37 @@ import time
import pyquarantine
-
def _get_quarantine_obj(config, quarantine):
- if quarantine not in config.keys():
+ try:
+ quarantine_obj = next((q["quarantine_obj"] for q in config if q["name"] == quarantine))
+ except StopIteration:
raise RuntimeError("invalid quarantine '{}'".format(quarantine))
- return config[quarantine]["quarantine_obj"]
-
+ return quarantine_obj
def _get_whitelist_obj(config, quarantine):
- if quarantine not in config.keys():
+ try:
+ whitelist_obj = next((q["whitelist_obj"] for q in config if q["name"] == quarantine))
+ except StopIteration:
raise RuntimeError("invalid quarantine '{}'".format(quarantine))
- return config[quarantine]["whitelist_obj"]
+ return whitelist_obj
-
-def print_table(headers, keys, data):
- if len(data) == 0:
+def print_table(columns, rows):
+ if not rows:
return
- # calculate length of each column
column_lengths = []
column_formats = []
- for idx, header in enumerate(headers):
- length = len(header)
- key = keys[idx]
- value_length=len((max(data.items(), key=lambda (k, v): len(v[key])))[1][key])
- if value_length > length: length = value_length
+
+ # iterate columns to display
+ for header, key in columns:
+ # get the length of the header string
+ lengths = [len(header)]
+ # get the length of the longest value
+ lengths.append(len(str(max(rows, key=lambda x: len(str(x[key])))[key])))
+ # use the the longer one
+ length = max(lengths)
column_lengths.append(length)
column_formats.append("{{:<{}}}".format(length))
@@ -63,29 +67,26 @@ def print_table(headers, keys, data):
separator = "-+-".join(separators)
# print header and separator
- print(row_format.format(*headers))
+ print(row_format.format(*[column[0] for column in columns]))
print(separator)
- # print data
- for key, value in data.items():
+ keys = [ entry[1] for entry in columns ]
+ # print rows
+ for entry in rows:
row = []
- for entry in keys:
- row.append(value[entry])
+ for key in keys:
+ row.append(entry[key])
print(row_format.format(*row))
-
def list_quarantines(config, args):
if args.batch:
- print("\n".join(config.keys()))
+ print("\n".join([ quarantine["name"] for quarantine in config ]))
else:
print_table(
- ["Name", "Quarantine", "Notification", "Action"],
- ["name", "quarantine_type", "notification_type", "action"],
- config
+ [("Name", "name"), ("Quarantine", "quarantine_type"), ("Notification", "notification_type"), ("Action", "action")],
+ config
)
- return 0
-
def list_quarantine_emails(config, args):
@@ -104,15 +105,15 @@ def list_quarantine_emails(config, args):
emails[quarantine_id]["date_str"] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(metadata["date"]))
if args.batch:
+ # batch mode, print quarantine IDs, each on a new line
print("\n".join(emails.keys()))
- else:
- if len(emails) == 0: logger.info("quarantine '{}' is empty".format(args.quarantine))
- print_table(
- ["Quarantine-ID", "From", "Recipient(s)", "Date"],
- ["quarantine_id", "from", "recipient_str", "date_str"],
- emails
- )
+ return
+ if not emails: logger.info("quarantine '{}' is empty".format(args.quarantine))
+ print_table(
+ [("Quarantine-ID", "quarantine_id"), ("From", "from"), ("Recipient(s)", "recipient_str"), ("Date", "date_str")],
+ emails.values()
+ )
def list_whitelist(config, args):
@@ -125,22 +126,23 @@ def list_whitelist(config, args):
# find whitelist entries
entries = whitelist.find(mailfrom=args.mailfrom, recipients=args.recipients, older_than=args.older_than)
- if len(entries) == 0:
+ if not entries:
logger.info("whitelist of quarantine '{}' is empty".format(args.quarantine))
- else:
- # transform some values to strings
- for entry_id, entry in entries.items():
- entries[entry_id]["id"] = str(entry["id"])
- entries[entry_id]["created_str"] = entry["created"].strftime('%Y-%m-%d %H:%M:%S')
- entries[entry_id]["last_used_str"] = entry["last_used"].strftime('%Y-%m-%d %H:%M:%S')
- entries[entry_id]["permanent_str"] = str(entry["permanent"])
+ return
- print_table(
- ["ID", "From", "To", "Created", "Last used", "Comment", "Permanent"],
- ["id", "mailfrom", "recipient", "created_str", "last_used_str", "comment", "permanent_str"],
- entries
- )
+ # transform some values to strings
+ for entry_id, entry in entries.items():
+ entries[entry_id]["permanent_str"] = str(entry["permanent"])
+ entries[entry_id]["created_str"] = entry["created"].strftime('%Y-%m-%d %H:%M:%S')
+ entries[entry_id]["last_used_str"] = entry["last_used"].strftime('%Y-%m-%d %H:%M:%S')
+ print_table(
+ [
+ ("ID", "id"), ("From", "mailfrom"), ("To", "recipient"), ("Created", "created_str"),
+ ("Last used", "last_used_str"), ("Comment", "comment"), ("Permanent", "permanent_str")
+ ],
+ entries.values()
+ )
def add_whitelist_entry(config, args):
@@ -153,58 +155,67 @@ def add_whitelist_entry(config, args):
# check existing entries
entries = whitelist.check(args.mailfrom, args.recipient)
- if len(entries) > 0:
+ if entries:
# check if the exact entry exists already
for entry in entries.values():
if entry["mailfrom"] == args.mailfrom and entry["recipient"] == args.recipient:
raise RuntimeError("an entry with this from/to combination already exists")
+
if not args.force:
# the entry is already covered by others
for entry_id, entry in entries.items():
+ entries[entry_id]["permanent_str"] = str(entry["permanent"])
entries[entry_id]["created_str"] = entry["created"].strftime('%Y-%m-%d %H:%M:%S')
entries[entry_id]["last_used_str"] = entry["last_used"].strftime('%Y-%m-%d %H:%M:%S')
- entries[entry_id]["permanent_str"] = str(entry["permanent"])
print_table(
- ["From", "To", "Created", "Last used", "Comment", "Permanent"],
- ["mailfrom", "recipient", "created_str", "last_used_str", "comment", "permanent_str"],
- entries
+ [
+ ("ID", "id"), ("From", "mailfrom"), ("To", "recipient"), ("Created", "created_str"),
+ ("Last used", "last_used_str"), ("Comment", "comment"), ("Permanent", "permanent_str")
+ ],
+ entries.values()
)
print("")
raise RuntimeError("from/to combination is already covered by the entries above, use --force to override.")
+
# add entry to whitelist
whitelist.add(args.mailfrom, args.recipient, args.comment, args.permanent)
logger.info("successfully added whitelist entry")
-
def delete_whitelist_entry(config, args):
logger = logging.getLogger(__name__)
+
whitelist = _get_whitelist_obj(config, args.quarantine)
if whitelist == None:
raise RuntimeError("whitelist type is set to None, unable to delete entries")
+
whitelist.delete(args.whitelist_id)
logger.info("successfully deleted whitelist entry")
-
def release_email(config, args):
logger = logging.getLogger(__name__)
+
quarantine = _get_quarantine_obj(config, args.quarantine)
if quarantine == None:
raise RuntimeError("quarantine type is set to None, unable to release e-mail")
+
quarantine.release(args.quarantine_id, args.recipient)
logger.info("successfully released e-mail [quarantine-id: {}] to '{}' from quarantine '{}'".format(args.quarantine_id, args.recipient, args.quarantine))
-
def delete_email(config, args):
logger = logging.getLogger(__name__)
+
quarantine = _get_quarantine_obj(config, args.quarantine)
if quarantine == None:
raise RuntimeError("quarantine type is set to None, unable to delete e-mail")
- quarantine.delete(args.quarantine_id, args.recipient)
- logger.info("successfully deleted e-mail [quarantine-id: {}] to '{}' from quarantine '{}'".format(args.quarantine_id, args.recipient, args.quarantine))
+ quarantine.delete(args.quarantine_id, args.recipient)
+ if args.recipient:
+ logger.info("successfully deleted e-mail [quarantine-id: {}] to '{}' from quarantine '{}'".format(args.quarantine_id, args.recipient, args.quarantine))
+ else:
+ logger.info("successfully deleted e-mail [quarantine-id: {}] from quarantine '{}'".format(args.quarantine_id, args.quarantine))
class StdErrFilter(logging.Filter):
@@ -212,13 +223,11 @@ class StdErrFilter(logging.Filter):
return rec.levelno in (logging.ERROR, logging.WARNING)
-
class StdOutFilter(logging.Filter):
def filter(self, rec):
return rec.levelno in (logging.DEBUG, logging.INFO)
-
def main():
"PyQuarantine command-line interface."
# parse command line
@@ -229,6 +238,7 @@ def main():
parser.add_argument("-d", "--debug", help="Log debugging messages.", action="store_true")
parser.set_defaults(syslog=False)
subparsers = parser.add_subparsers()
+
# list command
list_parser = subparsers.add_parser("list", help="List available quarantines.", formatter_class=formatter_class)
list_parser.add_argument("-b", "--batch", help="Print results using only quarantine names, each on a new line.", action="store_true")
@@ -236,10 +246,10 @@ def main():
# quarantine command group
quarantine_parser = subparsers.add_parser("quarantine", description="Manage quarantines.", help="Manage quarantines.", formatter_class=formatter_class)
+ quarantine_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
quarantine_subparsers = quarantine_parser.add_subparsers()
# quarantine list command
quarantine_list_parser = quarantine_subparsers.add_parser("list", description="List e-mails in quarantines.", help="List e-mails in quarantine.", formatter_class=formatter_class)
- quarantine_list_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
quarantine_list_parser.add_argument("-f", "--from", dest="mailfrom", help="Filter e-mails by from address.", default=None, nargs="+")
quarantine_list_parser.add_argument("-t", "--to", dest="recipients", help="Filter e-mails by recipient address.", default=None, nargs="+")
quarantine_list_parser.add_argument("-o", "--older-than", dest="older_than", help="Filter e-mails by age (days).", default=None, type=float)
@@ -247,7 +257,6 @@ def main():
quarantine_list_parser.set_defaults(func=list_quarantine_emails)
# quarantine release command
quarantine_release_parser = quarantine_subparsers.add_parser("release", description="Release e-mail from quarantine.", help="Release e-mail from quarantine.", formatter_class=formatter_class)
- quarantine_release_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
quarantine_release_parser.add_argument("quarantine_id", metavar="ID", help="Quarantine ID.")
quarantine_release_parser.add_argument("-n", "--disable-syslog", dest="syslog", help="Disable syslog messages.", action="store_false")
quarantine_release_parser_group = quarantine_release_parser.add_mutually_exclusive_group(required=True)
@@ -256,7 +265,6 @@ def main():
quarantine_release_parser.set_defaults(func=release_email)
# quarantine delete command
quarantine_delete_parser = quarantine_subparsers.add_parser("delete", description="Delete e-mail from quarantine.", help="Delete e-mail from quarantine.", formatter_class=formatter_class)
- quarantine_delete_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
quarantine_delete_parser.add_argument("quarantine_id", metavar="ID", help="Quarantine ID.")
quarantine_delete_parser.add_argument("-n", "--disable-syslog", dest="syslog", help="Disable syslog messages.", action="store_false")
quarantine_delete_parser_group = quarantine_delete_parser.add_mutually_exclusive_group(required=True)
@@ -266,26 +274,24 @@ def main():
# whitelist command group
whitelist_parser = subparsers.add_parser("whitelist", description="Manage whitelists.", help="Manage whitelists.", formatter_class=formatter_class)
+ whitelist_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
whitelist_subparsers = whitelist_parser.add_subparsers()
# whitelist list command
whitelist_list_parser = whitelist_subparsers.add_parser("list", description="List whitelist entries.", help="List whitelist entries.", formatter_class=formatter_class)
- whitelist_list_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
whitelist_list_parser.add_argument("-f", "--from", dest="mailfrom", help="Filter entries by from address.", default=None, nargs="+")
whitelist_list_parser.add_argument("-t", "--to", dest="recipients", help="Filter entries by recipient address.", default=None, nargs="+")
whitelist_list_parser.add_argument("-o", "--older-than", dest="older_than", help="Filter e-mails by last used date (days).", default=None, type=float)
whitelist_list_parser.set_defaults(func=list_whitelist)
# whitelist add command
whitelist_add_parser = whitelist_subparsers.add_parser("add", description="Add whitelist entry.", help="Add whitelist entry.", formatter_class=formatter_class)
- whitelist_add_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
whitelist_add_parser.add_argument("-f", "--from", dest="mailfrom", help="From address.", required=True)
whitelist_add_parser.add_argument("-t", "--to", dest="recipient", help="Recipient address.", required=True)
- whitelist_add_parser.add_argument("-c", "--comment", help="Comment.", default="added by CLI", required=False)
+ whitelist_add_parser.add_argument("-c", "--comment", help="Comment.", default="added by CLI")
whitelist_add_parser.add_argument("-p", "--permanent", help="Add a permanent entry.", action="store_true")
whitelist_add_parser.add_argument("--force", help="Force adding an entry, even if already covered by another entry.", action="store_true")
whitelist_add_parser.set_defaults(func=add_whitelist_entry)
# whitelist delete command
whitelist_delete_parser = whitelist_subparsers.add_parser("delete", description="Delete whitelist entry.", help="Delete whitelist entry.", formatter_class=formatter_class)
- whitelist_delete_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
whitelist_delete_parser.add_argument("whitelist_id", metavar="ID", help="Whitelist ID.")
whitelist_delete_parser.set_defaults(func=delete_whitelist_entry)
@@ -317,7 +323,7 @@ def main():
# try to generate milter configs
try:
- config = pyquarantine.generate_milter_config(config_files=args.config, configtest=True)
+ global_config, config = pyquarantine.generate_milter_config(config_files=args.config, configtest=True)
except RuntimeError as e:
logger.error(e)
sys.exit(255)
@@ -341,6 +347,5 @@ def main():
sys.exit(1)
-
if __name__ == "__main__":
main()
diff --git a/pyquarantine/mailer.py b/pyquarantine/mailer.py
index 19e8484..1c970c5 100644
--- a/pyquarantine/mailer.py
+++ b/pyquarantine/mailer.py
@@ -25,50 +25,52 @@ queue = Queue(maxsize=50)
process = None
-
def smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail):
s = smtplib.SMTP(host=smtp_host, port=smtp_port)
s.sendmail(mailfrom, [recipient], mail)
s.quit()
-
def mailprocess():
"Mailer process to send emails asynchronously."
global logger
global queue
+
try:
while True:
m = queue.get()
if not m: break
- smtp_host, smtp_port, queueid, mailfrom, recipient, mail = m
+
+ smtp_host, smtp_port, queueid, mailfrom, recipient, mail, emailtype = m
try:
smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail)
except Exception as e:
- logger.error("{}: error while sending email to <{}> via {}: {}".format(queueid, recipient, smtp_host, e))
+ logger.error("{}: error while sending {} to '{}': {}".format(queueid, emailtype, recipient, e))
else:
- logger.info("{}: email to <{}> sent successfully".format(queueid, recipient))
+ logger.info("{}: successfully sent {} to: {}".format(queueid, emailtype, recipient))
except KeyboardInterrupt:
pass
logger.debug("mailer process terminated")
-
-def sendmail(smtp_host, smtp_port, queueid, mailfrom, recipients, mail):
+def sendmail(smtp_host, smtp_port, queueid, mailfrom, recipients, mail, emailtype="email"):
"Send an email."
global logger
global process
global queue
+
if type(recipients) == str:
recipients = [recipients]
+
# start mailprocess if it is not started yet
if process == None:
process = Process(target=mailprocess)
process.daemon = True
logger.debug("starting mailer process")
process.start()
+
for recipient in recipients:
try:
- queue.put((smtp_host, smtp_port, queueid, mailfrom, recipient, mail), timeout=30)
+ queue.put((smtp_host, smtp_port, queueid, mailfrom, recipient, mail, emailtype), timeout=30)
except Queue.Full as e:
- raise RuntimeError("e-mail queue is full")
+ raise RuntimeError("email queue is full")
diff --git a/pyquarantine/notifications.py b/pyquarantine/notifications.py
index b7870a8..54752d5 100644
--- a/pyquarantine/notifications.py
+++ b/pyquarantine/notifications.py
@@ -20,25 +20,25 @@ import mailer
import re
from bs4 import BeautifulSoup
+from cgi import escape
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.image import MIMEImage
-
class BaseNotification(object):
"Notification base class"
- def __init__(self, quarantine_name, config, configtest=False):
- self.quarantine_name = quarantine_name
- self.config = config[quarantine_name]
- self.global_config = config["global"]
+ def __init__(self, global_config, config, configtest=False):
+ self.quarantine_name = config["name"]
+ self.global_config = global_config
+ self.config = config
self.logger = logging.getLogger(__name__)
def notify(self, queueid, quarantine_id, subject, mailfrom, recipients, fp):
+ fp.seek(0)
pass
-
class EMailNotification(BaseNotification):
"Notification class to send notifications via mail."
_html_text = "text/html"
@@ -84,10 +84,13 @@ class EMailNotification(BaseNotification):
"border",
"cellpadding",
"cellspacing",
+ "color",
"colspan",
"dir",
+ "face",
"headers",
"height",
+ "id",
"name",
"rowspan",
"size",
@@ -95,26 +98,33 @@ class EMailNotification(BaseNotification):
"style",
"title",
"type",
+ "valign",
"value",
"width"
]
- def __init__(self, quarantine_name, config, configtest=False):
- super(EMailNotification, self).__init__(quarantine_name, config, configtest)
+ def __init__(self, global_config, config, configtest=False):
+ super(EMailNotification, self).__init__(global_config, config, configtest)
+
# check if mandatory options are present in config
for option in ["smtp_host", "smtp_port", "notification_email_from", "notification_email_subject", "notification_email_template", "notification_email_replacement_img"]:
if option not in self.config.keys() and option in self.global_config.keys():
self.config[option] = self.global_config[option]
if option not in self.config.keys():
raise RuntimeError("mandatory option '{}' not present in config section '{}' or 'global'".format(option, self.quarantine_name))
+
self.smtp_host = self.config["smtp_host"]
self.smtp_port = self.config["smtp_port"]
self.mailfrom = self.config["notification_email_from"]
self.subject = self.config["notification_email_subject"]
+
+ # read email notification template
try:
self.template = open(self.config["notification_email_template"], "rb").read()
except IOError as e:
raise RuntimeError("error reading template: {}".format(e))
+
+ # read email replacement image
try:
self.replacement_img = MIMEImage(open(self.config["notification_email_replacement_img"], "rb").read())
except IOError as e:
@@ -122,95 +132,130 @@ class EMailNotification(BaseNotification):
else:
self.replacement_img.add_header("Content-ID", "")
- def get_text(self, part):
+ def get_text(self, queueid, part):
"Get the mail text in html form from email part."
mimetype = part.get_content_type()
- text = part.get_payload(decode=True)
- if mimetype == EMailNotification._plain_text:
- text = re.sub(r"^(.*)$", r"\1
\n", text, flags=re.MULTILINE)
- soup = BeautifulSoup(text, "lxml", from_encoding=part.get_content_charset())
- return soup
- def get_text_multipart(self, msg, preferred=_html_text):
+ self.logger.debug("{}: extracting content of email text part".format(queueid))
+ text = part.get_payload(decode=True)
+
+ if mimetype == EMailNotification._plain_text:
+ self.logger.debug("{}: content mimetype is {}, converting to {}".format(queueid, mimetype, self._html_text))
+ text = re.sub(r"^(.*)$", r"\1
\n", text, flags=re.MULTILINE)
+ else:
+ self.logger.debug("{}: content mimetype is {}".format(queueid, mimetype))
+
+ return BeautifulSoup(text, "lxml", from_encoding=part.get_content_charset())
+
+ def get_text_multipart(self, queueid, msg, preferred=_html_text):
"Get the mail text of a multipart email in html form."
soup = None
+
for part in msg.get_payload():
mimetype = part.get_content_type()
if mimetype in [EMailNotification._plain_text, EMailNotification._html_text]:
- soup = self.get_text(part)
+ soup = self.get_text(queueid, part)
elif mimetype.startswith("multipart"):
- soup = self.get_text_multipart(part, preferred)
+ soup = self.get_text_multipart(queueid, part, preferred)
+
if soup != None and mimetype == preferred:
break
return soup
- def sanitize(self, soup):
+ def sanitize(self, queueid, soup):
"Sanitize mail html text."
+ self.logger.debug("{}: sanitizing email text".format(queueid))
+
# completly remove bad elements
for element in soup(EMailNotification._bad_tags):
+ self.logger.debug("{}: removing dangerous tag '{}' and its content".format(queueid, element.name))
element.extract()
+
# remove not whitelisted elements, but keep their content
for element in soup.find_all(True):
if element.name not in EMailNotification._good_tags:
+ self.logger.debug("{}: removing tag '{}', keep its content".format(queueid, element.name))
element.replaceWithChildren()
+
# remove not whitelisted attributes
for element in soup.find_all(True):
for attribute in element.attrs.keys():
if attribute not in EMailNotification.good_attributes:
- del(element.attrs[attribute])
- # set href attribute for all a-tags to #
- for element in soup("a"):
- element["href"] = "#"
+ if element.name == "a" and attribute == "href":
+ self.logger.debug("{}: setting attribute href to '#' on tag '{}'".format(queueid, element.name))
+ element["href"] = "#"
+ else:
+ self.logger.debug("{}: removing attribute '{}' from tag '{}'".format(queueid, attribute, element.name))
+ del(element.attrs[attribute])
return soup
- def get_html_text_part(self, msg):
+ def get_html_text_part(self, queueid, msg):
"Get the mail text of an email in html form."
soup = None
mimetype = msg.get_content_type()
+
+ self.logger.debug("{}: trying to find text part of email".format(queueid))
if mimetype in [EMailNotification._plain_text, EMailNotification._html_text]:
- soup = self.get_text(msg)
+ soup = self.get_text(queueid, msg)
elif mimetype.startswith("multipart"):
- soup = self.get_text_multipart(msg)
+ soup = self.get_text_multipart(queueid, msg)
+
if soup == None:
+ self.logger.error("{}: unable to extract text part of email".format(queueid))
text = "ERROR: unable to extract text from email body"
soup = BeautifulSoup(text, "lxml", "UTF-8")
+
return soup
def notify(self, queueid, quarantine_id, subject, mailfrom, recipients, fp):
"Notify recipients via email."
super(EMailNotification, self).notify(queueid, quarantine_id, subject, mailfrom, recipients, fp)
- self.logger.debug("{}: generating notification email".format(queueid))
+
# extract html text from email
self.logger.debug("{}: extraction email text from original email".format(queueid))
- soup = self.get_html_text_part(email.message_from_file(fp))
+ soup = self.get_html_text_part(queueid, email.message_from_file(fp))
+
# replace picture sources
picture_replaced = False
for element in soup("img"):
- if "src" in element:
+ if "src" in element.attrs.keys():
self.logger.debug("{}: replacing image: {}".format(queueid, element["src"]))
element["src"] = "cid:removed_for_security_reasons"
picture_replaced = True
+
+ # sanitizing email text of original email
+ sanitized_text = self.sanitize(queueid, soup)
+
+ # escape possible html entities in subject
+ subject = escape(subject)
+
+ # sending email notifications
for recipient in recipients:
- self.logger.debug("{}: sending notification to <{}>".format(queueid, recipient))
+ self.logger.debug("{}: generating notification email for '{}'".format(queueid, recipient))
self.logger.debug("{}: parsing email template".format(queueid))
+
htmltext = self.template.format( \
- EMAIL_HTML_TEXT=self.sanitize(soup), \
- EMAIL_FROM=mailfrom, \
- EMAIL_TO=recipient, \
+ EMAIL_HTML_TEXT=sanitized_text, \
+ EMAIL_FROM=escape(mailfrom), \
+ EMAIL_TO=escape(recipient), \
EMAIL_SUBJECT=subject, \
EMAIL_QUARANTINE_ID=quarantine_id
)
+
msg = MIMEMultipart('alternative')
msg["Subject"] = self.subject
msg["From"] = "<{}>".format(self.mailfrom)
msg["To"] = "<{}>".format(recipient)
msg["Date"] = email.utils.formatdate()
msg.attach(MIMEText(htmltext, "html", 'UTF-8'))
- if picture_replaced:
- msg.attach(self.replacement_img)
- mailer.sendmail(self.smtp_host, self.smtp_port, queueid, self.mailfrom, recipient, msg.as_string())
+ if picture_replaced:
+ self.logger.debug("{}: attaching notification_replacement_img".format(queueid))
+ msg.attach(self.replacement_img)
+
+ self.logger.debug("{}: sending notification email to: {}".format(queueid, recipient))
+ mailer.sendmail(self.smtp_host, self.smtp_port, queueid, self.mailfrom, recipient, msg.as_string(), "notification email")
# list of notification types and their related notification classes
-notification_types = {"email": EMailNotification}
+TYPES = {"email": EMailNotification}
diff --git a/pyquarantine/quarantines.py b/pyquarantine/quarantines.py
index fac5fa8..1bca19b 100644
--- a/pyquarantine/quarantines.py
+++ b/pyquarantine/quarantines.py
@@ -26,16 +26,18 @@ from time import gmtime
import mailer
+
class BaseQuarantine(object):
"Quarantine base class"
- def __init__(self, name, config, configtest=False):
- self.name = name
- self.config = config[name]
- self.global_config = config["global"]
+ def __init__(self, global_config, config, configtest=False):
+ self.name = config["name"]
+ self.global_config = global_config
+ self.config = config
self.logger = logging.getLogger(__name__)
def add(self, queueid, mailfrom, recipients, fp):
"Add e-mail to quarantine."
+ fp.seek(0)
return ""
def find(self, mailfrom=None, recipients=None, older_than=None):
@@ -55,11 +57,11 @@ class BaseQuarantine(object):
return
-
class FileQuarantine(BaseQuarantine):
"Quarantine class to store mails on filesystem."
- def __init__(self, name, config, configtest=False):
- super(FileQuarantine, self).__init__(name, config, configtest)
+ def __init__(self, global_config, config, configtest=False):
+ super(FileQuarantine, self).__init__(global_config, config, configtest)
+
# check if mandatory options are present in config
for option in ["directory"]:
if option not in self.config.keys() and option in self.global_config.keys():
@@ -67,6 +69,7 @@ class FileQuarantine(BaseQuarantine):
if option not in self.config.keys():
raise RuntimeError("mandatory option '{}' not present in config section '{}' or 'global'".format(option, self.name))
self.directory = self.config["directory"]
+
# check if quarantine directory exists and is writable
if not os.path.isdir(self.directory) or not os.access(self.directory, os.W_OK):
raise RuntimeError("file quarantine directory '{}' does not exist or is not writable".format(self.directory))
@@ -91,10 +94,12 @@ class FileQuarantine(BaseQuarantine):
def _remove(self, quarantine_id):
datafile = os.path.join(self.directory, quarantine_id)
metafile = "{}{}".format(datafile, self._metadata_suffix)
+
try:
os.remove(metafile)
except IOError as e:
raise RuntimeError("unable to remove metadata file: {}".format(e))
+
try:
os.remove(datafile)
except IOError as e:
@@ -104,8 +109,10 @@ class FileQuarantine(BaseQuarantine):
"Add e-mail to file quarantine and return quarantine-id."
super(FileQuarantine, self).add(queueid, mailfrom, recipients, fp)
quarantine_id = "{}_{}".format(datetime.now().strftime("%Y%m%d%H%M%S"), queueid)
+
# save mail
self._save_datafile(quarantine_id, fp)
+
# save metadata
metadata = {
"from": mailfrom,
@@ -119,19 +126,23 @@ class FileQuarantine(BaseQuarantine):
datafile = os.path.join(self.directory, quarantine_id)
os.remove(datafile)
raise e
+
return quarantine_id
def get_metadata(self, quarantine_id):
"Return metadata of quarantined e-mail."
super(FileQuarantine, self).get_metadata(quarantine_id)
+
metafile = os.path.join(self.directory, "{}{}".format(quarantine_id, self._metadata_suffix))
if not os.path.isfile(metafile):
raise RuntimeError("invalid quarantine id '{}'".format(quarantine_id))
+
try:
with open(metafile, "rb") as f:
metadata = json.load(f)
except IOError as e:
raise RuntimeError("unable to read metadata file: {}".format(e))
+
return metadata
def find(self, mailfrom=None, recipients=None, older_than=None):
@@ -139,40 +150,49 @@ class FileQuarantine(BaseQuarantine):
super(FileQuarantine, self).find(mailfrom, recipients, older_than)
if type(mailfrom) == str: mailfrom = [mailfrom]
if type(recipients) == str: recipients = [recipients]
+
emails = {}
metafiles = glob(os.path.join(self.directory, "*{}".format(self._metadata_suffix)))
for metafile in metafiles:
if not os.path.isfile(metafile): continue
+
quarantine_id = os.path.basename(metafile[:-len(self._metadata_suffix)])
metadata = self.get_metadata(quarantine_id)
if older_than != None:
if timegm(gmtime()) - metadata["date"] < (older_than * 24 * 3600):
continue
+
if mailfrom != None:
if metadata["from"] not in mailfrom:
continue
+
if recipients != None:
if len(recipients) == 1 and recipients[0] not in metadata["recipients"]:
continue
elif len(set(recipients + metadata["recipients"])) == len(recipients + metadata["recipients"]):
continue
+
emails[quarantine_id] = metadata
+
return emails
def delete(self, quarantine_id, recipient=None):
"Delete e-mail in quarantine."
super(FileQuarantine, self).delete(quarantine_id, recipient)
+
try:
metadata = self.get_metadata(quarantine_id)
except RuntimeError as e:
raise RuntimeError("unable to delete e-mail: {}".format(e))
+
if recipient == None:
self._remove(quarantine_id)
else:
if recipient not in metadata["recipients"]:
raise RuntimeError("invalid recipient '{}'".format(recipient))
+
metadata["recipients"].remove(recipient)
- if len(metadata["recipients"]) == 0:
+ if not metadata["recipients"]:
self._remove(quarantine_id)
else:
self._save_metafile(quarantine_id, metadata)
@@ -180,10 +200,12 @@ class FileQuarantine(BaseQuarantine):
def release(self, quarantine_id, recipient=None):
"Release e-mail from quarantine."
super(FileQuarantine, self).release(quarantine_id, recipient)
+
try:
metadata = self.get_metadata(quarantine_id)
except RuntimeError as e:
raise RuntimeError("unable to release e-mail: {}".format(e))
+
datafile = os.path.join(self.directory, quarantine_id)
if recipient != None:
if recipient not in metadata["recipients"]:
@@ -191,19 +213,21 @@ class FileQuarantine(BaseQuarantine):
recipients = [recipient]
else:
recipients = metadata["recipients"]
+
try:
with open(datafile, "rb") as f:
mail = f.read()
except IOError as e:
raise RuntimeError("unable to read data file: {}".format(e))
+
for recipient in recipients:
try:
mailer.smtp_send(self.config["smtp_host"], self.config["smtp_port"], metadata["from"], recipient, mail)
except Exception as e:
raise RuntimeError("error while sending e-mail to '{}': {}".format(recipient, e))
+
self.delete(quarantine_id, recipient)
-
# list of quarantine types and their related quarantine classes
-quarantine_types = {"file": FileQuarantine}
+TYPES = {"file": FileQuarantine}
diff --git a/pyquarantine/run.py b/pyquarantine/run.py
index ac0d891..33e1eb0 100644
--- a/pyquarantine/run.py
+++ b/pyquarantine/run.py
@@ -23,7 +23,6 @@ import sys
import pyquarantine
-
def main():
"Run PyQuarantine-Milter."
# parse command line
@@ -35,6 +34,7 @@ def main():
parser.add_argument("-d", "--debug", help="Log debugging messages.", action="store_true")
parser.add_argument("-t", "--test", help="Check configuration.", action="store_true")
args = parser.parse_args()
+
# setup logging
loglevel = logging.INFO
logname = "pyquarantine-milter"
@@ -43,10 +43,12 @@ def main():
loglevel = logging.DEBUG
logname = "{}[%(name)s]".format(logname)
syslog_name = "{}: [%(name)s] %(levelname)s".format(syslog_name)
+
# set config files for milter class
pyquarantine.QuarantineMilter.set_configfiles(args.config)
root_logger = logging.getLogger()
root_logger.setLevel(loglevel)
+
# setup console log
stdouthandler = logging.StreamHandler(sys.stdout)
stdouthandler.setLevel(logging.DEBUG)
@@ -63,26 +65,32 @@ def main():
sys.exit(255)
else:
sys.exit(0)
- formatter = logging.Formatter("%(asctime)s {}: %(levelname)s - %(message)s".format(logname), datefmt="%Y-%m-%d %H:%M:%S")
+ formatter = logging.Formatter("%(asctime)s {}: [%(levelname)s] %(message)s".format(logname), datefmt="%Y-%m-%d %H:%M:%S")
stdouthandler.setFormatter(formatter)
+
# setup syslog
sysloghandler = logging.handlers.SysLogHandler(address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL)
sysloghandler.setLevel(loglevel)
formatter = logging.Formatter("{}: %(message)s".format(syslog_name))
sysloghandler.setFormatter(formatter)
root_logger.addHandler(sysloghandler)
+
logger.info("PyQuarantine-Milter starting")
try:
# generate milter config
- config = pyquarantine.generate_milter_config()
+ global_config, config = pyquarantine.generate_milter_config()
except RuntimeError as e:
logger.error(e)
sys.exit(255)
+
+ pyquarantine.QuarantineMilter.global_config = global_config
pyquarantine.QuarantineMilter.config = config
+
# register to have the Milter factory create instances of your class:
Milter.factory = pyquarantine.QuarantineMilter
Milter.set_exception_policy(Milter.TEMPFAIL)
#Milter.set_flags(0) # tell sendmail which features we use
+
# run milter
rc = 0
try:
@@ -95,6 +103,5 @@ def main():
sys.exit(rc)
-
if __name__ == "__main__":
main()
diff --git a/pyquarantine/whitelists.py b/pyquarantine/whitelists.py
index 79da6ff..33654e6 100644
--- a/pyquarantine/whitelists.py
+++ b/pyquarantine/whitelists.py
@@ -22,19 +22,18 @@ import sys
from playhouse.db_url import connect
-
class WhitelistBase(object):
"Whitelist base class"
- def __init__(self, name, config, configtest=False):
- self.name = name
- self.config = config[name]
+ def __init__(self, global_config, config, configtest=False):
+ self.global_config = global_config
+ self.config = config
self.configtest = configtest
- self.global_config = config["global"]
+ self.name = config["name"]
self.logger = logging.getLogger(__name__)
self.valid_entry_regex = re.compile(r"^[a-zA-Z0-9_.+-]*?(@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)?$")
def check(self, mailfrom, recipient):
- # check if mailfrom/recipient combination is whitelisted
+ "Check if mailfrom/recipient combination is whitelisted."
return
def find(self, mailfrom=None, recipients=None, older_than=None):
@@ -55,7 +54,6 @@ class WhitelistBase(object):
return
-
class WhitelistModel(peewee.Model):
mailfrom = peewee.CharField()
recipient = peewee.CharField()
@@ -65,51 +63,60 @@ class WhitelistModel(peewee.Model):
permanent = peewee.BooleanField(default=False)
-
class Meta(object):
indexes = (
(('mailfrom', 'recipient'), True), # trailing comma is mandatory if only one index should be created
)
-
class DatabaseWhitelist(WhitelistBase):
"Whitelist class to store whitelist in a database"
- _whitelists = {}
+ _db_connections = {}
+ _db_tables = {}
+
+ def __init__(self, global_config, config, configtest=False):
+ super(DatabaseWhitelist, self).__init__(global_config, config, configtest)
- def __init__(self, name, config, configtest=False):
- super(DatabaseWhitelist, self).__init__(name, config, configtest)
# check if mandatory options are present in config
for option in ["whitelist_db_connection", "whitelist_db_table"]:
if option not in self.config.keys() and option in self.global_config.keys():
self.config[option] = self.global_config[option]
if option not in self.config.keys():
raise RuntimeError("mandatory option '{}' not present in config section '{}' or 'global'".format(option, self.name))
- self.tablename = self.config["whitelist_db_table"]
+
+ tablename = self.config["whitelist_db_table"]
connection_string = self.config["whitelist_db_connection"]
- if connection_string in DatabaseWhitelist._whitelists.keys():
- new_connection = False
- self.db = DatabaseWhitelist._whitelists[connection_string]
+
+ if connection_string in DatabaseWhitelist._db_connections.keys():
+ db = DatabaseWhitelist._db_connections[connection_string]
else:
- new_connection = True
try:
# connect to database
self.logger.debug("connecting to database '{}'".format(re.sub(r"(.*?://.*?):.*?(@.*)", r"\1:\2", connection_string)))
- self.db = connect(connection_string)
+ db = connect(connection_string)
except Exception as e:
raise RuntimeError("unable to connect to database: {}".format(e))
- DatabaseWhitelist._whitelists[connection_string] = self.db
+
+ DatabaseWhitelist._db_connections[connection_string] = db
+
+ # generate model meta class
self.meta = Meta
- self.meta.database = self.db
- self.meta.table_name = self.tablename
- self.model = type("WhitelistModel_{}".format(name), (WhitelistModel,), {
+ self.meta.database = db
+ self.meta.table_name = tablename
+ self.model = type("WhitelistModel_{}".format(self.name), (WhitelistModel,), {
"Meta": self.meta
})
- if new_connection and not self.configtest:
- try:
- self.db.create_tables([self.model])
- except Exception as e:
- raise RuntimeError("unable to initialize table '{}': {}".format(self.tablename, e))
+
+ if connection_string not in DatabaseWhitelist._db_tables.keys():
+ DatabaseWhitelist._db_tables[connection_string] = []
+
+ if tablename not in DatabaseWhitelist._db_tables[connection_string]:
+ DatabaseWhitelist._db_tables[connection_string].append(tablename)
+ if not self.configtest:
+ try:
+ db.create_tables([self.model])
+ except Exception as e:
+ raise RuntimeError("unable to initialize table '{}': {}".format(tablename, e))
def _entry_to_dict(self, entry):
result = {}
@@ -136,27 +143,33 @@ class DatabaseWhitelist(WhitelistBase):
def check(self, mailfrom, recipient):
# check if mailfrom/recipient combination is whitelisted
super(DatabaseWhitelist, self).check(mailfrom, recipient)
+
# generate list of possible mailfroms
self.logger.debug("query database for whitelist entries from <{}> to <{}>".format(mailfrom, recipient))
mailfroms = [""]
if "@" in mailfrom and not mailfrom.startswith("@"):
mailfroms.append("@{}".format(mailfrom.split("@")[1]))
mailfroms.append(mailfrom)
+
# generate list of possible recipients
recipients = [""]
if "@" in recipient and not recipient.startswith("@"):
recipients.append("@{}".format(recipient.split("@")[1]))
recipients.append(recipient)
+
# query the database
try:
entries = list(self.model.select().where(self.model.mailfrom.in_(mailfroms), self.model.recipient.in_(recipients)))
except Exception as e:
raise RuntimeError("unable to query database: {}".format(e))
- if len(entries) == 0:
+
+ if not entries:
# no whitelist entry found
return {}
+
if len(entries) > 1:
entries.sort(key=lambda x: self.get_weight(x), reverse=True)
+
# use entry with the highest weight
entry = entries[0]
entry.last_used = datetime.datetime.now()
@@ -164,33 +177,41 @@ class DatabaseWhitelist(WhitelistBase):
result = {}
for entry in entries:
result.update(self._entry_to_dict(entry))
+
return result
def find(self, mailfrom=None, recipients=None, older_than=None):
"Find whitelist entries."
super(DatabaseWhitelist, self).find(mailfrom, recipients, older_than)
+
if type(mailfrom) == str: mailfrom = [mailfrom]
if type(recipients) == str: recipients = [recipients]
+
entries = {}
try:
for entry in list(self.model.select()):
if older_than != None:
if (datetime.datetime.now() - entry.last_used).total_seconds() < (older_than * 24 * 3600):
continue
+
if mailfrom != None:
if entry.mailfrom not in mailfrom:
continue
+
if recipients != None:
if entry.recipient not in recipients:
continue
+
entries.update(self._entry_to_dict(entry))
except Exception as e:
raise RuntimeError("unable to query database: {}".format(e))
+
return entries
def add(self, mailfrom, recipient, comment, permanent):
"Add entry to whitelist."
super(DatabaseWhitelist, self).add(mailfrom, recipient, comment, permanent)
+
try:
self.model.create(mailfrom=mailfrom, recipient=recipient, comment=comment, permanent=permanent)
except Exception as e:
@@ -199,16 +220,17 @@ class DatabaseWhitelist(WhitelistBase):
def delete(self, whitelist_id):
"Delete entry from whitelist."
super(DatabaseWhitelist, self).delete(whitelist_id)
+
try:
query = self.model.delete().where(self.model.id == whitelist_id)
deleted = query.execute()
except Exception as e:
raise RuntimeError("unable to delete entry from database: {}".format(e))
+
if deleted == 0:
raise RuntimeError("invalid whitelist id")
-
class WhitelistCache(object):
def __init__(self):
self.cache = {}
@@ -226,9 +248,8 @@ class WhitelistCache(object):
def get_whitelisted_recipients(self, whitelist, mailfrom, recipients):
self.load(whitelist, mailfrom, recipients)
- return filter(lambda x: len(self.cache[whitelist][x]) > 0, self.cache[whitelist].keys())
-
+ return filter(lambda x: self.cache[whitelist][x], self.cache[whitelist].keys())
# list of whitelist types and their related whitelist classes
-whitelist_types = {"db": DatabaseWhitelist}
+TYPES = {"db": DatabaseWhitelist}