spacefreak/pyquarantine-milter
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Improved CLI, quarantine- and error handling

Browse Source
This commit is contained in:
Thomas Oettli
2019-03-13 09:27:58 +01:00
parent da6f08ee25
commit 88cea0e127
10 changed files with 508 additions and 301 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
README.md
View File

@@ -24,10 +24,12 @@ Any available configuration option can be set in the global section as default i
The following configuration options are mandatory in the global section: The following configuration options are mandatory in the global section:
* **quarantines** * **quarantines**
Comma-separated, ordered list of active quarantines. For each, there must be a section of the same name in the configuration. Comma-separated, ordered list of active quarantines. For each, there must be a section of the same name in the configuration.
* **smtp_host** * **preferred_quarantin_action**
SMTP host to inject original e-mails. This is needed if not all recipients of an e-mail are whitelisted Defines which quarantine action should be preferred if multiple quarantines are matching for multiple recipients.
* **smtp_port** If at least one recipient receives the original e-mail due to whitelisting, the action is always ACCEPT.
SMTP port Possible values are:
* **first**
* **last**
### Quarantine sections ### Quarantine sections
The following configuration options are mandatory in each quarantine section: The following configuration options are mandatory in each quarantine section:
@@ -41,6 +43,10 @@ The following configuration options are mandatory in each quarantine section:
One of the notification types described below. One of the notification types described below.
* **whitelist_type** * **whitelist_type**
One of the whitelist types described below. One of the whitelist types described below.
* **smtp_host**
SMTP host to inject original e-mails. This is needed if not all recipients of an e-mail are whitelisted
* **smtp_port**
SMTP port
### Quarantine types ### Quarantine types
* **NONE** * **NONE**

9
docs/pyquarantine.conf.example
View File

@@ -19,6 +19,15 @@
# #
quarantines = spam quarantines = spam
# Option: quarantine_action_precedence
# Notes: Set if the action of the first or the last matching quarantine should
# be used if multiple recipients match multiple quarantines. If an original
# email is delivered to at least one recipient due to whitelisting, the
# email will always be accepted.
# Values: [ first | last ]
#
preferred_quarantine_action = last
# Option: smtp_host # Option: smtp_host
# Notes: Set the SMTP host. It will be used to (re-)inject emails. # Notes: Set the SMTP host. It will be used to (re-)inject emails.
# Values: [ HOSTNAME | IP_ADDRESS ] # Values: [ HOSTNAME | IP_ADDRESS ]

27
docs/templates/notification.template vendored
View File

@@ -1,10 +1,25 @@
<html> <html>
<body> <body>
<h1>This is a spam notification mail</h1> <h1>Quarantine ntification</h1>
<b>From:</b> {EMAIL_FROM}<br/> <table>
<b>To:</b> {EMAIL_TO}<br/> <tr>
<b>Subject:</b> {EMAIL_SUBJECT}<br/> <td><b>From:</b></td>
<b>Quarantine-ID:</b> {EMAIL_QUARANTINE_ID}<br/><br/><br/> <td>{EMAIL_FROM}</td>
{EMAIL_HTML_TEXT} </tr>
<tr>
<td><b>To:</b></td>
<td>{EMAIL_TO}</td>
</tr>
<tr>
<td><b>Subject:</b></td>
<td>{EMAIL_SUBJECT}</td>
</tr>
<tr>
<td><b>Quarantine ID:</b></td>
<td>{EMAIL_QUARANTINE_ID}</td>
</tr>
</table><br/>
<h2>Preview of the original e-mail</h2>
{EMAIL_HTML_TEXT}
</body> </body>
</html> </html>

325
pyquarantine/__init__.py
View File

@@ -30,7 +30,7 @@ import notifications
import whitelists import whitelists
from Milter.utils import parse_addr from Milter.utils import parse_addr
from itertools import groupby
class QuarantineMilter(Milter.Base): class QuarantineMilter(Milter.Base):
@@ -40,18 +40,27 @@ class QuarantineMilter(Milter.Base):
""" """
config = None config = None
global_config = None
# list of default config files # list of default config files
_config_files = ["/etc/pyquarantine/pyquarantine.conf", os.path.expanduser('~/pyquarantine.conf'), "pyquarantine.conf"] _config_files = ["/etc/pyquarantine/pyquarantine.conf", os.path.expanduser('~/pyquarantine.conf'), "pyquarantine.conf"]
# list of possible actions # list of possible actions
_actions = {"ACCEPT": Milter.ACCEPT, "REJECT": Milter.REJECT, "DISCARD": Milter.DISCARD} _actions = {"ACCEPT": Milter.ACCEPT, "REJECT": Milter.REJECT, "DISCARD": Milter.DISCARD}
def __init__(self): def __init__(self):
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
# save config, it must not change during runtime # save config, it must not change during runtime
self.global_config = QuarantineMilter.global_config
self.config = QuarantineMilter.config self.config = QuarantineMilter.config
def _get_preferred_quarantine(self):
matching_quarantines = [q for q in self.recipients_quarantines.values() if q]
if self.global_config["preferred_quarantine_action"] == "first":
quarantine = sorted(matching_quarantines, key=lambda x: x["index"])[0]
else:
quarantine = sorted(matching_quarantines, key=lambda x: x["index"], reverse=True)[0]
return quarantine
@staticmethod @staticmethod
def get_configfiles(): def get_configfiles():
return QuarantineMilter._config_files return QuarantineMilter._config_files
@@ -93,57 +102,89 @@ class QuarantineMilter(Milter.Base):
def eoh(self): def eoh(self):
try: try:
self.matched = None
self.whitelist_cache = whitelists.WhitelistCache() self.whitelist_cache = whitelists.WhitelistCache()
# initialize a dict to set quaranines per recipient
self.recipients_quarantines = {}
# iterate email headers # iterate email headers
recipients_to_check = self.recipients[:]
for header in self.headers: for header in self.headers:
self.logger.debug("{}: checking header '{}' against regex of every configured quarantine".format(self.queueid, header)) self.logger.debug("{}: checking header against configured quarantines: {}".format(self.queueid, header))
# iterate quarantines # iterate quarantines
for name, quarantine in self.config.items(): for quarantine in self.config:
if self.matched != None and quarantine["index"] == self.matched["index"]: if len(self.recipients_quarantines) == len(self.recipients):
# a quarantine with higher precedence already matched, skip checks of quarantines with lower precedence # every recipient matched a quarantine already
self.logger.debug("{}: quarantine '{}' matched already, skip further checks of this header".format(self.queueid, name)) if max([q["index"] for q in self.recipients_quarantines.values()]) <= quarantine["index"]:
# every recipient matched a quarantine with at least the same precedence already, skip checks against quarantines with lower precedence
self.logger.debug("{}: {}: skip further checks of this header".format(self.queueid, quarantine["name"]))
break break
self.logger.debug("{}: checking header against quarantine '{}'".format(self.queueid, name))
# check if header matches regex # check email header against quarantine regex
self.logger.debug("{}: {}: checking header against regex '{}'".format(self.queueid, quarantine["name"], quarantine["regex"]))
if quarantine["regex_compiled"].match(header): if quarantine["regex_compiled"].match(header):
self.logger.debug("{}: {}: header matched regex".format(self.queueid, quarantine["name"]))
# check for whitelisted recipients
whitelist = quarantine["whitelist_obj"] whitelist = quarantine["whitelist_obj"]
if whitelist != None: if whitelist != None:
try: try:
whitelisted_recipients = self.whitelist_cache.get_whitelisted_recipients(whitelist, self.mailfrom, self.recipients) whitelisted_recipients = self.whitelist_cache.get_whitelisted_recipients(whitelist, self.mailfrom, recipients_to_check)
except RuntimeError as e: except RuntimeError as e:
self.logger.error("{}: unable to query whitelist: {}".format(self.queueid, e)) self.logger.error("{}: {}: unable to query whitelist: {}".format(self.queueid, quarantine["name"], e))
return Milter.TEMPFAIL return Milter.TEMPFAIL
if len(whitelisted_recipients) == len(self.recipients): else:
# all recipients are whitelisted, continue with header checks whitelisted_recipients = {}
self.logger.debug("{}: header matched regex, but all recipients are whitelisted in quarantine '{}', continue checking this header".format(self.queueid, name))
# iterate recipients
for recipient in recipients_to_check[:]:
if recipient in whitelisted_recipients:
# recipient is whitelisted in this quarantine
self.logger.debug("{}: {}: recipient '{}' is whitelisted".format(self.queueid, quarantine["name"], recipient))
continue continue
self.matched = quarantine
# skip checks of this header with quarantines with lower precedence if recipient not in self.recipients_quarantines.keys() or self.recipients_quarantines[recipient]["index"] > quarantine["index"]:
self.logger.debug("{}: header matched regex in quarantine '{}', further checks of this header will be skipped".format(self.queueid, name)) self.logger.debug("{}: {}: set quarantine for recipient '{}'".format(self.queueid, quarantine["name"], recipient))
self.recipients_quarantines[recipient] = quarantine
if quarantine["index"] == 0:
# we do not need to check recipients which matched the quarantine with the highest precedence already
recipients_to_check.remove(recipient)
else:
self.logger.debug("{}: {}: a quarantine with the same or higher precedence matched already for recipient '{}'".format(self.queueid, quarantine["name"], recipient))
if not recipients_to_check:
self.logger.debug("{}: all recipients matched the first quarantine, skipping all remaining header checks".format(self.queueid))
break break
if self.matched != None and self.matched["index"] == 0:
self.logger.debug("{}: skipping checks of remaining headers, the quarantine with highest precedence matched already".format(self.queueid)) # check if no quarantine has matched for all recipients
if not self.recipients_quarantines:
# accept email
self.logger.info("{}: passed clean for all recipients".format(self.queueid))
return Milter.ACCEPT
# check if the email body is needed
keep_body = False
for recipient, quarantine in self.recipients_quarantines.items():
if quarantine["quarantine_obj"] or quarantine["notification_obj"]:
keep_body = True
break break
if self.matched != None:
self.logger.info("{}: email matched quarantine '{}'".format(self.queueid, self.matched["name"])) if keep_body:
# one of the configured quarantines matched
if self.matched["quarantine_obj"] != None or self.matched["notification_obj"] != None:
self.logger.debug("{}: initializing memory buffer to save email data".format(self.queueid)) self.logger.debug("{}: initializing memory buffer to save email data".format(self.queueid))
# quarantine or notification configured, initialize memory buffer to save mail # initialize memory buffer to save email data
self.fp = StringIO.StringIO() self.fp = StringIO.StringIO()
# write email headers to memory buffer # write email headers to memory buffer
self.fp.write("{}\n".format("\n".join(self.headers))) self.fp.write("{}\n".format("\n".join(self.headers)))
else: else:
# quarantine and notification disabled, return configured action # quarantine and notification are disabled on all matching quarantines, return configured action
self.logger.debug("{}: ".format(self.queueid)) quarantine = self._get_preferred_quarantine()
self.logger.info("{}: quarantine and notification disabled, responding with configured action: {}".format(self.queueid, self.matched["action"].upper())) self.logger.info("{}: {} matching quarantine is '{}', performing milter action {}".format(self.queueid, self.global_config["preferred_quarantine_action"],
return self.matched["milter_action"] quarantine["name"], quarantine["action"].upper()))
else: return quarantine["milter_action"]
# no quarantine matched, accept mail
self.logger.info("{}: email passed clean".format(self.queueid))
return Milter.ACCEPT
return Milter.CONTINUE return Milter.CONTINUE
except Exception as e: except Exception as e:
self.logger.exception("an exception occured in eoh function: {}".format(e)) self.logger.exception("an exception occured in eoh function: {}".format(e))
return Milter.TEMPFAIL return Milter.TEMPFAIL
@@ -159,46 +200,54 @@ class QuarantineMilter(Milter.Base):
def eom(self): def eom(self):
try: try:
if self.matched["whitelist_obj"] != None: # processing recipients grouped by quarantines
try: quarantines = []
whitelisted_recipients = self.whitelist_cache.get_whitelisted_recipients(self.matched["whitelist_obj"], self.mailfrom, self.recipients) keyfunc = lambda x: self.recipients_quarantines[x]
except RuntimeError as e: for quarantine, recipients in groupby(sorted(self.recipients_quarantines, key=keyfunc), keyfunc):
self.logger.error("{}: unable to query whitelist: {}".format(self.queueid, e)) quarantines.append((quarantine, list(recipients)))
return Milter.TEMPFAIL
if len(whitelisted_recipients) > 0: # iterate quarantines sorted by index
for recipient in whitelisted_recipients: for quarantine, recipients in sorted(quarantines, key=lambda x: x[0]["index"]):
self.recipients.remove(recipient)
self.fp.seek(0)
self.logger.info("{}: sending original email to whitelisted recipient(s): {}".format(self.queueid, "<{}>".format(">,<".join(whitelisted_recipients))))
try:
mailer.sendmail(self.matched["smtp_host"], self.matched["smtp_port"], self.queueid, self.mailfrom, whitelisted_recipients, self.fp.read())
except RuntimeError as e:
self.logger.error("{}: unable to send original email: {}".format(self.queueid, e))
return Milter.TEMPFAIL
if len(self.recipients) > 0:
quarantine_id = "" quarantine_id = ""
if self.matched["quarantine_obj"] != None:
# check if a quarantine is configured
if quarantine["quarantine_obj"] != None:
# add email to quarantine # add email to quarantine
self.fp.seek(0) self.logger.info("{}: adding to quarantine '{}' for: {}".format(self.queueid, quarantine["name"], ", ".join(recipients)))
self.logger.info("{}: adding email to quarantine of recipient(s): {}".format(self.queueid, "<{}>".format(">,<".join(self.recipients))))
try: try:
quarantine_id = self.matched["quarantine_obj"].add(self.queueid, self.mailfrom, self.recipients, fp=self.fp) quarantine_id = quarantine["quarantine_obj"].add(self.queueid, self.mailfrom, recipients, fp=self.fp)
except RuntimeError as e: except RuntimeError as e:
self.logger.error("{}: unable to add email to quarantine: {}".format(self.queueid, e)) self.logger.error("{}: unable to add to quarantine '{}': {}".format(self.queueid, quarantine["name"], e))
return Milter.TEMPFAIL return Milter.TEMPFAIL
if self.matched["notification_obj"] != None:
# check if a notification is configured
if quarantine["notification_obj"] != None:
# notify # notify
self.fp.seek(0) self.logger.info("{}: sending notification for quarantine '{}' to: {}".format(self.queueid, quarantine["name"], ", ".join(recipients)))
self.logger.info("{}: sending notification(s) to: {}".format(self.queueid, "<{}>".format(">,<".join(self.recipients))))
try: try:
self.matched["notification_obj"].notify(self.queueid, quarantine_id, self.subject, self.mailfrom, self.recipients, fp=self.fp) quarantine["notification_obj"].notify(self.queueid, quarantine_id, self.subject, self.mailfrom, recipients, fp=self.fp)
except RuntimeError as e: except RuntimeError as e:
self.logger.error("{}: unable to send notification(s): {}".format(self.queueid, e)) self.logger.error("{}: unable to send notification for quarantine '{}': {}".format(self.queueid, quarantine["name"], e))
return Milter.TEMPFAIL return Milter.TEMPFAIL
# remove processed recipient
for recipient in recipients:
self.delrcpt(recipient)
self.recipients.remove(recipient)
self.fp.close() self.fp.close()
# return configured action
self.logger.info("{}: responding with configured action: {}".format(self.queueid, self.matched["action"].upper())) # email passed clean for at least one recipient, accepting email
return self.matched["milter_action"] if self.recipients:
self.logger.info("{}: passed clean for: {}".format(self.queueid, ", ".join(self.recipients)))
return Milter.ACCEPT
## return configured action
quarantine = self._get_preferred_quarantine()
self.logger.info("{}: {} matching quarantine is '{}', performing milter action {}".format(self.queueid, self.global_config["preferred_quarantine_action"],
quarantine["name"], quarantine["action"].upper()))
return quarantine["milter_action"]
except Exception as e: except Exception as e:
self.logger.exception("an exception occured in eom function: {}".format(e)) self.logger.exception("an exception occured in eom function: {}".format(e))
return Milter.TEMPFAIL return Milter.TEMPFAIL
@@ -207,107 +256,131 @@ class QuarantineMilter(Milter.Base):
def generate_milter_config(configtest=False, config_files=[]): def generate_milter_config(configtest=False, config_files=[]):
"Generate the configuration for QuarantineMilter class." "Generate the configuration for QuarantineMilter class."
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# read config file # read config file
parser = ConfigParser.ConfigParser() parser = ConfigParser.ConfigParser()
if len(config_files) == 0: if not config_files:
config_files = parser.read(QuarantineMilter.get_configfiles()) config_files = parser.read(QuarantineMilter.get_configfiles())
else: else:
config_files = parser.read(config_files) config_files = parser.read(config_files)
if len(config_files) == 0: if not config_files:
raise RuntimeError("config file not found") raise RuntimeError("config file not found")
QuarantineMilter.set_configfiles(config_files) QuarantineMilter.set_configfiles(config_files)
os.chdir(os.path.dirname(config_files[0])) os.chdir(os.path.dirname(config_files[0]))
# check if mandatory config options in global section are present # check if mandatory config options in global section are present
if "global" not in parser.sections(): if "global" not in parser.sections():
raise RuntimeError("mandatory section 'global' not present in config file") raise RuntimeError("mandatory section 'global' not present in config file")
for option in ["quarantines"]: for option in ["quarantines", "preferred_quarantine_action"]:
if not parser.has_option("global", option): if not parser.has_option("global", option):
raise RuntimeError("mandatory option '{}' not present in config section 'global'".format(option)) raise RuntimeError("mandatory option '{}' not present in config section 'global'".format(option))
config = {}
config["global"] = dict(parser.items("global")) # read global config section
# iterate configured quarantines global_config = dict(parser.items("global"))
quarantine_names = list(set(map(str.strip, parser.get("global", "quarantines").split(",")))) global_config["preferred_quarantine_action"] = global_config["preferred_quarantine_action"].lower()
if global_config["preferred_quarantine_action"] not in ["first", "last"]:
raise RuntimeError("option preferred_quarantine_action has illegal value")
# read active quarantine names
quarantine_names = list(set(map(str.strip, global_config["quarantines"].split(","))))
if "global" in quarantine_names: if "global" in quarantine_names:
quarantine_names.remove("global")
logger.warning("removed illegal quarantine name 'global' from list of active quarantines") logger.warning("removed illegal quarantine name 'global' from list of active quarantines")
del(quarantine_names["global"]) if not quarantine_names:
if len(quarantine_names) == 0:
raise RuntimeError("no quarantines configured") raise RuntimeError("no quarantines configured")
idx = 0
for name in quarantine_names: milter_config = []
name = name.strip()
# check if config section for current quarantine is present logger.debug("preparing milter configuration ...")
if name not in parser.sections(): # iterate quarantine names
raise RuntimeError("config section '{}' is not present".format(name)) for index, quarantine_name in enumerate(quarantine_names):
config[name] = dict(parser.items(name))
config[name]["name"] = name # check if config section for current quarantine exists
if quarantine_name not in parser.sections():
raise RuntimeError("config section '{}' does not exist".format(quarantine_name))
config = dict(parser.items(quarantine_name))
# check if mandatory config options are present in config # check if mandatory config options are present in config
for option in ["regex", "quarantine_type", "notification_type", "action", "whitelist_type", "smtp_host", "smtp_port"]: for option in ["regex", "quarantine_type", "notification_type", "action", "whitelist_type", "smtp_host", "smtp_port"]:
if option not in config[name].keys() and \ if option not in config.keys() and \
option in config["global"].keys(): option in global_config.keys():
config[name][option] = config["global"][option] config[option] = global_config[option]
if option not in config[name].keys(): if option not in config.keys():
raise RuntimeError("mandatory option '{}' not present in config section '{}' or 'global'".format(option, name)) raise RuntimeError("mandatory option '{}' not present in config section '{}' or 'global'".format(option, quarantine_name))
logger.debug("preparing configuration for quarantine '{}' ...".format(name))
## add the index # set quarantine name
config[name]["index"] = idx config["name"] = quarantine_name
idx += 1
# compile regex # set the index
regex = config[name]["regex"] config["index"] = index
logger.debug("=> compiling regex '{}'".format(regex))
config[name]["regex_compiled"] = re.compile(regex) # pre-compile regex
logger.debug("{}: compiling regex '{}'".format(quarantine_name, config["regex"]))
config["regex_compiled"] = re.compile(config["regex"])
# create quarantine instance # create quarantine instance
quarantine_type = config[name]["quarantine_type"].lower() quarantine_type = config["quarantine_type"].lower()
if quarantine_type in quarantines.quarantine_types.keys(): if quarantine_type in quarantines.TYPES.keys():
logger.debug("=> initializing quarantine type '{}'".format(quarantine_type)) logger.debug("{}: initializing quarantine type '{}'".format(quarantine_name, quarantine_type.upper()))
quarantine = quarantines.quarantine_types[quarantine_type](name, config, configtest) quarantine = quarantines.TYPES[quarantine_type](global_config, config, configtest)
elif quarantine_type == "none": elif quarantine_type == "none":
logger.debug("=> setting quarantine to NONE") logger.debug("{}: quarantine is NONE".format(quarantine_name))
quarantine = None quarantine = None
else: else:
raise RuntimeError("unknown quarantine_type '{}'".format(quarantine_type)) raise RuntimeError("{}: unknown quarantine type '{}'".format(quarantine_name, quarantine_type))
config[name]["quarantine_obj"] = quarantine
config["quarantine_obj"] = quarantine
# create whitelist instance # create whitelist instance
whitelist_type = config[name]["whitelist_type"].lower() whitelist_type = config["whitelist_type"].lower()
if whitelist_type in whitelists.whitelist_types.keys(): if whitelist_type in whitelists.TYPES.keys():
logger.debug("=> initializing whitelist database") logger.debug("{}: initializing whitelist type '{}'".format(quarantine_name, whitelist_type.upper()))
whitelist = whitelists.whitelist_types[whitelist_type](name, config, configtest) whitelist = whitelists.TYPES[whitelist_type](global_config, config, configtest)
else: elif whitelist_type == "none":
logger.debug("=> setting whitelist to NONE") logger.debug("{}: whitelist is NONE".format(quarantine_name))
whitelist = None whitelist = None
config[name]["whitelist_obj"] = whitelist else:
raise RuntimeError("{}: unknown whitelist type '{}'".format(quarantine_name, whitelist_type))
config["whitelist_obj"] = whitelist
# create notification instance # create notification instance
notification_type = config[name]["notification_type"].lower() notification_type = config["notification_type"].lower()
if notification_type in notifications.notification_types.keys(): if notification_type in notifications.TYPES.keys():
logger.debug("=> initializing notification type '{}'".format(notification_type)) logger.debug("{}: initializing notification type '{}'".format(quarantine_name, notification_type.upper()))
notification = notifications.notification_types[notification_type](name, config, configtest) notification = notifications.TYPES[notification_type](global_config, config, configtest)
elif notification_type == "none": elif notification_type == "none":
logger.debug("=> setting notification to NONE") logger.debug("{}: notification is NONE".format(quarantine_name))
notification = None notification = None
else: else:
raise RuntimeError("unknown notification type '{}'".format(notification_type)) raise RuntimeError("{}: unknown notification type '{}'".format(quarantine_name, notification_type))
config[name]["notification_obj"] = notification
# determining milter action for this quarantine
action = config[name]["action"].upper()
if action in QuarantineMilter.get_actions().keys():
logger.debug("=> setting action to {}".format(action))
config[name]["milter_action"] = QuarantineMilter.get_actions()[action]
else:
raise RuntimeError("unknown action '{}' configured for quarantine '{}'".format(action, name))
# remove global section from config, every section should be a quarantine
del(config["global"])
return config
config["notification_obj"] = notification
# determining milter action for this quarantine
action = config["action"].upper()
if action in QuarantineMilter.get_actions().keys():
logger.debug("{}: action is {}".format(quarantine_name, action))
config["milter_action"] = QuarantineMilter.get_actions()[action]
else:
raise RuntimeError("{}: unknown action '{}'".format(quarantine_name, action))
milter_config.append(config)
return global_config, milter_config
def reload_config(): def reload_config():
"Reload the configuration of QuarantineMilter class." "Reload the configuration of QuarantineMilter class."
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
logger.debug("received SIGUSR1")
try: try:
config = generate_milter_config() global_config, config = generate_milter_config()
except RuntimeError as e: except RuntimeError as e:
logger.info(e) logger.info(e)
logger.info("daemon is still running with previous configuration") logger.info("daemon is still running with previous configuration")
else: else:
logger.info("reloading configuration") logger.info("reloading configuration")
QuarantineMilter.global_config = global_config
QuarantineMilter.config = config QuarantineMilter.config = config

129
pyquarantine/cli.py
View File

@@ -23,33 +23,37 @@ import time
import pyquarantine import pyquarantine
def _get_quarantine_obj(config, quarantine): def _get_quarantine_obj(config, quarantine):
if quarantine not in config.keys(): try:
quarantine_obj = next((q["quarantine_obj"] for q in config if q["name"] == quarantine))
except StopIteration:
raise RuntimeError("invalid quarantine '{}'".format(quarantine)) raise RuntimeError("invalid quarantine '{}'".format(quarantine))
return config[quarantine]["quarantine_obj"] return quarantine_obj
def _get_whitelist_obj(config, quarantine): def _get_whitelist_obj(config, quarantine):
if quarantine not in config.keys(): try:
whitelist_obj = next((q["whitelist_obj"] for q in config if q["name"] == quarantine))
except StopIteration:
raise RuntimeError("invalid quarantine '{}'".format(quarantine)) raise RuntimeError("invalid quarantine '{}'".format(quarantine))
return config[quarantine]["whitelist_obj"] return whitelist_obj
def print_table(columns, rows):
def print_table(headers, keys, data): if not rows:
if len(data) == 0:
return return
# calculate length of each column
column_lengths = [] column_lengths = []
column_formats = [] column_formats = []
for idx, header in enumerate(headers):
length = len(header) # iterate columns to display
key = keys[idx] for header, key in columns:
value_length=len((max(data.items(), key=lambda (k, v): len(v[key])))[1][key]) # get the length of the header string
if value_length > length: length = value_length lengths = [len(header)]
# get the length of the longest value
lengths.append(len(str(max(rows, key=lambda x: len(str(x[key])))[key])))
# use the the longer one
length = max(lengths)
column_lengths.append(length) column_lengths.append(length)
column_formats.append("{{:<{}}}".format(length)) column_formats.append("{{:<{}}}".format(length))
@@ -63,29 +67,26 @@ def print_table(headers, keys, data):
separator = "-+-".join(separators) separator = "-+-".join(separators)
# print header and separator # print header and separator
print(row_format.format(*headers)) print(row_format.format(*[column[0] for column in columns]))
print(separator) print(separator)
# print data keys = [ entry[1] for entry in columns ]
for key, value in data.items(): # print rows
for entry in rows:
row = [] row = []
for entry in keys: for key in keys:
row.append(value[entry]) row.append(entry[key])
print(row_format.format(*row)) print(row_format.format(*row))
def list_quarantines(config, args): def list_quarantines(config, args):
if args.batch: if args.batch:
print("\n".join(config.keys())) print("\n".join([ quarantine["name"] for quarantine in config ]))
else: else:
print_table( print_table(
["Name", "Quarantine", "Notification", "Action"], [("Name", "name"), ("Quarantine", "quarantine_type"), ("Notification", "notification_type"), ("Action", "action")],
["name", "quarantine_type", "notification_type", "action"],
config config
) )
return 0
def list_quarantine_emails(config, args): def list_quarantine_emails(config, args):
@@ -104,15 +105,15 @@ def list_quarantine_emails(config, args):
emails[quarantine_id]["date_str"] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(metadata["date"])) emails[quarantine_id]["date_str"] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(metadata["date"]))
if args.batch: if args.batch:
# batch mode, print quarantine IDs, each on a new line
print("\n".join(emails.keys())) print("\n".join(emails.keys()))
else: return
if len(emails) == 0: logger.info("quarantine '{}' is empty".format(args.quarantine))
print_table(
["Quarantine-ID", "From", "Recipient(s)", "Date"],
["quarantine_id", "from", "recipient_str", "date_str"],
emails
)
if not emails: logger.info("quarantine '{}' is empty".format(args.quarantine))
print_table(
[("Quarantine-ID", "quarantine_id"), ("From", "from"), ("Recipient(s)", "recipient_str"), ("Date", "date_str")],
emails.values()
)
def list_whitelist(config, args): def list_whitelist(config, args):
@@ -125,24 +126,25 @@ def list_whitelist(config, args):
# find whitelist entries # find whitelist entries
entries = whitelist.find(mailfrom=args.mailfrom, recipients=args.recipients, older_than=args.older_than) entries = whitelist.find(mailfrom=args.mailfrom, recipients=args.recipients, older_than=args.older_than)
if len(entries) == 0: if not entries:
logger.info("whitelist of quarantine '{}' is empty".format(args.quarantine)) logger.info("whitelist of quarantine '{}' is empty".format(args.quarantine))
else: return
# transform some values to strings # transform some values to strings
for entry_id, entry in entries.items(): for entry_id, entry in entries.items():
entries[entry_id]["id"] = str(entry["id"]) entries[entry_id]["permanent_str"] = str(entry["permanent"])
entries[entry_id]["created_str"] = entry["created"].strftime('%Y-%m-%d %H:%M:%S') entries[entry_id]["created_str"] = entry["created"].strftime('%Y-%m-%d %H:%M:%S')
entries[entry_id]["last_used_str"] = entry["last_used"].strftime('%Y-%m-%d %H:%M:%S') entries[entry_id]["last_used_str"] = entry["last_used"].strftime('%Y-%m-%d %H:%M:%S')
entries[entry_id]["permanent_str"] = str(entry["permanent"])
print_table( print_table(
["ID", "From", "To", "Created", "Last used", "Comment", "Permanent"], [
["id", "mailfrom", "recipient", "created_str", "last_used_str", "comment", "permanent_str"], ("ID", "id"), ("From", "mailfrom"), ("To", "recipient"), ("Created", "created_str"),
entries ("Last used", "last_used_str"), ("Comment", "comment"), ("Permanent", "permanent_str")
],
entries.values()
) )
def add_whitelist_entry(config, args): def add_whitelist_entry(config, args):
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -153,58 +155,67 @@ def add_whitelist_entry(config, args):
# check existing entries # check existing entries
entries = whitelist.check(args.mailfrom, args.recipient) entries = whitelist.check(args.mailfrom, args.recipient)
if len(entries) > 0: if entries:
# check if the exact entry exists already # check if the exact entry exists already
for entry in entries.values(): for entry in entries.values():
if entry["mailfrom"] == args.mailfrom and entry["recipient"] == args.recipient: if entry["mailfrom"] == args.mailfrom and entry["recipient"] == args.recipient:
raise RuntimeError("an entry with this from/to combination already exists") raise RuntimeError("an entry with this from/to combination already exists")
if not args.force: if not args.force:
# the entry is already covered by others # the entry is already covered by others
for entry_id, entry in entries.items(): for entry_id, entry in entries.items():
entries[entry_id]["permanent_str"] = str(entry["permanent"])
entries[entry_id]["created_str"] = entry["created"].strftime('%Y-%m-%d %H:%M:%S') entries[entry_id]["created_str"] = entry["created"].strftime('%Y-%m-%d %H:%M:%S')
entries[entry_id]["last_used_str"] = entry["last_used"].strftime('%Y-%m-%d %H:%M:%S') entries[entry_id]["last_used_str"] = entry["last_used"].strftime('%Y-%m-%d %H:%M:%S')
entries[entry_id]["permanent_str"] = str(entry["permanent"])
print_table( print_table(
["From", "To", "Created", "Last used", "Comment", "Permanent"], [
["mailfrom", "recipient", "created_str", "last_used_str", "comment", "permanent_str"], ("ID", "id"), ("From", "mailfrom"), ("To", "recipient"), ("Created", "created_str"),
entries ("Last used", "last_used_str"), ("Comment", "comment"), ("Permanent", "permanent_str")
],
entries.values()
) )
print("") print("")
raise RuntimeError("from/to combination is already covered by the entries above, use --force to override.") raise RuntimeError("from/to combination is already covered by the entries above, use --force to override.")
# add entry to whitelist # add entry to whitelist
whitelist.add(args.mailfrom, args.recipient, args.comment, args.permanent) whitelist.add(args.mailfrom, args.recipient, args.comment, args.permanent)
logger.info("successfully added whitelist entry") logger.info("successfully added whitelist entry")
def delete_whitelist_entry(config, args): def delete_whitelist_entry(config, args):
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
whitelist = _get_whitelist_obj(config, args.quarantine) whitelist = _get_whitelist_obj(config, args.quarantine)
if whitelist == None: if whitelist == None:
raise RuntimeError("whitelist type is set to None, unable to delete entries") raise RuntimeError("whitelist type is set to None, unable to delete entries")
whitelist.delete(args.whitelist_id) whitelist.delete(args.whitelist_id)
logger.info("successfully deleted whitelist entry") logger.info("successfully deleted whitelist entry")
def release_email(config, args): def release_email(config, args):
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
quarantine = _get_quarantine_obj(config, args.quarantine) quarantine = _get_quarantine_obj(config, args.quarantine)
if quarantine == None: if quarantine == None:
raise RuntimeError("quarantine type is set to None, unable to release e-mail") raise RuntimeError("quarantine type is set to None, unable to release e-mail")
quarantine.release(args.quarantine_id, args.recipient) quarantine.release(args.quarantine_id, args.recipient)
logger.info("successfully released e-mail [quarantine-id: {}] to '{}' from quarantine '{}'".format(args.quarantine_id, args.recipient, args.quarantine)) logger.info("successfully released e-mail [quarantine-id: {}] to '{}' from quarantine '{}'".format(args.quarantine_id, args.recipient, args.quarantine))
def delete_email(config, args): def delete_email(config, args):
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
quarantine = _get_quarantine_obj(config, args.quarantine) quarantine = _get_quarantine_obj(config, args.quarantine)
if quarantine == None: if quarantine == None:
raise RuntimeError("quarantine type is set to None, unable to delete e-mail") raise RuntimeError("quarantine type is set to None, unable to delete e-mail")
quarantine.delete(args.quarantine_id, args.recipient)
logger.info("successfully deleted e-mail [quarantine-id: {}] to '{}' from quarantine '{}'".format(args.quarantine_id, args.recipient, args.quarantine))
quarantine.delete(args.quarantine_id, args.recipient)
if args.recipient:
logger.info("successfully deleted e-mail [quarantine-id: {}] to '{}' from quarantine '{}'".format(args.quarantine_id, args.recipient, args.quarantine))
else:
logger.info("successfully deleted e-mail [quarantine-id: {}] from quarantine '{}'".format(args.quarantine_id, args.quarantine))
class StdErrFilter(logging.Filter): class StdErrFilter(logging.Filter):
@@ -212,13 +223,11 @@ class StdErrFilter(logging.Filter):
return rec.levelno in (logging.ERROR, logging.WARNING) return rec.levelno in (logging.ERROR, logging.WARNING)
class StdOutFilter(logging.Filter): class StdOutFilter(logging.Filter):
def filter(self, rec): def filter(self, rec):
return rec.levelno in (logging.DEBUG, logging.INFO) return rec.levelno in (logging.DEBUG, logging.INFO)
def main(): def main():
"PyQuarantine command-line interface." "PyQuarantine command-line interface."
# parse command line # parse command line
@@ -229,6 +238,7 @@ def main():
parser.add_argument("-d", "--debug", help="Log debugging messages.", action="store_true") parser.add_argument("-d", "--debug", help="Log debugging messages.", action="store_true")
parser.set_defaults(syslog=False) parser.set_defaults(syslog=False)
subparsers = parser.add_subparsers() subparsers = parser.add_subparsers()
# list command # list command
list_parser = subparsers.add_parser("list", help="List available quarantines.", formatter_class=formatter_class) list_parser = subparsers.add_parser("list", help="List available quarantines.", formatter_class=formatter_class)
list_parser.add_argument("-b", "--batch", help="Print results using only quarantine names, each on a new line.", action="store_true") list_parser.add_argument("-b", "--batch", help="Print results using only quarantine names, each on a new line.", action="store_true")
@@ -236,10 +246,10 @@ def main():
# quarantine command group # quarantine command group
quarantine_parser = subparsers.add_parser("quarantine", description="Manage quarantines.", help="Manage quarantines.", formatter_class=formatter_class) quarantine_parser = subparsers.add_parser("quarantine", description="Manage quarantines.", help="Manage quarantines.", formatter_class=formatter_class)
quarantine_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
quarantine_subparsers = quarantine_parser.add_subparsers() quarantine_subparsers = quarantine_parser.add_subparsers()
# quarantine list command # quarantine list command
quarantine_list_parser = quarantine_subparsers.add_parser("list", description="List e-mails in quarantines.", help="List e-mails in quarantine.", formatter_class=formatter_class) quarantine_list_parser = quarantine_subparsers.add_parser("list", description="List e-mails in quarantines.", help="List e-mails in quarantine.", formatter_class=formatter_class)
quarantine_list_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
quarantine_list_parser.add_argument("-f", "--from", dest="mailfrom", help="Filter e-mails by from address.", default=None, nargs="+") quarantine_list_parser.add_argument("-f", "--from", dest="mailfrom", help="Filter e-mails by from address.", default=None, nargs="+")
quarantine_list_parser.add_argument("-t", "--to", dest="recipients", help="Filter e-mails by recipient address.", default=None, nargs="+") quarantine_list_parser.add_argument("-t", "--to", dest="recipients", help="Filter e-mails by recipient address.", default=None, nargs="+")
quarantine_list_parser.add_argument("-o", "--older-than", dest="older_than", help="Filter e-mails by age (days).", default=None, type=float) quarantine_list_parser.add_argument("-o", "--older-than", dest="older_than", help="Filter e-mails by age (days).", default=None, type=float)
@@ -247,7 +257,6 @@ def main():
quarantine_list_parser.set_defaults(func=list_quarantine_emails) quarantine_list_parser.set_defaults(func=list_quarantine_emails)
# quarantine release command # quarantine release command
quarantine_release_parser = quarantine_subparsers.add_parser("release", description="Release e-mail from quarantine.", help="Release e-mail from quarantine.", formatter_class=formatter_class) quarantine_release_parser = quarantine_subparsers.add_parser("release", description="Release e-mail from quarantine.", help="Release e-mail from quarantine.", formatter_class=formatter_class)
quarantine_release_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
quarantine_release_parser.add_argument("quarantine_id", metavar="ID", help="Quarantine ID.") quarantine_release_parser.add_argument("quarantine_id", metavar="ID", help="Quarantine ID.")
quarantine_release_parser.add_argument("-n", "--disable-syslog", dest="syslog", help="Disable syslog messages.", action="store_false") quarantine_release_parser.add_argument("-n", "--disable-syslog", dest="syslog", help="Disable syslog messages.", action="store_false")
quarantine_release_parser_group = quarantine_release_parser.add_mutually_exclusive_group(required=True) quarantine_release_parser_group = quarantine_release_parser.add_mutually_exclusive_group(required=True)
@@ -256,7 +265,6 @@ def main():
quarantine_release_parser.set_defaults(func=release_email) quarantine_release_parser.set_defaults(func=release_email)
# quarantine delete command # quarantine delete command
quarantine_delete_parser = quarantine_subparsers.add_parser("delete", description="Delete e-mail from quarantine.", help="Delete e-mail from quarantine.", formatter_class=formatter_class) quarantine_delete_parser = quarantine_subparsers.add_parser("delete", description="Delete e-mail from quarantine.", help="Delete e-mail from quarantine.", formatter_class=formatter_class)
quarantine_delete_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
quarantine_delete_parser.add_argument("quarantine_id", metavar="ID", help="Quarantine ID.") quarantine_delete_parser.add_argument("quarantine_id", metavar="ID", help="Quarantine ID.")
quarantine_delete_parser.add_argument("-n", "--disable-syslog", dest="syslog", help="Disable syslog messages.", action="store_false") quarantine_delete_parser.add_argument("-n", "--disable-syslog", dest="syslog", help="Disable syslog messages.", action="store_false")
quarantine_delete_parser_group = quarantine_delete_parser.add_mutually_exclusive_group(required=True) quarantine_delete_parser_group = quarantine_delete_parser.add_mutually_exclusive_group(required=True)
@@ -266,26 +274,24 @@ def main():
# whitelist command group # whitelist command group
whitelist_parser = subparsers.add_parser("whitelist", description="Manage whitelists.", help="Manage whitelists.", formatter_class=formatter_class) whitelist_parser = subparsers.add_parser("whitelist", description="Manage whitelists.", help="Manage whitelists.", formatter_class=formatter_class)
whitelist_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
whitelist_subparsers = whitelist_parser.add_subparsers() whitelist_subparsers = whitelist_parser.add_subparsers()
# whitelist list command # whitelist list command
whitelist_list_parser = whitelist_subparsers.add_parser("list", description="List whitelist entries.", help="List whitelist entries.", formatter_class=formatter_class) whitelist_list_parser = whitelist_subparsers.add_parser("list", description="List whitelist entries.", help="List whitelist entries.", formatter_class=formatter_class)
whitelist_list_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
whitelist_list_parser.add_argument("-f", "--from", dest="mailfrom", help="Filter entries by from address.", default=None, nargs="+") whitelist_list_parser.add_argument("-f", "--from", dest="mailfrom", help="Filter entries by from address.", default=None, nargs="+")
whitelist_list_parser.add_argument("-t", "--to", dest="recipients", help="Filter entries by recipient address.", default=None, nargs="+") whitelist_list_parser.add_argument("-t", "--to", dest="recipients", help="Filter entries by recipient address.", default=None, nargs="+")
whitelist_list_parser.add_argument("-o", "--older-than", dest="older_than", help="Filter e-mails by last used date (days).", default=None, type=float) whitelist_list_parser.add_argument("-o", "--older-than", dest="older_than", help="Filter e-mails by last used date (days).", default=None, type=float)
whitelist_list_parser.set_defaults(func=list_whitelist) whitelist_list_parser.set_defaults(func=list_whitelist)
# whitelist add command # whitelist add command
whitelist_add_parser = whitelist_subparsers.add_parser("add", description="Add whitelist entry.", help="Add whitelist entry.", formatter_class=formatter_class) whitelist_add_parser = whitelist_subparsers.add_parser("add", description="Add whitelist entry.", help="Add whitelist entry.", formatter_class=formatter_class)
whitelist_add_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
whitelist_add_parser.add_argument("-f", "--from", dest="mailfrom", help="From address.", required=True) whitelist_add_parser.add_argument("-f", "--from", dest="mailfrom", help="From address.", required=True)
whitelist_add_parser.add_argument("-t", "--to", dest="recipient", help="Recipient address.", required=True) whitelist_add_parser.add_argument("-t", "--to", dest="recipient", help="Recipient address.", required=True)
whitelist_add_parser.add_argument("-c", "--comment", help="Comment.", default="added by CLI", required=False) whitelist_add_parser.add_argument("-c", "--comment", help="Comment.", default="added by CLI")
whitelist_add_parser.add_argument("-p", "--permanent", help="Add a permanent entry.", action="store_true") whitelist_add_parser.add_argument("-p", "--permanent", help="Add a permanent entry.", action="store_true")
whitelist_add_parser.add_argument("--force", help="Force adding an entry, even if already covered by another entry.", action="store_true") whitelist_add_parser.add_argument("--force", help="Force adding an entry, even if already covered by another entry.", action="store_true")
whitelist_add_parser.set_defaults(func=add_whitelist_entry) whitelist_add_parser.set_defaults(func=add_whitelist_entry)
# whitelist delete command # whitelist delete command
whitelist_delete_parser = whitelist_subparsers.add_parser("delete", description="Delete whitelist entry.", help="Delete whitelist entry.", formatter_class=formatter_class) whitelist_delete_parser = whitelist_subparsers.add_parser("delete", description="Delete whitelist entry.", help="Delete whitelist entry.", formatter_class=formatter_class)
whitelist_delete_parser.add_argument("quarantine", metavar="QUARANTINE", help="Quarantine name.")
whitelist_delete_parser.add_argument("whitelist_id", metavar="ID", help="Whitelist ID.") whitelist_delete_parser.add_argument("whitelist_id", metavar="ID", help="Whitelist ID.")
whitelist_delete_parser.set_defaults(func=delete_whitelist_entry) whitelist_delete_parser.set_defaults(func=delete_whitelist_entry)
@@ -317,7 +323,7 @@ def main():
# try to generate milter configs # try to generate milter configs
try: try:
config = pyquarantine.generate_milter_config(config_files=args.config, configtest=True) global_config, config = pyquarantine.generate_milter_config(config_files=args.config, configtest=True)
except RuntimeError as e: except RuntimeError as e:
logger.error(e) logger.error(e)
sys.exit(255) sys.exit(255)
@@ -341,6 +347,5 @@ def main():
sys.exit(1) sys.exit(1)
if __name__ == "__main__": if __name__ == "__main__":
main() main()

20
pyquarantine/mailer.py
View File

@@ -25,50 +25,52 @@ queue = Queue(maxsize=50)
process = None process = None
def smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail): def smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail):
s = smtplib.SMTP(host=smtp_host, port=smtp_port) s = smtplib.SMTP(host=smtp_host, port=smtp_port)
s.sendmail(mailfrom, [recipient], mail) s.sendmail(mailfrom, [recipient], mail)
s.quit() s.quit()
def mailprocess(): def mailprocess():
"Mailer process to send emails asynchronously." "Mailer process to send emails asynchronously."
global logger global logger
global queue global queue
try: try:
while True: while True:
m = queue.get() m = queue.get()
if not m: break if not m: break
smtp_host, smtp_port, queueid, mailfrom, recipient, mail = m
smtp_host, smtp_port, queueid, mailfrom, recipient, mail, emailtype = m
try: try:
smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail) smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail)
except Exception as e: except Exception as e:
logger.error("{}: error while sending email to <{}> via {}: {}".format(queueid, recipient, smtp_host, e)) logger.error("{}: error while sending {} to '{}': {}".format(queueid, emailtype, recipient, e))
else: else:
logger.info("{}: email to <{}> sent successfully".format(queueid, recipient)) logger.info("{}: successfully sent {} to: {}".format(queueid, emailtype, recipient))
except KeyboardInterrupt: except KeyboardInterrupt:
pass pass
logger.debug("mailer process terminated") logger.debug("mailer process terminated")
def sendmail(smtp_host, smtp_port, queueid, mailfrom, recipients, mail, emailtype="email"):
def sendmail(smtp_host, smtp_port, queueid, mailfrom, recipients, mail):
"Send an email." "Send an email."
global logger global logger
global process global process
global queue global queue
if type(recipients) == str: if type(recipients) == str:
recipients = [recipients] recipients = [recipients]
# start mailprocess if it is not started yet # start mailprocess if it is not started yet
if process == None: if process == None:
process = Process(target=mailprocess) process = Process(target=mailprocess)
process.daemon = True process.daemon = True
logger.debug("starting mailer process") logger.debug("starting mailer process")
process.start() process.start()
for recipient in recipients: for recipient in recipients:
try: try:
queue.put((smtp_host, smtp_port, queueid, mailfrom, recipient, mail), timeout=30) queue.put((smtp_host, smtp_port, queueid, mailfrom, recipient, mail, emailtype), timeout=30)
except Queue.Full as e: except Queue.Full as e:
raise RuntimeError("e-mail queue is full") raise RuntimeError("email queue is full")

115
pyquarantine/notifications.py
View File

@@ -20,25 +20,25 @@ import mailer
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from cgi import escape
from email.mime.multipart import MIMEMultipart from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText from email.mime.text import MIMEText
from email.mime.image import MIMEImage from email.mime.image import MIMEImage
class BaseNotification(object): class BaseNotification(object):
"Notification base class" "Notification base class"
def __init__(self, quarantine_name, config, configtest=False): def __init__(self, global_config, config, configtest=False):
self.quarantine_name = quarantine_name self.quarantine_name = config["name"]
self.config = config[quarantine_name] self.global_config = global_config
self.global_config = config["global"] self.config = config
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
def notify(self, queueid, quarantine_id, subject, mailfrom, recipients, fp): def notify(self, queueid, quarantine_id, subject, mailfrom, recipients, fp):
fp.seek(0)
pass pass
class EMailNotification(BaseNotification): class EMailNotification(BaseNotification):
"Notification class to send notifications via mail." "Notification class to send notifications via mail."
_html_text = "text/html" _html_text = "text/html"
@@ -84,10 +84,13 @@ class EMailNotification(BaseNotification):
"border", "border",
"cellpadding", "cellpadding",
"cellspacing", "cellspacing",
"color",
"colspan", "colspan",
"dir", "dir",
"face",
"headers", "headers",
"height", "height",
"id",
"name", "name",
"rowspan", "rowspan",
"size", "size",
@@ -95,26 +98,33 @@ class EMailNotification(BaseNotification):
"style", "style",
"title", "title",
"type", "type",
"valign",
"value", "value",
"width" "width"
] ]
def __init__(self, quarantine_name, config, configtest=False): def __init__(self, global_config, config, configtest=False):
super(EMailNotification, self).__init__(quarantine_name, config, configtest) super(EMailNotification, self).__init__(global_config, config, configtest)
# check if mandatory options are present in config # check if mandatory options are present in config
for option in ["smtp_host", "smtp_port", "notification_email_from", "notification_email_subject", "notification_email_template", "notification_email_replacement_img"]: for option in ["smtp_host", "smtp_port", "notification_email_from", "notification_email_subject", "notification_email_template", "notification_email_replacement_img"]:
if option not in self.config.keys() and option in self.global_config.keys(): if option not in self.config.keys() and option in self.global_config.keys():
self.config[option] = self.global_config[option] self.config[option] = self.global_config[option]
if option not in self.config.keys(): if option not in self.config.keys():
raise RuntimeError("mandatory option '{}' not present in config section '{}' or 'global'".format(option, self.quarantine_name)) raise RuntimeError("mandatory option '{}' not present in config section '{}' or 'global'".format(option, self.quarantine_name))
self.smtp_host = self.config["smtp_host"] self.smtp_host = self.config["smtp_host"]
self.smtp_port = self.config["smtp_port"] self.smtp_port = self.config["smtp_port"]
self.mailfrom = self.config["notification_email_from"] self.mailfrom = self.config["notification_email_from"]
self.subject = self.config["notification_email_subject"] self.subject = self.config["notification_email_subject"]
# read email notification template
try: try:
self.template = open(self.config["notification_email_template"], "rb").read() self.template = open(self.config["notification_email_template"], "rb").read()
except IOError as e: except IOError as e:
raise RuntimeError("error reading template: {}".format(e)) raise RuntimeError("error reading template: {}".format(e))
# read email replacement image
try: try:
self.replacement_img = MIMEImage(open(self.config["notification_email_replacement_img"], "rb").read()) self.replacement_img = MIMEImage(open(self.config["notification_email_replacement_img"], "rb").read())
except IOError as e: except IOError as e:
@@ -122,95 +132,130 @@ class EMailNotification(BaseNotification):
else: else:
self.replacement_img.add_header("Content-ID", "<removed_for_security_reasons>") self.replacement_img.add_header("Content-ID", "<removed_for_security_reasons>")
def get_text(self, part): def get_text(self, queueid, part):
"Get the mail text in html form from email part." "Get the mail text in html form from email part."
mimetype = part.get_content_type() mimetype = part.get_content_type()
text = part.get_payload(decode=True)
if mimetype == EMailNotification._plain_text:
text = re.sub(r"^(.*)$", r"\1<br/>\n", text, flags=re.MULTILINE)
soup = BeautifulSoup(text, "lxml", from_encoding=part.get_content_charset())
return soup
def get_text_multipart(self, msg, preferred=_html_text): self.logger.debug("{}: extracting content of email text part".format(queueid))
text = part.get_payload(decode=True)
if mimetype == EMailNotification._plain_text:
self.logger.debug("{}: content mimetype is {}, converting to {}".format(queueid, mimetype, self._html_text))
text = re.sub(r"^(.*)$", r"\1<br/>\n", text, flags=re.MULTILINE)
else:
self.logger.debug("{}: content mimetype is {}".format(queueid, mimetype))
return BeautifulSoup(text, "lxml", from_encoding=part.get_content_charset())
def get_text_multipart(self, queueid, msg, preferred=_html_text):
"Get the mail text of a multipart email in html form." "Get the mail text of a multipart email in html form."
soup = None soup = None
for part in msg.get_payload(): for part in msg.get_payload():
mimetype = part.get_content_type() mimetype = part.get_content_type()
if mimetype in [EMailNotification._plain_text, EMailNotification._html_text]: if mimetype in [EMailNotification._plain_text, EMailNotification._html_text]:
soup = self.get_text(part) soup = self.get_text(queueid, part)
elif mimetype.startswith("multipart"): elif mimetype.startswith("multipart"):
soup = self.get_text_multipart(part, preferred) soup = self.get_text_multipart(queueid, part, preferred)
if soup != None and mimetype == preferred: if soup != None and mimetype == preferred:
break break
return soup return soup
def sanitize(self, soup): def sanitize(self, queueid, soup):
"Sanitize mail html text." "Sanitize mail html text."
self.logger.debug("{}: sanitizing email text".format(queueid))
# completly remove bad elements # completly remove bad elements
for element in soup(EMailNotification._bad_tags): for element in soup(EMailNotification._bad_tags):
self.logger.debug("{}: removing dangerous tag '{}' and its content".format(queueid, element.name))
element.extract() element.extract()
# remove not whitelisted elements, but keep their content # remove not whitelisted elements, but keep their content
for element in soup.find_all(True): for element in soup.find_all(True):
if element.name not in EMailNotification._good_tags: if element.name not in EMailNotification._good_tags:
self.logger.debug("{}: removing tag '{}', keep its content".format(queueid, element.name))
element.replaceWithChildren() element.replaceWithChildren()
# remove not whitelisted attributes # remove not whitelisted attributes
for element in soup.find_all(True): for element in soup.find_all(True):
for attribute in element.attrs.keys(): for attribute in element.attrs.keys():
if attribute not in EMailNotification.good_attributes: if attribute not in EMailNotification.good_attributes:
del(element.attrs[attribute]) if element.name == "a" and attribute == "href":
# set href attribute for all a-tags to # self.logger.debug("{}: setting attribute href to '#' on tag '{}'".format(queueid, element.name))
for element in soup("a"):
element["href"] = "#" element["href"] = "#"
else:
self.logger.debug("{}: removing attribute '{}' from tag '{}'".format(queueid, attribute, element.name))
del(element.attrs[attribute])
return soup return soup
def get_html_text_part(self, msg): def get_html_text_part(self, queueid, msg):
"Get the mail text of an email in html form." "Get the mail text of an email in html form."
soup = None soup = None
mimetype = msg.get_content_type() mimetype = msg.get_content_type()
self.logger.debug("{}: trying to find text part of email".format(queueid))
if mimetype in [EMailNotification._plain_text, EMailNotification._html_text]: if mimetype in [EMailNotification._plain_text, EMailNotification._html_text]:
soup = self.get_text(msg) soup = self.get_text(queueid, msg)
elif mimetype.startswith("multipart"): elif mimetype.startswith("multipart"):
soup = self.get_text_multipart(msg) soup = self.get_text_multipart(queueid, msg)
if soup == None: if soup == None:
self.logger.error("{}: unable to extract text part of email".format(queueid))
text = "ERROR: unable to extract text from email body" text = "ERROR: unable to extract text from email body"
soup = BeautifulSoup(text, "lxml", "UTF-8") soup = BeautifulSoup(text, "lxml", "UTF-8")
return soup return soup
def notify(self, queueid, quarantine_id, subject, mailfrom, recipients, fp): def notify(self, queueid, quarantine_id, subject, mailfrom, recipients, fp):
"Notify recipients via email." "Notify recipients via email."
super(EMailNotification, self).notify(queueid, quarantine_id, subject, mailfrom, recipients, fp) super(EMailNotification, self).notify(queueid, quarantine_id, subject, mailfrom, recipients, fp)
self.logger.debug("{}: generating notification email".format(queueid))
# extract html text from email # extract html text from email
self.logger.debug("{}: extraction email text from original email".format(queueid)) self.logger.debug("{}: extraction email text from original email".format(queueid))
soup = self.get_html_text_part(email.message_from_file(fp)) soup = self.get_html_text_part(queueid, email.message_from_file(fp))
# replace picture sources # replace picture sources
picture_replaced = False picture_replaced = False
for element in soup("img"): for element in soup("img"):
if "src" in element: if "src" in element.attrs.keys():
self.logger.debug("{}: replacing image: {}".format(queueid, element["src"])) self.logger.debug("{}: replacing image: {}".format(queueid, element["src"]))
element["src"] = "cid:removed_for_security_reasons" element["src"] = "cid:removed_for_security_reasons"
picture_replaced = True picture_replaced = True
# sanitizing email text of original email
sanitized_text = self.sanitize(queueid, soup)
# escape possible html entities in subject
subject = escape(subject)
# sending email notifications
for recipient in recipients: for recipient in recipients:
self.logger.debug("{}: sending notification to <{}>".format(queueid, recipient)) self.logger.debug("{}: generating notification email for '{}'".format(queueid, recipient))
self.logger.debug("{}: parsing email template".format(queueid)) self.logger.debug("{}: parsing email template".format(queueid))
htmltext = self.template.format( \ htmltext = self.template.format( \
EMAIL_HTML_TEXT=self.sanitize(soup), \ EMAIL_HTML_TEXT=sanitized_text, \
EMAIL_FROM=mailfrom, \ EMAIL_FROM=escape(mailfrom), \
EMAIL_TO=recipient, \ EMAIL_TO=escape(recipient), \
EMAIL_SUBJECT=subject, \ EMAIL_SUBJECT=subject, \
EMAIL_QUARANTINE_ID=quarantine_id EMAIL_QUARANTINE_ID=quarantine_id
) )
msg = MIMEMultipart('alternative') msg = MIMEMultipart('alternative')
msg["Subject"] = self.subject msg["Subject"] = self.subject
msg["From"] = "<{}>".format(self.mailfrom) msg["From"] = "<{}>".format(self.mailfrom)
msg["To"] = "<{}>".format(recipient) msg["To"] = "<{}>".format(recipient)
msg["Date"] = email.utils.formatdate() msg["Date"] = email.utils.formatdate()
msg.attach(MIMEText(htmltext, "html", 'UTF-8')) msg.attach(MIMEText(htmltext, "html", 'UTF-8'))
if picture_replaced:
msg.attach(self.replacement_img)
mailer.sendmail(self.smtp_host, self.smtp_port, queueid, self.mailfrom, recipient, msg.as_string())
if picture_replaced:
self.logger.debug("{}: attaching notification_replacement_img".format(queueid))
msg.attach(self.replacement_img)
self.logger.debug("{}: sending notification email to: {}".format(queueid, recipient))
mailer.sendmail(self.smtp_host, self.smtp_port, queueid, self.mailfrom, recipient, msg.as_string(), "notification email")
# list of notification types and their related notification classes # list of notification types and their related notification classes
notification_types = {"email": EMailNotification} TYPES = {"email": EMailNotification}

44
pyquarantine/quarantines.py
View File

@@ -26,16 +26,18 @@ from time import gmtime
import mailer import mailer
class BaseQuarantine(object): class BaseQuarantine(object):
"Quarantine base class" "Quarantine base class"
def __init__(self, name, config, configtest=False): def __init__(self, global_config, config, configtest=False):
self.name = name self.name = config["name"]
self.config = config[name] self.global_config = global_config
self.global_config = config["global"] self.config = config
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
def add(self, queueid, mailfrom, recipients, fp): def add(self, queueid, mailfrom, recipients, fp):
"Add e-mail to quarantine." "Add e-mail to quarantine."
fp.seek(0)
return "" return ""
def find(self, mailfrom=None, recipients=None, older_than=None): def find(self, mailfrom=None, recipients=None, older_than=None):
@@ -55,11 +57,11 @@ class BaseQuarantine(object):
return return
class FileQuarantine(BaseQuarantine): class FileQuarantine(BaseQuarantine):
"Quarantine class to store mails on filesystem." "Quarantine class to store mails on filesystem."
def __init__(self, name, config, configtest=False): def __init__(self, global_config, config, configtest=False):
super(FileQuarantine, self).__init__(name, config, configtest) super(FileQuarantine, self).__init__(global_config, config, configtest)
# check if mandatory options are present in config # check if mandatory options are present in config
for option in ["directory"]: for option in ["directory"]:
if option not in self.config.keys() and option in self.global_config.keys(): if option not in self.config.keys() and option in self.global_config.keys():
@@ -67,6 +69,7 @@ class FileQuarantine(BaseQuarantine):
if option not in self.config.keys(): if option not in self.config.keys():
raise RuntimeError("mandatory option '{}' not present in config section '{}' or 'global'".format(option, self.name)) raise RuntimeError("mandatory option '{}' not present in config section '{}' or 'global'".format(option, self.name))
self.directory = self.config["directory"] self.directory = self.config["directory"]
# check if quarantine directory exists and is writable # check if quarantine directory exists and is writable
if not os.path.isdir(self.directory) or not os.access(self.directory, os.W_OK): if not os.path.isdir(self.directory) or not os.access(self.directory, os.W_OK):
raise RuntimeError("file quarantine directory '{}' does not exist or is not writable".format(self.directory)) raise RuntimeError("file quarantine directory '{}' does not exist or is not writable".format(self.directory))
@@ -91,10 +94,12 @@ class FileQuarantine(BaseQuarantine):
def _remove(self, quarantine_id): def _remove(self, quarantine_id):
datafile = os.path.join(self.directory, quarantine_id) datafile = os.path.join(self.directory, quarantine_id)
metafile = "{}{}".format(datafile, self._metadata_suffix) metafile = "{}{}".format(datafile, self._metadata_suffix)
try: try:
os.remove(metafile) os.remove(metafile)
except IOError as e: except IOError as e:
raise RuntimeError("unable to remove metadata file: {}".format(e)) raise RuntimeError("unable to remove metadata file: {}".format(e))
try: try:
os.remove(datafile) os.remove(datafile)
except IOError as e: except IOError as e:
@@ -104,8 +109,10 @@ class FileQuarantine(BaseQuarantine):
"Add e-mail to file quarantine and return quarantine-id." "Add e-mail to file quarantine and return quarantine-id."
super(FileQuarantine, self).add(queueid, mailfrom, recipients, fp) super(FileQuarantine, self).add(queueid, mailfrom, recipients, fp)
quarantine_id = "{}_{}".format(datetime.now().strftime("%Y%m%d%H%M%S"), queueid) quarantine_id = "{}_{}".format(datetime.now().strftime("%Y%m%d%H%M%S"), queueid)
# save mail # save mail
self._save_datafile(quarantine_id, fp) self._save_datafile(quarantine_id, fp)
# save metadata # save metadata
metadata = { metadata = {
"from": mailfrom, "from": mailfrom,
@@ -119,19 +126,23 @@ class FileQuarantine(BaseQuarantine):
datafile = os.path.join(self.directory, quarantine_id) datafile = os.path.join(self.directory, quarantine_id)
os.remove(datafile) os.remove(datafile)
raise e raise e
return quarantine_id return quarantine_id
def get_metadata(self, quarantine_id): def get_metadata(self, quarantine_id):
"Return metadata of quarantined e-mail." "Return metadata of quarantined e-mail."
super(FileQuarantine, self).get_metadata(quarantine_id) super(FileQuarantine, self).get_metadata(quarantine_id)
metafile = os.path.join(self.directory, "{}{}".format(quarantine_id, self._metadata_suffix)) metafile = os.path.join(self.directory, "{}{}".format(quarantine_id, self._metadata_suffix))
if not os.path.isfile(metafile): if not os.path.isfile(metafile):
raise RuntimeError("invalid quarantine id '{}'".format(quarantine_id)) raise RuntimeError("invalid quarantine id '{}'".format(quarantine_id))
try: try:
with open(metafile, "rb") as f: with open(metafile, "rb") as f:
metadata = json.load(f) metadata = json.load(f)
except IOError as e: except IOError as e:
raise RuntimeError("unable to read metadata file: {}".format(e)) raise RuntimeError("unable to read metadata file: {}".format(e))
return metadata return metadata
def find(self, mailfrom=None, recipients=None, older_than=None): def find(self, mailfrom=None, recipients=None, older_than=None):
@@ -139,40 +150,49 @@ class FileQuarantine(BaseQuarantine):
super(FileQuarantine, self).find(mailfrom, recipients, older_than) super(FileQuarantine, self).find(mailfrom, recipients, older_than)
if type(mailfrom) == str: mailfrom = [mailfrom] if type(mailfrom) == str: mailfrom = [mailfrom]
if type(recipients) == str: recipients = [recipients] if type(recipients) == str: recipients = [recipients]
emails = {} emails = {}
metafiles = glob(os.path.join(self.directory, "*{}".format(self._metadata_suffix))) metafiles = glob(os.path.join(self.directory, "*{}".format(self._metadata_suffix)))
for metafile in metafiles: for metafile in metafiles:
if not os.path.isfile(metafile): continue if not os.path.isfile(metafile): continue
quarantine_id = os.path.basename(metafile[:-len(self._metadata_suffix)]) quarantine_id = os.path.basename(metafile[:-len(self._metadata_suffix)])
metadata = self.get_metadata(quarantine_id) metadata = self.get_metadata(quarantine_id)
if older_than != None: if older_than != None:
if timegm(gmtime()) - metadata["date"] < (older_than * 24 * 3600): if timegm(gmtime()) - metadata["date"] < (older_than * 24 * 3600):
continue continue
if mailfrom != None: if mailfrom != None:
if metadata["from"] not in mailfrom: if metadata["from"] not in mailfrom:
continue continue
if recipients != None: if recipients != None:
if len(recipients) == 1 and recipients[0] not in metadata["recipients"]: if len(recipients) == 1 and recipients[0] not in metadata["recipients"]:
continue continue
elif len(set(recipients + metadata["recipients"])) == len(recipients + metadata["recipients"]): elif len(set(recipients + metadata["recipients"])) == len(recipients + metadata["recipients"]):
continue continue
emails[quarantine_id] = metadata emails[quarantine_id] = metadata
return emails return emails
def delete(self, quarantine_id, recipient=None): def delete(self, quarantine_id, recipient=None):
"Delete e-mail in quarantine." "Delete e-mail in quarantine."
super(FileQuarantine, self).delete(quarantine_id, recipient) super(FileQuarantine, self).delete(quarantine_id, recipient)
try: try:
metadata = self.get_metadata(quarantine_id) metadata = self.get_metadata(quarantine_id)
except RuntimeError as e: except RuntimeError as e:
raise RuntimeError("unable to delete e-mail: {}".format(e)) raise RuntimeError("unable to delete e-mail: {}".format(e))
if recipient == None: if recipient == None:
self._remove(quarantine_id) self._remove(quarantine_id)
else: else:
if recipient not in metadata["recipients"]: if recipient not in metadata["recipients"]:
raise RuntimeError("invalid recipient '{}'".format(recipient)) raise RuntimeError("invalid recipient '{}'".format(recipient))
metadata["recipients"].remove(recipient) metadata["recipients"].remove(recipient)
if len(metadata["recipients"]) == 0: if not metadata["recipients"]:
self._remove(quarantine_id) self._remove(quarantine_id)
else: else:
self._save_metafile(quarantine_id, metadata) self._save_metafile(quarantine_id, metadata)
@@ -180,10 +200,12 @@ class FileQuarantine(BaseQuarantine):
def release(self, quarantine_id, recipient=None): def release(self, quarantine_id, recipient=None):
"Release e-mail from quarantine." "Release e-mail from quarantine."
super(FileQuarantine, self).release(quarantine_id, recipient) super(FileQuarantine, self).release(quarantine_id, recipient)
try: try:
metadata = self.get_metadata(quarantine_id) metadata = self.get_metadata(quarantine_id)
except RuntimeError as e: except RuntimeError as e:
raise RuntimeError("unable to release e-mail: {}".format(e)) raise RuntimeError("unable to release e-mail: {}".format(e))
datafile = os.path.join(self.directory, quarantine_id) datafile = os.path.join(self.directory, quarantine_id)
if recipient != None: if recipient != None:
if recipient not in metadata["recipients"]: if recipient not in metadata["recipients"]:
@@ -191,19 +213,21 @@ class FileQuarantine(BaseQuarantine):
recipients = [recipient] recipients = [recipient]
else: else:
recipients = metadata["recipients"] recipients = metadata["recipients"]
try: try:
with open(datafile, "rb") as f: with open(datafile, "rb") as f:
mail = f.read() mail = f.read()
except IOError as e: except IOError as e:
raise RuntimeError("unable to read data file: {}".format(e)) raise RuntimeError("unable to read data file: {}".format(e))
for recipient in recipients: for recipient in recipients:
try: try:
mailer.smtp_send(self.config["smtp_host"], self.config["smtp_port"], metadata["from"], recipient, mail) mailer.smtp_send(self.config["smtp_host"], self.config["smtp_port"], metadata["from"], recipient, mail)
except Exception as e: except Exception as e:
raise RuntimeError("error while sending e-mail to '{}': {}".format(recipient, e)) raise RuntimeError("error while sending e-mail to '{}': {}".format(recipient, e))
self.delete(quarantine_id, recipient) self.delete(quarantine_id, recipient)
# list of quarantine types and their related quarantine classes # list of quarantine types and their related quarantine classes
quarantine_types = {"file": FileQuarantine} TYPES = {"file": FileQuarantine}

15
pyquarantine/run.py
View File

@@ -23,7 +23,6 @@ import sys
import pyquarantine import pyquarantine
def main(): def main():
"Run PyQuarantine-Milter." "Run PyQuarantine-Milter."
# parse command line # parse command line
@@ -35,6 +34,7 @@ def main():
parser.add_argument("-d", "--debug", help="Log debugging messages.", action="store_true") parser.add_argument("-d", "--debug", help="Log debugging messages.", action="store_true")
parser.add_argument("-t", "--test", help="Check configuration.", action="store_true") parser.add_argument("-t", "--test", help="Check configuration.", action="store_true")
args = parser.parse_args() args = parser.parse_args()
# setup logging # setup logging
loglevel = logging.INFO loglevel = logging.INFO
logname = "pyquarantine-milter" logname = "pyquarantine-milter"
@@ -43,10 +43,12 @@ def main():
loglevel = logging.DEBUG loglevel = logging.DEBUG
logname = "{}[%(name)s]".format(logname) logname = "{}[%(name)s]".format(logname)
syslog_name = "{}: [%(name)s] %(levelname)s".format(syslog_name) syslog_name = "{}: [%(name)s] %(levelname)s".format(syslog_name)
# set config files for milter class # set config files for milter class
pyquarantine.QuarantineMilter.set_configfiles(args.config) pyquarantine.QuarantineMilter.set_configfiles(args.config)
root_logger = logging.getLogger() root_logger = logging.getLogger()
root_logger.setLevel(loglevel) root_logger.setLevel(loglevel)
# setup console log # setup console log
stdouthandler = logging.StreamHandler(sys.stdout) stdouthandler = logging.StreamHandler(sys.stdout)
stdouthandler.setLevel(logging.DEBUG) stdouthandler.setLevel(logging.DEBUG)
@@ -63,26 +65,32 @@ def main():
sys.exit(255) sys.exit(255)
else: else:
sys.exit(0) sys.exit(0)
formatter = logging.Formatter("%(asctime)s {}: %(levelname)s - %(message)s".format(logname), datefmt="%Y-%m-%d %H:%M:%S") formatter = logging.Formatter("%(asctime)s {}: [%(levelname)s] %(message)s".format(logname), datefmt="%Y-%m-%d %H:%M:%S")
stdouthandler.setFormatter(formatter) stdouthandler.setFormatter(formatter)
# setup syslog # setup syslog
sysloghandler = logging.handlers.SysLogHandler(address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL) sysloghandler = logging.handlers.SysLogHandler(address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL)
sysloghandler.setLevel(loglevel) sysloghandler.setLevel(loglevel)
formatter = logging.Formatter("{}: %(message)s".format(syslog_name)) formatter = logging.Formatter("{}: %(message)s".format(syslog_name))
sysloghandler.setFormatter(formatter) sysloghandler.setFormatter(formatter)
root_logger.addHandler(sysloghandler) root_logger.addHandler(sysloghandler)
logger.info("PyQuarantine-Milter starting") logger.info("PyQuarantine-Milter starting")
try: try:
# generate milter config # generate milter config
config = pyquarantine.generate_milter_config() global_config, config = pyquarantine.generate_milter_config()
except RuntimeError as e: except RuntimeError as e:
logger.error(e) logger.error(e)
sys.exit(255) sys.exit(255)
pyquarantine.QuarantineMilter.global_config = global_config
pyquarantine.QuarantineMilter.config = config pyquarantine.QuarantineMilter.config = config
# register to have the Milter factory create instances of your class: # register to have the Milter factory create instances of your class:
Milter.factory = pyquarantine.QuarantineMilter Milter.factory = pyquarantine.QuarantineMilter
Milter.set_exception_policy(Milter.TEMPFAIL) Milter.set_exception_policy(Milter.TEMPFAIL)
#Milter.set_flags(0) # tell sendmail which features we use #Milter.set_flags(0) # tell sendmail which features we use
# run milter # run milter
rc = 0 rc = 0
try: try:
@@ -95,6 +103,5 @@ def main():
sys.exit(rc) sys.exit(rc)
if __name__ == "__main__": if __name__ == "__main__":
main() main()

81
pyquarantine/whitelists.py
View File

@@ -22,19 +22,18 @@ import sys
from playhouse.db_url import connect from playhouse.db_url import connect
class WhitelistBase(object): class WhitelistBase(object):
"Whitelist base class" "Whitelist base class"
def __init__(self, name, config, configtest=False): def __init__(self, global_config, config, configtest=False):
self.name = name self.global_config = global_config
self.config = config[name] self.config = config
self.configtest = configtest self.configtest = configtest
self.global_config = config["global"] self.name = config["name"]
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
self.valid_entry_regex = re.compile(r"^[a-zA-Z0-9_.+-]*?(@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)?$") self.valid_entry_regex = re.compile(r"^[a-zA-Z0-9_.+-]*?(@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)?$")
def check(self, mailfrom, recipient): def check(self, mailfrom, recipient):
# check if mailfrom/recipient combination is whitelisted "Check if mailfrom/recipient combination is whitelisted."
return return
def find(self, mailfrom=None, recipients=None, older_than=None): def find(self, mailfrom=None, recipients=None, older_than=None):
@@ -55,7 +54,6 @@ class WhitelistBase(object):
return return
class WhitelistModel(peewee.Model): class WhitelistModel(peewee.Model):
mailfrom = peewee.CharField() mailfrom = peewee.CharField()
recipient = peewee.CharField() recipient = peewee.CharField()
@@ -65,51 +63,60 @@ class WhitelistModel(peewee.Model):
permanent = peewee.BooleanField(default=False) permanent = peewee.BooleanField(default=False)
class Meta(object): class Meta(object):
indexes = ( indexes = (
(('mailfrom', 'recipient'), True), # trailing comma is mandatory if only one index should be created (('mailfrom', 'recipient'), True), # trailing comma is mandatory if only one index should be created
) )
class DatabaseWhitelist(WhitelistBase): class DatabaseWhitelist(WhitelistBase):
"Whitelist class to store whitelist in a database" "Whitelist class to store whitelist in a database"
_whitelists = {} _db_connections = {}
_db_tables = {}
def __init__(self, global_config, config, configtest=False):
super(DatabaseWhitelist, self).__init__(global_config, config, configtest)
def __init__(self, name, config, configtest=False):
super(DatabaseWhitelist, self).__init__(name, config, configtest)
# check if mandatory options are present in config # check if mandatory options are present in config
for option in ["whitelist_db_connection", "whitelist_db_table"]: for option in ["whitelist_db_connection", "whitelist_db_table"]:
if option not in self.config.keys() and option in self.global_config.keys(): if option not in self.config.keys() and option in self.global_config.keys():
self.config[option] = self.global_config[option] self.config[option] = self.global_config[option]
if option not in self.config.keys(): if option not in self.config.keys():
raise RuntimeError("mandatory option '{}' not present in config section '{}' or 'global'".format(option, self.name)) raise RuntimeError("mandatory option '{}' not present in config section '{}' or 'global'".format(option, self.name))
self.tablename = self.config["whitelist_db_table"]
tablename = self.config["whitelist_db_table"]
connection_string = self.config["whitelist_db_connection"] connection_string = self.config["whitelist_db_connection"]
if connection_string in DatabaseWhitelist._whitelists.keys():
new_connection = False if connection_string in DatabaseWhitelist._db_connections.keys():
self.db = DatabaseWhitelist._whitelists[connection_string] db = DatabaseWhitelist._db_connections[connection_string]
else: else:
new_connection = True
try: try:
# connect to database # connect to database
self.logger.debug("connecting to database '{}'".format(re.sub(r"(.*?://.*?):.*?(@.*)", r"\1:<PASSWORD>\2", connection_string))) self.logger.debug("connecting to database '{}'".format(re.sub(r"(.*?://.*?):.*?(@.*)", r"\1:<PASSWORD>\2", connection_string)))
self.db = connect(connection_string) db = connect(connection_string)
except Exception as e: except Exception as e:
raise RuntimeError("unable to connect to database: {}".format(e)) raise RuntimeError("unable to connect to database: {}".format(e))
DatabaseWhitelist._whitelists[connection_string] = self.db
DatabaseWhitelist._db_connections[connection_string] = db
# generate model meta class
self.meta = Meta self.meta = Meta
self.meta.database = self.db self.meta.database = db
self.meta.table_name = self.tablename self.meta.table_name = tablename
self.model = type("WhitelistModel_{}".format(name), (WhitelistModel,), { self.model = type("WhitelistModel_{}".format(self.name), (WhitelistModel,), {
"Meta": self.meta "Meta": self.meta
}) })
if new_connection and not self.configtest:
if connection_string not in DatabaseWhitelist._db_tables.keys():
DatabaseWhitelist._db_tables[connection_string] = []
if tablename not in DatabaseWhitelist._db_tables[connection_string]:
DatabaseWhitelist._db_tables[connection_string].append(tablename)
if not self.configtest:
try: try:
self.db.create_tables([self.model]) db.create_tables([self.model])
except Exception as e: except Exception as e:
raise RuntimeError("unable to initialize table '{}': {}".format(self.tablename, e)) raise RuntimeError("unable to initialize table '{}': {}".format(tablename, e))
def _entry_to_dict(self, entry): def _entry_to_dict(self, entry):
result = {} result = {}
@@ -136,27 +143,33 @@ class DatabaseWhitelist(WhitelistBase):
def check(self, mailfrom, recipient): def check(self, mailfrom, recipient):
# check if mailfrom/recipient combination is whitelisted # check if mailfrom/recipient combination is whitelisted
super(DatabaseWhitelist, self).check(mailfrom, recipient) super(DatabaseWhitelist, self).check(mailfrom, recipient)
# generate list of possible mailfroms # generate list of possible mailfroms
self.logger.debug("query database for whitelist entries from <{}> to <{}>".format(mailfrom, recipient)) self.logger.debug("query database for whitelist entries from <{}> to <{}>".format(mailfrom, recipient))
mailfroms = [""] mailfroms = [""]
if "@" in mailfrom and not mailfrom.startswith("@"): if "@" in mailfrom and not mailfrom.startswith("@"):
mailfroms.append("@{}".format(mailfrom.split("@")[1])) mailfroms.append("@{}".format(mailfrom.split("@")[1]))
mailfroms.append(mailfrom) mailfroms.append(mailfrom)
# generate list of possible recipients # generate list of possible recipients
recipients = [""] recipients = [""]
if "@" in recipient and not recipient.startswith("@"): if "@" in recipient and not recipient.startswith("@"):
recipients.append("@{}".format(recipient.split("@")[1])) recipients.append("@{}".format(recipient.split("@")[1]))
recipients.append(recipient) recipients.append(recipient)
# query the database # query the database
try: try:
entries = list(self.model.select().where(self.model.mailfrom.in_(mailfroms), self.model.recipient.in_(recipients))) entries = list(self.model.select().where(self.model.mailfrom.in_(mailfroms), self.model.recipient.in_(recipients)))
except Exception as e: except Exception as e:
raise RuntimeError("unable to query database: {}".format(e)) raise RuntimeError("unable to query database: {}".format(e))
if len(entries) == 0:
if not entries:
# no whitelist entry found # no whitelist entry found
return {} return {}
if len(entries) > 1: if len(entries) > 1:
entries.sort(key=lambda x: self.get_weight(x), reverse=True) entries.sort(key=lambda x: self.get_weight(x), reverse=True)
# use entry with the highest weight # use entry with the highest weight
entry = entries[0] entry = entries[0]
entry.last_used = datetime.datetime.now() entry.last_used = datetime.datetime.now()
@@ -164,33 +177,41 @@ class DatabaseWhitelist(WhitelistBase):
result = {} result = {}
for entry in entries: for entry in entries:
result.update(self._entry_to_dict(entry)) result.update(self._entry_to_dict(entry))
return result return result
def find(self, mailfrom=None, recipients=None, older_than=None): def find(self, mailfrom=None, recipients=None, older_than=None):
"Find whitelist entries." "Find whitelist entries."
super(DatabaseWhitelist, self).find(mailfrom, recipients, older_than) super(DatabaseWhitelist, self).find(mailfrom, recipients, older_than)
if type(mailfrom) == str: mailfrom = [mailfrom] if type(mailfrom) == str: mailfrom = [mailfrom]
if type(recipients) == str: recipients = [recipients] if type(recipients) == str: recipients = [recipients]
entries = {} entries = {}
try: try:
for entry in list(self.model.select()): for entry in list(self.model.select()):
if older_than != None: if older_than != None:
if (datetime.datetime.now() - entry.last_used).total_seconds() < (older_than * 24 * 3600): if (datetime.datetime.now() - entry.last_used).total_seconds() < (older_than * 24 * 3600):
continue continue
if mailfrom != None: if mailfrom != None:
if entry.mailfrom not in mailfrom: if entry.mailfrom not in mailfrom:
continue continue
if recipients != None: if recipients != None:
if entry.recipient not in recipients: if entry.recipient not in recipients:
continue continue
entries.update(self._entry_to_dict(entry)) entries.update(self._entry_to_dict(entry))
except Exception as e: except Exception as e:
raise RuntimeError("unable to query database: {}".format(e)) raise RuntimeError("unable to query database: {}".format(e))
return entries return entries
def add(self, mailfrom, recipient, comment, permanent): def add(self, mailfrom, recipient, comment, permanent):
"Add entry to whitelist." "Add entry to whitelist."
super(DatabaseWhitelist, self).add(mailfrom, recipient, comment, permanent) super(DatabaseWhitelist, self).add(mailfrom, recipient, comment, permanent)
try: try:
self.model.create(mailfrom=mailfrom, recipient=recipient, comment=comment, permanent=permanent) self.model.create(mailfrom=mailfrom, recipient=recipient, comment=comment, permanent=permanent)
except Exception as e: except Exception as e:
@@ -199,16 +220,17 @@ class DatabaseWhitelist(WhitelistBase):
def delete(self, whitelist_id): def delete(self, whitelist_id):
"Delete entry from whitelist." "Delete entry from whitelist."
super(DatabaseWhitelist, self).delete(whitelist_id) super(DatabaseWhitelist, self).delete(whitelist_id)
try: try:
query = self.model.delete().where(self.model.id == whitelist_id) query = self.model.delete().where(self.model.id == whitelist_id)
deleted = query.execute() deleted = query.execute()
except Exception as e: except Exception as e:
raise RuntimeError("unable to delete entry from database: {}".format(e)) raise RuntimeError("unable to delete entry from database: {}".format(e))
if deleted == 0: if deleted == 0:
raise RuntimeError("invalid whitelist id") raise RuntimeError("invalid whitelist id")
class WhitelistCache(object): class WhitelistCache(object):
def __init__(self): def __init__(self):
self.cache = {} self.cache = {}
@@ -226,9 +248,8 @@ class WhitelistCache(object):
def get_whitelisted_recipients(self, whitelist, mailfrom, recipients): def get_whitelisted_recipients(self, whitelist, mailfrom, recipients):
self.load(whitelist, mailfrom, recipients) self.load(whitelist, mailfrom, recipients)
return filter(lambda x: len(self.cache[whitelist][x]) > 0, self.cache[whitelist].keys()) return filter(lambda x: self.cache[whitelist][x], self.cache[whitelist].keys())
# list of whitelist types and their related whitelist classes # list of whitelist types and their related whitelist classes
whitelist_types = {"db": DatabaseWhitelist} TYPES = {"db": DatabaseWhitelist}