spacefreak/pyquarantine-milter
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Restructure code and rename some config variables

Browse Source
This commit is contained in:
Thomas Oettli
2020-02-03 02:03:59 +01:00
parent 182ca2bad7
commit ab16c9f83e
8 changed files with 584 additions and 538 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
README.md
View File

@@ -36,8 +36,8 @@ The following configuration options are mandatory in the global section:
The following configuration options are mandatory in each quarantine section: The following configuration options are mandatory in each quarantine section:
* **regex** * **regex**
Case insensitive regular expression to filter e-mail headers. Case insensitive regular expression to filter e-mail headers.
* **quarantine_type** * **storage_type**
One of the quarantine-types described below. One of the storage types described below.
* **action** * **action**
One of the actions described below. One of the actions described below.
* **notification_type** * **notification_type**
@@ -50,13 +50,13 @@ The following configuration options are mandatory in each quarantine section:
SMTP port SMTP port
The following configuration options are optional in each quarantine section: The following configuration options are optional in each quarantine section:
* **ignore_hosts** * **host_whitelist**
Comma-separated list of host and network addresses to be ignored by this quarantine. Comma-separated list of host and network addresses to be ignored by this quarantine.
* **reject_reason** * **reject_reason**
Reason to return to the client if action is set to reject. Reason to return to the client if action is set to reject.
### Quarantine types ### Storage types
* **NONE** * **NONE**
Original e-mails scrapped, sent to nirvana, black-holed or however you want to call it. Original e-mails scrapped, sent to nirvana, black-holed or however you want to call it.
@@ -64,7 +64,7 @@ The following configuration options are optional in each quarantine section:
Original e-mails are stored on the filesystem with a unique filename. The filename is available as a Original e-mails are stored on the filesystem with a unique filename. The filename is available as a
template variable used in notifiaction templates. template variable used in notifiaction templates.
The following configuration options are mandatory for this quarantine type: The following configuration options are mandatory for this quarantine type:
* **quarantine_directory** * **storage_directory**
The directory in which quarantined e-mails are stored. The directory in which quarantined e-mails are stored.

14
docs/pyquarantine.conf.example
View File

@@ -30,12 +30,12 @@ preferred_quarantine_action = last
[spam] [spam]
# Option: ignore_hosts # Option: host_whitelist
# Notes: Set a list of host and network addresses to be ignored by this quarantine. # Notes: Set a list of host and network addresses to be ignored by this quarantine.
# All the common host/network notations are supported, including IPv6. # All the common host/network notations are supported, including IPv6.
# Value: [ HOST ] # Value: [ HOST ]
# #
ignore_hosts = 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16 host_whitelist = 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
# Option: regex # Option: regex
# Notes: Set the case insensitive regular expression to match against email headers. # Notes: Set the case insensitive regular expression to match against email headers.
@@ -57,18 +57,18 @@ smtp_host = 127.0.0.1
# #
smtp_port = 25 smtp_port = 25
# Option: quarantine_type # Option: storage_type
# Notes: Set the quarantine type. # Notes: Set the storage type.
# Values: [ file | none ] # Values: [ file | none ]
# #
quarantine_type = file storage_type = file
# Option: quarantine_directory # Option: storage_directory
# Notes: Set the directory to store quarantined emails. # Notes: Set the directory to store quarantined emails.
# This option is needed by quarantine type 'file'. # This option is needed by quarantine type 'file'.
# Values: [ DIRECTORY ] # Values: [ DIRECTORY ]
# #
quarantine_directory = /var/lib/pyquarantine/spam storage_directory = /var/lib/pyquarantine/spam
# Option: action # Option: action
# Notes: Set the milter action to perform if email is processed by this quarantine. # Notes: Set the milter action to perform if email is processed by this quarantine.

558
pyquarantine/__init__.py
View File

@@ -13,13 +13,14 @@
# #
__all__ = [ __all__ = [
"Quarantine",
"QuarantineMilter", "QuarantineMilter",
"generate_milter_config", "setup_milter",
"reload_config", "reload_config",
"cli", "cli",
"mailer", "mailer",
"notifications", "notifications",
"quarantines", "storages",
"run", "run",
"version", "version",
"whitelists"] "whitelists"]
@@ -38,78 +39,295 @@ from collections import defaultdict
from io import BytesIO from io import BytesIO
from itertools import groupby from itertools import groupby
from netaddr import IPAddress, IPNetwork from netaddr import IPAddress, IPNetwork
from pyquarantine import quarantines from pyquarantine import mailer
from pyquarantine import notifications from pyquarantine import notifications
from pyquarantine import storages
from pyquarantine import whitelists from pyquarantine import whitelists
class QuarantineMilter(Milter.Base): class Quarantine(object):
"""QuarantineMilter based on Milter.Base to implement milter communication """Quarantine class suitable for QuarantineMilter
The class variable config needs to be filled with the result of the generate_milter_config function. The class holds all the objects and functions needed for QuarantineMilter quarantine.
""" """
config = None
global_config = None
# list of default config files
_config_files = [
"/etc/pyquarantine/pyquarantine.conf",
os.path.expanduser('~/pyquarantine.conf'),
"pyquarantine.conf"]
# list of possible actions # list of possible actions
_actions = { _actions = {
"ACCEPT": Milter.ACCEPT, "ACCEPT": Milter.ACCEPT,
"REJECT": Milter.REJECT, "REJECT": Milter.REJECT,
"DISCARD": Milter.DISCARD} "DISCARD": Milter.DISCARD}
def __init__(self, name, index=0, regex=None, storage=None, whitelist=None,
host_whitelist=[], notification=None, action="ACCEPT",
reject_reason=None):
self.logger = logging.getLogger(__name__)
self.name = name
self.index = index
if regex:
self.regex = re.compile(
regex, re.MULTILINE + re.DOTALL + re.IGNORECASE)
self.storage = storage
self.whitelist = whitelist
self.host_whitelist = host_whitelist
self.notification = notification
action = action.upper()
assert action in self._actions
self.action = action
self.milter_action = self._actions[action]
self.reject_reason = reject_reason
def setup_from_cfg(self, global_cfg, cfg, test=False):
defaults = {
"action": "accept",
"reject_reason": "Message rejected",
"storage_type": "none",
"notification_type": "none",
"whitelist_type": "none",
"host_whitelist": ""
}
# check config
for opt in ["regex", "smtp_host", "smtp_port"] + list(defaults.keys()):
if opt in cfg:
continue
if opt in global_cfg:
cfg[opt] = global_cfg[opt]
elif opt in defaults:
cfg[opt] = defaults[opt]
else:
raise RuntimeError(
"mandatory option '{}' not present in config section '{}' or 'global'".format(
opt, self.name))
# pre-compile regex
self.logger.debug(
"{}: compiling regex '{}'".format(
self.name, cfg["regex"]))
self.regex = re.compile(
cfg["regex"], re.MULTILINE + re.DOTALL + re.IGNORECASE)
self.smtp_host = cfg["smtp_host"]
self.smtp_port = cfg["smtp_port"]
# create storage instance
storage_type = cfg["storage_type"].lower()
if storage_type in storages.TYPES:
self.logger.debug(
"{}: initializing storage type '{}'".format(
self.name,
storage_type.upper()))
self.storage = storages.TYPES[storage_type](
self.name, global_cfg, cfg, test)
elif storage_type == "none":
self.logger.debug("{}: storage is NONE".format(self.name))
self.storage = None
else:
raise RuntimeError(
"{}: unknown storage type '{}'".format(
self.name, storage_type))
# create whitelist instance
whitelist_type = cfg["whitelist_type"].lower()
if whitelist_type in whitelists.TYPES:
self.logger.debug(
"{}: initializing whitelist type '{}'".format(
self.name,
whitelist_type.upper()))
self.whitelist = whitelists.TYPES[whitelist_type](
self.name, global_cfg, cfg, test)
elif whitelist_type == "none":
logger.debug("{}: whitelist is NONE".format(self.name))
self.whitelist = None
else:
raise RuntimeError(
"{}: unknown whitelist type '{}'".format(
self.name, whitelist_type))
# create notification instance
notification_type = cfg["notification_type"].lower()
if notification_type in notifications.TYPES:
self.logger.debug(
"{}: initializing notification type '{}'".format(
self.name,
notification_type.upper()))
self.notification = notifications.TYPES[notification_type](
self.name, global_cfg, cfg, test)
elif notification_type == "none":
self.logger.debug("{}: notification is NONE".format(self.name))
self.notification = None
else:
raise RuntimeError(
"{}: unknown notification type '{}'".format(
self.name, notification_type))
# determining milter action for this quarantine
action = cfg["action"].upper()
if action in self._actions:
self.logger.debug("{}: action is {}".format(self.name, action))
self.action = action
self.milter_action = self._actions[action]
else:
raise RuntimeError(
"{}: unknown action '{}'".format(self._name, action))
self.reject_reason = cfg["reject_reason"]
# create host/network whitelist
self.host_whitelist = []
host_whitelist = set([p.strip()
for p in cfg["host_whitelist"].split(",") if p])
for host in host_whitelist:
if not host:
continue
# parse network notation
try:
net = IPNetwork(host)
except AddrFormatError as e:
raise RuntimeError("{}: error parsing host_whitelist: {}".format(
self.name, e))
else:
self.host_whitelist.append(net)
if self.host_whitelist:
self.logger.debug(
"{}: ignore hosts: {}".format(
self.name,
", ".join(ignored)))
def notify(self, storage_id, recipient=None, synchronous=True):
"Notify recipient about email in storage."
if not self.storage:
raise RuntimeError(
"storage type is set to None, unable to send notification")
if not self.notification:
raise RuntimeError(
"notification type is set to None, unable to send notification")
fp, metadata = self.storage.get_mail(storage_id)
if recipient is not None:
if recipient not in metadata["recipients"]:
raise RuntimeError("invalid recipient '{}'".format(recipient))
recipients = [recipient]
else:
recipients = metadata["recipients"]
self.notification.notify(
metadata["queue_id"], storage_id, metadata["mailfrom"],
recipients, metadata["headers"], fp,
metadata["subgroups"], metadata["named_subgroups"],
synchronous)
fp.close()
def release(self, storage_id, recipients=None):
"Release email from storage."
if not self.storage:
raise RuntimeError(
"storage type is set to None, unable to release email")
fp, metadata = self.storage.get_mail(storage_id)
try:
mail = fp.read()
fp.close()
except IOError as e:
raise RuntimeError("unable to read data file: {}".format(e))
if recipients and type(recipients) == str:
recipients = [recipients]
else:
recipients = metadata["recipients"]
for recipient in recipients:
if recipient not in metadata["recipients"]:
raise RuntimeError("invalid recipient '{}'".format(recipient))
try:
mailer.smtp_send(
self.smtp_host,
self.smtp_port,
metadata["mailfrom"],
recipient,
mail)
except Exception as e:
raise RuntimeError(
"error while sending email to '{}': {}".format(
recipient, e))
self.storage.delete(storage_id, recipient)
def get_storage(self):
return self.storage
def get_notification(self):
return self.notification
def get_whitelist(self):
return self.whitelist
def host_in_whitelist(self, hostaddr):
ip = IPAddress(hostaddr[0])
for entry in self.host_whitelist:
if ip in entry:
return true
return False
def match(self, header):
return self.regex.search(header)
class QuarantineMilter(Milter.Base):
"""QuarantineMilter based on Milter.Base to implement milter communication
The class variable quarantines needs to be filled by runng the setup_milter function.
"""
quarantines = []
preferred_quarantine_action = "first"
# list of default config files
_cfg_files = [
"/etc/pyquarantine/pyquarantine.conf",
os.path.expanduser('~/pyquarantine.conf'),
"pyquarantine.conf"]
def __init__(self): def __init__(self):
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
# save config, it must not change during runtime # save runtime config, it must not change during runtime
self.global_config = QuarantineMilter.global_config self.quarantines = QuarantineMilter.quarantines
self.config = QuarantineMilter.config
def _get_preferred_quarantine(self): def _get_preferred_quarantine(self):
matching_quarantines = [ matching_quarantines = [
q for q in self.recipients_quarantines.values() if q] q for q in self.recipients_quarantines.values() if q]
if self.global_config["preferred_quarantine_action"] == "first": if self.preferred_quarantine_action == "first":
quarantine = sorted( quarantine = sorted(
matching_quarantines, matching_quarantines,
key=lambda x: x["index"])[0] key=lambda q: q.index)[0]
else: else:
quarantine = sorted( quarantine = sorted(
matching_quarantines, matching_quarantines,
key=lambda x: x["index"], key=lambda q: q.index,
reverse=True)[0] reverse=True)[0]
return quarantine return quarantine
@staticmethod @staticmethod
def get_configfiles(): def get_cfg_files():
return QuarantineMilter._config_files return QuarantineMilter._cfg_files
@staticmethod @staticmethod
def get_actions(): def set_cfg_files(cfg_files):
return QuarantineMilter._actions QuarantineMilter._cfg_files = cfg_files
@staticmethod
def set_configfiles(config_files):
QuarantineMilter._config_files = config_files
def connect(self, hostname, family, hostaddr): def connect(self, hostname, family, hostaddr):
self.hostaddr = hostaddr self.hostaddr = hostaddr
self.logger.debug( self.logger.debug(
"accepted milter connection from {} port {}".format( "accepted milter connection from {} port {}".format(
*hostaddr)) *hostaddr))
ip = IPAddress(hostaddr[0]) for quarantine in self.quarantines.copy():
for quarantine in self.config.copy(): if quarantine.host_in_whitelist(hostaddr):
for ignore in quarantine["ignore_hosts_list"]:
if ip in ignore:
self.logger.debug( self.logger.debug(
"host {} is ignored by quarantine {}".format( "host {} is ignored by quarantine {}".format(
hostaddr[0], quarantine["name"])) hostaddr[0], quarantine["name"]))
self.config.remove(quarantine) self.quarantines.remove(quarantine)
break if not self.quarantines:
if not self.config:
self.logger.debug( self.logger.debug(
"host {} is ignored by all quarantines, " "host {} is ignored by all quarantines, "
"skip further processing", "skip further processing",
@@ -176,39 +394,39 @@ class QuarantineMilter(Milter.Base):
"{}: checking header against configured quarantines: {}".format( "{}: checking header against configured quarantines: {}".format(
self.queueid, header)) self.queueid, header))
# iterate quarantines # iterate quarantines
for quarantine in self.config: for quarantine in self.quarantines:
if len(self.recipients_quarantines) == len( if len(self.recipients_quarantines) == len(
self.recipients): self.recipients):
# every recipient matched a quarantine already # every recipient matched a quarantine already
if quarantine["index"] >= max( if quarantine.index >= max(
[q["index"] for q in self.recipients_quarantines.values()]): [q.index for q in self.recipients_quarantines.values()]):
# all recipients matched a quarantine with at least # all recipients matched a quarantine with at least
# the same precedence already, skip checks against # the same precedence already, skip checks against
# quarantines with lower precedence # quarantines with lower precedence
self.logger.debug( self.logger.debug(
"{}: {}: skip further checks of this header".format( "{}: {}: skip further checks of this header".format(
self.queueid, quarantine["name"])) self.queueid, quarantine.name))
break break
# check email header against quarantine regex # check email header against quarantine regex
self.logger.debug( self.logger.debug(
"{}: {}: checking header against regex '{}'".format( "{}: {}: checking header against regex '{}'".format(
self.queueid, quarantine["name"], quarantine["regex"])) self.queueid, quarantine.name, quarantine.regex))
match = quarantine["regex_compiled"].search(header) match = quarantine.match(header)
if match: if match:
self.logger.debug( self.logger.debug(
"{}: {}: header matched regex".format( "{}: {}: header matched regex".format(
self.queueid, quarantine["name"])) self.queueid, quarantine.name))
# check for whitelisted recipients # check for whitelisted recipients
whitelist = quarantine["whitelist_obj"] whitelist = quarantine.get_whitelist()
if whitelist is not None: if whitelist:
try: try:
whitelisted_recipients = self.whitelist_cache.get_whitelisted_recipients( whitelisted_recipients = self.whitelist_cache.get_whitelisted_recipients(
whitelist, self.mailfrom, recipients_to_check) whitelist, self.mailfrom, recipients_to_check)
except RuntimeError as e: except RuntimeError as e:
self.logger.error( self.logger.error(
"{}: {}: unable to query whitelist: {}".format( "{}: {}: unable to query whitelist: {}".format(
self.queueid, quarantine["name"], e)) self.queueid, quarantine.name, e))
return Milter.TEMPFAIL return Milter.TEMPFAIL
else: else:
whitelisted_recipients = {} whitelisted_recipients = {}
@@ -219,19 +437,19 @@ class QuarantineMilter(Milter.Base):
# recipient is whitelisted in this quarantine # recipient is whitelisted in this quarantine
self.logger.debug( self.logger.debug(
"{}: {}: recipient '{}' is whitelisted".format( "{}: {}: recipient '{}' is whitelisted".format(
self.queueid, quarantine["name"], recipient)) self.queueid, quarantine.name, recipient))
continue continue
if recipient not in self.recipients_quarantines.keys() or \ if recipient not in self.recipients_quarantines.keys() or \
self.recipients_quarantines[recipient]["index"] > quarantine["index"]: self.recipients_quarantines[recipient].index > quarantine.index:
self.logger.debug( self.logger.debug(
"{}: {}: set quarantine for recipient '{}'".format( "{}: {}: set quarantine for recipient '{}'".format(
self.queueid, quarantine["name"], recipient)) self.queueid, quarantine.name, recipient))
# save match for later use as template # save match for later use as template
# variables # variables
self.quarantines_matches[quarantine["name"]] = match self.quarantines_matches[quarantine.name] = match
self.recipients_quarantines[recipient] = quarantine self.recipients_quarantines[recipient] = quarantine
if quarantine["index"] == 0: if quarantine.index == 0:
# we do not need to check recipients which # we do not need to check recipients which
# matched the quarantine with the highest # matched the quarantine with the highest
# precedence already # precedence already
@@ -240,7 +458,7 @@ class QuarantineMilter(Milter.Base):
self.logger.debug( self.logger.debug(
"{}: {}: a quarantine with same or higher precedence " "{}: {}: a quarantine with same or higher precedence "
"matched already for recipient '{}'".format( "matched already for recipient '{}'".format(
self.queueid, quarantine["name"], recipient)) self.queueid, quarantine.name, recipient))
if not recipients_to_check: if not recipients_to_check:
self.logger.debug( self.logger.debug(
@@ -259,7 +477,7 @@ class QuarantineMilter(Milter.Base):
# check if the mail body is needed # check if the mail body is needed
for recipient, quarantine in self.recipients_quarantines.items(): for recipient, quarantine in self.recipients_quarantines.items():
if quarantine["quarantine_obj"] or quarantine["notification_obj"]: if quarantine.get_storage() or quarantine.get_notification():
# mail body is needed, continue processing # mail body is needed, continue processing
return Milter.CONTINUE return Milter.CONTINUE
@@ -269,12 +487,12 @@ class QuarantineMilter(Milter.Base):
self.logger.info( self.logger.info(
"{}: {} matching quarantine is '{}', performing milter action {}".format( "{}: {} matching quarantine is '{}', performing milter action {}".format(
self.queueid, self.queueid,
self.global_config["preferred_quarantine_action"], self.preferred_quarantine_action,
quarantine["name"], quarantine.name,
quarantine["action"].upper())) quarantine.action))
if quarantine["action"] == "reject": if quarantine.action == "REJECT":
self.setreply("554", "5.7.0", quarantine["reject_reason"]) self.setreply("554", "5.7.0", quarantine.reject_reason)
return quarantine["milter_action"] return quarantine.milter_action
except Exception as e: except Exception as e:
self.logger.exception( self.logger.exception(
@@ -297,52 +515,54 @@ class QuarantineMilter(Milter.Base):
quarantines = [] quarantines = []
for quarantine, recipients in groupby( for quarantine, recipients in groupby(
sorted(self.recipients_quarantines, sorted(self.recipients_quarantines,
key=lambda x: self.recipients_quarantines[x]["index"]), key=lambda x: self.recipients_quarantines[x].index),
lambda x: self.recipients_quarantines[x]): lambda x: self.recipients_quarantines[x]):
quarantines.append((quarantine, list(recipients))) quarantines.append((quarantine, list(recipients)))
# iterate quarantines sorted by index # iterate quarantines sorted by index
for quarantine, recipients in sorted( for quarantine, recipients in sorted(
quarantines, key=lambda x: x[0]["index"]): quarantines, key=lambda x: x[0].index):
quarantine_id = ""
headers = defaultdict(str) headers = defaultdict(str)
for name, value in self.headers: for name, value in self.headers:
headers[name.lower()] = value headers[name.lower()] = value
subgroups = self.quarantines_matches[quarantine["name"]].groups( subgroups = self.quarantines_matches[quarantine.name].groups(
default="") default="")
named_subgroups = self.quarantines_matches[quarantine["name"]].groupdict( named_subgroups = self.quarantines_matches[quarantine.name].groupdict(
default="") default="")
# check if a quarantine is configured # check if a storage is configured
if quarantine["quarantine_obj"] is not None: storage_id = ""
storage = quarantine.get_storage()
if storage:
# add email to quarantine # add email to quarantine
self.logger.info("{}: adding to quarantine '{}' for: {}".format( self.logger.info("{}: adding to quarantine '{}' for: {}".format(
self.queueid, quarantine["name"], ", ".join(recipients))) self.queueid, quarantine.name, ", ".join(recipients)))
try: try:
quarantine_id = quarantine["quarantine_obj"].add( storage_id = storage.add(
self.queueid, self.mailfrom, recipients, headers, self.fp, self.queueid, self.mailfrom, recipients, headers, self.fp,
subgroups, named_subgroups) subgroups, named_subgroups)
except RuntimeError as e: except RuntimeError as e:
self.logger.error( self.logger.error(
"{}: unable to add to quarantine '{}': {}".format( "{}: unable to add to quarantine '{}': {}".format(
self.queueid, quarantine["name"], e)) self.queueid, quarantine.name, e))
return Milter.TEMPFAIL return Milter.TEMPFAIL
# check if a notification is configured # check if a notification is configured
if quarantine["notification_obj"] is not None: notification = quarantine.get_notification()
if notification:
# notify # notify
self.logger.info( self.logger.info(
"{}: sending notification for quarantine '{}' to: {}".format( "{}: sending notification for quarantine '{}' to: {}".format(
self.queueid, quarantine["name"], ", ".join(recipients))) self.queueid, quarantine.name, ", ".join(recipients)))
try: try:
quarantine["notification_obj"].notify( notification.notify(
self.queueid, quarantine_id, self.queueid, storage_id,
self.mailfrom, recipients, headers, self.fp, self.mailfrom, recipients, headers, self.fp,
subgroups, named_subgroups) subgroups, named_subgroups)
except RuntimeError as e: except RuntimeError as e:
self.logger.error( self.logger.error(
"{}: unable to send notification for quarantine '{}': {}".format( "{}: unable to send notification for quarantine '{}': {}".format(
self.queueid, quarantine["name"], e)) self.queueid, quarantine.name, e))
return Milter.TEMPFAIL return Milter.TEMPFAIL
# remove processed recipient # remove processed recipient
@@ -365,12 +585,12 @@ class QuarantineMilter(Milter.Base):
self.logger.info( self.logger.info(
"{}: {} matching quarantine is '{}', performing milter action {}".format( "{}: {} matching quarantine is '{}', performing milter action {}".format(
self.queueid, self.queueid,
self.global_config["preferred_quarantine_action"], self.preferred_quarantine_action,
quarantine["name"], quarantine.name,
quarantine["action"].upper())) quarantine.action))
if quarantine["action"] == "reject": if quarantine.action == "REJECT":
self.setreply("554", "5.7.0", quarantine["reject_reason"]) self.setreply("554", "5.7.0", quarantine.reject_reason)
return quarantine["milter_action"] return quarantine.milter_action
except Exception as e: except Exception as e:
self.logger.exception( self.logger.exception(
@@ -381,23 +601,24 @@ class QuarantineMilter(Milter.Base):
self.logger.debug( self.logger.debug(
"disconnect from {} port {}".format( "disconnect from {} port {}".format(
*self.hostaddr)) *self.hostaddr))
return Milter.CONTINUE
def generate_milter_config(configtest=False, config_files=[]): def setup_milter(test=False, cfg_files=[]):
"Generate the configuration for QuarantineMilter class." "Generate the configuration for QuarantineMilter class."
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# read config file # read config file
parser = configparser.ConfigParser() parser = configparser.ConfigParser()
if not config_files: if not cfg_files:
config_files = parser.read(QuarantineMilter.get_configfiles()) cfg_files = parser.read(QuarantineMilter.get_cfg_files())
else: else:
config_files = parser.read(config_files) cfg_files = parser.read(cfg_files)
if not config_files: if not cfg_files:
raise RuntimeError("config file not found") raise RuntimeError("config file not found")
QuarantineMilter.set_configfiles(config_files) QuarantineMilter.set_cfg_files(cfg_files)
os.chdir(os.path.dirname(config_files[0])) os.chdir(os.path.dirname(cfg_files[0]))
# check if mandatory config options in global section are present # check if mandatory config options in global section are present
if "global" not in parser.sections(): if "global" not in parser.sections():
@@ -409,164 +630,41 @@ def generate_milter_config(configtest=False, config_files=[]):
"mandatory option '{}' not present in config section 'global'".format(option)) "mandatory option '{}' not present in config section 'global'".format(option))
# read global config section # read global config section
global_config = dict(parser.items("global")) global_cfg = dict(parser.items("global"))
global_config["preferred_quarantine_action"] = global_config["preferred_quarantine_action"].lower() preferred_quarantine_action = global_cfg["preferred_quarantine_action"].lower()
if global_config["preferred_quarantine_action"] not in ["first", "last"]: if preferred_quarantine_action not in ["first", "last"]:
raise RuntimeError( raise RuntimeError(
"option preferred_quarantine_action has illegal value") "option preferred_quarantine_action has illegal value")
# read active quarantine names # read active quarantine names
quarantine_names = [ quarantines = [
q.strip() for q in global_config["quarantines"].split(",")] q.strip() for q in global_cfg["quarantines"].split(",")]
if len(quarantine_names) != len(set(quarantine_names)): if len(quarantines) != len(set(quarantines)):
raise RuntimeError( raise RuntimeError(
"at least one quarantine is specified multiple times in quarantines option") "at least one quarantine is specified multiple times in quarantines option")
if "global" in quarantine_names: if "global" in quarantines:
quarantine_names.remove("global") quarantines.remove("global")
logger.warning( logger.warning(
"removed illegal quarantine name 'global' from list of active quarantines") "removed illegal quarantine name 'global' from list of active quarantines")
if not quarantine_names: if not quarantines:
raise RuntimeError("no quarantines configured") raise RuntimeError("no quarantines configured")
milter_config = [] milter_quarantines = []
logger.debug("preparing milter configuration ...") logger.debug("preparing milter configuration ...")
# iterate quarantine names # iterate quarantine names
for index, quarantine_name in enumerate(quarantine_names): for index, name in enumerate(quarantines):
# check if config section for current quarantine exists # check if config section for current quarantine exists
if quarantine_name not in parser.sections(): if name not in parser.sections():
raise RuntimeError( raise RuntimeError(
"config section '{}' does not exist".format(quarantine_name)) "config section '{}' does not exist".format(name))
config = dict(parser.items(quarantine_name))
# check if mandatory config options are present in config cfg = dict(parser.items(name))
for option in ["regex", "quarantine_type", "notification_type", quarantine = Quarantine(name, index)
"action", "whitelist_type", "smtp_host", "smtp_port"]: quarantine.setup_from_cfg(global_cfg, cfg, test)
if option not in config.keys() and \ milter_quarantines.append(quarantine)
option in global_config.keys():
config[option] = global_config[option]
if option not in config.keys():
raise RuntimeError(
"mandatory option '{}' not present in config section '{}' or 'global'".format(
option, quarantine_name))
# check if optional config options are present in config QuarantineMilter.preferred_quarantine_action = preferred_quarantine_action
defaults = { QuarantineMilter.quarantines = milter_quarantines
"reject_reason": "Message rejected",
"ignore_hosts": ""
}
for option in defaults.keys():
if option not in config.keys() and \
option in global_config.keys():
config[option] = global_config[option]
if option not in config.keys():
config[option] = defaults[option]
# set quarantine name
config["name"] = quarantine_name
# set the index
config["index"] = index
# pre-compile regex
logger.debug(
"{}: compiling regex '{}'".format(
quarantine_name,
config["regex"]))
config["regex_compiled"] = re.compile(
config["regex"], re.MULTILINE + re.DOTALL + re.IGNORECASE)
# create quarantine instance
quarantine_type = config["quarantine_type"].lower()
if quarantine_type in quarantines.TYPES.keys():
logger.debug(
"{}: initializing quarantine type '{}'".format(
quarantine_name,
quarantine_type.upper()))
quarantine = quarantines.TYPES[quarantine_type](
global_config, config, configtest)
elif quarantine_type == "none":
logger.debug("{}: quarantine is NONE".format(quarantine_name))
quarantine = None
else:
raise RuntimeError(
"{}: unknown quarantine type '{}'".format(
quarantine_name, quarantine_type))
config["quarantine_obj"] = quarantine
# create whitelist instance
whitelist_type = config["whitelist_type"].lower()
if whitelist_type in whitelists.TYPES.keys():
logger.debug(
"{}: initializing whitelist type '{}'".format(
quarantine_name,
whitelist_type.upper()))
whitelist = whitelists.TYPES[whitelist_type](
global_config, config, configtest)
elif whitelist_type == "none":
logger.debug("{}: whitelist is NONE".format(quarantine_name))
whitelist = None
else:
raise RuntimeError(
"{}: unknown whitelist type '{}'".format(
quarantine_name, whitelist_type))
config["whitelist_obj"] = whitelist
# create notification instance
notification_type = config["notification_type"].lower()
if notification_type in notifications.TYPES.keys():
logger.debug(
"{}: initializing notification type '{}'".format(
quarantine_name,
notification_type.upper()))
notification = notifications.TYPES[notification_type](
global_config, config, configtest)
elif notification_type == "none":
logger.debug("{}: notification is NONE".format(quarantine_name))
notification = None
else:
raise RuntimeError(
"{}: unknown notification type '{}'".format(
quarantine_name, notification_type))
config["notification_obj"] = notification
# determining milter action for this quarantine
action = config["action"].upper()
if action in QuarantineMilter.get_actions().keys():
logger.debug("{}: action is {}".format(quarantine_name, action))
config["milter_action"] = QuarantineMilter.get_actions()[action]
else:
raise RuntimeError(
"{}: unknown action '{}'".format(
quarantine_name, action))
# create host/network whitelist
config["ignore_hosts_list"] = []
ignored = set([p.strip()
for p in config["ignore_hosts"].split(",") if p])
for ignore in ignored:
if not ignore:
continue
# parse network notation
try:
net = IPNetwork(ignore)
except AddrFormatError as e:
raise RuntimeError("error parsing ignore_hosts: {}".format(e))
else:
config["ignore_hosts_list"].append(net)
if config["ignore_hosts_list"]:
logger.debug(
"{}: ignore hosts: {}".format(
quarantine_name,
", ".join(ignored)))
milter_config.append(config)
return global_config, milter_config
def reload_config(): def reload_config():
@@ -574,11 +672,9 @@ def reload_config():
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
try: try:
global_config, config = generate_milter_config() setup_milter()
except RuntimeError as e: except RuntimeError as e:
logger.info(e) logger.info(e)
logger.info("daemon is still running with previous configuration") logger.info("daemon is still running with previous configuration")
else: else:
logger.info("reloading configuration") logger.info("reloaded configuration")
QuarantineMilter.global_config = global_config
QuarantineMilter.config = config

179
pyquarantine/cli.py
View File

@@ -26,23 +26,36 @@ import pyquarantine
from pyquarantine.version import __version__ as version from pyquarantine.version import __version__ as version
def _get_quarantine_obj(config, quarantine): def _get_quarantine(quarantines, name):
try: try:
quarantine_obj = next((q["quarantine_obj"] quarantine = next((q for q in quarantines if q.name == name))
for q in config if q["name"] == quarantine))
except StopIteration: except StopIteration:
raise RuntimeError("invalid quarantine '{}'".format(quarantine)) raise RuntimeError("invalid quarantine '{}'".format(name))
return quarantine_obj return quarantine
def _get_storage(quarantines, name):
quarantine = _get_quarantine(quarantines, name)
storage = quarantine.get_storage()
if not storage:
raise RuntimeError(
"storage type is set to NONE")
return storage
def _get_whitelist_obj(config, quarantine): def _get_notification(quarantines, name):
try: quarantine = _get_quarantine(quarantines, name)
whitelist_obj = next((q["whitelist_obj"] notification = quarantine.get_notification()
for q in config if q["name"] == quarantine)) if not notification:
except StopIteration: raise RuntimeError(
raise RuntimeError("invalid quarantine '{}'".format(quarantine)) "notification type is set to NONE")
return whitelist_obj return notification
def _get_whitelist(quarantines, name):
quarantine = _get_quarantine(quarantines, name)
whitelist = quarantine.get_whitelist()
if not whitelist:
raise RuntimeError(
"whitelist type is set to NONE")
return whitelist
def print_table(columns, rows): def print_table(columns, rows):
if not rows: if not rows:
@@ -85,51 +98,72 @@ def print_table(columns, rows):
print(row_format.format(*row)) print(row_format.format(*row))
def list_quarantines(config, args): def list_quarantines(quarantines, args):
if args.batch: if args.batch:
print("\n".join([quarantine["name"] for quarantine in config])) print("\n".join([q.name for q in quarantines]))
else: else:
qlist = []
for q in quarantines:
storage = q.get_storage()
if storage:
storage_type = q.get_storage().storage_type
else:
storage_type = "NONE"
notification = q.get_notification()
if notification:
notification_type = q.get_notification().notification_type
else:
notification_type = "NONE"
whitelist = q.get_whitelist()
if whitelist:
whitelist_type = q.get_whitelist().whitelist_type
else:
whitelist_type = "NONE"
qlist.append({
"name": q.name,
"storage": storage_type,
"notification": notification_type,
"whitelist": whitelist_type,
"action": q.action})
print_table( print_table(
[("Name", "name"), ("Quarantine", "quarantine_type"), [("Name", "name"),
("Notification", "notification_type"), ("Action", "action")], ("Storage", "storage"),
config ("Notification", "notification"),
("Whitelist", "whitelist"),
("Action", "action")],
qlist
) )
def list_quarantine_emails(config, args): def list_quarantine_emails(quarantines, args):
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
storage = _get_storage(quarantines, args.quarantine)
# get quarantine object
quarantine = _get_quarantine_obj(config, args.quarantine)
if quarantine is None:
raise RuntimeError(
"quarantine type is set to None, unable to list emails")
# find emails and transform some metadata values to strings # find emails and transform some metadata values to strings
rows = [] rows = []
emails = quarantine.find( emails = storage.find(
mailfrom=args.mailfrom, args.mailfrom, args.recipients, args.older_than)
recipients=args.recipients, for storage_id, metadata in emails.items():
older_than=args.older_than) row = emails[storage_id]
for quarantine_id, metadata in emails.items(): row["storage_id"] = storage_id
row = emails[quarantine_id]
row["quarantine_id"] = quarantine_id
row["date"] = time.strftime( row["date"] = time.strftime(
'%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M:%S',
time.localtime( time.localtime(
metadata["date"])) metadata["date"]))
row["mailfrom"] = metadata["mailfrom"] row["mailfrom"] = metadata["mailfrom"]
row["recipient"] = metadata["recipients"].pop(0) row["recipient"] = metadata["recipients"].pop(0)
if "subject" not in emails[quarantine_id]["headers"].keys(): if "subject" not in emails[storage_id]["headers"].keys():
emails[quarantine_id]["headers"]["subject"] = "" emails[storage_id]["headers"]["subject"] = ""
row["subject"] = str(make_header(decode_header( row["subject"] = str(make_header(decode_header(
emails[quarantine_id]["headers"]["subject"])))[:60].replace( emails[storage_id]["headers"]["subject"])))[:60].replace(
"\r", "").replace("\n", "").strip() "\r", "").replace("\n", "").strip()
rows.append(row) rows.append(row)
if metadata["recipients"]: if metadata["recipients"]:
row = { row = {
"quarantine_id": "", "storage_id": "",
"date": "", "date": "",
"mailfrom": "", "mailfrom": "",
"recipient": metadata["recipients"].pop(0), "recipient": metadata["recipients"].pop(0),
@@ -145,21 +179,16 @@ def list_quarantine_emails(config, args):
if not emails: if not emails:
logger.info("quarantine '{}' is empty".format(args.quarantine)) logger.info("quarantine '{}' is empty".format(args.quarantine))
print_table( print_table(
[("Quarantine-ID", "quarantine_id"), ("Date", "date"), [("Quarantine-ID", "storage_id"), ("Date", "date"),
("From", "mailfrom"), ("Recipient(s)", "recipient"), ("From", "mailfrom"), ("Recipient(s)", "recipient"),
("Subject", "subject")], ("Subject", "subject")],
rows rows
) )
def list_whitelist(config, args): def list_whitelist(quarantines, args):
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
whitelist = _get_whitelist(quarantines, args.quarantine)
# get whitelist object
whitelist = _get_whitelist_obj(config, args.quarantine)
if whitelist is None:
raise RuntimeError(
"whitelist type is set to None, unable to list entries")
# find whitelist entries # find whitelist entries
entries = whitelist.find( entries = whitelist.find(
@@ -190,14 +219,9 @@ def list_whitelist(config, args):
) )
def add_whitelist_entry(config, args): def add_whitelist_entry(quarantines, args):
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
whitelist = _get_whitelist(quarantines, args.quarantine)
# get whitelist object
whitelist = _get_whitelist_obj(config, args.quarantine)
if whitelist is None:
raise RuntimeError(
"whitelist type is set to None, unable to add entries")
# check existing entries # check existing entries
entries = whitelist.check(args.mailfrom, args.recipient) entries = whitelist.check(args.mailfrom, args.recipient)
@@ -235,50 +259,31 @@ def add_whitelist_entry(config, args):
logger.info("whitelist entry added successfully") logger.info("whitelist entry added successfully")
def delete_whitelist_entry(config, args): def delete_whitelist_entry(quarantines, args):
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
whitelist = _get_whitelist(quarantines, args.quarantine)
whitelist = _get_whitelist_obj(config, args.quarantine)
if whitelist is None:
raise RuntimeError(
"whitelist type is set to None, unable to delete entries")
whitelist.delete(args.whitelist_id) whitelist.delete(args.whitelist_id)
logger.info("whitelist entry deleted successfully") logger.info("whitelist entry deleted successfully")
def notify_email(config, args): def notify(quarantines, args):
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
quarantine = _get_quarantine(quarantines, args.quarantine)
quarantine = _get_quarantine_obj(config, args.quarantine)
if quarantine is None:
raise RuntimeError(
"quarantine type is set to None, unable to send notification")
quarantine.notify(args.quarantine_id, args.recipient) quarantine.notify(args.quarantine_id, args.recipient)
logger.info("sent notification successfully") logger.info("notification sent successfully")
def release_email(config, args): def release(quarantines, args):
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
quarantine = _get_quarantine(quarantines, args.quarantine)
quarantine = _get_quarantine_obj(config, args.quarantine)
if quarantine is None:
raise RuntimeError(
"quarantine type is set to None, unable to release email")
quarantine.release(args.quarantine_id, args.recipient) quarantine.release(args.quarantine_id, args.recipient)
logger.info("quarantined email released successfully") logger.info("quarantined email released successfully")
def delete_email(config, args): def delete(quarantines, args):
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
storage = _get_storage(quarantines, args.quarantine)
quarantine = _get_quarantine_obj(config, args.quarantine) storage.delete(args.quarantine_id, args.recipient)
if quarantine is None:
raise RuntimeError(
"quarantine type is set to None, unable to delete email")
quarantine.delete(args.quarantine_id, args.recipient)
logger.info("quarantined email deleted successfully") logger.info("quarantined email deleted successfully")
@@ -304,7 +309,7 @@ def main():
"-c", "--config", "-c", "--config",
help="Config files to read.", help="Config files to read.",
nargs="+", metavar="CFG", nargs="+", metavar="CFG",
default=pyquarantine.QuarantineMilter.get_configfiles()) default=pyquarantine.QuarantineMilter.get_cfg_files())
parser.add_argument( parser.add_argument(
"-d", "--debug", "-d", "--debug",
help="Log debugging messages.", help="Log debugging messages.",
@@ -394,7 +399,7 @@ def main():
"-a", "--all", "-a", "--all",
help="Release email for all recipients.", help="Release email for all recipients.",
action="store_true") action="store_true")
quarantine_notify_parser.set_defaults(func=notify_email) quarantine_notify_parser.set_defaults(func=notify)
# quarantine release command # quarantine release command
quarantine_release_parser = quarantine_subparsers.add_parser( quarantine_release_parser = quarantine_subparsers.add_parser(
"release", "release",
@@ -421,7 +426,7 @@ def main():
"-a", "--all", "-a", "--all",
help="Release email for all recipients.", help="Release email for all recipients.",
action="store_true") action="store_true")
quarantine_release_parser.set_defaults(func=release_email) quarantine_release_parser.set_defaults(func=release)
# quarantine delete command # quarantine delete command
quarantine_delete_parser = quarantine_subparsers.add_parser( quarantine_delete_parser = quarantine_subparsers.add_parser(
"delete", "delete",
@@ -447,7 +452,7 @@ def main():
"-a", "--all", "-a", "--all",
help="Delete email for all recipients.", help="Delete email for all recipients.",
action="store_true") action="store_true")
quarantine_delete_parser.set_defaults(func=delete_email) quarantine_delete_parser.set_defaults(func=delete)
# whitelist command group # whitelist command group
whitelist_parser = subparsers.add_parser( whitelist_parser = subparsers.add_parser(
@@ -558,8 +563,8 @@ def main():
# try to generate milter configs # try to generate milter configs
try: try:
global_config, config = pyquarantine.generate_milter_config( pyquarantine.setup_milter(
config_files=args.config, configtest=True) cfg_files=args.config, test=True)
except RuntimeError as e: except RuntimeError as e:
logger.error(e) logger.error(e)
sys.exit(255) sys.exit(255)
@@ -580,7 +585,7 @@ def main():
# call the commands function # call the commands function
try: try:
args.func(config, args) args.func(pyquarantine.QuarantineMilter.quarantines, args)
except RuntimeError as e: except RuntimeError as e:
logger.error(e) logger.error(e)
sys.exit(1) sys.exit(1)

76
pyquarantine/notifications.py
View File

@@ -31,14 +31,13 @@ from pyquarantine import mailer
class BaseNotification(object): class BaseNotification(object):
"Notification base class" "Notification base class"
notification_type = "base"
def __init__(self, global_config, config, configtest=False): def __init__(self, name, global_cfg, cfg, test=False):
self.quarantine_name = config["name"] self.name = name
self.global_config = global_config
self.config = config
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
def notify(self, queueid, quarantine_id, mailfrom, recipients, headers, def notify(self, queueid, storage_id, mailfrom, recipients, headers,
fp, subgroups=None, named_subgroups=None, synchronous=False): fp, subgroups=None, named_subgroups=None, synchronous=False):
fp.seek(0) fp.seek(0)
pass pass
@@ -46,6 +45,7 @@ class BaseNotification(object):
class EMailNotification(BaseNotification): class EMailNotification(BaseNotification):
"Notification class to send notifications via mail." "Notification class to send notifications via mail."
notification_type = "email"
_html_text = "text/html" _html_text = "text/html"
_plain_text = "text/plain" _plain_text = "text/plain"
_bad_tags = [ _bad_tags = [
@@ -111,44 +111,40 @@ class EMailNotification(BaseNotification):
"width" "width"
] ]
def __init__(self, global_config, config, configtest=False): def __init__(self, name, global_cfg, cfg, test=False):
super(EMailNotification, self).__init__( super(EMailNotification, self).__init__(
global_config, config, configtest) name, global_cfg, cfg, test)
# check if mandatory options are present in config defaults = {
for option in [ "notification_email_replacement_img": "",
"notification_email_strip_images": "false",
"notification_email_parser_lib": "lxml"
}
# check config
for opt in [
"notification_email_smtp_host", "notification_email_smtp_host",
"notification_email_smtp_port", "notification_email_smtp_port",
"notification_email_envelope_from", "notification_email_envelope_from",
"notification_email_from", "notification_email_from",
"notification_email_subject", "notification_email_subject",
"notification_email_template", "notification_email_template",
"notification_email_embedded_imgs"]: "notification_email_embedded_imgs"] + list(defaults.keys()):
if option not in self.config.keys() and option in self.global_config.keys(): if opt in cfg:
self.config[option] = self.global_config[option] continue
if option not in self.config.keys(): if opt in global_cfg:
cfg[opt] = global_cfg[opt]
elif opt in defaults:
cfg[opt] = defaults[opt]
else:
raise RuntimeError( raise RuntimeError(
"mandatory option '{}' not present in config section '{}' or 'global'".format( "mandatory option '{}' not present in config section '{}' or 'global'".format(
option, self.quarantine_name)) opt, self.name))
# check if optional config options are present in config self.smtp_host = cfg["notification_email_smtp_host"]
defaults = { self.smtp_port = cfg["notification_email_smtp_port"]
"notification_email_replacement_img": "", self.mailfrom = cfg["notification_email_envelope_from"]
"notification_email_strip_images": "false", self.from_header = cfg["notification_email_from"]
"notification_email_parser_lib": "lxml" self.subject = cfg["notification_email_subject"]
}
for option in defaults.keys():
if option not in config.keys() and \
option in global_config.keys():
config[option] = global_config[option]
if option not in config.keys():
config[option] = defaults[option]
self.smtp_host = self.config["notification_email_smtp_host"]
self.smtp_port = self.config["notification_email_smtp_port"]
self.mailfrom = self.config["notification_email_envelope_from"]
self.from_header = self.config["notification_email_from"]
self.subject = self.config["notification_email_subject"]
testvars = defaultdict(str, test="TEST") testvars = defaultdict(str, test="TEST")
@@ -169,14 +165,14 @@ class EMailNotification(BaseNotification):
# read and parse email notification template # read and parse email notification template
try: try:
self.template = open( self.template = open(
self.config["notification_email_template"], "r").read() cfg["notification_email_template"], "r").read()
self.template.format_map(testvars) self.template.format_map(testvars)
except IOError as e: except IOError as e:
raise RuntimeError("error reading template: {}".format(e)) raise RuntimeError("error reading template: {}".format(e))
except ValueError as e: except ValueError as e:
raise RuntimeError("error parsing template: {}".format(e)) raise RuntimeError("error parsing template: {}".format(e))
strip_images = self.config["notification_email_strip_images"].strip().upper() strip_images = cfg["notification_email_strip_images"].strip().upper()
if strip_images in ["TRUE", "ON", "YES"]: if strip_images in ["TRUE", "ON", "YES"]:
self.strip_images = True self.strip_images = True
elif strip_images in ["FALSE", "OFF", "NO"]: elif strip_images in ["FALSE", "OFF", "NO"]:
@@ -184,12 +180,12 @@ class EMailNotification(BaseNotification):
else: else:
raise RuntimeError("error parsing notification_email_strip_images: unknown value") raise RuntimeError("error parsing notification_email_strip_images: unknown value")
self.parser_lib = self.config["notification_email_parser_lib"].strip() self.parser_lib = cfg["notification_email_parser_lib"].strip()
if self.parser_lib not in ["lxml", "html.parser"]: if self.parser_lib not in ["lxml", "html.parser"]:
raise RuntimeError("error parsing notification_email_parser_lib: unknown value") raise RuntimeError("error parsing notification_email_parser_lib: unknown value")
# read email replacement image if specified # read email replacement image if specified
replacement_img = self.config["notification_email_replacement_img"].strip() replacement_img = cfg["notification_email_replacement_img"].strip()
if not self.strip_images and replacement_img: if not self.strip_images and replacement_img:
try: try:
self.replacement_img = MIMEImage( self.replacement_img = MIMEImage(
@@ -205,7 +201,7 @@ class EMailNotification(BaseNotification):
# read images to embed if specified # read images to embed if specified
embedded_img_paths = [ embedded_img_paths = [
p.strip() for p in self.config["notification_email_embedded_imgs"].split(",") if p] p.strip() for p in cfg["notification_email_embedded_imgs"].split(",") if p]
self.embedded_imgs = [] self.embedded_imgs = []
for img_path in embedded_img_paths: for img_path in embedded_img_paths:
# read image # read image
@@ -291,14 +287,14 @@ class EMailNotification(BaseNotification):
del(element.attrs[attribute]) del(element.attrs[attribute])
return soup return soup
def notify(self, queueid, quarantine_id, mailfrom, recipients, headers, fp, def notify(self, queueid, storage_id, mailfrom, recipients, headers, fp,
subgroups=None, named_subgroups=None, synchronous=False): subgroups=None, named_subgroups=None, synchronous=False):
"Notify recipients via email." "Notify recipients via email."
super( super(
EMailNotification, EMailNotification,
self).notify( self).notify(
queueid, queueid,
quarantine_id, storage_id,
mailfrom, mailfrom,
recipients, recipients,
headers, headers,
@@ -372,7 +368,7 @@ class EMailNotification(BaseNotification):
EMAIL_ENVELOPE_TO=escape(recipient), EMAIL_ENVELOPE_TO=escape(recipient),
EMAIL_ENVELOPE_TO_URL=escape(quote(recipient)), EMAIL_ENVELOPE_TO_URL=escape(quote(recipient)),
EMAIL_SUBJECT=escape(decoded_headers["subject"]), EMAIL_SUBJECT=escape(decoded_headers["subject"]),
EMAIL_QUARANTINE_ID=quarantine_id) EMAIL_QUARANTINE_ID=storage_id)
if subgroups: if subgroups:
number = 0 number = 0

203
pyquarantine/quarantines.py
View File

@@ -22,63 +22,60 @@ from glob import glob
from shutil import copyfileobj from shutil import copyfileobj
from time import gmtime from time import gmtime
from pyquarantine import mailer
class BaseMailStorage(object):
"Mail storage base class"
storage_type = "base"
class BaseQuarantine(object): def __init__(self, name, global_cfg, cfg, test=False):
"Quarantine base class" self.name = name
def __init__(self, global_config, config, configtest=False):
self.name = config["name"]
self.global_config = global_config
self.config = config
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
def add(self, queueid, mailfrom, recipients, headers, def add(self, queueid, mailfrom, recipients, headers,
fp, subgroups=None, named_subgroups=None): fp, subgroups=None, named_subgroups=None):
"Add email to quarantine." "Add email to storage."
fp.seek(0) fp.seek(0)
return "" return ""
def find(self, mailfrom=None, recipients=None, older_than=None): def find(self, mailfrom=None, recipients=None, older_than=None):
"Find emails in quarantine." "Find emails in storage."
return return
def get_metadata(self, quarantine_id): def get_metadata(self, storage_id):
"Return metadata of quarantined email." "Return metadata of email in storage."
return return
def delete(self, quarantine_id, recipient=None): def delete(self, storage_id, recipients=None):
"Delete email from quarantine." "Delete email from storage."
return return
def notify(self, quarantine_id, recipient=None): def get_mail(self, storage_id):
"Notify recipient about email in quarantine." "Return a file pointer to the email and metadata."
if not self.config["notification_obj"]:
raise RuntimeError(
"notification type is set to None, unable to send notifications")
return
def release(self, quarantine_id, recipient=None):
"Release email from quarantine."
return return
class FileQuarantine(BaseQuarantine): class FileMailStorage(BaseMailStorage):
"Quarantine class to store mails on filesystem." "Storage class to store mails on filesystem."
storage_type = "file"
def __init__(self, global_config, config, configtest=False): def __init__(self, name, global_cfg, cfg, test=False):
super(FileQuarantine, self).__init__(global_config, config, configtest) super(FileMailStorage, self).__init__(name, global_cfg, cfg, test)
# check if mandatory options are present in config defaults = {}
for option in ["quarantine_directory"]: # check config
if option not in self.config.keys() and option in self.global_config.keys():
self.config[option] = self.global_config[option] for opt in ["storage_directory"] + list(defaults.keys()):
if option not in self.config.keys(): if opt in cfg:
continue
if opt in global_cfg:
cfg[opt] = global_cfg[opt]
elif opt in defaults:
cfg[opt] = defaults[opt]
else:
raise RuntimeError( raise RuntimeError(
"mandatory option '{}' not present in config section '{}' or 'global'".format( "mandatory option '{}' not present in config section '{}' or 'global'".format(
option, self.name)) opt, self.name))
self.directory = self.config["quarantine_directory"] self.directory = cfg["storage_directory"]
# check if quarantine directory exists and is writable # check if quarantine directory exists and is writable
if not os.path.isdir(self.directory) or not os.access( if not os.path.isdir(self.directory) or not os.access(
@@ -88,26 +85,26 @@ class FileQuarantine(BaseQuarantine):
self.directory)) self.directory))
self._metadata_suffix = ".metadata" self._metadata_suffix = ".metadata"
def _save_datafile(self, quarantine_id, fp): def _save_datafile(self, storage_id, fp):
datafile = os.path.join(self.directory, quarantine_id) datafile = os.path.join(self.directory, storage_id)
try: try:
with open(datafile, "wb") as f: with open(datafile, "wb") as f:
copyfileobj(fp, f) copyfileobj(fp, f)
except IOError as e: except IOError as e:
raise RuntimeError("unable save data file: {}".format(e)) raise RuntimeError("unable save data file: {}".format(e))
def _save_metafile(self, quarantine_id, metadata): def _save_metafile(self, storage_id, metadata):
metafile = os.path.join( metafile = os.path.join(
self.directory, "{}{}".format( self.directory, "{}{}".format(
quarantine_id, self._metadata_suffix)) storage_id, self._metadata_suffix))
try: try:
with open(metafile, "w") as f: with open(metafile, "w") as f:
json.dump(metadata, f, indent=2) json.dump(metadata, f, indent=2)
except IOError as e: except IOError as e:
raise RuntimeError("unable to save metadata file: {}".format(e)) raise RuntimeError("unable to save metadata file: {}".format(e))
def _remove(self, quarantine_id): def _remove(self, storage_id):
datafile = os.path.join(self.directory, quarantine_id) datafile = os.path.join(self.directory, storage_id)
metafile = "{}{}".format(datafile, self._metadata_suffix) metafile = "{}{}".format(datafile, self._metadata_suffix)
try: try:
@@ -122,9 +119,9 @@ class FileQuarantine(BaseQuarantine):
def add(self, queueid, mailfrom, recipients, headers, def add(self, queueid, mailfrom, recipients, headers,
fp, subgroups=None, named_subgroups=None): fp, subgroups=None, named_subgroups=None):
"Add email to file quarantine and return quarantine-id." "Add email to file storage and return storage id."
super( super(
FileQuarantine, FileMailStorage,
self).add( self).add(
queueid, queueid,
mailfrom, mailfrom,
@@ -133,11 +130,11 @@ class FileQuarantine(BaseQuarantine):
fp, fp,
subgroups, subgroups,
named_subgroups) named_subgroups)
quarantine_id = "{}_{}".format( storage_id = "{}_{}".format(
datetime.now().strftime("%Y%m%d%H%M%S"), queueid) datetime.now().strftime("%Y%m%d%H%M%S"), queueid)
# save mail # save mail
self._save_datafile(quarantine_id, fp) self._save_datafile(storage_id, fp)
# save metadata # save metadata
metadata = { metadata = {
@@ -150,24 +147,24 @@ class FileQuarantine(BaseQuarantine):
"named_subgroups": named_subgroups "named_subgroups": named_subgroups
} }
try: try:
self._save_metafile(quarantine_id, metadata) self._save_metafile(storage_id, metadata)
except RuntimeError as e: except RuntimeError as e:
datafile = os.path.join(self.directory, quarantine_id) datafile = os.path.join(self.directory, storage_id)
os.remove(datafile) os.remove(datafile)
raise e raise e
return quarantine_id return storage_id
def get_metadata(self, quarantine_id): def get_metadata(self, storage_id):
"Return metadata of quarantined email." "Return metadata of email in storage."
super(FileQuarantine, self).get_metadata(quarantine_id) super(FileMailStorage, self).get_metadata(storage_id)
metafile = os.path.join( metafile = os.path.join(
self.directory, "{}{}".format( self.directory, "{}{}".format(
quarantine_id, self._metadata_suffix)) storage_id, self._metadata_suffix))
if not os.path.isfile(metafile): if not os.path.isfile(metafile):
raise RuntimeError( raise RuntimeError(
"invalid quarantine id '{}'".format(quarantine_id)) "invalid storage id '{}'".format(storage_id))
try: try:
with open(metafile, "r") as f: with open(metafile, "r") as f:
@@ -182,8 +179,8 @@ class FileQuarantine(BaseQuarantine):
return metadata return metadata
def find(self, mailfrom=None, recipients=None, older_than=None): def find(self, mailfrom=None, recipients=None, older_than=None):
"Find emails in quarantine." "Find emails in storage."
super(FileQuarantine, self).find(mailfrom, recipients, older_than) super(FileMailStorage, self).find(mailfrom, recipients, older_than)
if isinstance(mailfrom, str): if isinstance(mailfrom, str):
mailfrom = [mailfrom] mailfrom = [mailfrom]
if isinstance(recipients, str): if isinstance(recipients, str):
@@ -196,9 +193,9 @@ class FileQuarantine(BaseQuarantine):
if not os.path.isfile(metafile): if not os.path.isfile(metafile):
continue continue
quarantine_id = os.path.basename( storage_id = os.path.basename(
metafile[:-len(self._metadata_suffix)]) metafile[:-len(self._metadata_suffix)])
metadata = self.get_metadata(quarantine_id) metadata = self.get_metadata(storage_id)
if older_than is not None: if older_than is not None:
if timegm(gmtime()) - metadata["date"] < (older_than * 86400): if timegm(gmtime()) - metadata["date"] < (older_than * 86400):
continue continue
@@ -214,96 +211,44 @@ class FileQuarantine(BaseQuarantine):
elif len(set(recipients + metadata["recipients"])) == len(recipients + metadata["recipients"]): elif len(set(recipients + metadata["recipients"])) == len(recipients + metadata["recipients"]):
continue continue
emails[quarantine_id] = metadata emails[storage_id] = metadata
return emails return emails
def delete(self, quarantine_id, recipient=None): def delete(self, storage_id, recipients=None):
"Delete email in quarantine." "Delete email from storage."
super(FileQuarantine, self).delete(quarantine_id, recipient) super(FileMailStorage, self).delete(storage_id, recipients)
try: try:
metadata = self.get_metadata(quarantine_id) metadata = self.get_metadata(storage_id)
except RuntimeError as e: except RuntimeError as e:
raise RuntimeError("unable to delete email: {}".format(e)) raise RuntimeError("unable to delete email: {}".format(e))
if recipient is None: if not recipients:
self._remove(quarantine_id) self._remove(storage_id)
else: else:
if type(recipients) == str:
recipients = [recipients]
for recipient in recipients:
if recipient not in metadata["recipients"]: if recipient not in metadata["recipients"]:
raise RuntimeError("invalid recipient '{}'".format(recipient)) raise RuntimeError("invalid recipient '{}'".format(recipient))
metadata["recipients"].remove(recipient) metadata["recipients"].remove(recipient)
if not metadata["recipients"]: if not metadata["recipients"]:
self._remove(quarantine_id) self._remove(storage_id)
else: else:
self._save_metafile(quarantine_id, metadata) self._save_metafile(storage_id, metadata)
def notify(self, quarantine_id, recipient=None): def get_mail(self, storage_id):
"Notify recipient about email in quarantine." super(FileMailStorage, self).get_mail(storage_id)
super(FileQuarantine, self).notify(quarantine_id, recipient)
metadata = self.get_metadata(storage_id)
datafile = os.path.join(self.directory, storage_id)
try: try:
metadata = self.get_metadata(quarantine_id) fp = open(datafile, "rb")
except RuntimeError as e:
raise RuntimeError("unable to release email: {}".format(e))
if recipient is not None:
if recipient not in metadata["recipients"]:
raise RuntimeError("invalid recipient '{}'".format(recipient))
recipients = [recipient]
else:
recipients = metadata["recipients"]
datafile = os.path.join(self.directory, quarantine_id)
try:
with open(datafile, "rb") as fp:
self.config["notification_obj"].notify(
metadata["queue_id"], quarantine_id, metadata["mailfrom"],
recipients, metadata["headers"], fp,
metadata["subgroups"], metadata["named_subgroups"],
synchronous=True)
except IOError as e: except IOError as e:
raise RuntimeError raise RuntimeError("unable to open email data file: {}".format(e))
return (fp, metadata)
def release(self, quarantine_id, recipient=None):
"Release email from quarantine."
super(FileQuarantine, self).release(quarantine_id, recipient)
try:
metadata = self.get_metadata(quarantine_id)
except RuntimeError as e:
raise RuntimeError("unable to release email: {}".format(e))
if recipient is not None:
if recipient not in metadata["recipients"]:
raise RuntimeError("invalid recipient '{}'".format(recipient))
recipients = [recipient]
else:
recipients = metadata["recipients"]
datafile = os.path.join(self.directory, quarantine_id)
try:
with open(datafile, "rb") as f:
mail = f.read()
except IOError as e:
raise RuntimeError("unable to read data file: {}".format(e))
for recipient in recipients:
try:
mailer.smtp_send(
self.config["smtp_host"],
self.config["smtp_port"],
metadata["mailfrom"],
recipient,
mail)
except Exception as e:
raise RuntimeError(
"error while sending email to '{}': {}".format(
recipient, e))
self.delete(quarantine_id, recipient)
# list of quarantine types and their related quarantine classes # list of storage types and their related storage classes
TYPES = {"file": FileQuarantine} TYPES = {"file": FileMailStorage}

15
pyquarantine/run.py
View File

@@ -35,11 +35,11 @@ def main():
"-c", "--config", "-c", "--config",
help="List of config files to read.", help="List of config files to read.",
nargs="+", nargs="+",
default=pyquarantine.QuarantineMilter.get_configfiles()) default=pyquarantine.QuarantineMilter.get_cfg_files())
parser.add_argument( parser.add_argument(
"-s", "--socket", "-s", "--socket",
help="Socket used to communicate with the MTA.", help="Socket used to communicate with the MTA.",
required=True) default="inet:8899@127.0.0.1")
parser.add_argument( parser.add_argument(
"-d", "--debug", "-d", "--debug",
help="Log debugging messages.", help="Log debugging messages.",
@@ -65,7 +65,7 @@ def main():
syslog_name = "{}: [%(name)s] %(levelname)s".format(syslog_name) syslog_name = "{}: [%(name)s] %(levelname)s".format(syslog_name)
# set config files for milter class # set config files for milter class
pyquarantine.QuarantineMilter.set_configfiles(args.config) pyquarantine.QuarantineMilter.set_cfg_files(args.config)
root_logger = logging.getLogger() root_logger = logging.getLogger()
root_logger.setLevel(loglevel) root_logger.setLevel(loglevel)
@@ -78,7 +78,7 @@ def main():
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
if args.test: if args.test:
try: try:
pyquarantine.generate_milter_config(args.test) pyquarantine.setup_milter(test=args.test)
print("Configuration ok") print("Configuration ok")
except RuntimeError as e: except RuntimeError as e:
logger.error(e) logger.error(e)
@@ -101,18 +101,14 @@ def main():
logger.info("PyQuarantine-Milter starting") logger.info("PyQuarantine-Milter starting")
try: try:
# generate milter config # generate milter config
global_config, config = pyquarantine.generate_milter_config() pyquarantine.setup_milter()
except RuntimeError as e: except RuntimeError as e:
logger.error(e) logger.error(e)
sys.exit(255) sys.exit(255)
pyquarantine.QuarantineMilter.global_config = global_config
pyquarantine.QuarantineMilter.config = config
# register to have the Milter factory create instances of your class: # register to have the Milter factory create instances of your class:
Milter.factory = pyquarantine.QuarantineMilter Milter.factory = pyquarantine.QuarantineMilter
Milter.set_exception_policy(Milter.TEMPFAIL) Milter.set_exception_policy(Milter.TEMPFAIL)
# Milter.set_flags(0) # tell sendmail which features we use
# run milter # run milter
rc = 0 rc = 0
@@ -122,6 +118,7 @@ def main():
except Milter.milter.error as e: except Milter.milter.error as e:
logger.error(e) logger.error(e)
rc = 255 rc = 255
pyquarantine.mailer.queue.put(None) pyquarantine.mailer.queue.put(None)
logger.info("PyQuarantine-Milter terminated") logger.info("PyQuarantine-Milter terminated")
sys.exit(rc) sys.exit(rc)

43
pyquarantine/whitelists.py
View File

@@ -24,11 +24,11 @@ from playhouse.db_url import connect
class WhitelistBase(object): class WhitelistBase(object):
"Whitelist base class" "Whitelist base class"
def __init__(self, global_config, config, configtest=False): whitelist_type = "base"
self.global_config = global_config
self.config = config def __init__(self, name, global_cfg, cfg, test=False):
self.configtest = configtest self.name = name
self.name = config["name"] self.test = test
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
self.valid_entry_regex = re.compile( self.valid_entry_regex = re.compile(
r"^[a-zA-Z0-9_.+-]*?(@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)?$") r"^[a-zA-Z0-9_.+-]*?(@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)?$")
@@ -73,28 +73,35 @@ class Meta(object):
class DatabaseWhitelist(WhitelistBase): class DatabaseWhitelist(WhitelistBase):
"Whitelist class to store whitelist in a database" "Whitelist class to store whitelist in a database"
whitelist_type = "db"
_db_connections = {} _db_connections = {}
_db_tables = {} _db_tables = {}
def __init__(self, global_config, config, configtest=False): def __init__(self, name, global_cfg, cfg, test=False):
super( super(
DatabaseWhitelist, DatabaseWhitelist,
self).__init__( self).__init__(
global_config, global_cfg,
config, cfg,
configtest) test)
# check if mandatory options are present in config defaults = {}
for option in ["whitelist_db_connection", "whitelist_db_table"]:
if option not in self.config.keys() and option in self.global_config.keys(): # check config
self.config[option] = self.global_config[option] for opt in ["whitelist_db_connection", "whitelist_db_table"] + list(defaults.keys()):
if option not in self.config.keys(): if opt in cfg:
continue
if opt in global_cfg:
cfg[opt] = global_cfg[opt]
elif opt in defaults:
cfg[opt] = defaults[opt]
else:
raise RuntimeError( raise RuntimeError(
"mandatory option '{}' not present in config section '{}' or 'global'".format( "mandatory option '{}' not present in config section '{}' or 'global'".format(
option, self.name)) opt, self.name))
tablename = self.config["whitelist_db_table"] tablename = cfg["whitelist_db_table"]
connection_string = self.config["whitelist_db_connection"] connection_string = cfg["whitelist_db_connection"]
if connection_string in DatabaseWhitelist._db_connections.keys(): if connection_string in DatabaseWhitelist._db_connections.keys():
db = DatabaseWhitelist._db_connections[connection_string] db = DatabaseWhitelist._db_connections[connection_string]
@@ -127,7 +134,7 @@ class DatabaseWhitelist(WhitelistBase):
if tablename not in DatabaseWhitelist._db_tables[connection_string]: if tablename not in DatabaseWhitelist._db_tables[connection_string]:
DatabaseWhitelist._db_tables[connection_string].append(tablename) DatabaseWhitelist._db_tables[connection_string].append(tablename)
if not self.configtest: if not self.test:
try: try:
db.create_tables([self.model]) db.create_tables([self.model])
except Exception as e: except Exception as e: