From 1bcfbb2414db9b2a1388d7cd2b0adcddd6376f88 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 22 Apr 2020 19:50:25 +0200 Subject: [PATCH] switch config to JSON format and new rules/modifications logic --- README.md | 99 +++--- docs/pymodmilter.conf.example | 206 ++++++++----- pymodmilter/__init__.py | 558 ++++++++++++++-------------------- pymodmilter/run.py | 189 ++++++++++++ pymodmilter/version.py | 1 + setup.py | 7 +- test-pymodmilter | 9 + 7 files changed, 619 insertions(+), 450 deletions(-) create mode 100644 pymodmilter/run.py create mode 100644 pymodmilter/version.py create mode 100755 test-pymodmilter diff --git a/README.md b/README.md index acf0810..6b1a9fa 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,82 @@ # pymodmilter A pymilter based sendmail/postfix pre-queue filter with the ability to add, remove and modify e-mail headers. -The project is currently in beta status, but it is already used in a productive enterprise environment which processes about a million e-mails per month. +This project is currently in beta status, but it is already used in a productive enterprise environment which processes about a million e-mails per month. +The basic idea is to define rules with conditions and do modifications make changes when all conditions are met. ## Requirements * pymilter * netaddr ## Configuration -The pymodmilter uses an INI-style configuration file. The sections are described below. +Pymodmilter uses a configuration file in JSON format. The options are described below. Make a copy of the example configuration file in the docs folder to start with. -### Section "global" -Any available configuration option can be set in the global section as default instead of in a rule section. +### global (Object) +The following optional global configuration options are available: +* **local_addrs (Array of Strings)** + A list of hosts and network addresses which are considered local. It is used to for the condition option 'local'. This option may be overriden within a rule object. +* **log (Bool)** + Enable or disable logging. This option may be overriden by a rule or modification object. -The following configuration options are mandatory in the global section: -* **rules** - Comma-separated, ordered list of active rules. For each, there must be a section of the same name in the configuration. +### rules (Array) +A mandatory list of rule objects which are processed in the given order. -### Rule sections +### rule (Object) The following configuration options are mandatory for each rule: -* **action** - Set the action of this rule. Possible values are: - * **add** - * **del** - * **mod** -* **header** - Name of the header in case of adding a header, regular expression to match whole header lines in case of deleting or modifying a header. - -The following configuration options are mandatory for an add-rule: -* **value** - Value of the header. - -The following configuration options are mandatory for a mod-rule: -* **search** - Regular expression to match the value of header lines. You may use subgroups or named subgroups (python syntax) to include parts of the original value in the new value. -* **value** - New value of the header. +* **modifications (Array of Objects)** + A list of modification objects which are processed in the given order. The following configuration options are optional for each rule: -* **ignore_envfrom** - Regular expression to match envelop-from addresses. The rule will be skipped if the expression matches. -* **ignore_hosts** - Comma-separated list of host and network addresses. The rule will be skipped if the sending host is included here. -* **only_hosts** - Comma-separated list of host and network addresses. The rule will be skipped if the sending host is not included here. If a is included in both **ignore_hosts** and **only_hosts**, the rule will be skipped. -* **log** - Enable or disable logging of this rule. Possible values are: - * **true** - * **false** +* **name (String)** + Name of the rule. +* **conditions (Object)** + A list of conditions which all have to be true to process the rule. +* **local_addrs (Array of Strings)** + As described above in the global object section. +* **log (Bool)** + As described above in the global object section. + +### modification (Object) +The following configuration options are mandatory for each modification: +* **type (String)** + Set the modification type. Possible values are: + * **add_header** + * **del_header** + * **mod_header** + +Additional parameters are mandatory based on the modification type. +* **add_header** + * **header (String)** + Name of the header. + * **value (String)** + Value of the header. + +* **del_header** + * **header (String)** + Regular expression to match against header lines. + +* **mod_header** + * **header (String)** + Regular expression to match against header lines. + * **search (String)** + Regular expression to match against the value of header lines. You may use subgroups or named subgroups (python syntax) to include parts of the original value in the new value. + * **value (String)** + New value of the header. + +The following configuration options are optional for each modification: +* **name (String)** + Name of the modification. +* **log (Bool)** + As described above in the global object section. + +### conditions (Object) +The following configuration options are optional: +* **local (Bool)** + If set to true, the rule is only executed for emails originating from addresses defined in local_addrs and vice versa. +* **hosts (Array of Strings)** + A list of hosts and network addresses for which the rule should be executed. +* **envfrom (String)** + A regular expression to match against the evenlope-from addresses for which the rule should be executed. ## Developer information Everyone who wants to improve or extend this project is very welcome. diff --git a/docs/pymodmilter.conf.example b/docs/pymodmilter.conf.example index 3f0a350..a5d3035 100644 --- a/docs/pymodmilter.conf.example +++ b/docs/pymodmilter.conf.example @@ -1,103 +1,143 @@ # This is an example /etc/pymodmilter.conf file. # Copy it into place before use. # -# Comments: use '#' for comment lines and ';' (following a space) for inline comments. +# The file is in JSON format. # -# If an option is not present in a modification section, it will be read from -# the global section. +# The global option 'log' can be overriden per rule or per modification. # +{ + # Section: global + # Notes: Set default options. + # + "global": { + # Option: local_addrs + # Type: List + # Notes: Set a list of local hosts and networks. + # Value: [ LIST ] + # + "local_addrs": ["127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"], + # Option: log + # Type: Bool + # Notes: Set if processing of rules and modifications is logged. + # Value: [ true | false ] + # + "log": true + }, -[global] + # Section: rules + # Notes: Set rules and related modifications. + # + "rules": [ + { + # Option: name + # Type: String + # Notes: Set the name of the rule. + # Value: [ NAME ] + # + "name": "MyRule", -# Option: rules -# Notes: Set active rules (comma-separated). -# Each rule must have a section with the same name below. -# The rule name 'global' is forbidden and will be ignored. -# Rule names must be unique. -# Values: [ ACTIVE ] -# -rules = add_header,del_header,mod_header + # Section: conditions + # Notes: Optionally set conditions to run the rule. + # If multiple conditions are specified, they all + # have to be true to run the rule. + # + "conditions": { + # Option: local + # Type: Bool + # Notes: Set a condition on the senders host address. + # Set to true to execute the rule only for emails originating + # from addresses defined in local_addrs and vice versa. + # Value: [ true | false ] + # + "local": false, -# Option: ignore_envfrom -# Notes: Set a regular expression to match envelope-from addresses to be ignored. -# Value: [ REGEX ] -# -ignore_envfrom = ^.*@localhost$ + # Option: hosts + # Type: String + # Notes: Set a condition on the senders host address. + # The rule will only be executed if the list contains the + # senders host address. + # Value: [ LIST ] + # + "hosts": [ "127.0.0.1" ], -# Option: ignore_hosts -# Notes: Set a list of host and network addresses to be ignored. -# All the common host/network notations are supported, including IPv6. -# Value: [ HOST ] -# -ignore_hosts = 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 + # Option: envfrom + # Type: String + # Notes: Set a regular expression to match against the envelope-from address. + # Value: [ REGEX ] + # + "envfrom": "^(?!.+@mycompany\\.com).+$" + }, -# Option: only_hosts -# Notes: Set a list of host and network addresses. -# All the common host/network notations are supported, including IPv6. -# Value: [ HOST ] -# -only_hosts = 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 + # Section: modifications + # Notes: Set modifications for the rule. + # + "modifications": [ + { + # Option: name + # Type: String + # Notes: Set the name of the modification. + # Value: [ NAME ] + # + "name": "AddHeader", -# Option: log -# Notes: Set if modifications are logged. -# Value: [ true | false ] -# -log = true + # Option: type + # Type: String + # Notes: Set the modification type. + # Value: [ add_header | del_header | mod_header ] + # + "type": "add_header", + # Option: header + # Type: String + # Notes: Set the name of the new header. + # Value: [ NAME ] + # + "header": "X-Test-Header", -[add_header] + # Option: value + # Type: String + # Notes: Set the value of the new header. + # Value: [ VALUE ] + # + "value": "true" + }, { + "name": "ModifyHeader", -# Option: action -# Notes: Set the modification action. -# Values: [ add ] -action = add + "type": "mod_header", -# Option: header -# Notes: Set the name of the header. -# Values: [ NAME ] -# -header = X-My-Own-Header + # Option: header + # Type: String + # Notes: Set a regular expression to match against header lines (e.g. Subject: Test-Subject). + # Value: [ REGEX ] + # + "header": "^Subject:", -# Option: value -# Notes: Set the value of the header. -# Values: [ VALUE ] -value = my own value + # Option: search + # Type: String + # Notes: Set a regular expression to match against the headers value. + # Values: [ VALUE ] + # + "search": "(?P.*)", + # Option: value + # Type: String + # Notes: Set the value of the header. + # Values: [ VALUE ] + "value": "[EXTERNAL] \\g" + }, { + "name": "DeleteHeader", -[del_header] + "type": "del_header", -# Option: action -# Notes: Set the modification action. -# Values: [ del ] -action = del - -# Option: header -# Notes: Set a regular expression to match the header lines to delete. -# Values: [ REGEX ] -# -header = ^Received:.* - - -[mod_header] - -# Option: action -# Notes: Set the modification action. -# Values: [ mod ] -action = mod - -# Option: header -# Notes: Set a regular expression to match the header lines to modify. -# Values: [ REGEX ] -# -header = ^Subject:.* - -# Option: search -# Notes: Set a regular expression to match the headers value. -# Values: [ VALUE ] -search = (?P.*) - -# Option: value -# Notes: Set the value of the header. -# Values: [ VALUE ] -value = [SPAM] \g + # Option: header + # Type: String + # Notes: Set a regular expression to match against header lines (e.g. Subject: Test-Subject). + # Value: [ REGEX ] + # + "header": "^Received:" + } + ] + } + ] +} diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index c46c955..781f79c 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -15,16 +15,16 @@ __all__ = [ "make_header", "replace_illegal_chars", - "HeaderRule", - "HeaderMilter"] + "run", + "version", + "Modification", + "Rule", + "ModifyMilter"] import Milter -import argparse -import configparser import logging import logging.handlers import re -import sys from Milter.utils import parse_addr from email.charset import Charset @@ -61,108 +61,74 @@ def replace_illegal_chars(string): "\n", "") -class HeaderRule: - """HeaderRule to implement a rule to apply on e-mail headers.""" +class Modification: + """Modification to implement a modification to apply on e-mail headers.""" - def __init__(self, name, action, header, search="", value="", - ignore_hosts=[], ignore_envfrom=None, only_hosts=[], - log=True): + types = { + "add_header": ["header", "value"], + "del_header": ["header"], + "mod_header": ["header", "search", "value"] + } + + def __init__(self, name, mod_type, log, **params): self.logger = logging.getLogger(__name__) + self.logger.debug(f"initializing modification '{name}'") self.name = name - self.action = action - self.header = header - self.search = search - self.value = value - self.ignore_hosts = ignore_hosts - self.ignore_envfrom = ignore_envfrom - self.only_hosts = only_hosts self.log = log + # check mod_type + if mod_type not in Modification.types: + raise RuntimeError( + f"{self.name}: invalid modification type '{mod_type}'") + self.mod_type = mod_type + # check if mandatory modification options are present in config + for option in Modification.types[self.mod_type]: + if option not in params: + raise RuntimeError( + f"{self.name}: mandatory config " + f"option '{option}' not found") + if option == "value" and not params["value"]: + raise RuntimeError( + f"{self.name}: empty value specified") - if action in ["del", "mod"]: + if mod_type == "add_header": + self.header = params["header"] + self.value = params["value"] + elif mod_type in ["del_header", "mod_header"]: # compile header regex try: self.header = re.compile( - header, re.MULTILINE + re.DOTALL + re.IGNORECASE) + params["header"], re.MULTILINE + re.DOTALL + re.IGNORECASE) except re.error as e: raise RuntimeError( - f"unable to parse option 'header' of rule '{name}': {e}") + f"{self.name}: unable to parse regular expression of " + f"option 'header': {e}") - if action == "mod": + if mod_type == "mod_header": # compile search regex try: self.search = re.compile( - search, re.MULTILINE + re.DOTALL + re.IGNORECASE) + params["search"], + re.MULTILINE + re.DOTALL + re.IGNORECASE) except re.error as e: raise RuntimeError( - f"unable to parse option 'search' of " - f"rule '{name}': {e}") + f"{self.name}: unable to parse regular expression of " + f"option 'search': {e}") + self.value = params["value"] - if action in ["add", "mod"] and not value: - raise RuntimeError("value of option 'value' is empty") - - # replace strings in ignore_hosts and only_hosts with IPNetwork - # instances - try: - for index, ignore in enumerate(ignore_hosts): - self.ignore_hosts[index] = IPNetwork(ignore) - except AddrFormatError as e: - raise RuntimeError( - f"unable to parse option 'ignore_hosts' of rule '{name}': {e}") - - if self.ignore_envfrom: - try: - self.ignore_envfrom = re.compile(ignore_envfrom, re.IGNORECASE) - except re.error as e: - raise RuntimeError( - f"unable to parse option 'ignore_envfrom' of " - f"rule '{name}': {e}") - - try: - for index, only in enumerate(only_hosts): - self.only_hosts[index] = IPNetwork(only) - except AddrFormatError as e: - raise RuntimeError( - f"unable to parse option 'only_hosts' of rule '{name}': {e}") - - def ignore_host(self, host): - ip = IPAddress(host) - ignore = False - - # check if host matches ignore_hosts - for ignored in self.ignore_hosts: - if ip in ignored: - ignore = True - break - - if not ignore and self.only_hosts: - # host does not match ignore_hosts, check if it matches only_hosts - ignore = True - for only in self.only_hosts: - if ip in only: - ignore = False - break - - if ignore: - self.logger.debug(f"host {host} is ignored by rule {self.name}") - return ignore - - def ignore_from(self, envfrom): - ignore = False - - if self.ignore_envfrom: - if self.ignore_envfrom.search(envfrom): - ignore = True - self.logger.debug( - f"envelope-from {envfrom} is ignored by rule {self.name}") - return ignore - - def execute(self, headers): + def execute(self, qid, headers): """ Execute rule on given headers and return list with modified headers. """ - if self.action == "add": - return [(self.header, self.value, 0, 1)] + if self.mod_type == "add_header": + header = f"{self.header}: {self.value}" + if self.log: + self.logger.info( + f"{qid}: {self.name}: add_header: {header[0:70]}") + else: + self.logger.debug( + f"{qid}: {self.name}: add_header: {header}") + return [(self.mod_type, self.header, self.value, 0, 1)] modified = [] index = 0 @@ -178,32 +144,184 @@ class HeaderRule: occurrences[name] += 1 # check if header line matches regex - if self.header.search(f"{name}: {value}"): - if self.action == "del": + header = f"{name}: {value}" + if self.header.search(header): + if self.mod_type == "del_header": # set an empty value to delete the header new_value = "" + if self.log: + self.logger.info( + f"{qid}: {self.name}: del_header: " + f"{header[0:70]}") + else: + self.logger.debug( + f"{qid}: {self.name}: del_header: " + f"(occ. {occurrences[name]}): {header}") else: + old_header = header new_value = self.search.sub(self.value, value) - if value != new_value: - modified.append( - (name, new_value, index, occurrences[name])) + if value == new_value: + continue + header = f"{name}: {new_value}" + if self.log: + self.logger.info( + f"{qid}: {self.name}: mod_header: " + f"{old_header[0:70]}: {header[0:70]}") + else: + self.logger.debug( + f"{qid}: {self.name}: mod_header: " + f"(occ. {occurrences[name]}): {old_header}: " + f"{header}") + modified.append( + (self.mod_type, name, new_value, index, occurrences[name])) index += 1 return modified -class HeaderMilter(Milter.Base): - """HeaderMilter based on Milter.Base to implement milter communication""" +class Rule: + def __init__(self, name, modifications, local_addrs, log, conditions={}): + self.logger = logging.getLogger(__name__) + self.name = name + self.log = log + + self.logger.debug(f"initializing rule '{self.name}'") + + self._local_addrs = [] + # replace strings in local_addrs list with IPNetwork instances + try: + for addr in local_addrs: + self._local_addrs.append(IPNetwork(addr)) + except AddrFormatError as e: + raise RuntimeError( + f"{self.name}: unable to parse entry of " + f"option local_addrs: {e}") + + self.conditions = {} + for option, value in conditions.items(): + if option == "local": + self.conditions[option] = value + self.logger.debug( + f"{self.name}: added condition: {option} = {value}") + elif option == "hosts": + self.conditions[option] = [] + try: + for host in value: + self.conditions[option].append(IPNetwork(host)) + except AddrFormatError as e: + raise RuntimeError( + f"{self.name}: unable to parse entry of " + f"condition '{option}': {e}") + self.logger.debug( + f"{self.name}: added condition: {option} = {value}") + elif option == "envfrom": + try: + self.conditions[option] = re.compile(value, re.IGNORECASE) + except re.error as e: + raise RuntimeError( + f"{self.name}: unable to parse regular expression of " + f"condition '{option}': {e}") + self.logger.debug( + f"{self.name}: added condition: {option} = {value}") + + self.modifications = [] + for mod_idx, mod in enumerate(modifications): + params = {} + # set default values if not specified in config + if "name" not in mod: + mod["name"] = f"Modification #{mod_idx}" + + if self.name: + params["name"] = f"{self.name}/{mod['name']}" + else: + params["name"] = mod["name"] + + if "log" in mod: + params["log"] = mod["log"] + else: + params["log"] = self.log + + if "type" in mod: + params["mod_type"] = mod["type"] + else: + raise RuntimeError( + f"{params['name']}: mandatory config " + f"option 'type' not found") + + if "header" in mod: + params["header"] = mod["header"] + + if "search" in mod: + params["search"] = mod["search"] + + if "value" in mod: + params["value"] = mod["value"] + + self.modifications.append(Modification(**params)) + self.logger.debug( + f"{self.name}: added modification: {mod['name']}") + + def ignore_host(self, host): + ip = IPAddress(host) + + if "local" in self.conditions: + is_local = False + for addr in self._local_addrs: + if ip in addr: + is_local = True + break + if is_local != self.conditions["local"]: + return True + + if "hosts" in self.conditions: + # check if host is in list + for accepted in self.conditions["hosts"]: + if ip in accepted: + return False + return True + + return False + + def ignore_envfrom(self, envfrom): + if "envfrom" in self.conditions: + if not self.conditions["envfrom"].search(envfrom): + return True + return False + + def execute(self, qid, headers): + changes = [] + if self.log: + self.logger.info(f"{qid}: executing rule '{self.name}'") + else: + self.logger.debug(f"{qid}: executing rule '{self.name}'") + + for mod in self.modifications: + self.logger.debug(f"{qid}: executing modification '{mod.name}'") + result = mod.execute(qid, headers) + changes += result + for mod_type, name, value, index, occurrence in result: + if mod_type == "add_header": + headers.append((name, value)) + else: + if mod_type == "mod_header": + headers[index] = (name, value) + elif mod_type == "del_header": + del headers[index] + return changes + + +class ModifyMilter(Milter.Base): + """ModifyMilter based on Milter.Base to implement milter communication""" _rules = [] @staticmethod def set_rules(rules): - HeaderMilter._rules = rules + ModifyMilter._rules = rules def __init__(self): self.logger = logging.getLogger(__name__) # save rules, it must not change during runtime - self.rules = HeaderMilter._rules.copy() + self.rules = ModifyMilter._rules.copy() def connect(self, IPname, family, hostaddr): self.logger.debug( @@ -214,6 +332,8 @@ class HeaderMilter(Milter.Base): # remove rules which ignore this host for rule in self.rules.copy(): if rule.ignore_host(ip): + self.logger.debug( + f"host {hostaddr[0]} is ignored by rule '{rule.name}'") self.rules.remove(rule) if not self.rules: @@ -226,7 +346,10 @@ class HeaderMilter(Milter.Base): def envfrom(self, mailfrom, *str): mailfrom = "@".join(parse_addr(mailfrom)).lower() for rule in self.rules.copy(): - if rule.ignore_from(mailfrom): + if rule.ignore_envfrom(mailfrom): + self.logger.debug( + f"envelope-from {mailfrom} is ignored by " + f"rule '{rule.name}'") self.rules.remove(rule) if not self.rules: @@ -263,246 +386,21 @@ class HeaderMilter(Milter.Base): def eom(self): try: for rule in self.rules: - self.logger.debug(f"{self.qid}: executing rule '{rule.name}'") - modified = rule.execute(self.headers) - for name, value, index, occurrence in modified: - header = f"{name}: {value}" + changes = rule.execute(self.qid, self.headers) + for mod_type, name, value, index, occurrence in changes: enc_value = replace_illegal_chars( Header(s=value).encode()) - if rule.action == "add": - if rule.log: - self.logger.info( - f"{self.qid}: add: header: " - f"{header[0:70]}") - else: - self.logger.debug( - f"{self.qid}: add: header: " - f"{header}") - self.headers.insert(0, (name, value)) - self.addheader(name, enc_value, 1) + if mod_type == "add_header": + self.logger.debug(f"{self.qid}: milter: adding " + f"header: {name}: {enc_value}") + self.addheader(name, enc_value, -1) else: - if rule.action == "mod": - old_header = "{}: {}".format(*self.headers[index]) - if rule.log: - self.logger.info( - f"{self.qid}: modify: header: " - f"{old_header[0:70]}: {header[0:70]}") - else: - self.logger.debug( - f"{self.qid}: modify: header " - f"(occ. {occurrence}): {old_header}: " - f"{header}") - self.headers[index] = (name, value) - elif rule.action == "del": - if rule.log: - self.logger.info( - f"{self.qid}: delete: header: " - f"{header[0:70]}") - else: - self.logger.debug( - f"{self.qid}: delete: header " - f"(occ. {occurrence}): {header}") - del self.headers[index] - + self.logger.debug(f"{self.qid}: milter: modify " + f"header (occ. {occurrence}): " + f"{name}: {enc_value}") self.chgheader(name, occurrence, enc_value) return Milter.ACCEPT except Exception as e: self.logger.exception( f"an exception occured in eom function: {e}") return Milter.TEMPFAIL - - -def main(): - "Run PyMod-Milter." - # parse command line - parser = argparse.ArgumentParser( - description="PyMod milter daemon", - formatter_class=lambda prog: argparse.HelpFormatter( - prog, max_help_position=45, width=140)) - parser.add_argument( - "-c", "--config", help="Config file to read.", - default="/etc/pymodmilter.conf") - parser.add_argument( - "-s", - "--socket", - help="Socket used to communicate with the MTA.", - required=True) - parser.add_argument( - "-d", - "--debug", - help="Log debugging messages.", - action="store_true") - parser.add_argument( - "-t", - "--test", - help="Check configuration.", - action="store_true") - args = parser.parse_args() - - # setup logging - loglevel = logging.INFO - logname = "pymodmilter" - syslog_name = logname - if args.debug: - loglevel = logging.DEBUG - logname = f"{logname}[%(name)s]" - syslog_name = f"{syslog_name}: [%(name)s] %(levelname)s" - - # set config files for milter class - root_logger = logging.getLogger() - root_logger.setLevel(loglevel) - - # setup console log - stdouthandler = logging.StreamHandler(sys.stdout) - stdouthandler.setLevel(logging.DEBUG) - formatter = logging.Formatter("%(message)s") - stdouthandler.setFormatter(formatter) - root_logger.addHandler(stdouthandler) - logger = logging.getLogger(__name__) - - try: - # read config file - parser = configparser.ConfigParser() - if not parser.read(args.config): - raise RuntimeError("config file not found") - - # check if mandatory config options in global section are present - if "global" not in parser.sections(): - raise RuntimeError( - "mandatory section 'global' not present in config file") - for option in ["rules"]: - if not parser.has_option("global", option): - raise RuntimeError( - f"mandatory option '{option}' not present in config " - f"section 'global'") - - # read global config section - global_config = dict(parser.items("global")) - - # read active rules - active_rules = [r.strip() for r in global_config["rules"].split(",")] - if len(active_rules) != len(set(active_rules)): - raise RuntimeError( - "at least one rule is specified multiple times " - "in 'rules' option") - if "global" in active_rules: - active_rules.remove("global") - logger.warning( - "removed illegal rule name 'global' from list of " - "active rules") - if not active_rules: - raise RuntimeError("no rules configured") - - logger.debug("preparing milter configuration ...") - rules = [] - # iterate active rules - for rule_name in active_rules: - # check if config section exists - if rule_name not in parser.sections(): - raise RuntimeError( - f"config section '{rule_name}' does not exist") - config = dict(parser.items(rule_name)) - - # check if mandatory option action is present in config - option = "action" - if option not in config.keys() and \ - option in global_config.keys(): - config[option] = global_config[option] - if option not in config.keys(): - raise RuntimeError( - f"mandatory option '{option}' not specified for " - f"rule '{rule_name}'") - config["action"] = config["action"].lower() - if config["action"] not in ["add", "del", "mod"]: - raise RuntimeError( - f"invalid action specified for rule '{rule_name}'") - - # check if mandatory options are present in config - mandatory = ["header"] - if config["action"] == "add": - mandatory += ["value"] - elif config["action"] == "mod": - mandatory += ["search", "value"] - for option in mandatory: - if option not in config.keys() and \ - option in global_config.keys(): - config[option] = global_config[option] - if option not in config.keys(): - raise RuntimeError( - f"mandatory option '{option}' not specified for " - f"rule '{rule_name}'") - - # check if optional config options are present in config - defaults = { - "ignore_hosts": [], - "ignore_envfrom": None, - "only_hosts": [], - "log": "true" - } - for option in defaults.keys(): - if option not in config.keys() and \ - option in global_config.keys(): - config[option] = global_config[option] - if option not in config.keys(): - config[option] = defaults[option] - if config["ignore_hosts"]: - config["ignore_hosts"] = [ - h.strip() for h in config["ignore_hosts"].split(",")] - if config["only_hosts"]: - config["only_hosts"] = [ - h.strip() for h in config["only_hosts"].split(",")] - config["log"] = config["log"].lower() - if config["log"] == "true": - config["log"] = True - elif config["log"] == "false": - config["log"] = False - else: - raise RuntimeError( - f"invalid value specified for option 'log' for " - f"rule '{rule_name}'") - - # add rule - logging.debug(f"adding rule '{rule_name}'") - rules.append(HeaderRule(name=rule_name, **config)) - - except RuntimeError as e: - logger.error(e) - sys.exit(255) - - if args.test: - print("Configuration ok") - sys.exit(0) - - # change log format for runtime - formatter = logging.Formatter( - f"%(asctime)s {logname}: [%(levelname)s] %(message)s", - datefmt="%Y-%m-%d %H:%M:%S") - stdouthandler.setFormatter(formatter) - - # setup syslog - sysloghandler = logging.handlers.SysLogHandler( - address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL) - sysloghandler.setLevel(loglevel) - formatter = logging.Formatter(f"{syslog_name}: %(message)s") - sysloghandler.setFormatter(formatter) - root_logger.addHandler(sysloghandler) - - logger.info("pymodmilter starting") - HeaderMilter.set_rules(rules) - - # register milter factory class - Milter.factory = HeaderMilter - Milter.set_exception_policy(Milter.TEMPFAIL) - - rc = 0 - try: - Milter.runmilter("pymodmilter", socketname=args.socket, timeout=30) - except Milter.milter.error as e: - logger.error(e) - rc = 255 - logger.info("pymodmilter terminated") - sys.exit(rc) - - -if __name__ == "__main__": - main() diff --git a/pymodmilter/run.py b/pymodmilter/run.py new file mode 100644 index 0000000..3afd970 --- /dev/null +++ b/pymodmilter/run.py @@ -0,0 +1,189 @@ +# PyMod-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyMod-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyMod-Milter. If not, see . +# + + +import Milter +import argparse +import logging +import logging.handlers +import sys + +from json import loads +from re import sub + +from pymodmilter import Rule, ModifyMilter +from pymodmilter.version import __version__ as version + + +def main(): + "Run PyMod-Milter." + # parse command line + parser = argparse.ArgumentParser( + description="PyMod milter daemon", + formatter_class=lambda prog: argparse.HelpFormatter( + prog, max_help_position=45, width=140)) + parser.add_argument( + "-c", "--config", help="Config file to read.", + default="/etc/pymodmilter.conf") + parser.add_argument( + "-s", + "--socket", + help="Socket used to communicate with the MTA.", + required=True) + parser.add_argument( + "-d", + "--debug", + help="Log debugging messages.", + action="store_true") + parser.add_argument( + "-t", + "--test", + help="Check configuration.", + action="store_true") + parser.add_argument( + "-v", "--version", + help="Print version.", + action="version", + version=f"%(prog)s ({version})") + args = parser.parse_args() + + # setup logging + loglevel = logging.INFO + logname = "pymodmilter" + syslog_name = logname + if args.debug: + loglevel = logging.DEBUG + logname = f"{logname}[%(name)s]" + syslog_name = f"{syslog_name}: [%(name)s] %(levelname)s" + + root_logger = logging.getLogger() + root_logger.setLevel(loglevel) + + # setup console log + stdouthandler = logging.StreamHandler(sys.stdout) + stdouthandler.setLevel(logging.DEBUG) + formatter = logging.Formatter("%(message)s") + stdouthandler.setFormatter(formatter) + root_logger.addHandler(stdouthandler) + logger = logging.getLogger(__name__) + + try: + # read config file + logger.debug("parsing config file") + try: + with open(args.config, "r") as fh: + config = loads( + sub(r"(?m)^\s*#.*\n?", "", fh.read())) + except Exception as e: + raise RuntimeError( + f"unable to parse config file: {e}") + + logger.debug("preparing milter configuration ...") + + # default values for global config if not set + if "global" not in config: + config["global"] = {} + + if "local_addrs" not in config["global"]: + config["global"]["local_addrs"] = [ + "127.0.0.0/8", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16"] + + if "log" not in config["global"]: + config["global"]["log"] = True + + # check if mandatory sections are present in config + for section in ["rules"]: + if section not in config: + raise RuntimeError( + f"mandatory config section '{section}' not found") + + if not config["rules"]: + raise RuntimeError("no rules configured") + + rules = [] + # iterate configured rules + for rule_idx, rule in enumerate(config["rules"]): + params = {} + # set default values if not specified in config + if "name" in rule: + params["name"] = rule["name"] + else: + params["name"] = f"Rule #{rule_idx}" + + if "log" in rule: + params["log"] = rule["log"] + else: + params["log"] = config["global"]["log"] + + if "local_addrs" in rule: + params["local_addrs"] = rule["local_addrs"] + else: + params["local_addrs"] = config["global"]["local_addrs"] + + if "conditions" in rule: + params["conditions"] = rule["conditions"] + + if "modifications" in rule: + params["modifications"] = rule["modifications"] + else: + raise RuntimeError( + f"{rule['name']}: mandatory config section " + f"'modifications' not found") + + rules.append(Rule(**params)) + + except RuntimeError as e: + logger.error(e) + sys.exit(255) + + if args.test: + print("Configuration ok") + sys.exit(0) + + # change log format for runtime + formatter = logging.Formatter( + f"%(asctime)s {logname}: [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S") + stdouthandler.setFormatter(formatter) + + # setup syslog + sysloghandler = logging.handlers.SysLogHandler( + address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL) + sysloghandler.setLevel(loglevel) + formatter = logging.Formatter(f"{syslog_name}: %(message)s") + sysloghandler.setFormatter(formatter) + root_logger.addHandler(sysloghandler) + + logger.info("pymodmilter starting") + ModifyMilter.set_rules(rules) + + # register milter factory class + Milter.factory = ModifyMilter + Milter.set_exception_policy(Milter.TEMPFAIL) + + rc = 0 + try: + Milter.runmilter("pymodmilter", socketname=args.socket, timeout=30) + except Milter.milter.error as e: + logger.error(e) + rc = 255 + logger.info("pymodmilter terminated") + sys.exit(rc) + + +if __name__ == "__main__": + main() diff --git a/pymodmilter/version.py b/pymodmilter/version.py new file mode 100644 index 0000000..a73339b --- /dev/null +++ b/pymodmilter/version.py @@ -0,0 +1 @@ +__version__ = "0.0.8" diff --git a/setup.py b/setup.py index 962c6b7..ba29fe6 100644 --- a/setup.py +++ b/setup.py @@ -4,8 +4,11 @@ def read_file(fname): with open(fname, 'r') as f: return f.read() +version = {} +exec(read_file("pymodmilter/version.py"), version) + setup(name = "pymodmilter", - version = "0.0.8", + version = version["__version__"], author = "Thomas Oettli", author_email = "spacefreak@noop.ch", description = "A pymilter based sendmail/postfix pre-queue filter.", @@ -28,7 +31,7 @@ setup(name = "pymodmilter", ], entry_points = { "console_scripts": [ - "pymodmilter=pymodmilter:main" + "pymodmilter=pymodmilter.run:main" ] }, install_requires = ["pymilter", "netaddr"], diff --git a/test-pymodmilter b/test-pymodmilter new file mode 100755 index 0000000..fbe25a8 --- /dev/null +++ b/test-pymodmilter @@ -0,0 +1,9 @@ +#!/usr/bin/env python + +import sys +import pymodmilter.run + +if __name__ == '__main__': + sys.exit( + pymodmilter.run.main() + )