From b4986af1c2b04f09fb5b7717a782ceb688d5398e Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Tue, 9 Mar 2021 12:14:48 +0100 Subject: [PATCH] switch to new config objects --- pymodmilter/__init__.py | 48 +++-- pymodmilter/actions.py | 129 ++++--------- pymodmilter/conditions.py | 56 +----- pymodmilter/config.py | 36 ++-- pymodmilter/docs/pymodmilter.conf.example | 20 +- pymodmilter/run.py | 155 ++-------------- pymodmilter/test.conf | 214 ++++++++++++++++++++++ pymodmilter/test.py | 5 + 8 files changed, 331 insertions(+), 332 deletions(-) create mode 100644 pymodmilter/test.conf create mode 100755 pymodmilter/test.py diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 719760e..03e7e7e 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -36,6 +36,7 @@ from email.message import MIMEPart from email.parser import BytesFeedParser from email.policy import default as default_policy, SMTP +from pymodmilter.actions import Action from pymodmilter.conditions import Conditions @@ -58,28 +59,26 @@ class Rule: Rule to implement multiple actions on emails. """ - def __init__(self, name, local_addrs, conditions, actions, pretend=False, - loglevel=logging.INFO): - logger = logging.getLogger(name) - self.logger = CustomLogger(logger, {"name": name}) - self.logger.setLevel(loglevel) + def __init__(self, milter_cfg, cfg): + logger = logging.getLogger(cfg["name"]) + self.logger = CustomLogger(logger, {"name": cfg["name"]}) + self.logger.setLevel(cfg["loglevel"]) - if logger is None: - logger = logging.getLogger(__name__) - - self.logger = CustomLogger(logger, {"name": name}) - self.conditions = Conditions( - local_addrs=local_addrs, - args=conditions, - logger=self.logger) - self.actions = actions - self.pretend = pretend + if cfg["conditions"] is None: + self.conditions = None + else: + self.conditions = Conditions(milter_cfg, cfg["conditions"]) self._need_body = False - for action in actions: + + self.actions = [] + for action_cfg in cfg["actions"]: + action = Action(milter_cfg, action_cfg) + self.actions.append(action) if action.need_body(): self._need_body = True - break + + self.pretend = cfg["pretend"] def need_body(self): """Return True if this rule needs the message body.""" @@ -97,9 +96,9 @@ class Rule: if envto is not None: args["envto"] = envto - if self.conditions.match(args): + if self.conditions is None or self.conditions.match(args): for action in self.actions: - if action.conditions.match(args): + if action.conditions is None or action.conditions.match(args): return False return True @@ -160,12 +159,11 @@ class ModifyMilter(Milter.Base): _loglevel = logging.INFO @staticmethod - def set_rules(rules): - ModifyMilter._rules = rules - - @staticmethod - def set_loglevel(level): - ModifyMilter._loglevel = level + def set_config(cfg): + ModifyMilter._loglevel = cfg["loglevel"] + for rule_cfg in cfg["rules"]: + ModifyMilter._rules.append( + Rule(cfg, rule_cfg)) def __init__(self): self.logger = logging.getLogger(__name__) diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 96283d1..1aa26a5 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -269,7 +269,7 @@ def add_disclaimer(milter, text, html, action, policy, pretend=False, def rewrite_links(milter, repl, pretend=False, - logger=logging.getLogger(__name__)): + logger=logging.getLogger(__name__)): """Rewrite link targets in the mail html body.""" html_body, html_content = _get_body_content(milter.msg, "html") @@ -318,106 +318,41 @@ def store(milter, directory, pretend=False, class Action: """Action to implement a pre-configured action to perform on e-mails.""" - _need_body_map = { - "add_header": False, - "del_header": False, - "mod_header": False, - "add_disclaimer": True, - "rewrite_links": True, - "store": True} - def __init__(self, name, local_addrs, conditions, action_type, args, - loglevel=logging.INFO, pretend=False): - logger = logging.getLogger(name) - self.logger = CustomLogger(logger, {"name": name}) - self.logger.setLevel(loglevel) + def __init__(self, milter_cfg, cfg): + logger = logging.getLogger(cfg["name"]) + self.logger = CustomLogger(logger, {"name": cfg["name"]}) + self.logger.setLevel(cfg["loglevel"]) - self.conditions = Conditions( - local_addrs=local_addrs, - args=conditions, - logger=self.logger) - self.pretend = pretend - self._args = {} + if cfg["conditions"] is None: + self.conditions = None + else: + self.conditions = Conditions(milter_cfg, cfg["conditions"]) - if action_type not in self._need_body_map: - raise RuntimeError(f"invalid action type '{action_type}'") - self._need_body = self._need_body_map[action_type] + self.pretend = cfg["pretend"] + self._args = cfg["args"] - try: - if action_type == "add_header": - self._func = add_header - self._args["field"] = args["header"] - self._args["value"] = args["value"] - if "idx" in args: - self._args["idx"] = args["idx"] - - elif action_type in ["mod_header", "del_header"]: - args["field"] = args["header"] - del args["header"] - regex_args = ["field"] - - if action_type == "mod_header": - self._func = mod_header - self._args["value"] = args["value"] - regex_args.append("search") - elif action_type == "del_header": - self._func = del_header - if "value" in args: - regex_args.append("value") - - for arg in regex_args: - try: - self._args[arg] = re.compile( - args[arg], - re.MULTILINE + re.DOTALL + re.IGNORECASE) - except re.error as e: - raise RuntimeError( - f"unable to parse {arg} regex: {e}") - - elif action_type == "add_disclaimer": - self._func = add_disclaimer - if args["action"] not in ["append", "prepend"]: - raise RuntimeError(f"invalid action '{args['action']}'") - - self._args["action"] = args["action"] - - if args["error_policy"] not in ["wrap", "ignore", "reject"]: - raise RuntimeError(f"invalid policy '{args['policy']}'") - - self._args["policy"] = args["error_policy"] - - try: - with open(args["html_file"], "r") as f: - html = BeautifulSoup( - f.read(), "html.parser") - body = html.find('body') - if body: - # just use content within the body tag if present - html = body - self._args["html"] = html - with open(args["text_file"], "r") as f: - self._args["text"] = f.read() - except IOError as e: - raise RuntimeError(f"unable to read template: {e}") - - elif action_type == "rewrite_links": - self._func = rewrite_links - self._args["repl"] = args["repl"] - - elif action_type == "store": - self._func = store - if args["storage_type"] not in ["file"]: - raise RuntimeError( - "invalid storage_type 'args['storage_type']'") - - if args["storage_type"] == "file": - self._args["directory"] = args["directory"] - else: - raise RuntimeError(f"invalid action type: {action_type}") - - except KeyError as e: - raise RuntimeError( - f"mandatory argument not found: {e}") + action_type = cfg["type"] + if action_type == "add_header": + self._func = add_header + self._need_body = False + elif action_type == "mod_header": + self._func = mod_header + self._need_body = False + elif action_type == "del_header": + self._func = del_header + self._need_body = False + elif action_type == "add_disclaimer": + self._func = add_disclaimer + self._need_body = True + elif action_type == "rewrite_links": + self._func = rewrite_links + self._need_body = True + elif action_type == "store": + self._func = store + self._need_body = True + else: + raise ValueError(f"invalid action type: {action_type}") def need_body(self): """Return the needs of this action.""" diff --git a/pymodmilter/conditions.py b/pymodmilter/conditions.py index c0fbf9a..335baf5 100644 --- a/pymodmilter/conditions.py +++ b/pymodmilter/conditions.py @@ -13,61 +13,21 @@ # import logging -import re -from netaddr import IPAddress, IPNetwork, AddrFormatError +from netaddr import IPAddress +from pymodmilter import CustomLogger class Conditions: """Conditions to implement conditions for rules and actions.""" - def __init__(self, local_addrs, args, logger=None): - if logger is None: - logger = logging.getLogger(__name__) + def __init__(self, milter_cfg, cfg): + logger = logging.getLogger(cfg["name"]) + self.logger = CustomLogger(logger, {"name": cfg["name"]}) + self.logger.setLevel(cfg["loglevel"]) - self._local_addrs = [] - self.logger = logger - self._args = {} - - try: - for addr in local_addrs: - self._local_addrs.append(IPNetwork(addr)) - except AddrFormatError as e: - raise RuntimeError(f"invalid address in local_addrs: {e}") - - try: - if "local" in args: - logger.debug(f"condition: local = {args['local']}") - self._args["local"] = args["local"] - - if "hosts" in args: - logger.debug(f"condition: hosts = {args['hosts']}") - self._args["hosts"] = [] - try: - for host in args["hosts"]: - self._args["hosts"].append(IPNetwork(host)) - except AddrFormatError as e: - raise RuntimeError(f"invalid address in hosts: {e}") - - if "envfrom" in args: - logger.debug(f"condition: envfrom = {args['envfrom']}") - try: - self._args["envfrom"] = re.compile( - args["envfrom"], re.IGNORECASE) - except re.error as e: - raise RuntimeError(f"unable to parse envfrom regex: {e}") - - if "envto" in args: - logger.debug(f"condition: envto = {args['envto']}") - try: - self._args["envto"] = re.compile( - args["envto"], re.IGNORECASE) - except re.error as e: - raise RuntimeError(f"unable to parse envto regex: {e}") - - except KeyError as e: - raise RuntimeError( - f"mandatory argument not found: {e}") + self._local_addrs = milter_cfg["local_addrs"] + self._args = cfg["args"] def match(self, args): if "host" in args: diff --git a/pymodmilter/config.py b/pymodmilter/config.py index 2c4a705..3d1b651 100644 --- a/pymodmilter/config.py +++ b/pymodmilter/config.py @@ -27,7 +27,12 @@ from netaddr import IPNetwork, AddrFormatError class BaseConfig: - def __init__(self, cfg={}, debug=False): + def __init__(self, cfg={}, debug=False, logger=None): + if logger is None: + logger = logging.getLogger(__name__) + + self.logger = logger + self._cfg = {} if "name" in cfg: assert isinstance(cfg["name"], str), \ @@ -39,10 +44,13 @@ class BaseConfig: if debug: self["loglevel"] = logging.DEBUG elif "loglevel" in cfg: - level = getattr(logging, cfg["loglevel"].upper(), None) - assert isinstance(level, int), \ - f"{self['name']}: loglevel: invalid value" - self["loglevel"] = level + if isinstance(cfg["loglevel"], int): + self["loglevel"] = cfg["loglevel"] + else: + level = getattr(logging, cfg["loglevel"].upper(), None) + assert isinstance(level, int), \ + f"{self['name']}: loglevel: invalid value" + self["loglevel"] = level else: self["loglevel"] = logging.INFO @@ -137,7 +145,6 @@ class ActionConfig(BaseConfig): self["pretend"] = rule_cfg["pretend"] self["conditions"] = None self["type"] = "" - self["need_body"] = False if "pretend" in cfg: pretend = cfg["pretend"] @@ -217,22 +224,23 @@ class ActionConfig(BaseConfig): raise RuntimeError( f"{self['name']}: unable to open/read template file: {e}") - self["need_body"] = True - elif self["type"] == "rewrite_links": self.add_string_arg(cfg, "repl") - self["need_body"] = True elif self["type"] == "store": self.add_string_arg(cfg, "storage_type") - assert self["storage_type"] in ("file"), \ + assert self["args"]["storage_type"] in ("file"), \ f"{self['name']}: storage_type: invalid value, " \ f"should be 'file'" if self["args"]["storage_type"] == "file": self.add_string_arg(cfg, "directory") - self["need_body"] = True + if "conditions" in cfg: + conditions = cfg["conditions"] + assert isinstance(conditions, dict), \ + f"{self['name']}: conditions: invalid value, should be dict" + self["conditions"] = ConditionsConfig(self, conditions, debug) class RuleConfig(BaseConfig): @@ -265,6 +273,12 @@ class RuleConfig(BaseConfig): self["actions"].append( ActionConfig(idx, self, action_cfg, debug)) + if "conditions" in cfg: + conditions = cfg["conditions"] + assert isinstance(conditions, dict), \ + f"{self['name']}: conditions: invalid value, should be dict" + self["conditions"] = ConditionsConfig(self, conditions, debug) + class ModifyMilterConfig(BaseConfig): def __init__(self, cfgfile, debug=False): diff --git a/pymodmilter/docs/pymodmilter.conf.example b/pymodmilter/docs/pymodmilter.conf.example index 04e887f..98967ea 100644 --- a/pymodmilter/docs/pymodmilter.conf.example +++ b/pymodmilter/docs/pymodmilter.conf.example @@ -111,12 +111,12 @@ # "type": "add_header", - # Option: header + # Option: field # Type: String # Notes: Name of the header. # Value: [ NAME ] # - "header": "X-Test-Header", + "field": "X-Test-Header", # Option: value # Type: String @@ -129,12 +129,12 @@ "type": "mod_header", - # Option: header + # Option: field # Type: String # Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject). # Value: [ REGEX ] # - "header": "^Subject$", + "field": "^Subject$", # Option: search # Type: String @@ -153,12 +153,12 @@ "type": "del_header", - # Option: header + # Option: field # Type: String # Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject). # Value: [ REGEX ] # - "header": "^Received$" + "field": "^Received$" }, { "name": "add_disclaimer", @@ -171,19 +171,19 @@ # "action": "prepend", - # Option: html_file + # Option: html_template # Type: String # Notes: Path to a file which contains the html representation of the disclaimer. # Value: [ FILE_PATH ] # - "html_file": "/etc/pymodmilter/templates/disclaimer_html.template", + "html_template": "/etc/pymodmilter/templates/disclaimer_html.template", - # Option: text_file + # Option: text_template # Type: String # Notes: Path to a file which contains the text representation of the disclaimer. # Value: [ FILE_PATH ] # - "text_file": "/etc/pymodmilter/templates/disclaimer_text.template", + "text_template": "/etc/pymodmilter/templates/disclaimer_text.template", # Option: error_policy # Type: String diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 4c174ab..b89bebe 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -18,12 +18,9 @@ import logging import logging.handlers import sys -from json import loads -from re import sub - -from pymodmilter import Rule, ModifyMilter +from pymodmilter import ModifyMilter from pymodmilter import __version__ as version -from pymodmilter.actions import Action +from pymodmilter.config import ModifyMilterConfig def main(): @@ -63,13 +60,6 @@ def main(): args = parser.parse_args() - loglevels = { - "error": logging.ERROR, - "warning": logging.WARNING, - "info": logging.INFO, - "debug": logging.DEBUG - } - root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) @@ -92,148 +82,32 @@ def main(): logger.setLevel(logging.INFO) try: - try: - with open(args.config, "r") as fh: - config = sub(r"(?m)^\s*#.*\n?", "", fh.read()) - config = loads(config) - except Exception as e: - for num, line in enumerate(config.splitlines()): - logger.error(f"{num+1}: {line}") - raise RuntimeError( - f"unable to parse config file: {e}") - - if "global" not in config: - config["global"] = {} - - if args.debug: - loglevel = logging.DEBUG - else: - if "loglevel" not in config["global"]: - config["global"]["loglevel"] = "info" - loglevel = loglevels[config["global"]["loglevel"]] - - logger.setLevel(loglevel) - logger.debug("prepar milter configuration") + cfg = ModifyMilterConfig(args.cfgfile, args.debug) - if "pretend" not in config["global"]: - config["global"]["pretend"] = False + if not args.debug: + logger.setLevel(cfg["loglevel"]) if args.socket: socket = args.socket - elif "socket" in config["global"]: - socket = config["global"]["socket"] + elif "socket" in cfg: + socket = cfg["socket"] else: raise RuntimeError( "listening socket is neither specified on the command line " "nor in the configuration file") - if "local_addrs" in config["global"]: - local_addrs = config["global"]["local_addrs"] - else: - local_addrs = [ - "::1/128", - "127.0.0.0/8", - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16"] - - if "rules" not in config: - raise RuntimeError( - "mandatory config section 'rules' not found") - - if not config["rules"]: + if not cfg["rules"]: raise RuntimeError("no rules configured") - logger.debug("initialize rules ...") + logger.debug("initializing rules ...") - rules = [] - for rule_idx, rule in enumerate(config["rules"]): - if "name" in rule: - rule_name = rule["name"] - else: - rule_name = f"Rule #{rule_idx}" - - logger.debug(f"prepare rule {rule_name} ...") - - if "actions" not in rule: + for rule_cfg in cfg["rules"]: + if not rule_cfg["actions"]: raise RuntimeError( - f"{rule_name}: mandatory config " - f"section 'actions' not found") + f"{rule_cfg['name']}: no actions configured") - if not rule["actions"]: - raise RuntimeError("{rule_name}: no actions configured") - - if args.debug: - rule_loglevel = logging.DEBUG - elif "loglevel" in rule: - rule_loglevel = loglevels[rule["loglevel"]] - else: - rule_loglevel = loglevels[config["global"]["loglevel"]] - - if "pretend" in rule: - rule_pretend = rule["pretend"] - else: - rule_pretend = config["global"]["pretend"] - - actions = [] - for action_idx, action in enumerate(rule["actions"]): - if "name" in action: - action_name = f"{rule_name}: {action['name']}" - else: - action_name = f"Action #{action_idx}" - - if args.debug: - action_loglevel = logging.DEBUG - elif "loglevel" in action: - action_loglevel = loglevels[action["loglevel"]] - else: - action_loglevel = rule_loglevel - - if "pretend" in action: - action_pretend = action["pretend"] - else: - action_pretend = rule_pretend - - if "type" not in action: - raise RuntimeError( - f"{rule_name}: {action_name}: mandatory config " - f"section 'actions' not found") - - if "conditions" not in action: - action["conditions"] = {} - - try: - actions.append( - Action( - name=action_name, - local_addrs=local_addrs, - conditions=action["conditions"], - action_type=action["type"], - args=action, - loglevel=action_loglevel, - pretend=action_pretend)) - except RuntimeError as e: - logger.error(f"{action_name}: {e}") - sys.exit(253) - - if "conditions" not in rule: - rule["conditions"] = {} - - try: - rules.append( - Rule( - name=rule_name, - local_addrs=local_addrs, - conditions=rule["conditions"], - actions=actions, - loglevel=rule_loglevel, - pretend=rule_pretend)) - except RuntimeError as e: - logger.error(f"{rule_name}: {e}") - sys.exit(254) - - except RuntimeError as e: + except (RuntimeError, AssertionError) as e: logger.error(e) sys.exit(255) @@ -247,8 +121,7 @@ def main(): stdouthandler.setLevel(logging.DEBUG) logger.info("pymodmilter starting") - ModifyMilter.set_rules(rules) - ModifyMilter.set_loglevel(loglevel) + ModifyMilter.set_config(cfg) # register milter factory class Milter.factory = ModifyMilter diff --git a/pymodmilter/test.conf b/pymodmilter/test.conf new file mode 100644 index 0000000..998e537 --- /dev/null +++ b/pymodmilter/test.conf @@ -0,0 +1,214 @@ +# This is an example /etc/pymodmilter.conf file. +# Copy it into place before use. +# +# The file is in JSON format. +# +# The global option 'log' can be overriden per rule or per modification. +# +{ + # Section: global + # Notes: Global options. + # + "global": { + # Option: socket + # Type: String + # Notes: The socket used to communicate with the MTA. + # + # Examples: + # unix:/path/to/socket a named pipe + # inet:8899 listen on ANY interface + # inet:8899@localhost listen on a specific interface + # inet6:8899 listen on ANY interface + # inet6:8899@[2001:db8:1234::1] listen on a specific interface + # Value: [ SOCKET ] + "socket": "inet:8898@127.0.0.1", + + # Option: local_addrs + # Type: List + # Notes: A list of local hosts and networks. + # Value: [ LIST ] + # + "local_addrs": ["::1/128", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"], + + # Option: loglevel + # Type: String + # Notes: Set loglevel for rules and actions. + # Value: [ error | warning | info | debug ] + # + "loglevel": "info", + + # Option: pretend + # Type: Bool + # Notes: Just pretend to do the actions, for test purposes. + # Value: [ true | false ] + # + "pretend": true + }, + + # Section: rules + # Notes: Rules and related actions. + # + "rules": [ + { + # Option: name + # Type: String + # Notes: Name of the rule. + # Value: [ NAME ] + # + "name": "myrule", + + # Section: conditions + # Notes: Optional conditions to process the rule. + # If multiple conditions are set, they all + # have to be true to process the rule. + # + "conditions": { + # Option: local + # Type: Bool + # Notes: Condition wheter the senders host address is listed in local_addrs. + # Value: [ true | false ] + # + "local": false, + + # Option: hosts + # Type: String + # Notes: Condition wheter the senders host address is listed in this list. + # Value: [ LIST ] + # + "hosts": [ "127.0.0.1" ], + + # Option: envfrom + # Type: String + # Notes: Condition wheter the envelop-from address matches this regular expression. + # Value: [ REGEX ] + # + "envfrom": "^.+@mypartner\\.com$", + + # Option: envto + # Type: String + # Notes: Condition wheter the envelop-to address matches this regular expression. + # Value: [ REGEX ] + # + "envto": "^postmaster@.+$" + }, + + # Section: actions + # Notes: Actions of the rule. + # + "actions": [ + { + # Option: name + # Type: String + # Notes: Name of the modification. + # Value: [ NAME ] + # + "name": "add_test_header", + + # Option: type + # Type: String + # Notes: Type of the modification. + # Value: [ add_header | del_header | mod_header ] + # + "type": "add_header", + + # Option: field + # Type: String + # Notes: Name of the header. + # Value: [ NAME ] + # + "field": "X-Test-Header", + + # Option: value + # Type: String + # Notes: Value of the header. + # Value: [ VALUE ] + # + "value": "true" + }, { + "name": "modify_subject", + + "type": "mod_header", + + # Option: field + # Type: String + # Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject). + # Value: [ REGEX ] + # + "field": "^Subject$", + + # Option: search + # Type: String + # Notes: Regular expression to match against the headers value. + # Values: [ VALUE ] + # + "search": "(?P.*)", + + # Option: value + # Type: String + # Notes: New value of the header. + # Values: [ VALUE ] + "value": "[EXTERNAL] \\g" + }, { + "name": "delete_received_header", + + "type": "del_header", + + # Option: field + # Type: String + # Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject). + # Value: [ REGEX ] + # + "field": "^Received$" + }, { + "name": "add_disclaimer", + + "type": "add_disclaimer", + + # Option: action + # Type: String + # Notes: Action to perform with the disclaimer. + # Value: [ append | prepend ] + # + "action": "prepend", + + # Option: html_template + # Type: String + # Notes: Path to a file which contains the html representation of the disclaimer. + # Value: [ FILE_PATH ] + # + "html_template": "docs/templates/disclaimer_html.template", + + # Option: text_template + # Type: String + # Notes: Path to a file which contains the text representation of the disclaimer. + # Value: [ FILE_PATH ] + # + "text_template": "docs/templates/disclaimer_text.template", + + # Option: error_policy + # Type: String + # Notes: Set what should be done if the modification fails (e.g. no message body present). + # Value: [ wrap | ignore | reject ] + # + "error_policy": "wrap" + }, { + "name": "store_message", + + "type": "store", + + # Option: storage_type + # Type: String + # Notes: The storage type used to store e-mails. + # Value: [ file ] + "storage_type": "file", + + # Option: directory + # Type: String + # Notes: Directory used to store e-mails. + # Value: [ file ] + "directory": "/mnt/messages" + } + ] + } + ] +} diff --git a/pymodmilter/test.py b/pymodmilter/test.py new file mode 100755 index 0000000..dbd784f --- /dev/null +++ b/pymodmilter/test.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 + +import config + +a = config.ModifyMilterConfig("test.conf", debug=True)