switch to new config objects

This commit is contained in:
2021-03-09 12:14:48 +01:00
parent 915fa509b5
commit b4986af1c2
8 changed files with 331 additions and 332 deletions

View File

@@ -36,6 +36,7 @@ from email.message import MIMEPart
from email.parser import BytesFeedParser
from email.policy import default as default_policy, SMTP
from pymodmilter.actions import Action
from pymodmilter.conditions import Conditions
@@ -58,28 +59,26 @@ class Rule:
Rule to implement multiple actions on emails.
"""
def __init__(self, name, local_addrs, conditions, actions, pretend=False,
loglevel=logging.INFO):
logger = logging.getLogger(name)
self.logger = CustomLogger(logger, {"name": name})
self.logger.setLevel(loglevel)
def __init__(self, milter_cfg, cfg):
logger = logging.getLogger(cfg["name"])
self.logger = CustomLogger(logger, {"name": cfg["name"]})
self.logger.setLevel(cfg["loglevel"])
if logger is None:
logger = logging.getLogger(__name__)
self.logger = CustomLogger(logger, {"name": name})
self.conditions = Conditions(
local_addrs=local_addrs,
args=conditions,
logger=self.logger)
self.actions = actions
self.pretend = pretend
if cfg["conditions"] is None:
self.conditions = None
else:
self.conditions = Conditions(milter_cfg, cfg["conditions"])
self._need_body = False
for action in actions:
self.actions = []
for action_cfg in cfg["actions"]:
action = Action(milter_cfg, action_cfg)
self.actions.append(action)
if action.need_body():
self._need_body = True
break
self.pretend = cfg["pretend"]
def need_body(self):
"""Return True if this rule needs the message body."""
@@ -97,9 +96,9 @@ class Rule:
if envto is not None:
args["envto"] = envto
if self.conditions.match(args):
if self.conditions is None or self.conditions.match(args):
for action in self.actions:
if action.conditions.match(args):
if action.conditions is None or action.conditions.match(args):
return False
return True
@@ -160,12 +159,11 @@ class ModifyMilter(Milter.Base):
_loglevel = logging.INFO
@staticmethod
def set_rules(rules):
ModifyMilter._rules = rules
@staticmethod
def set_loglevel(level):
ModifyMilter._loglevel = level
def set_config(cfg):
ModifyMilter._loglevel = cfg["loglevel"]
for rule_cfg in cfg["rules"]:
ModifyMilter._rules.append(
Rule(cfg, rule_cfg))
def __init__(self):
self.logger = logging.getLogger(__name__)

View File

@@ -318,106 +318,41 @@ def store(milter, directory, pretend=False,
class Action:
"""Action to implement a pre-configured action to perform on e-mails."""
_need_body_map = {
"add_header": False,
"del_header": False,
"mod_header": False,
"add_disclaimer": True,
"rewrite_links": True,
"store": True}
def __init__(self, name, local_addrs, conditions, action_type, args,
loglevel=logging.INFO, pretend=False):
logger = logging.getLogger(name)
self.logger = CustomLogger(logger, {"name": name})
self.logger.setLevel(loglevel)
def __init__(self, milter_cfg, cfg):
logger = logging.getLogger(cfg["name"])
self.logger = CustomLogger(logger, {"name": cfg["name"]})
self.logger.setLevel(cfg["loglevel"])
self.conditions = Conditions(
local_addrs=local_addrs,
args=conditions,
logger=self.logger)
self.pretend = pretend
self._args = {}
if cfg["conditions"] is None:
self.conditions = None
else:
self.conditions = Conditions(milter_cfg, cfg["conditions"])
if action_type not in self._need_body_map:
raise RuntimeError(f"invalid action type '{action_type}'")
self._need_body = self._need_body_map[action_type]
self.pretend = cfg["pretend"]
self._args = cfg["args"]
try:
action_type = cfg["type"]
if action_type == "add_header":
self._func = add_header
self._args["field"] = args["header"]
self._args["value"] = args["value"]
if "idx" in args:
self._args["idx"] = args["idx"]
elif action_type in ["mod_header", "del_header"]:
args["field"] = args["header"]
del args["header"]
regex_args = ["field"]
if action_type == "mod_header":
self._need_body = False
elif action_type == "mod_header":
self._func = mod_header
self._args["value"] = args["value"]
regex_args.append("search")
self._need_body = False
elif action_type == "del_header":
self._func = del_header
if "value" in args:
regex_args.append("value")
for arg in regex_args:
try:
self._args[arg] = re.compile(
args[arg],
re.MULTILINE + re.DOTALL + re.IGNORECASE)
except re.error as e:
raise RuntimeError(
f"unable to parse {arg} regex: {e}")
self._need_body = False
elif action_type == "add_disclaimer":
self._func = add_disclaimer
if args["action"] not in ["append", "prepend"]:
raise RuntimeError(f"invalid action '{args['action']}'")
self._args["action"] = args["action"]
if args["error_policy"] not in ["wrap", "ignore", "reject"]:
raise RuntimeError(f"invalid policy '{args['policy']}'")
self._args["policy"] = args["error_policy"]
try:
with open(args["html_file"], "r") as f:
html = BeautifulSoup(
f.read(), "html.parser")
body = html.find('body')
if body:
# just use content within the body tag if present
html = body
self._args["html"] = html
with open(args["text_file"], "r") as f:
self._args["text"] = f.read()
except IOError as e:
raise RuntimeError(f"unable to read template: {e}")
self._need_body = True
elif action_type == "rewrite_links":
self._func = rewrite_links
self._args["repl"] = args["repl"]
self._need_body = True
elif action_type == "store":
self._func = store
if args["storage_type"] not in ["file"]:
raise RuntimeError(
"invalid storage_type 'args['storage_type']'")
if args["storage_type"] == "file":
self._args["directory"] = args["directory"]
self._need_body = True
else:
raise RuntimeError(f"invalid action type: {action_type}")
except KeyError as e:
raise RuntimeError(
f"mandatory argument not found: {e}")
raise ValueError(f"invalid action type: {action_type}")
def need_body(self):
"""Return the needs of this action."""

View File

@@ -13,61 +13,21 @@
#
import logging
import re
from netaddr import IPAddress, IPNetwork, AddrFormatError
from netaddr import IPAddress
from pymodmilter import CustomLogger
class Conditions:
"""Conditions to implement conditions for rules and actions."""
def __init__(self, local_addrs, args, logger=None):
if logger is None:
logger = logging.getLogger(__name__)
def __init__(self, milter_cfg, cfg):
logger = logging.getLogger(cfg["name"])
self.logger = CustomLogger(logger, {"name": cfg["name"]})
self.logger.setLevel(cfg["loglevel"])
self._local_addrs = []
self.logger = logger
self._args = {}
try:
for addr in local_addrs:
self._local_addrs.append(IPNetwork(addr))
except AddrFormatError as e:
raise RuntimeError(f"invalid address in local_addrs: {e}")
try:
if "local" in args:
logger.debug(f"condition: local = {args['local']}")
self._args["local"] = args["local"]
if "hosts" in args:
logger.debug(f"condition: hosts = {args['hosts']}")
self._args["hosts"] = []
try:
for host in args["hosts"]:
self._args["hosts"].append(IPNetwork(host))
except AddrFormatError as e:
raise RuntimeError(f"invalid address in hosts: {e}")
if "envfrom" in args:
logger.debug(f"condition: envfrom = {args['envfrom']}")
try:
self._args["envfrom"] = re.compile(
args["envfrom"], re.IGNORECASE)
except re.error as e:
raise RuntimeError(f"unable to parse envfrom regex: {e}")
if "envto" in args:
logger.debug(f"condition: envto = {args['envto']}")
try:
self._args["envto"] = re.compile(
args["envto"], re.IGNORECASE)
except re.error as e:
raise RuntimeError(f"unable to parse envto regex: {e}")
except KeyError as e:
raise RuntimeError(
f"mandatory argument not found: {e}")
self._local_addrs = milter_cfg["local_addrs"]
self._args = cfg["args"]
def match(self, args):
if "host" in args:

View File

@@ -27,7 +27,12 @@ from netaddr import IPNetwork, AddrFormatError
class BaseConfig:
def __init__(self, cfg={}, debug=False):
def __init__(self, cfg={}, debug=False, logger=None):
if logger is None:
logger = logging.getLogger(__name__)
self.logger = logger
self._cfg = {}
if "name" in cfg:
assert isinstance(cfg["name"], str), \
@@ -39,6 +44,9 @@ class BaseConfig:
if debug:
self["loglevel"] = logging.DEBUG
elif "loglevel" in cfg:
if isinstance(cfg["loglevel"], int):
self["loglevel"] = cfg["loglevel"]
else:
level = getattr(logging, cfg["loglevel"].upper(), None)
assert isinstance(level, int), \
f"{self['name']}: loglevel: invalid value"
@@ -137,7 +145,6 @@ class ActionConfig(BaseConfig):
self["pretend"] = rule_cfg["pretend"]
self["conditions"] = None
self["type"] = ""
self["need_body"] = False
if "pretend" in cfg:
pretend = cfg["pretend"]
@@ -217,22 +224,23 @@ class ActionConfig(BaseConfig):
raise RuntimeError(
f"{self['name']}: unable to open/read template file: {e}")
self["need_body"] = True
elif self["type"] == "rewrite_links":
self.add_string_arg(cfg, "repl")
self["need_body"] = True
elif self["type"] == "store":
self.add_string_arg(cfg, "storage_type")
assert self["storage_type"] in ("file"), \
assert self["args"]["storage_type"] in ("file"), \
f"{self['name']}: storage_type: invalid value, " \
f"should be 'file'"
if self["args"]["storage_type"] == "file":
self.add_string_arg(cfg, "directory")
self["need_body"] = True
if "conditions" in cfg:
conditions = cfg["conditions"]
assert isinstance(conditions, dict), \
f"{self['name']}: conditions: invalid value, should be dict"
self["conditions"] = ConditionsConfig(self, conditions, debug)
class RuleConfig(BaseConfig):
@@ -265,6 +273,12 @@ class RuleConfig(BaseConfig):
self["actions"].append(
ActionConfig(idx, self, action_cfg, debug))
if "conditions" in cfg:
conditions = cfg["conditions"]
assert isinstance(conditions, dict), \
f"{self['name']}: conditions: invalid value, should be dict"
self["conditions"] = ConditionsConfig(self, conditions, debug)
class ModifyMilterConfig(BaseConfig):
def __init__(self, cfgfile, debug=False):

View File

@@ -111,12 +111,12 @@
#
"type": "add_header",
# Option: header
# Option: field
# Type: String
# Notes: Name of the header.
# Value: [ NAME ]
#
"header": "X-Test-Header",
"field": "X-Test-Header",
# Option: value
# Type: String
@@ -129,12 +129,12 @@
"type": "mod_header",
# Option: header
# Option: field
# Type: String
# Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject).
# Value: [ REGEX ]
#
"header": "^Subject$",
"field": "^Subject$",
# Option: search
# Type: String
@@ -153,12 +153,12 @@
"type": "del_header",
# Option: header
# Option: field
# Type: String
# Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject).
# Value: [ REGEX ]
#
"header": "^Received$"
"field": "^Received$"
}, {
"name": "add_disclaimer",
@@ -171,19 +171,19 @@
#
"action": "prepend",
# Option: html_file
# Option: html_template
# Type: String
# Notes: Path to a file which contains the html representation of the disclaimer.
# Value: [ FILE_PATH ]
#
"html_file": "/etc/pymodmilter/templates/disclaimer_html.template",
"html_template": "/etc/pymodmilter/templates/disclaimer_html.template",
# Option: text_file
# Option: text_template
# Type: String
# Notes: Path to a file which contains the text representation of the disclaimer.
# Value: [ FILE_PATH ]
#
"text_file": "/etc/pymodmilter/templates/disclaimer_text.template",
"text_template": "/etc/pymodmilter/templates/disclaimer_text.template",
# Option: error_policy
# Type: String

View File

@@ -18,12 +18,9 @@ import logging
import logging.handlers
import sys
from json import loads
from re import sub
from pymodmilter import Rule, ModifyMilter
from pymodmilter import ModifyMilter
from pymodmilter import __version__ as version
from pymodmilter.actions import Action
from pymodmilter.config import ModifyMilterConfig
def main():
@@ -63,13 +60,6 @@ def main():
args = parser.parse_args()
loglevels = {
"error": logging.ERROR,
"warning": logging.WARNING,
"info": logging.INFO,
"debug": logging.DEBUG
}
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
@@ -92,148 +82,32 @@ def main():
logger.setLevel(logging.INFO)
try:
try:
with open(args.config, "r") as fh:
config = sub(r"(?m)^\s*#.*\n?", "", fh.read())
config = loads(config)
except Exception as e:
for num, line in enumerate(config.splitlines()):
logger.error(f"{num+1}: {line}")
raise RuntimeError(
f"unable to parse config file: {e}")
if "global" not in config:
config["global"] = {}
if args.debug:
loglevel = logging.DEBUG
else:
if "loglevel" not in config["global"]:
config["global"]["loglevel"] = "info"
loglevel = loglevels[config["global"]["loglevel"]]
logger.setLevel(loglevel)
logger.debug("prepar milter configuration")
cfg = ModifyMilterConfig(args.cfgfile, args.debug)
if "pretend" not in config["global"]:
config["global"]["pretend"] = False
if not args.debug:
logger.setLevel(cfg["loglevel"])
if args.socket:
socket = args.socket
elif "socket" in config["global"]:
socket = config["global"]["socket"]
elif "socket" in cfg:
socket = cfg["socket"]
else:
raise RuntimeError(
"listening socket is neither specified on the command line "
"nor in the configuration file")
if "local_addrs" in config["global"]:
local_addrs = config["global"]["local_addrs"]
else:
local_addrs = [
"::1/128",
"127.0.0.0/8",
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16"]
if "rules" not in config:
raise RuntimeError(
"mandatory config section 'rules' not found")
if not config["rules"]:
if not cfg["rules"]:
raise RuntimeError("no rules configured")
logger.debug("initialize rules ...")
logger.debug("initializing rules ...")
rules = []
for rule_idx, rule in enumerate(config["rules"]):
if "name" in rule:
rule_name = rule["name"]
else:
rule_name = f"Rule #{rule_idx}"
logger.debug(f"prepare rule {rule_name} ...")
if "actions" not in rule:
for rule_cfg in cfg["rules"]:
if not rule_cfg["actions"]:
raise RuntimeError(
f"{rule_name}: mandatory config "
f"section 'actions' not found")
f"{rule_cfg['name']}: no actions configured")
if not rule["actions"]:
raise RuntimeError("{rule_name}: no actions configured")
if args.debug:
rule_loglevel = logging.DEBUG
elif "loglevel" in rule:
rule_loglevel = loglevels[rule["loglevel"]]
else:
rule_loglevel = loglevels[config["global"]["loglevel"]]
if "pretend" in rule:
rule_pretend = rule["pretend"]
else:
rule_pretend = config["global"]["pretend"]
actions = []
for action_idx, action in enumerate(rule["actions"]):
if "name" in action:
action_name = f"{rule_name}: {action['name']}"
else:
action_name = f"Action #{action_idx}"
if args.debug:
action_loglevel = logging.DEBUG
elif "loglevel" in action:
action_loglevel = loglevels[action["loglevel"]]
else:
action_loglevel = rule_loglevel
if "pretend" in action:
action_pretend = action["pretend"]
else:
action_pretend = rule_pretend
if "type" not in action:
raise RuntimeError(
f"{rule_name}: {action_name}: mandatory config "
f"section 'actions' not found")
if "conditions" not in action:
action["conditions"] = {}
try:
actions.append(
Action(
name=action_name,
local_addrs=local_addrs,
conditions=action["conditions"],
action_type=action["type"],
args=action,
loglevel=action_loglevel,
pretend=action_pretend))
except RuntimeError as e:
logger.error(f"{action_name}: {e}")
sys.exit(253)
if "conditions" not in rule:
rule["conditions"] = {}
try:
rules.append(
Rule(
name=rule_name,
local_addrs=local_addrs,
conditions=rule["conditions"],
actions=actions,
loglevel=rule_loglevel,
pretend=rule_pretend))
except RuntimeError as e:
logger.error(f"{rule_name}: {e}")
sys.exit(254)
except RuntimeError as e:
except (RuntimeError, AssertionError) as e:
logger.error(e)
sys.exit(255)
@@ -247,8 +121,7 @@ def main():
stdouthandler.setLevel(logging.DEBUG)
logger.info("pymodmilter starting")
ModifyMilter.set_rules(rules)
ModifyMilter.set_loglevel(loglevel)
ModifyMilter.set_config(cfg)
# register milter factory class
Milter.factory = ModifyMilter

214
pymodmilter/test.conf Normal file
View File

@@ -0,0 +1,214 @@
# This is an example /etc/pymodmilter.conf file.
# Copy it into place before use.
#
# The file is in JSON format.
#
# The global option 'log' can be overriden per rule or per modification.
#
{
# Section: global
# Notes: Global options.
#
"global": {
# Option: socket
# Type: String
# Notes: The socket used to communicate with the MTA.
#
# Examples:
# unix:/path/to/socket a named pipe
# inet:8899 listen on ANY interface
# inet:8899@localhost listen on a specific interface
# inet6:8899 listen on ANY interface
# inet6:8899@[2001:db8:1234::1] listen on a specific interface
# Value: [ SOCKET ]
"socket": "inet:8898@127.0.0.1",
# Option: local_addrs
# Type: List
# Notes: A list of local hosts and networks.
# Value: [ LIST ]
#
"local_addrs": ["::1/128", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"],
# Option: loglevel
# Type: String
# Notes: Set loglevel for rules and actions.
# Value: [ error | warning | info | debug ]
#
"loglevel": "info",
# Option: pretend
# Type: Bool
# Notes: Just pretend to do the actions, for test purposes.
# Value: [ true | false ]
#
"pretend": true
},
# Section: rules
# Notes: Rules and related actions.
#
"rules": [
{
# Option: name
# Type: String
# Notes: Name of the rule.
# Value: [ NAME ]
#
"name": "myrule",
# Section: conditions
# Notes: Optional conditions to process the rule.
# If multiple conditions are set, they all
# have to be true to process the rule.
#
"conditions": {
# Option: local
# Type: Bool
# Notes: Condition wheter the senders host address is listed in local_addrs.
# Value: [ true | false ]
#
"local": false,
# Option: hosts
# Type: String
# Notes: Condition wheter the senders host address is listed in this list.
# Value: [ LIST ]
#
"hosts": [ "127.0.0.1" ],
# Option: envfrom
# Type: String
# Notes: Condition wheter the envelop-from address matches this regular expression.
# Value: [ REGEX ]
#
"envfrom": "^.+@mypartner\\.com$",
# Option: envto
# Type: String
# Notes: Condition wheter the envelop-to address matches this regular expression.
# Value: [ REGEX ]
#
"envto": "^postmaster@.+$"
},
# Section: actions
# Notes: Actions of the rule.
#
"actions": [
{
# Option: name
# Type: String
# Notes: Name of the modification.
# Value: [ NAME ]
#
"name": "add_test_header",
# Option: type
# Type: String
# Notes: Type of the modification.
# Value: [ add_header | del_header | mod_header ]
#
"type": "add_header",
# Option: field
# Type: String
# Notes: Name of the header.
# Value: [ NAME ]
#
"field": "X-Test-Header",
# Option: value
# Type: String
# Notes: Value of the header.
# Value: [ VALUE ]
#
"value": "true"
}, {
"name": "modify_subject",
"type": "mod_header",
# Option: field
# Type: String
# Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject).
# Value: [ REGEX ]
#
"field": "^Subject$",
# Option: search
# Type: String
# Notes: Regular expression to match against the headers value.
# Values: [ VALUE ]
#
"search": "(?P<subject>.*)",
# Option: value
# Type: String
# Notes: New value of the header.
# Values: [ VALUE ]
"value": "[EXTERNAL] \\g<subject>"
}, {
"name": "delete_received_header",
"type": "del_header",
# Option: field
# Type: String
# Notes: Regular expression to match against header lines (e.g. Subject: Test-Subject).
# Value: [ REGEX ]
#
"field": "^Received$"
}, {
"name": "add_disclaimer",
"type": "add_disclaimer",
# Option: action
# Type: String
# Notes: Action to perform with the disclaimer.
# Value: [ append | prepend ]
#
"action": "prepend",
# Option: html_template
# Type: String
# Notes: Path to a file which contains the html representation of the disclaimer.
# Value: [ FILE_PATH ]
#
"html_template": "docs/templates/disclaimer_html.template",
# Option: text_template
# Type: String
# Notes: Path to a file which contains the text representation of the disclaimer.
# Value: [ FILE_PATH ]
#
"text_template": "docs/templates/disclaimer_text.template",
# Option: error_policy
# Type: String
# Notes: Set what should be done if the modification fails (e.g. no message body present).
# Value: [ wrap | ignore | reject ]
#
"error_policy": "wrap"
}, {
"name": "store_message",
"type": "store",
# Option: storage_type
# Type: String
# Notes: The storage type used to store e-mails.
# Value: [ file ]
"storage_type": "file",
# Option: directory
# Type: String
# Notes: Directory used to store e-mails.
# Value: [ file ]
"directory": "/mnt/messages"
}
]
}
]
}

5
pymodmilter/test.py Executable file
View File

@@ -0,0 +1,5 @@
#!/usr/bin/env python3
import config
a = config.ModifyMilterConfig("test.conf", debug=True)