remove surrogates from headers again, the cause troubles

MANIFEST.in

@@ -1,3 +1,3 @@
 include LICENSE README.md
-recursive-include docs *
-recursive-include misc *
+recursive-include pymodmilter/docs *
+recursive-include pymodmilter/misc *

README.md

@@ -1,6 +1,6 @@
 # pymodmilter
 A pymilter based sendmail/postfix pre-queue filter with the ability to add, remove and modify e-mail headers.  
-The project is currently in beta status, but it is already used in a productive enterprise environment which processes about a million e-mails per month.  
+The project is currently in beta status, but it is already used in a productive enterprise environment that processes about a million e-mails per month.  
 
 The basic idea is to define rules with conditions and actions which are processed when all conditions are true.
 
@@ -8,6 +8,7 @@ The basic idea is to define rules with conditions and actions which are processe
 Pymodmilter is depending on these python packages, but they are installed automatically if you are working with pip.
 * [pymilter](https://pythonhosted.org/pymilter/)
 * [netaddr](https://github.com/drkjam/netaddr/)
+* [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/)
 
 ## Installation
 * Install pymodmilter with pip and copy the example config file.
@@ -72,19 +73,19 @@ Config options for **action** objects:
   As described above in the [Global](#Global) section.
 
 Config options for **add_header** actions:
-  * **header**  
+  * **field**  
     Name of the header.
   * **value**  
     Value of the header.
 
 Config options for **del_header** actions:
-  * **header**  
+  * **field**  
     Regular expression to match against header names.
   * **value** (optional)
     Regular expression to match against the headers value.
 
 Config options for **mod_header** actions:
-  * **header**  
+  * **field**  
     Regular expression to match against header names.
   * **search** (optional)  
     Regular expression to match against header values. You may use subgroups or named subgroups (python syntax) to include parts of the original value in the new value.
@@ -96,9 +97,9 @@ Config options for **add_disclaimer** actions:
     Action to perform with the disclaimer. Possible values are:
     * append
     * prepend
-  * **html_file**  
+  * **html_template**  
     Path to a file which contains the html representation of the disclaimer.
-  * **text_file**  
+  * **text_template**  
     Path to a file which contains the text representation of the disclaimer.
   * **error_policy** (optional)  
     Set the error policy in case the disclaimer cannot be added (e.g. if no body part is present in the e-mail). Possible values are:

pymodmilter/__init__.py

@@ -14,214 +14,112 @@
 
 __all__ = [
     "actions",
+    "base",
     "conditions",
+    "rules",
     "run",
-    "CustomLogger",
-    "Rule",
+    "ModifyMilterConfig",
     "ModifyMilter"]
 
-__version__ = "1.1.1"
+__version__ = "1.1.6"
+
+from pymodmilter import _runtime_patches
 
 import Milter
 import logging
-import encodings
+import re
+import json
 
 from Milter.utils import parse_addr
-from email.message import MIMEPart
+from collections import defaultdict
+from email.header import Header
 from email.parser import BytesFeedParser
-from email.policy import default as default_policy
+from email.policy import default as default_policy, SMTP
+from netaddr import IPNetwork, AddrFormatError
 
-from pymodmilter.conditions import Conditions
-
-########################################################
-#  monkey-patch pythons email library bug 27257,30988  #
-########################################################
-#
-# https://bugs.python.org/issue27257
-# https://bugs.python.org/issue30988
-#
-# fix: https://github.com/python/cpython/pull/15600
-
-import email._header_value_parser
-from email._header_value_parser import TokenList, NameAddr
-from email._header_value_parser import get_display_name, get_angle_addr
-from email._header_value_parser import get_cfws, errors
-from email._header_value_parser import CFWS_LEADER, PHRASE_ENDS
+from pymodmilter.base import CustomLogger, BaseConfig, MilterMessage
+from pymodmilter.base import replace_illegal_chars
+from pymodmilter.rules import RuleConfig, Rule
 
 
-class DisplayName(email._header_value_parser.DisplayName):
-    @property
-    def display_name(self):
-        res = TokenList(self)
-        if len(res) == 0:
-            return res.value
-        if res[0].token_type == 'cfws':
-            res.pop(0)
-        else:
-            if isinstance(res[0], TokenList) and \
-                    res[0][0].token_type == 'cfws':
-                res[0] = TokenList(res[0][1:])
-        if res[-1].token_type == 'cfws':
-            res.pop()
-        else:
-            if isinstance(res[-1], TokenList) and \
-                    res[-1][-1].token_type == 'cfws':
-                res[-1] = TokenList(res[-1][:-1])
-        return res.value
+class ModifyMilterConfig(BaseConfig):
+    def __init__(self, cfgfile, debug=False):
+        try:
+            with open(cfgfile, "r") as fh:
+                # remove lines with leading # (comments), they
+                # are not allowed in json
+                cfg = re.sub(r"(?m)^\s*#.*\n?", "", fh.read())
+        except IOError as e:
+            raise RuntimeError(f"unable to open/read config file: {e}")
 
+        try:
+            cfg = json.loads(cfg)
+        except json.JSONDecodeError as e:
+            cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())]
+            msg = "\n".join(cfg_text)
+            raise RuntimeError(f"{e}\n{msg}")
 
-def get_name_addr(value):
-    """ name-addr = [display-name] angle-addr
+        if "global" in cfg:
+            assert isinstance(cfg["global"], dict), \
+                "global: invalid type, should be dict"
 
-    """
-    name_addr = NameAddr()
-    # Both the optional display name and the angle-addr can start with cfws.
-    leader = None
-    if value[0] in CFWS_LEADER:
-        leader, value = get_cfws(value)
-        if not value:
-            raise errors.HeaderParseError(
-                "expected name-addr but found '{}'".format(leader))
-    if value[0] != '<':
-        if value[0] in PHRASE_ENDS:
-            raise errors.HeaderParseError(
-                "expected name-addr but found '{}'".format(value))
-        token, value = get_display_name(value)
-        if not value:
-            raise errors.HeaderParseError(
-                "expected name-addr but found '{}'".format(token))
-        if leader is not None:
-            if isinstance(token[0], TokenList):
-                token[0][:0] = [leader]
+            cfg["global"]["name"] = "global"
+            super().__init__(cfg["global"], debug)
+
+            self.logger.debug("initialize config")
+
+            if "pretend" in cfg["global"]:
+                pretend = cfg["global"]["pretend"]
+                assert isinstance(pretend, bool), \
+                    "global: pretend: invalid value, should be bool"
+                self["pretend"] = pretend
             else:
-                token[:0] = [leader]
-            leader = None
-        name_addr.append(token)
-    token, value = get_angle_addr(value)
-    if leader is not None:
-        token[:0] = [leader]
-    name_addr.append(token)
-    return name_addr, value
+                self["pretend"] = False
 
-
-setattr(email._header_value_parser, "DisplayName", DisplayName)
-setattr(email._header_value_parser, "get_name_addr", get_name_addr)
-
-################################################
-#  add charset alias for windows-874 encoding  #
-################################################
-
-for alias in ["windows-874", "windows_874"]:
-    if alias not in encodings.aliases.aliases:
-        encodings.aliases.aliases[alias] = "cp874"
-
-################################################
-
-
-class CustomLogger(logging.LoggerAdapter):
-    def process(self, msg, kwargs):
-        if "name" in self.extra:
-            msg = "{}: {}".format(self.extra["name"], msg)
-
-        if "qid" in self.extra:
-            msg = "{}: {}".format(self.extra["qid"], msg)
-
-        if self.logger.getEffectiveLevel() != logging.DEBUG:
-            msg = msg.replace("\n", "").replace("\r", "")
-
-        return msg, kwargs
-
-
-class Rule:
-    """
-    Rule to implement multiple actions on emails.
-    """
-
-    def __init__(self, name, local_addrs, conditions, actions, pretend=False,
-                 loglevel=logging.INFO):
-        logger = logging.getLogger(name)
-        self.logger = CustomLogger(logger, {"name": name})
-        self.logger.setLevel(loglevel)
-
-        if logger is None:
-            logger = logging.getLogger(__name__)
-
-        self.logger = CustomLogger(logger, {"name": name})
-        self.conditions = Conditions(
-            local_addrs=local_addrs,
-            args=conditions,
-            logger=self.logger)
-        self.actions = actions
-        self.pretend = pretend
-
-        self._need_body = False
-        for action in actions:
-            if action.need_body():
-                self._need_body = True
-                break
-
-    def need_body(self):
-        """Return the if this rule needs the message body."""
-        return self._need_body
-
-    def ignores(self, host=None, envfrom=None, envto=None):
-        args = {}
-
-        if host is not None:
-            args["host"] = host
-
-        if envfrom is not None:
-            args["envfrom"] = envfrom
-
-        if envto is not None:
-            args["envto"] = envto
-
-        if self.conditions.match(args):
-            for action in self.actions:
-                if action.conditions.match(args):
-                    return False
-
-        return True
-
-    def execute(self, milter, pretend=None):
-        """Execute all actions of this rule."""
-        if pretend is None:
-            pretend = self.pretend
-
-        for action in self.actions:
-            milter_action = action.execute(milter, pretend=pretend)
-            if milter_action is not None:
-                return milter_action
-
-
-class MilterMessage(MIMEPart):
-    def replace_header(self, _name, _value, occ=None):
-        _name = _name.lower()
-        counter = 0
-        for i, (k, v) in zip(range(len(self._headers)), self._headers):
-            if k.lower() == _name:
-                counter += 1
-                if not occ or counter == occ:
-                    self._headers[i] = self.policy.header_store_parse(
-                        k, _value)
-                    break
-
-        else:
-            raise KeyError(_name)
-
-    def remove_header(self, name, occ=None):
-        name = name.lower()
-        newheaders = []
-        counter = 0
-        for k, v in self._headers:
-            if k.lower() == name:
-                counter += 1
-                if counter != occ:
-                    newheaders.append((k, v))
+            if "socket" in cfg["global"]:
+                socket = cfg["global"]["socket"]
+                assert isinstance(socket, str), \
+                    "global: socket: invalid value, should be string"
+                self["socket"] = socket
             else:
-                newheaders.append((k, v))
+                self["socket"] = None
 
-        self._headers = newheaders
+            if "local_addrs" in cfg["global"]:
+                local_addrs = cfg["global"]["local_addrs"]
+                assert isinstance(local_addrs, list) and all(
+                    [isinstance(addr, str) for addr in local_addrs]), \
+                    "global: local_addrs: invalid value, " \
+                    "should be list of strings"
+            else:
+                local_addrs = [
+                    "::1/128",
+                    "127.0.0.0/8",
+                    "10.0.0.0/8",
+                    "172.16.0.0/12",
+                    "192.168.0.0/16"]
+
+            self["local_addrs"] = []
+            try:
+                for addr in local_addrs:
+                    self["local_addrs"].append(IPNetwork(addr))
+            except AddrFormatError as e:
+                raise ValueError(f"{self['name']}: local_addrs: {e}")
+
+            self.logger.debug(f"socket={self['socket']}, "
+                              f"local_addrs={self['local_addrs']}, "
+                              f"pretend={self['pretend']}, "
+                              f"loglevel={self['loglevel']}")
+
+        assert "rules" in cfg, \
+            "mandatory parameter 'rules' not found"
+        assert isinstance(cfg["rules"], list), \
+            "rules: invalid value, should be list"
+
+        self.logger.debug("initialize rules config")
+        self["rules"] = []
+        for idx, rule_cfg in enumerate(cfg["rules"]):
+            self["rules"].append(
+                RuleConfig(idx, self, rule_cfg, debug))
 
 
 class ModifyMilter(Milter.Base):
@@ -231,11 +129,11 @@ class ModifyMilter(Milter.Base):
     _loglevel = logging.INFO
 
     @staticmethod
-    def set_rules(rules):
-        ModifyMilter._rules = rules
-
-    def set_loglevel(level):
-        ModifyMilter._loglevel = level
+    def set_config(cfg):
+        ModifyMilter._loglevel = cfg["loglevel"]
+        for rule_cfg in cfg["rules"]:
+            ModifyMilter._rules.append(
+                Rule(cfg, rule_cfg))
 
     def __init__(self):
         self.logger = logging.getLogger(__name__)
@@ -244,6 +142,47 @@ class ModifyMilter(Milter.Base):
         # save rules, it must not change during runtime
         self.rules = ModifyMilter._rules.copy()
 
+        self.msg = None
+        self._replace_body = False
+
+    def addheader(self, field, value, idx=-1):
+        value = replace_illegal_chars(Header(s=value).encode())
+        self.logger.debug(f"milter: addheader: {field}: {value}")
+        super().addheader(field, value, idx)
+
+    def chgheader(self, field, value, idx=1):
+        value = replace_illegal_chars(Header(s=value).encode())
+        if value:
+            self.logger.debug(f"milter: chgheader: {field}[{idx}]: {value}")
+        else:
+            self.logger.debug(f"milter: delheader: {field}[{idx}]")
+        super().chgheader(field, idx, value)
+
+    def update_headers(self, old_headers):
+        if self.msg.is_multipart() and not self.msg["MIME-Version"]:
+            self.msg.add_header("MIME-Version", "1.0")
+
+        # serialize the message object so it updates its internal strucure
+        self.msg.as_bytes()
+
+        old_headers = [(f, f.lower(), v) for f, v in old_headers]
+        headers = [(f, f.lower(), v) for f, v in self.msg.items()]
+
+        idx = defaultdict(int)
+        for field, field_lower, value in old_headers:
+            idx[field_lower] += 1
+            if (field, field_lower, value) not in headers:
+                self.chgheader(field, "", idx=idx[field_lower])
+                idx[field] -= 1
+
+        for field, value in self.msg.items():
+            field_lower = field.lower()
+            if (field, field_lower, value) not in old_headers:
+                self.addheader(field, value)
+
+    def replacebody(self):
+        self._replace_body = True
+
     def connect(self, IPname, family, hostaddr):
         try:
             if hostaddr is None:
@@ -292,7 +231,6 @@ class ModifyMilter(Milter.Base):
 
         return Milter.CONTINUE
 
-    @Milter.noreply
     def envrcpt(self, to, *str):
         try:
             self.recipients.add("@".join(parse_addr(to)).lower())
@@ -376,19 +314,30 @@ class ModifyMilter(Milter.Base):
     def eom(self):
         try:
             self.msg = self._fp.close()
+            milter_action = None
             for rule in self.rules:
                 milter_action = rule.execute(self)
 
                 if milter_action is not None:
-                    if milter_action["action"] == "reject":
-                        self.setreply("554", "5.7.0", milter_action["reason"])
-                        return Milter.REJECT
+                    break
 
-                    if milter_action["action"] == "accept":
-                        return Milter.ACCEPT
+            if self._replace_body:
+                data = self.msg.as_bytes(policy=SMTP)
+                body_pos = data.find(b"\r\n\r\n") + 4
+                self.logger.debug("milter: replacebody")
+                super().replacebody(data[body_pos:])
+                del data
 
-                    if milter_action["action"] == "discard":
-                        return Milter.DISCARD
+            if milter_action is not None:
+                if milter_action["action"] == "reject":
+                    self.setreply("554", "5.7.0", milter_action["reason"])
+                    return Milter.REJECT
+
+                if milter_action["action"] == "accept":
+                    return Milter.ACCEPT
+
+                if milter_action["action"] == "discard":
+                    return Milter.DISCARD
 
         except Exception as e:
             self.logger.exception(

pymodmilter/_runtime_patches.py

@@ -0,0 +1,169 @@
+# PyMod-Milter is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# PyMod-Milter is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PyMod-Milter.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import encodings
+
+
+#####################################
+#    patch pythons email library    #
+#####################################
+#
+# https://bugs.python.org/issue27257
+# https://bugs.python.org/issue30988
+#
+# fix: https://github.com/python/cpython/pull/15600
+
+import email._header_value_parser
+from email._header_value_parser import TokenList, NameAddr
+from email._header_value_parser import get_display_name, get_angle_addr
+from email._header_value_parser import get_cfws, errors
+from email._header_value_parser import CFWS_LEADER, PHRASE_ENDS
+
+
+class DisplayName(email._header_value_parser.DisplayName):
+    @property
+    def display_name(self):
+        res = TokenList(self)
+        if len(res) == 0:
+            return res.value
+        if res[0].token_type == 'cfws':
+            res.pop(0)
+        else:
+            if isinstance(res[0], TokenList) and \
+                    res[0][0].token_type == 'cfws':
+                res[0] = TokenList(res[0][1:])
+        if res[-1].token_type == 'cfws':
+            res.pop()
+        else:
+            if isinstance(res[-1], TokenList) and \
+                    res[-1][-1].token_type == 'cfws':
+                res[-1] = TokenList(res[-1][:-1])
+        return res.value
+
+
+def get_name_addr(value):
+    """ name-addr = [display-name] angle-addr
+
+    """
+    name_addr = NameAddr()
+    # Both the optional display name and the angle-addr can start with cfws.
+    leader = None
+    if value[0] in CFWS_LEADER:
+        leader, value = get_cfws(value)
+        if not value:
+            raise errors.HeaderParseError(
+                "expected name-addr but found '{}'".format(leader))
+    if value[0] != '<':
+        if value[0] in PHRASE_ENDS:
+            raise errors.HeaderParseError(
+                "expected name-addr but found '{}'".format(value))
+        token, value = get_display_name(value)
+        if not value:
+            raise errors.HeaderParseError(
+                "expected name-addr but found '{}'".format(token))
+        if leader is not None:
+            if isinstance(token[0], TokenList):
+                token[0][:0] = [leader]
+            else:
+                token[:0] = [leader]
+            leader = None
+        name_addr.append(token)
+    token, value = get_angle_addr(value)
+    if leader is not None:
+        token[:0] = [leader]
+    name_addr.append(token)
+    return name_addr, value
+
+
+setattr(email._header_value_parser, "DisplayName", DisplayName)
+setattr(email._header_value_parser, "get_name_addr", get_name_addr)
+
+
+# https://bugs.python.org/issue30681
+#
+# fix: https://github.com/python/cpython/pull/2254
+
+import email.errors
+from email.errors import HeaderDefect
+
+
+class InvalidDateDefect(HeaderDefect):
+    """Header has unparseable or invalid date"""
+
+
+setattr(email.errors, "InvalidDateDefect", InvalidDateDefect)
+
+
+import email.utils
+from email.utils import _parsedate_tz
+import datetime
+
+
+def parsedate_to_datetime(data):
+    parsed_date_tz = _parsedate_tz(data)
+    if parsed_date_tz is None:
+        raise ValueError('Invalid date value or format "%s"' % str(data))
+    *dtuple, tz = parsed_date_tz
+    if tz is None:
+        return datetime.datetime(*dtuple[:6])
+    return datetime.datetime(*dtuple[:6],
+            tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
+
+
+setattr(email.utils, "parsedate_to_datetime", parsedate_to_datetime)
+
+
+import email.headerregistry
+from email import utils, _header_value_parser as parser
+
+@classmethod
+def parse(cls, value, kwds):
+    if not value:
+        kwds['defects'].append(errors.HeaderMissingRequiredValue())
+        kwds['datetime'] = None
+        kwds['decoded'] = ''
+        kwds['parse_tree'] = parser.TokenList()
+        return
+    if isinstance(value, str):
+        kwds['decoded'] = value
+        try:
+            value = utils.parsedate_to_datetime(value)
+        except ValueError:
+            kwds['defects'].append(errors.InvalidDateDefect('Invalid date value or format'))
+            kwds['datetime'] = None
+            kwds['parse_tree'] = parser.TokenList()
+            return
+    kwds['datetime'] = value
+    kwds['decoded'] = utils.format_datetime(kwds['datetime'])
+    kwds['parse_tree'] = cls.value_parser(kwds['decoded'])
+
+
+setattr(email.headerregistry.DateHeader, "parse", parse)
+
+
+#######################################
+#  add charset alias for windows-874  #
+#######################################
+#
+# https://bugs.python.org/issue17254
+#
+# fix: https://github.com/python/cpython/pull/10237
+
+aliases = encodings.aliases.aliases
+
+for alias in ["windows-874", "windows_874"]:
+    if alias not in aliases:
+        aliases[alias] = "cp874"
+
+setattr(encodings.aliases, "aliases", aliases)

pymodmilter/actions.py

@@ -12,34 +12,30 @@
 # along with PyMod-Milter.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+__all__ = [
+    "add_header",
+    "mod_header",
+    "del_header",
+    "add_disclaimer",
+    "rewrite_links",
+    "store",
+    "ActionConfig",
+    "Action"]
+
 import logging
 import os
 import re
 
+from base64 import b64encode
 from bs4 import BeautifulSoup
 from collections import defaultdict
 from copy import copy
 from datetime import datetime
-from email.header import Header
 from email.message import MIMEPart
-from email.policy import SMTP
 
-from pymodmilter import CustomLogger, Conditions
-
-
-def _replace_illegal_chars(string):
-    """Replace illegal characters in header values."""
-    return string.replace(
-        "\x00", "").replace(
-        "\r", "").replace(
-        "\n", "")
-
-
-def _add_header(milter, field, value, idx=-1):
-    value = _replace_illegal_chars(
-        Header(s=value).encode())
-    milter.logger.debug(f"milter: addheader: {field}: {value}")
-    milter.addheader(field, value, idx)
+from pymodmilter import CustomLogger, BaseConfig
+from pymodmilter.conditions import ConditionsConfig, Conditions
+from pymodmilter import replace_illegal_chars
 
 
 def add_header(milter, field, value, pretend=False,
@@ -51,20 +47,10 @@ def add_header(milter, field, value, pretend=False,
     else:
         logger.info(f"add_header: {header[0:70]}")
 
-    milter.msg.add_header(field, _replace_illegal_chars(value))
+    milter.msg.add_header(field, replace_illegal_chars(value))
 
-    if pretend:
-        return
-
-    _add_header(milter, field, value)
-
-
-def _mod_header(milter, field, value, occ=1):
-    value = _replace_illegal_chars(
-        Header(s=value).encode())
-    milter.logger.debug(
-        f"milter: chgheader: {field}[{occ}]: {value}")
-    milter.chgheader(field, occ, value)
+    if not pretend:
+        milter.addheader(field, value)
 
 
 def mod_header(milter, field, value, search=None, pretend=False,
@@ -76,11 +62,11 @@ def mod_header(milter, field, value, search=None, pretend=False,
     if isinstance(search, str):
         search = re.compile(search, re.MULTILINE + re.DOTALL + re.IGNORECASE)
 
-    occ = defaultdict(int)
+    idx = defaultdict(int)
 
     for i, (f, v) in enumerate(milter.msg.items()):
         f_lower = f.lower()
-        occ[f_lower] += 1
+        idx[f_lower] += 1
 
         if not field.match(f):
             continue
@@ -109,18 +95,10 @@ def mod_header(milter, field, value, search=None, pretend=False,
             logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}")
 
         milter.msg.replace_header(
-            f, _replace_illegal_chars(new_value), occ=occ[f_lower])
+            f, replace_illegal_chars(new_value), idx=idx[f_lower])
 
-        if pretend:
-            continue
-
-        _mod_header(milter, f, new_value, occ=occ[f_lower])
-
-
-def _del_header(milter, field, occ=1):
-    milter.logger.debug(
-        f"milter: delheader: {field}[{occ}]")
-    milter.chgheader(field, occ, "")
+        if not pretend:
+            milter.chgheader(f, new_value, idx=idx[f_lower])
 
 
 def del_header(milter, field, value=None, pretend=False,
@@ -132,11 +110,11 @@ def del_header(milter, field, value=None, pretend=False,
     if isinstance(value, str):
         value = re.compile(value, re.MULTILINE + re.DOTALL + re.IGNORECASE)
 
-    occ = defaultdict(int)
+    idx = defaultdict(int)
 
     for f, v in milter.msg.items():
         f_lower = f.lower()
-        occ[f_lower] += 1
+        idx[f_lower] += 1
 
         if not field.match(f):
             continue
@@ -149,32 +127,12 @@ def del_header(milter, field, value=None, pretend=False,
             logger.debug(f"del_header: {header}")
         else:
             logger.info(f"del_header: {header[0:70]}")
-        milter.msg.remove_header(f, occ=occ[f_lower])
+        milter.msg.remove_header(f, idx=idx[f_lower])
 
         if not pretend:
-            _del_header(milter, f, occ=occ[f_lower])
+            milter.chgheader(f, "", idx=idx[f_lower])
 
-        occ[f_lower] -= 1
-
-
-def _serialize_msg(msg, logger):
-    if msg.is_multipart() and not msg["MIME-Version"]:
-        msg.add_header("MIME-Version", "1.0")
-
-    try:
-        logger.debug("serialize message as bytes")
-        data = msg.as_bytes(policy=SMTP)
-    except Exception as e:
-        logger.waring(
-            f"unable to serialize message as bytes: {e}")
-        try:
-            logger.warning("try to serialize message as string")
-            data = msg.as_string(policy=SMTP)
-            data = data.encode("ascii", errors="replace")
-        except Exception as e:
-            raise e
-
-    return data
+        idx[f_lower] -= 1
 
 
 def _get_body_content(msg, pref):
@@ -202,7 +160,7 @@ def _has_content_before_body_tag(soup):
     return False
 
 
-def _patch_message_body(milter, action, text, html, logger):
+def _patch_message_body(milter, action, text_template, html_template, logger):
     text_body, text_content = _get_body_content(milter.msg, "plain")
     html_body, html_content = _get_body_content(milter.msg, "html")
 
@@ -213,9 +171,9 @@ def _patch_message_body(milter, action, text, html, logger):
         logger.info(f"{action} text disclaimer")
 
         if action == "prepend":
-            content = f"{text}{text_content}"
+            content = f"{text_template}{text_content}"
         else:
-            content = f"{text_content}{text}"
+            content = f"{text_content}{text_template}"
 
         text_body.set_content(
             content.encode(), maintype="text", subtype="plain")
@@ -234,9 +192,9 @@ def _patch_message_body(milter, action, text, html, logger):
             body = soup
 
         if action == "prepend":
-            body.insert(0, copy(html))
+            body.insert(0, copy(html_template))
         else:
-            body.append(html)
+            body.append(html_template)
 
         html_body.set_content(
             str(soup).encode(), maintype="text", subtype="html")
@@ -244,37 +202,6 @@ def _patch_message_body(milter, action, text, html, logger):
         del html_body["MIME-Version"]
 
 
-def _update_body(milter, logger):
-    data = _serialize_msg(milter.msg, logger)
-    body_pos = data.find(b"\r\n\r\n") + 4
-    logger.debug("milter: replacebody")
-    milter.replacebody(data[body_pos:])
-    del data
-
-
-def _update_headers(milter, original_headers, logger):
-    if milter.msg.is_multipart() and not milter.msg["MIME-Version"]:
-        milter.msg.add_header("MIME-Version", "1.0")
-
-    # serialize the message object so it updates its internal strucure
-    milter.msg.as_bytes()
-
-    original_headers = [(f, f.lower(), v) for f, v in original_headers]
-    headers = [(f, f.lower(), v) for f, v in milter.msg.items()]
-
-    occ = defaultdict(int)
-    for field, field_lower, value in original_headers:
-        occ[field_lower] += 1
-        if (field, field_lower, value) not in headers:
-            _del_header(milter, field, occ=occ[field_lower])
-            occ[field] -= 1
-
-    for field, value in milter.msg.items():
-        field_lower = field.lower()
-        if (field, field_lower, value) not in original_headers:
-            _add_header(milter, field, value)
-
-
 def _wrap_message(milter, logger):
     attachment = MIMEPart()
     attachment.set_content(milter.msg.as_bytes(),
@@ -312,26 +239,28 @@ def _inject_body(milter):
         milter.msg.attach(attachment)
 
 
-def add_disclaimer(milter, text, html, action, policy, pretend=False,
-                   logger=logging.getLogger(__name__)):
+def add_disclaimer(milter, text_template, html_template, action, error_policy,
+                   pretend=False, logger=logging.getLogger(__name__)):
     """Append or prepend a disclaimer to the mail body."""
-    original_headers = milter.msg.items()
+    old_headers = milter.msg.items()
 
     try:
         try:
-            _patch_message_body(milter, action, text, html, logger)
+            _patch_message_body(
+                milter, action, text_template, html_template, logger)
         except RuntimeError as e:
             logger.info(f"{e}, inject empty plain and html body")
             _inject_body(milter)
-            _patch_message_body(milter, action, text, html, logger)
+            _patch_message_body(
+                milter, action, text_template, html_template, logger)
     except Exception as e:
         logger.warning(e)
-        if policy == "ignore":
+        if error_policy == "ignore":
             logger.info(
                 "unable to add disclaimer to message body, "
                 "ignore error according to policy")
             return
-        elif policy == "reject":
+        elif error_policy == "reject":
             logger.info(
                 "unable to add disclaimer to message body, "
                 "reject message according to policy")
@@ -341,18 +270,51 @@ def add_disclaimer(milter, text, html, action, policy, pretend=False,
         logger.info("wrap original message in a new message envelope")
         try:
             _wrap_message(milter, logger)
-            _patch_message_body(milter, action, text, html, logger)
+            _patch_message_body(
+                milter, action, text_template, html_template, logger)
         except Exception as e:
             logger.error(e)
             raise Exception(
                 "unable to wrap message in a new message envelope, "
                 "give up ...")
 
-    if pretend:
-        return
+    if not pretend:
+        milter.update_headers(old_headers)
+        milter.replacebody()
 
-    _update_headers(milter, original_headers, logger)
-    _update_body(milter, logger)
+
+def rewrite_links(milter, repl, pretend=False,
+                  logger=logging.getLogger(__name__)):
+    """Rewrite link targets in the mail html body."""
+
+    html_body, html_content = _get_body_content(milter.msg, "html")
+    if html_content is not None:
+        soup = BeautifulSoup(html_content, "html.parser")
+
+        rewritten = 0
+        for link in soup.find_all("a", href=True):
+            if not link["href"]:
+                continue
+
+            if "{URL_B64}" in repl:
+                url_b64 = b64encode(link["href"].encode()).decode()
+                target = repl.replace("{URL_B64}", url_b64)
+            else:
+                target = repl
+
+            link["href"] = target
+            rewritten += 1
+
+        if rewritten:
+            logger.info(f"rewrote {rewritten} link(s) in html body")
+
+            html_body.set_content(
+                str(soup).encode(), maintype="text", subtype="html")
+            html_body.set_param("charset", "UTF-8", header="Content-Type")
+            del html_body["MIME-Version"]
+
+            if not pretend:
+                milter.replacebody()
 
 
 def store(milter, directory, pretend=False,
@@ -369,102 +331,180 @@ def store(milter, directory, pretend=False,
         raise RuntimeError(f"unable to store message: {e}")
 
 
-class Action:
-    """Action to implement a pre-configured action to perform on e-mails."""
-    _need_body_map = {
-        "add_header": False,
-        "del_header": False,
-        "mod_header": False,
-        "add_disclaimer": True,
-        "store": True}
+class ActionConfig(BaseConfig):
+    def __init__(self, idx, rule_cfg, cfg, debug):
+        if "name" in cfg:
+            assert isinstance(cfg["name"], str), \
+                f"{rule_cfg['name']}: Action #{idx}: name: invalid value, " \
+                f"should be string"
+            cfg["name"] = f"{rule_cfg['name']}: {cfg['name']}"
+        else:
+            cfg["name"] = f"{rule_cfg['name']}: Action #{idx}"
 
-    def __init__(self, name, local_addrs, conditions, action_type, args,
-                 loglevel=logging.INFO, pretend=False):
-        logger = logging.getLogger(name)
-        self.logger = CustomLogger(logger, {"name": name})
-        self.logger.setLevel(loglevel)
+        if "loglevel" not in cfg:
+            cfg["loglevel"] = rule_cfg["loglevel"]
 
-        self.conditions = Conditions(
-            local_addrs=local_addrs,
-            args=conditions,
-            logger=self.logger)
-        self.pretend = pretend
-        self._args = {}
+        super().__init__(cfg, debug)
 
-        if action_type not in self._need_body_map:
-            raise RuntimeError(f"invalid action type '{action_type}'")
-        self._need_body = self._need_body_map[action_type]
+        self["pretend"] = rule_cfg["pretend"]
+        self["conditions"] = None
+        self["type"] = ""
 
-        try:
-            if action_type == "add_header":
-                self._func = add_header
-                self._args["field"] = args["header"]
-                self._args["value"] = args["value"]
-                if "idx" in args:
-                    self._args["idx"] = args["idx"]
+        if "pretend" in cfg:
+            pretend = cfg["pretend"]
+            assert isinstance(pretend, bool), \
+                f"{self['name']}: pretend: invalid value, should be bool"
+            self["pretend"] = pretend
 
-            elif action_type in ["mod_header", "del_header"]:
-                args["field"] = args["header"]
-                del args["header"]
-                regex_args = ["field"]
+        assert "type" in cfg, \
+            f"{self['name']}: mandatory parameter 'type' not found"
+        assert isinstance(cfg["type"], str), \
+            f"{self['name']}: type: invalid value, should be string"
+        self["type"] = cfg["type"]
 
-                if action_type == "mod_header":
-                    self._func = mod_header
-                    self._args["value"] = args["value"]
-                    regex_args.append("search")
-                elif action_type == "del_header":
-                    self._func = del_header
-                    if "value" in args:
-                        regex_args.append("value")
+        if self["type"] == "add_header":
+            self["func"] = add_header
+            self["need_body"] = False
 
-                for arg in regex_args:
+            if "field" not in cfg and "header" in cfg:
+                cfg["field"] = cfg["header"]
+
+            self.add_string_arg(cfg, ("field", "value"))
+
+        elif self["type"] == "mod_header":
+            self["func"] = mod_header
+            self["need_body"] = False
+
+            if "field" not in cfg and "header" in cfg:
+                cfg["field"] = cfg["header"]
+
+            args = ["field", "value"]
+            if "search" in cfg:
+                args.append("search")
+
+            for arg in args:
+                self.add_string_arg(cfg, arg)
+                if arg in ("field", "search"):
                     try:
-                        self._args[arg] = re.compile(
-                            args[arg],
+                        self["args"][arg] = re.compile(
+                            self["args"][arg],
                             re.MULTILINE + re.DOTALL + re.IGNORECASE)
                     except re.error as e:
-                        raise RuntimeError(
-                            f"unable to parse {arg} regex: {e}")
+                        raise ValueError(f"{self['name']}: {arg}: {e}")
 
-            elif action_type == "add_disclaimer":
-                self._func = add_disclaimer
-                if args["action"] not in ["append", "prepend"]:
-                    raise RuntimeError(f"invalid action '{args['action']}'")
+        elif self["type"] == "del_header":
+            self["func"] = del_header
+            self["need_body"] = False
 
-                self._args["action"] = args["action"]
+            if "field" not in cfg and "header" in cfg:
+                cfg["field"] = cfg["header"]
 
-                if args["error_policy"] not in ["wrap", "ignore", "reject"]:
-                    raise RuntimeError(f"invalid policy '{args['policy']}'")
-
-                self._args["policy"] = args["error_policy"]
+            args = ["field"]
+            if "value" in cfg:
+                args.append("value")
 
+            for arg in args:
+                self.add_string_arg(cfg, arg)
                 try:
-                    with open(args["html_file"], "r") as f:
-                        html = BeautifulSoup(
-                            f.read(), "html.parser")
-                        body = html.find('body')
-                        if body:
-                            # just use content within the body tag if present
-                            html = body
-                        self._args["html"] = html
-                    with open(args["text_file"], "r") as f:
-                        self._args["text"] = f.read()
-                except IOError as e:
-                    raise RuntimeError(f"unable to read template: {e}")
-            elif action_type == "store":
-                self._func = store
-                if args["storage_type"] not in ["file"]:
-                    raise RuntimeError(
-                        "invalid storage_type 'args['storage_type']'")
+                    self["args"][arg] = re.compile(
+                        self["args"][arg],
+                        re.MULTILINE + re.DOTALL + re.IGNORECASE)
+                except re.error as e:
+                    raise ValueError(f"{self['name']}: {arg}: {e}")
 
-                if args["storage_type"] == "file":
-                    self._args["directory"] = args["directory"]
+        elif self["type"] == "add_disclaimer":
+            self["func"] = add_disclaimer
+            self["need_body"] = True
+
+            if "html_template" not in cfg and "html_file" in cfg:
+                cfg["html_template"] = cfg["html_file"]
+
+            if "text_template" not in cfg and "text_file" in cfg:
+                cfg["text_template"] = cfg["text_file"]
+
+            if "error_policy" not in cfg:
+                cfg["error_policy"] = "wrap"
+
+            self.add_string_arg(
+                cfg, ("action", "html_template", "text_template",
+                      "error_policy"))
+            assert self["args"]["action"] in ("append", "prepend"), \
+                f"{self['name']}: action: invalid value, " \
+                f"should be 'append' or 'prepend'"
+            assert self["args"]["error_policy"] in ("wrap",
+                                                    "ignore",
+                                                    "reject"), \
+                f"{self['name']}: error_policy: invalid value, " \
+                f"should be 'wrap', 'ignore' or 'reject'"
+
+            try:
+                with open(self["args"]["html_template"], "r") as f:
+                    html = BeautifulSoup(f.read(), "html.parser")
+                    body = html.find('body')
+                    if body:
+                        # just use content within the body tag if present
+                        html = body
+                    self["args"]["html_template"] = html
+
+                with open(self["args"]["text_template"], "r") as f:
+                    self["args"]["text_template"] = f.read()
+
+            except IOError as e:
+                raise RuntimeError(
+                    f"{self['name']}: unable to open/read template file: {e}")
+
+        elif self["type"] == "rewrite_links":
+            self["func"] = rewrite_links
+            self["need_body"] = True
+            self.add_string_arg(cfg, "repl")
+
+        elif self["type"] == "store":
+            self["func"] = store
+            self["need_body"] = True
+
+            assert "storage_type" in cfg, \
+                f"{self['name']}: mandatory parameter 'storage_type' not found"
+            assert isinstance(cfg["type"], str), \
+                f"{self['name']}: storage_type: invalid value, " \
+                f"should be string"
+            self["storage_type"] = cfg["storage_type"]
+            if self["storage_type"] == "file":
+                self.add_string_arg(cfg, "directory")
             else:
-                raise RuntimeError(f"invalid action type: {action_type}")
+                raise RuntimeError(
+                    f"{self['name']}: storage_type: invalid storage type")
 
-        except KeyError as e:
-            raise RuntimeError(
-                f"mandatory argument not found: {e}")
+        else:
+            raise RuntimeError(f"{self['name']}: type: invalid action type")
+
+        if "conditions" in cfg:
+            conditions = cfg["conditions"]
+            assert isinstance(conditions, dict), \
+                f"{self['name']}: conditions: invalid value, should be dict"
+            self["conditions"] = ConditionsConfig(self, conditions, debug)
+
+        self.logger.debug(f"pretend={self['pretend']}, "
+                          f"loglevel={self['loglevel']}, "
+                          f"type={self['type']}, "
+                          f"args={self['args']}")
+
+
+class Action:
+    """Action to implement a pre-configured action to perform on e-mails."""
+
+    def __init__(self, milter_cfg, cfg):
+        self.logger = cfg.logger
+
+        if cfg["conditions"] is None:
+            self.conditions = None
+        else:
+            self.conditions = Conditions(milter_cfg, cfg["conditions"])
+
+        self.pretend = cfg["pretend"]
+        self._name = cfg["name"]
+        self._func = cfg["func"]
+        self._args = cfg["args"]
+        self._need_body = cfg["need_body"]
 
     def need_body(self):
         """Return the needs of this action."""
@@ -475,7 +515,8 @@ class Action:
         if pretend is None:
             pretend = self.pretend
 
-        logger = CustomLogger(self.logger, {"qid": milter.qid})
+        logger = CustomLogger(
+            self.logger, {"name": self._name, "qid": milter.qid})
 
         return self._func(milter=milter, pretend=pretend,
                           logger=logger, **self._args)

pymodmilter/base.py

@@ -0,0 +1,140 @@
+# PyMod-Milter is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# PyMod-Milter is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PyMod-Milter.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = [
+    "CustomLogger",
+    "BaseConfig",
+    "MilterMessage",
+    "replace_illegal_chars"]
+
+import logging
+
+from email.message import MIMEPart
+
+
+class CustomLogger(logging.LoggerAdapter):
+    def process(self, msg, kwargs):
+        if "name" in self.extra:
+            msg = f"{self.extra['name']}: {msg}"
+
+        if "qid" in self.extra:
+            msg = f"{self.extra['qid']}: {msg}"
+
+        if self.logger.getEffectiveLevel() != logging.DEBUG:
+            msg = msg.replace("\n", "").replace("\r", "")
+
+        return msg, kwargs
+
+
+class BaseConfig:
+    def __init__(self, cfg={}, debug=False, logger=None):
+        self._cfg = {}
+        if "name" in cfg:
+            assert isinstance(cfg["name"], str), \
+                "rule: name: invalid value, should be string"
+            self["name"] = cfg["name"]
+        else:
+            self["name"] = __name__
+
+        if debug:
+            self["loglevel"] = logging.DEBUG
+        elif "loglevel" in cfg:
+            if isinstance(cfg["loglevel"], int):
+                self["loglevel"] = cfg["loglevel"]
+            else:
+                level = getattr(logging, cfg["loglevel"].upper(), None)
+                assert isinstance(level, int), \
+                    f"{self['name']}: loglevel: invalid value"
+                self["loglevel"] = level
+        else:
+            self["loglevel"] = logging.INFO
+
+        if logger is None:
+            logger = logging.getLogger(self["name"])
+            logger.setLevel(self["loglevel"])
+
+        self.logger = logger
+
+        # the keys/values of args are used as parameters
+        # to functions
+        self["args"] = {}
+
+    def __setitem__(self, key, value):
+        self._cfg[key] = value
+
+    def __getitem__(self, key):
+        return self._cfg[key]
+
+    def __delitem__(self, key):
+        del self._cfg[key]
+
+    def __contains__(self, key):
+        return key in self._cfg
+
+    def add_string_arg(self, cfg, args):
+        if isinstance(args, str):
+            args = [args]
+
+        for arg in args:
+            assert arg in cfg, \
+                f"{self['name']}: mandatory parameter '{arg}' not found"
+            assert isinstance(cfg[arg], str), \
+                f"{self['name']}: {arg}: invalid value, should be string"
+            self["args"][arg] = cfg[arg]
+
+    def add_bool_arg(self, cfg, args):
+        if isinstance(args, str):
+            args = [args]
+
+        for arg in args:
+            assert arg in cfg, \
+                f"{self['name']}: mandatory parameter '{arg}' not found"
+            assert isinstance(cfg[arg], bool), \
+                f"{self['name']}: {arg}: invalid value, should be bool"
+            self["args"][arg] = cfg[arg]
+
+
+class MilterMessage(MIMEPart):
+    def replace_header(self, _name, _value, idx=None):
+        _name = _name.lower()
+        counter = 0
+        for i, (k, v) in zip(range(len(self._headers)), self._headers):
+            if k.lower() == _name:
+                counter += 1
+                if not idx or counter == idx:
+                    self._headers[i] = self.policy.header_store_parse(
+                        k, _value)
+                    break
+
+        else:
+            raise KeyError(_name)
+
+    def remove_header(self, name, idx=None):
+        name = name.lower()
+        newheaders = []
+        counter = 0
+        for k, v in self._headers:
+            if k.lower() == name:
+                counter += 1
+                if counter != idx:
+                    newheaders.append((k, v))
+            else:
+                newheaders.append((k, v))
+
+        self._headers = newheaders
+
+
+def replace_illegal_chars(string):
+    """Remove illegal characters from header values."""
+    return "".join(string.replace("\x00", "").splitlines())

pymodmilter/conditions.py

@@ -12,62 +12,65 @@
 # along with PyMod-Milter.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-import logging
+__all__ = [
+    "ConditionsConfig",
+    "Conditions"]
+
 import re
 
 from netaddr import IPAddress, IPNetwork, AddrFormatError
+from pymodmilter import BaseConfig
+
+
+class ConditionsConfig(BaseConfig):
+    def __init__(self, parent_cfg, cfg, debug):
+        if "loglevel" not in cfg:
+            cfg["loglevel"] = parent_cfg["loglevel"]
+
+        cfg["name"] = f"{parent_cfg['name']}: condition"
+
+        super().__init__(cfg, debug)
+
+        if "local" in cfg:
+            self.add_bool_arg(cfg, "local")
+
+        if "hosts" in cfg:
+            hosts = cfg["hosts"]
+            assert isinstance(hosts, list) and all(
+                [isinstance(host, str) for host in hosts]), \
+                f"{self['name']}: hosts: invalid value, " \
+                f"should be list of strings"
+
+            self["args"]["hosts"] = []
+            try:
+                for host in cfg["hosts"]:
+                    self["args"]["hosts"].append(IPNetwork(host))
+            except AddrFormatError as e:
+                raise ValueError(f"{self['name']}: hosts: {e}")
+
+        for arg in ("envfrom", "envto"):
+            if arg in cfg:
+                self.add_string_arg(cfg, arg)
+                try:
+                    self["args"][arg] = re.compile(
+                        self["args"][arg],
+                        re.IGNORECASE)
+                except re.error as e:
+                    raise ValueError(f"{self['name']}: {arg}: {e}")
+
+        self.logger.debug(f"{self['name']}: "
+                          f"loglevel={self['loglevel']}, "
+                          f"args={self['args']}")
 
 
 class Conditions:
     """Conditions to implement conditions for rules and actions."""
 
-    def __init__(self, local_addrs, args, logger=None):
-        if logger is None:
-            logger = logging.getLogger(__name__)
+    def __init__(self, milter_cfg, cfg):
+        self.logger = cfg.logger
 
-        self._local_addrs = []
-        self.logger = logger
-        self._args = {}
-
-        try:
-            for addr in local_addrs:
-                self._local_addrs.append(IPNetwork(addr))
-        except AddrFormatError as e:
-            raise RuntimeError(f"invalid address in local_addrs: {e}")
-
-        try:
-            if "local" in args:
-                logger.debug(f"condition: local = {args['local']}")
-                self._args["local"] = args["local"]
-
-            if "hosts" in args:
-                logger.debug(f"condition: hosts = {args['hosts']}")
-                self._args["hosts"] = []
-                try:
-                    for host in args["hosts"]:
-                        self._args["hosts"].append(IPNetwork(host))
-                except AddrFormatError as e:
-                    raise RuntimeError(f"invalid address in hosts: {e}")
-
-            if "envfrom" in args:
-                logger.debug(f"condition: envfrom = {args['envfrom']}")
-                try:
-                    self._args["envfrom"] = re.compile(
-                        args["envfrom"], re.IGNORECASE)
-                except re.error as e:
-                    raise RuntimeError(f"unable to parse envfrom regex: {e}")
-
-            if "envto" in args:
-                logger.debug(f"condition: envto = {args['envto']}")
-                try:
-                    self._args["envto"] = re.compile(
-                        args["envto"], re.IGNORECASE)
-                except re.error as e:
-                    raise RuntimeError(f"unable to parse envto regex: {e}")
-
-        except KeyError as e:
-            raise RuntimeError(
-                f"mandatory argument not found: {e}")
+        self._local_addrs = milter_cfg["local_addrs"]
+        self._args = cfg["args"]
 
     def match(self, args):
         if "host" in args:

pymodmilter/docs/pymodmilter.conf.example

@@ -111,12 +111,12 @@
                     #
                     "type": "add_header",
 
-                    # Option: header
+                    # Option: field
                     # Type:   String
                     # Notes:  Name of the header.
                     # Value:  [ NAME ]
                     #
-                    "header": "X-Test-Header",
+                    "field": "X-Test-Header",
 
                     # Option: value
                     # Type:   String
@@ -129,12 +129,12 @@
 
                     "type": "mod_header",
 
-                    # Option: header
+                    # Option: field
                     # Type:   String
                     # Notes:  Regular expression to match against header lines (e.g. Subject: Test-Subject).
                     # Value:  [ REGEX ]
                     #
-                    "header": "^Subject$",
+                    "field": "^Subject$",
 
                     # Option: search
                     # Type:   String
@@ -153,12 +153,12 @@
 
                     "type": "del_header",
 
-                    # Option: header
+                    # Option: field
                     # Type:   String
                     # Notes:  Regular expression to match against header lines (e.g. Subject: Test-Subject).
                     # Value:  [ REGEX ]
                     #
-                    "header": "^Received$"
+                    "field": "^Received$"
                 }, {
                     "name": "add_disclaimer",
 
@@ -171,19 +171,19 @@
                     #
                     "action": "prepend",
 
-                    # Option: html_file
+                    # Option: html_template
                     # Type:   String
                     # Notes:  Path to a file which contains the html representation of the disclaimer.
                     # Value:  [ FILE_PATH ]
                     #
-                    "html_file": "/etc/pymodmilter/templates/disclaimer_html.template",
+                    "html_template": "/etc/pymodmilter/templates/disclaimer_html.template",
 
-                    # Option: text_file
+                    # Option: text_template
                     # Type:   String
                     # Notes:  Path to a file which contains the text representation of the disclaimer.
                     # Value:  [ FILE_PATH ]
                     #
-                    "text_file": "/etc/pymodmilter/templates/disclaimer_text.template",
+                    "text_template": "/etc/pymodmilter/templates/disclaimer_text.template",
 
                     # Option: error_policy
                     # Type:   String

pymodmilter/misc/openrc/pymodmilter.initd

@@ -7,9 +7,9 @@ pidfile="/run/${RC_SVCNAME}.pid"
 command="/usr/bin/pymodmilter"
 command_args="${milter_opts}"
 command_background=true
-start_stop_daemon_args="--user ${user}"
-
+command_user="${user}"
 extra_commands="configtest"
+start_stop_daemon_args="--wait 500"
 
 depend() {
 	need net
@@ -40,7 +40,7 @@ start_pre() {
 }
 
 stop_pre() {
-	if [ "${RC_CMD}" != "restart" ]; then
+	if [ "${RC_CMD}" == "restart" ]; then
 		checkconfig || return $?
 	fi
 }

pymodmilter/rules.py

@@ -0,0 +1,122 @@
+# PyMod-Milter is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# PyMod-Milter is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PyMod-Milter.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = [
+    "RuleConfig",
+    "Rule"]
+
+from pymodmilter import BaseConfig
+from pymodmilter.actions import ActionConfig, Action
+from pymodmilter.conditions import ConditionsConfig, Conditions
+
+
+class RuleConfig(BaseConfig):
+    def __init__(self, idx, milter_cfg, cfg, debug=False):
+        if "name" in cfg:
+            assert isinstance(cfg["name"], str), \
+                f"Rule #{idx}: name: invalid value, should be string"
+        else:
+            cfg["name"] = f"Rule #{idx}"
+
+        if "loglevel" not in cfg:
+            cfg["loglevel"] = milter_cfg["loglevel"]
+
+        super().__init__(cfg, debug)
+
+        self["pretend"] = milter_cfg["pretend"]
+        self["conditions"] = None
+        self["actions"] = []
+
+        if "pretend" in cfg:
+            pretend = cfg["pretend"]
+            assert isinstance(pretend, bool), \
+                f"{self['name']}: pretend: invalid value, should be bool"
+            self["pretend"] = pretend
+
+        assert "actions" in cfg, \
+            f"{self['name']}: mandatory parameter 'actions' not found"
+        actions = cfg["actions"]
+        assert isinstance(actions, list), \
+            f"{self['name']}: actions: invalid value, should be list"
+
+        self.logger.debug(f"pretend={self['pretend']}, "
+                          f"loglevel={self['loglevel']}")
+
+        if "conditions" in cfg:
+            conditions = cfg["conditions"]
+            assert isinstance(conditions, dict), \
+                f"{self['name']}: conditions: invalid value, should be dict"
+            self["conditions"] = ConditionsConfig(self, conditions, debug)
+
+        for idx, action_cfg in enumerate(cfg["actions"]):
+            self["actions"].append(
+                ActionConfig(idx, self, action_cfg, debug))
+
+
+class Rule:
+    """
+    Rule to implement multiple actions on emails.
+    """
+
+    def __init__(self, milter_cfg, cfg):
+        self.logger = cfg.logger
+
+        if cfg["conditions"] is None:
+            self.conditions = None
+        else:
+            self.conditions = Conditions(milter_cfg, cfg["conditions"])
+
+        self._need_body = False
+
+        self.actions = []
+        for action_cfg in cfg["actions"]:
+            action = Action(milter_cfg, action_cfg)
+            self.actions.append(action)
+            if action.need_body():
+                self._need_body = True
+
+        self.pretend = cfg["pretend"]
+
+    def need_body(self):
+        """Return True if this rule needs the message body."""
+        return self._need_body
+
+    def ignores(self, host=None, envfrom=None, envto=None):
+        args = {}
+
+        if host is not None:
+            args["host"] = host
+
+        if envfrom is not None:
+            args["envfrom"] = envfrom
+
+        if envto is not None:
+            args["envto"] = envto
+
+        if self.conditions is None or self.conditions.match(args):
+            for action in self.actions:
+                if action.conditions is None or action.conditions.match(args):
+                    return False
+
+        return True
+
+    def execute(self, milter, pretend=None):
+        """Execute all actions of this rule."""
+        if pretend is None:
+            pretend = self.pretend
+
+        for action in self.actions:
+            milter_action = action.execute(milter, pretend=pretend)
+            if milter_action is not None:
+                return milter_action

pymodmilter/run.py

@@ -12,21 +12,23 @@
 # along with PyMod-Milter.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+__all__ = ["main"]
+
 import Milter
 import argparse
 import logging
 import logging.handlers
 import sys
 
-from json import loads
-from re import sub
-
-from pymodmilter import Rule, ModifyMilter
+from pymodmilter import ModifyMilter
 from pymodmilter import __version__ as version
-from pymodmilter.actions import Action
+from pymodmilter import ModifyMilterConfig
 
 
 def main():
+    python_version = ".".join([str(v) for v in sys.version_info[0:3]])
+    python_version = f"{python_version}-{sys.version_info[3]}"
+
     "Run PyMod-Milter."
     parser = argparse.ArgumentParser(
         description="PyMod milter daemon",
@@ -59,26 +61,61 @@ def main():
         "-v", "--version",
         help="Print version.",
         action="version",
-        version=f"%(prog)s ({version})")
+        version=f"%(prog)s {version} (python {python_version})")
 
     args = parser.parse_args()
 
-    loglevels = {
-        "error": logging.ERROR,
-        "warning": logging.WARNING,
-        "info": logging.INFO,
-        "debug": logging.DEBUG
-    }
-
     root_logger = logging.getLogger()
     root_logger.setLevel(logging.DEBUG)
 
     # setup console log
     stdouthandler = logging.StreamHandler(sys.stdout)
-    stdouthandler.setFormatter(
-        logging.Formatter("%(asctime)s - %(levelname)s: %(message)s"))
+    formatter = logging.Formatter("%(levelname)s: %(message)s")
+    stdouthandler.setFormatter(formatter)
     root_logger.addHandler(stdouthandler)
 
+    logger = logging.getLogger(__name__)
+
+    if not args.debug:
+        logger.setLevel(logging.INFO)
+
+    try:
+        logger.debug("prepar milter configuration")
+        cfg = ModifyMilterConfig(args.config, args.debug)
+
+        if not args.debug:
+            logger.setLevel(cfg["loglevel"])
+
+        if args.socket:
+            socket = args.socket
+        elif "socket" in cfg:
+            socket = cfg["socket"]
+        else:
+            raise RuntimeError(
+                "listening socket is neither specified on the command line "
+                "nor in the configuration file")
+
+        if not cfg["rules"]:
+            raise RuntimeError("no rules configured")
+
+        for rule_cfg in cfg["rules"]:
+            if not rule_cfg["actions"]:
+                raise RuntimeError(
+                    f"{rule_cfg['name']}: no actions configured")
+
+    except (RuntimeError, AssertionError) as e:
+        logger.error(e)
+        sys.exit(255)
+
+    if args.test:
+        print("Configuration OK")
+        sys.exit(0)
+
+    # setup console log for runtime
+    formatter = logging.Formatter("%(asctime)s - %(levelname)s: %(message)s")
+    stdouthandler.setFormatter(formatter)
+    stdouthandler.setLevel(logging.DEBUG)
+
     # setup syslog
     sysloghandler = logging.handlers.SysLogHandler(
         address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL)
@@ -86,169 +123,8 @@ def main():
         logging.Formatter("pymodmilter: %(message)s"))
     root_logger.addHandler(sysloghandler)
 
-    logger = logging.getLogger(__name__)
-
-    if not args.debug:
-        logger.setLevel(logging.INFO)
-
-    try:
-        try:
-            with open(args.config, "r") as fh:
-                config = sub(r"(?m)^\s*#.*\n?", "", fh.read())
-                config = loads(config)
-        except Exception as e:
-            for num, line in enumerate(config.splitlines()):
-                logger.error(f"{num+1}: {line}")
-            raise RuntimeError(
-                f"unable to parse config file: {e}")
-
-        if "global" not in config:
-            config["global"] = {}
-
-        if args.debug:
-            loglevel = logging.DEBUG
-        else:
-            if "loglevel" not in config["global"]:
-                config["global"]["loglevel"] = "info"
-            loglevel = loglevels[config["global"]["loglevel"]]
-
-        logger.setLevel(loglevel)
-
-        logger.debug("prepar milter configuration")
-
-        if "pretend" not in config["global"]:
-            config["global"]["pretend"] = False
-
-        if args.socket:
-            socket = args.socket
-        elif "socket" in config["global"]:
-            socket = config["global"]["socket"]
-        else:
-            raise RuntimeError(
-                "listening socket is neither specified on the command line "
-                "nor in the configuration file")
-
-        if "local_addrs" in config["global"]:
-            local_addrs = config["global"]["local_addrs"]
-        else:
-            local_addrs = [
-                "::1/128",
-                "127.0.0.0/8",
-                "10.0.0.0/8",
-                "172.16.0.0/12",
-                "192.168.0.0/16"]
-
-        if "rules" not in config:
-            raise RuntimeError(
-                "mandatory config section 'rules' not found")
-
-        if not config["rules"]:
-            raise RuntimeError("no rules configured")
-
-        logger.debug("initialize rules ...")
-
-        rules = []
-        for rule_idx, rule in enumerate(config["rules"]):
-            if "name" in rule:
-                rule_name = rule["name"]
-            else:
-                rule_name = f"Rule #{rule_idx}"
-
-            logger.debug(f"prepare rule {rule_name} ...")
-
-            if "actions" not in rule:
-                raise RuntimeError(
-                    f"{rule_name}: mandatory config "
-                    f"section 'actions' not found")
-
-            if not rule["actions"]:
-                raise RuntimeError("{rule_name}: no actions configured")
-
-            if args.debug:
-                rule_loglevel = logging.DEBUG
-            elif "loglevel" in rule:
-                rule_loglevel = loglevels[rule["loglevel"]]
-            else:
-                rule_loglevel = loglevels[config["global"]["loglevel"]]
-
-            if "pretend" in rule:
-                rule_pretend = rule["pretend"]
-            else:
-                rule_pretend = config["global"]["pretend"]
-
-            actions = []
-            for action_idx, action in enumerate(rule["actions"]):
-                if "name" in action:
-                    action_name = f"{rule_name}: {action['name']}"
-                else:
-                    action_name = f"Action #{action_idx}"
-
-                if args.debug:
-                    action_loglevel = logging.DEBUG
-                elif "loglevel" in action:
-                    action_loglevel = loglevels[action["loglevel"]]
-                else:
-                    action_loglevel = rule_loglevel
-
-                if "pretend" in action:
-                    action_pretend = action["pretend"]
-                else:
-                    action_pretend = rule_pretend
-
-                if "type" not in action:
-                    raise RuntimeError(
-                        f"{rule_name}: {action_name}: mandatory config "
-                        f"section 'actions' not found")
-
-                if "conditions" not in action:
-                    action["conditions"] = {}
-
-                try:
-                    actions.append(
-                        Action(
-                            name=action_name,
-                            local_addrs=local_addrs,
-                            conditions=action["conditions"],
-                            action_type=action["type"],
-                            args=action,
-                            loglevel=action_loglevel,
-                            pretend=action_pretend))
-                except RuntimeError as e:
-                    logger.error(f"{action_name}: {e}")
-                    sys.exit(253)
-
-            if "conditions" not in rule:
-                rule["conditions"] = {}
-
-            try:
-                rules.append(
-                    Rule(
-                        name=rule_name,
-                        local_addrs=local_addrs,
-                        conditions=rule["conditions"],
-                        actions=actions,
-                        loglevel=rule_loglevel,
-                        pretend=rule_pretend))
-            except RuntimeError as e:
-                logger.error(f"{rule_name}: {e}")
-                sys.exit(254)
-
-    except RuntimeError as e:
-        logger.error(e)
-        sys.exit(255)
-
-    if args.test:
-        print("Configuration ok")
-        sys.exit(0)
-
-    # setup console log for runtime
-    formatter = logging.Formatter("%(asctime)s - %(levelname)s: %(message)s")
-    stdouthandler.setFormatter(formatter)
-    stdouthandler.setLevel(logging.DEBUG)
-
     logger.info("pymodmilter starting")
-    ModifyMilter.set_rules(rules)
-    ModifyMilter.set_loglevel(loglevel)
+    ModifyMilter.set_config(cfg)
 
     # register milter factory class
     Milter.factory = ModifyMilter
@@ -259,10 +135,12 @@ def main():
 
     rc = 0
     try:
-        Milter.runmilter("pymodmilter", socketname=socket, timeout=30)
+        Milter.runmilter("pymodmilter", socketname=socket, timeout=600)
+        logger.info("pymodmilter stopped")
     except Milter.milter.error as e:
         logger.error(e)
         rc = 255
+
     sys.exit(rc)
 
 

setup.py

@@ -36,16 +36,16 @@ setup(name = "pymodmilter",
         (
             "/etc/pymodmilter",
             [
-                "docs/pymodmilter.conf.example"
+                "pymodmilter/docs/pymodmilter.conf.example"
             ]
         ), (
             "/etc/pymodmilter/templates",
             [
-                "docs/templates/disclaimer_html.template",
-                "docs/templates/disclaimer_text.template"
+                "pymodmilter/docs/templates/disclaimer_html.template",
+                "pymodmilter/docs/templates/disclaimer_text.template"
             ]
         )
     ],
-    install_requires = ["pymilter", "netaddr"],
+    install_requires = ["pymilter", "netaddr", "beautifulsoup4[lxml]"],
     python_requires = ">=3.6"
 )