prepare to merge with pyquarantine project

pyquarantine/__init__.py

@@ -0,0 +1,324 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = [
+    "action",
+    "base",
+    "cli",
+    "conditions",
+    "config",
+    "mailer",
+    "modify",
+    "notify",
+    "rule",
+    "run",
+    "storage",
+    "whitelist",
+    "ModifyMilter"]
+
+__version__ = "2.0.0"
+
+from pyquarantine import _runtime_patches
+
+import Milter
+import logging
+
+from Milter.utils import parse_addr
+from collections import defaultdict
+from copy import copy
+from email import message_from_binary_file
+from email.header import Header, decode_header, make_header
+from email.headerregistry import AddressHeader, _default_header_map
+from email.policy import SMTPUTF8
+from io import BytesIO
+from netaddr import IPNetwork, AddrFormatError
+
+from pyquarantine.base import CustomLogger, MilterMessage
+from pyquarantine.base import replace_illegal_chars
+from pyquarantine.rule import Rule
+
+
+class ModifyMilter(Milter.Base):
+    """ModifyMilter based on Milter.Base to implement milter communication"""
+
+    _rules = []
+    _loglevel = logging.INFO
+    _addr_fields = [f for f, v in _default_header_map.items()
+                    if issubclass(v, AddressHeader)]
+
+    @staticmethod
+    def set_config(cfg, debug):
+        ModifyMilter._loglevel = cfg.get_loglevel(debug)
+
+        try:
+            local_addrs = []
+            for addr in cfg["local_addrs"]:
+                local_addrs.append(IPNetwork(addr))
+        except AddrFormatError as e:
+            raise RuntimeError(e)
+
+        logger = logging.getLogger(__name__)
+        logger.setLevel(ModifyMilter._loglevel)
+        for idx, rule_cfg in enumerate(cfg["rules"]):
+            if "name" not in rule_cfg:
+                rule_cfg["name"] = f"rule#{idx}"
+            if "loglevel" not in rule_cfg:
+                rule_cfg["loglevel"] = cfg["loglevel"]
+            if "pretend" not in rule_cfg:
+                rule_cfg["pretend"] = cfg["pretend"]
+            rule = Rule(rule_cfg, local_addrs, debug)
+
+            logger.debug(rule)
+            ModifyMilter._rules.append(rule)
+
+    def __init__(self):
+        self.logger = logging.getLogger(__name__)
+        self.logger.setLevel(ModifyMilter._loglevel)
+
+    def addheader(self, field, value, idx=-1):
+        value = replace_illegal_chars(Header(s=value).encode())
+        self.logger.debug(f"addheader: {field}: {value}")
+        super().addheader(field, value, idx)
+
+    def chgheader(self, field, value, idx=1):
+        value = replace_illegal_chars(Header(s=value).encode())
+        if value:
+            self.logger.debug(f"chgheader: {field}[{idx}]: {value}")
+        else:
+            self.logger.debug(f"delheader: {field}[{idx}]")
+        super().chgheader(field, idx, value)
+
+    def update_headers(self, old_headers):
+        if self.msg.is_multipart() and not self.msg["MIME-Version"]:
+            self.msg.add_header("MIME-Version", "1.0")
+
+        # serialize the message object so it updates its internal strucure
+        self.msg.as_bytes()
+
+        old_headers = [(f, f.lower(), v) for f, v in old_headers]
+        headers = [(f, f.lower(), v) for f, v in self.msg.items()]
+
+        idx = defaultdict(int)
+        for field, field_lower, value in old_headers:
+            idx[field_lower] += 1
+            if (field, field_lower, value) not in headers:
+                self.chgheader(field, "", idx=idx[field_lower])
+                idx[field] -= 1
+
+        for field, value in self.msg.items():
+            field_lower = field.lower()
+            if (field, field_lower, value) not in old_headers:
+                self.addheader(field, value)
+
+    def replacebody(self):
+        self._body_changed = True
+
+    def _replacebody(self):
+        if not self._body_changed:
+            return
+        data = self.msg.as_bytes()
+        body_pos = data.find(b"\r\n\r\n") + 4
+        self.logger.debug("replace body")
+        super().replacebody(data[body_pos:])
+        del data
+
+    def delrcpt(self, rcpts):
+        "Remove recipient. May be called from eom callback only."
+        if not isinstance(rcpts, list):
+            rcpts = [rcpts]
+        for rcpt in rcpts:
+            self.logger.debug(f"delrcpt: {rcpt}")
+            self.msginfo["rcpts"].remove(rcpt)
+            super().delrcpt(rcpt)
+
+    def connect(self, IPname, family, hostaddr):
+        try:
+            if hostaddr is None:
+                self.logger.error(f"received invalid host address {hostaddr}, "
+                                  f"unable to proceed")
+                return Milter.TEMPFAIL
+
+            self.IP = hostaddr[0]
+            self.port = hostaddr[1]
+            self.logger.debug(
+                f"accepted milter connection from {self.IP} "
+                f"port {self.port}")
+
+            # pre-filter rules and actions by the host condition
+            # also check if the mail body is needed by any upcoming action.
+            self.rules = []
+            self._headersonly = True
+            for rule in ModifyMilter._rules:
+                if rule.conditions is None or \
+                        rule.conditions.match_host(self.IP):
+                    actions = []
+                    for action in rule.actions:
+                        if action.conditions is None or \
+                                action.conditions.match_host(self.IP):
+                            actions.append(action)
+                            if not action.headersonly():
+                                self._headersonly = False
+
+                    if actions:
+                        # copy needed rules to preserve configured actions
+                        rule = copy(rule)
+                        rule.actions = actions
+                        self.rules.append(rule)
+
+            if not self.rules:
+                self.logger.debug(
+                    "host is ignored by all rules, skip further processing")
+                return Milter.ACCEPT
+
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in connect method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.CONTINUE
+
+    def hello(self, heloname):
+        try:
+            self.heloname = heloname
+            self.logger.debug(f"received HELO name: {heloname}")
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in hello method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.CONTINUE
+
+    def envfrom(self, mailfrom, *str):
+        try:
+            self.mailfrom = "@".join(parse_addr(mailfrom)).lower()
+            self.rcpts = set()
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in envfrom method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.CONTINUE
+
+    def envrcpt(self, to, *str):
+        try:
+            self.rcpts.add("@".join(parse_addr(to)).lower())
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in envrcpt method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.CONTINUE
+
+    def data(self):
+        try:
+            self.qid = self.getsymval('i')
+            self.logger = CustomLogger(
+                self.logger, {"qid": self.qid, "name": "milter"})
+            self.logger.debug("received queue-id from MTA")
+            self.fp = BytesIO()
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in data method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.CONTINUE
+
+    def header(self, field, value):
+        try:
+            # remove CR and LF from address fields, otherwise pythons
+            # email library throws an exception
+            if field.lower() in ModifyMilter._addr_fields:
+                try:
+                    v = str(make_header(decode_header(value)))
+                except Exception as e:
+                    self.logger.error(
+                        f"unable to decode field '{field}': {e}")
+                else:
+                    if any(c in v for c in ["\r", "\n"]):
+                        v = v.replace("\r", "").replace("\n", "")
+                        value = Header(s=v).encode()
+
+            # remove surrogates
+            field = field.encode("ascii", errors="replace")
+            value = value.encode("ascii", errors="replace")
+
+            self.fp.write(field + b": " + value + b"\r\n")
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in header method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.CONTINUE
+
+    def eoh(self):
+        try:
+            self.fp.write(b"\r\n")
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in eoh method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.CONTINUE
+
+    def body(self, chunk):
+        try:
+            if not self._headersonly:
+                self.fp.write(chunk)
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in body method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.CONTINUE
+
+    def eom(self):
+        try:
+            # msg and msginfo contain the runtime data that
+            # is read/modified by actions
+            self.fp.seek(0)
+            self.msg = message_from_binary_file(
+                self.fp, _class=MilterMessage, policy=SMTPUTF8.clone(
+                    refold_source='none'))
+            self.msginfo = {
+                "mailfrom": self.mailfrom,
+                "rcpts": [*self.rcpts],
+                "vars": {}}
+
+            self._body_changed = False
+            milter_action = None
+            for rule in self.rules:
+                milter_action = rule.execute(self)
+                self.logger.debug(
+                    f"current template variables: {self.msginfo['vars']}")
+                if milter_action is not None:
+                    break
+
+            if milter_action is None:
+                self._replacebody()
+            else:
+                action, reason = milter_action
+                if action == "ACCEPT":
+                    self._replacebody()
+                elif action == "REJECT":
+                    self.setreply("554", "5.7.0", reason)
+                    return Milter.REJECT
+                elif action == "DISCARD":
+                    return Milter.DISCARD
+
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in eom method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.ACCEPT

pyquarantine/_runtime_patches.py

@@ -0,0 +1,229 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import encodings
+
+
+#####################################
+#    patch pythons email library    #
+#####################################
+#
+# https://bugs.python.org/issue27257
+# https://bugs.python.org/issue30988
+#
+# fix: https://github.com/python/cpython/pull/15600
+
+import email._header_value_parser
+from email._header_value_parser import TokenList, NameAddr
+from email._header_value_parser import get_display_name, get_angle_addr
+from email._header_value_parser import get_cfws, errors
+from email._header_value_parser import CFWS_LEADER, PHRASE_ENDS
+
+
+class DisplayName(email._header_value_parser.DisplayName):
+    @property
+    def display_name(self):
+        res = TokenList(self)
+        if len(res) == 0:
+            return res.value
+        if res[0].token_type == 'cfws':
+            res.pop(0)
+        else:
+            if isinstance(res[0], TokenList) and \
+                    res[0][0].token_type == 'cfws':
+                res[0] = TokenList(res[0][1:])
+        if res[-1].token_type == 'cfws':
+            res.pop()
+        else:
+            if isinstance(res[-1], TokenList) and \
+                    res[-1][-1].token_type == 'cfws':
+                res[-1] = TokenList(res[-1][:-1])
+        return res.value
+
+
+def get_name_addr(value):
+    """ name-addr = [display-name] angle-addr
+
+    """
+    name_addr = NameAddr()
+    # Both the optional display name and the angle-addr can start with cfws.
+    leader = None
+    if value[0] in CFWS_LEADER:
+        leader, value = get_cfws(value)
+        if not value:
+            raise errors.HeaderParseError(
+                "expected name-addr but found '{}'".format(leader))
+    if value[0] != '<':
+        if value[0] in PHRASE_ENDS:
+            raise errors.HeaderParseError(
+                "expected name-addr but found '{}'".format(value))
+        token, value = get_display_name(value)
+        if not value:
+            raise errors.HeaderParseError(
+                "expected name-addr but found '{}'".format(token))
+        if leader is not None:
+            if isinstance(token[0], TokenList):
+                token[0][:0] = [leader]
+            else:
+                token[:0] = [leader]
+            leader = None
+        name_addr.append(token)
+    token, value = get_angle_addr(value)
+    if leader is not None:
+        token[:0] = [leader]
+    name_addr.append(token)
+    return name_addr, value
+
+
+setattr(email._header_value_parser, "DisplayName", DisplayName)
+setattr(email._header_value_parser, "get_name_addr", get_name_addr)
+
+
+# https://bugs.python.org/issue42484
+#
+# fix: https://github.com/python/cpython/pull/24669
+
+from email._header_value_parser import DOT, ObsLocalPart, ValueTerminal, get_word
+
+
+def get_obs_local_part(value):
+    """ obs-local-part = word *("." word)
+    """
+    obs_local_part = ObsLocalPart()
+    last_non_ws_was_dot = False
+    while value and (value[0]=='\\' or value[0] not in PHRASE_ENDS):
+        if value[0] == '.':
+            if last_non_ws_was_dot:
+                obs_local_part.defects.append(errors.InvalidHeaderDefect(
+                    "invalid repeated '.'"))
+            obs_local_part.append(DOT)
+            last_non_ws_was_dot = True
+            value = value[1:]
+            continue
+        elif value[0]=='\\':
+            obs_local_part.append(ValueTerminal(value[0],
+                                                'misplaced-special'))
+            value = value[1:]
+            obs_local_part.defects.append(errors.InvalidHeaderDefect(
+                "'\\' character outside of quoted-string/ccontent"))
+            last_non_ws_was_dot = False
+            continue
+        if obs_local_part and obs_local_part[-1].token_type != 'dot':
+            obs_local_part.defects.append(errors.InvalidHeaderDefect(
+                "missing '.' between words"))
+        try:
+            token, value = get_word(value)
+            last_non_ws_was_dot = False
+        except errors.HeaderParseError:
+            if value[0] not in CFWS_LEADER:
+                raise
+            token, value = get_cfws(value)
+        obs_local_part.append(token)
+    if not obs_local_part:
+        return obs_local_part, value
+    if (obs_local_part[0].token_type == 'dot' or
+            obs_local_part[0].token_type=='cfws' and
+            obs_local_part[1].token_type=='dot'):
+        obs_local_part.defects.append(errors.InvalidHeaderDefect(
+            "Invalid leading '.' in local part"))
+    if (obs_local_part[-1].token_type == 'dot' or
+            obs_local_part[-1].token_type=='cfws' and
+            obs_local_part[-2].token_type=='dot'):
+        obs_local_part.defects.append(errors.InvalidHeaderDefect(
+            "Invalid trailing '.' in local part"))
+    if obs_local_part.defects:
+        obs_local_part.token_type = 'invalid-obs-local-part'
+    return obs_local_part, value
+
+
+setattr(email._header_value_parser, "get_obs_local_part", get_obs_local_part)
+
+
+# https://bugs.python.org/issue30681
+#
+# fix: https://github.com/python/cpython/pull/22090
+
+import email.errors
+from email.errors import HeaderDefect
+
+
+class InvalidDateDefect(HeaderDefect):
+    """Header has unparseable or invalid date"""
+
+
+setattr(email.errors, "InvalidDateDefect", InvalidDateDefect)
+
+
+import email.utils
+from email.utils import _parsedate_tz
+import datetime
+
+
+def parsedate_to_datetime(data):
+    parsed_date_tz = _parsedate_tz(data)
+    if parsed_date_tz is None:
+        raise ValueError('Invalid date value or format "%s"' % str(data))
+    *dtuple, tz = parsed_date_tz
+    if tz is None:
+        return datetime.datetime(*dtuple[:6])
+    return datetime.datetime(*dtuple[:6],
+            tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
+
+
+setattr(email.utils, "parsedate_to_datetime", parsedate_to_datetime)
+
+
+import email.headerregistry
+from email import utils, _header_value_parser as parser
+
+@classmethod
+def parse(cls, value, kwds):
+    if not value:
+        kwds['defects'].append(errors.HeaderMissingRequiredValue())
+        kwds['datetime'] = None
+        kwds['decoded'] = ''
+        kwds['parse_tree'] = parser.TokenList()
+        return
+    if isinstance(value, str):
+        kwds['decoded'] = value
+        try:
+            value = utils.parsedate_to_datetime(value)
+        except ValueError:
+            kwds['defects'].append(errors.InvalidDateDefect('Invalid date value or format'))
+            kwds['datetime'] = None
+            kwds['parse_tree'] = parser.TokenList()
+            return
+    kwds['datetime'] = value
+    kwds['decoded'] = utils.format_datetime(kwds['datetime'])
+    kwds['parse_tree'] = cls.value_parser(kwds['decoded'])
+
+
+setattr(email.headerregistry.DateHeader, "parse", parse)
+
+
+#######################################
+#  add charset alias for windows-874  #
+#######################################
+#
+# https://bugs.python.org/issue17254
+#
+# fix: https://github.com/python/cpython/pull/10237
+
+aliases = encodings.aliases.aliases
+
+for alias in ["windows-874", "windows_874"]:
+    if alias not in aliases:
+        aliases[alias] = "cp874"
+
+setattr(encodings.aliases, "aliases", aliases)

pyquarantine/action.py

@@ -0,0 +1,64 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = ["Action"]
+
+from pyquarantine import modify, notify, storage
+from pyquarantine.conditions import Conditions
+
+
+class Action:
+    """Action to implement a pre-configured action to perform on e-mails."""
+    ACTION_TYPES = {
+        "add_header": modify.Modify,
+        "mod_header": modify.Modify,
+        "del_header": modify.Modify,
+        "add_disclaimer": modify.Modify,
+        "rewrite_links": modify.Modify,
+        "store": storage.Store,
+        "notify": notify.Notify,
+        "quarantine": storage.Quarantine}
+
+    def __init__(self, cfg, local_addrs, debug):
+        self.cfg = cfg
+
+        self.conditions = cfg["conditions"] if "conditions" in cfg else None
+        if self.conditions is not None:
+            self.conditions["name"] = f"{cfg['name']}: conditions"
+            self.conditions["loglevel"] = cfg["loglevel"]
+            self.conditions = Conditions(self.conditions, local_addrs, debug)
+
+        action_type = cfg["type"]
+        self.action = self.ACTION_TYPES[action_type](
+            cfg, local_addrs, debug)
+
+    def __str__(self):
+        cfg = []
+        for key in ["name", "loglevel", "pretend", "type"]:
+            value = self.cfg[key]
+            cfg.append(f"{key}={value}")
+        if self.conditions is not None:
+            cfg.append(f"conditions={self.conditions}")
+        cfg.append(f"action={self.action}")
+        return "Action(" + ", ".join(cfg) + ")"
+
+    def headersonly(self):
+        """Return the needs of this action."""
+        return self.action._headersonly
+
+    def execute(self, milter):
+        """Execute configured action."""
+        if self.conditions is None or \
+                self.conditions.match(milter):
+            return self.action.execute(milter)

pyquarantine/base.py

@@ -0,0 +1,71 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = [
+    "CustomLogger",
+    "MilterMessage",
+    "replace_illegal_chars"]
+
+import logging
+
+from email.message import MIMEPart
+
+
+class CustomLogger(logging.LoggerAdapter):
+    def process(self, msg, kwargs):
+        if "name" in self.extra:
+            msg = f"{self.extra['name']}: {msg}"
+
+        if "qid" in self.extra:
+            msg = f"{self.extra['qid']}: {msg}"
+
+        if self.logger.getEffectiveLevel() != logging.DEBUG:
+            msg = msg.replace("\n", "").replace("\r", "")
+
+        return msg, kwargs
+
+
+class MilterMessage(MIMEPart):
+    def replace_header(self, _name, _value, idx=None):
+        _name = _name.lower()
+        counter = 0
+        for i, (k, v) in zip(range(len(self._headers)), self._headers):
+            if k.lower() == _name:
+                counter += 1
+                if not idx or counter == idx:
+                    self._headers[i] = self.policy.header_store_parse(
+                        k, _value)
+                    break
+
+        else:
+            raise KeyError(_name)
+
+    def remove_header(self, name, idx=None):
+        name = name.lower()
+        newheaders = []
+        counter = 0
+        for k, v in self._headers:
+            if k.lower() == name:
+                counter += 1
+                if counter != idx:
+                    newheaders.append((k, v))
+            else:
+                newheaders.append((k, v))
+
+        self._headers = newheaders
+
+
+def replace_illegal_chars(string):
+    """Remove illegal characters from header values."""
+    return "".join(string.replace("\x00", "").splitlines())

pyquarantine/cli.py

@@ -0,0 +1,628 @@
+#!/usr/bin/env python
+#
+# PyQuarantine-Milter is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# PyQuarantine-Milter is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PyQuarantineMilter.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import argparse
+import logging
+import logging.handlers
+import sys
+import time
+
+from pyquarantine.config import get_milter_config
+from pyquarantine import __version__ as version
+
+
+def _get_quarantine(quarantines, name):
+    try:
+        quarantine = next((q for q in quarantines if q.name == name))
+    except StopIteration:
+        raise RuntimeError("invalid quarantine 'name'")
+    return quarantine
+
+
+def _get_storage(quarantines, name):
+    quarantine = _get_quarantine(quarantines, name)
+    storage = quarantine.get_storage()
+    if not storage:
+        raise RuntimeError(
+                "storage type is set to NONE")
+    return storage
+
+
+def _get_notification(quarantines, name):
+    quarantine = _get_quarantine(quarantines, name)
+    notification = quarantine.get_notification()
+    if not notification:
+        raise RuntimeError(
+                "notification type is set to NONE")
+    return notification
+
+
+def _get_whitelist(quarantines, name):
+    quarantine = _get_quarantine(quarantines, name)
+    whitelist = quarantine.get_whitelist()
+    if not whitelist:
+        raise RuntimeError(
+                "whitelist type is set to NONE")
+    return whitelist
+
+
+def print_table(columns, rows):
+    if not rows:
+        return
+
+    column_lengths = []
+    column_formats = []
+
+    # iterate columns to display
+    for header, key in columns:
+        # get the length of the header string
+        lengths = [len(header)]
+        # get the length of the longest value
+        lengths.append(
+            len(str(max(rows, key=lambda x: len(str(x[key])))[key])))
+        # use the longer one
+        length = max(lengths)
+        column_lengths.append(length)
+        column_formats.append(f"{{:<{length}}}")
+
+    # define row format
+    row_format = " | ".join(column_formats)
+
+    # define header/body separator
+    separators = []
+    for length in column_lengths:
+        separators.append("-" * length)
+    separator = "-+-".join(separators)
+
+    # print header and separator
+    print(row_format.format(*[column[0] for column in columns]))
+    print(separator)
+
+    keys = [entry[1] for entry in columns]
+    # print rows
+    for entry in rows:
+        row = []
+        for key in keys:
+            row.append(entry[key])
+        print(row_format.format(*row))
+
+
+def list_quarantines(quarantines, args):
+    if args.batch:
+        print("\n".join([q.name for q in quarantines]))
+    else:
+        qlist = []
+        for q in quarantines:
+            storage = q.get_storage()
+            if storage:
+                storage_type = q.get_storage().storage_type
+            else:
+                storage_type = "NONE"
+
+            notification = q.get_notification()
+            if notification:
+                notification_type = q.get_notification().notification_type
+            else:
+                notification_type = "NONE"
+
+            whitelist = q.get_whitelist()
+            if whitelist:
+                whitelist_type = q.get_whitelist().whitelist_type
+            else:
+                whitelist_type = "NONE"
+
+            qlist.append({
+                "name": q.name,
+                "storage": storage_type,
+                "notification": notification_type,
+                "whitelist": whitelist_type,
+                "action": q.action})
+        print_table(
+            [("Name", "name"),
+             ("Storage", "storage"),
+             ("Notification", "notification"),
+             ("Whitelist", "whitelist"),
+             ("Action", "action")],
+            qlist
+        )
+
+
+def list_quarantine_emails(quarantines, args):
+    logger = logging.getLogger(__name__)
+    storage = _get_storage(quarantines, args.quarantine)
+    # find emails and transform some metadata values to strings
+    rows = []
+    emails = storage.find(
+        args.mailfrom, args.recipients, args.older_than)
+    for storage_id, metadata in emails.items():
+        row = emails[storage_id]
+        row["storage_id"] = storage_id
+        row["date"] = time.strftime(
+            '%Y-%m-%d %H:%M:%S',
+            time.localtime(
+                metadata["date"]))
+        row["mailfrom"] = metadata["mailfrom"]
+        row["recipient"] = metadata["recipients"].pop(0)
+        if "subject" not in emails[storage_id]["headers"].keys():
+            emails[storage_id]["headers"]["subject"] = ""
+        row["subject"] = emails[storage_id]["headers"]["subject"][:60].strip()
+        rows.append(row)
+
+        if metadata["recipients"]:
+            row = {
+                "storage_id": "",
+                "date": "",
+                "mailfrom": "",
+                "recipient": metadata["recipients"].pop(0),
+                "subject": ""
+            }
+            rows.append(row)
+
+    if args.batch:
+        # batch mode, print quarantine IDs, each on a new line
+        print("\n".join(emails.keys()))
+        return
+
+    if not emails:
+        logger.info(f"quarantine '{args.quarantine}' is empty")
+    print_table(
+        [("Quarantine-ID", "storage_id"), ("Date", "date"),
+         ("From", "mailfrom"), ("Recipient(s)", "recipient"),
+         ("Subject", "subject")],
+        rows
+    )
+
+
+def list_whitelist(quarantines, args):
+    logger = logging.getLogger(__name__)
+    whitelist = _get_whitelist(quarantines, args.quarantine)
+
+    # find whitelist entries
+    entries = whitelist.find(
+        mailfrom=args.mailfrom,
+        recipients=args.recipients,
+        older_than=args.older_than)
+    if not entries:
+        logger.info(
+            f"whitelist of quarantine '{args.quarantine}' is empty")
+        return
+
+    # transform some values to strings
+    for entry_id, entry in entries.items():
+        entries[entry_id]["permanent_str"] = str(entry["permanent"])
+        entries[entry_id]["created_str"] = entry["created"].strftime(
+            '%Y-%m-%d %H:%M:%S')
+        entries[entry_id]["last_used_str"] = entry["last_used"].strftime(
+            '%Y-%m-%d %H:%M:%S')
+
+    print_table(
+        [
+            ("ID", "id"), ("From", "mailfrom"), ("To", "recipient"),
+            ("Created", "created_str"), ("Last used", "last_used_str"),
+            ("Comment", "comment"), ("Permanent", "permanent_str")
+        ],
+        entries.values()
+    )
+
+
+def add_whitelist_entry(quarantines, args):
+    logger = logging.getLogger(__name__)
+    whitelist = _get_whitelist(quarantines, args.quarantine)
+
+    # check existing entries
+    entries = whitelist.check(args.mailfrom, args.recipient)
+    if entries:
+        # check if the exact entry exists already
+        for entry in entries.values():
+            if entry["mailfrom"] == args.mailfrom and \
+                    entry["recipient"] == args.recipient:
+                raise RuntimeError(
+                    "an entry with this from/to combination already exists")
+
+        if not args.force:
+            # the entry is already covered by others
+            for entry_id, entry in entries.items():
+                entries[entry_id]["permanent_str"] = str(entry["permanent"])
+                entries[entry_id]["created_str"] = entry["created"].strftime(
+                    '%Y-%m-%d %H:%M:%S')
+                entries[entry_id]["last_used_str"] = entry["last_used"].strftime(
+                    '%Y-%m-%d %H:%M:%S')
+            print_table(
+                [
+                    ("ID", "id"), ("From", "mailfrom"), ("To", "recipient"),
+                    ("Created", "created_str"), ("Last used", "last_used_str"),
+                    ("Comment", "comment"), ("Permanent", "permanent_str")
+                ],
+                entries.values()
+            )
+            print("")
+            raise RuntimeError(
+                "from/to combination is already covered by the entries above, "
+                "use --force to override.")
+
+    # add entry to whitelist
+    whitelist.add(args.mailfrom, args.recipient, args.comment, args.permanent)
+    logger.info("whitelist entry added successfully")
+
+
+def delete_whitelist_entry(quarantines, args):
+    logger = logging.getLogger(__name__)
+    whitelist = _get_whitelist(quarantines, args.quarantine)
+    whitelist.delete(args.whitelist_id)
+    logger.info("whitelist entry deleted successfully")
+
+
+def notify(quarantines, args):
+    logger = logging.getLogger(__name__)
+    quarantine = _get_quarantine(quarantines, args.quarantine)
+    quarantine.notify(args.quarantine_id, args.recipient)
+    logger.info("notification sent successfully")
+
+
+def release(quarantines, args):
+    logger = logging.getLogger(__name__)
+    quarantine = _get_quarantine(quarantines, args.quarantine)
+    quarantine.release(args.quarantine_id, args.recipient)
+    logger.info("quarantined email released successfully")
+
+
+def delete(quarantines, args):
+    logger = logging.getLogger(__name__)
+    storage = _get_storage(quarantines, args.quarantine)
+    storage.delete(args.quarantine_id, args.recipient)
+    logger.info("quarantined email deleted successfully")
+
+
+def get(quarantines, args):
+    storage = _get_storage(quarantines, args.quarantine)
+    fp, _ = storage.get_mail(args.quarantine_id)
+    print(fp.read().decode())
+    fp.close()
+
+
+class StdErrFilter(logging.Filter):
+    def filter(self, rec):
+        return rec.levelno in (logging.ERROR, logging.WARNING)
+
+
+class StdOutFilter(logging.Filter):
+    def filter(self, rec):
+        return rec.levelno in (logging.DEBUG, logging.INFO)
+
+
+def main():
+    python_version = ".".join([str(v) for v in sys.version_info[0:3]])
+    python_version = f"{python_version}-{sys.version_info[3]}"
+
+    "PyQuarantine command-line interface."
+    # parse command line
+    def formatter_class(prog): return argparse.HelpFormatter(
+        prog, max_help_position=50, width=140)
+    parser = argparse.ArgumentParser(
+        description="PyQuarantine CLI",
+        formatter_class=formatter_class)
+    parser.add_argument(
+        "-c", "--config", help="Config file to read.",
+        default="/etc/pyquarantine/pyquarantine.conf")
+    parser.add_argument(
+        "-d", "--debug",
+        help="Log debugging messages.",
+        action="store_true")
+    parser.add_argument(
+        "-v", "--version",
+        help="Print version.",
+        action="version",
+        version=f"%(prog)s {version} (python {python_version})")
+    parser.set_defaults(syslog=False)
+    subparsers = parser.add_subparsers(
+        dest="command",
+        title="Commands")
+    subparsers.required = True
+
+    # list command
+    list_parser = subparsers.add_parser(
+        "list",
+        help="List available quarantines.",
+        formatter_class=formatter_class)
+    list_parser.add_argument(
+        "-b", "--batch",
+        help="Print results using only quarantine names, each on a new line.",
+        action="store_true")
+    list_parser.set_defaults(func=list_quarantines)
+
+    # quarantine command group
+    quarantine_parser = subparsers.add_parser(
+        "quarantine",
+        description="Manage quarantines.",
+        help="Manage quarantines.",
+        formatter_class=formatter_class)
+    quarantine_parser.add_argument(
+        "quarantine",
+        metavar="QUARANTINE",
+        help="Quarantine name.")
+    quarantine_subparsers = quarantine_parser.add_subparsers(
+        dest="command",
+        title="Quarantine commands")
+    quarantine_subparsers.required = True
+    # quarantine list command
+    quarantine_list_parser = quarantine_subparsers.add_parser(
+        "list",
+        description="List emails in quarantines.",
+        help="List emails in quarantine.",
+        formatter_class=formatter_class)
+    quarantine_list_parser.add_argument(
+        "-f", "--from",
+        dest="mailfrom",
+        help="Filter emails by from address.",
+        default=None,
+        nargs="+")
+    quarantine_list_parser.add_argument(
+        "-t", "--to",
+        dest="recipients",
+        help="Filter emails by recipient address.",
+        default=None,
+        nargs="+")
+    quarantine_list_parser.add_argument(
+        "-o", "--older-than",
+        dest="older_than",
+        help="Filter emails by age (days).",
+        default=None,
+        type=float)
+    quarantine_list_parser.add_argument(
+        "-b", "--batch",
+        help="Print results using only email quarantine IDs, each on a new line.",
+        action="store_true")
+    quarantine_list_parser.set_defaults(func=list_quarantine_emails)
+    # quarantine notify command
+    quarantine_notify_parser = quarantine_subparsers.add_parser(
+        "notify",
+        description="Notify recipient about email in quarantine.",
+        help="Notify recipient about email in quarantine.",
+        formatter_class=formatter_class)
+    quarantine_notify_parser.add_argument(
+        "quarantine_id",
+        metavar="ID",
+        help="Quarantine ID.")
+    quarantine_notify_parser_group = quarantine_notify_parser.add_mutually_exclusive_group(
+        required=True)
+    quarantine_notify_parser_group.add_argument(
+        "-t", "--to",
+        dest="recipient",
+        help="Release email for one recipient address.")
+    quarantine_notify_parser_group.add_argument(
+        "-a", "--all",
+        help="Release email for all recipients.",
+        action="store_true")
+    quarantine_notify_parser.set_defaults(func=notify)
+    # quarantine release command
+    quarantine_release_parser = quarantine_subparsers.add_parser(
+        "release",
+        description="Release email from quarantine.",
+        help="Release email from quarantine.",
+        formatter_class=formatter_class)
+    quarantine_release_parser.add_argument(
+        "quarantine_id",
+        metavar="ID",
+        help="Quarantine ID.")
+    quarantine_release_parser.add_argument(
+        "-n",
+        "--disable-syslog",
+        dest="syslog",
+        help="Disable syslog messages.",
+        action="store_false")
+    quarantine_release_parser_group = quarantine_release_parser.add_mutually_exclusive_group(
+        required=True)
+    quarantine_release_parser_group.add_argument(
+        "-t", "--to",
+        dest="recipient",
+        help="Release email for one recipient address.")
+    quarantine_release_parser_group.add_argument(
+        "-a", "--all",
+        help="Release email for all recipients.",
+        action="store_true")
+    quarantine_release_parser.set_defaults(func=release)
+    # quarantine delete command
+    quarantine_delete_parser = quarantine_subparsers.add_parser(
+        "delete",
+        description="Delete email from quarantine.",
+        help="Delete email from quarantine.",
+        formatter_class=formatter_class)
+    quarantine_delete_parser.add_argument(
+        "quarantine_id",
+        metavar="ID",
+        help="Quarantine ID.")
+    quarantine_delete_parser.add_argument(
+        "-n", "--disable-syslog",
+        dest="syslog",
+        help="Disable syslog messages.",
+        action="store_false")
+    quarantine_delete_parser_group = quarantine_delete_parser.add_mutually_exclusive_group(
+        required=True)
+    quarantine_delete_parser_group.add_argument(
+        "-t", "--to",
+        dest="recipient",
+        help="Delete email for one recipient address.")
+    quarantine_delete_parser_group.add_argument(
+        "-a", "--all",
+        help="Delete email for all recipients.",
+        action="store_true")
+    quarantine_delete_parser.set_defaults(func=delete)
+    # quarantine get command
+    quarantine_get_parser = quarantine_subparsers.add_parser(
+        "get",
+        description="Get email from quarantine.",
+        help="Get email from quarantine",
+        formatter_class=formatter_class)
+    quarantine_get_parser.add_argument(
+        "quarantine_id",
+        metavar="ID",
+        help="Quarantine ID.")
+    quarantine_get_parser.set_defaults(func=get)
+
+    # whitelist command group
+    whitelist_parser = subparsers.add_parser(
+        "whitelist",
+        description="Manage whitelists.",
+        help="Manage whitelists.",
+        formatter_class=formatter_class)
+    whitelist_parser.add_argument(
+        "quarantine",
+        metavar="QUARANTINE",
+        help="Quarantine name.")
+    whitelist_subparsers = whitelist_parser.add_subparsers(
+        dest="command",
+        title="Whitelist commands")
+    whitelist_subparsers.required = True
+    # whitelist list command
+    whitelist_list_parser = whitelist_subparsers.add_parser(
+        "list",
+        description="List whitelist entries.",
+        help="List whitelist entries.",
+        formatter_class=formatter_class)
+    whitelist_list_parser.add_argument(
+        "-f", "--from",
+        dest="mailfrom",
+        help="Filter entries by from address.",
+        default=None,
+        nargs="+")
+    whitelist_list_parser.add_argument(
+        "-t", "--to",
+        dest="recipients",
+        help="Filter entries by recipient address.",
+        default=None,
+        nargs="+")
+    whitelist_list_parser.add_argument(
+        "-o", "--older-than",
+        dest="older_than",
+        help="Filter emails by last used date (days).",
+        default=None,
+        type=float)
+    whitelist_list_parser.set_defaults(func=list_whitelist)
+    # whitelist add command
+    whitelist_add_parser = whitelist_subparsers.add_parser(
+        "add",
+        description="Add whitelist entry.",
+        help="Add whitelist entry.",
+        formatter_class=formatter_class)
+    whitelist_add_parser.add_argument(
+        "-f", "--from",
+        dest="mailfrom",
+        help="From address.",
+        required=True)
+    whitelist_add_parser.add_argument(
+        "-t", "--to",
+        dest="recipient",
+        help="Recipient address.",
+        required=True)
+    whitelist_add_parser.add_argument(
+        "-c", "--comment",
+        help="Comment.",
+        default="added by CLI")
+    whitelist_add_parser.add_argument(
+        "-p", "--permanent",
+        help="Add a permanent entry.",
+        action="store_true")
+    whitelist_add_parser.add_argument(
+        "--force",
+        help="Force adding an entry, even if already covered by another entry.",
+        action="store_true")
+    whitelist_add_parser.set_defaults(func=add_whitelist_entry)
+    # whitelist delete command
+    whitelist_delete_parser = whitelist_subparsers.add_parser(
+        "delete",
+        description="Delete whitelist entry.",
+        help="Delete whitelist entry.",
+        formatter_class=formatter_class)
+    whitelist_delete_parser.add_argument(
+        "whitelist_id",
+        metavar="ID",
+        help="Whitelist ID.")
+    whitelist_delete_parser.set_defaults(func=delete_whitelist_entry)
+
+    args = parser.parse_args()
+
+    # setup logging
+    loglevel = logging.INFO
+    root_logger = logging.getLogger()
+    root_logger.setLevel(loglevel)
+
+    # setup console log
+    if args.debug:
+        formatter = logging.Formatter(
+            "%(levelname)s: [%(name)s] - %(message)s")
+    else:
+        formatter = logging.Formatter("%(levelname)s: %(message)s")
+    # stdout
+    stdouthandler = logging.StreamHandler(sys.stdout)
+    stdouthandler.setLevel(logging.DEBUG)
+    stdouthandler.setFormatter(formatter)
+    stdouthandler.addFilter(StdOutFilter())
+    root_logger.addHandler(stdouthandler)
+    # stderr
+    stderrhandler = logging.StreamHandler(sys.stderr)
+    stderrhandler.setLevel(logging.WARNING)
+    stderrhandler.setFormatter(formatter)
+    stderrhandler.addFilter(StdErrFilter())
+    root_logger.addHandler(stderrhandler)
+    logger = logging.getLogger(__name__)
+
+    try:
+        logger.debug("read milter configuration")
+        cfg = get_milter_config(args.config)
+        if not cfg["rules"]:
+            raise RuntimeError("no rules configured")
+
+        for rule in cfg["rules"]:
+            if not rule["actions"]:
+                raise RuntimeError(
+                    f"{rule['name']}: no actions configured")
+    except (RuntimeError, AssertionError) as e:
+        logger.error(f"config error: {e}")
+        sys.exit(255)
+
+    quarantines = []
+    for rule in cfg["rules"]:
+        for action in rule["actions"]:
+            if action["type"] == "quarantine":
+                quarantines.append(action)
+
+    print(quarantines)
+    sys.exit(0)
+
+    if args.syslog:
+        # setup syslog
+        sysloghandler = logging.handlers.SysLogHandler(
+            address="/dev/log",
+            facility=logging.handlers.SysLogHandler.LOG_MAIL)
+        sysloghandler.setLevel(loglevel)
+        if args.debug:
+            formatter = logging.Formatter(
+                "pyquarantine: [%(name)s] [%(levelname)s] %(message)s")
+        else:
+            formatter = logging.Formatter("pyquarantine: %(message)s")
+        sysloghandler.setFormatter(formatter)
+        root_logger.addHandler(sysloghandler)
+
+    # call the commands function
+    try:
+        args.func(cfg, args)
+    except RuntimeError as e:
+        logger.error(e)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

pyquarantine/conditions.py

@@ -0,0 +1,193 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = ["Conditions"]
+
+import logging
+import re
+
+from netaddr import IPAddress, IPNetwork, AddrFormatError
+from pyquarantine import CustomLogger
+from pyquarantine.whitelist import DatabaseWhitelist
+
+
+class Conditions:
+    """Conditions to implement conditions for rules and actions."""
+
+    def __init__(self, cfg, local_addrs, debug):
+        self.cfg = cfg
+        self.local_addrs = local_addrs
+
+        self.logger = logging.getLogger(cfg["name"])
+        self.logger.setLevel(cfg.get_loglevel(debug))
+
+        for arg in ("local", "hosts", "envfrom", "envto", "header", "metavar",
+                    "var"):
+            if arg not in cfg:
+                setattr(self, arg, None)
+                continue
+
+            if arg == "hosts":
+                try:
+                    self.hosts = []
+                    for host in cfg["hosts"]:
+                        self.hosts.append(IPNetwork(host))
+                except AddrFormatError as e:
+                    raise RuntimeError(e)
+            elif arg in ("envfrom", "envto"):
+                try:
+                    setattr(self, arg, re.compile(
+                        cfg[arg], re.IGNORECASE))
+                except re.error as e:
+                    raise RuntimeError(e)
+            elif arg == "header":
+                try:
+                    self.header = re.compile(
+                        cfg["header"],
+                        re.IGNORECASE + re.DOTALL + re.MULTILINE)
+                except re.error as e:
+                    raise RuntimeError(e)
+            else:
+                setattr(self, arg, cfg[arg])
+
+        self.whitelist = cfg["whitelist"] if "whitelist" in cfg else None
+        if self.whitelist is not None:
+            self.whitelist["name"] = f"{cfg['name']}: whitelist"
+            self.whitelist["loglevel"] = cfg["loglevel"]
+            if self.whitelist["type"] == "db":
+                self.whitelist = DatabaseWhitelist(self.whitelist, debug)
+            else:
+                raise RuntimeError("invalid whitelist type")
+
+    def __str__(self):
+        cfg = []
+        for arg in ("local", "hosts", "envfrom", "envto", "header",
+                    "var", "metavar"):
+            if arg in self.cfg:
+                cfg.append(f"{arg}={self.cfg[arg]}")
+        if self.whitelist is not None:
+            cfg.append(f"whitelist={self.whitelist}")
+        return "Conditions(" + ", ".join(cfg) + ")"
+
+    def match_host(self, host):
+        logger = CustomLogger(
+            self.logger, {"name": self.cfg["name"]})
+        ip = IPAddress(host)
+
+        if self.local is not None:
+            is_local = False
+            for addr in self.local_addrs:
+                if ip in addr:
+                    is_local = True
+                    break
+
+            if is_local != self.local:
+                logger.debug(
+                    f"ignore host {host}, "
+                    f"local does not match")
+                return False
+
+            logger.debug(
+                f"local matches for host {host}")
+
+        if self.hosts is not None:
+            found = False
+            for addr in self.hosts:
+                if ip in addr:
+                    found = True
+                    break
+
+            if not found:
+                logger.debug(
+                    f"ignore host {host}, "
+                    f"hosts does not match")
+                return False
+
+            logger.debug(
+                f"hosts matches for host {host}")
+
+        return True
+
+    def get_wl_rcpts(self, mailfrom, rcpts, logger):
+        if not self.whitelist:
+            return {}
+
+        wl_rcpts = []
+        for rcpt in rcpts:
+            if self.whitelist.check(mailfrom, rcpt, logger):
+                wl_rcpts.append(rcpt)
+
+        return wl_rcpts
+
+    def match(self, milter):
+        logger = CustomLogger(
+            self.logger, {"qid": milter.qid, "name": self.cfg["name"]})
+
+        if self.envfrom is not None:
+            envfrom = milter.msginfo["mailfrom"]
+            if not self.envfrom.match(envfrom):
+                logger.debug(
+                    f"ignore envelope-from address {envfrom}, "
+                    f"envfrom does not match")
+                return False
+
+            logger.debug(
+                f"envfrom matches for "
+                f"envelope-from address {envfrom}")
+
+        if self.envto is not None:
+            envto = milter.msginfo["rcpts"]
+            if not isinstance(envto, list):
+                envto = [envto]
+
+            for to in envto:
+                if not self.envto.match(to):
+                    logger.debug(
+                        f"ignore envelope-to address {envto}, "
+                        f"envto does not match")
+                    return False
+
+            logger.debug(
+                f"envto matches for "
+                f"envelope-to address {envto}")
+
+        if self.header is not None:
+            match = None
+            for field, value in milter.msg.items():
+                header = f"{field}: {value}"
+                match = self.header.search(header)
+                if match:
+                    logger.debug(
+                        f"header matches for "
+                        f"header: {header}")
+                    if self.metavar is not None:
+                        named_subgroups = match.groupdict(default=None)
+                        for group, value in named_subgroups.items():
+                            if value is None:
+                                continue
+                            name = f"{self.metavar}_{group}"
+                            milter.msginfo["vars"][name] = value
+                    break
+
+            if not match:
+                logger.debug(
+                    "ignore message, "
+                    "header does not match")
+                return False
+
+        if self.var is not None:
+            if self.var not in milter.msginfo["vars"]:
+                return False
+
+        return True

pyquarantine/config.py

@@ -0,0 +1,354 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = [
+    "BaseConfig",
+    "ConditionsConfig",
+    "AddHeaderConfig",
+    "ModHeaderConfig",
+    "DelHeaderConfig",
+    "AddDisclaimerConfig",
+    "RewriteLinksConfig",
+    "StoreConfig",
+    "NotifyConfig",
+    "WhitelistConfig",
+    "QuarantineConfig",
+    "ActionConfig",
+    "RuleConfig",
+    "MilterConfig",
+    "get_milter_config"]
+
+import json
+import jsonschema
+import logging
+import re
+
+
+class BaseConfig:
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": [],
+        "additionalProperties": True,
+        "properties": {
+            "loglevel": {"type": "string", "default": "info"}}}
+
+    def __init__(self, config):
+        required = self.JSON_SCHEMA["required"]
+        properties = self.JSON_SCHEMA["properties"]
+        for p in properties.keys():
+            if p in required:
+                continue
+            elif p not in config and "default" in properties[p]:
+                config[p] = properties[p]["default"]
+        try:
+            jsonschema.validate(config, self.JSON_SCHEMA)
+        except jsonschema.exceptions.ValidationError as e:
+            raise RuntimeError(e)
+
+        self._config = config
+
+    def __getitem__(self, key):
+        return self._config[key]
+
+    def __setitem__(self, key, value):
+        self._config[key] = value
+
+    def __delitem__(self, key):
+        del self._config[key]
+
+    def __contains__(self, key):
+        return key in self._config
+
+    def keys(self):
+        return self._config.keys()
+
+    def items(self):
+        return self._config.items()
+
+    def get_loglevel(self, debug):
+        if debug:
+            level = logging.DEBUG
+        else:
+            level = getattr(logging, self["loglevel"].upper(), None)
+            assert isinstance(level, int), \
+                "loglevel: invalid value"
+        return level
+
+    def get_config(self):
+        return self._config
+
+
+class WhitelistConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["type"],
+        "additionalProperties": True,
+        "properties": {
+            "type": {"enum": ["db"]}},
+        "if": {"properties": {"type": {"const": "db"}}},
+        "then": {
+            "required": ["connection", "table"],
+            "additionalProperties": False,
+            "properties": {
+                "type": {"type": "string"},
+                "connection": {"type": "string"},
+                "table": {"type": "string"}}}}
+
+
+class ConditionsConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": [],
+        "additionalProperties": False,
+        "properties": {
+            "metavar": {"type": "string"},
+            "local": {"type": "boolean"},
+            "hosts": {"type": "array",
+                      "items": {"type": "string"}},
+            "envfrom": {"type": "string"},
+            "envto": {"type": "string"},
+            "header": {"type": "string"},
+            "var": {"type": "string"},
+            "whitelist": {"type": "object"}}}
+
+    def __init__(self, config, rec=True):
+        super().__init__(config)
+        if rec:
+            if "whitelist" in self:
+                self["whitelist"] = WhitelistConfig(self["whitelist"])
+
+
+class AddHeaderConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["field", "value"],
+        "additionalProperties": False,
+        "properties": {
+            "field": {"type": "string"},
+            "value": {"type": "string"}}}
+
+
+class ModHeaderConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["field", "value"],
+        "additionalProperties": False,
+        "properties": {
+            "field": {"type": "string"},
+            "value": {"type": "string"},
+            "search": {"type": "string"}}}
+
+
+class DelHeaderConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["field"],
+        "additionalProperties": False,
+        "properties": {
+            "field": {"type": "string"},
+            "value": {"type": "string"}}}
+
+
+class AddDisclaimerConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["action", "html_template", "text_template"],
+        "additionalProperties": False,
+        "properties": {
+            "action": {"type": "string"},
+            "html_template": {"type": "string"},
+            "text_template": {"type": "string"},
+            "error_policy": {"type": "string"}}}
+
+
+class RewriteLinksConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["repl"],
+        "additionalProperties": False,
+        "properties": {
+            "repl": {"type": "string"}}}
+
+
+class StoreConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["type"],
+        "additionalProperties": True,
+        "properties": {
+            "type": {"enum": ["file"]}},
+        "if": {"properties": {"type": {"const": "file"}}},
+        "then": {
+            "required": ["directory"],
+            "additionalProperties": False,
+            "properties": {
+                "type": {"type": "string"},
+                "directory": {"type": "string"},
+                "mode": {"type": "string"},
+                "metavar": {"type": "string"},
+                "original": {"type": "boolean", "default": True}}}}
+
+
+class NotifyConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["type"],
+        "additionalProperties": True,
+        "properties": {
+            "type": {"enum": ["email"]}},
+        "if": {"properties": {"type": {"const": "email"}}},
+        "then": {
+            "required": ["smtp_host", "smtp_port", "envelope_from",
+                         "from_header", "subject", "template"],
+            "additionalProperties": False,
+            "properties": {
+                "type": {"type": "string"},
+                "smtp_host": {"type": "string"},
+                "smtp_port": {"type": "number"},
+                "envelope_from": {"type": "string"},
+                "from_header": {"type": "string"},
+                "subject": {"type": "string"},
+                "template": {"type": "string"},
+                "repl_img": {"type": "string"},
+                "embed_imgs": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "default": True}}}}
+
+
+class QuarantineConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["store"],
+        "additionalProperties": False,
+        "properties": {
+            "name": {"type": "string"},
+            "notify": {"type": "object"},
+            "milter_action": {"type": "string"},
+            "reject_reason": {"type": "string"},
+            "whitelist": {"type": "object"},
+            "store": {"type": "object"}}}
+
+    def __init__(self, config, rec=True):
+        super().__init__(config)
+        if rec:
+            self["store"] = StoreConfig(self["store"])
+            if "notify" in self:
+                self["notify"] = NotifyConfig(self["notify"])
+            if "whitelist" in self:
+                self["whitelist"] = ConditionsConfig(
+                    {"whitelist": self["whitelist"]}, rec)
+
+
+class ActionConfig(BaseConfig):
+    ACTION_TYPES = {
+        "add_header": AddHeaderConfig,
+        "mod_header": ModHeaderConfig,
+        "del_header": DelHeaderConfig,
+        "add_disclaimer": AddDisclaimerConfig,
+        "rewrite_links": RewriteLinksConfig,
+        "store": StoreConfig,
+        "notify": NotifyConfig,
+        "quarantine": QuarantineConfig}
+
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["type", "args"],
+        "additionalProperties": False,
+        "properties": {
+            "name": {"type": "string", "default": "action"},
+            "loglevel": {"type": "string", "default": "info"},
+            "pretend": {"type": "boolean", "default": False},
+            "conditions": {"type": "object"},
+            "type": {"enum": list(ACTION_TYPES.keys())},
+            "args": {"type": "object"}}}
+
+    def __init__(self, config, rec=True):
+        super().__init__(config)
+        if rec:
+            if "conditions" in self:
+                self["conditions"] = ConditionsConfig(self["conditions"])
+            self["action"] = self.ACTION_TYPES[self["type"]](self["args"])
+
+
+class RuleConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["actions"],
+        "additionalProperties": False,
+        "properties": {
+            "name": {"type": "string", "default": "rule"},
+            "loglevel": {"type": "string", "default": "info"},
+            "pretend": {"type": "boolean", "default": False},
+            "conditions": {"type": "object"},
+            "actions": {"type": "array"}}}
+
+    def __init__(self, config, rec=True):
+        super().__init__(config)
+        if rec:
+            if "conditions" in self:
+                self["conditions"] = ConditionsConfig(self["conditions"])
+
+            actions = []
+            for idx, action in enumerate(self["actions"]):
+                actions.append(ActionConfig(action, rec))
+            self["actions"] = actions
+
+
+class MilterConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["rules"],
+        "additionalProperties": False,
+        "properties": {
+            "socket": {"type": "string"},
+            "local_addrs": {"type": "array",
+                            "items": {"type": "string"},
+                            "default": [
+                                "fe80::/64",
+                                "::1/128",
+                                "127.0.0.0/8",
+                                "10.0.0.0/8",
+                                "172.16.0.0/12",
+                                "192.168.0.0/16"]},
+            "loglevel": {"type": "string", "default": "info"},
+            "pretend": {"type": "boolean", "default": False},
+            "rules": {"type": "array"}}}
+
+    def __init__(self, config, rec=True):
+        super().__init__(config)
+        if rec:
+            rules = []
+            for idx, rule in enumerate(self["rules"]):
+                rules.append(RuleConfig(rule, rec))
+            self["rules"] = rules
+
+
+def get_milter_config(cfgfile):
+    try:
+        with open(cfgfile, "r") as fh:
+            # remove lines with leading # (comments), they
+            # are not allowed in json
+            cfg = re.sub(r"(?m)^\s*#.*\n?", "", fh.read())
+    except IOError as e:
+        raise RuntimeError(f"unable to open/read config file: {e}")
+
+    try:
+        cfg = json.loads(cfg)
+    except json.JSONDecodeError as e:
+        cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())]
+        msg = "\n".join(cfg_text)
+        raise RuntimeError(f"{e}\n{msg}")
+
+    return MilterConfig(cfg)

pyquarantine/docs/pyquarantine.conf.example

@@ -0,0 +1,221 @@
+# This is an example /etc/pyquarantine/pyquarantine.conf file.
+# Copy it into place before use.
+#
+# The file is in JSON format.
+#
+# The global option 'log' can be overriden per rule or per modification.
+#
+{
+    # Section: global
+    # Notes:   Global options.
+    #
+    "global": {
+        # Option: socket
+        # Type:   String
+        # Notes:  The socket used to communicate with the MTA.
+        #
+        #         Examples:
+        #         unix:/path/to/socket             a named pipe
+        #         inet:8899                        listen on ANY interface
+        #         inet:8899@localhost              listen on a specific interface
+        #         inet6:8899                       listen on ANY interface
+        #         inet6:8899@[2001:db8:1234::1]    listen on a specific interface
+        # Value:  [ SOCKET ]
+        "socket": "inet:8898@127.0.0.1",
+
+        # Option: local_addrs
+        # Type:   List
+        # Notes:  A list of local hosts and networks.
+        # Value:  [ LIST ]
+        #
+        "local_addrs": ["fe80::/64", "::1/128", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"],
+
+        # Option: loglevel
+        # Type:   String
+        # Notes:  Set loglevel for rules and actions.
+        # Value:  [ error | warning | info | debug ]
+        #
+        "loglevel": "info",
+
+        # Option: pretend
+        # Type:   Bool
+        # Notes:  Just pretend to do the actions, for test purposes.
+        # Value:  [ true | false ]
+        #
+        "pretend": true
+    },
+
+    # Section: rules
+    # Notes:   Rules and related actions.
+    #
+    "rules": [
+        {
+            # Option: name
+            # Type:   String
+            # Notes:  Name of the rule.
+            # Value:  [ NAME ]
+            #
+            "name": "myrule",
+
+            # Section: conditions
+            # Notes:   Optional conditions to process the rule.
+            #          If multiple conditions are set, they all
+            #          have to be true to process the rule.
+            #
+            "conditions": {
+                # Option: local
+                # Type:   Bool
+                # Notes:  Condition wheter the senders host address is listed in local_addrs.
+                # Value:  [ true | false ]
+                #
+                "local": false,
+
+                # Option: hosts
+                # Type:   String
+                # Notes:  Condition wheter the senders host address is listed in this list.
+                # Value: [ LIST ]
+                #
+                "hosts": [ "127.0.0.1" ],
+
+                # Option: envfrom
+                # Type:   String
+                # Notes:  Condition wheter the envelop-from address matches this regular expression.
+                # Value:  [ REGEX ]
+                #
+                "envfrom": "^.+@mypartner\\.com$",
+
+                # Option: envto
+                # Type:   String
+                # Notes:  Condition wheter the envelop-to address matches this regular expression.
+                # Value:  [ REGEX ]
+                #
+                "envto": "^postmaster@.+$"
+            },
+
+            # Section: actions
+            # Notes:   Actions of the rule.
+            #
+            "actions": [
+                {
+                    # Option: name
+                    # Type:   String
+                    # Notes:  Name of the modification.
+                    # Value:  [ NAME ]
+                    #
+                    "name": "add_test_header",
+
+                    # Option: type
+                    # Type:   String
+                    # Notes:  Type of the modification.
+                    # Value:  [ add_header | del_header | mod_header ]
+                    #
+                    "type": "add_header",
+
+                    # Option: field
+                    # Type:   String
+                    # Notes:  Name of the header.
+                    # Value:  [ NAME ]
+                    #
+                    "field": "X-Test-Header",
+
+                    # Option: value
+                    # Type:   String
+                    # Notes:  Value of the header.
+                    # Value:  [ VALUE ]
+                    #
+                    "value": "true"
+                }, {
+                    "name": "modify_subject",
+
+                    "type": "mod_header",
+
+                    # Option: field
+                    # Type:   String
+                    # Notes:  Regular expression to match against header lines (e.g. Subject: Test-Subject).
+                    # Value:  [ REGEX ]
+                    #
+                    "field": "^Subject$",
+
+                    # Option: search
+                    # Type:   String
+                    # Notes:  Regular expression to match against the headers value.
+                    # Values: [ VALUE ]
+                    #
+                    "search": "(?P<subject>.*)",
+
+                    # Option: value
+                    # Type:   String
+                    # Notes:  New value of the header.
+                    # Values: [ VALUE ]
+                    "value": "[EXTERNAL] \\g<subject>"
+                }, {
+                    "name": "delete_received_header",
+
+                    "type": "del_header",
+
+                    # Option: field
+                    # Type:   String
+                    # Notes:  Regular expression to match against header lines (e.g. Subject: Test-Subject).
+                    # Value:  [ REGEX ]
+                    #
+                    "field": "^Received$"
+                }, {
+                    "name": "add_disclaimer",
+
+                    "type": "add_disclaimer",
+
+                    # Option: action
+                    # Type:   String
+                    # Notes:  Action to perform with the disclaimer.
+                    # Value:  [ append | prepend ]
+                    #
+                    "action": "prepend",
+
+                    # Option: html_template
+                    # Type:   String
+                    # Notes:  Path to a file which contains the html representation of the disclaimer.
+                    # Value:  [ FILE_PATH ]
+                    #
+                    "html_template": "/etc/pyquarantine/templates/disclaimer_html.template",
+
+                    # Option: text_template
+                    # Type:   String
+                    # Notes:  Path to a file which contains the text representation of the disclaimer.
+                    # Value:  [ FILE_PATH ]
+                    #
+                    "text_template": "/etc/pyquarantine/templates/disclaimer_text.template",
+
+                    # Option: error_policy
+                    # Type:   String
+                    # Notes:  Set what should be done if the modification fails (e.g. no message body present).
+                    # Value:  [ wrap | ignore | reject ]
+                    #
+                    "error_policy": "wrap"
+                }, {
+                    "name": "store_message",
+
+                    "type": "store",
+
+                    # Option: storage_type
+                    # Type:   String
+                    # Notes:  The storage type used to store e-mails.
+                    # Value: [ file ]
+                    "storage_type": "file",
+
+                    # Option: directory
+                    # Type:   String
+                    # Notes:  Directory used to store e-mails.
+                    # Value: [ file ]
+                    "directory": "/mnt/messages",
+
+                    # Option: original
+                    # Type:   Bool
+                    # Notes:  If set to true, store the message as received by the MTA instead of storing the current state
+                    #         of the message, that may was modified already by other actions.
+                    # Value: [ true | false ]
+                    "original": true
+                }
+            ]
+        }
+    ]
+}

pyquarantine/docs/templates/disclaimer_html.template

@@ -0,0 +1,9 @@
+<table style="border: 1px solid; background-color: #F8E898; border-color: #885800; font-family: Arial; font-size: 11pt;">
+ <tr>
+  <td>
+   <span style="font-weight: bold; color: #905800;">CAUTION:</span> This email originated from outside the organization.
+   Do not follow guidance, click links or open attachments unless you recognize the sender and know the content is safe.
+  </td>
+ </tr>
+</table>
+<br/><br/>

pyquarantine/docs/templates/disclaimer_text.template

@@ -0,0 +1,4 @@
+
+CAUTION: This email originated from outside the organization. Do not follow guidance, click links or open attachments unless you recognize the sender and know the content is safe.
+
+

pyquarantine/docs/templates/notification.template

@@ -0,0 +1,29 @@
+<html>
+ <body>
+  <h1>Quarantine notification</h1>
+  <table>
+   <tr>
+    <td><b>Envelope-From:</b></td>
+    <td>{ENVELOPE_FROM}</td>
+   </tr>
+   <tr>
+    <td><b>From:</b></td>
+    <td>{FROM}</td>
+   </tr>
+   <tr>
+    <td><b>Envelope-To:</b></td>
+    <td>{ENVELOPE_TO}</td>
+   </tr>
+   <tr>
+    <td><b>To:</b></td>
+    <td>{TO}</td>
+   </tr>
+   <tr>
+    <td><b>Subject:</b></td>
+    <td>{SUBJECT}</td>
+   </tr>
+  </table><br/>
+  <h2>Preview of the original e-mail</h2>
+  {HTML_TEXT}
+ </body>
+</html>

pyquarantine/mailer.py

@@ -0,0 +1,86 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import logging
+import smtplib
+
+from multiprocessing import Process, Queue
+
+
+logger = logging.getLogger(__name__)
+queue = Queue(maxsize=50)
+process = None
+
+
+def smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail):
+    s = smtplib.SMTP(host=smtp_host, port=smtp_port)
+    s.ehlo()
+    if s.has_extn("STARTTLS"):
+        s.starttls()
+        s.ehlo()
+    s.sendmail(mailfrom, [recipient], mail)
+    s.quit()
+
+
+def mailprocess():
+    "Mailer process to send emails asynchronously."
+    global logger
+    global queue
+
+    try:
+        while True:
+            m = queue.get()
+            if not m:
+                break
+
+            smtp_host, smtp_port, qid, mailfrom, recipient, mail, emailtype = m
+            try:
+                smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail)
+            except Exception as e:
+                logger.error(
+                    f"{qid}: error while sending {emailtype} "
+                    f"to '{recipient}': {e}")
+            else:
+                logger.info(
+                    f"{qid}: successfully sent {emailtype} to: {recipient}")
+    except KeyboardInterrupt:
+        pass
+    logger.debug("mailer process terminated")
+
+
+def sendmail(smtp_host, smtp_port, qid, mailfrom, recipients, mail,
+             emailtype="email"):
+    "Send an email."
+    global logger
+    global process
+    global queue
+
+    if isinstance(recipients, str):
+        recipients = [recipients]
+
+    # start mailprocess if it is not started yet
+    if process is None:
+        process = Process(target=mailprocess)
+        process.daemon = True
+        logger.debug("starting mailer process")
+        process.start()
+
+    for recipient in recipients:
+        try:
+            queue.put(
+                (smtp_host, smtp_port, qid, mailfrom, recipient, mail,
+                 emailtype),
+                timeout=30)
+        except Queue.Full:
+            raise RuntimeError("email queue is full")

pyquarantine/misc/openrc/pyquarantine-milter.confd

@@ -0,0 +1,8 @@
+# /etc/conf.d/pyquarantine: config file for /etc/init.d/pyquarantine
+
+# Start the daemon as the user. You can optionally append a group name here also.
+# USER="daemon"
+# USER="daemon:nobody"
+
+# Optional parameters for pyquarantine
+# MILTER_OPTS=""

pyquarantine/misc/openrc/pyquarantine-milter.initd

@@ -0,0 +1,46 @@
+#!/sbin/openrc-run
+
+user=${USER:-daemon}
+milter_opts="${MILTER_OPTS:-}"
+
+pidfile="/run/${RC_SVCNAME}.pid"
+command="/usr/bin/pyquarantine-milter"
+command_args="${milter_opts}"
+command_background=true
+command_user="${user}"
+extra_commands="configtest"
+start_stop_daemon_args="--wait 500"
+
+depend() {
+	need net
+	before mta
+}
+
+checkconfig() {
+	OUTPUT=$( ${command} ${command_args} -t 2>&1 )
+	ret=$?
+	if [ $ret -ne 0 ]; then
+		eerror "${SVCNAME} has detected an error in your configuration:"
+		printf "%s\n" "${OUTPUT}"
+	fi
+
+	return $ret
+}
+
+configtest() {
+	ebegin "Checking ${SVCNAME} configuration"
+	checkconfig
+	eend $?
+}
+
+start_pre() {
+	if [ "${RC_CMD}" != "restart" ]; then
+		checkconfig || return $?
+	fi
+}
+
+stop_pre() {
+	if [ "${RC_CMD}" == "restart" ]; then
+		checkconfig || return $?
+	fi
+}

pyquarantine/modify.py

@@ -0,0 +1,391 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = [
+    "AddHeader",
+    "ModHeader",
+    "DelHeader",
+    "AddDisclaimer",
+    "RewriteLinks",
+    "Modify"]
+
+import logging
+import re
+
+from base64 import b64encode
+from bs4 import BeautifulSoup
+from collections import defaultdict
+from copy import copy
+from email.message import MIMEPart
+from email.policy import SMTPUTF8
+
+from pyquarantine import replace_illegal_chars
+from pyquarantine.base import CustomLogger
+
+
+class AddHeader:
+    """Add a mail header field."""
+    _headersonly = True
+
+    def __init__(self, field, value, pretend=False):
+        self.field = field
+        self.value = value
+        self.pretend = pretend
+
+    def execute(self, milter, logger):
+        header = f"{self.field}: {self.value}"
+        if logger.getEffectiveLevel() == logging.DEBUG:
+            logger.debug(f"add_header: {header}")
+        else:
+            logger.info(f"add_header: {header[0:70]}")
+
+        milter.msg.add_header(self.field, self.value)
+        if not self.pretend:
+            milter.addheader(self.field, self.value)
+
+
+class ModHeader:
+    """Change the value of a mail header field."""
+    _headersonly = True
+
+    def __init__(self, field, value, search=None, pretend=False):
+        try:
+            self.field = re.compile(field, re.IGNORECASE)
+            self.search = search
+            if self.search is not None:
+                self.search = re.compile(
+                    self.search, re.MULTILINE + re.DOTALL + re.IGNORECASE)
+
+        except re.error as e:
+            raise RuntimeError(e)
+
+        self.value = value
+        self.pretend = pretend
+
+    def execute(self, milter, logger):
+        idx = defaultdict(int)
+
+        for i, (field, value) in enumerate(milter.msg.items()):
+            field_lower = field.lower()
+            idx[field_lower] += 1
+
+            if not self.field.match(field):
+                continue
+
+            new_value = value
+            if self.search is not None:
+                new_value = self.search.sub(self.value, value).strip()
+            else:
+                new_value = self.value
+
+            if not new_value:
+                logger.warning(
+                    "mod_header: resulting value is empty, "
+                    "skip modification")
+                continue
+
+            if new_value == value:
+                continue
+
+            header = f"{field}: {value}"
+            new_header = f"{field}: {new_value}"
+
+            if logger.getEffectiveLevel() == logging.DEBUG:
+                logger.debug(f"mod_header: {header}: {new_header}")
+            else:
+                logger.info(
+                    f"mod_header: {header[0:70]}: {new_header[0:70]}")
+
+            milter.msg.replace_header(
+                field, replace_illegal_chars(new_value), idx=idx[field_lower])
+
+            if not self.pretend:
+                milter.chgheader(field, new_value, idx=idx[field_lower])
+
+
+class DelHeader:
+    """Delete a mail header field."""
+    _headersonly = True
+
+    def __init__(self, field, value=None, pretend=False):
+        try:
+            self.field = re.compile(field, re.IGNORECASE)
+            self.value = value
+            if self.value is not None:
+                self.value = re.compile(
+                    value, re.MULTILINE + re.DOTALL + re.IGNORECASE)
+        except re.error as e:
+            raise RuntimeError(e)
+
+        self.pretend = pretend
+
+    def execute(self, milter, logger):
+        idx = defaultdict(int)
+
+        for field, value in milter.msg.items():
+            field_lower = field.lower()
+            idx[field_lower] += 1
+
+            if not self.field.match(field):
+                continue
+
+            if self.value is not None and not self.value.search(value):
+                continue
+
+            header = f"{field}: {value}"
+            if logger.getEffectiveLevel() == logging.DEBUG:
+                logger.debug(f"del_header: {header}")
+            else:
+                logger.info(f"del_header: {header[0:70]}")
+            milter.msg.remove_header(field, idx=idx[field_lower])
+
+            if not self.pretend:
+                milter.chgheader(field, "", idx=idx[field_lower])
+
+            idx[field_lower] -= 1
+
+
+def _get_body_content(msg, pref):
+    part = None
+    content = None
+    if not msg.is_multipart() and msg.get_content_type() == f"text/{pref}":
+        part = msg
+    else:
+        part = msg.get_body(preferencelist=(pref))
+
+    if part is not None:
+        content = part.get_content()
+
+    return (part, content)
+
+
+def _has_content_before_body_tag(soup):
+    s = copy(soup)
+    for element in s.find_all("head") + s.find_all("body"):
+        element.extract()
+
+    if len(s.text.strip()) > 0:
+        return True
+
+    return False
+
+
+def _inject_body(milter):
+    if not milter.msg.is_multipart():
+        milter.msg.make_mixed()
+
+    attachments = []
+    for attachment in milter.msg.iter_attachments():
+        if "content-disposition" not in attachment:
+            attachment["Content-Disposition"] = "attachment"
+        attachments.append(attachment)
+
+    milter.msg.clear_content()
+    milter.msg.set_content("")
+    milter.msg.add_alternative("", subtype="html")
+    milter.msg.make_mixed()
+
+    for attachment in attachments:
+        milter.msg.attach(attachment)
+
+
+def _wrap_message(milter):
+    attachment = MIMEPart(policy=SMTPUTF8)
+    attachment.set_content(milter.msg.as_bytes(),
+                           maintype="plain", subtype="text",
+                           disposition="attachment",
+                           filename=f"{milter.qid}.eml",
+                           params={"name": f"{milter.qid}.eml"})
+
+    milter.msg.clear_content()
+    milter.msg.set_content(
+        "Please see the original email attached.")
+    milter.msg.add_alternative(
+        "<html><body>Please see the original email attached.</body></html>",
+        subtype="html")
+    milter.msg.make_mixed()
+    milter.msg.attach(attachment)
+
+
+class AddDisclaimer:
+    """Append or prepend a disclaimer to the mail body."""
+    _headersonly = False
+
+    def __init__(self, text_template, html_template, action, error_policy,
+                 pretend=False):
+        self.text_template_path = text_template
+        self.html_template_path = html_template
+        try:
+            with open(text_template, "r") as f:
+                self.text_template = f.read()
+
+            with open(html_template, "r") as f:
+                html = BeautifulSoup(f.read(), "html.parser")
+
+        except IOError as e:
+            raise RuntimeError(e)
+        body = html.find('body')
+        self.html_template = body or html
+        self.action = action
+        self.error_policy = error_policy
+        self.pretend = pretend
+
+    def patch_message_body(self, milter, logger):
+        text_body, text_content = _get_body_content(milter.msg, "plain")
+        html_body, html_content = _get_body_content(milter.msg, "html")
+
+        if text_content is None and html_content is None:
+            raise RuntimeError("message does not contain any body part")
+
+        if text_content is not None:
+            logger.info(f"{self.action} text disclaimer")
+
+            if self.action == "prepend":
+                content = f"{self.text_template}{text_content}"
+            else:
+                content = f"{text_content}{self.text_template}"
+
+            text_body.set_content(
+                content.encode(), maintype="text", subtype="plain")
+            text_body.set_param("charset", "UTF-8", header="Content-Type")
+            del text_body["MIME-Version"]
+
+        if html_content is not None:
+            logger.info(f"{self.action} html disclaimer")
+
+            soup = BeautifulSoup(html_content, "html.parser")
+
+            body = soup.find('body')
+            if not body:
+                body = soup
+            elif _has_content_before_body_tag(soup):
+                body = soup
+
+            if self.action == "prepend":
+                body.insert(0, copy(self.html_template))
+            else:
+                body.append(self.html_template)
+
+            html_body.set_content(
+                str(soup).encode(), maintype="text", subtype="html")
+            html_body.set_param("charset", "UTF-8", header="Content-Type")
+            del html_body["MIME-Version"]
+
+    def execute(self, milter, logger):
+        old_headers = milter.msg.items()
+
+        try:
+            try:
+                self.patch_message_body(milter, logger)
+            except RuntimeError as e:
+                logger.info(f"{e}, inject empty plain and html body")
+                _inject_body(milter)
+                self.patch_message_body(milter, logger)
+
+        except Exception as e:
+            logger.warning(e)
+            if self.error_policy == "ignore":
+                logger.info(
+                    "unable to add disclaimer to message body, "
+                    "ignore error according to policy")
+                return
+            elif self.error_policy == "reject":
+                logger.info(
+                    "unable to add disclaimer to message body, "
+                    "reject message according to policy")
+                return [
+                    ("reject", "Message rejected due to error")]
+
+            logger.info("wrap original message in a new message envelope")
+            try:
+                _wrap_message(milter)
+                self.patch_message_body(milter, logger)
+            except Exception as e:
+                logger.error(e)
+                raise Exception(
+                    "unable to wrap message in a new message envelope, "
+                    "give up ...")
+
+        if not self.pretend:
+            milter.update_headers(old_headers)
+            milter.replacebody()
+
+
+class RewriteLinks:
+    """Rewrite link targets in the mail html body."""
+    _headersonly = False
+
+    def __init__(self, repl, pretend=False):
+        self.repl = repl
+        self.pretend = pretend
+
+    def execute(self, milter, logger):
+        html_body, html_content = _get_body_content(milter.msg, "html")
+        if html_content is not None:
+            soup = BeautifulSoup(html_content, "html.parser")
+
+            rewritten = 0
+            for link in soup.find_all("a", href=True):
+                if not link["href"]:
+                    continue
+
+                if "{URL_B64}" in self.repl:
+                    url_b64 = b64encode(link["href"].encode()).decode()
+                    target = self.repl.replace("{URL_B64}", url_b64)
+                else:
+                    target = self.repl
+
+                link["href"] = target
+                rewritten += 1
+
+            if rewritten:
+                logger.info(f"rewrote {rewritten} link(s) in html body")
+
+                html_body.set_content(
+                    str(soup).encode(), maintype="text", subtype="html")
+                html_body.set_param("charset", "UTF-8", header="Content-Type")
+                del html_body["MIME-Version"]
+
+                if not self.pretend:
+                    milter.replacebody()
+
+
+class Modify:
+    MODIFICATION_TYPES = {
+        "add_header": AddHeader,
+        "mod_header": ModHeader,
+        "del_header": DelHeader,
+        "add_disclaimer": AddDisclaimer,
+        "rewrite_links": RewriteLinks}
+
+    def __init__(self, cfg, local_addrs, debug):
+        self.cfg = cfg
+        self.logger = logging.getLogger(cfg["name"])
+        self.logger.setLevel(cfg.get_loglevel(debug))
+        cfg["args"]["pretend"] = cfg["pretend"]
+        self._modification = self.MODIFICATION_TYPES[cfg["type"]](
+            **cfg["args"])
+        self._headersonly = self._modification._headersonly
+
+    def __str__(self):
+        cfg = []
+        for key, value in self.cfg["args"].items():
+            cfg.append(f"{key}={value}")
+        class_name = type(self._modification).__name__
+        return f"{class_name}(" + ", ".join(cfg) + ")"
+
+    def execute(self, milter):
+        logger = CustomLogger(
+            self.logger, {"name": self.cfg["name"], "qid": milter.qid})
+        self._modification.execute(milter, logger)

pyquarantine/notify.py

@@ -0,0 +1,342 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = [
+    "BaseNotification",
+    "EMailNotification",
+    "Notify"]
+
+import email
+import logging
+import re
+
+from bs4 import BeautifulSoup
+from collections import defaultdict
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.mime.image import MIMEImage
+from html import escape
+from os.path import basename
+from urllib.parse import quote
+
+from pyquarantine.base import CustomLogger
+from pyquarantine import mailer
+
+
+class BaseNotification:
+    "Notification base class"
+    _headersonly = True
+
+    def __init__(self, pretend=False):
+        self.pretend = pretend
+
+    def execute(self, milter, logger):
+        return
+
+
+class EMailNotification(BaseNotification):
+    "Notification class to send notifications via mail."
+    _headersonly = False
+    _bad_tags = [
+        "applet",
+        "embed",
+        "frame",
+        "frameset",
+        "head",
+        "iframe",
+        "script"
+    ]
+    _good_tags = [
+        "a",
+        "b",
+        "br",
+        "center",
+        "div",
+        "font",
+        "h1",
+        "h2",
+        "h3",
+        "h4",
+        "h5",
+        "h6",
+        "i",
+        "img",
+        "li",
+        "p",
+        "pre",
+        "span",
+        "table",
+        "td",
+        "th",
+        "tr",
+        "tt",
+        "u",
+        "ul"
+    ]
+    _good_attributes = [
+        "align",
+        "alt",
+        "bgcolor",
+        "border",
+        "cellpadding",
+        "cellspacing",
+        "class",
+        "color",
+        "colspan",
+        "dir",
+        "face",
+        "headers",
+        "height",
+        "id",
+        "name",
+        "rowspan",
+        "size",
+        "src",
+        "style",
+        "title",
+        "type",
+        "valign",
+        "value",
+        "width"
+    ]
+
+    def __init__(self, smtp_host, smtp_port, envelope_from, from_header,
+                 subject, template, embed_imgs=[], repl_img=None,
+                 strip_imgs=False, parser_lib="lxml", pretend=False):
+        super().__init__(pretend)
+        self.smtp_host = smtp_host
+        self.smtp_port = smtp_port
+        self.mailfrom = envelope_from
+        self.from_header = from_header
+        self.subject = subject
+        try:
+            self.template = open(template, "r").read()
+            self.embed_imgs = []
+            for img_path in embed_imgs:
+                img = MIMEImage(open(img_path, "rb").read())
+                filename = basename(img_path)
+                img.add_header("Content-ID", f"<{filename}>")
+                self.embed_imgs.append(img)
+
+            self.replacement_img = repl_img
+            self.strip_images = strip_imgs
+
+            if not strip_imgs and repl_img:
+                self.replacement_img = MIMEImage(
+                    open(repl_img, "rb").read())
+                self.replacement_img.add_header(
+                    "Content-ID", "<removed_for_security_reasons>")
+
+        except IOError as e:
+            raise RuntimeError(e)
+
+        self.parser_lib = parser_lib
+
+    def get_email_body_soup(self, msg, logger):
+        "Extract and decode email body and return it as BeautifulSoup object."
+        # try to find the body part
+        logger.debug("trying to find email body")
+        try:
+            body = msg.get_body(preferencelist=("html", "plain"))
+        except Exception as e:
+            logger.error(
+                f"an error occured in email.message.EmailMessage.get_body: "
+                f"{e}")
+            body = None
+
+        if body:
+            charset = body.get_content_charset() or "utf-8"
+            content = body.get_payload(decode=True)
+            try:
+                content = content.decode(encoding=charset, errors="replace")
+            except LookupError:
+                logger.info(
+                    f"unknown encoding '{charset}', falling back to UTF-8")
+                content = content.decode("utf-8", errors="replace")
+            content_type = body.get_content_type()
+            if content_type == "text/plain":
+                # convert text/plain to text/html
+                logger.debug(
+                    f"content type is {content_type}, "
+                    f"converting to text/html")
+                content = re.sub(r"^(.*)$", r"\1<br/>",
+                                 escape(content, quote=False),
+                                 flags=re.MULTILINE)
+            else:
+                logger.debug(f"content type is {content_type}")
+        else:
+            logger.error("unable to find email body")
+            content = "ERROR: unable to find email body"
+
+        # create BeautifulSoup object
+        length = len(content)
+        logger.debug(
+            f"trying to create BeatufilSoup object with "
+            f"parser lib {self.parser_lib}, "
+            f"text length is {length} bytes")
+        soup = BeautifulSoup(content, self.parser_lib)
+        logger.debug("sucessfully created BeautifulSoup object")
+
+        return soup
+
+    def sanitize(self, soup, logger):
+        "Sanitize mail html text."
+        logger.debug("sanitizing email text")
+
+        # completly remove bad elements
+        for element in soup(EMailNotification._bad_tags):
+            logger.debug(
+                f"removing dangerous tag '{element.name}' "
+                f"and its content")
+            element.extract()
+
+        # remove not whitelisted elements, but keep their content
+        for element in soup.find_all(True):
+            if element.name not in EMailNotification._good_tags:
+                logger.debug(
+                    f"removing tag '{element.name}', keep its content")
+                element.replaceWithChildren()
+
+        # remove not whitelisted attributes
+        for element in soup.find_all(True):
+            for attribute in list(element.attrs.keys()):
+                if attribute not in EMailNotification._good_attributes:
+                    if element.name == "a" and attribute == "href":
+                        logger.debug(
+                            f"setting attribute href to '#' "
+                            f"on tag '{element.name}'")
+                        element["href"] = "#"
+                    else:
+                        logger.debug(
+                            f"removing attribute '{attribute}' "
+                            f"from tag '{element.name}'")
+                        del(element.attrs[attribute])
+        return soup
+
+    def notify(self, msg, qid, mailfrom, recipients, logger,
+               template_vars=defaultdict(str), synchronous=False):
+        "Notify recipients via email."
+        # extract body from email
+        soup = self.get_email_body_soup(msg, logger)
+
+        # replace picture sources
+        image_replaced = False
+        if self.strip_images:
+            logger.debug("looking for images to strip")
+            for element in soup("img"):
+                if "src" in element.attrs.keys():
+                    logger.debug(f"strip image: {element['src']}")
+                element.extract()
+        elif self.replacement_img:
+            logger.debug("looking for images to replace")
+            for element in soup("img"):
+                if "src" in element.attrs.keys():
+                    logger.debug(f"replacing image: {element['src']}")
+                element["src"] = "cid:removed_for_security_reasons"
+                image_replaced = True
+
+        # sanitizing email text of original email
+        sanitized_text = self.sanitize(soup, logger)
+        del soup
+
+        # sending email notifications
+        for recipient in recipients:
+            logger.debug(
+                f"generating notification email for '{recipient}'")
+            logger.debug("parsing email template")
+
+            # generate dict containing all template variables
+            variables = defaultdict(str, template_vars)
+            variables.update({
+                "HTML_TEXT": sanitized_text,
+                "FROM": escape(msg["from"], quote=False),
+                "ENVELOPE_FROM": escape(mailfrom, quote=False),
+                "ENVELOPE_FROM_URL": escape(quote(mailfrom),
+                                            quote=False),
+                "TO": escape(msg["to"], quote=False),
+                "ENVELOPE_TO": escape(recipient, quote=False),
+                "ENVELOPE_TO_URL": escape(quote(recipient)),
+                "SUBJECT": escape(msg["subject"], quote=False)})
+
+            # parse template
+            htmltext = self.template.format_map(variables)
+
+            newmsg = MIMEMultipart('related')
+            newmsg["From"] = self.from_header.format_map(
+                defaultdict(str, FROM=msg["from"]))
+            newmsg["To"] = msg["to"]
+            newmsg["Subject"] = self.subject.format_map(
+                defaultdict(str, SUBJECT=msg["subject"]))
+            newmsg["Date"] = email.utils.formatdate()
+            newmsg.attach(MIMEText(htmltext, "html", 'UTF-8'))
+
+            if image_replaced:
+                logger.debug("attaching notification_replacement_img")
+                newmsg.attach(self.replacement_img)
+
+            for img in self.embed_imgs:
+                logger.debug("attaching imgage")
+                newmsg.attach(img)
+
+            logger.debug(f"sending notification email to: {recipient}")
+            if synchronous:
+                try:
+                    mailer.smtp_send(self.smtp_host, self.smtp_port,
+                                     self.mailfrom, recipient,
+                                     newmsg.as_string())
+                except Exception as e:
+                    raise RuntimeError(
+                        f"error while sending email to '{recipient}': {e}")
+            else:
+                mailer.sendmail(self.smtp_host, self.smtp_port, qid,
+                                self.mailfrom, recipient, newmsg.as_string(),
+                                "notification email")
+
+    def execute(self, milter, logger):
+        super().execute(milter, logger)
+
+        self.notify(msg=milter.msg, qid=milter.qid,
+                    mailfrom=milter.msginfo["mailfrom"],
+                    recipients=milter.msginfo["rcpts"],
+                    template_vars=milter.msginfo["vars"],
+                    logger=logger)
+
+
+class Notify:
+    NOTIFICATION_TYPES = {
+        "email": EMailNotification}
+
+    def __init__(self, cfg, local_addrs, debug):
+        self.cfg = cfg
+        self.logger = logging.getLogger(cfg["name"])
+        self.logger.setLevel(cfg.get_loglevel(debug))
+
+        nodification_type = cfg["args"]["type"]
+        del cfg["args"]["type"]
+        cfg["args"]["pretend"] = cfg["pretend"]
+        self._notification = self.NOTIFICATION_TYPES[nodification_type](
+            **cfg["args"])
+        self._headersonly = self._notification._headersonly
+
+    def __str__(self):
+        cfg = []
+        for key, value in self.cfg["args"].items():
+            cfg.append(f"{key}={value}")
+        class_name = type(self._notification).__name__
+        return f"{class_name}(" + ", ".join(cfg) + ")"
+
+    def execute(self, milter):
+        logger = CustomLogger(
+            self.logger, {"name": self.cfg["name"], "qid": milter.qid})
+        self._notification.execute(milter, logger)

pyquarantine/rule.py

@@ -0,0 +1,66 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = ["Rule"]
+
+from pyquarantine.action import Action
+from pyquarantine.conditions import Conditions
+
+
+class Rule:
+    """
+    Rule to implement multiple actions on emails.
+    """
+    def __init__(self, cfg, local_addrs, debug):
+        self.cfg = cfg
+
+        self.conditions = cfg["conditions"] if "conditions" in cfg else None
+        if self.conditions is not None:
+            self.conditions["name"] = f"{cfg['name']}: condition"
+            self.conditions["loglevel"] = cfg["loglevel"]
+            self.conditions = Conditions(self.conditions, local_addrs, debug)
+
+        self.actions = []
+        for idx, action_cfg in enumerate(cfg["actions"]):
+            if "name" in action_cfg:
+                action_cfg["name"] = f"{cfg['name']}: {action_cfg['name']}"
+            else:
+                action_cfg["name"] = f"action#{idx}"
+            if "loglevel" not in action_cfg:
+                action_cfg["loglevel"] = cfg["loglevel"]
+            if "pretend" not in action_cfg:
+                action_cfg["pretend"] = cfg["pretend"]
+            self.actions.append(Action(action_cfg, local_addrs, debug))
+
+    def __str__(self):
+        cfg = []
+        for key in ["name", "loglevel", "pretend"]:
+            value = self.cfg[key]
+            cfg.append(f"{key}={value}")
+        if self.conditions is not None:
+            cfg.append(f"conditions={self.conditions}")
+        actions = []
+        for action in self.actions:
+            actions.append(str(action))
+        cfg.append("actions=[" + ", ".join(actions) + "]")
+        return "Rule(" + ", ".join(cfg) + ")"
+
+    def execute(self, milter):
+        """Execute all actions of this rule."""
+        if self.conditions is None or \
+                self.conditions.match(milter):
+            for action in self.actions:
+                milter_action = action.execute(milter)
+                if milter_action is not None:
+                    return milter_action

pyquarantine/run.py

@@ -0,0 +1,154 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = ["main"]
+
+import Milter
+import argparse
+import logging
+import logging.handlers
+import sys
+
+from pyquarantine import mailer
+from pyquarantine import ModifyMilter
+from pyquarantine import __version__ as version
+from pyquarantine.config import get_milter_config
+
+
+def main():
+    python_version = ".".join([str(v) for v in sys.version_info[0:3]])
+    python_version = f"{python_version}-{sys.version_info[3]}"
+
+    "Run pyquarantine."
+    parser = argparse.ArgumentParser(
+        description="pyquarantine-milter daemon",
+        formatter_class=lambda prog: argparse.HelpFormatter(
+            prog, max_help_position=45, width=140))
+
+    parser.add_argument(
+        "-c", "--config", help="Config file to read.",
+        default="/etc/pyquarantine/pyquarantine.conf")
+
+    parser.add_argument(
+        "-s",
+        "--socket",
+        help="Socket used to communicate with the MTA.",
+        default="")
+
+    parser.add_argument(
+        "-d",
+        "--debug",
+        help="Log debugging messages.",
+        action="store_true")
+
+    parser.add_argument(
+        "-t",
+        "--test",
+        help="Check configuration.",
+        action="store_true")
+
+    parser.add_argument(
+        "-v", "--version",
+        help="Print version.",
+        action="version",
+        version=f"%(prog)s {version} (python {python_version})")
+
+    args = parser.parse_args()
+
+    root_logger = logging.getLogger()
+    root_logger.setLevel(logging.DEBUG)
+
+    # setup console log
+    stdouthandler = logging.StreamHandler(sys.stdout)
+    formatter = logging.Formatter("%(levelname)s: %(message)s")
+    stdouthandler.setFormatter(formatter)
+    root_logger.addHandler(stdouthandler)
+
+    logger = logging.getLogger(__name__)
+
+    if not args.debug:
+        logger.setLevel(logging.INFO)
+
+    try:
+        logger.debug("read milter configuration")
+        cfg = get_milter_config(args.config)
+        logger.setLevel(cfg.get_loglevel(args.debug))
+
+        if args.socket:
+            socket = args.socket
+        elif "socket" in cfg:
+            socket = cfg["socket"]
+        else:
+            raise RuntimeError(
+                "listening socket is neither specified on the command line "
+                "nor in the configuration file")
+
+        if not cfg["rules"]:
+            raise RuntimeError("no rules configured")
+
+        for rule in cfg["rules"]:
+            if not rule["actions"]:
+                raise RuntimeError(
+                    f"{rule['name']}: no actions configured")
+
+    except (RuntimeError, AssertionError) as e:
+        logger.error(f"config error: {e}")
+        sys.exit(255)
+
+    try:
+        ModifyMilter.set_config(cfg, args.debug)
+    except RuntimeError as e:
+        logger.error(e)
+        sys.exit(254)
+
+    if args.test:
+        print("Configuration OK")
+        sys.exit(0)
+
+    # setup console log for runtime
+    formatter = logging.Formatter("%(asctime)s - %(levelname)s: %(message)s")
+    stdouthandler.setFormatter(formatter)
+    stdouthandler.setLevel(logging.DEBUG)
+
+    # setup syslog
+    sysloghandler = logging.handlers.SysLogHandler(
+        address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL)
+    sysloghandler.setFormatter(
+        logging.Formatter("pyquarantine: %(message)s"))
+    root_logger.addHandler(sysloghandler)
+
+    logger.info("pyquarantine-milter starting")
+
+    # register milter factory class
+    Milter.factory = ModifyMilter
+    Milter.set_exception_policy(Milter.TEMPFAIL)
+
+    if args.debug:
+        Milter.setdbg(1)
+
+    rc = 0
+    try:
+        Milter.runmilter("pyquarantine", socketname=socket, timeout=600)
+    except Milter.milter.error as e:
+        logger.error(e)
+        rc = 255
+
+    mailer.queue.put(None)
+    logger.info("pyquarantine-milter stopped")
+
+    sys.exit(rc)
+
+
+if __name__ == "__main__":
+    main()

pyquarantine/storage.py

@@ -0,0 +1,431 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = [
+    "BaseMailStorage",
+    "FileMailStorage",
+    "Store",
+    "Quarantine"]
+
+import json
+import logging
+import os
+
+from calendar import timegm
+from datetime import datetime
+from glob import glob
+from time import gmtime
+
+from pyquarantine.base import CustomLogger
+from pyquarantine.conditions import Conditions
+from pyquarantine.config import ActionConfig
+from pyquarantine.notify import Notify
+
+
+class BaseMailStorage:
+    "Mail storage base class"
+    _headersonly = True
+
+    def __init__(self, original=False, metadata=False, metavar=None,
+                 pretend=False):
+        self.original = original
+        self.metadata = metadata
+        self.metavar = metavar
+        self.pretend = False
+
+    def add(self, data, qid, mailfrom="", recipients=[]):
+        "Add email to storage."
+        return ("", "")
+
+    def execute(self, milter, logger):
+        return
+
+    def find(self, mailfrom=None, recipients=None, older_than=None):
+        "Find emails in storage."
+        return
+
+    def get_metadata(self, storage_id):
+        "Return metadata of email in storage."
+        return
+
+    def delete(self, storage_id, recipients=None):
+        "Delete email from storage."
+        return
+
+    def get_mail(self, storage_id):
+        "Return email and metadata."
+        return
+
+
+class FileMailStorage(BaseMailStorage):
+    "Storage class to store mails on filesystem."
+    _headersonly = False
+
+    def __init__(self, directory, original=False, metadata=False, metavar=None,
+                 mode=None, pretend=False):
+        super().__init__(original, metadata, metavar, pretend)
+        # check if directory exists and is writable
+        if not os.path.isdir(directory) or \
+                not os.access(directory, os.W_OK):
+            raise RuntimeError(
+                f"directory '{directory}' does not exist or is "
+                f"not writable")
+        self.directory = directory
+        try:
+            self.mode = int(mode, 8) if mode is not None else None
+            if self.mode is not None and self.mode > 511:
+                raise ValueError
+        except ValueError:
+            raise RuntimeError(f"invalid mode '{mode}'")
+
+        self._metadata_suffix = ".metadata"
+
+    def __str__(self):
+        cfg = []
+        cfg.append(f"metadata={self.metadata}")
+        cfg.append(f"metavar={self.metavar}")
+        cfg.append(f"pretend={self.pretend}")
+        cfg.append(f"directory={self.directory}")
+        cfg.append(f"original={self.original}")
+        return "FileMailStorage(" + ", ".join(cfg) + ")"
+
+    def get_storageid(self, qid):
+        timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
+        return f"{timestamp}_{qid}"
+
+    def _get_file_paths(self, storage_id):
+        datafile = os.path.join(self.directory, storage_id)
+        metafile = f"{datafile}{self._metadata_suffix}"
+        return metafile, datafile
+
+    def _save_datafile(self, datafile, data):
+        try:
+            if self.mode is None:
+                with open(datafile, "wb") as f:
+                    f.write(data)
+            else:
+                umask = os.umask(0)
+                with open(
+                        os.open(datafile, os.O_CREAT | os.O_WRONLY, self.mode),
+                        "wb") as f:
+                    f.write(data)
+                os.umask(umask)
+
+        except IOError as e:
+            raise RuntimeError(f"unable save data file: {e}")
+
+    def _save_metafile(self, metafile, metadata):
+        try:
+            if self.mode is None:
+                with open(metafile, "w") as f:
+                    json.dump(metadata, f, indent=2)
+            else:
+                umask = os.umask(0)
+                with open(
+                        os.open(metafile, os.O_CREAT | os.O_WRONLY, self.mode),
+                        "w") as f:
+                    json.dump(metadata, f, indent=2)
+                os.umask(umask)
+
+        except IOError as e:
+            raise RuntimeError(f"unable to save metadata file: {e}")
+
+    def _remove(self, storage_id):
+        metafile, datafile = self._get_file_paths(storage_id)
+
+        try:
+            if self.metadata:
+                os.remove(metafile)
+
+            os.remove(datafile)
+        except IOError as e:
+            raise RuntimeError(f"unable to remove file: {e}")
+
+    def add(self, data, qid, mailfrom="", recipients=[], subject=""):
+        "Add email to file storage and return storage id."
+        super().add(data, qid, mailfrom, recipients)
+
+        storage_id = self.get_storageid(qid)
+        metafile, datafile = self._get_file_paths(storage_id)
+
+        # save mail
+        self._save_datafile(datafile, data)
+
+        if not self.metadata:
+            return storage_id, None, datafile
+
+        # save metadata
+        metadata = {
+            "mailfrom": mailfrom,
+            "recipients": recipients,
+            "subject": subject,
+            "timestamp": timegm(gmtime()),
+            "queue_id": qid}
+
+        try:
+            self._save_metafile(metafile, metadata)
+        except RuntimeError as e:
+            os.remove(datafile)
+            raise e
+
+        return storage_id, metafile, datafile
+
+    def execute(self, milter, logger):
+        if self.original:
+            milter.fp.seek(0)
+            data = milter.fp.read
+            mailfrom = milter.mailfrom
+            recipients = list(milter.rcpts)
+            subject = ""
+        else:
+            data = milter.msg.as_bytes
+            mailfrom = milter.msginfo["mailfrom"]
+            recipients = list(milter.msginfo["rcpts"])
+            subject = milter.msg["subject"] or ""
+
+        if not self.pretend:
+            storage_id, metafile, datafile = self.add(
+                data(), milter.qid, mailfrom, recipients, subject)
+            logger.info(f"stored message in file {datafile}")
+        else:
+            storage_id = self.get_storageid(milter.qid)
+            metafile, datafile = self._get_file_paths(storage_id)
+
+        if self.metavar:
+            milter.msginfo["vars"][f"{self.metavar}_ID"] = storage_id
+            milter.msginfo["vars"][f"{self.metavar}_DATAFILE"] = datafile
+            if self.metadata:
+                milter.msginfo["vars"][f"{self.metavar}_METAFILE"] = metafile
+
+    def get_metadata(self, storage_id):
+        "Return metadata of email in storage."
+        super().get_metadata(storage_id)
+
+        if not self.metadata:
+            return None
+
+        metafile, _ = self._get_file_paths(storage_id)
+        if not os.path.isfile(metafile):
+            raise RuntimeError(
+                f"invalid storage id '{storage_id}'")
+
+        try:
+            with open(metafile, "r") as f:
+                metadata = json.load(f)
+        except IOError as e:
+            raise RuntimeError(f"unable to read metadata file: {e}")
+        except json.JSONDecodeError as e:
+            raise RuntimeError(
+                f"invalid metafile '{metafile}': {e}")
+
+        return metadata
+
+    def find(self, mailfrom=None, recipients=None, older_than=None):
+        "Find emails in storage."
+        super().find(mailfrom, recipients, older_than)
+        if isinstance(mailfrom, str):
+            mailfrom = [mailfrom]
+        if isinstance(recipients, str):
+            recipients = [recipients]
+
+        if not self.metadata:
+            return {}
+
+        emails = {}
+        metafiles = glob(os.path.join(
+            self.directory, f"*{self._metadata_suffix}"))
+        for metafile in metafiles:
+            if not os.path.isfile(metafile):
+                continue
+
+            storage_id = os.path.basename(
+                metafile[:-len(self._metadata_suffix)])
+            metadata = self.get_metadata(storage_id)
+            if older_than is not None:
+                if timegm(gmtime()) - metadata["date"] < (older_than * 86400):
+                    continue
+
+            if mailfrom is not None:
+                if metadata["mailfrom"] not in mailfrom:
+                    continue
+
+            if recipients is not None:
+                if len(recipients) == 1 and \
+                        recipients[0] not in metadata["recipients"]:
+                    continue
+                elif len(set(recipients + metadata["recipients"])) == \
+                        len(recipients + metadata["recipients"]):
+                    continue
+
+            emails[storage_id] = metadata
+
+        return emails
+
+    def delete(self, storage_id, recipients=None):
+        "Delete email from storage."
+        super().delete(storage_id, recipients)
+
+        if not recipients or not self.metadata:
+            self._remove(storage_id)
+            return
+
+        try:
+            metadata = self.get_metadata(storage_id)
+        except RuntimeError as e:
+            raise RuntimeError(f"unable to delete email: {e}")
+
+        metafile, _ = self._get_file_paths(storage_id)
+
+        if type(recipients) == str:
+            recipients = [recipients]
+
+        for recipient in recipients:
+            if recipient not in metadata["recipients"]:
+                raise RuntimeError(f"invalid recipient '{recipient}'")
+            metadata["recipients"].remove(recipient)
+            if not metadata["recipients"]:
+                self._remove(storage_id)
+            else:
+                self._save_metafile(metafile, metadata)
+
+    def get_mail(self, storage_id):
+        super().get_mail(storage_id)
+
+        metadata = self.get_metadata(storage_id)
+        _, datafile = self._get_file_paths(storage_id)
+        try:
+            data = open(datafile, "rb").read()
+        except IOError as e:
+            raise RuntimeError(f"unable to open email data file: {e}")
+        return (metadata, data)
+
+
+class Store:
+    STORAGE_TYPES = {
+        "file": FileMailStorage}
+
+    def __init__(self, cfg, local_addrs, debug):
+        self.cfg = cfg
+        self.logger = logging.getLogger(cfg["name"])
+        self.logger.setLevel(cfg.get_loglevel(debug))
+
+        storage_type = cfg["args"]["type"]
+        del cfg["args"]["type"]
+        cfg["args"]["pretend"] = cfg["pretend"]
+        self._storage = self.STORAGE_TYPES[storage_type](
+            **cfg["args"])
+        self._headersonly = self._storage._headersonly
+
+    def __str__(self):
+        cfg = []
+        for key, value in self.cfg["args"].items():
+            cfg.append(f"{key}={value}")
+        class_name = type(self._storage).__name__
+        return f"{class_name}(" + ", ".join(cfg) + ")"
+
+    def execute(self, milter):
+        logger = CustomLogger(
+            self.logger, {"name": self.cfg["name"], "qid": milter.qid})
+        self._storage.execute(milter, logger)
+
+
+class Quarantine:
+    "Quarantine class."
+    _headersonly = False
+
+    def __init__(self, cfg, local_addrs, debug):
+        self.cfg = cfg
+        self.logger = logging.getLogger(cfg["name"])
+        self.logger.setLevel(cfg.get_loglevel(debug))
+
+        store_cfg = ActionConfig({
+            "name": cfg["name"],
+            "loglevel": cfg["loglevel"],
+            "pretend": cfg["pretend"],
+            "type": "store",
+            "args": cfg["args"]["store"].get_config()})
+        store_cfg["args"]["metadata"] = True
+        self.store = Store(store_cfg, local_addrs, debug)
+
+        self.notify = None
+        if "notify" in cfg["args"]:
+            notify_cfg = ActionConfig({
+                "name": cfg["name"],
+                "loglevel": cfg["loglevel"],
+                "pretend": cfg["pretend"],
+                "type": "notify",
+                "args": cfg["args"]["notify"].get_config()})
+            self.notify = Notify(notify_cfg, local_addrs, debug)
+
+        self.whitelist = None
+        if "whitelist" in cfg["args"]:
+            whitelist_cfg = cfg["args"]["whitelist"]
+            whitelist_cfg["name"] = cfg["name"]
+            whitelist_cfg["loglevel"] = cfg["loglevel"]
+            self.whitelist = Conditions(
+                whitelist_cfg,
+                local_addrs=[],
+                debug=debug)
+
+        self.milter_action = None
+        if "milter_action" in cfg["args"]:
+            self.milter_action = cfg["args"]["milter_action"]
+        self.reject_reason = None
+        if "reject_reason" in cfg["args"]:
+            self.reject_reason = cfg["args"]["reject_reason"]
+
+    def __str__(self):
+        cfg = []
+        cfg.append(f"store={str(self.store)}")
+        if self.notify is not None:
+            cfg.append(f"notify={str(self.notify)}")
+        if self.whitelist is not None:
+            cfg.append(f"whitelist={str(self.whitelist)}")
+        for key in ["milter_action", "reject_reason"]:
+            if key not in self.cfg["args"]:
+                continue
+            value = self.cfg["args"][key]
+            cfg.append(f"{key}={value}")
+        class_name = type(self).__name__
+        return f"{class_name}(" + ", ".join(cfg) + ")"
+
+    def execute(self, milter):
+        logger = CustomLogger(
+            self.logger, {"name": self.cfg["name"], "qid": milter.qid})
+        wl_rcpts = []
+        if self.whitelist:
+            wl_rcpts = self.whitelist.get_wl_rcpts(
+                milter.msginfo["mailfrom"], milter.msginfo["rcpts"], logger)
+            logger.info(f"whitelisted recipients: {wl_rcpts}")
+
+        rcpts = [
+            rcpt for rcpt in milter.msginfo["rcpts"] if rcpt not in wl_rcpts]
+
+        if not rcpts:
+            # all recipients whitelisted
+            return
+
+        logger.info(f"add to quarantine for recipients: {rcpts}")
+        milter.msginfo["rcpts"] = rcpts
+
+        self.store.execute(milter)
+
+        if self.notify is not None:
+            self.notify.execute(milter)
+
+        milter.msginfo["rcpts"].extend(wl_rcpts)
+        milter.delrcpt(rcpts)
+
+        if self.milter_action is not None and not milter.msginfo["rcpts"]:
+            return (self.milter_action, self.reject_reason)

pyquarantine/whitelist.py

@@ -0,0 +1,274 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = [
+    "DatabaseWhitelist",
+    "WhitelistBase"]
+
+import logging
+import peewee
+import re
+
+from datetime import datetime
+from playhouse.db_url import connect
+
+
+class WhitelistBase:
+    "Whitelist base class"
+    def __init__(self, cfg, debug):
+        self.cfg = cfg
+        self.logger = logging.getLogger(cfg["name"])
+        self.logger.setLevel(cfg.get_loglevel(debug))
+
+        peewee_logger = logging.getLogger("peewee")
+        peewee_logger.setLevel(cfg.get_loglevel(debug))
+
+        self.valid_entry_regex = re.compile(
+            r"^[a-zA-Z0-9_.=+-]*?(@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)?$")
+        self.batv_regex = re.compile(
+            r"^prvs=[0-9]{4}[0-9A-Fa-f]{6}=(?P<LEFT_PART>.+?)@")
+
+    def remove_batv(self, addr):
+        return self.batv_regex.sub(r"\g<LEFT_PART>", addr, count=1)
+
+    def check(self, mailfrom, recipient):
+        "Check if mailfrom/recipient combination is whitelisted."
+        return
+
+    def find(self, mailfrom=None, recipients=None, older_than=None):
+        "Find whitelist entries."
+        return
+
+    def add(self, mailfrom, recipient, comment, permanent):
+        "Add entry to whitelist."
+        # check if mailfrom and recipient are valid
+        if not self.valid_entry_regex.match(mailfrom):
+            raise RuntimeError("invalid from address")
+        if not self.valid_entry_regex.match(recipient):
+            raise RuntimeError("invalid recipient")
+        return
+
+    def delete(self, whitelist_id):
+        "Delete entry from whitelist."
+        return
+
+
+class WhitelistModel(peewee.Model):
+    mailfrom = peewee.CharField()
+    recipient = peewee.CharField()
+    created = peewee.DateTimeField(default=datetime.now)
+    last_used = peewee.DateTimeField(default=datetime.now)
+    comment = peewee.TextField(default="")
+    permanent = peewee.BooleanField(default=False)
+
+
+class Meta:
+    indexes = (
+        # trailing comma is mandatory if only one index should be created
+        (('mailfrom', 'recipient'), True),
+    )
+
+
+class DatabaseWhitelist(WhitelistBase):
+    "Whitelist class to store whitelist in a database"
+    whitelist_type = "db"
+    _db_connections = {}
+    _db_tables = {}
+
+    def __init__(self, cfg, debug):
+        super().__init__(cfg, debug)
+
+        tablename = cfg["table"]
+        connection_string = cfg["connection"]
+
+        if connection_string in DatabaseWhitelist._db_connections.keys():
+            db = DatabaseWhitelist._db_connections[connection_string]
+        else:
+            try:
+                # connect to database
+                conn = re.sub(
+                    r"(.*?://.*?):.*?(@.*)",
+                    r"\1:<PASSWORD>\2",
+                    connection_string)
+                self.logger.debug(
+                    f"connecting to database '{conn}'")
+                db = connect(connection_string)
+            except Exception as e:
+                raise RuntimeError(
+                    f"unable to connect to database: {e}")
+
+            DatabaseWhitelist._db_connections[connection_string] = db
+
+        # generate model meta class
+        self.meta = Meta
+        self.meta.database = db
+        self.meta.table_name = tablename
+        self.model = type(
+            f"WhitelistModel_{self.cfg['name']}",
+            (WhitelistModel,),
+            {"Meta": self.meta})
+
+        if connection_string not in DatabaseWhitelist._db_tables.keys():
+            DatabaseWhitelist._db_tables[connection_string] = []
+
+        if tablename not in DatabaseWhitelist._db_tables[connection_string]:
+            DatabaseWhitelist._db_tables[connection_string].append(tablename)
+            try:
+                db.create_tables([self.model])
+            except Exception as e:
+                raise RuntimeError(
+                    f"unable to initialize table '{tablename}': {e}")
+
+    def __str__(self):
+        cfg = []
+        for arg in ("connection", "table"):
+            if arg in self.cfg:
+                cfg.append(f"{arg}={self.cfg[arg]}")
+        return "DatabaseWhitelist(" + ", ".join(cfg) + ")"
+
+    def _entry_to_dict(self, entry):
+        result = {}
+        result[entry.id] = {
+            "id": entry.id,
+            "mailfrom": entry.mailfrom,
+            "recipient": entry.recipient,
+            "created": entry.created,
+            "last_used": entry.last_used,
+            "comment": entry.comment,
+            "permanent": entry.permanent
+        }
+        return result
+
+    def get_weight(self, entry):
+        value = 0
+        for address in [entry.mailfrom, entry.recipient]:
+            if address == "":
+                value += 2
+            elif address[0] == "@":
+                value += 1
+        return value
+
+    def check(self, mailfrom, recipient, logger):
+        # check if mailfrom/recipient combination is whitelisted
+        super().check(mailfrom, recipient)
+        mailfrom = self.remove_batv(mailfrom)
+        recipient = self.remove_batv(recipient)
+
+        # generate list of possible mailfroms
+        logger.debug(
+            f"query database for whitelist entries from <{mailfrom}> "
+            f"to <{recipient}>")
+        mailfroms = [""]
+        if "@" in mailfrom and not mailfrom.startswith("@"):
+            domain = mailfrom.split("@")[1]
+            mailfroms.append(f"@{domain}")
+        mailfroms.append(mailfrom)
+
+        # generate list of possible recipients
+        recipients = [""]
+        if "@" in recipient and not recipient.startswith("@"):
+            domain = recipient.split("@")[1]
+            recipients.append(f"@{domain}")
+        recipients.append(recipient)
+
+        # query the database
+        try:
+            entries = list(
+                self.model.select().where(
+                    self.model.mailfrom.in_(mailfroms),
+                    self.model.recipient.in_(recipients)))
+        except Exception as e:
+            raise RuntimeError(f"unable to query database: {e}")
+
+        if not entries:
+            # no whitelist entry found
+            return {}
+
+        if len(entries) > 1:
+            entries.sort(key=lambda x: self.get_weight(x), reverse=True)
+
+        # use entry with the highest weight
+        entry = entries[0]
+        entry.last_used = datetime.now()
+        entry.save()
+        result = {}
+        for entry in entries:
+            result.update(self._entry_to_dict(entry))
+
+        return result
+
+    def find(self, mailfrom=None, recipients=None, older_than=None):
+        "Find whitelist entries."
+        super().find(mailfrom, recipients, older_than)
+
+        if isinstance(mailfrom, str):
+            mailfrom = [mailfrom]
+        if isinstance(recipients, str):
+            recipients = [recipients]
+
+        entries = {}
+        try:
+            for entry in list(self.model.select()):
+                if older_than is not None:
+                    delta = (datetime.now() - entry.last_used).total_seconds()
+                    if delta < (older_than * 86400):
+                        continue
+
+                if mailfrom is not None:
+                    if entry.mailfrom not in mailfrom:
+                        continue
+
+                if recipients is not None:
+                    if entry.recipient not in recipients:
+                        continue
+
+                entries.update(self._entry_to_dict(entry))
+        except Exception as e:
+            raise RuntimeError(f"unable to query database: {e}")
+
+        return entries
+
+    def add(self, mailfrom, recipient, comment, permanent):
+        "Add entry to whitelist."
+        super().add(
+            mailfrom,
+            recipient,
+            comment,
+            permanent)
+
+        mailfrom = self.remove_batv(mailfrom)
+        recipient = self.remove_batv(recipient)
+
+        try:
+            self.model.create(
+                mailfrom=mailfrom,
+                recipient=recipient,
+                comment=comment,
+                permanent=permanent)
+        except Exception as e:
+            raise RuntimeError(f"unable to add entry to database: {e}")
+
+    def delete(self, whitelist_id):
+        "Delete entry from whitelist."
+        super().delete(whitelist_id)
+
+        try:
+            query = self.model.delete().where(self.model.id == whitelist_id)
+            deleted = query.execute()
+        except Exception as e:
+            raise RuntimeError(
+                f"unable to delete entry from database: {e}")
+
+        if deleted == 0:
+            raise RuntimeError("invalid whitelist id")