diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index d1f032f..aac9594 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -32,8 +32,6 @@ from pymodmilter import _runtime_patches import Milter import logging -import re -import json from Milter.utils import parse_addr from collections import defaultdict @@ -45,95 +43,9 @@ from email.policy import SMTPUTF8 from io import BytesIO from netaddr import IPNetwork, AddrFormatError -from pymodmilter.base import CustomLogger, BaseConfig, MilterMessage +from pymodmilter.base import CustomLogger, MilterMessage from pymodmilter.base import replace_illegal_chars -from pymodmilter.rule import RuleConfig, Rule - - -class ModifyMilterConfig(BaseConfig): - def __init__(self, cfgfile, debug=False): - try: - with open(cfgfile, "r") as fh: - # remove lines with leading # (comments), they - # are not allowed in json - cfg = re.sub(r"(?m)^\s*#.*\n?", "", fh.read()) - except IOError as e: - raise RuntimeError(f"unable to open/read config file: {e}") - - try: - cfg = json.loads(cfg) - except json.JSONDecodeError as e: - cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())] - msg = "\n".join(cfg_text) - raise RuntimeError(f"{e}\n{msg}") - - if "global" in cfg: - assert isinstance(cfg["global"], dict), \ - "global: invalid type, should be dict" - - cfg["global"]["name"] = "global" - super().__init__(cfg["global"], debug) - - self.logger.debug("initialize config") - - if "pretend" in cfg["global"]: - pretend = cfg["global"]["pretend"] - assert isinstance(pretend, bool), \ - "global: pretend: invalid value, should be bool" - self.pretend = pretend - else: - self.pretend = False - - if "socket" in cfg["global"]: - socket = cfg["global"]["socket"] - assert isinstance(socket, str), \ - "global: socket: invalid value, should be string" - self.socket = socket - else: - self.socket = None - - if "local_addrs" in cfg["global"]: - local_addrs = cfg["global"]["local_addrs"] - assert isinstance(local_addrs, list) and all( - [isinstance(addr, str) for addr in local_addrs]), \ - "global: local_addrs: invalid value, " \ - "should be list of strings" - else: - local_addrs = [ - "fe80::/64", - "::1/128", - "127.0.0.0/8", - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16"] - - self.local_addrs = [] - try: - for addr in local_addrs: - self.local_addrs.append(IPNetwork(addr)) - except AddrFormatError as e: - raise ValueError(f"{self.name}: local_addrs: {e}") - - self.logger.debug(f"socket={self.socket}, " - f"local_addrs={self.local_addrs}, " - f"pretend={self.pretend}, " - f"loglevel={self.loglevel}") - - assert "rules" in cfg, \ - "mandatory parameter 'rules' not found" - assert isinstance(cfg["rules"], list), \ - "rules: invalid value, should be list" - - self.logger.debug("initialize rules config") - self.rules = [] - for idx, rule_cfg in enumerate(cfg["rules"]): - if "name" not in rule_cfg: - rule_cfg["name"] = "Rule #{idx}" - if "loglevel" not in rule_cfg: - rule_cfg["loglevel"] = self.loglevel - if "pretend" not in rule_cfg: - rule_cfg["pretend"] = self.pretend - self.rules.append(RuleConfig(rule_cfg, self.local_addrs, debug)) +from pymodmilter.rule import Rule class ModifyMilter(Milter.Base): @@ -145,11 +57,29 @@ class ModifyMilter(Milter.Base): if issubclass(v, AddressHeader)] @staticmethod - def set_config(cfg): - ModifyMilter._loglevel = cfg.loglevel - for rule_cfg in cfg.rules: - ModifyMilter._rules.append( - Rule(rule_cfg)) + def set_config(cfg, debug): + ModifyMilter._loglevel = cfg.get_loglevel(debug) + + try: + local_addrs = [] + for addr in cfg["local_addrs"]: + local_addrs.append(IPNetwork(addr)) + except AddrFormatError as e: + raise RuntimeError(e) + + logger = logging.getLogger(__name__) + logger.setLevel(ModifyMilter._loglevel) + for idx, rule_cfg in enumerate(cfg["rules"]): + if "name" not in rule_cfg: + rule_cfg["name"] = f"rule#{idx}" + if "loglevel" not in rule_cfg: + rule_cfg["loglevel"] = cfg["loglevel"] + if "pretend" not in rule_cfg: + rule_cfg["pretend"] = cfg["pretend"] + rule = Rule(rule_cfg, local_addrs, debug) + + logger.debug(rule) + ModifyMilter._rules.append(rule) def __init__(self): self.logger = logging.getLogger(__name__) diff --git a/pymodmilter/action.py b/pymodmilter/action.py index d5ef9ee..a2bc31a 100644 --- a/pymodmilter/action.py +++ b/pymodmilter/action.py @@ -12,204 +12,51 @@ # along with PyMod-Milter. If not, see . # -__all__ = [ - "ActionConfig", - "Action"] +__all__ = ["Action"] + +import logging -from pymodmilter import BaseConfig from pymodmilter import modify, notify, storage from pymodmilter.base import CustomLogger -from pymodmilter.conditions import ConditionsConfig, Conditions - - -class ActionConfig(BaseConfig): - TYPES = {"add_header": "_add_header", - "mod_header": "_mod_header", - "del_header": "_del_header", - "add_disclaimer": "_add_disclaimer", - "rewrite_links": "_rewrite_links", - "store": "_store", - "notify": "_notify", - "quarantine": "_quarantine"} - - def __init__(self, cfg, local_addrs, debug): - super().__init__(cfg, debug) - - self.local_addrs = local_addrs - self.debug = debug - - self.pretend = False - if "pretend" in cfg: - assert isinstance(cfg["pretend"], bool), \ - f"{self.name}: pretend: invalid value, should be bool" - self.pretend = cfg["pretend"] - - assert "type" in cfg, \ - f"{self.name}: mandatory parameter 'type' not found" - assert isinstance(cfg["type"], str), \ - f"{self.name}: type: invalid value, should be string" - assert cfg["type"] in ActionConfig.TYPES, \ - f"{self.name}: type: invalid action type" - - getattr(self, ActionConfig.TYPES[cfg["type"]])(cfg) - - if "conditions" in cfg: - assert isinstance(cfg["conditions"], dict), \ - f"{self.name}: conditions: invalid value, should be dict" - cfg["conditions"]["name"] = f"{self.name}: condition" - if "loglevel" not in cfg["conditions"]: - cfg["conditions"]["loglevel"] = self.loglevel - self.conditions = ConditionsConfig( - cfg["conditions"], local_addrs, debug) - else: - self.conditions = None - - self.logger.debug(f"{self.name}: pretend={self.pretend}, " - f"loglevel={self.loglevel}, " - f"type={cfg['type']}, " - f"args={self.args}") - - def _add_header(self, cfg): - self.action = modify.AddHeader - self.add_string_arg(cfg, ["field", "value"]) - - def _mod_header(self, cfg): - self.action = modify.ModHeader - args = ["field", "value"] - if "search" in cfg: - args.append("search") - - self.add_string_arg(cfg, args) - - def _del_header(self, cfg): - self.action = modify.DelHeader - args = ["field"] - if "value" in cfg: - args.append("value") - - self.add_string_arg(cfg, args) - - def _add_disclaimer(self, cfg): - self.action = modify.AddDisclaimer - if "error_policy" not in cfg: - cfg["error_policy"] = "wrap" - - self.add_string_arg( - cfg, ["action", "html_template", "text_template", - "error_policy"]) - assert self.args["action"] in ["append", "prepend"], \ - f"{self.name}: action: invalid value, " \ - f"should be 'append' or 'prepend'" - assert self.args["error_policy"] in ("wrap", - "ignore", - "reject"), \ - f"{self.name}: error_policy: invalid value, " \ - f"should be 'wrap', 'ignore' or 'reject'" - - def _rewrite_links(self, cfg): - self.action = modify.RewriteLinks - self.add_string_arg(cfg, "repl") - - def _store(self, cfg): - assert "storage_type" in cfg, \ - f"{self.name}: mandatory parameter 'storage_type' not found" - assert isinstance(cfg["storage_type"], str), \ - f"{self.name}: storage_type: invalid value, " \ - f"should be string" - - if "original" in cfg: - self.add_bool_arg(cfg, "original") - - if cfg["storage_type"] == "file": - self.action = storage.FileMailStorage - self.add_string_arg(cfg, "directory") - - if "metavar" in cfg: - self.add_string_arg(cfg, "metavar") - - else: - raise RuntimeError( - f"{self.name}: storage_type: invalid storage type") - - def _notify(self, cfg): - self.action = notify.EMailNotification - - args = ["smtp_host", "envelope_from", "from_header", "subject", - "template"] - if "repl_img" in cfg: - args.append("repl_img") - self.add_string_arg(cfg, args) - - self.add_int_arg(cfg, "smtp_port") - - if "embed_imgs" in cfg: - assert isinstance(cfg["embed_imgs"], list) and all( - [isinstance(img, str) for img in cfg["embed_imgs"]]), \ - f"{self.name}: embed_imgs: invalid value, " \ - f"should be list of strings" - self.args["embed_imgs"] = cfg["embed_imgs"] - - def _quarantine(self, cfg): - self.action = storage.Quarantine - assert "storage" in cfg, \ - f"{self.name}: mandatory parameter 'storage' not found" - assert isinstance(cfg["storage"], dict), \ - f"{self.name}: storage: invalid value, " \ - f"should be dict" - cfg["storage"]["type"] = "store" - cfg["storage"]["name"] = f"{self.name}: storage" - - args = ["storage"] - if "notification" in cfg: - assert isinstance(cfg["notification"], dict), \ - f"{self.name}: notification: invalid value, " \ - f"should be dict" - cfg["notification"]["type"] = "notify" - cfg["notification"]["name"] = f"{self.name}: notification" - args.append("notification") - - for arg in args: - if "loglevel" not in cfg[arg]: - cfg[arg]["loglevel"] = self.loglevel - if "pretend" not in cfg[arg]: - cfg[arg]["pretend"] = self.pretend - - self.args[arg] = ActionConfig( - cfg[arg], self.local_addrs, self.debug) - - if "milter_action" in cfg: - self.add_string_arg(cfg, "milter_action") - self.args["milter_action"] = self.args["milter_action"].upper() - assert self.args["milter_action"] in ["REJECT", "DISCARD", - "ACCEPT"], \ - f"{self.name}: milter_action: invalid value, " \ - f"should be 'ACCEPT', 'REJECT' or 'DISCARD'" - if self.args["milter_action"] == "REJECT": - if "reject_reason" in cfg: - self.add_string_arg(cfg, "reject_reason") - - if "whitelist" in cfg: - wl = {"whitelist": cfg["whitelist"]} - wl["name"] = f"{self.name}: whitelist" - if "loglevel" not in wl: - wl["loglevel"] = self.loglevel - self.args["whitelist"] = ConditionsConfig( - wl, self.local_addrs, self.debug) +from pymodmilter.conditions import Conditions class Action: """Action to implement a pre-configured action to perform on e-mails.""" + ACTION_TYPES = { + "add_header": modify.Modify, + "mod_header": modify.Modify, + "del_header": modify.Modify, + "add_disclaimer": modify.Modify, + "rewrite_links": modify.Modify, + "store": storage.Store, + "notify": notify.Notify, + "quarantine": storage.Quarantine} - def __init__(self, cfg): - self.logger = cfg.logger - if cfg.conditions is None: - self.conditions = None - else: - self.conditions = Conditions(cfg.conditions) + def __init__(self, cfg, local_addrs, debug): + self.cfg = cfg + self.logger = logging.getLogger(cfg["name"]) + self.logger.setLevel(cfg.get_loglevel(debug)) - self.pretend = cfg.pretend - self.name = cfg.name - self.action = cfg.action(**cfg.args) + self.conditions = cfg["conditions"] if "conditions" in cfg else None + if self.conditions is not None: + self.conditions["name"] = f"{cfg['name']}: conditions" + self.conditions["loglevel"] = cfg["loglevel"] + self.conditions = Conditions(self.conditions, local_addrs, debug) + + action_type = cfg["type"] + self.action = self.ACTION_TYPES[action_type]( + cfg, local_addrs, debug) + + def __str__(self): + cfg = [] + for key in ["name", "loglevel", "pretend", "type"]: + value = self.cfg[key] + cfg.append(f"{key}={value}") + if self.conditions is not None: + cfg.append(f"conditions={self.conditions}") + cfg.append(f"action={self.action}") + return "Action(" + ", ".join(cfg) + ")" def headersonly(self): """Return the needs of this action.""" @@ -218,8 +65,7 @@ class Action: def execute(self, milter): """Execute configured action.""" logger = CustomLogger( - self.logger, {"qid": milter.qid, "name": self.name}) + self.logger, {"qid": milter.qid, "name": self.cfg["name"]}) if self.conditions is None or \ self.conditions.match(milter): - return self.action.execute( - milter=milter, pretend=self.pretend, logger=logger) + return self.action.execute(milter) diff --git a/pymodmilter/base.py b/pymodmilter/base.py index eb1363d..a88671a 100644 --- a/pymodmilter/base.py +++ b/pymodmilter/base.py @@ -14,10 +14,8 @@ __all__ = [ "CustomLogger", - "BaseConfig", "MilterMessage", - "replace_illegal_chars", - "config_schema"] + "replace_illegal_chars"] import logging @@ -38,70 +36,6 @@ class CustomLogger(logging.LoggerAdapter): return msg, kwargs -class BaseConfig: - def __init__(self, cfg={}, debug=False): - if "name" in cfg: - assert isinstance(cfg["name"], str), \ - "name: invalid value, should be string" - self.name = cfg["name"] - else: - self.name = __name__ - - self.logger = logging.getLogger(self.name) - if debug: - self.loglevel = logging.DEBUG - elif "loglevel" in cfg: - if isinstance(cfg["loglevel"], int): - self.loglevel = cfg["loglevel"] - else: - level = getattr(logging, cfg["loglevel"].upper(), None) - assert isinstance(level, int), \ - f"{self.name}: loglevel: invalid value" - self.loglevel = level - else: - self.loglevel = logging.INFO - - self.logger.setLevel(self.loglevel) - self.debug = debug - - # the keys/values in args are used as parameters - # to initialize action classes - self.args = {} - - def add_string_arg(self, cfg, args): - if isinstance(args, str): - args = [args] - - for arg in args: - assert arg in cfg, \ - f"{self.name}: mandatory parameter '{arg}' not found" - assert isinstance(cfg[arg], str), \ - f"{self.name}: {arg}: invalid value, should be string" - self.args[arg] = cfg[arg] - - def add_bool_arg(self, cfg, args): - if isinstance(args, str): - args = [args] - - for arg in args: - assert arg in cfg, \ - f"{self.name}: mandatory parameter '{arg}' not found" - assert isinstance(cfg[arg], bool), \ - f"{self.name}: {arg}: invalid value, should be bool" - self.args[arg] = cfg[arg] - - def add_int_arg(self, cfg, args): - if isinstance(args, str): - args = [args] - - for arg in args: - assert arg in cfg, \ - f"{self.name}: mandatory parameter '{arg}' not found" - assert isinstance(cfg[arg], int), \ - f"{self.name}: {arg}: invalid value, should be integer" - self.args[arg] = cfg[arg] - - class MilterMessage(MIMEPart): def replace_header(self, _name, _value, idx=None): _name = _name.lower() @@ -135,462 +69,3 @@ class MilterMessage(MIMEPart): def replace_illegal_chars(string): """Remove illegal characters from header values.""" return "".join(string.replace("\x00", "").splitlines()) - - -JSON_CONFIG_SCHEMA = """ -{ - "$id": "https://example.com/schemas/config", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Root", - "type": "object", - "required": ["rules"], - "additionalProperties": false, - "properties": { - "global": { - "title": "Section global", - "type": "object", - "additionalProperties": false, - "properties": { - "local_addrs": { "$ref": "/schemas/config/hosts" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "socket": { - "title": "Socket", - "type": "string", - "pattern": "^((unix|local):.+|inet6?:[0-9]{1,5}(@.+)?)$" - } - } - }, - "rules": { - "title": "Section rules", - "type": "array", - "items": { - "title": "Rules", - "type": "object", - "required": [ - "actions" - ], - "additionalProperties": false, - "properties": { - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "actions": { - "title": "Section actions", - "type": "array", - "items": { - "title": "Actions", - "type": "object", - "required": ["type"], - "properties": { - "type": { "$ref": "/schemas/config/actiontype" } - }, - "if": { "properties": { "type": { "const": "add_header" } } }, - "then": { "$ref": "/schemas/config/add_header" }, - "else": { - "if": { "properties": { "type": { "const": "mod_header" } } }, - "then": { "$ref": "/schemas/config/mod_header" }, - "else": { - "if": { "properties": { "type": { "const": "del_header" } } }, - "then": { "$ref": "/schemas/config/del_header" }, - "else": { - "if": { "properties": { "type": { "const": "add_disclaimer" } } }, - "then": { "$ref": "/schemas/config/add_disclaimer" }, - "else": { - "if": { "properties": { "type": { "const": "rewrite_links" } } }, - "then": { "$ref": "/schemas/config/rewrite_links" }, - "else": { - "if": { "properties": { "type": { "const": "store" } } }, - "then": { "$ref": "/schemas/config/store" }, - "else": { - "if": { "properties": { "type": { "const": "notify" } } }, - "then": { "$ref": "/schemas/config/notify" }, - "else": { - "if": { "properties": { "type": { "const": "quarantine" } } }, - "then": { "$ref": "/schemas/config/quarantine" }, - "else": { - "additionalProperties": false - } - } - } - } - } - } - } - } - } - } - } - } - } - }, - "$defs": { - "name": { - "$id": "/schemas/config/name", - "title": "Name", - "type": "string", - "pattern": "^.+$" - }, - "hosts": { - "$id": "/schemas/config/hosts", - "title": "Hosts/networks", - "type": "array", - "items": { - "title": "Hosts/Networks", - "type": "string", - "pattern": "^.+$" - } - }, - "pretend": { - "$id": "/schemas/config/pretend", - "title": "Pretend", - "type": "boolean" - }, - "loglevel": { - "$id": "/schemas/config/loglevel", - "title": "Loglevel", - "type": "string", - "pattern": "^(critical|error|warning|info|debug)$" - }, - "actiontype": { - "$id": "/schemas/config/actiontype", - "title": "Action type", - "enum": [ - "add_header", "mod_header", "del_header", "add_disclaimer", - "rewrite_links", "store", "notify", "quarantine"] - }, - "storagetype": { - "$id": "/schemas/config/storagetype", - "title": "Storage type", - "enum": ["file"] - }, - "whitelisttype": { - "$id": "/schemas/config/whitelisttype", - "title": "Whitelist type", - "enum": ["db"] - }, - "field": { - "$id": "/schemas/config/field", - "title": "Field", - "type": "string", - "pattern": "^.+$" - }, - "value": { - "$id": "/schemas/config/value", - "title": "Value", - "type": "string", - "pattern": "^.+$" - }, - "original": { - "$id": "/schemas/config/original", - "title": "Original", - "type": "boolean" - }, - "metavar": { - "$id": "/schemas/config/metavar", - "title": "Meta variable", - "type": "string", - "pattern": "^.+$" - }, - "conditions": { - "$id": "/schemas/config/conditions", - "title": "Conditions", - "type": "object", - "properties": { - "metavar": { "$ref": "/schemas/config/metavar" }, - "local": { - "title": "Local", - "type": "boolean" - }, - "hosts": { - "title": "Hosts/Networks", - "type": "array", - "items":{ - "title": "Host/Network", - "type": "string", - "pattern": "^.+$" - } - }, - "envfrom": { - "title": "Envelope from", - "type": "string", - "pattern": "^.+$" - }, - "envto": { - "title": "Envelope to", - "type": "string", - "pattern": "^.+$" - }, - "header": { - "title": "Header", - "type": "string", - "pattern": "^.+$" - }, - "var": { - "title": "Variable", - "type": "string", - "pattern": "^.+$" - } - }, - "additionalProperties": false, - "anyOf": [ - {"required": ["local"]}, - {"required": ["hosts"]}, - {"required": ["envfrom"]}, - {"required": ["envto"]}, - {"required": ["header"]}, - {"required": ["var"]} - ] - }, - "add_header": { - "$id": "/schemas/config/add_header", - "title": "Add header", - "type": "object", - "required": ["type", "field", "value"], - "properties": { - "type": { "$ref": "/schemas/config/actiontype" }, - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "field": { "$ref": "/schemas/config/field" }, - "value": { "$ref": "/schemas/config/value" } - }, - "additionalProperties": false - }, - "mod_header": { - "$id": "/schemas/config/mod_header", - "title": "Modify header", - "type": "object", - "required": ["type", "field", "value"], - "properties": { - "type": { "$ref": "/schemas/config/actiontype" }, - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "field": { "$ref": "/schemas/config/field" }, - "value": { "$ref": "/schemas/config/value" }, - "search": { - "title": "Search", - "type": "string", - "pattern": "^.+$" - } - }, - "additionalProperties": false - }, - "del_header": { - "$id": "/schemas/config/del_header", - "title": "Delete header", - "type": "object", - "required": ["type", "field"], - "properties": { - "type": { "$ref": "/schemas/config/actiontype" }, - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "field": { "$ref": "/schemas/config/field" }, - "value": { "$ref": "/schemas/config/value" } - }, - "additionalProperties": false - }, - "add_disclaimer": { - "$id": "/schemas/config/add_disclaimer", - "title": "Add disclaimer", - "type": "object", - "required": ["type", "action", "html_template", "text_template"], - "properties": { - "type": { "$ref": "/schemas/config/actiontype" }, - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "action": { - "title": "Action", - "enum": ["append", "prepend"] - }, - "html_template": { - "title": "HTML template", - "type": "string", - "pattern": "^.+$" - }, - "text_template": { - "title": "Text template", - "type": "string", - "pattern": "^.+$" - }, - "error_policy": { - "title": "Action", - "enum": [ - "wrap", "ignore", "reject", - "WRAP", "IGNORE", "REJECT"] - } - }, - "additionalProperties": false - }, - "rewrite_links": { - "$id": "/schemas/config/rewrite_links", - "title": "Rewrite links", - "type": "object", - "required": ["type", "repl"], - "properties": { - "type": { "$ref": "/schemas/config/actiontype" }, - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "repl": { - "title": "Replacement", - "type": "string", - "pattern": "^.+$" - } - }, - "additionalProperties": false - }, - "store": { - "$id": "/schemas/config/store", - "title": "Store", - "type": "object", - "required": ["storage_type"], - "properties": { - "storage_type": { "$ref": "/schemas/config/storagetype" } - }, - "if": { "properties": { "storage_type": { "const": "file" } } }, - "then": { - "properties": { - "type": { "$ref": "/schemas/config/actiontype" }, - "storage_type": { "$ref": "/schemas/config/storagetype" }, - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "original": { "$ref": "/schemas/config/original" }, - "metavar": { "$ref": "/schemas/config/metavar" }, - "directory": { - "title": "Directory", - "type": "string", - "pattern": "^.+$" - } - }, - "additionalProperties": false - }, - "else": { - "additionalProperties": false - } - }, - "notify": { - "$id": "/schemas/config/notify", - "title": "Notify", - "type": "object", - "required": ["smtp_host", "smtp_port", "envelope_from", "from_header", "subject", "template"], - "properties": { - "type": { "$ref": "/schemas/config/actiontype" }, - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "smtp_host": { - "title": "SMTP host", - "type": "string", - "pattern": "^.+$" - }, - "smtp_port": { - "title": "SMTP port", - "type": "number" - }, - "envelope_from": { - "title": "Envelope from", - "type": "string", - "pattern": "^.+$" - }, - "from_header": { - "title": "From-Header", - "type": "string", - "pattern": "^.+$" - }, - "subject": { - "title": "Subject", - "type": "string", - "pattern": "^.+$" - }, - "template": { - "title": "Template", - "type": "string", - "pattern": "^.+$" - }, - "repl_img": { - "title": "Replacement image", - "type": "string", - "pattern": "^.+$" - }, - "embed_imgs": { - "title": "Embedded images", - "type": "array", - "items": { - "title": "Embedded image", - "type": "string", - "pattern": "^.+$" - } - } - }, - "additionalProperties": false - }, - "quarantine": { - "$id": "/schemas/config/quarantine", - "title": "Quarantine", - "type": "object", - "required": ["storage"], - "properties": { - "type": { "$ref": "/schemas/config/actiontype" }, - "name": { "$ref": "/schemas/config/name" }, - "pretend": { "$ref": "/schemas/config/pretend" }, - "conditions": { "$ref": "/schemas/config/conditions" }, - "loglevel": { "$ref": "/schemas/config/loglevel" }, - "storage": { "$ref": "/schemas/config/store" }, - "notification": { "$ref": "/schemas/config/notify" }, - "milter_action": { - "title": "Milter action", - "enum": [ - "reject", "discard", "accept", - "REJECT", "DISCARD", "ACCEPT"] - }, - "reject_reason": { - "title": "Reject reason", - "type": "string", - "pattern": "^.+$" - }, - "whitelist": { - "title": "Whitelist", - "type": "object", - "required": ["type"], - "properties": { - "type": { "$ref": "/schemas/config/whitelisttype" } - }, - "if": { "properties": { "type": { "const": "db" } } }, - "then": { - "required": ["connection", "table"], - "properties": { - "type": { "$ref": "/schemas/config/whitelisttype" }, - "connection": { - "title": "DB connection", - "type": "string", - "pattern": "^.+$" - }, - "table": { - "title": "DB table", - "type": "string", - "pattern": "^.+$" - } - }, - "additionalProperties": false - }, - "else": { - "additionalProperties": false - } - } - } - } - } -} -""" diff --git a/pymodmilter/conditions.py b/pymodmilter/conditions.py index f097ff6..9761c2f 100644 --- a/pymodmilter/conditions.py +++ b/pymodmilter/conditions.py @@ -12,127 +12,77 @@ # along with PyMod-Milter. If not, see . # -__all__ = [ - "ConditionsConfig", - "Conditions"] +__all__ = ["Conditions"] +import logging import re from netaddr import IPAddress, IPNetwork, AddrFormatError -from pymodmilter import BaseConfig, CustomLogger +from pymodmilter import CustomLogger from pymodmilter.whitelist import DatabaseWhitelist -class ConditionsConfig(BaseConfig): - def __init__(self, cfg, local_addrs, debug): - super().__init__(cfg, debug) - - self.local_addrs = local_addrs - - if "local" in cfg: - self.add_bool_arg(cfg, "local") - - if "hosts" in cfg: - assert isinstance(cfg["hosts"], list) and all( - [isinstance(host, str) for host in cfg["hosts"]]), \ - f"{self.name}: hosts: invalid value, " \ - f"should be list of strings" - - self.args["hosts"] = cfg["hosts"] - - for arg in ("envfrom", "envto"): - if arg in cfg: - self.add_string_arg(cfg, arg) - - if "header" in cfg: - self.add_string_arg(cfg, "header") - - if "var" in cfg: - self.add_string_arg(cfg, "var") - - if "metavar" in cfg: - self.add_string_arg(cfg, "metavar") - - if "whitelist" in cfg: - assert isinstance(cfg["whitelist"], dict), \ - f"{self.name}: whitelist: invalid value, " \ - f"should be dict" - whitelist = cfg["whitelist"] - assert "type" in whitelist, \ - f"{self.name}: whitelist: mandatory parameter 'type' not found" - assert isinstance(whitelist["type"], str), \ - f"{self.name}: whitelist: type: invalid value, " \ - f"should be string" - self.args["whitelist"] = { - "type": whitelist["type"], - "name": f"{self.name}: whitelist"} - if whitelist["type"] == "db": - for arg in ["connection", "table"]: - assert arg in whitelist, \ - f"{self.name}: whitelist: mandatory parameter " \ - f"'{arg}' not found" - assert isinstance(whitelist[arg], str), \ - f"{self.name}: whitelist: {arg}: invalid value, " \ - f"should be string" - self.args["whitelist"][arg] = whitelist[arg] - - else: - raise RuntimeError( - f"{self.name}: whitelist: type: invalid type") - - self.logger.debug(f"{self.name}: " - f"loglevel={self.loglevel}, " - f"args={self.args}") - - class Conditions: """Conditions to implement conditions for rules and actions.""" - def __init__(self, cfg): - self.logger = cfg.logger - self.name = cfg.name - self.local_addrs = cfg.local_addrs + def __init__(self, cfg, local_addrs, debug): + self.cfg = cfg + self.local_addrs = local_addrs + + self.logger = logging.getLogger(cfg["name"]) + self.logger.setLevel(cfg.get_loglevel(debug)) for arg in ("local", "hosts", "envfrom", "envto", "header", "metavar", "var"): - value = cfg.args[arg] if arg in cfg.args else None - setattr(self, arg, value) - if value is None: + if arg not in cfg: + setattr(self, arg, None) continue - elif arg == "hosts": + + if arg == "hosts": try: - hosts = [] - for host in self.hosts: - hosts.append(IPNetwork(host)) + self.hosts = [] + for host in cfg["hosts"]: + self.hosts.append(IPNetwork(host)) except AddrFormatError as e: raise RuntimeError(e) - - self.hosts = hosts elif arg in ("envfrom", "envto"): try: setattr(self, arg, re.compile( - getattr(self, arg), re.IGNORECASE)) + cfg[arg], re.IGNORECASE)) except re.error as e: raise RuntimeError(e) - elif arg == "header": try: self.header = re.compile( - self.header, re.IGNORECASE + re.DOTALL + re.MULTILINE) + cfg["header"], + re.IGNORECASE + re.DOTALL + re.MULTILINE) except re.error as e: raise RuntimeError(e) - - if "whitelist" in cfg.args: - wl_cfg = cfg.args["whitelist"] - if wl_cfg["type"] == "db": - self.whitelist = DatabaseWhitelist(wl_cfg) else: - raise RuntimeError("invalid storage type") + setattr(self, arg, cfg[arg]) + + self.whitelist = cfg["whitelist"] if "whitelist" in cfg else None + if self.whitelist is not None: + self.whitelist["name"] = f"{cfg['name']}: whitelist" + self.whitelist["loglevel"] = cfg["loglevel"] + if self.whitelist["type"] == "db": + self.whitelist = DatabaseWhitelist(self.whitelist, debug) + else: + raise RuntimeError("invalid whitelist type") + + def __str__(self): + cfg = [] + for arg in ("local", "hosts", "envfrom", "envto", "header", + "var", "metavar"): + if arg in self.cfg: + cfg.append(f"{arg}={self.cfg[arg]}") + if self.whitelist is not None: + cfg.append(f"whitelist={self.whitelist}") + return "Conditions(" + ", ".join(cfg) + ")" def match_host(self, host): logger = CustomLogger( - self.logger, {"name": self.name}) - + self.logger, {"name": self.cfg["name"]}) ip = IPAddress(host) if self.local is not None: @@ -145,11 +95,11 @@ class Conditions: if is_local != self.local: logger.debug( f"ignore host {host}, " - f"condition local does not match") + f"local does not match") return False logger.debug( - f"condition local matches for host {host}") + f"local matches for host {host}") if self.hosts is not None: found = False @@ -161,39 +111,39 @@ class Conditions: if not found: logger.debug( f"ignore host {host}, " - f"condition hosts does not match") + f"hosts does not match") return False logger.debug( - f"condition hosts matches for host {host}") + f"hosts matches for host {host}") return True - def get_wl_rcpts(self, mailfrom, rcpts): + def get_wl_rcpts(self, mailfrom, rcpts, logger): if not self.whitelist: return {} wl_rcpts = [] for rcpt in rcpts: - if self.whitelist.check(mailfrom, rcpt): + if self.whitelist.check(mailfrom, rcpt, logger): wl_rcpts.append(rcpt) return wl_rcpts def match(self, milter): logger = CustomLogger( - self.logger, {"qid": milter.qid, "name": self.name}) + self.logger, {"qid": milter.qid, "name": self.cfg["name"]}) if self.envfrom is not None: envfrom = milter.msginfo["mailfrom"] if not self.envfrom.match(envfrom): logger.debug( f"ignore envelope-from address {envfrom}, " - f"condition envfrom does not match") + f"envfrom does not match") return False logger.debug( - f"condition envfrom matches for " + f"envfrom matches for " f"envelope-from address {envfrom}") if self.envto is not None: @@ -205,11 +155,11 @@ class Conditions: if not self.envto.match(to): logger.debug( f"ignore envelope-to address {envto}, " - f"condition envto does not match") + f"envto does not match") return False logger.debug( - f"condition envto matches for " + f"envto matches for " f"envelope-to address {envto}") if self.header is not None: @@ -219,7 +169,7 @@ class Conditions: match = self.header.search(header) if match: logger.debug( - f"condition header matches for " + f"header matches for " f"header: {header}") if self.metavar is not None: named_subgroups = match.groupdict(default=None) @@ -233,7 +183,7 @@ class Conditions: if not match: logger.debug( "ignore message, " - "condition header does not match") + "header does not match") return False if self.var is not None: diff --git a/pymodmilter/config.py b/pymodmilter/config.py new file mode 100644 index 0000000..48020f5 --- /dev/null +++ b/pymodmilter/config.py @@ -0,0 +1,330 @@ +# PyMod-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyMod-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyMod-Milter. If not, see . +# + +__all__ = [ + "BaseConfig", + "ConditionsConfig", + "AddHeaderConfig", + "ModHeaderConfig", + "DelHeaderConfig", + "AddDisclaimerConfig", + "RewriteLinksConfig", + "StoreConfig", + "NotifyConfig", + "WhitelistConfig", + "QuarantineConfig", + "ActionConfig", + "RuleConfig", + "MilterConfig"] + +import jsonschema +import logging + + +class BaseConfig: + JSON_SCHEMA = { + "type": "object", + "required": [], + "additionalProperties": True, + "properties": { + "loglevel": {"type": "string", "default": "info"}}} + + def __init__(self, config): + required = self.JSON_SCHEMA["required"] + properties = self.JSON_SCHEMA["properties"] + for p in properties.keys(): + if p in required: + continue + elif p not in config and "default" in properties[p]: + config[p] = properties[p]["default"] + try: + jsonschema.validate(config, self.JSON_SCHEMA) + except jsonschema.exceptions.ValidationError as e: + raise RuntimeError(e) + + self._config = config + + def __getitem__(self, key): + return self._config[key] + + def __setitem__(self, key, value): + self._config[key] = value + + def __delitem__(self, key): + del self._config[key] + + def __contains__(self, key): + return key in self._config + + def keys(self): + return self._config.keys() + + def items(self): + return self._config.items() + + def get_loglevel(self, debug): + if debug: + level = logging.DEBUG + else: + level = getattr(logging, self["loglevel"].upper(), None) + assert isinstance(level, int), \ + "loglevel: invalid value" + return level + + def get_config(self): + return self._config + +class WhitelistConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["type"], + "additionalProperties": True, + "properties": { + "type": {"enum": ["db"]}}, + "if": {"properties": {"type": {"const": "db"}}}, + "then": { + "required": ["connection", "table"], + "additionalProperties": False, + "properties": { + "type": {"type": "string"}, + "connection": {"type": "string"}, + "table": {"type": "string"}}}} + + +class ConditionsConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": [], + "additionalProperties": False, + "properties": { + "metavar": {"type": "string"}, + "local": {"type": "boolean"}, + "hosts": {"type": "array", + "items": {"type": "string"}}, + "envfrom": {"type": "string"}, + "envto": {"type": "string"}, + "header": {"type": "string"}, + "var": {"type": "string"}, + "whitelist": {"type": "object"}}} + + def __init__(self, config, rec=True): + super().__init__(config) + if rec: + if "whitelist" in self: + self["whitelist"] = WhitelistConfig(self["whitelist"]) + + +class AddHeaderConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["field", "value"], + "additionalProperties": False, + "properties": { + "field": {"type": "string"}, + "value": {"type": "string"}}} + + +class ModHeaderConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["field", "value"], + "additionalProperties": False, + "properties": { + "field": {"type": "string"}, + "value": {"type": "string"}, + "search": {"type": "string"}}} + + +class DelHeaderConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["field"], + "additionalProperties": False, + "properties": { + "field": {"type": "string"}, + "value": {"type": "string"}}} + + +class AddDisclaimerConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["action", "html_template", "text_template"], + "additionalProperties": False, + "properties": { + "action": {"type": "string"}, + "html_template": {"type": "string"}, + "text_template": {"type": "string"}, + "error_policy": {"type": "string"}}} + + +class RewriteLinksConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["repl"], + "additionalProperties": False, + "properties": { + "repl": {"type": "string"}}} + + +class StoreConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["type"], + "additionalProperties": True, + "properties": { + "type": {"enum": ["file"]}}, + "if": {"properties": {"type": {"const": "file"}}}, + "then": { + "required": ["directory"], + "additionalProperties": False, + "properties": { + "type": {"type": "string"}, + "directory": {"type": "string"}, + "metavar": {"type": "string"}, + "original": {"type": "boolean", "default": True}}}} + + +class NotifyConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["type"], + "additionalProperties": True, + "properties": { + "type": {"enum": ["email"]}}, + "if": {"properties": {"type": {"const": "email"}}}, + "then": { + "required": ["smtp_host", "smtp_port", "envelope_from", + "from_header", "subject", "template"], + "additionalProperties": False, + "properties": { + "type": {"type": "string"}, + "smtp_host": {"type": "string"}, + "smtp_port": {"type": "number"}, + "envelope_from": {"type": "string"}, + "from_header": {"type": "string"}, + "subject": {"type": "string"}, + "template": {"type": "string"}, + "repl_img": {"type": "string"}, + "embed_imgs": { + "type": "array", + "items": {"type": "string"}, + "default": True}}}} + + +class QuarantineConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["store"], + "additionalProperties": False, + "properties": { + "name": {"type": "string"}, + "notify": {"type": "object"}, + "milter_action": {"type": "string"}, + "reject_reason": {"type": "string"}, + "whitelist": {"type": "object"}, + "store": {"type": "object"}}} + + def __init__(self, config, rec=True): + super().__init__(config) + if rec: + self["store"] = StoreConfig(self["store"]) + if "notify" in self: + self["notify"] = NotifyConfig(self["notify"]) + if "whitelist" in self: + self["whitelist"] = ConditionsConfig( + {"whitelist": self["whitelist"]}, rec) + + +class ActionConfig(BaseConfig): + ACTION_TYPES = { + "add_header": AddHeaderConfig, + "mod_header": ModHeaderConfig, + "del_header": DelHeaderConfig, + "add_disclaimer": AddDisclaimerConfig, + "rewrite_links": RewriteLinksConfig, + "store": StoreConfig, + "notify": NotifyConfig, + "quarantine": QuarantineConfig} + + JSON_SCHEMA = { + "type": "object", + "required": ["type", "args"], + "additionalProperties": False, + "properties": { + "name": {"type": "string", "default": "action"}, + "loglevel": {"type": "string", "default": "info"}, + "pretend": {"type": "boolean", "default": False}, + "conditions": {"type": "object"}, + "type": {"enum": list(ACTION_TYPES.keys())}, + "args": {"type": "object"}}} + + def __init__(self, config, rec=True): + super().__init__(config) + if rec: + if "conditions" in self: + self["conditions"] = ConditionsConfig(self["conditions"]) + self["action"] = self.ACTION_TYPES[self["type"]](self["args"]) + + +class RuleConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["actions"], + "additionalProperties": False, + "properties": { + "name": {"type": "string", "default": "rule"}, + "loglevel": {"type": "string", "default": "info"}, + "pretend": {"type": "boolean", "default": False}, + "conditions": {"type": "object"}, + "actions": {"type": "array"}}} + + def __init__(self, config, rec=True): + super().__init__(config) + if rec: + if "conditions" in self: + self["conditions"] = ConditionsConfig(self["conditions"]) + + actions = [] + for idx, action in enumerate(self["actions"]): + actions.append(ActionConfig(action, rec)) + self["actions"] = actions + + +class MilterConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["rules"], + "additionalProperties": False, + "properties": { + "socket": {"type": "string"}, + "local_addrs": {"type": "array", + "items": {"type": "string"}, + "default": [ + "fe80::/64", + "::1/128", + "127.0.0.0/8", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16"]}, + "loglevel": {"type": "string", "default": "info"}, + "pretend": {"type": "boolean", "default": False}, + "rules": {"type": "array"}}} + + def __init__(self, config, rec=True): + super().__init__(config) + if rec: + rules = [] + for idx, rule in enumerate(self["rules"]): + rules.append(RuleConfig(rule, rec)) + self["rules"] = rules diff --git a/pymodmilter/modify.py b/pymodmilter/modify.py index e1d947f..8222a81 100644 --- a/pymodmilter/modify.py +++ b/pymodmilter/modify.py @@ -30,18 +30,19 @@ from email.message import MIMEPart from email.policy import SMTPUTF8 from pymodmilter import replace_illegal_chars +from pymodmilter.base import CustomLogger class AddHeader: """Add a mail header field.""" _headersonly = True - def __init__(self, field, value): + def __init__(self, field, value, pretend=False): self.field = field self.value = value + self.pretend = pretend - def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + def execute(self, milter, logger): header = f"{self.field}: {self.value}" if logger.getEffectiveLevel() == logging.DEBUG: logger.debug(f"add_header: {header}") @@ -49,8 +50,7 @@ class AddHeader: logger.info(f"add_header: {header[0:70]}") milter.msg.add_header(self.field, self.value) - - if not pretend: + if not self.pretend: milter.addheader(self.field, self.value) @@ -58,21 +58,21 @@ class ModHeader: """Change the value of a mail header field.""" _headersonly = True - def __init__(self, field, value, search=None): - self.value = value - + def __init__(self, field, value, search=None, pretend=False): try: self.field = re.compile(field, re.IGNORECASE) - if search is not None: + self.search = search + if self.search is not None: self.search = re.compile( - search, re.MULTILINE + re.DOTALL + re.IGNORECASE) - else: - self.search = search + self.search, re.MULTILINE + re.DOTALL + re.IGNORECASE) + except re.error as e: raise RuntimeError(e) - def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + self.value = value + self.pretend = pretend + + def execute(self, milter, logger): idx = defaultdict(int) for i, (field, value) in enumerate(milter.msg.items()): @@ -103,12 +103,13 @@ class ModHeader: if logger.getEffectiveLevel() == logging.DEBUG: logger.debug(f"mod_header: {header}: {new_header}") else: - logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}") + logger.info( + f"mod_header: {header[0:70]}: {new_header[0:70]}") milter.msg.replace_header( field, replace_illegal_chars(new_value), idx=idx[field_lower]) - if not pretend: + if not self.pretend: milter.chgheader(field, new_value, idx=idx[field_lower]) @@ -116,19 +117,19 @@ class DelHeader: """Delete a mail header field.""" _headersonly = True - def __init__(self, field, value=None): + def __init__(self, field, value=None, pretend=False): try: self.field = re.compile(field, re.IGNORECASE) - if value is not None: + self.value = value + if self.value is not None: self.value = re.compile( value, re.MULTILINE + re.DOTALL + re.IGNORECASE) - else: - self.value = value except re.error as e: raise RuntimeError(e) - def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + self.pretend = pretend + + def execute(self, milter, logger): idx = defaultdict(int) for field, value in milter.msg.items(): @@ -148,7 +149,7 @@ class DelHeader: logger.info(f"del_header: {header[0:70]}") milter.msg.remove_header(field, idx=idx[field_lower]) - if not pretend: + if not self.pretend: milter.chgheader(field, "", idx=idx[field_lower]) idx[field_lower] -= 1 @@ -220,7 +221,10 @@ class AddDisclaimer: """Append or prepend a disclaimer to the mail body.""" _headersonly = False - def __init__(self, text_template, html_template, action, error_policy): + def __init__(self, text_template, html_template, action, error_policy, + pretend=False): + self.text_template_path = text_template + self.html_template_path = html_template try: with open(text_template, "r") as f: self.text_template = f.read() @@ -230,11 +234,11 @@ class AddDisclaimer: except IOError as e: raise RuntimeError(e) - body = html.find('body') self.html_template = body or html self.action = action self.error_policy = error_policy + self.pretend = pretend def patch_message_body(self, milter, logger): text_body, text_content = _get_body_content(milter.msg, "plain") @@ -277,8 +281,7 @@ class AddDisclaimer: html_body.set_param("charset", "UTF-8", header="Content-Type") del html_body["MIME-Version"] - def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + def execute(self, milter, logger): old_headers = milter.msg.items() try: @@ -313,7 +316,7 @@ class AddDisclaimer: "unable to wrap message in a new message envelope, " "give up ...") - if not pretend: + if not self.pretend: milter.update_headers(old_headers) milter.replacebody() @@ -322,11 +325,11 @@ class RewriteLinks: """Rewrite link targets in the mail html body.""" _headersonly = False - def __init__(self, repl): + def __init__(self, repl, pretend=False): self.repl = repl + self.pretend = pretend - def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + def execute(self, milter, logger): html_body, html_content = _get_body_content(milter.msg, "html") if html_content is not None: soup = BeautifulSoup(html_content, "html.parser") @@ -353,5 +356,35 @@ class RewriteLinks: html_body.set_param("charset", "UTF-8", header="Content-Type") del html_body["MIME-Version"] - if not pretend: + if not self.pretend: milter.replacebody() + + +class Modify: + MODIFICATION_TYPES = { + "add_header": AddHeader, + "mod_header": ModHeader, + "del_header": DelHeader, + "add_disclaimer": AddDisclaimer, + "rewrite_links": RewriteLinks} + + def __init__(self, cfg, local_addrs, debug): + self.cfg = cfg + self.logger = logging.getLogger(cfg["name"]) + self.logger.setLevel(cfg.get_loglevel(debug)) + cfg["args"]["pretend"] = cfg["pretend"] + self._modification = self.MODIFICATION_TYPES[cfg["type"]]( + **cfg["args"]) + self._headersonly = self._modification._headersonly + + def __str__(self): + cfg = [] + for key, value in self.cfg["args"].items(): + cfg.append(f"{key}={value}") + class_name = type(self._modification).__name__ + return f"{class_name}(" + ", ".join(cfg) + ")" + + def execute(self, milter): + logger = CustomLogger( + self.logger, {"name": self.cfg["name"], "qid": milter.qid}) + self._modification.execute(milter, logger) diff --git a/pymodmilter/notify.py b/pymodmilter/notify.py index 0c436d5..32d3270 100644 --- a/pymodmilter/notify.py +++ b/pymodmilter/notify.py @@ -29,6 +29,7 @@ from html import escape from os.path import basename from urllib.parse import quote +from pymodmilter.base import CustomLogger from pymodmilter import mailer @@ -36,11 +37,10 @@ class BaseNotification: "Notification base class" _headersonly = True - def __init__(self): - self.logger = logging.getLogger(__name__) - return + def __init__(self, pretend=False): + self.pretend = pretend - def execute(self, milter, pretend=False, logger=None): + def execute(self, milter, logger): return @@ -112,9 +112,8 @@ class EMailNotification(BaseNotification): def __init__(self, smtp_host, smtp_port, envelope_from, from_header, subject, template, embed_imgs=[], repl_img=None, - strip_imgs=False, parser_lib="lxml"): - super().__init__() - + strip_imgs=False, parser_lib="lxml", pretend=False): + super().__init__(pretend) self.smtp_host = smtp_host self.smtp_port = smtp_port self.mailfrom = envelope_from @@ -143,11 +142,8 @@ class EMailNotification(BaseNotification): self.parser_lib = parser_lib - def get_email_body_soup(self, msg, logger=None): + def get_email_body_soup(self, msg, logger): "Extract and decode email body and return it as BeautifulSoup object." - if logger is None: - logger = self.logger - # try to find the body part logger.debug("trying to find email body") try: @@ -193,11 +189,8 @@ class EMailNotification(BaseNotification): return soup - def sanitize(self, soup, logger=None): + def sanitize(self, soup, logger): "Sanitize mail html text." - if logger is None: - logger = self.logger - logger.debug("sanitizing email text") # completly remove bad elements @@ -230,13 +223,9 @@ class EMailNotification(BaseNotification): del(element.attrs[attribute]) return soup - def notify(self, msg, qid, mailfrom, recipients, - template_vars=defaultdict(str), synchronous=False, - logger=None): + def notify(self, msg, qid, mailfrom, recipients, logger, + template_vars=defaultdict(str), synchronous=False): "Notify recipients via email." - if logger is None: - logger = self.logger - # extract body from email soup = self.get_email_body_soup(msg, logger) @@ -262,7 +251,8 @@ class EMailNotification(BaseNotification): # sending email notifications for recipient in recipients: - logger.debug(f"generating notification email for '{recipient}'") + logger.debug( + f"generating notification email for '{recipient}'") logger.debug("parsing email template") # generate dict containing all template variables @@ -313,15 +303,40 @@ class EMailNotification(BaseNotification): self.mailfrom, recipient, newmsg.as_string(), "notification email") - def execute(self, milter, pretend=False, - logger=None): - super().execute(milter, pretend, logger) - - if logger is None: - logger = self.logger + def execute(self, milter, logger): + super().execute(milter, logger) self.notify(msg=milter.msg, qid=milter.qid, mailfrom=milter.msginfo["mailfrom"], recipients=milter.msginfo["rcpts"], template_vars=milter.msginfo["vars"], logger=logger) + + +class Notify: + NOTIFICATION_TYPES = { + "email": EMailNotification} + + def __init__(self, cfg, local_addrs, debug): + self.cfg = cfg + self.logger = logging.getLogger(cfg["name"]) + self.logger.setLevel(cfg.get_loglevel(debug)) + + nodification_type = cfg["args"]["type"] + del cfg["args"]["type"] + cfg["args"]["pretend"] = cfg["pretend"] + self._notification = self.NOTIFICATION_TYPES[nodification_type]( + **cfg["args"]) + self._headersonly = self._notification._headersonly + + def __str__(self): + cfg = [] + for key, value in self.cfg["args"].items(): + cfg.append(f"{key}={value}") + class_name = type(self._notification).__name__ + return f"{class_name}(" + ", ".join(cfg) + ")" + + def execute(self, milter): + logger = CustomLogger( + self.logger, {"name": self.cfg["name"], "qid": milter.qid}) + self._notification.execute(milter, logger) diff --git a/pymodmilter/rule.py b/pymodmilter/rule.py index 5052d2e..5a831c2 100644 --- a/pymodmilter/rule.py +++ b/pymodmilter/rule.py @@ -12,83 +12,53 @@ # along with PyMod-Milter. If not, see . # -__all__ = [ - "RuleConfig", - "Rule"] +__all__ = ["Rule"] -from pymodmilter import BaseConfig -from pymodmilter.action import ActionConfig, Action -from pymodmilter.conditions import ConditionsConfig, Conditions +import logging - -class RuleConfig(BaseConfig): - def __init__(self, cfg, local_addrs, debug=False): - super().__init__(cfg, debug) - - self.conditions = None - self.actions = [] - - self.pretend = False - if "pretend" in cfg: - assert isinstance(cfg["pretend"], bool), \ - f"{self.name}: pretend: invalid value, should be bool" - self.pretend = cfg["pretend"] - - assert "actions" in cfg, \ - f"{self.name}: mandatory parameter 'actions' not found" - actions = cfg["actions"] - assert isinstance(actions, list), \ - f"{self.name}: actions: invalid value, should be list" - - self.logger.debug(f"{self.name}: pretend={self.pretend}, " - f"loglevel={self.loglevel}") - - if "conditions" in cfg: - assert isinstance(cfg["conditions"], dict), \ - f"{self.name}: conditions: invalid value, should be dict" - cfg["conditions"]["name"] = f"{self.name}: condition" - if "loglevel" not in cfg["conditions"]: - cfg["conditions"]["loglevel"] = self.loglevel - self.conditions = ConditionsConfig( - cfg["conditions"], local_addrs, debug) - else: - self.conditions = None - - for idx, action_cfg in enumerate(cfg["actions"]): - if "name" in action_cfg: - assert isinstance(action_cfg["name"], str), \ - f"{self.name}: Action #{idx}: name: invalid value, " \ - f"should be string" - action_cfg["name"] = f"{self.name}: {action_cfg['name']}" - else: - action_cfg["name"] = f"{self.name}: Action #{idx}" - - if "loglevel" not in action_cfg: - action_cfg["loglevel"] = self.loglevel - if "pretend" not in action_cfg: - action_cfg["pretend"] = self.pretend - self.actions.append( - ActionConfig(action_cfg, local_addrs, debug)) +from pymodmilter.action import Action +from pymodmilter.conditions import Conditions class Rule: """ Rule to implement multiple actions on emails. """ + def __init__(self, cfg, local_addrs, debug): + self.cfg = cfg + self.logger = logging.getLogger(cfg["name"]) + self.logger.setLevel(cfg.get_loglevel(debug)) - def __init__(self, cfg): - self.logger = cfg.logger - - if cfg.conditions is None: - self.conditions = None - else: - self.conditions = Conditions(cfg.conditions) + self.conditions = cfg["conditions"] if "conditions" in cfg else None + if self.conditions is not None: + self.conditions["name"] = f"{cfg['name']}: condition" + self.conditions["loglevel"] = cfg["loglevel"] + self.conditions = Conditions(self.conditions, local_addrs, debug) self.actions = [] - for action_cfg in cfg.actions: - self.actions.append(Action(action_cfg)) + for idx, action_cfg in enumerate(cfg["actions"]): + if "name" in action_cfg: + action_cfg["name"] = f"{cfg['name']}: {action_cfg['name']}" + else: + action_cfg["name"] = f"action#{idx}" + if "loglevel" not in action_cfg: + action_cfg["loglevel"] = cfg["loglevel"] + if "pretend" not in action_cfg: + action_cfg["pretend"] = cfg["pretend"] + self.actions.append(Action(action_cfg, local_addrs, debug)) - self.pretend = cfg.pretend + def __str__(self): + cfg = [] + for key in ["name", "loglevel", "pretend"]: + value = self.cfg[key] + cfg.append(f"{key}={value}") + if self.conditions is not None: + cfg.append(f"conditions={self.conditions}") + actions = [] + for action in self.actions: + actions.append(str(action)) + cfg.append("actions=[" + ", ".join(actions) + "]") + return "Rule(" + ", ".join(cfg) + ")" def execute(self, milter): """Execute all actions of this rule.""" diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 6bd6304..bc93699 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -16,13 +16,17 @@ __all__ = ["main"] import Milter import argparse +import json import logging import logging.handlers +import os +import re import sys from pymodmilter import mailer -from pymodmilter import ModifyMilterConfig, ModifyMilter +from pymodmilter import ModifyMilter from pymodmilter import __version__ as version +from pymodmilter.config import MilterConfig def main(): @@ -80,36 +84,52 @@ def main(): logger.setLevel(logging.INFO) try: - logger.debug("prepare milter configuration") - cfg = ModifyMilterConfig(args.config, args.debug) + logger.debug("read milter configuration") + + try: + with open(args.config, "r") as fh: + # remove lines with leading # (comments), they + # are not allowed in json + cfg = re.sub(r"(?m)^\s*#.*\n?", "", fh.read()) + except IOError as e: + raise RuntimeError(f"unable to open/read config file: {e}") + + try: + cfg = json.loads(cfg) + except json.JSONDecodeError as e: + cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())] + msg = "\n".join(cfg_text) + raise RuntimeError(f"{e}\n{msg}") + + cfg = MilterConfig(cfg) if not args.debug: - logger.setLevel(cfg.loglevel) + logger.setLevel(cfg.get_loglevel(args.debug)) if args.socket: socket = args.socket - elif cfg.socket: - socket = cfg.socket + elif cfg["socket"]: + socket = cfg["socket"] else: raise RuntimeError( "listening socket is neither specified on the command line " "nor in the configuration file") - if not cfg.rules: + if not cfg["rules"]: raise RuntimeError("no rules configured") - for rule_cfg in cfg.rules: - if not rule_cfg.actions: + for rule in cfg["rules"]: + if not rule["actions"]: raise RuntimeError( - f"{rule_cfg.name}: no actions configured") + f"{rule['name']}: no actions configured") except (RuntimeError, AssertionError) as e: - logger.error(e) + logger.error(f"error in config file: {e}") sys.exit(255) try: - ModifyMilter.set_config(cfg) - except (RuntimeError, ValueError) as e: + ModifyMilter.set_config(cfg, args.debug) + except RuntimeError as e: logger.error(e) sys.exit(254) @@ -147,6 +167,7 @@ def main(): mailer.queue.put(None) logger.info("pymodmilter stopped") + sys.exit(rc) diff --git a/pymodmilter/storage.py b/pymodmilter/storage.py index de4a2a9..f64d1a0 100644 --- a/pymodmilter/storage.py +++ b/pymodmilter/storage.py @@ -28,24 +28,26 @@ from time import gmtime from pymodmilter.base import CustomLogger from pymodmilter.conditions import Conditions +from pymodmilter.config import ActionConfig +from pymodmilter.notify import Notify class BaseMailStorage: "Mail storage base class" _headersonly = True - def __init__(self, original=False, metadata=False, metavar=None): + def __init__(self, original=False, metadata=False, metavar=None, + pretend=False): self.original = original self.metadata = metadata self.metavar = metavar - return + self.pretend = False def add(self, data, qid, mailfrom="", recipients=[]): "Add email to storage." return ("", "") - def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + def execute(self, milter, logger): return def find(self, mailfrom=None, recipients=None, older_than=None): @@ -69,9 +71,9 @@ class FileMailStorage(BaseMailStorage): "Storage class to store mails on filesystem." _headersonly = False - def __init__(self, directory, original=False, metadata=False, - metavar=None): - super().__init__(original, metadata, metavar) + def __init__(self, directory, original=False, metadata=False, metavar=None, + pretend=False): + super().__init__(original, metadata, metavar, pretend) # check if directory exists and is writable if not os.path.isdir(directory) or \ not os.access(directory, os.W_OK): @@ -81,6 +83,15 @@ class FileMailStorage(BaseMailStorage): self.directory = directory self._metadata_suffix = ".metadata" + def __str__(self): + cfg = [] + cfg.append(f"metadata={self.metadata}") + cfg.append(f"metavar={self.metavar}") + cfg.append(f"pretend={self.pretend}") + cfg.append(f"directory={self.directory}") + cfg.append(f"original={self.original}") + return "FileMailStorage(" + ", ".join(cfg) + ")" + def get_storageid(self, qid): timestamp = datetime.now().strftime("%Y%m%d%H%M%S") return f"{timestamp}_{qid}" @@ -144,8 +155,7 @@ class FileMailStorage(BaseMailStorage): return storage_id, metafile, datafile - def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + def execute(self, milter, logger): if self.original: milter.fp.seek(0) data = milter.fp.read @@ -158,7 +168,7 @@ class FileMailStorage(BaseMailStorage): recipients = list(milter.msginfo["rcpts"]) subject = milter.msg["subject"] or "" - if not pretend: + if not self.pretend: storage_id, metafile, datafile = self.add( data(), milter.qid, mailfrom, recipients, subject) logger.info(f"stored message in file {datafile}") @@ -174,7 +184,7 @@ class FileMailStorage(BaseMailStorage): def get_metadata(self, storage_id): "Return metadata of email in storage." - super(FileMailStorage, self).get_metadata(storage_id) + super().get_metadata(storage_id) if not self.metadata: return None @@ -197,7 +207,7 @@ class FileMailStorage(BaseMailStorage): def find(self, mailfrom=None, recipients=None, older_than=None): "Find emails in storage." - super(FileMailStorage, self).find(mailfrom, recipients, older_than) + super().find(mailfrom, recipients, older_than) if isinstance(mailfrom, str): mailfrom = [mailfrom] if isinstance(recipients, str): @@ -238,7 +248,7 @@ class FileMailStorage(BaseMailStorage): def delete(self, storage_id, recipients=None): "Delete email from storage." - super(FileMailStorage, self).delete(storage_id, recipients) + super().delete(storage_id, recipients) if not recipients or not self.metadata: self._remove(storage_id) @@ -264,7 +274,7 @@ class FileMailStorage(BaseMailStorage): self._save_metafile(metafile, metadata) def get_mail(self, storage_id): - super(FileMailStorage, self).get_mail(storage_id) + super().get_mail(storage_id) metadata = self.get_metadata(storage_id) _, datafile = self._get_file_paths(storage_id) @@ -275,31 +285,101 @@ class FileMailStorage(BaseMailStorage): return (metadata, data) +class Store: + STORAGE_TYPES = { + "file": FileMailStorage} + + def __init__(self, cfg, local_addrs, debug): + self.cfg = cfg + self.logger = logging.getLogger(cfg["name"]) + self.logger.setLevel(cfg.get_loglevel(debug)) + + storage_type = cfg["args"]["type"] + del cfg["args"]["type"] + cfg["args"]["pretend"] = cfg["pretend"] + self._storage = self.STORAGE_TYPES[storage_type]( + **cfg["args"]) + self._headersonly = self._storage._headersonly + + def __str__(self): + cfg = [] + for key, value in self.cfg["args"].items(): + cfg.append(f"{key}={value}") + class_name = type(self._storage).__name__ + return f"{class_name}(" + ", ".join(cfg) + ")" + + def execute(self, milter): + logger = CustomLogger( + self.logger, {"name": self.cfg["name"], "qid": milter.qid}) + self._storage.execute(milter, logger) + + class Quarantine: "Quarantine class." _headersonly = False - def __init__(self, storage, notification=None, whitelist=None, - milter_action=None, reject_reason="Message rejected"): - self.storage = storage.action(**storage.args, metadata=True) - self.storage_name = storage.name - self.storage_logger = storage.logger + def __init__(self, cfg, local_addrs, debug): + self.cfg = cfg + self.logger = logging.getLogger(cfg["name"]) + self.logger.setLevel(cfg.get_loglevel(debug)) - self.notification = notification - if self.notification is not None: - self.notification = notification.action(**notification.args) - self.notification_name = notification.name - self.notification_logger = notification.logger - self.whitelist = Conditions(whitelist) - self.milter_action = milter_action - self.reject_reason = reject_reason + store_cfg = ActionConfig({ + "name": cfg["name"], + "loglevel": cfg["loglevel"], + "pretend": cfg["pretend"], + "type": "store", + "args": cfg["args"]["store"].get_config()}) + self.store = Store(store_cfg, local_addrs, debug) - def execute(self, milter, pretend=False, - logger=logging.getLogger(__name__)): + self.notify = None + if "notify" in cfg["args"]: + notify_cfg = ActionConfig({ + "name": cfg["name"], + "loglevel": cfg["loglevel"], + "pretend": cfg["pretend"], + "type": "notify", + "args": cfg["args"]["notify"].get_config()}) + self.notify = Notify(notify_cfg, local_addrs, debug) + + self.whitelist = None + if "whitelist" in cfg["args"]: + whitelist_cfg = cfg["args"]["whitelist"] + whitelist_cfg["name"] = cfg["name"] + whitelist_cfg["loglevel"] = cfg["loglevel"] + self.whitelist = Conditions( + whitelist_cfg, + local_addrs=[], + debug=debug) + + self.milter_action = None + if "milter_action" in cfg["args"]: + self.milter_action = cfg["args"]["milter_action"] + self.reject_reason = None + if "reject_reason" in cfg["args"]: + self.reject_reason = cfg["args"]["reject_reason"] + + def __str__(self): + cfg = [] + cfg.append(f"store={str(self.store)}") + if self.notify is not None: + cfg.append(f"notify={str(self.notify)}") + if self.whitelist is not None: + cfg.append(f"whitelist={str(self.whitelist)}") + for key in ["milter_action", "reject_reason"]: + if key not in self.cfg["args"]: + continue + value = self.cfg["args"][key] + cfg.append(f"{key}={value}") + class_name = type(self).__name__ + return f"{class_name}(" + ", ".join(cfg) + ")" + + def execute(self, milter): + logger = CustomLogger( + self.logger, {"name": self.cfg["name"], "qid": milter.qid}) wl_rcpts = [] if self.whitelist: wl_rcpts = self.whitelist.get_wl_rcpts( - milter.msginfo["mailfrom"], milter.msginfo["rcpts"]) + milter.msginfo["mailfrom"], milter.msginfo["rcpts"], logger) logger.info(f"whitelisted recipients: {wl_rcpts}") rcpts = [ @@ -312,14 +392,10 @@ class Quarantine: logger.info(f"add to quarantine for recipients: {rcpts}") milter.msginfo["rcpts"] = rcpts - custom_logger = CustomLogger( - self.storage_logger, {"name": self.storage_name}) - self.storage.execute(milter, pretend, custom_logger) + self.store.execute(milter) - if self.notification is not None: - custom_logger = CustomLogger( - self.notification_logger, {"name": self.notification_name}) - self.notification.execute(milter, pretend, custom_logger) + if self.notify is not None: + self.notify.execute(milter) milter.msginfo["rcpts"].extend(wl_rcpts) milter.delrcpt(rcpts) diff --git a/pymodmilter/whitelist.py b/pymodmilter/whitelist.py index 4a5ac8a..ca34851 100644 --- a/pymodmilter/whitelist.py +++ b/pymodmilter/whitelist.py @@ -26,17 +26,21 @@ from playhouse.db_url import connect class WhitelistBase: "Whitelist base class" - def __init__(self, cfg): - self.name = cfg["name"] - self.logger = logging.getLogger(__name__) + def __init__(self, cfg, debug): + self.cfg = cfg + self.logger = logging.getLogger(cfg["name"]) + self.logger.setLevel(cfg.get_loglevel(debug)) + + peewee_logger = logging.getLogger("peewee") + peewee_logger.setLevel(cfg.get_loglevel(debug)) + self.valid_entry_regex = re.compile( r"^[a-zA-Z0-9_.=+-]*?(@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)?$") self.batv_regex = re.compile( r"^prvs=[0-9]{4}[0-9A-Fa-f]{6}=(?P.+?)@") def remove_batv(self, addr): - return self.batv_regex.sub("\g", addr, count=1) - + return self.batv_regex.sub(r"\g", addr, count=1) def check(self, mailfrom, recipient): "Check if mailfrom/recipient combination is whitelisted." @@ -82,8 +86,8 @@ class DatabaseWhitelist(WhitelistBase): _db_connections = {} _db_tables = {} - def __init__(self, cfg): - super().__init__(cfg) + def __init__(self, cfg, debug): + super().__init__(cfg, debug) tablename = cfg["table"] connection_string = cfg["connection"] @@ -110,9 +114,10 @@ class DatabaseWhitelist(WhitelistBase): self.meta = Meta self.meta.database = db self.meta.table_name = tablename - self.model = type(f"WhitelistModel_{self.name}", (WhitelistModel,), { - "Meta": self.meta - }) + self.model = type( + f"WhitelistModel_{self.cfg['name']}", + (WhitelistModel,), + {"Meta": self.meta}) if connection_string not in DatabaseWhitelist._db_tables.keys(): DatabaseWhitelist._db_tables[connection_string] = [] @@ -125,6 +130,13 @@ class DatabaseWhitelist(WhitelistBase): raise RuntimeError( f"unable to initialize table '{tablename}': {e}") + def __str__(self): + cfg = [] + for arg in ("connection", "table"): + if arg in self.cfg: + cfg.append(f"{arg}={self.cfg[arg]}") + return "DatabaseWhitelist(" + ", ".join(cfg) + ")" + def _entry_to_dict(self, entry): result = {} result[entry.id] = { @@ -147,14 +159,14 @@ class DatabaseWhitelist(WhitelistBase): value += 1 return value - def check(self, mailfrom, recipient): + def check(self, mailfrom, recipient, logger): # check if mailfrom/recipient combination is whitelisted super().check(mailfrom, recipient) mailfrom = self.remove_batv(mailfrom) recipient = self.remove_batv(recipient) # generate list of possible mailfroms - self.logger.debug( + logger.debug( f"query database for whitelist entries from <{mailfrom}> " f"to <{recipient}>") mailfroms = [""] diff --git a/setup.py b/setup.py index 5fae1b5..f9d6689 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,6 @@ setup(name = "pymodmilter", ] ) ], - install_requires = ["pymilter", "netaddr", "beautifulsoup4[lxml]", "peewee"], + install_requires = ["pymilter", "jsonschema", "netaddr", "beautifulsoup4[lxml]", "peewee"], python_requires = ">=3.8" )