spacefreak/pyquarantine-milter
1
0
Fork 0
Code Issues Pull Requests Releases Activity

use jsonschema to validate config and massive refactor

Browse Source
This commit is contained in:
Thomas Oettli
2021-09-30 01:07:21 +02:00
parent 4d9baa79f7
commit 08b6ae6377
12 changed files with 762 additions and 1104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
pymodmilter/__init__.py
View File

@@ -32,8 +32,6 @@ from pymodmilter import _runtime_patches
import Milter
import logging
import re
import json
from Milter.utils import parse_addr
from collections import defaultdict
@@ -45,95 +43,9 @@ from email.policy import SMTPUTF8
from io import BytesIO
from netaddr import IPNetwork, AddrFormatError
from pymodmilter.base import CustomLogger, BaseConfig, MilterMessage
from pymodmilter.base import CustomLogger, MilterMessage
from pymodmilter.base import replace_illegal_chars
from pymodmilter.rule import RuleConfig, Rule
class ModifyMilterConfig(BaseConfig):
def __init__(self, cfgfile, debug=False):
try:
with open(cfgfile, "r") as fh:
# remove lines with leading # (comments), they
# are not allowed in json
cfg = re.sub(r"(?m)^\s*#.*\n?", "", fh.read())
except IOError as e:
raise RuntimeError(f"unable to open/read config file: {e}")
try:
cfg = json.loads(cfg)
except json.JSONDecodeError as e:
cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())]
msg = "\n".join(cfg_text)
raise RuntimeError(f"{e}\n{msg}")
if "global" in cfg:
assert isinstance(cfg["global"], dict), \
"global: invalid type, should be dict"
cfg["global"]["name"] = "global"
super().__init__(cfg["global"], debug)
self.logger.debug("initialize config")
if "pretend" in cfg["global"]:
pretend = cfg["global"]["pretend"]
assert isinstance(pretend, bool), \
"global: pretend: invalid value, should be bool"
self.pretend = pretend
else:
self.pretend = False
if "socket" in cfg["global"]:
socket = cfg["global"]["socket"]
assert isinstance(socket, str), \
"global: socket: invalid value, should be string"
self.socket = socket
else:
self.socket = None
if "local_addrs" in cfg["global"]:
local_addrs = cfg["global"]["local_addrs"]
assert isinstance(local_addrs, list) and all(
[isinstance(addr, str) for addr in local_addrs]), \
"global: local_addrs: invalid value, " \
"should be list of strings"
else:
local_addrs = [
"fe80::/64",
"::1/128",
"127.0.0.0/8",
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16"]
self.local_addrs = []
try:
for addr in local_addrs:
self.local_addrs.append(IPNetwork(addr))
except AddrFormatError as e:
raise ValueError(f"{self.name}: local_addrs: {e}")
self.logger.debug(f"socket={self.socket}, "
f"local_addrs={self.local_addrs}, "
f"pretend={self.pretend}, "
f"loglevel={self.loglevel}")
assert "rules" in cfg, \
"mandatory parameter 'rules' not found"
assert isinstance(cfg["rules"], list), \
"rules: invalid value, should be list"
self.logger.debug("initialize rules config")
self.rules = []
for idx, rule_cfg in enumerate(cfg["rules"]):
if "name" not in rule_cfg:
rule_cfg["name"] = "Rule #{idx}"
if "loglevel" not in rule_cfg:
rule_cfg["loglevel"] = self.loglevel
if "pretend" not in rule_cfg:
rule_cfg["pretend"] = self.pretend
self.rules.append(RuleConfig(rule_cfg, self.local_addrs, debug))
from pymodmilter.rule import Rule
class ModifyMilter(Milter.Base):
@@ -145,11 +57,29 @@ class ModifyMilter(Milter.Base):
if issubclass(v, AddressHeader)]
@staticmethod
def set_config(cfg):
ModifyMilter._loglevel = cfg.loglevel
for rule_cfg in cfg.rules:
ModifyMilter._rules.append(
Rule(rule_cfg))
def set_config(cfg, debug):
ModifyMilter._loglevel = cfg.get_loglevel(debug)
try:
local_addrs = []
for addr in cfg["local_addrs"]:
local_addrs.append(IPNetwork(addr))
except AddrFormatError as e:
raise RuntimeError(e)
logger = logging.getLogger(__name__)
logger.setLevel(ModifyMilter._loglevel)
for idx, rule_cfg in enumerate(cfg["rules"]):
if "name" not in rule_cfg:
rule_cfg["name"] = f"rule#{idx}"
if "loglevel" not in rule_cfg:
rule_cfg["loglevel"] = cfg["loglevel"]
if "pretend" not in rule_cfg:
rule_cfg["pretend"] = cfg["pretend"]
rule = Rule(rule_cfg, local_addrs, debug)
logger.debug(rule)
ModifyMilter._rules.append(rule)
def __init__(self):
self.logger = logging.getLogger(__name__)

230
pymodmilter/action.py
View File

@@ -12,204 +12,51 @@
# along with PyMod-Milter. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = [
"ActionConfig",
"Action"]
__all__ = ["Action"]
import logging
from pymodmilter import BaseConfig
from pymodmilter import modify, notify, storage
from pymodmilter.base import CustomLogger
from pymodmilter.conditions import ConditionsConfig, Conditions
class ActionConfig(BaseConfig):
TYPES = {"add_header": "_add_header",
"mod_header": "_mod_header",
"del_header": "_del_header",
"add_disclaimer": "_add_disclaimer",
"rewrite_links": "_rewrite_links",
"store": "_store",
"notify": "_notify",
"quarantine": "_quarantine"}
def __init__(self, cfg, local_addrs, debug):
super().__init__(cfg, debug)
self.local_addrs = local_addrs
self.debug = debug
self.pretend = False
if "pretend" in cfg:
assert isinstance(cfg["pretend"], bool), \
f"{self.name}: pretend: invalid value, should be bool"
self.pretend = cfg["pretend"]
assert "type" in cfg, \
f"{self.name}: mandatory parameter 'type' not found"
assert isinstance(cfg["type"], str), \
f"{self.name}: type: invalid value, should be string"
assert cfg["type"] in ActionConfig.TYPES, \
f"{self.name}: type: invalid action type"
getattr(self, ActionConfig.TYPES[cfg["type"]])(cfg)
if "conditions" in cfg:
assert isinstance(cfg["conditions"], dict), \
f"{self.name}: conditions: invalid value, should be dict"
cfg["conditions"]["name"] = f"{self.name}: condition"
if "loglevel" not in cfg["conditions"]:
cfg["conditions"]["loglevel"] = self.loglevel
self.conditions = ConditionsConfig(
cfg["conditions"], local_addrs, debug)
else:
self.conditions = None
self.logger.debug(f"{self.name}: pretend={self.pretend}, "
f"loglevel={self.loglevel}, "
f"type={cfg['type']}, "
f"args={self.args}")
def _add_header(self, cfg):
self.action = modify.AddHeader
self.add_string_arg(cfg, ["field", "value"])
def _mod_header(self, cfg):
self.action = modify.ModHeader
args = ["field", "value"]
if "search" in cfg:
args.append("search")
self.add_string_arg(cfg, args)
def _del_header(self, cfg):
self.action = modify.DelHeader
args = ["field"]
if "value" in cfg:
args.append("value")
self.add_string_arg(cfg, args)
def _add_disclaimer(self, cfg):
self.action = modify.AddDisclaimer
if "error_policy" not in cfg:
cfg["error_policy"] = "wrap"
self.add_string_arg(
cfg, ["action", "html_template", "text_template",
"error_policy"])
assert self.args["action"] in ["append", "prepend"], \
f"{self.name}: action: invalid value, " \
f"should be 'append' or 'prepend'"
assert self.args["error_policy"] in ("wrap",
"ignore",
"reject"), \
f"{self.name}: error_policy: invalid value, " \
f"should be 'wrap', 'ignore' or 'reject'"
def _rewrite_links(self, cfg):
self.action = modify.RewriteLinks
self.add_string_arg(cfg, "repl")
def _store(self, cfg):
assert "storage_type" in cfg, \
f"{self.name}: mandatory parameter 'storage_type' not found"
assert isinstance(cfg["storage_type"], str), \
f"{self.name}: storage_type: invalid value, " \
f"should be string"
if "original" in cfg:
self.add_bool_arg(cfg, "original")
if cfg["storage_type"] == "file":
self.action = storage.FileMailStorage
self.add_string_arg(cfg, "directory")
if "metavar" in cfg:
self.add_string_arg(cfg, "metavar")
else:
raise RuntimeError(
f"{self.name}: storage_type: invalid storage type")
def _notify(self, cfg):
self.action = notify.EMailNotification
args = ["smtp_host", "envelope_from", "from_header", "subject",
"template"]
if "repl_img" in cfg:
args.append("repl_img")
self.add_string_arg(cfg, args)
self.add_int_arg(cfg, "smtp_port")
if "embed_imgs" in cfg:
assert isinstance(cfg["embed_imgs"], list) and all(
[isinstance(img, str) for img in cfg["embed_imgs"]]), \
f"{self.name}: embed_imgs: invalid value, " \
f"should be list of strings"
self.args["embed_imgs"] = cfg["embed_imgs"]
def _quarantine(self, cfg):
self.action = storage.Quarantine
assert "storage" in cfg, \
f"{self.name}: mandatory parameter 'storage' not found"
assert isinstance(cfg["storage"], dict), \
f"{self.name}: storage: invalid value, " \
f"should be dict"
cfg["storage"]["type"] = "store"
cfg["storage"]["name"] = f"{self.name}: storage"
args = ["storage"]
if "notification" in cfg:
assert isinstance(cfg["notification"], dict), \
f"{self.name}: notification: invalid value, " \
f"should be dict"
cfg["notification"]["type"] = "notify"
cfg["notification"]["name"] = f"{self.name}: notification"
args.append("notification")
for arg in args:
if "loglevel" not in cfg[arg]:
cfg[arg]["loglevel"] = self.loglevel
if "pretend" not in cfg[arg]:
cfg[arg]["pretend"] = self.pretend
self.args[arg] = ActionConfig(
cfg[arg], self.local_addrs, self.debug)
if "milter_action" in cfg:
self.add_string_arg(cfg, "milter_action")
self.args["milter_action"] = self.args["milter_action"].upper()
assert self.args["milter_action"] in ["REJECT", "DISCARD",
"ACCEPT"], \
f"{self.name}: milter_action: invalid value, " \
f"should be 'ACCEPT', 'REJECT' or 'DISCARD'"
if self.args["milter_action"] == "REJECT":
if "reject_reason" in cfg:
self.add_string_arg(cfg, "reject_reason")
if "whitelist" in cfg:
wl = {"whitelist": cfg["whitelist"]}
wl["name"] = f"{self.name}: whitelist"
if "loglevel" not in wl:
wl["loglevel"] = self.loglevel
self.args["whitelist"] = ConditionsConfig(
wl, self.local_addrs, self.debug)
from pymodmilter.conditions import Conditions
class Action:
"""Action to implement a pre-configured action to perform on e-mails."""
ACTION_TYPES = {
"add_header": modify.Modify,
"mod_header": modify.Modify,
"del_header": modify.Modify,
"add_disclaimer": modify.Modify,
"rewrite_links": modify.Modify,
"store": storage.Store,
"notify": notify.Notify,
"quarantine": storage.Quarantine}
def __init__(self, cfg):
self.logger = cfg.logger
if cfg.conditions is None:
self.conditions = None
else:
self.conditions = Conditions(cfg.conditions)
def __init__(self, cfg, local_addrs, debug):
self.cfg = cfg
self.logger = logging.getLogger(cfg["name"])
self.logger.setLevel(cfg.get_loglevel(debug))
self.pretend = cfg.pretend
self.name = cfg.name
self.action = cfg.action(**cfg.args)
self.conditions = cfg["conditions"] if "conditions" in cfg else None
if self.conditions is not None:
self.conditions["name"] = f"{cfg['name']}: conditions"
self.conditions["loglevel"] = cfg["loglevel"]
self.conditions = Conditions(self.conditions, local_addrs, debug)
action_type = cfg["type"]
self.action = self.ACTION_TYPES[action_type](
cfg, local_addrs, debug)
def __str__(self):
cfg = []
for key in ["name", "loglevel", "pretend", "type"]:
value = self.cfg[key]
cfg.append(f"{key}={value}")
if self.conditions is not None:
cfg.append(f"conditions={self.conditions}")
cfg.append(f"action={self.action}")
return "Action(" + ", ".join(cfg) + ")"
def headersonly(self):
"""Return the needs of this action."""
@@ -218,8 +65,7 @@ class Action:
def execute(self, milter):
"""Execute configured action."""
logger = CustomLogger(
self.logger, {"qid": milter.qid, "name": self.name})
self.logger, {"qid": milter.qid, "name": self.cfg["name"]})
if self.conditions is None or \
self.conditions.match(milter):
return self.action.execute(
milter=milter, pretend=self.pretend, logger=logger)
return self.action.execute(milter)

527
pymodmilter/base.py
View File

@@ -14,10 +14,8 @@
__all__ = [
"CustomLogger",
"BaseConfig",
"MilterMessage",
"replace_illegal_chars",
"config_schema"]
"replace_illegal_chars"]
import logging
@@ -38,70 +36,6 @@ class CustomLogger(logging.LoggerAdapter):
return msg, kwargs
class BaseConfig:
def __init__(self, cfg={}, debug=False):
if "name" in cfg:
assert isinstance(cfg["name"], str), \
"name: invalid value, should be string"
self.name = cfg["name"]
else:
self.name = __name__
self.logger = logging.getLogger(self.name)
if debug:
self.loglevel = logging.DEBUG
elif "loglevel" in cfg:
if isinstance(cfg["loglevel"], int):
self.loglevel = cfg["loglevel"]
else:
level = getattr(logging, cfg["loglevel"].upper(), None)
assert isinstance(level, int), \
f"{self.name}: loglevel: invalid value"
self.loglevel = level
else:
self.loglevel = logging.INFO
self.logger.setLevel(self.loglevel)
self.debug = debug
# the keys/values in args are used as parameters
# to initialize action classes
self.args = {}
def add_string_arg(self, cfg, args):
if isinstance(args, str):
args = [args]
for arg in args:
assert arg in cfg, \
f"{self.name}: mandatory parameter '{arg}' not found"
assert isinstance(cfg[arg], str), \
f"{self.name}: {arg}: invalid value, should be string"
self.args[arg] = cfg[arg]
def add_bool_arg(self, cfg, args):
if isinstance(args, str):
args = [args]
for arg in args:
assert arg in cfg, \
f"{self.name}: mandatory parameter '{arg}' not found"
assert isinstance(cfg[arg], bool), \
f"{self.name}: {arg}: invalid value, should be bool"
self.args[arg] = cfg[arg]
def add_int_arg(self, cfg, args):
if isinstance(args, str):
args = [args]
for arg in args:
assert arg in cfg, \
f"{self.name}: mandatory parameter '{arg}' not found"
assert isinstance(cfg[arg], int), \
f"{self.name}: {arg}: invalid value, should be integer"
self.args[arg] = cfg[arg]
class MilterMessage(MIMEPart):
def replace_header(self, _name, _value, idx=None):
_name = _name.lower()
@@ -135,462 +69,3 @@ class MilterMessage(MIMEPart):
def replace_illegal_chars(string):
"""Remove illegal characters from header values."""
return "".join(string.replace("\x00", "").splitlines())
JSON_CONFIG_SCHEMA = """
{
"$id": "https://example.com/schemas/config",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Root",
"type": "object",
"required": ["rules"],
"additionalProperties": false,
"properties": {
"global": {
"title": "Section global",
"type": "object",
"additionalProperties": false,
"properties": {
"local_addrs": { "$ref": "/schemas/config/hosts" },
"loglevel": { "$ref": "/schemas/config/loglevel" },
"socket": {
"title": "Socket",
"type": "string",
"pattern": "^((unix|local):.+|inet6?:[0-9]{1,5}(@.+)?)$"
}
}
},
"rules": {
"title": "Section rules",
"type": "array",
"items": {
"title": "Rules",
"type": "object",
"required": [
"actions"
],
"additionalProperties": false,
"properties": {
"name": { "$ref": "/schemas/config/name" },
"pretend": { "$ref": "/schemas/config/pretend" },
"conditions": { "$ref": "/schemas/config/conditions" },
"loglevel": { "$ref": "/schemas/config/loglevel" },
"actions": {
"title": "Section actions",
"type": "array",
"items": {
"title": "Actions",
"type": "object",
"required": ["type"],
"properties": {
"type": { "$ref": "/schemas/config/actiontype" }
},
"if": { "properties": { "type": { "const": "add_header" } } },
"then": { "$ref": "/schemas/config/add_header" },
"else": {
"if": { "properties": { "type": { "const": "mod_header" } } },
"then": { "$ref": "/schemas/config/mod_header" },
"else": {
"if": { "properties": { "type": { "const": "del_header" } } },
"then": { "$ref": "/schemas/config/del_header" },
"else": {
"if": { "properties": { "type": { "const": "add_disclaimer" } } },
"then": { "$ref": "/schemas/config/add_disclaimer" },
"else": {
"if": { "properties": { "type": { "const": "rewrite_links" } } },
"then": { "$ref": "/schemas/config/rewrite_links" },
"else": {
"if": { "properties": { "type": { "const": "store" } } },
"then": { "$ref": "/schemas/config/store" },
"else": {
"if": { "properties": { "type": { "const": "notify" } } },
"then": { "$ref": "/schemas/config/notify" },
"else": {
"if": { "properties": { "type": { "const": "quarantine" } } },
"then": { "$ref": "/schemas/config/quarantine" },
"else": {
"additionalProperties": false
}
}
}
}
}
}
}
}
}
}
}
}
}
},
"$defs": {
"name": {
"$id": "/schemas/config/name",
"title": "Name",
"type": "string",
"pattern": "^.+$"
},
"hosts": {
"$id": "/schemas/config/hosts",
"title": "Hosts/networks",
"type": "array",
"items": {
"title": "Hosts/Networks",
"type": "string",
"pattern": "^.+$"
}
},
"pretend": {
"$id": "/schemas/config/pretend",
"title": "Pretend",
"type": "boolean"
},
"loglevel": {
"$id": "/schemas/config/loglevel",
"title": "Loglevel",
"type": "string",
"pattern": "^(critical|error|warning|info|debug)$"
},
"actiontype": {
"$id": "/schemas/config/actiontype",
"title": "Action type",
"enum": [
"add_header", "mod_header", "del_header", "add_disclaimer",
"rewrite_links", "store", "notify", "quarantine"]
},
"storagetype": {
"$id": "/schemas/config/storagetype",
"title": "Storage type",
"enum": ["file"]
},
"whitelisttype": {
"$id": "/schemas/config/whitelisttype",
"title": "Whitelist type",
"enum": ["db"]
},
"field": {
"$id": "/schemas/config/field",
"title": "Field",
"type": "string",
"pattern": "^.+$"
},
"value": {
"$id": "/schemas/config/value",
"title": "Value",
"type": "string",
"pattern": "^.+$"
},
"original": {
"$id": "/schemas/config/original",
"title": "Original",
"type": "boolean"
},
"metavar": {
"$id": "/schemas/config/metavar",
"title": "Meta variable",
"type": "string",
"pattern": "^.+$"
},
"conditions": {
"$id": "/schemas/config/conditions",
"title": "Conditions",
"type": "object",
"properties": {
"metavar": { "$ref": "/schemas/config/metavar" },
"local": {
"title": "Local",
"type": "boolean"
},
"hosts": {
"title": "Hosts/Networks",
"type": "array",
"items":{
"title": "Host/Network",
"type": "string",
"pattern": "^.+$"
}
},
"envfrom": {
"title": "Envelope from",
"type": "string",
"pattern": "^.+$"
},
"envto": {
"title": "Envelope to",
"type": "string",
"pattern": "^.+$"
},
"header": {
"title": "Header",
"type": "string",
"pattern": "^.+$"
},
"var": {
"title": "Variable",
"type": "string",
"pattern": "^.+$"
}
},
"additionalProperties": false,
"anyOf": [
{"required": ["local"]},
{"required": ["hosts"]},
{"required": ["envfrom"]},
{"required": ["envto"]},
{"required": ["header"]},
{"required": ["var"]}
]
},
"add_header": {
"$id": "/schemas/config/add_header",
"title": "Add header",
"type": "object",
"required": ["type", "field", "value"],
"properties": {
"type": { "$ref": "/schemas/config/actiontype" },
"name": { "$ref": "/schemas/config/name" },
"pretend": { "$ref": "/schemas/config/pretend" },
"conditions": { "$ref": "/schemas/config/conditions" },
"loglevel": { "$ref": "/schemas/config/loglevel" },
"field": { "$ref": "/schemas/config/field" },
"value": { "$ref": "/schemas/config/value" }
},
"additionalProperties": false
},
"mod_header": {
"$id": "/schemas/config/mod_header",
"title": "Modify header",
"type": "object",
"required": ["type", "field", "value"],
"properties": {
"type": { "$ref": "/schemas/config/actiontype" },
"name": { "$ref": "/schemas/config/name" },
"pretend": { "$ref": "/schemas/config/pretend" },
"conditions": { "$ref": "/schemas/config/conditions" },
"loglevel": { "$ref": "/schemas/config/loglevel" },
"field": { "$ref": "/schemas/config/field" },
"value": { "$ref": "/schemas/config/value" },
"search": {
"title": "Search",
"type": "string",
"pattern": "^.+$"
}
},
"additionalProperties": false
},
"del_header": {
"$id": "/schemas/config/del_header",
"title": "Delete header",
"type": "object",
"required": ["type", "field"],
"properties": {
"type": { "$ref": "/schemas/config/actiontype" },
"name": { "$ref": "/schemas/config/name" },
"pretend": { "$ref": "/schemas/config/pretend" },
"conditions": { "$ref": "/schemas/config/conditions" },
"loglevel": { "$ref": "/schemas/config/loglevel" },
"field": { "$ref": "/schemas/config/field" },
"value": { "$ref": "/schemas/config/value" }
},
"additionalProperties": false
},
"add_disclaimer": {
"$id": "/schemas/config/add_disclaimer",
"title": "Add disclaimer",
"type": "object",
"required": ["type", "action", "html_template", "text_template"],
"properties": {
"type": { "$ref": "/schemas/config/actiontype" },
"name": { "$ref": "/schemas/config/name" },
"pretend": { "$ref": "/schemas/config/pretend" },
"conditions": { "$ref": "/schemas/config/conditions" },
"loglevel": { "$ref": "/schemas/config/loglevel" },
"action": {
"title": "Action",
"enum": ["append", "prepend"]
},
"html_template": {
"title": "HTML template",
"type": "string",
"pattern": "^.+$"
},
"text_template": {
"title": "Text template",
"type": "string",
"pattern": "^.+$"
},
"error_policy": {
"title": "Action",
"enum": [
"wrap", "ignore", "reject",
"WRAP", "IGNORE", "REJECT"]
}
},
"additionalProperties": false
},
"rewrite_links": {
"$id": "/schemas/config/rewrite_links",
"title": "Rewrite links",
"type": "object",
"required": ["type", "repl"],
"properties": {
"type": { "$ref": "/schemas/config/actiontype" },
"name": { "$ref": "/schemas/config/name" },
"pretend": { "$ref": "/schemas/config/pretend" },
"conditions": { "$ref": "/schemas/config/conditions" },
"loglevel": { "$ref": "/schemas/config/loglevel" },
"repl": {
"title": "Replacement",
"type": "string",
"pattern": "^.+$"
}
},
"additionalProperties": false
},
"store": {
"$id": "/schemas/config/store",
"title": "Store",
"type": "object",
"required": ["storage_type"],
"properties": {
"storage_type": { "$ref": "/schemas/config/storagetype" }
},
"if": { "properties": { "storage_type": { "const": "file" } } },
"then": {
"properties": {
"type": { "$ref": "/schemas/config/actiontype" },
"storage_type": { "$ref": "/schemas/config/storagetype" },
"name": { "$ref": "/schemas/config/name" },
"pretend": { "$ref": "/schemas/config/pretend" },
"conditions": { "$ref": "/schemas/config/conditions" },
"loglevel": { "$ref": "/schemas/config/loglevel" },
"original": { "$ref": "/schemas/config/original" },
"metavar": { "$ref": "/schemas/config/metavar" },
"directory": {
"title": "Directory",
"type": "string",
"pattern": "^.+$"
}
},
"additionalProperties": false
},
"else": {
"additionalProperties": false
}
},
"notify": {
"$id": "/schemas/config/notify",
"title": "Notify",
"type": "object",
"required": ["smtp_host", "smtp_port", "envelope_from", "from_header", "subject", "template"],
"properties": {
"type": { "$ref": "/schemas/config/actiontype" },
"name": { "$ref": "/schemas/config/name" },
"pretend": { "$ref": "/schemas/config/pretend" },
"conditions": { "$ref": "/schemas/config/conditions" },
"loglevel": { "$ref": "/schemas/config/loglevel" },
"smtp_host": {
"title": "SMTP host",
"type": "string",
"pattern": "^.+$"
},
"smtp_port": {
"title": "SMTP port",
"type": "number"
},
"envelope_from": {
"title": "Envelope from",
"type": "string",
"pattern": "^.+$"
},
"from_header": {
"title": "From-Header",
"type": "string",
"pattern": "^.+$"
},
"subject": {
"title": "Subject",
"type": "string",
"pattern": "^.+$"
},
"template": {
"title": "Template",
"type": "string",
"pattern": "^.+$"
},
"repl_img": {
"title": "Replacement image",
"type": "string",
"pattern": "^.+$"
},
"embed_imgs": {
"title": "Embedded images",
"type": "array",
"items": {
"title": "Embedded image",
"type": "string",
"pattern": "^.+$"
}
}
},
"additionalProperties": false
},
"quarantine": {
"$id": "/schemas/config/quarantine",
"title": "Quarantine",
"type": "object",
"required": ["storage"],
"properties": {
"type": { "$ref": "/schemas/config/actiontype" },
"name": { "$ref": "/schemas/config/name" },
"pretend": { "$ref": "/schemas/config/pretend" },
"conditions": { "$ref": "/schemas/config/conditions" },
"loglevel": { "$ref": "/schemas/config/loglevel" },
"storage": { "$ref": "/schemas/config/store" },
"notification": { "$ref": "/schemas/config/notify" },
"milter_action": {
"title": "Milter action",
"enum": [
"reject", "discard", "accept",
"REJECT", "DISCARD", "ACCEPT"]
},
"reject_reason": {
"title": "Reject reason",
"type": "string",
"pattern": "^.+$"
},
"whitelist": {
"title": "Whitelist",
"type": "object",
"required": ["type"],
"properties": {
"type": { "$ref": "/schemas/config/whitelisttype" }
},
"if": { "properties": { "type": { "const": "db" } } },
"then": {
"required": ["connection", "table"],
"properties": {
"type": { "$ref": "/schemas/config/whitelisttype" },
"connection": {
"title": "DB connection",
"type": "string",
"pattern": "^.+$"
},
"table": {
"title": "DB table",
"type": "string",
"pattern": "^.+$"
}
},
"additionalProperties": false
},
"else": {
"additionalProperties": false
}
}
}
}
}
}
"""

156
pymodmilter/conditions.py
View File

@@ -12,127 +12,77 @@
# along with PyMod-Milter. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = [
"ConditionsConfig",
"Conditions"]
__all__ = ["Conditions"]
import logging
import re
from netaddr import IPAddress, IPNetwork, AddrFormatError
from pymodmilter import BaseConfig, CustomLogger
from pymodmilter import CustomLogger
from pymodmilter.whitelist import DatabaseWhitelist
class ConditionsConfig(BaseConfig):
def __init__(self, cfg, local_addrs, debug):
super().__init__(cfg, debug)
self.local_addrs = local_addrs
if "local" in cfg:
self.add_bool_arg(cfg, "local")
if "hosts" in cfg:
assert isinstance(cfg["hosts"], list) and all(
[isinstance(host, str) for host in cfg["hosts"]]), \
f"{self.name}: hosts: invalid value, " \
f"should be list of strings"
self.args["hosts"] = cfg["hosts"]
for arg in ("envfrom", "envto"):
if arg in cfg:
self.add_string_arg(cfg, arg)
if "header" in cfg:
self.add_string_arg(cfg, "header")
if "var" in cfg:
self.add_string_arg(cfg, "var")
if "metavar" in cfg:
self.add_string_arg(cfg, "metavar")
if "whitelist" in cfg:
assert isinstance(cfg["whitelist"], dict), \
f"{self.name}: whitelist: invalid value, " \
f"should be dict"
whitelist = cfg["whitelist"]
assert "type" in whitelist, \
f"{self.name}: whitelist: mandatory parameter 'type' not found"
assert isinstance(whitelist["type"], str), \
f"{self.name}: whitelist: type: invalid value, " \
f"should be string"
self.args["whitelist"] = {
"type": whitelist["type"],
"name": f"{self.name}: whitelist"}
if whitelist["type"] == "db":
for arg in ["connection", "table"]:
assert arg in whitelist, \
f"{self.name}: whitelist: mandatory parameter " \
f"'{arg}' not found"
assert isinstance(whitelist[arg], str), \
f"{self.name}: whitelist: {arg}: invalid value, " \
f"should be string"
self.args["whitelist"][arg] = whitelist[arg]
else:
raise RuntimeError(
f"{self.name}: whitelist: type: invalid type")
self.logger.debug(f"{self.name}: "
f"loglevel={self.loglevel}, "
f"args={self.args}")
class Conditions:
"""Conditions to implement conditions for rules and actions."""
def __init__(self, cfg):
self.logger = cfg.logger
self.name = cfg.name
self.local_addrs = cfg.local_addrs
def __init__(self, cfg, local_addrs, debug):
self.cfg = cfg
self.local_addrs = local_addrs
self.logger = logging.getLogger(cfg["name"])
self.logger.setLevel(cfg.get_loglevel(debug))
for arg in ("local", "hosts", "envfrom", "envto", "header", "metavar",
"var"):
value = cfg.args[arg] if arg in cfg.args else None
setattr(self, arg, value)
if value is None:
if arg not in cfg:
setattr(self, arg, None)
continue
elif arg == "hosts":
if arg == "hosts":
try:
hosts = []
for host in self.hosts:
hosts.append(IPNetwork(host))
self.hosts = []
for host in cfg["hosts"]:
self.hosts.append(IPNetwork(host))
except AddrFormatError as e:
raise RuntimeError(e)
self.hosts = hosts
elif arg in ("envfrom", "envto"):
try:
setattr(self, arg, re.compile(
getattr(self, arg), re.IGNORECASE))
cfg[arg], re.IGNORECASE))
except re.error as e:
raise RuntimeError(e)
elif arg == "header":
try:
self.header = re.compile(
self.header, re.IGNORECASE + re.DOTALL + re.MULTILINE)
cfg["header"],
re.IGNORECASE + re.DOTALL + re.MULTILINE)
except re.error as e:
raise RuntimeError(e)
if "whitelist" in cfg.args:
wl_cfg = cfg.args["whitelist"]
if wl_cfg["type"] == "db":
self.whitelist = DatabaseWhitelist(wl_cfg)
else:
raise RuntimeError("invalid storage type")
setattr(self, arg, cfg[arg])
self.whitelist = cfg["whitelist"] if "whitelist" in cfg else None
if self.whitelist is not None:
self.whitelist["name"] = f"{cfg['name']}: whitelist"
self.whitelist["loglevel"] = cfg["loglevel"]
if self.whitelist["type"] == "db":
self.whitelist = DatabaseWhitelist(self.whitelist, debug)
else:
raise RuntimeError("invalid whitelist type")
def __str__(self):
cfg = []
for arg in ("local", "hosts", "envfrom", "envto", "header",
"var", "metavar"):
if arg in self.cfg:
cfg.append(f"{arg}={self.cfg[arg]}")
if self.whitelist is not None:
cfg.append(f"whitelist={self.whitelist}")
return "Conditions(" + ", ".join(cfg) + ")"
def match_host(self, host):
logger = CustomLogger(
self.logger, {"name": self.name})
self.logger, {"name": self.cfg["name"]})
ip = IPAddress(host)
if self.local is not None:
@@ -145,11 +95,11 @@ class Conditions:
if is_local != self.local:
logger.debug(
f"ignore host {host}, "
f"condition local does not match")
f"local does not match")
return False
logger.debug(
f"condition local matches for host {host}")
f"local matches for host {host}")
if self.hosts is not None:
found = False
@@ -161,39 +111,39 @@ class Conditions:
if not found:
logger.debug(
f"ignore host {host}, "
f"condition hosts does not match")
f"hosts does not match")
return False
logger.debug(
f"condition hosts matches for host {host}")
f"hosts matches for host {host}")
return True
def get_wl_rcpts(self, mailfrom, rcpts):
def get_wl_rcpts(self, mailfrom, rcpts, logger):
if not self.whitelist:
return {}
wl_rcpts = []
for rcpt in rcpts:
if self.whitelist.check(mailfrom, rcpt):
if self.whitelist.check(mailfrom, rcpt, logger):
wl_rcpts.append(rcpt)
return wl_rcpts
def match(self, milter):
logger = CustomLogger(
self.logger, {"qid": milter.qid, "name": self.name})
self.logger, {"qid": milter.qid, "name": self.cfg["name"]})
if self.envfrom is not None:
envfrom = milter.msginfo["mailfrom"]
if not self.envfrom.match(envfrom):
logger.debug(
f"ignore envelope-from address {envfrom}, "
f"condition envfrom does not match")
f"envfrom does not match")
return False
logger.debug(
f"condition envfrom matches for "
f"envfrom matches for "
f"envelope-from address {envfrom}")
if self.envto is not None:
@@ -205,11 +155,11 @@ class Conditions:
if not self.envto.match(to):
logger.debug(
f"ignore envelope-to address {envto}, "
f"condition envto does not match")
f"envto does not match")
return False
logger.debug(
f"condition envto matches for "
f"envto matches for "
f"envelope-to address {envto}")
if self.header is not None:
@@ -219,7 +169,7 @@ class Conditions:
match = self.header.search(header)
if match:
logger.debug(
f"condition header matches for "
f"header matches for "
f"header: {header}")
if self.metavar is not None:
named_subgroups = match.groupdict(default=None)
@@ -233,7 +183,7 @@ class Conditions:
if not match:
logger.debug(
"ignore message, "
"condition header does not match")
"header does not match")
return False
if self.var is not None:

330
pymodmilter/config.py Normal file
View File

@@ -0,0 +1,330 @@
# PyMod-Milter is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyMod-Milter is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with PyMod-Milter. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = [
"BaseConfig",
"ConditionsConfig",
"AddHeaderConfig",
"ModHeaderConfig",
"DelHeaderConfig",
"AddDisclaimerConfig",
"RewriteLinksConfig",
"StoreConfig",
"NotifyConfig",
"WhitelistConfig",
"QuarantineConfig",
"ActionConfig",
"RuleConfig",
"MilterConfig"]
import jsonschema
import logging
class BaseConfig:
JSON_SCHEMA = {
"type": "object",
"required": [],
"additionalProperties": True,
"properties": {
"loglevel": {"type": "string", "default": "info"}}}
def __init__(self, config):
required = self.JSON_SCHEMA["required"]
properties = self.JSON_SCHEMA["properties"]
for p in properties.keys():
if p in required:
continue
elif p not in config and "default" in properties[p]:
config[p] = properties[p]["default"]
try:
jsonschema.validate(config, self.JSON_SCHEMA)
except jsonschema.exceptions.ValidationError as e:
raise RuntimeError(e)
self._config = config
def __getitem__(self, key):
return self._config[key]
def __setitem__(self, key, value):
self._config[key] = value
def __delitem__(self, key):
del self._config[key]
def __contains__(self, key):
return key in self._config
def keys(self):
return self._config.keys()
def items(self):
return self._config.items()
def get_loglevel(self, debug):
if debug:
level = logging.DEBUG
else:
level = getattr(logging, self["loglevel"].upper(), None)
assert isinstance(level, int), \
"loglevel: invalid value"
return level
def get_config(self):
return self._config
class WhitelistConfig(BaseConfig):
JSON_SCHEMA = {
"type": "object",
"required": ["type"],
"additionalProperties": True,
"properties": {
"type": {"enum": ["db"]}},
"if": {"properties": {"type": {"const": "db"}}},
"then": {
"required": ["connection", "table"],
"additionalProperties": False,
"properties": {
"type": {"type": "string"},
"connection": {"type": "string"},
"table": {"type": "string"}}}}
class ConditionsConfig(BaseConfig):
JSON_SCHEMA = {
"type": "object",
"required": [],
"additionalProperties": False,
"properties": {
"metavar": {"type": "string"},
"local": {"type": "boolean"},
"hosts": {"type": "array",
"items": {"type": "string"}},
"envfrom": {"type": "string"},
"envto": {"type": "string"},
"header": {"type": "string"},
"var": {"type": "string"},
"whitelist": {"type": "object"}}}
def __init__(self, config, rec=True):
super().__init__(config)
if rec:
if "whitelist" in self:
self["whitelist"] = WhitelistConfig(self["whitelist"])
class AddHeaderConfig(BaseConfig):
JSON_SCHEMA = {
"type": "object",
"required": ["field", "value"],
"additionalProperties": False,
"properties": {
"field": {"type": "string"},
"value": {"type": "string"}}}
class ModHeaderConfig(BaseConfig):
JSON_SCHEMA = {
"type": "object",
"required": ["field", "value"],
"additionalProperties": False,
"properties": {
"field": {"type": "string"},
"value": {"type": "string"},
"search": {"type": "string"}}}
class DelHeaderConfig(BaseConfig):
JSON_SCHEMA = {
"type": "object",
"required": ["field"],
"additionalProperties": False,
"properties": {
"field": {"type": "string"},
"value": {"type": "string"}}}
class AddDisclaimerConfig(BaseConfig):
JSON_SCHEMA = {
"type": "object",
"required": ["action", "html_template", "text_template"],
"additionalProperties": False,
"properties": {
"action": {"type": "string"},
"html_template": {"type": "string"},
"text_template": {"type": "string"},
"error_policy": {"type": "string"}}}
class RewriteLinksConfig(BaseConfig):
JSON_SCHEMA = {
"type": "object",
"required": ["repl"],
"additionalProperties": False,
"properties": {
"repl": {"type": "string"}}}
class StoreConfig(BaseConfig):
JSON_SCHEMA = {
"type": "object",
"required": ["type"],
"additionalProperties": True,
"properties": {
"type": {"enum": ["file"]}},
"if": {"properties": {"type": {"const": "file"}}},
"then": {
"required": ["directory"],
"additionalProperties": False,
"properties": {
"type": {"type": "string"},
"directory": {"type": "string"},
"metavar": {"type": "string"},
"original": {"type": "boolean", "default": True}}}}
class NotifyConfig(BaseConfig):
JSON_SCHEMA = {
"type": "object",
"required": ["type"],
"additionalProperties": True,
"properties": {
"type": {"enum": ["email"]}},
"if": {"properties": {"type": {"const": "email"}}},
"then": {
"required": ["smtp_host", "smtp_port", "envelope_from",
"from_header", "subject", "template"],
"additionalProperties": False,
"properties": {
"type": {"type": "string"},
"smtp_host": {"type": "string"},
"smtp_port": {"type": "number"},
"envelope_from": {"type": "string"},
"from_header": {"type": "string"},
"subject": {"type": "string"},
"template": {"type": "string"},
"repl_img": {"type": "string"},
"embed_imgs": {
"type": "array",
"items": {"type": "string"},
"default": True}}}}
class QuarantineConfig(BaseConfig):
JSON_SCHEMA = {
"type": "object",
"required": ["store"],
"additionalProperties": False,
"properties": {
"name": {"type": "string"},
"notify": {"type": "object"},
"milter_action": {"type": "string"},
"reject_reason": {"type": "string"},
"whitelist": {"type": "object"},
"store": {"type": "object"}}}
def __init__(self, config, rec=True):
super().__init__(config)
if rec:
self["store"] = StoreConfig(self["store"])
if "notify" in self:
self["notify"] = NotifyConfig(self["notify"])
if "whitelist" in self:
self["whitelist"] = ConditionsConfig(
{"whitelist": self["whitelist"]}, rec)
class ActionConfig(BaseConfig):
ACTION_TYPES = {
"add_header": AddHeaderConfig,
"mod_header": ModHeaderConfig,
"del_header": DelHeaderConfig,
"add_disclaimer": AddDisclaimerConfig,
"rewrite_links": RewriteLinksConfig,
"store": StoreConfig,
"notify": NotifyConfig,
"quarantine": QuarantineConfig}
JSON_SCHEMA = {
"type": "object",
"required": ["type", "args"],
"additionalProperties": False,
"properties": {
"name": {"type": "string", "default": "action"},
"loglevel": {"type": "string", "default": "info"},
"pretend": {"type": "boolean", "default": False},
"conditions": {"type": "object"},
"type": {"enum": list(ACTION_TYPES.keys())},
"args": {"type": "object"}}}
def __init__(self, config, rec=True):
super().__init__(config)
if rec:
if "conditions" in self:
self["conditions"] = ConditionsConfig(self["conditions"])
self["action"] = self.ACTION_TYPES[self["type"]](self["args"])
class RuleConfig(BaseConfig):
JSON_SCHEMA = {
"type": "object",
"required": ["actions"],
"additionalProperties": False,
"properties": {
"name": {"type": "string", "default": "rule"},
"loglevel": {"type": "string", "default": "info"},
"pretend": {"type": "boolean", "default": False},
"conditions": {"type": "object"},
"actions": {"type": "array"}}}
def __init__(self, config, rec=True):
super().__init__(config)
if rec:
if "conditions" in self:
self["conditions"] = ConditionsConfig(self["conditions"])
actions = []
for idx, action in enumerate(self["actions"]):
actions.append(ActionConfig(action, rec))
self["actions"] = actions
class MilterConfig(BaseConfig):
JSON_SCHEMA = {
"type": "object",
"required": ["rules"],
"additionalProperties": False,
"properties": {
"socket": {"type": "string"},
"local_addrs": {"type": "array",
"items": {"type": "string"},
"default": [
"fe80::/64",
"::1/128",
"127.0.0.0/8",
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16"]},
"loglevel": {"type": "string", "default": "info"},
"pretend": {"type": "boolean", "default": False},
"rules": {"type": "array"}}}
def __init__(self, config, rec=True):
super().__init__(config)
if rec:
rules = []
for idx, rule in enumerate(self["rules"]):
rules.append(RuleConfig(rule, rec))
self["rules"] = rules

97
pymodmilter/modify.py
View File

@@ -30,18 +30,19 @@ from email.message import MIMEPart
from email.policy import SMTPUTF8
from pymodmilter import replace_illegal_chars
from pymodmilter.base import CustomLogger
class AddHeader:
"""Add a mail header field."""
_headersonly = True
def __init__(self, field, value):
def __init__(self, field, value, pretend=False):
self.field = field
self.value = value
self.pretend = pretend
def execute(self, milter, pretend=False,
logger=logging.getLogger(__name__)):
def execute(self, milter, logger):
header = f"{self.field}: {self.value}"
if logger.getEffectiveLevel() == logging.DEBUG:
logger.debug(f"add_header: {header}")
@@ -49,8 +50,7 @@ class AddHeader:
logger.info(f"add_header: {header[0:70]}")
milter.msg.add_header(self.field, self.value)
if not pretend:
if not self.pretend:
milter.addheader(self.field, self.value)
@@ -58,21 +58,21 @@ class ModHeader:
"""Change the value of a mail header field."""
_headersonly = True
def __init__(self, field, value, search=None):
self.value = value
def __init__(self, field, value, search=None, pretend=False):
try:
self.field = re.compile(field, re.IGNORECASE)
if search is not None:
self.search = re.compile(
search, re.MULTILINE + re.DOTALL + re.IGNORECASE)
else:
self.search = search
if self.search is not None:
self.search = re.compile(
self.search, re.MULTILINE + re.DOTALL + re.IGNORECASE)
except re.error as e:
raise RuntimeError(e)
def execute(self, milter, pretend=False,
logger=logging.getLogger(__name__)):
self.value = value
self.pretend = pretend
def execute(self, milter, logger):
idx = defaultdict(int)
for i, (field, value) in enumerate(milter.msg.items()):
@@ -103,12 +103,13 @@ class ModHeader:
if logger.getEffectiveLevel() == logging.DEBUG:
logger.debug(f"mod_header: {header}: {new_header}")
else:
logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}")
logger.info(
f"mod_header: {header[0:70]}: {new_header[0:70]}")
milter.msg.replace_header(
field, replace_illegal_chars(new_value), idx=idx[field_lower])
if not pretend:
if not self.pretend:
milter.chgheader(field, new_value, idx=idx[field_lower])
@@ -116,19 +117,19 @@ class DelHeader:
"""Delete a mail header field."""
_headersonly = True
def __init__(self, field, value=None):
def __init__(self, field, value=None, pretend=False):
try:
self.field = re.compile(field, re.IGNORECASE)
if value is not None:
self.value = value
if self.value is not None:
self.value = re.compile(
value, re.MULTILINE + re.DOTALL + re.IGNORECASE)
else:
self.value = value
except re.error as e:
raise RuntimeError(e)
def execute(self, milter, pretend=False,
logger=logging.getLogger(__name__)):
self.pretend = pretend
def execute(self, milter, logger):
idx = defaultdict(int)
for field, value in milter.msg.items():
@@ -148,7 +149,7 @@ class DelHeader:
logger.info(f"del_header: {header[0:70]}")
milter.msg.remove_header(field, idx=idx[field_lower])
if not pretend:
if not self.pretend:
milter.chgheader(field, "", idx=idx[field_lower])
idx[field_lower] -= 1
@@ -220,7 +221,10 @@ class AddDisclaimer:
"""Append or prepend a disclaimer to the mail body."""
_headersonly = False
def __init__(self, text_template, html_template, action, error_policy):
def __init__(self, text_template, html_template, action, error_policy,
pretend=False):
self.text_template_path = text_template
self.html_template_path = html_template
try:
with open(text_template, "r") as f:
self.text_template = f.read()
@@ -230,11 +234,11 @@ class AddDisclaimer:
except IOError as e:
raise RuntimeError(e)
body = html.find('body')
self.html_template = body or html
self.action = action
self.error_policy = error_policy
self.pretend = pretend
def patch_message_body(self, milter, logger):
text_body, text_content = _get_body_content(milter.msg, "plain")
@@ -277,8 +281,7 @@ class AddDisclaimer:
html_body.set_param("charset", "UTF-8", header="Content-Type")
del html_body["MIME-Version"]
def execute(self, milter, pretend=False,
logger=logging.getLogger(__name__)):
def execute(self, milter, logger):
old_headers = milter.msg.items()
try:
@@ -313,7 +316,7 @@ class AddDisclaimer:
"unable to wrap message in a new message envelope, "
"give up ...")
if not pretend:
if not self.pretend:
milter.update_headers(old_headers)
milter.replacebody()
@@ -322,11 +325,11 @@ class RewriteLinks:
"""Rewrite link targets in the mail html body."""
_headersonly = False
def __init__(self, repl):
def __init__(self, repl, pretend=False):
self.repl = repl
self.pretend = pretend
def execute(self, milter, pretend=False,
logger=logging.getLogger(__name__)):
def execute(self, milter, logger):
html_body, html_content = _get_body_content(milter.msg, "html")
if html_content is not None:
soup = BeautifulSoup(html_content, "html.parser")
@@ -353,5 +356,35 @@ class RewriteLinks:
html_body.set_param("charset", "UTF-8", header="Content-Type")
del html_body["MIME-Version"]
if not pretend:
if not self.pretend:
milter.replacebody()
class Modify:
MODIFICATION_TYPES = {
"add_header": AddHeader,
"mod_header": ModHeader,
"del_header": DelHeader,
"add_disclaimer": AddDisclaimer,
"rewrite_links": RewriteLinks}
def __init__(self, cfg, local_addrs, debug):
self.cfg = cfg
self.logger = logging.getLogger(cfg["name"])
self.logger.setLevel(cfg.get_loglevel(debug))
cfg["args"]["pretend"] = cfg["pretend"]
self._modification = self.MODIFICATION_TYPES[cfg["type"]](
**cfg["args"])
self._headersonly = self._modification._headersonly
def __str__(self):
cfg = []
for key, value in self.cfg["args"].items():
cfg.append(f"{key}={value}")
class_name = type(self._modification).__name__
return f"{class_name}(" + ", ".join(cfg) + ")"
def execute(self, milter):
logger = CustomLogger(
self.logger, {"name": self.cfg["name"], "qid": milter.qid})
self._modification.execute(milter, logger)

71
pymodmilter/notify.py
View File

@@ -29,6 +29,7 @@ from html import escape
from os.path import basename
from urllib.parse import quote
from pymodmilter.base import CustomLogger
from pymodmilter import mailer
@@ -36,11 +37,10 @@ class BaseNotification:
"Notification base class"
_headersonly = True
def __init__(self):
self.logger = logging.getLogger(__name__)
return
def __init__(self, pretend=False):
self.pretend = pretend
def execute(self, milter, pretend=False, logger=None):
def execute(self, milter, logger):
return
@@ -112,9 +112,8 @@ class EMailNotification(BaseNotification):
def __init__(self, smtp_host, smtp_port, envelope_from, from_header,
subject, template, embed_imgs=[], repl_img=None,
strip_imgs=False, parser_lib="lxml"):
super().__init__()
strip_imgs=False, parser_lib="lxml", pretend=False):
super().__init__(pretend)
self.smtp_host = smtp_host
self.smtp_port = smtp_port
self.mailfrom = envelope_from
@@ -143,11 +142,8 @@ class EMailNotification(BaseNotification):
self.parser_lib = parser_lib
def get_email_body_soup(self, msg, logger=None):
def get_email_body_soup(self, msg, logger):
"Extract and decode email body and return it as BeautifulSoup object."
if logger is None:
logger = self.logger
# try to find the body part
logger.debug("trying to find email body")
try:
@@ -193,11 +189,8 @@ class EMailNotification(BaseNotification):
return soup
def sanitize(self, soup, logger=None):
def sanitize(self, soup, logger):
"Sanitize mail html text."
if logger is None:
logger = self.logger
logger.debug("sanitizing email text")
# completly remove bad elements
@@ -230,13 +223,9 @@ class EMailNotification(BaseNotification):
del(element.attrs[attribute])
return soup
def notify(self, msg, qid, mailfrom, recipients,
template_vars=defaultdict(str), synchronous=False,
logger=None):
def notify(self, msg, qid, mailfrom, recipients, logger,
template_vars=defaultdict(str), synchronous=False):
"Notify recipients via email."
if logger is None:
logger = self.logger
# extract body from email
soup = self.get_email_body_soup(msg, logger)
@@ -262,7 +251,8 @@ class EMailNotification(BaseNotification):
# sending email notifications
for recipient in recipients:
logger.debug(f"generating notification email for '{recipient}'")
logger.debug(
f"generating notification email for '{recipient}'")
logger.debug("parsing email template")
# generate dict containing all template variables
@@ -313,15 +303,40 @@ class EMailNotification(BaseNotification):
self.mailfrom, recipient, newmsg.as_string(),
"notification email")
def execute(self, milter, pretend=False,
logger=None):
super().execute(milter, pretend, logger)
if logger is None:
logger = self.logger
def execute(self, milter, logger):
super().execute(milter, logger)
self.notify(msg=milter.msg, qid=milter.qid,
mailfrom=milter.msginfo["mailfrom"],
recipients=milter.msginfo["rcpts"],
template_vars=milter.msginfo["vars"],
logger=logger)
class Notify:
NOTIFICATION_TYPES = {
"email": EMailNotification}
def __init__(self, cfg, local_addrs, debug):
self.cfg = cfg
self.logger = logging.getLogger(cfg["name"])
self.logger.setLevel(cfg.get_loglevel(debug))
nodification_type = cfg["args"]["type"]
del cfg["args"]["type"]
cfg["args"]["pretend"] = cfg["pretend"]
self._notification = self.NOTIFICATION_TYPES[nodification_type](
**cfg["args"])
self._headersonly = self._notification._headersonly
def __str__(self):
cfg = []
for key, value in self.cfg["args"].items():
cfg.append(f"{key}={value}")
class_name = type(self._notification).__name__
return f"{class_name}(" + ", ".join(cfg) + ")"
def execute(self, milter):
logger = CustomLogger(
self.logger, {"name": self.cfg["name"], "qid": milter.qid})
self._notification.execute(milter, logger)

100
pymodmilter/rule.py
View File

@@ -12,83 +12,53 @@
# along with PyMod-Milter. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = [
"RuleConfig",
"Rule"]
__all__ = ["Rule"]
from pymodmilter import BaseConfig
from pymodmilter.action import ActionConfig, Action
from pymodmilter.conditions import ConditionsConfig, Conditions
import logging
class RuleConfig(BaseConfig):
def __init__(self, cfg, local_addrs, debug=False):
super().__init__(cfg, debug)
self.conditions = None
self.actions = []
self.pretend = False
if "pretend" in cfg:
assert isinstance(cfg["pretend"], bool), \
f"{self.name}: pretend: invalid value, should be bool"
self.pretend = cfg["pretend"]
assert "actions" in cfg, \
f"{self.name}: mandatory parameter 'actions' not found"
actions = cfg["actions"]
assert isinstance(actions, list), \
f"{self.name}: actions: invalid value, should be list"
self.logger.debug(f"{self.name}: pretend={self.pretend}, "
f"loglevel={self.loglevel}")
if "conditions" in cfg:
assert isinstance(cfg["conditions"], dict), \
f"{self.name}: conditions: invalid value, should be dict"
cfg["conditions"]["name"] = f"{self.name}: condition"
if "loglevel" not in cfg["conditions"]:
cfg["conditions"]["loglevel"] = self.loglevel
self.conditions = ConditionsConfig(
cfg["conditions"], local_addrs, debug)
else:
self.conditions = None
for idx, action_cfg in enumerate(cfg["actions"]):
if "name" in action_cfg:
assert isinstance(action_cfg["name"], str), \
f"{self.name}: Action #{idx}: name: invalid value, " \
f"should be string"
action_cfg["name"] = f"{self.name}: {action_cfg['name']}"
else:
action_cfg["name"] = f"{self.name}: Action #{idx}"
if "loglevel" not in action_cfg:
action_cfg["loglevel"] = self.loglevel
if "pretend" not in action_cfg:
action_cfg["pretend"] = self.pretend
self.actions.append(
ActionConfig(action_cfg, local_addrs, debug))
from pymodmilter.action import Action
from pymodmilter.conditions import Conditions
class Rule:
"""
Rule to implement multiple actions on emails.
"""
def __init__(self, cfg, local_addrs, debug):
self.cfg = cfg
self.logger = logging.getLogger(cfg["name"])
self.logger.setLevel(cfg.get_loglevel(debug))
def __init__(self, cfg):
self.logger = cfg.logger
if cfg.conditions is None:
self.conditions = None
else:
self.conditions = Conditions(cfg.conditions)
self.conditions = cfg["conditions"] if "conditions" in cfg else None
if self.conditions is not None:
self.conditions["name"] = f"{cfg['name']}: condition"
self.conditions["loglevel"] = cfg["loglevel"]
self.conditions = Conditions(self.conditions, local_addrs, debug)
self.actions = []
for action_cfg in cfg.actions:
self.actions.append(Action(action_cfg))
for idx, action_cfg in enumerate(cfg["actions"]):
if "name" in action_cfg:
action_cfg["name"] = f"{cfg['name']}: {action_cfg['name']}"
else:
action_cfg["name"] = f"action#{idx}"
if "loglevel" not in action_cfg:
action_cfg["loglevel"] = cfg["loglevel"]
if "pretend" not in action_cfg:
action_cfg["pretend"] = cfg["pretend"]
self.actions.append(Action(action_cfg, local_addrs, debug))
self.pretend = cfg.pretend
def __str__(self):
cfg = []
for key in ["name", "loglevel", "pretend"]:
value = self.cfg[key]
cfg.append(f"{key}={value}")
if self.conditions is not None:
cfg.append(f"conditions={self.conditions}")
actions = []
for action in self.actions:
actions.append(str(action))
cfg.append("actions=[" + ", ".join(actions) + "]")
return "Rule(" + ", ".join(cfg) + ")"
def execute(self, milter):
"""Execute all actions of this rule."""

47
pymodmilter/run.py
View File

@@ -16,13 +16,17 @@ __all__ = ["main"]
import Milter
import argparse
import json
import logging
import logging.handlers
import os
import re
import sys
from pymodmilter import mailer
from pymodmilter import ModifyMilterConfig, ModifyMilter
from pymodmilter import ModifyMilter
from pymodmilter import __version__ as version
from pymodmilter.config import MilterConfig
def main():
@@ -80,36 +84,52 @@ def main():
logger.setLevel(logging.INFO)
try:
logger.debug("prepare milter configuration")
cfg = ModifyMilterConfig(args.config, args.debug)
logger.debug("read milter configuration")
try:
with open(args.config, "r") as fh:
# remove lines with leading # (comments), they
# are not allowed in json
cfg = re.sub(r"(?m)^\s*#.*\n?", "", fh.read())
except IOError as e:
raise RuntimeError(f"unable to open/read config file: {e}")
try:
cfg = json.loads(cfg)
except json.JSONDecodeError as e:
cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())]
msg = "\n".join(cfg_text)
raise RuntimeError(f"{e}\n{msg}")
cfg = MilterConfig(cfg)
if not args.debug:
logger.setLevel(cfg.loglevel)
logger.setLevel(cfg.get_loglevel(args.debug))
if args.socket:
socket = args.socket
elif cfg.socket:
socket = cfg.socket
elif cfg["socket"]:
socket = cfg["socket"]
else:
raise RuntimeError(
"listening socket is neither specified on the command line "
"nor in the configuration file")
if not cfg.rules:
if not cfg["rules"]:
raise RuntimeError("no rules configured")
for rule_cfg in cfg.rules:
if not rule_cfg.actions:
for rule in cfg["rules"]:
if not rule["actions"]:
raise RuntimeError(
f"{rule_cfg.name}: no actions configured")
f"{rule['name']}: no actions configured")
except (RuntimeError, AssertionError) as e:
logger.error(e)
logger.error(f"error in config file: {e}")
sys.exit(255)
try:
ModifyMilter.set_config(cfg)
except (RuntimeError, ValueError) as e:
ModifyMilter.set_config(cfg, args.debug)
except RuntimeError as e:
logger.error(e)
sys.exit(254)
@@ -147,6 +167,7 @@ def main():
mailer.queue.put(None)
logger.info("pymodmilter stopped")
sys.exit(rc)

150
pymodmilter/storage.py
View File

@@ -28,24 +28,26 @@ from time import gmtime
from pymodmilter.base import CustomLogger
from pymodmilter.conditions import Conditions
from pymodmilter.config import ActionConfig
from pymodmilter.notify import Notify
class BaseMailStorage:
"Mail storage base class"
_headersonly = True
def __init__(self, original=False, metadata=False, metavar=None):
def __init__(self, original=False, metadata=False, metavar=None,
pretend=False):
self.original = original
self.metadata = metadata
self.metavar = metavar
return
self.pretend = False
def add(self, data, qid, mailfrom="", recipients=[]):
"Add email to storage."
return ("", "")
def execute(self, milter, pretend=False,
logger=logging.getLogger(__name__)):
def execute(self, milter, logger):
return
def find(self, mailfrom=None, recipients=None, older_than=None):
@@ -69,9 +71,9 @@ class FileMailStorage(BaseMailStorage):
"Storage class to store mails on filesystem."
_headersonly = False
def __init__(self, directory, original=False, metadata=False,
metavar=None):
super().__init__(original, metadata, metavar)
def __init__(self, directory, original=False, metadata=False, metavar=None,
pretend=False):
super().__init__(original, metadata, metavar, pretend)
# check if directory exists and is writable
if not os.path.isdir(directory) or \
not os.access(directory, os.W_OK):
@@ -81,6 +83,15 @@ class FileMailStorage(BaseMailStorage):
self.directory = directory
self._metadata_suffix = ".metadata"
def __str__(self):
cfg = []
cfg.append(f"metadata={self.metadata}")
cfg.append(f"metavar={self.metavar}")
cfg.append(f"pretend={self.pretend}")
cfg.append(f"directory={self.directory}")
cfg.append(f"original={self.original}")
return "FileMailStorage(" + ", ".join(cfg) + ")"
def get_storageid(self, qid):
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
return f"{timestamp}_{qid}"
@@ -144,8 +155,7 @@ class FileMailStorage(BaseMailStorage):
return storage_id, metafile, datafile
def execute(self, milter, pretend=False,
logger=logging.getLogger(__name__)):
def execute(self, milter, logger):
if self.original:
milter.fp.seek(0)
data = milter.fp.read
@@ -158,7 +168,7 @@ class FileMailStorage(BaseMailStorage):
recipients = list(milter.msginfo["rcpts"])
subject = milter.msg["subject"] or ""
if not pretend:
if not self.pretend:
storage_id, metafile, datafile = self.add(
data(), milter.qid, mailfrom, recipients, subject)
logger.info(f"stored message in file {datafile}")
@@ -174,7 +184,7 @@ class FileMailStorage(BaseMailStorage):
def get_metadata(self, storage_id):
"Return metadata of email in storage."
super(FileMailStorage, self).get_metadata(storage_id)
super().get_metadata(storage_id)
if not self.metadata:
return None
@@ -197,7 +207,7 @@ class FileMailStorage(BaseMailStorage):
def find(self, mailfrom=None, recipients=None, older_than=None):
"Find emails in storage."
super(FileMailStorage, self).find(mailfrom, recipients, older_than)
super().find(mailfrom, recipients, older_than)
if isinstance(mailfrom, str):
mailfrom = [mailfrom]
if isinstance(recipients, str):
@@ -238,7 +248,7 @@ class FileMailStorage(BaseMailStorage):
def delete(self, storage_id, recipients=None):
"Delete email from storage."
super(FileMailStorage, self).delete(storage_id, recipients)
super().delete(storage_id, recipients)
if not recipients or not self.metadata:
self._remove(storage_id)
@@ -264,7 +274,7 @@ class FileMailStorage(BaseMailStorage):
self._save_metafile(metafile, metadata)
def get_mail(self, storage_id):
super(FileMailStorage, self).get_mail(storage_id)
super().get_mail(storage_id)
metadata = self.get_metadata(storage_id)
_, datafile = self._get_file_paths(storage_id)
@@ -275,31 +285,101 @@ class FileMailStorage(BaseMailStorage):
return (metadata, data)
class Store:
STORAGE_TYPES = {
"file": FileMailStorage}
def __init__(self, cfg, local_addrs, debug):
self.cfg = cfg
self.logger = logging.getLogger(cfg["name"])
self.logger.setLevel(cfg.get_loglevel(debug))
storage_type = cfg["args"]["type"]
del cfg["args"]["type"]
cfg["args"]["pretend"] = cfg["pretend"]
self._storage = self.STORAGE_TYPES[storage_type](
**cfg["args"])
self._headersonly = self._storage._headersonly
def __str__(self):
cfg = []
for key, value in self.cfg["args"].items():
cfg.append(f"{key}={value}")
class_name = type(self._storage).__name__
return f"{class_name}(" + ", ".join(cfg) + ")"
def execute(self, milter):
logger = CustomLogger(
self.logger, {"name": self.cfg["name"], "qid": milter.qid})
self._storage.execute(milter, logger)
class Quarantine:
"Quarantine class."
_headersonly = False
def __init__(self, storage, notification=None, whitelist=None,
milter_action=None, reject_reason="Message rejected"):
self.storage = storage.action(**storage.args, metadata=True)
self.storage_name = storage.name
self.storage_logger = storage.logger
def __init__(self, cfg, local_addrs, debug):
self.cfg = cfg
self.logger = logging.getLogger(cfg["name"])
self.logger.setLevel(cfg.get_loglevel(debug))
self.notification = notification
if self.notification is not None:
self.notification = notification.action(**notification.args)
self.notification_name = notification.name
self.notification_logger = notification.logger
self.whitelist = Conditions(whitelist)
self.milter_action = milter_action
self.reject_reason = reject_reason
store_cfg = ActionConfig({
"name": cfg["name"],
"loglevel": cfg["loglevel"],
"pretend": cfg["pretend"],
"type": "store",
"args": cfg["args"]["store"].get_config()})
self.store = Store(store_cfg, local_addrs, debug)
def execute(self, milter, pretend=False,
logger=logging.getLogger(__name__)):
self.notify = None
if "notify" in cfg["args"]:
notify_cfg = ActionConfig({
"name": cfg["name"],
"loglevel": cfg["loglevel"],
"pretend": cfg["pretend"],
"type": "notify",
"args": cfg["args"]["notify"].get_config()})
self.notify = Notify(notify_cfg, local_addrs, debug)
self.whitelist = None
if "whitelist" in cfg["args"]:
whitelist_cfg = cfg["args"]["whitelist"]
whitelist_cfg["name"] = cfg["name"]
whitelist_cfg["loglevel"] = cfg["loglevel"]
self.whitelist = Conditions(
whitelist_cfg,
local_addrs=[],
debug=debug)
self.milter_action = None
if "milter_action" in cfg["args"]:
self.milter_action = cfg["args"]["milter_action"]
self.reject_reason = None
if "reject_reason" in cfg["args"]:
self.reject_reason = cfg["args"]["reject_reason"]
def __str__(self):
cfg = []
cfg.append(f"store={str(self.store)}")
if self.notify is not None:
cfg.append(f"notify={str(self.notify)}")
if self.whitelist is not None:
cfg.append(f"whitelist={str(self.whitelist)}")
for key in ["milter_action", "reject_reason"]:
if key not in self.cfg["args"]:
continue
value = self.cfg["args"][key]
cfg.append(f"{key}={value}")
class_name = type(self).__name__
return f"{class_name}(" + ", ".join(cfg) + ")"
def execute(self, milter):
logger = CustomLogger(
self.logger, {"name": self.cfg["name"], "qid": milter.qid})
wl_rcpts = []
if self.whitelist:
wl_rcpts = self.whitelist.get_wl_rcpts(
milter.msginfo["mailfrom"], milter.msginfo["rcpts"])
milter.msginfo["mailfrom"], milter.msginfo["rcpts"], logger)
logger.info(f"whitelisted recipients: {wl_rcpts}")
rcpts = [
@@ -312,14 +392,10 @@ class Quarantine:
logger.info(f"add to quarantine for recipients: {rcpts}")
milter.msginfo["rcpts"] = rcpts
custom_logger = CustomLogger(
self.storage_logger, {"name": self.storage_name})
self.storage.execute(milter, pretend, custom_logger)
self.store.execute(milter)
if self.notification is not None:
custom_logger = CustomLogger(
self.notification_logger, {"name": self.notification_name})
self.notification.execute(milter, pretend, custom_logger)
if self.notify is not None:
self.notify.execute(milter)
milter.msginfo["rcpts"].extend(wl_rcpts)
milter.delrcpt(rcpts)

36
pymodmilter/whitelist.py
View File

@@ -26,17 +26,21 @@ from playhouse.db_url import connect
class WhitelistBase:
"Whitelist base class"
def __init__(self, cfg):
self.name = cfg["name"]
self.logger = logging.getLogger(__name__)
def __init__(self, cfg, debug):
self.cfg = cfg
self.logger = logging.getLogger(cfg["name"])
self.logger.setLevel(cfg.get_loglevel(debug))
peewee_logger = logging.getLogger("peewee")
peewee_logger.setLevel(cfg.get_loglevel(debug))
self.valid_entry_regex = re.compile(
r"^[a-zA-Z0-9_.=+-]*?(@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)?$")
self.batv_regex = re.compile(
r"^prvs=[0-9]{4}[0-9A-Fa-f]{6}=(?P<LEFT_PART>.+?)@")
def remove_batv(self, addr):
return self.batv_regex.sub("\g<LEFT_PART>", addr, count=1)
return self.batv_regex.sub(r"\g<LEFT_PART>", addr, count=1)
def check(self, mailfrom, recipient):
"Check if mailfrom/recipient combination is whitelisted."
@@ -82,8 +86,8 @@ class DatabaseWhitelist(WhitelistBase):
_db_connections = {}
_db_tables = {}
def __init__(self, cfg):
super().__init__(cfg)
def __init__(self, cfg, debug):
super().__init__(cfg, debug)
tablename = cfg["table"]
connection_string = cfg["connection"]
@@ -110,9 +114,10 @@ class DatabaseWhitelist(WhitelistBase):
self.meta = Meta
self.meta.database = db
self.meta.table_name = tablename
self.model = type(f"WhitelistModel_{self.name}", (WhitelistModel,), {
"Meta": self.meta
})
self.model = type(
f"WhitelistModel_{self.cfg['name']}",
(WhitelistModel,),
{"Meta": self.meta})
if connection_string not in DatabaseWhitelist._db_tables.keys():
DatabaseWhitelist._db_tables[connection_string] = []
@@ -125,6 +130,13 @@ class DatabaseWhitelist(WhitelistBase):
raise RuntimeError(
f"unable to initialize table '{tablename}': {e}")
def __str__(self):
cfg = []
for arg in ("connection", "table"):
if arg in self.cfg:
cfg.append(f"{arg}={self.cfg[arg]}")
return "DatabaseWhitelist(" + ", ".join(cfg) + ")"
def _entry_to_dict(self, entry):
result = {}
result[entry.id] = {
@@ -147,14 +159,14 @@ class DatabaseWhitelist(WhitelistBase):
value += 1
return value
def check(self, mailfrom, recipient):
def check(self, mailfrom, recipient, logger):
# check if mailfrom/recipient combination is whitelisted
super().check(mailfrom, recipient)
mailfrom = self.remove_batv(mailfrom)
recipient = self.remove_batv(recipient)
# generate list of possible mailfroms
self.logger.debug(
logger.debug(
f"query database for whitelist entries from <{mailfrom}> "
f"to <{recipient}>")
mailfroms = [""]

2
setup.py
View File

@@ -45,6 +45,6 @@ setup(name = "pymodmilter",
]
)
],
install_requires = ["pymilter", "netaddr", "beautifulsoup4[lxml]", "peewee"],
install_requires = ["pymilter", "jsonschema", "netaddr", "beautifulsoup4[lxml]", "peewee"],
python_requires = ">=3.8"
)