spacefreak/pyquarantine-milter
1
0
Fork 0
Code Issues Pull Requests Releases Activity

add option to store unparse msg and rework rules logic

Browse Source
This commit is contained in:
Thomas Oettli
2021-09-06 15:11:21 +02:00
parent 6b8ad1b078
commit d20e868452
4 changed files with 80 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@@ -28,7 +28,7 @@ Config options in **global** section:
The socket used to communicate with the MTA. If it is not specified in the config, it has to be set as command line option. The socket used to communicate with the MTA. If it is not specified in the config, it has to be set as command line option.
* **local_addrs** (optional) * **local_addrs** (optional)
A list of hosts and network addresses which are considered local. It is used to for the condition option [local](#Conditions). A list of hosts and network addresses which are considered local. It is used to for the condition option [local](#Conditions).
Default: **::1/128, 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16** Default: **fe80::/64, ::1/128, 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16**
* **loglevel** (optional) * **loglevel** (optional)
Set the log level. This option may be overriden by any rule or action object. Possible values are: Set the log level. This option may be overriden by any rule or action object. Possible values are:
* **error** * **error**
@@ -115,6 +115,9 @@ Config options for **store** actions:
* **storage_type** * **storage_type**
Storage type. Possible values are: Storage type. Possible values are:
* **file** * **file**
* **original** (optional)
Default: **false**
If set to true, store the message as received by the MTA instead of storing the current state of the message, that may was modified already by other actions.
Config options for **file** storage: Config options for **file** storage:
* **directory** * **directory**

116
pymodmilter/__init__.py
View File

@@ -32,9 +32,10 @@ import json
from Milter.utils import parse_addr from Milter.utils import parse_addr
from collections import defaultdict from collections import defaultdict
from email import message_from_binary_file
from email.header import Header from email.header import Header
from email.parser import BytesFeedParser
from email.policy import default as default_policy, SMTP from email.policy import default as default_policy, SMTP
from io import BytesIO
from netaddr import IPNetwork, AddrFormatError from netaddr import IPNetwork, AddrFormatError
from pymodmilter.base import CustomLogger, BaseConfig, MilterMessage from pymodmilter.base import CustomLogger, BaseConfig, MilterMessage
@@ -92,6 +93,7 @@ class ModifyMilterConfig(BaseConfig):
"should be list of strings" "should be list of strings"
else: else:
local_addrs = [ local_addrs = [
"fe80::/64",
"::1/128", "::1/128",
"127.0.0.0/8", "127.0.0.0/8",
"10.0.0.0/8", "10.0.0.0/8",
@@ -139,12 +141,6 @@ class ModifyMilter(Milter.Base):
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
self.logger.setLevel(ModifyMilter._loglevel) self.logger.setLevel(ModifyMilter._loglevel)
# save rules, it must not change during runtime
self.rules = ModifyMilter._rules.copy()
self.msg = None
self._replace_body = False
def addheader(self, field, value, idx=-1): def addheader(self, field, value, idx=-1):
value = replace_illegal_chars(Header(s=value).encode()) value = replace_illegal_chars(Header(s=value).encode())
self.logger.debug(f"milter: addheader: {field}: {value}") self.logger.debug(f"milter: addheader: {field}: {value}")
@@ -181,28 +177,20 @@ class ModifyMilter(Milter.Base):
self.addheader(field, value) self.addheader(field, value)
def replacebody(self): def replacebody(self):
self._replace_body = True self._replacebody = True
def connect(self, IPname, family, hostaddr): def connect(self, IPname, family, hostaddr):
try: try:
if hostaddr is None: if hostaddr is None:
self.logger.error("unable to proceed, host address is None") self.logger.error(f"received invalid host address {hostaddr}, "
f"unable to proceed")
return Milter.TEMPFAIL return Milter.TEMPFAIL
self.IP = hostaddr[0]
self.port = hostaddr[1]
self.logger.debug( self.logger.debug(
f"accepted milter connection from {hostaddr[0]} " f"accepted milter connection from {self.IP} "
f"port {hostaddr[1]}") f"port {self.port}")
# remove rules which ignore this host
for rule in self.rules.copy():
if rule.ignores(host=hostaddr[0]):
self.rules.remove(rule)
if not self.rules:
self.logger.debug(
f"host {hostaddr[0]} is ignored by all rules, "
f"skip further processing")
return Milter.ACCEPT
except Exception as e: except Exception as e:
self.logger.exception( self.logger.exception(
f"an exception occured in connect method: {e}") f"an exception occured in connect method: {e}")
@@ -210,20 +198,21 @@ class ModifyMilter(Milter.Base):
return Milter.CONTINUE return Milter.CONTINUE
def hello(self, heloname):
try:
self.heloname = heloname
self.logger.debug(f"received HELO name: {heloname}")
except Exception as e:
self.logger.exception(
f"an exception occured in hello method: {e}")
return Milter.TEMPFAIL
return Milter.CONTINUE
def envfrom(self, mailfrom, *str): def envfrom(self, mailfrom, *str):
try: try:
mailfrom = "@".join(parse_addr(mailfrom)).lower() self.mailfrom = "@".join(parse_addr(mailfrom)).lower()
for rule in self.rules.copy(): self.rcpts = set()
if rule.ignores(envfrom=mailfrom):
self.rules.remove(rule)
if not self.rules:
self.logger.debug(
f"envelope-from address {mailfrom} is ignored by "
f"all rules, skip further processing")
return Milter.ACCEPT
self.recipients = set()
except Exception as e: except Exception as e:
self.logger.exception( self.logger.exception(
f"an exception occured in envfrom method: {e}") f"an exception occured in envfrom method: {e}")
@@ -233,7 +222,7 @@ class ModifyMilter(Milter.Base):
def envrcpt(self, to, *str): def envrcpt(self, to, *str):
try: try:
self.recipients.add("@".join(parse_addr(to)).lower()) self.rcpts.add("@".join(parse_addr(to)).lower())
except Exception as e: except Exception as e:
self.logger.exception( self.logger.exception(
f"an exception occured in envrcpt method: {e}") f"an exception occured in envrcpt method: {e}")
@@ -243,32 +232,25 @@ class ModifyMilter(Milter.Base):
def data(self): def data(self):
try: try:
for rule in self.rules.copy():
if rule.ignores(envto=[*self.recipients]):
self.rules.remove(rule)
if not self.rules:
self.logger.debug(
"envelope-to addresses are ignored by all rules, "
"skip further processing")
return Milter.ACCEPT
self.qid = self.getsymval('i') self.qid = self.getsymval('i')
self.logger = CustomLogger(self.logger, {"qid": self.qid}) self.logger = CustomLogger(self.logger, {"qid": self.qid})
self.logger.debug("received queue-id from MTA") self.logger.debug("received queue-id from MTA")
self.fields = None self.rules = []
self.fields_bytes = None self._headersonly = True
self.body_data = None for rule in ModifyMilter._rules:
if not rule.ignores(host=self.IP, envfrom=self.mailfrom,
self._fp = BytesFeedParser( envto=[*self.rcpts]):
_factory=MilterMessage, policy=default_policy) self.rules.append(rule)
self._keep_body = False
for rule in self.rules:
if rule.need_body(): if rule.need_body():
self._keep_body = True self._headersonly = False
break
if not self.rules:
self.logger.debug(
"message is ignored by all rules, skip further processing")
return Milter.ACCEPT
self.fp = BytesIO()
except Exception as e: except Exception as e:
self.logger.exception( self.logger.exception(
f"an exception occured in data method: {e}") f"an exception occured in data method: {e}")
@@ -282,7 +264,7 @@ class ModifyMilter(Milter.Base):
field = field.encode("ascii", errors="replace") field = field.encode("ascii", errors="replace")
value = value.encode("ascii", errors="replace") value = value.encode("ascii", errors="replace")
self._fp.feed(field + b": " + value + b"\r\n") self.fp.write(field + b": " + value + b"\r\n")
except Exception as e: except Exception as e:
self.logger.exception( self.logger.exception(
f"an exception occured in header method: {e}") f"an exception occured in header method: {e}")
@@ -292,7 +274,7 @@ class ModifyMilter(Milter.Base):
def eoh(self): def eoh(self):
try: try:
self._fp.feed(b"\r\n") self.fp.write(b"\r\n")
except Exception as e: except Exception as e:
self.logger.exception( self.logger.exception(
f"an exception occured in eoh method: {e}") f"an exception occured in eoh method: {e}")
@@ -302,8 +284,8 @@ class ModifyMilter(Milter.Base):
def body(self, chunk): def body(self, chunk):
try: try:
if self._keep_body: if not self._headersonly:
self._fp.feed(chunk) self.fp.write(chunk)
except Exception as e: except Exception as e:
self.logger.exception( self.logger.exception(
f"an exception occured in body method: {e}") f"an exception occured in body method: {e}")
@@ -313,7 +295,19 @@ class ModifyMilter(Milter.Base):
def eom(self): def eom(self):
try: try:
self.msg = self._fp.close() self.fp.seek(0)
self.msg = message_from_binary_file(
self.fp, _class=MilterMessage, policy=default_policy)
self.msg_info = defaultdict(str)
self.msg_info["ip"] = self.IP
self.msg_info["port"] = self.port
self.msg_info["heloname"] = self.heloname
self.msg_info["envfrom"] = self.mailfrom
self.msg_info["rcpts"] = self.rcpts
self.msg_info["qid"] = self.qid
self._replacebody = False
milter_action = None milter_action = None
for rule in self.rules: for rule in self.rules:
milter_action = rule.execute(self) milter_action = rule.execute(self)
@@ -321,7 +315,7 @@ class ModifyMilter(Milter.Base):
if milter_action is not None: if milter_action is not None:
break break
if self._replace_body: if self._replacebody:
data = self.msg.as_bytes(policy=SMTP) data = self.msg.as_bytes(policy=SMTP)
body_pos = data.find(b"\r\n\r\n") + 4 body_pos = data.find(b"\r\n\r\n") + 4
self.logger.debug("milter: replacebody") self.logger.debug("milter: replacebody")

15
pymodmilter/actions.py
View File

@@ -32,6 +32,7 @@ from collections import defaultdict
from copy import copy from copy import copy
from datetime import datetime from datetime import datetime
from email.message import MIMEPart from email.message import MIMEPart
from email.policy import SMTP
from pymodmilter import CustomLogger, BaseConfig from pymodmilter import CustomLogger, BaseConfig
from pymodmilter.conditions import ConditionsConfig, Conditions from pymodmilter.conditions import ConditionsConfig, Conditions
@@ -204,7 +205,7 @@ def _patch_message_body(milter, action, text_template, html_template, logger):
def _wrap_message(milter, logger): def _wrap_message(milter, logger):
attachment = MIMEPart() attachment = MIMEPart()
attachment.set_content(milter.msg.as_bytes(), attachment.set_content(milter.msg.as_bytes(policy=SMTP),
maintype="plain", subtype="text", maintype="plain", subtype="text",
disposition="attachment", disposition="attachment",
filename=f"{milter.qid}.eml", filename=f"{milter.qid}.eml",
@@ -317,7 +318,7 @@ def rewrite_links(milter, repl, pretend=False,
milter.replacebody() milter.replacebody()
def store(milter, directory, pretend=False, def store(milter, directory, original=False, pretend=False,
logger=logging.getLogger(__name__)): logger=logging.getLogger(__name__)):
timestamp = datetime.now().strftime("%Y%m%d%H%M%S") timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
store_id = f"{timestamp}_{milter.qid}" store_id = f"{timestamp}_{milter.qid}"
@@ -326,7 +327,11 @@ def store(milter, directory, pretend=False,
logger.info(f"store message in file {datafile}") logger.info(f"store message in file {datafile}")
try: try:
with open(datafile, "wb") as fp: with open(datafile, "wb") as fp:
fp.write(milter.msg.as_bytes()) if original:
milter.fp.seek(0)
fp.write(milter.fp.read())
else:
fp.write(milter.msg.as_bytes(policy=SMTP))
except IOError as e: except IOError as e:
raise RuntimeError(f"unable to store message: {e}") raise RuntimeError(f"unable to store message: {e}")
@@ -468,6 +473,10 @@ class ActionConfig(BaseConfig):
f"{self['name']}: storage_type: invalid value, " \ f"{self['name']}: storage_type: invalid value, " \
f"should be string" f"should be string"
self["storage_type"] = cfg["storage_type"] self["storage_type"] = cfg["storage_type"]
if "original" in cfg:
self.add_bool_arg(cfg, "original")
if self["storage_type"] == "file": if self["storage_type"] == "file":
self.add_string_arg(cfg, "directory") self.add_string_arg(cfg, "directory")
else: else:

11
pymodmilter/docs/pymodmilter.conf.example
View File

@@ -28,7 +28,7 @@
# Notes: A list of local hosts and networks. # Notes: A list of local hosts and networks.
# Value: [ LIST ] # Value: [ LIST ]
# #
"local_addrs": ["::1/128", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"], "local_addrs": ["fe80::/64", "::1/128", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"],
# Option: loglevel # Option: loglevel
# Type: String # Type: String
@@ -206,7 +206,14 @@
# Type: String # Type: String
# Notes: Directory used to store e-mails. # Notes: Directory used to store e-mails.
# Value: [ file ] # Value: [ file ]
"directory": "/mnt/messages" "directory": "/mnt/messages",
# Option: original
# Type: Bool
# Notes: If set to true, store the message as received by the MTA instead of storing the current state
# of the message, that may was modified already by other actions.
# Value: [ true | false ]
"original": true
} }
] ]
} }