add new file storage/quarantine and change version to 1.2.0
This commit is contained in:
@@ -16,12 +16,13 @@ __all__ = [
|
|||||||
"actions",
|
"actions",
|
||||||
"base",
|
"base",
|
||||||
"conditions",
|
"conditions",
|
||||||
|
"modifications",
|
||||||
"rules",
|
"rules",
|
||||||
"run",
|
"run",
|
||||||
"ModifyMilterConfig",
|
"ModifyMilterConfig",
|
||||||
"ModifyMilter"]
|
"ModifyMilter"]
|
||||||
|
|
||||||
__version__ = "1.1.7"
|
__version__ = "1.2.0"
|
||||||
|
|
||||||
from pymodmilter import _runtime_patches
|
from pymodmilter import _runtime_patches
|
||||||
|
|
||||||
@@ -327,18 +328,15 @@ class ModifyMilter(Milter.Base):
|
|||||||
|
|
||||||
def eom(self):
|
def eom(self):
|
||||||
try:
|
try:
|
||||||
# setup msg and msg_info to be read/modified by rules and actions
|
# msg and msginfo contain the runtime data that
|
||||||
|
# is read/modified by actions
|
||||||
self.fp.seek(0)
|
self.fp.seek(0)
|
||||||
self.msg = message_from_binary_file(
|
self.msg = message_from_binary_file(
|
||||||
self.fp, _class=MilterMessage, policy=SMTPUTF8.clone(
|
self.fp, _class=MilterMessage, policy=SMTPUTF8.clone(
|
||||||
refold_source='none'))
|
refold_source='none'))
|
||||||
self.msg_info = defaultdict(str)
|
self.msginfo = {
|
||||||
self.msg_info["ip"] = self.IP
|
"mailfrom": self.mailfrom,
|
||||||
self.msg_info["port"] = self.port
|
"rcpts": self.rcpts}
|
||||||
self.msg_info["heloname"] = self.heloname
|
|
||||||
self.msg_info["envfrom"] = self.mailfrom
|
|
||||||
self.msg_info["rcpts"] = self.rcpts
|
|
||||||
self.msg_info["qid"] = self.qid
|
|
||||||
|
|
||||||
self._replacebody = False
|
self._replacebody = False
|
||||||
milter_action = None
|
milter_action = None
|
||||||
|
|||||||
@@ -13,328 +13,17 @@
|
|||||||
#
|
#
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"add_header",
|
|
||||||
"mod_header",
|
|
||||||
"del_header",
|
|
||||||
"add_disclaimer",
|
|
||||||
"rewrite_links",
|
|
||||||
"store",
|
|
||||||
"ActionConfig",
|
"ActionConfig",
|
||||||
"Action"]
|
"Action"]
|
||||||
|
|
||||||
import logging
|
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from base64 import b64encode
|
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from collections import defaultdict
|
|
||||||
from copy import copy
|
|
||||||
from datetime import datetime
|
|
||||||
from email.message import MIMEPart
|
|
||||||
|
|
||||||
from pymodmilter import CustomLogger, BaseConfig
|
from pymodmilter import CustomLogger, BaseConfig
|
||||||
from pymodmilter.conditions import ConditionsConfig, Conditions
|
from pymodmilter.conditions import ConditionsConfig, Conditions
|
||||||
from pymodmilter import replace_illegal_chars
|
from pymodmilter import modifications, storages
|
||||||
|
|
||||||
|
|
||||||
def add_header(milter, field, value, pretend=False,
|
|
||||||
logger=logging.getLogger(__name__)):
|
|
||||||
"""Add a mail header field."""
|
|
||||||
header = f"{field}: {value}"
|
|
||||||
if logger.getEffectiveLevel() == logging.DEBUG:
|
|
||||||
logger.debug(f"add_header: {header}")
|
|
||||||
else:
|
|
||||||
logger.info(f"add_header: {header[0:70]}")
|
|
||||||
|
|
||||||
milter.msg.add_header(field, replace_illegal_chars(value))
|
|
||||||
|
|
||||||
if not pretend:
|
|
||||||
milter.addheader(field, value)
|
|
||||||
|
|
||||||
|
|
||||||
def mod_header(milter, field, value, search=None, pretend=False,
|
|
||||||
logger=logging.getLogger(__name__)):
|
|
||||||
"""Change the value of a mail header field."""
|
|
||||||
if isinstance(field, str):
|
|
||||||
field = re.compile(field, re.IGNORECASE)
|
|
||||||
|
|
||||||
if isinstance(search, str):
|
|
||||||
search = re.compile(search, re.MULTILINE + re.DOTALL + re.IGNORECASE)
|
|
||||||
|
|
||||||
idx = defaultdict(int)
|
|
||||||
|
|
||||||
for i, (f, v) in enumerate(milter.msg.items()):
|
|
||||||
f_lower = f.lower()
|
|
||||||
idx[f_lower] += 1
|
|
||||||
|
|
||||||
if not field.match(f):
|
|
||||||
continue
|
|
||||||
|
|
||||||
new_value = v
|
|
||||||
if search is not None:
|
|
||||||
new_value = search.sub(value, v).strip()
|
|
||||||
else:
|
|
||||||
new_value = value
|
|
||||||
|
|
||||||
if not new_value:
|
|
||||||
logger.warning(
|
|
||||||
"mod_header: resulting value is empty, "
|
|
||||||
"skip modification")
|
|
||||||
continue
|
|
||||||
|
|
||||||
if new_value == v:
|
|
||||||
continue
|
|
||||||
|
|
||||||
header = f"{f}: {v}"
|
|
||||||
new_header = f"{f}: {new_value}"
|
|
||||||
|
|
||||||
if logger.getEffectiveLevel() == logging.DEBUG:
|
|
||||||
logger.debug(f"mod_header: {header}: {new_header}")
|
|
||||||
else:
|
|
||||||
logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}")
|
|
||||||
|
|
||||||
milter.msg.replace_header(
|
|
||||||
f, replace_illegal_chars(new_value), idx=idx[f_lower])
|
|
||||||
|
|
||||||
if not pretend:
|
|
||||||
milter.chgheader(f, new_value, idx=idx[f_lower])
|
|
||||||
|
|
||||||
|
|
||||||
def del_header(milter, field, value=None, pretend=False,
|
|
||||||
logger=logging.getLogger(__name__)):
|
|
||||||
"""Delete a mail header field."""
|
|
||||||
if isinstance(field, str):
|
|
||||||
field = re.compile(field, re.IGNORECASE)
|
|
||||||
|
|
||||||
if isinstance(value, str):
|
|
||||||
value = re.compile(value, re.MULTILINE + re.DOTALL + re.IGNORECASE)
|
|
||||||
|
|
||||||
idx = defaultdict(int)
|
|
||||||
|
|
||||||
for f, v in milter.msg.items():
|
|
||||||
f_lower = f.lower()
|
|
||||||
idx[f_lower] += 1
|
|
||||||
|
|
||||||
if not field.match(f):
|
|
||||||
continue
|
|
||||||
|
|
||||||
if value is not None and not value.search(v):
|
|
||||||
continue
|
|
||||||
|
|
||||||
header = f"{f}: {v}"
|
|
||||||
if logger.getEffectiveLevel() == logging.DEBUG:
|
|
||||||
logger.debug(f"del_header: {header}")
|
|
||||||
else:
|
|
||||||
logger.info(f"del_header: {header[0:70]}")
|
|
||||||
milter.msg.remove_header(f, idx=idx[f_lower])
|
|
||||||
|
|
||||||
if not pretend:
|
|
||||||
milter.chgheader(f, "", idx=idx[f_lower])
|
|
||||||
|
|
||||||
idx[f_lower] -= 1
|
|
||||||
|
|
||||||
|
|
||||||
def _get_body_content(msg, pref):
|
|
||||||
part = None
|
|
||||||
content = None
|
|
||||||
if not msg.is_multipart() and msg.get_content_type() == f"text/{pref}":
|
|
||||||
part = msg
|
|
||||||
else:
|
|
||||||
part = msg.get_body(preferencelist=(pref))
|
|
||||||
|
|
||||||
if part is not None:
|
|
||||||
content = part.get_content()
|
|
||||||
|
|
||||||
return (part, content)
|
|
||||||
|
|
||||||
|
|
||||||
def _has_content_before_body_tag(soup):
|
|
||||||
s = copy(soup)
|
|
||||||
for element in s.find_all("head") + s.find_all("body"):
|
|
||||||
element.extract()
|
|
||||||
|
|
||||||
if len(s.text.strip()) > 0:
|
|
||||||
return True
|
|
||||||
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def _patch_message_body(milter, action, text_template, html_template, logger):
|
|
||||||
text_body, text_content = _get_body_content(milter.msg, "plain")
|
|
||||||
html_body, html_content = _get_body_content(milter.msg, "html")
|
|
||||||
|
|
||||||
if text_content is None and html_content is None:
|
|
||||||
raise RuntimeError("message does not contain any body part")
|
|
||||||
|
|
||||||
if text_content is not None:
|
|
||||||
logger.info(f"{action} text disclaimer")
|
|
||||||
|
|
||||||
if action == "prepend":
|
|
||||||
content = f"{text_template}{text_content}"
|
|
||||||
else:
|
|
||||||
content = f"{text_content}{text_template}"
|
|
||||||
|
|
||||||
text_body.set_content(
|
|
||||||
content.encode(), maintype="text", subtype="plain")
|
|
||||||
text_body.set_param("charset", "UTF-8", header="Content-Type")
|
|
||||||
del text_body["MIME-Version"]
|
|
||||||
|
|
||||||
if html_content is not None:
|
|
||||||
logger.info(f"{action} html disclaimer")
|
|
||||||
|
|
||||||
soup = BeautifulSoup(html_content, "html.parser")
|
|
||||||
|
|
||||||
body = soup.find('body')
|
|
||||||
if not body:
|
|
||||||
body = soup
|
|
||||||
elif _has_content_before_body_tag(soup):
|
|
||||||
body = soup
|
|
||||||
|
|
||||||
if action == "prepend":
|
|
||||||
body.insert(0, copy(html_template))
|
|
||||||
else:
|
|
||||||
body.append(html_template)
|
|
||||||
|
|
||||||
html_body.set_content(
|
|
||||||
str(soup).encode(), maintype="text", subtype="html")
|
|
||||||
html_body.set_param("charset", "UTF-8", header="Content-Type")
|
|
||||||
del html_body["MIME-Version"]
|
|
||||||
|
|
||||||
|
|
||||||
def _wrap_message(milter, logger):
|
|
||||||
attachment = MIMEPart()
|
|
||||||
attachment.set_content(milter.msg.as_bytes(),
|
|
||||||
maintype="plain", subtype="text",
|
|
||||||
disposition="attachment",
|
|
||||||
filename=f"{milter.qid}.eml",
|
|
||||||
params={"name": f"{milter.qid}.eml"})
|
|
||||||
|
|
||||||
milter.msg.clear_content()
|
|
||||||
milter.msg.set_content(
|
|
||||||
"Please see the original email attached.")
|
|
||||||
milter.msg.add_alternative(
|
|
||||||
"<html><body>Please see the original email attached.</body></html>",
|
|
||||||
subtype="html")
|
|
||||||
milter.msg.make_mixed()
|
|
||||||
milter.msg.attach(attachment)
|
|
||||||
|
|
||||||
|
|
||||||
def _inject_body(milter):
|
|
||||||
if not milter.msg.is_multipart():
|
|
||||||
milter.msg.make_mixed()
|
|
||||||
|
|
||||||
attachments = []
|
|
||||||
for attachment in milter.msg.iter_attachments():
|
|
||||||
if "content-disposition" not in attachment:
|
|
||||||
attachment["Content-Disposition"] = "attachment"
|
|
||||||
attachments.append(attachment)
|
|
||||||
|
|
||||||
milter.msg.clear_content()
|
|
||||||
milter.msg.set_content("")
|
|
||||||
milter.msg.add_alternative("", subtype="html")
|
|
||||||
milter.msg.make_mixed()
|
|
||||||
|
|
||||||
for attachment in attachments:
|
|
||||||
milter.msg.attach(attachment)
|
|
||||||
|
|
||||||
|
|
||||||
def add_disclaimer(milter, text_template, html_template, action, error_policy,
|
|
||||||
pretend=False, logger=logging.getLogger(__name__)):
|
|
||||||
"""Append or prepend a disclaimer to the mail body."""
|
|
||||||
old_headers = milter.msg.items()
|
|
||||||
|
|
||||||
try:
|
|
||||||
try:
|
|
||||||
_patch_message_body(
|
|
||||||
milter, action, text_template, html_template, logger)
|
|
||||||
except RuntimeError as e:
|
|
||||||
logger.info(f"{e}, inject empty plain and html body")
|
|
||||||
_inject_body(milter)
|
|
||||||
_patch_message_body(
|
|
||||||
milter, action, text_template, html_template, logger)
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(e)
|
|
||||||
if error_policy == "ignore":
|
|
||||||
logger.info(
|
|
||||||
"unable to add disclaimer to message body, "
|
|
||||||
"ignore error according to policy")
|
|
||||||
return
|
|
||||||
elif error_policy == "reject":
|
|
||||||
logger.info(
|
|
||||||
"unable to add disclaimer to message body, "
|
|
||||||
"reject message according to policy")
|
|
||||||
return [
|
|
||||||
("reject", "Message rejected due to error")]
|
|
||||||
|
|
||||||
logger.info("wrap original message in a new message envelope")
|
|
||||||
try:
|
|
||||||
_wrap_message(milter, logger)
|
|
||||||
_patch_message_body(
|
|
||||||
milter, action, text_template, html_template, logger)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(e)
|
|
||||||
raise Exception(
|
|
||||||
"unable to wrap message in a new message envelope, "
|
|
||||||
"give up ...")
|
|
||||||
|
|
||||||
if not pretend:
|
|
||||||
milter.update_headers(old_headers)
|
|
||||||
milter.replacebody()
|
|
||||||
|
|
||||||
|
|
||||||
def rewrite_links(milter, repl, pretend=False,
|
|
||||||
logger=logging.getLogger(__name__)):
|
|
||||||
"""Rewrite link targets in the mail html body."""
|
|
||||||
|
|
||||||
html_body, html_content = _get_body_content(milter.msg, "html")
|
|
||||||
if html_content is not None:
|
|
||||||
soup = BeautifulSoup(html_content, "html.parser")
|
|
||||||
|
|
||||||
rewritten = 0
|
|
||||||
for link in soup.find_all("a", href=True):
|
|
||||||
if not link["href"]:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if "{URL_B64}" in repl:
|
|
||||||
url_b64 = b64encode(link["href"].encode()).decode()
|
|
||||||
target = repl.replace("{URL_B64}", url_b64)
|
|
||||||
else:
|
|
||||||
target = repl
|
|
||||||
|
|
||||||
link["href"] = target
|
|
||||||
rewritten += 1
|
|
||||||
|
|
||||||
if rewritten:
|
|
||||||
logger.info(f"rewrote {rewritten} link(s) in html body")
|
|
||||||
|
|
||||||
html_body.set_content(
|
|
||||||
str(soup).encode(), maintype="text", subtype="html")
|
|
||||||
html_body.set_param("charset", "UTF-8", header="Content-Type")
|
|
||||||
del html_body["MIME-Version"]
|
|
||||||
|
|
||||||
if not pretend:
|
|
||||||
milter.replacebody()
|
|
||||||
|
|
||||||
|
|
||||||
def store(milter, directory, original=False, pretend=False,
|
|
||||||
logger=logging.getLogger(__name__)):
|
|
||||||
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
|
|
||||||
store_id = f"{timestamp}_{milter.qid}"
|
|
||||||
datafile = os.path.join(directory, store_id)
|
|
||||||
|
|
||||||
logger.info(f"store message in file {datafile}")
|
|
||||||
|
|
||||||
if not pretend:
|
|
||||||
try:
|
|
||||||
with open(datafile, "wb") as fp:
|
|
||||||
if original:
|
|
||||||
milter.fp.seek(0)
|
|
||||||
fp.write(milter.fp.read())
|
|
||||||
else:
|
|
||||||
fp.write(milter.msg.as_bytes())
|
|
||||||
except IOError as e:
|
|
||||||
raise RuntimeError(f"unable to store message: {e}")
|
|
||||||
|
|
||||||
|
|
||||||
class ActionConfig(BaseConfig):
|
class ActionConfig(BaseConfig):
|
||||||
@@ -369,7 +58,7 @@ class ActionConfig(BaseConfig):
|
|||||||
self["type"] = cfg["type"]
|
self["type"] = cfg["type"]
|
||||||
|
|
||||||
if self["type"] == "add_header":
|
if self["type"] == "add_header":
|
||||||
self["func"] = add_header
|
self["class"] = modifications.AddHeader
|
||||||
self["headersonly"] = True
|
self["headersonly"] = True
|
||||||
|
|
||||||
if "field" not in cfg and "header" in cfg:
|
if "field" not in cfg and "header" in cfg:
|
||||||
@@ -378,7 +67,7 @@ class ActionConfig(BaseConfig):
|
|||||||
self.add_string_arg(cfg, ("field", "value"))
|
self.add_string_arg(cfg, ("field", "value"))
|
||||||
|
|
||||||
elif self["type"] == "mod_header":
|
elif self["type"] == "mod_header":
|
||||||
self["func"] = mod_header
|
self["class"] = modifications.ModHeader
|
||||||
self["headersonly"] = True
|
self["headersonly"] = True
|
||||||
|
|
||||||
if "field" not in cfg and "header" in cfg:
|
if "field" not in cfg and "header" in cfg:
|
||||||
@@ -399,7 +88,7 @@ class ActionConfig(BaseConfig):
|
|||||||
raise ValueError(f"{self['name']}: {arg}: {e}")
|
raise ValueError(f"{self['name']}: {arg}: {e}")
|
||||||
|
|
||||||
elif self["type"] == "del_header":
|
elif self["type"] == "del_header":
|
||||||
self["func"] = del_header
|
self["class"] = modifications.DelHeader
|
||||||
self["headersonly"] = True
|
self["headersonly"] = True
|
||||||
|
|
||||||
if "field" not in cfg and "header" in cfg:
|
if "field" not in cfg and "header" in cfg:
|
||||||
@@ -419,7 +108,7 @@ class ActionConfig(BaseConfig):
|
|||||||
raise ValueError(f"{self['name']}: {arg}: {e}")
|
raise ValueError(f"{self['name']}: {arg}: {e}")
|
||||||
|
|
||||||
elif self["type"] == "add_disclaimer":
|
elif self["type"] == "add_disclaimer":
|
||||||
self["func"] = add_disclaimer
|
self["class"] = modifications.AddDisclaimer
|
||||||
self["headersonly"] = False
|
self["headersonly"] = False
|
||||||
|
|
||||||
if "html_template" not in cfg and "html_file" in cfg:
|
if "html_template" not in cfg and "html_file" in cfg:
|
||||||
@@ -460,12 +149,11 @@ class ActionConfig(BaseConfig):
|
|||||||
f"{self['name']}: unable to open/read template file: {e}")
|
f"{self['name']}: unable to open/read template file: {e}")
|
||||||
|
|
||||||
elif self["type"] == "rewrite_links":
|
elif self["type"] == "rewrite_links":
|
||||||
self["func"] = rewrite_links
|
self["class"] = modifications.RewriteLinks
|
||||||
self["headersonly"] = False
|
self["headersonly"] = False
|
||||||
self.add_string_arg(cfg, "repl")
|
self.add_string_arg(cfg, "repl")
|
||||||
|
|
||||||
elif self["type"] == "store":
|
elif self["type"] == "store":
|
||||||
self["func"] = store
|
|
||||||
self["headersonly"] = False
|
self["headersonly"] = False
|
||||||
|
|
||||||
assert "storage_type" in cfg, \
|
assert "storage_type" in cfg, \
|
||||||
@@ -479,7 +167,15 @@ class ActionConfig(BaseConfig):
|
|||||||
self.add_bool_arg(cfg, "original")
|
self.add_bool_arg(cfg, "original")
|
||||||
|
|
||||||
if self["storage_type"] == "file":
|
if self["storage_type"] == "file":
|
||||||
|
self["class"] = storages.FileMailStorage
|
||||||
self.add_string_arg(cfg, "directory")
|
self.add_string_arg(cfg, "directory")
|
||||||
|
# check if directory exists and is writable
|
||||||
|
if not os.path.isdir(self["args"]["directory"]) or \
|
||||||
|
not os.access(self["args"]["directory"], os.W_OK):
|
||||||
|
raise RuntimeError(
|
||||||
|
f"{self['name']}: file quarantine directory "
|
||||||
|
f"'{self['directory']}' does not exist or is "
|
||||||
|
f"not writable")
|
||||||
else:
|
else:
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
f"{self['name']}: storage_type: invalid storage type")
|
f"{self['name']}: storage_type: invalid storage type")
|
||||||
@@ -512,8 +208,7 @@ class Action:
|
|||||||
|
|
||||||
self.pretend = cfg["pretend"]
|
self.pretend = cfg["pretend"]
|
||||||
self._name = cfg["name"]
|
self._name = cfg["name"]
|
||||||
self._func = cfg["func"]
|
self._class = cfg["class"](**cfg["args"])
|
||||||
self._args = cfg["args"]
|
|
||||||
self._headersonly = cfg["headersonly"]
|
self._headersonly = cfg["headersonly"]
|
||||||
|
|
||||||
def headersonly(self):
|
def headersonly(self):
|
||||||
@@ -529,5 +224,5 @@ class Action:
|
|||||||
self.conditions.match(envfrom=milter.mailfrom,
|
self.conditions.match(envfrom=milter.mailfrom,
|
||||||
envto=[*milter.rcpts],
|
envto=[*milter.rcpts],
|
||||||
headers=milter.msg.items()):
|
headers=milter.msg.items()):
|
||||||
return self._func(milter=milter, pretend=self.pretend,
|
return self._class.execute(
|
||||||
logger=logger, **self._args)
|
milter=milter, pretend=self.pretend, logger=logger)
|
||||||
|
|||||||
320
pymodmilter/modifications.py
Normal file
320
pymodmilter/modifications.py
Normal file
@@ -0,0 +1,320 @@
|
|||||||
|
# PyMod-Milter is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# PyMod-Milter is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with PyMod-Milter. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
#
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"AddHeader",
|
||||||
|
"ModHeader",
|
||||||
|
"DelHeader",
|
||||||
|
"AddDisclaimer",
|
||||||
|
"RewriteLinks"]
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from base64 import b64encode
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from collections import defaultdict
|
||||||
|
from copy import copy
|
||||||
|
from email.message import MIMEPart
|
||||||
|
|
||||||
|
from pymodmilter import replace_illegal_chars
|
||||||
|
|
||||||
|
|
||||||
|
class AddHeader:
|
||||||
|
"""Add a mail header field."""
|
||||||
|
def __init__(self, field, value):
|
||||||
|
self.field = field
|
||||||
|
self.value = value
|
||||||
|
|
||||||
|
def execute(self, milter, pretend=False,
|
||||||
|
logger=logging.getLogger(__name__)):
|
||||||
|
header = f"{self.field}: {self.value}"
|
||||||
|
if logger.getEffectiveLevel() == logging.DEBUG:
|
||||||
|
logger.debug(f"add_header: {header}")
|
||||||
|
else:
|
||||||
|
logger.info(f"add_header: {header[0:70]}")
|
||||||
|
|
||||||
|
milter.msg.add_header(self.field, self.value)
|
||||||
|
|
||||||
|
if not pretend:
|
||||||
|
milter.addheader(self.field, self.value)
|
||||||
|
|
||||||
|
|
||||||
|
class ModHeader:
|
||||||
|
"""Change the value of a mail header field."""
|
||||||
|
def __init__(self, field, value, search=None):
|
||||||
|
self.field = field
|
||||||
|
self.value = value
|
||||||
|
self.search = search
|
||||||
|
|
||||||
|
def execute(self, milter, pretend=False,
|
||||||
|
logger=logging.getLogger(__name__)):
|
||||||
|
idx = defaultdict(int)
|
||||||
|
|
||||||
|
for i, (field, value) in enumerate(milter.msg.items()):
|
||||||
|
field_lower = field.lower()
|
||||||
|
idx[field_lower] += 1
|
||||||
|
|
||||||
|
if not self.field.match(field):
|
||||||
|
continue
|
||||||
|
|
||||||
|
new_value = value
|
||||||
|
if self.search is not None:
|
||||||
|
new_value = self.search.sub(self.value, value).strip()
|
||||||
|
else:
|
||||||
|
new_value = self.value
|
||||||
|
|
||||||
|
if not new_value:
|
||||||
|
logger.warning(
|
||||||
|
"mod_header: resulting value is empty, "
|
||||||
|
"skip modification")
|
||||||
|
continue
|
||||||
|
|
||||||
|
if new_value == value:
|
||||||
|
continue
|
||||||
|
|
||||||
|
header = f"{field}: {value}"
|
||||||
|
new_header = f"{field}: {new_value}"
|
||||||
|
|
||||||
|
if logger.getEffectiveLevel() == logging.DEBUG:
|
||||||
|
logger.debug(f"mod_header: {header}: {new_header}")
|
||||||
|
else:
|
||||||
|
logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}")
|
||||||
|
|
||||||
|
milter.msg.replace_header(
|
||||||
|
field, replace_illegal_chars(new_value), idx=idx[field_lower])
|
||||||
|
|
||||||
|
if not pretend:
|
||||||
|
milter.chgheader(field, new_value, idx=idx[field_lower])
|
||||||
|
|
||||||
|
|
||||||
|
class DelHeader:
|
||||||
|
"""Delete a mail header field."""
|
||||||
|
def __init__(self, field, value=None):
|
||||||
|
self.field = field
|
||||||
|
self.value = value
|
||||||
|
|
||||||
|
def execute(self, milter, pretend=False,
|
||||||
|
logger=logging.getLogger(__name__)):
|
||||||
|
idx = defaultdict(int)
|
||||||
|
|
||||||
|
for field, value in milter.msg.items():
|
||||||
|
field_lower = field.lower()
|
||||||
|
idx[field_lower] += 1
|
||||||
|
|
||||||
|
if not self.field.match(field):
|
||||||
|
continue
|
||||||
|
|
||||||
|
if self.value is not None and not self.value.search(value):
|
||||||
|
continue
|
||||||
|
|
||||||
|
header = f"{field}: {value}"
|
||||||
|
if logger.getEffectiveLevel() == logging.DEBUG:
|
||||||
|
logger.debug(f"del_header: {header}")
|
||||||
|
else:
|
||||||
|
logger.info(f"del_header: {header[0:70]}")
|
||||||
|
milter.msg.remove_header(field, idx=idx[field_lower])
|
||||||
|
|
||||||
|
if not pretend:
|
||||||
|
milter.chgheader(field, "", idx=idx[field_lower])
|
||||||
|
|
||||||
|
idx[field_lower] -= 1
|
||||||
|
|
||||||
|
|
||||||
|
def _get_body_content(msg, pref):
|
||||||
|
part = None
|
||||||
|
content = None
|
||||||
|
if not msg.is_multipart() and msg.get_content_type() == f"text/{pref}":
|
||||||
|
part = msg
|
||||||
|
else:
|
||||||
|
part = msg.get_body(preferencelist=(pref))
|
||||||
|
|
||||||
|
if part is not None:
|
||||||
|
content = part.get_content()
|
||||||
|
|
||||||
|
return (part, content)
|
||||||
|
|
||||||
|
|
||||||
|
def _has_content_before_body_tag(soup):
|
||||||
|
s = copy(soup)
|
||||||
|
for element in s.find_all("head") + s.find_all("body"):
|
||||||
|
element.extract()
|
||||||
|
|
||||||
|
if len(s.text.strip()) > 0:
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _inject_body(milter):
|
||||||
|
if not milter.msg.is_multipart():
|
||||||
|
milter.msg.make_mixed()
|
||||||
|
|
||||||
|
attachments = []
|
||||||
|
for attachment in milter.msg.iter_attachments():
|
||||||
|
if "content-disposition" not in attachment:
|
||||||
|
attachment["Content-Disposition"] = "attachment"
|
||||||
|
attachments.append(attachment)
|
||||||
|
|
||||||
|
milter.msg.clear_content()
|
||||||
|
milter.msg.set_content("")
|
||||||
|
milter.msg.add_alternative("", subtype="html")
|
||||||
|
milter.msg.make_mixed()
|
||||||
|
|
||||||
|
for attachment in attachments:
|
||||||
|
milter.msg.attach(attachment)
|
||||||
|
|
||||||
|
|
||||||
|
def _wrap_message(milter):
|
||||||
|
attachment = MIMEPart()
|
||||||
|
attachment.set_content(milter.msg.as_bytes(),
|
||||||
|
maintype="plain", subtype="text",
|
||||||
|
disposition="attachment",
|
||||||
|
filename=f"{milter.qid}.eml",
|
||||||
|
params={"name": f"{milter.qid}.eml"})
|
||||||
|
|
||||||
|
milter.msg.clear_content()
|
||||||
|
milter.msg.set_content(
|
||||||
|
"Please see the original email attached.")
|
||||||
|
milter.msg.add_alternative(
|
||||||
|
"<html><body>Please see the original email attached.</body></html>",
|
||||||
|
subtype="html")
|
||||||
|
milter.msg.make_mixed()
|
||||||
|
milter.msg.attach(attachment)
|
||||||
|
|
||||||
|
|
||||||
|
class AddDisclaimer:
|
||||||
|
"""Append or prepend a disclaimer to the mail body."""
|
||||||
|
def __init__(self, text_template, html_template, action, error_policy):
|
||||||
|
self.text_template = text_template
|
||||||
|
self.html_template = html_template
|
||||||
|
self.action = action
|
||||||
|
self.error_policy = error_policy
|
||||||
|
|
||||||
|
def patch_message_body(self, milter, logger):
|
||||||
|
text_body, text_content = _get_body_content(milter.msg, "plain")
|
||||||
|
html_body, html_content = _get_body_content(milter.msg, "html")
|
||||||
|
|
||||||
|
if text_content is None and html_content is None:
|
||||||
|
raise RuntimeError("message does not contain any body part")
|
||||||
|
|
||||||
|
if text_content is not None:
|
||||||
|
logger.info(f"{self.action} text disclaimer")
|
||||||
|
|
||||||
|
if self.action == "prepend":
|
||||||
|
content = f"{self.text_template}{text_content}"
|
||||||
|
else:
|
||||||
|
content = f"{text_content}{self.text_template}"
|
||||||
|
|
||||||
|
text_body.set_content(
|
||||||
|
content.encode(), maintype="text", subtype="plain")
|
||||||
|
text_body.set_param("charset", "UTF-8", header="Content-Type")
|
||||||
|
del text_body["MIME-Version"]
|
||||||
|
|
||||||
|
if html_content is not None:
|
||||||
|
logger.info(f"{self.action} html disclaimer")
|
||||||
|
|
||||||
|
soup = BeautifulSoup(html_content, "html.parser")
|
||||||
|
|
||||||
|
body = soup.find('body')
|
||||||
|
if not body:
|
||||||
|
body = soup
|
||||||
|
elif _has_content_before_body_tag(soup):
|
||||||
|
body = soup
|
||||||
|
|
||||||
|
if self.action == "prepend":
|
||||||
|
body.insert(0, copy(self.html_template))
|
||||||
|
else:
|
||||||
|
body.append(self.html_template)
|
||||||
|
|
||||||
|
html_body.set_content(
|
||||||
|
str(soup).encode(), maintype="text", subtype="html")
|
||||||
|
html_body.set_param("charset", "UTF-8", header="Content-Type")
|
||||||
|
del html_body["MIME-Version"]
|
||||||
|
|
||||||
|
def execute(self, milter, pretend=False,
|
||||||
|
logger=logging.getLogger(__name__)):
|
||||||
|
old_headers = milter.msg.items()
|
||||||
|
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
self.patch_message_body(milter, logger)
|
||||||
|
except RuntimeError as e:
|
||||||
|
logger.info(f"{e}, inject empty plain and html body")
|
||||||
|
_inject_body(milter)
|
||||||
|
self.patch_message_body(milter, logger)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(e)
|
||||||
|
if self.error_policy == "ignore":
|
||||||
|
logger.info(
|
||||||
|
"unable to add disclaimer to message body, "
|
||||||
|
"ignore error according to policy")
|
||||||
|
return
|
||||||
|
elif self.error_policy == "reject":
|
||||||
|
logger.info(
|
||||||
|
"unable to add disclaimer to message body, "
|
||||||
|
"reject message according to policy")
|
||||||
|
return [
|
||||||
|
("reject", "Message rejected due to error")]
|
||||||
|
|
||||||
|
logger.info("wrap original message in a new message envelope")
|
||||||
|
try:
|
||||||
|
_wrap_message(milter)
|
||||||
|
self.patch_message_body(milter, logger)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(e)
|
||||||
|
raise Exception(
|
||||||
|
"unable to wrap message in a new message envelope, "
|
||||||
|
"give up ...")
|
||||||
|
|
||||||
|
if not pretend:
|
||||||
|
milter.update_headers(old_headers)
|
||||||
|
milter.replacebody()
|
||||||
|
|
||||||
|
|
||||||
|
class RewriteLinks:
|
||||||
|
"""Rewrite link targets in the mail html body."""
|
||||||
|
def __init__(self, repl):
|
||||||
|
self.repl = repl
|
||||||
|
|
||||||
|
def execute(self, milter, pretend=False,
|
||||||
|
logger=logging.getLogger(__name__)):
|
||||||
|
html_body, html_content = _get_body_content(milter.msg, "html")
|
||||||
|
if html_content is not None:
|
||||||
|
soup = BeautifulSoup(html_content, "html.parser")
|
||||||
|
|
||||||
|
rewritten = 0
|
||||||
|
for link in soup.find_all("a", href=True):
|
||||||
|
if not link["href"]:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if "{URL_B64}" in self.repl:
|
||||||
|
url_b64 = b64encode(link["href"].encode()).decode()
|
||||||
|
target = self.repl.replace("{URL_B64}", url_b64)
|
||||||
|
else:
|
||||||
|
target = self.repl
|
||||||
|
|
||||||
|
link["href"] = target
|
||||||
|
rewritten += 1
|
||||||
|
|
||||||
|
if rewritten:
|
||||||
|
logger.info(f"rewrote {rewritten} link(s) in html body")
|
||||||
|
|
||||||
|
html_body.set_content(
|
||||||
|
str(soup).encode(), maintype="text", subtype="html")
|
||||||
|
html_body.set_param("charset", "UTF-8", header="Content-Type")
|
||||||
|
del html_body["MIME-Version"]
|
||||||
|
|
||||||
|
if not pretend:
|
||||||
|
milter.replacebody()
|
||||||
226
pymodmilter/storages.py
Normal file
226
pymodmilter/storages.py
Normal file
@@ -0,0 +1,226 @@
|
|||||||
|
# PyMod-Milter is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# PyMod-Milter is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with PyMod-Milter. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
#
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
|
||||||
|
from calendar import timegm
|
||||||
|
from datetime import datetime
|
||||||
|
from glob import glob
|
||||||
|
from time import gmtime
|
||||||
|
|
||||||
|
|
||||||
|
class BaseMailStorage(object):
|
||||||
|
"Mail storage base class"
|
||||||
|
def __init__(self):
|
||||||
|
return
|
||||||
|
|
||||||
|
def add(self, data, qid, mailfrom="", recipients=[]):
|
||||||
|
"Add email to storage."
|
||||||
|
return ("", "")
|
||||||
|
|
||||||
|
def find(self, mailfrom=None, recipients=None, older_than=None):
|
||||||
|
"Find emails in storage."
|
||||||
|
return
|
||||||
|
|
||||||
|
def get_metadata(self, storage_id):
|
||||||
|
"Return metadata of email in storage."
|
||||||
|
return
|
||||||
|
|
||||||
|
def delete(self, storage_id, recipients=None):
|
||||||
|
"Delete email from storage."
|
||||||
|
return
|
||||||
|
|
||||||
|
def get_mail(self, storage_id):
|
||||||
|
"Return email and metadata."
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
class FileMailStorage(BaseMailStorage):
|
||||||
|
"Storage class to store mails on filesystem."
|
||||||
|
def __init__(self, directory, original=False):
|
||||||
|
super().__init__()
|
||||||
|
self.directory = directory
|
||||||
|
self.original = original
|
||||||
|
self._metadata_suffix = ".metadata"
|
||||||
|
|
||||||
|
def _save_datafile(self, storage_id, data):
|
||||||
|
datafile = os.path.join(self.directory, storage_id)
|
||||||
|
try:
|
||||||
|
with open(datafile, "wb") as f:
|
||||||
|
f.write(data)
|
||||||
|
except IOError as e:
|
||||||
|
raise RuntimeError(f"unable save data file: {e}")
|
||||||
|
|
||||||
|
return datafile
|
||||||
|
|
||||||
|
def _save_metafile(self, storage_id, metadata):
|
||||||
|
metafile = os.path.join(
|
||||||
|
self.directory, f"{storage_id}{self._metadata_suffix}")
|
||||||
|
try:
|
||||||
|
with open(metafile, "w") as f:
|
||||||
|
json.dump(metadata, f, indent=2)
|
||||||
|
except IOError as e:
|
||||||
|
raise RuntimeError(f"unable to save metadata file: {e}")
|
||||||
|
|
||||||
|
def _remove(self, storage_id):
|
||||||
|
datafile = os.path.join(self.directory, storage_id)
|
||||||
|
metafile = f"{datafile}{self._metadata_suffix}"
|
||||||
|
|
||||||
|
try:
|
||||||
|
os.remove(metafile)
|
||||||
|
except IOError as e:
|
||||||
|
raise RuntimeError(f"unable to remove metadata file: {e}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
os.remove(datafile)
|
||||||
|
except IOError as e:
|
||||||
|
raise RuntimeError(f"unable to remove data file: {e}")
|
||||||
|
|
||||||
|
def add(self, data, qid, mailfrom="", recipients=[], subject=""):
|
||||||
|
"Add email to file storage and return storage id."
|
||||||
|
super().add(data, qid, mailfrom, recipients)
|
||||||
|
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||||
|
storage_id = f"{timestamp}_{qid}"
|
||||||
|
|
||||||
|
# save mail
|
||||||
|
datafile = self._save_datafile(storage_id, data)
|
||||||
|
|
||||||
|
# save metadata
|
||||||
|
metadata = {
|
||||||
|
"mailfrom": mailfrom,
|
||||||
|
"recipients": recipients,
|
||||||
|
"subject": subject,
|
||||||
|
"timestamp": timegm(gmtime()),
|
||||||
|
"queue_id": qid}
|
||||||
|
|
||||||
|
try:
|
||||||
|
self._save_metafile(storage_id, metadata)
|
||||||
|
except RuntimeError as e:
|
||||||
|
os.remove(datafile)
|
||||||
|
raise e
|
||||||
|
|
||||||
|
return (storage_id, datafile)
|
||||||
|
|
||||||
|
def execute(self, milter, pretend=False,
|
||||||
|
logger=logging.getLogger(__name__)):
|
||||||
|
if self.original:
|
||||||
|
milter.fp.seek(0)
|
||||||
|
data = milter.fp.read
|
||||||
|
mailfrom = milter.mailfrom
|
||||||
|
recipients = list(milter.rcpts)
|
||||||
|
subject = ""
|
||||||
|
else:
|
||||||
|
data = milter.msg.as_bytes
|
||||||
|
mailfrom = milter.msginfo["mailfrom"]
|
||||||
|
recipients = list(milter.msginfo["rcpts"])
|
||||||
|
subject = milter.msg["subject"] or ""
|
||||||
|
|
||||||
|
storage_id, datafile = self.add(
|
||||||
|
data(), milter.qid, mailfrom, recipients, subject)
|
||||||
|
logger.info(f"stored message in file {datafile}")
|
||||||
|
|
||||||
|
def get_metadata(self, storage_id):
|
||||||
|
"Return metadata of email in storage."
|
||||||
|
super(FileMailStorage, self).get_metadata(storage_id)
|
||||||
|
|
||||||
|
metafile = os.path.join(
|
||||||
|
self.directory, f"{storage_id}{self._metadata_suffix}")
|
||||||
|
if not os.path.isfile(metafile):
|
||||||
|
raise RuntimeError(
|
||||||
|
f"invalid storage id '{storage_id}'")
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(metafile, "r") as f:
|
||||||
|
metadata = json.load(f)
|
||||||
|
except IOError as e:
|
||||||
|
raise RuntimeError(f"unable to read metadata file: {e}")
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"invalid metafile '{metafile}': {e}")
|
||||||
|
|
||||||
|
return metadata
|
||||||
|
|
||||||
|
def find(self, mailfrom=None, recipients=None, older_than=None):
|
||||||
|
"Find emails in storage."
|
||||||
|
super(FileMailStorage, self).find(mailfrom, recipients, older_than)
|
||||||
|
if isinstance(mailfrom, str):
|
||||||
|
mailfrom = [mailfrom]
|
||||||
|
if isinstance(recipients, str):
|
||||||
|
recipients = [recipients]
|
||||||
|
|
||||||
|
emails = {}
|
||||||
|
metafiles = glob(os.path.join(
|
||||||
|
self.directory, f"*{self._metadata_suffix}"))
|
||||||
|
for metafile in metafiles:
|
||||||
|
if not os.path.isfile(metafile):
|
||||||
|
continue
|
||||||
|
|
||||||
|
storage_id = os.path.basename(
|
||||||
|
metafile[:-len(self._metadata_suffix)])
|
||||||
|
metadata = self.get_metadata(storage_id)
|
||||||
|
if older_than is not None:
|
||||||
|
if timegm(gmtime()) - metadata["date"] < (older_than * 86400):
|
||||||
|
continue
|
||||||
|
|
||||||
|
if mailfrom is not None:
|
||||||
|
if metadata["mailfrom"] not in mailfrom:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if recipients is not None:
|
||||||
|
if len(recipients) == 1 and \
|
||||||
|
recipients[0] not in metadata["recipients"]:
|
||||||
|
continue
|
||||||
|
elif len(set(recipients + metadata["recipients"])) == \
|
||||||
|
len(recipients + metadata["recipients"]):
|
||||||
|
continue
|
||||||
|
|
||||||
|
emails[storage_id] = metadata
|
||||||
|
|
||||||
|
return emails
|
||||||
|
|
||||||
|
def delete(self, storage_id, recipients=None):
|
||||||
|
"Delete email from storage."
|
||||||
|
super(FileMailStorage, self).delete(storage_id, recipients)
|
||||||
|
|
||||||
|
try:
|
||||||
|
metadata = self.get_metadata(storage_id)
|
||||||
|
except RuntimeError as e:
|
||||||
|
raise RuntimeError(f"unable to delete email: {e}")
|
||||||
|
|
||||||
|
if not recipients:
|
||||||
|
self._remove(storage_id)
|
||||||
|
else:
|
||||||
|
if type(recipients) == str:
|
||||||
|
recipients = [recipients]
|
||||||
|
for recipient in recipients:
|
||||||
|
if recipient not in metadata["recipients"]:
|
||||||
|
raise RuntimeError(f"invalid recipient '{recipient}'")
|
||||||
|
metadata["recipients"].remove(recipient)
|
||||||
|
if not metadata["recipients"]:
|
||||||
|
self._remove(storage_id)
|
||||||
|
else:
|
||||||
|
self._save_metafile(storage_id, metadata)
|
||||||
|
|
||||||
|
def get_mail(self, storage_id):
|
||||||
|
super(FileMailStorage, self).get_mail(storage_id)
|
||||||
|
|
||||||
|
metadata = self.get_metadata(storage_id)
|
||||||
|
datafile = os.path.join(self.directory, storage_id)
|
||||||
|
try:
|
||||||
|
fp = open(datafile, "rb")
|
||||||
|
except IOError as e:
|
||||||
|
raise RuntimeError(f"unable to open email data file: {e}")
|
||||||
|
return (fp, metadata)
|
||||||
Reference in New Issue
Block a user