add new file storage/quarantine and change version to 1.2.0
This commit is contained in:
@@ -13,328 +13,17 @@
|
||||
#
|
||||
|
||||
__all__ = [
|
||||
"add_header",
|
||||
"mod_header",
|
||||
"del_header",
|
||||
"add_disclaimer",
|
||||
"rewrite_links",
|
||||
"store",
|
||||
"ActionConfig",
|
||||
"Action"]
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
||||
from base64 import b64encode
|
||||
from bs4 import BeautifulSoup
|
||||
from collections import defaultdict
|
||||
from copy import copy
|
||||
from datetime import datetime
|
||||
from email.message import MIMEPart
|
||||
|
||||
from pymodmilter import CustomLogger, BaseConfig
|
||||
from pymodmilter.conditions import ConditionsConfig, Conditions
|
||||
from pymodmilter import replace_illegal_chars
|
||||
|
||||
|
||||
def add_header(milter, field, value, pretend=False,
|
||||
logger=logging.getLogger(__name__)):
|
||||
"""Add a mail header field."""
|
||||
header = f"{field}: {value}"
|
||||
if logger.getEffectiveLevel() == logging.DEBUG:
|
||||
logger.debug(f"add_header: {header}")
|
||||
else:
|
||||
logger.info(f"add_header: {header[0:70]}")
|
||||
|
||||
milter.msg.add_header(field, replace_illegal_chars(value))
|
||||
|
||||
if not pretend:
|
||||
milter.addheader(field, value)
|
||||
|
||||
|
||||
def mod_header(milter, field, value, search=None, pretend=False,
|
||||
logger=logging.getLogger(__name__)):
|
||||
"""Change the value of a mail header field."""
|
||||
if isinstance(field, str):
|
||||
field = re.compile(field, re.IGNORECASE)
|
||||
|
||||
if isinstance(search, str):
|
||||
search = re.compile(search, re.MULTILINE + re.DOTALL + re.IGNORECASE)
|
||||
|
||||
idx = defaultdict(int)
|
||||
|
||||
for i, (f, v) in enumerate(milter.msg.items()):
|
||||
f_lower = f.lower()
|
||||
idx[f_lower] += 1
|
||||
|
||||
if not field.match(f):
|
||||
continue
|
||||
|
||||
new_value = v
|
||||
if search is not None:
|
||||
new_value = search.sub(value, v).strip()
|
||||
else:
|
||||
new_value = value
|
||||
|
||||
if not new_value:
|
||||
logger.warning(
|
||||
"mod_header: resulting value is empty, "
|
||||
"skip modification")
|
||||
continue
|
||||
|
||||
if new_value == v:
|
||||
continue
|
||||
|
||||
header = f"{f}: {v}"
|
||||
new_header = f"{f}: {new_value}"
|
||||
|
||||
if logger.getEffectiveLevel() == logging.DEBUG:
|
||||
logger.debug(f"mod_header: {header}: {new_header}")
|
||||
else:
|
||||
logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}")
|
||||
|
||||
milter.msg.replace_header(
|
||||
f, replace_illegal_chars(new_value), idx=idx[f_lower])
|
||||
|
||||
if not pretend:
|
||||
milter.chgheader(f, new_value, idx=idx[f_lower])
|
||||
|
||||
|
||||
def del_header(milter, field, value=None, pretend=False,
|
||||
logger=logging.getLogger(__name__)):
|
||||
"""Delete a mail header field."""
|
||||
if isinstance(field, str):
|
||||
field = re.compile(field, re.IGNORECASE)
|
||||
|
||||
if isinstance(value, str):
|
||||
value = re.compile(value, re.MULTILINE + re.DOTALL + re.IGNORECASE)
|
||||
|
||||
idx = defaultdict(int)
|
||||
|
||||
for f, v in milter.msg.items():
|
||||
f_lower = f.lower()
|
||||
idx[f_lower] += 1
|
||||
|
||||
if not field.match(f):
|
||||
continue
|
||||
|
||||
if value is not None and not value.search(v):
|
||||
continue
|
||||
|
||||
header = f"{f}: {v}"
|
||||
if logger.getEffectiveLevel() == logging.DEBUG:
|
||||
logger.debug(f"del_header: {header}")
|
||||
else:
|
||||
logger.info(f"del_header: {header[0:70]}")
|
||||
milter.msg.remove_header(f, idx=idx[f_lower])
|
||||
|
||||
if not pretend:
|
||||
milter.chgheader(f, "", idx=idx[f_lower])
|
||||
|
||||
idx[f_lower] -= 1
|
||||
|
||||
|
||||
def _get_body_content(msg, pref):
|
||||
part = None
|
||||
content = None
|
||||
if not msg.is_multipart() and msg.get_content_type() == f"text/{pref}":
|
||||
part = msg
|
||||
else:
|
||||
part = msg.get_body(preferencelist=(pref))
|
||||
|
||||
if part is not None:
|
||||
content = part.get_content()
|
||||
|
||||
return (part, content)
|
||||
|
||||
|
||||
def _has_content_before_body_tag(soup):
|
||||
s = copy(soup)
|
||||
for element in s.find_all("head") + s.find_all("body"):
|
||||
element.extract()
|
||||
|
||||
if len(s.text.strip()) > 0:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _patch_message_body(milter, action, text_template, html_template, logger):
|
||||
text_body, text_content = _get_body_content(milter.msg, "plain")
|
||||
html_body, html_content = _get_body_content(milter.msg, "html")
|
||||
|
||||
if text_content is None and html_content is None:
|
||||
raise RuntimeError("message does not contain any body part")
|
||||
|
||||
if text_content is not None:
|
||||
logger.info(f"{action} text disclaimer")
|
||||
|
||||
if action == "prepend":
|
||||
content = f"{text_template}{text_content}"
|
||||
else:
|
||||
content = f"{text_content}{text_template}"
|
||||
|
||||
text_body.set_content(
|
||||
content.encode(), maintype="text", subtype="plain")
|
||||
text_body.set_param("charset", "UTF-8", header="Content-Type")
|
||||
del text_body["MIME-Version"]
|
||||
|
||||
if html_content is not None:
|
||||
logger.info(f"{action} html disclaimer")
|
||||
|
||||
soup = BeautifulSoup(html_content, "html.parser")
|
||||
|
||||
body = soup.find('body')
|
||||
if not body:
|
||||
body = soup
|
||||
elif _has_content_before_body_tag(soup):
|
||||
body = soup
|
||||
|
||||
if action == "prepend":
|
||||
body.insert(0, copy(html_template))
|
||||
else:
|
||||
body.append(html_template)
|
||||
|
||||
html_body.set_content(
|
||||
str(soup).encode(), maintype="text", subtype="html")
|
||||
html_body.set_param("charset", "UTF-8", header="Content-Type")
|
||||
del html_body["MIME-Version"]
|
||||
|
||||
|
||||
def _wrap_message(milter, logger):
|
||||
attachment = MIMEPart()
|
||||
attachment.set_content(milter.msg.as_bytes(),
|
||||
maintype="plain", subtype="text",
|
||||
disposition="attachment",
|
||||
filename=f"{milter.qid}.eml",
|
||||
params={"name": f"{milter.qid}.eml"})
|
||||
|
||||
milter.msg.clear_content()
|
||||
milter.msg.set_content(
|
||||
"Please see the original email attached.")
|
||||
milter.msg.add_alternative(
|
||||
"<html><body>Please see the original email attached.</body></html>",
|
||||
subtype="html")
|
||||
milter.msg.make_mixed()
|
||||
milter.msg.attach(attachment)
|
||||
|
||||
|
||||
def _inject_body(milter):
|
||||
if not milter.msg.is_multipart():
|
||||
milter.msg.make_mixed()
|
||||
|
||||
attachments = []
|
||||
for attachment in milter.msg.iter_attachments():
|
||||
if "content-disposition" not in attachment:
|
||||
attachment["Content-Disposition"] = "attachment"
|
||||
attachments.append(attachment)
|
||||
|
||||
milter.msg.clear_content()
|
||||
milter.msg.set_content("")
|
||||
milter.msg.add_alternative("", subtype="html")
|
||||
milter.msg.make_mixed()
|
||||
|
||||
for attachment in attachments:
|
||||
milter.msg.attach(attachment)
|
||||
|
||||
|
||||
def add_disclaimer(milter, text_template, html_template, action, error_policy,
|
||||
pretend=False, logger=logging.getLogger(__name__)):
|
||||
"""Append or prepend a disclaimer to the mail body."""
|
||||
old_headers = milter.msg.items()
|
||||
|
||||
try:
|
||||
try:
|
||||
_patch_message_body(
|
||||
milter, action, text_template, html_template, logger)
|
||||
except RuntimeError as e:
|
||||
logger.info(f"{e}, inject empty plain and html body")
|
||||
_inject_body(milter)
|
||||
_patch_message_body(
|
||||
milter, action, text_template, html_template, logger)
|
||||
except Exception as e:
|
||||
logger.warning(e)
|
||||
if error_policy == "ignore":
|
||||
logger.info(
|
||||
"unable to add disclaimer to message body, "
|
||||
"ignore error according to policy")
|
||||
return
|
||||
elif error_policy == "reject":
|
||||
logger.info(
|
||||
"unable to add disclaimer to message body, "
|
||||
"reject message according to policy")
|
||||
return [
|
||||
("reject", "Message rejected due to error")]
|
||||
|
||||
logger.info("wrap original message in a new message envelope")
|
||||
try:
|
||||
_wrap_message(milter, logger)
|
||||
_patch_message_body(
|
||||
milter, action, text_template, html_template, logger)
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
raise Exception(
|
||||
"unable to wrap message in a new message envelope, "
|
||||
"give up ...")
|
||||
|
||||
if not pretend:
|
||||
milter.update_headers(old_headers)
|
||||
milter.replacebody()
|
||||
|
||||
|
||||
def rewrite_links(milter, repl, pretend=False,
|
||||
logger=logging.getLogger(__name__)):
|
||||
"""Rewrite link targets in the mail html body."""
|
||||
|
||||
html_body, html_content = _get_body_content(milter.msg, "html")
|
||||
if html_content is not None:
|
||||
soup = BeautifulSoup(html_content, "html.parser")
|
||||
|
||||
rewritten = 0
|
||||
for link in soup.find_all("a", href=True):
|
||||
if not link["href"]:
|
||||
continue
|
||||
|
||||
if "{URL_B64}" in repl:
|
||||
url_b64 = b64encode(link["href"].encode()).decode()
|
||||
target = repl.replace("{URL_B64}", url_b64)
|
||||
else:
|
||||
target = repl
|
||||
|
||||
link["href"] = target
|
||||
rewritten += 1
|
||||
|
||||
if rewritten:
|
||||
logger.info(f"rewrote {rewritten} link(s) in html body")
|
||||
|
||||
html_body.set_content(
|
||||
str(soup).encode(), maintype="text", subtype="html")
|
||||
html_body.set_param("charset", "UTF-8", header="Content-Type")
|
||||
del html_body["MIME-Version"]
|
||||
|
||||
if not pretend:
|
||||
milter.replacebody()
|
||||
|
||||
|
||||
def store(milter, directory, original=False, pretend=False,
|
||||
logger=logging.getLogger(__name__)):
|
||||
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
store_id = f"{timestamp}_{milter.qid}"
|
||||
datafile = os.path.join(directory, store_id)
|
||||
|
||||
logger.info(f"store message in file {datafile}")
|
||||
|
||||
if not pretend:
|
||||
try:
|
||||
with open(datafile, "wb") as fp:
|
||||
if original:
|
||||
milter.fp.seek(0)
|
||||
fp.write(milter.fp.read())
|
||||
else:
|
||||
fp.write(milter.msg.as_bytes())
|
||||
except IOError as e:
|
||||
raise RuntimeError(f"unable to store message: {e}")
|
||||
from pymodmilter import modifications, storages
|
||||
|
||||
|
||||
class ActionConfig(BaseConfig):
|
||||
@@ -369,7 +58,7 @@ class ActionConfig(BaseConfig):
|
||||
self["type"] = cfg["type"]
|
||||
|
||||
if self["type"] == "add_header":
|
||||
self["func"] = add_header
|
||||
self["class"] = modifications.AddHeader
|
||||
self["headersonly"] = True
|
||||
|
||||
if "field" not in cfg and "header" in cfg:
|
||||
@@ -378,7 +67,7 @@ class ActionConfig(BaseConfig):
|
||||
self.add_string_arg(cfg, ("field", "value"))
|
||||
|
||||
elif self["type"] == "mod_header":
|
||||
self["func"] = mod_header
|
||||
self["class"] = modifications.ModHeader
|
||||
self["headersonly"] = True
|
||||
|
||||
if "field" not in cfg and "header" in cfg:
|
||||
@@ -399,7 +88,7 @@ class ActionConfig(BaseConfig):
|
||||
raise ValueError(f"{self['name']}: {arg}: {e}")
|
||||
|
||||
elif self["type"] == "del_header":
|
||||
self["func"] = del_header
|
||||
self["class"] = modifications.DelHeader
|
||||
self["headersonly"] = True
|
||||
|
||||
if "field" not in cfg and "header" in cfg:
|
||||
@@ -419,7 +108,7 @@ class ActionConfig(BaseConfig):
|
||||
raise ValueError(f"{self['name']}: {arg}: {e}")
|
||||
|
||||
elif self["type"] == "add_disclaimer":
|
||||
self["func"] = add_disclaimer
|
||||
self["class"] = modifications.AddDisclaimer
|
||||
self["headersonly"] = False
|
||||
|
||||
if "html_template" not in cfg and "html_file" in cfg:
|
||||
@@ -460,12 +149,11 @@ class ActionConfig(BaseConfig):
|
||||
f"{self['name']}: unable to open/read template file: {e}")
|
||||
|
||||
elif self["type"] == "rewrite_links":
|
||||
self["func"] = rewrite_links
|
||||
self["class"] = modifications.RewriteLinks
|
||||
self["headersonly"] = False
|
||||
self.add_string_arg(cfg, "repl")
|
||||
|
||||
elif self["type"] == "store":
|
||||
self["func"] = store
|
||||
self["headersonly"] = False
|
||||
|
||||
assert "storage_type" in cfg, \
|
||||
@@ -479,7 +167,15 @@ class ActionConfig(BaseConfig):
|
||||
self.add_bool_arg(cfg, "original")
|
||||
|
||||
if self["storage_type"] == "file":
|
||||
self["class"] = storages.FileMailStorage
|
||||
self.add_string_arg(cfg, "directory")
|
||||
# check if directory exists and is writable
|
||||
if not os.path.isdir(self["args"]["directory"]) or \
|
||||
not os.access(self["args"]["directory"], os.W_OK):
|
||||
raise RuntimeError(
|
||||
f"{self['name']}: file quarantine directory "
|
||||
f"'{self['directory']}' does not exist or is "
|
||||
f"not writable")
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"{self['name']}: storage_type: invalid storage type")
|
||||
@@ -512,8 +208,7 @@ class Action:
|
||||
|
||||
self.pretend = cfg["pretend"]
|
||||
self._name = cfg["name"]
|
||||
self._func = cfg["func"]
|
||||
self._args = cfg["args"]
|
||||
self._class = cfg["class"](**cfg["args"])
|
||||
self._headersonly = cfg["headersonly"]
|
||||
|
||||
def headersonly(self):
|
||||
@@ -529,5 +224,5 @@ class Action:
|
||||
self.conditions.match(envfrom=milter.mailfrom,
|
||||
envto=[*milter.rcpts],
|
||||
headers=milter.msg.items()):
|
||||
return self._func(milter=milter, pretend=self.pretend,
|
||||
logger=logger, **self._args)
|
||||
return self._class.execute(
|
||||
milter=milter, pretend=self.pretend, logger=logger)
|
||||
|
||||
Reference in New Issue
Block a user