diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py
index 5223a7a..f54d680 100644
--- a/pymodmilter/__init__.py
+++ b/pymodmilter/__init__.py
@@ -16,12 +16,13 @@ __all__ = [
"actions",
"base",
"conditions",
+ "modifications",
"rules",
"run",
"ModifyMilterConfig",
"ModifyMilter"]
-__version__ = "1.1.7"
+__version__ = "1.2.0"
from pymodmilter import _runtime_patches
@@ -327,18 +328,15 @@ class ModifyMilter(Milter.Base):
def eom(self):
try:
- # setup msg and msg_info to be read/modified by rules and actions
+ # msg and msginfo contain the runtime data that
+ # is read/modified by actions
self.fp.seek(0)
self.msg = message_from_binary_file(
self.fp, _class=MilterMessage, policy=SMTPUTF8.clone(
refold_source='none'))
- self.msg_info = defaultdict(str)
- self.msg_info["ip"] = self.IP
- self.msg_info["port"] = self.port
- self.msg_info["heloname"] = self.heloname
- self.msg_info["envfrom"] = self.mailfrom
- self.msg_info["rcpts"] = self.rcpts
- self.msg_info["qid"] = self.qid
+ self.msginfo = {
+ "mailfrom": self.mailfrom,
+ "rcpts": self.rcpts}
self._replacebody = False
milter_action = None
diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py
index 09733ef..ebe7d78 100644
--- a/pymodmilter/actions.py
+++ b/pymodmilter/actions.py
@@ -13,328 +13,17 @@
#
__all__ = [
- "add_header",
- "mod_header",
- "del_header",
- "add_disclaimer",
- "rewrite_links",
- "store",
"ActionConfig",
"Action"]
-import logging
import os
import re
-from base64 import b64encode
from bs4 import BeautifulSoup
-from collections import defaultdict
-from copy import copy
-from datetime import datetime
-from email.message import MIMEPart
from pymodmilter import CustomLogger, BaseConfig
from pymodmilter.conditions import ConditionsConfig, Conditions
-from pymodmilter import replace_illegal_chars
-
-
-def add_header(milter, field, value, pretend=False,
- logger=logging.getLogger(__name__)):
- """Add a mail header field."""
- header = f"{field}: {value}"
- if logger.getEffectiveLevel() == logging.DEBUG:
- logger.debug(f"add_header: {header}")
- else:
- logger.info(f"add_header: {header[0:70]}")
-
- milter.msg.add_header(field, replace_illegal_chars(value))
-
- if not pretend:
- milter.addheader(field, value)
-
-
-def mod_header(milter, field, value, search=None, pretend=False,
- logger=logging.getLogger(__name__)):
- """Change the value of a mail header field."""
- if isinstance(field, str):
- field = re.compile(field, re.IGNORECASE)
-
- if isinstance(search, str):
- search = re.compile(search, re.MULTILINE + re.DOTALL + re.IGNORECASE)
-
- idx = defaultdict(int)
-
- for i, (f, v) in enumerate(milter.msg.items()):
- f_lower = f.lower()
- idx[f_lower] += 1
-
- if not field.match(f):
- continue
-
- new_value = v
- if search is not None:
- new_value = search.sub(value, v).strip()
- else:
- new_value = value
-
- if not new_value:
- logger.warning(
- "mod_header: resulting value is empty, "
- "skip modification")
- continue
-
- if new_value == v:
- continue
-
- header = f"{f}: {v}"
- new_header = f"{f}: {new_value}"
-
- if logger.getEffectiveLevel() == logging.DEBUG:
- logger.debug(f"mod_header: {header}: {new_header}")
- else:
- logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}")
-
- milter.msg.replace_header(
- f, replace_illegal_chars(new_value), idx=idx[f_lower])
-
- if not pretend:
- milter.chgheader(f, new_value, idx=idx[f_lower])
-
-
-def del_header(milter, field, value=None, pretend=False,
- logger=logging.getLogger(__name__)):
- """Delete a mail header field."""
- if isinstance(field, str):
- field = re.compile(field, re.IGNORECASE)
-
- if isinstance(value, str):
- value = re.compile(value, re.MULTILINE + re.DOTALL + re.IGNORECASE)
-
- idx = defaultdict(int)
-
- for f, v in milter.msg.items():
- f_lower = f.lower()
- idx[f_lower] += 1
-
- if not field.match(f):
- continue
-
- if value is not None and not value.search(v):
- continue
-
- header = f"{f}: {v}"
- if logger.getEffectiveLevel() == logging.DEBUG:
- logger.debug(f"del_header: {header}")
- else:
- logger.info(f"del_header: {header[0:70]}")
- milter.msg.remove_header(f, idx=idx[f_lower])
-
- if not pretend:
- milter.chgheader(f, "", idx=idx[f_lower])
-
- idx[f_lower] -= 1
-
-
-def _get_body_content(msg, pref):
- part = None
- content = None
- if not msg.is_multipart() and msg.get_content_type() == f"text/{pref}":
- part = msg
- else:
- part = msg.get_body(preferencelist=(pref))
-
- if part is not None:
- content = part.get_content()
-
- return (part, content)
-
-
-def _has_content_before_body_tag(soup):
- s = copy(soup)
- for element in s.find_all("head") + s.find_all("body"):
- element.extract()
-
- if len(s.text.strip()) > 0:
- return True
-
- return False
-
-
-def _patch_message_body(milter, action, text_template, html_template, logger):
- text_body, text_content = _get_body_content(milter.msg, "plain")
- html_body, html_content = _get_body_content(milter.msg, "html")
-
- if text_content is None and html_content is None:
- raise RuntimeError("message does not contain any body part")
-
- if text_content is not None:
- logger.info(f"{action} text disclaimer")
-
- if action == "prepend":
- content = f"{text_template}{text_content}"
- else:
- content = f"{text_content}{text_template}"
-
- text_body.set_content(
- content.encode(), maintype="text", subtype="plain")
- text_body.set_param("charset", "UTF-8", header="Content-Type")
- del text_body["MIME-Version"]
-
- if html_content is not None:
- logger.info(f"{action} html disclaimer")
-
- soup = BeautifulSoup(html_content, "html.parser")
-
- body = soup.find('body')
- if not body:
- body = soup
- elif _has_content_before_body_tag(soup):
- body = soup
-
- if action == "prepend":
- body.insert(0, copy(html_template))
- else:
- body.append(html_template)
-
- html_body.set_content(
- str(soup).encode(), maintype="text", subtype="html")
- html_body.set_param("charset", "UTF-8", header="Content-Type")
- del html_body["MIME-Version"]
-
-
-def _wrap_message(milter, logger):
- attachment = MIMEPart()
- attachment.set_content(milter.msg.as_bytes(),
- maintype="plain", subtype="text",
- disposition="attachment",
- filename=f"{milter.qid}.eml",
- params={"name": f"{milter.qid}.eml"})
-
- milter.msg.clear_content()
- milter.msg.set_content(
- "Please see the original email attached.")
- milter.msg.add_alternative(
- "Please see the original email attached.",
- subtype="html")
- milter.msg.make_mixed()
- milter.msg.attach(attachment)
-
-
-def _inject_body(milter):
- if not milter.msg.is_multipart():
- milter.msg.make_mixed()
-
- attachments = []
- for attachment in milter.msg.iter_attachments():
- if "content-disposition" not in attachment:
- attachment["Content-Disposition"] = "attachment"
- attachments.append(attachment)
-
- milter.msg.clear_content()
- milter.msg.set_content("")
- milter.msg.add_alternative("", subtype="html")
- milter.msg.make_mixed()
-
- for attachment in attachments:
- milter.msg.attach(attachment)
-
-
-def add_disclaimer(milter, text_template, html_template, action, error_policy,
- pretend=False, logger=logging.getLogger(__name__)):
- """Append or prepend a disclaimer to the mail body."""
- old_headers = milter.msg.items()
-
- try:
- try:
- _patch_message_body(
- milter, action, text_template, html_template, logger)
- except RuntimeError as e:
- logger.info(f"{e}, inject empty plain and html body")
- _inject_body(milter)
- _patch_message_body(
- milter, action, text_template, html_template, logger)
- except Exception as e:
- logger.warning(e)
- if error_policy == "ignore":
- logger.info(
- "unable to add disclaimer to message body, "
- "ignore error according to policy")
- return
- elif error_policy == "reject":
- logger.info(
- "unable to add disclaimer to message body, "
- "reject message according to policy")
- return [
- ("reject", "Message rejected due to error")]
-
- logger.info("wrap original message in a new message envelope")
- try:
- _wrap_message(milter, logger)
- _patch_message_body(
- milter, action, text_template, html_template, logger)
- except Exception as e:
- logger.error(e)
- raise Exception(
- "unable to wrap message in a new message envelope, "
- "give up ...")
-
- if not pretend:
- milter.update_headers(old_headers)
- milter.replacebody()
-
-
-def rewrite_links(milter, repl, pretend=False,
- logger=logging.getLogger(__name__)):
- """Rewrite link targets in the mail html body."""
-
- html_body, html_content = _get_body_content(milter.msg, "html")
- if html_content is not None:
- soup = BeautifulSoup(html_content, "html.parser")
-
- rewritten = 0
- for link in soup.find_all("a", href=True):
- if not link["href"]:
- continue
-
- if "{URL_B64}" in repl:
- url_b64 = b64encode(link["href"].encode()).decode()
- target = repl.replace("{URL_B64}", url_b64)
- else:
- target = repl
-
- link["href"] = target
- rewritten += 1
-
- if rewritten:
- logger.info(f"rewrote {rewritten} link(s) in html body")
-
- html_body.set_content(
- str(soup).encode(), maintype="text", subtype="html")
- html_body.set_param("charset", "UTF-8", header="Content-Type")
- del html_body["MIME-Version"]
-
- if not pretend:
- milter.replacebody()
-
-
-def store(milter, directory, original=False, pretend=False,
- logger=logging.getLogger(__name__)):
- timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
- store_id = f"{timestamp}_{milter.qid}"
- datafile = os.path.join(directory, store_id)
-
- logger.info(f"store message in file {datafile}")
-
- if not pretend:
- try:
- with open(datafile, "wb") as fp:
- if original:
- milter.fp.seek(0)
- fp.write(milter.fp.read())
- else:
- fp.write(milter.msg.as_bytes())
- except IOError as e:
- raise RuntimeError(f"unable to store message: {e}")
+from pymodmilter import modifications, storages
class ActionConfig(BaseConfig):
@@ -369,7 +58,7 @@ class ActionConfig(BaseConfig):
self["type"] = cfg["type"]
if self["type"] == "add_header":
- self["func"] = add_header
+ self["class"] = modifications.AddHeader
self["headersonly"] = True
if "field" not in cfg and "header" in cfg:
@@ -378,7 +67,7 @@ class ActionConfig(BaseConfig):
self.add_string_arg(cfg, ("field", "value"))
elif self["type"] == "mod_header":
- self["func"] = mod_header
+ self["class"] = modifications.ModHeader
self["headersonly"] = True
if "field" not in cfg and "header" in cfg:
@@ -399,7 +88,7 @@ class ActionConfig(BaseConfig):
raise ValueError(f"{self['name']}: {arg}: {e}")
elif self["type"] == "del_header":
- self["func"] = del_header
+ self["class"] = modifications.DelHeader
self["headersonly"] = True
if "field" not in cfg and "header" in cfg:
@@ -419,7 +108,7 @@ class ActionConfig(BaseConfig):
raise ValueError(f"{self['name']}: {arg}: {e}")
elif self["type"] == "add_disclaimer":
- self["func"] = add_disclaimer
+ self["class"] = modifications.AddDisclaimer
self["headersonly"] = False
if "html_template" not in cfg and "html_file" in cfg:
@@ -460,12 +149,11 @@ class ActionConfig(BaseConfig):
f"{self['name']}: unable to open/read template file: {e}")
elif self["type"] == "rewrite_links":
- self["func"] = rewrite_links
+ self["class"] = modifications.RewriteLinks
self["headersonly"] = False
self.add_string_arg(cfg, "repl")
elif self["type"] == "store":
- self["func"] = store
self["headersonly"] = False
assert "storage_type" in cfg, \
@@ -479,7 +167,15 @@ class ActionConfig(BaseConfig):
self.add_bool_arg(cfg, "original")
if self["storage_type"] == "file":
+ self["class"] = storages.FileMailStorage
self.add_string_arg(cfg, "directory")
+ # check if directory exists and is writable
+ if not os.path.isdir(self["args"]["directory"]) or \
+ not os.access(self["args"]["directory"], os.W_OK):
+ raise RuntimeError(
+ f"{self['name']}: file quarantine directory "
+ f"'{self['directory']}' does not exist or is "
+ f"not writable")
else:
raise RuntimeError(
f"{self['name']}: storage_type: invalid storage type")
@@ -512,8 +208,7 @@ class Action:
self.pretend = cfg["pretend"]
self._name = cfg["name"]
- self._func = cfg["func"]
- self._args = cfg["args"]
+ self._class = cfg["class"](**cfg["args"])
self._headersonly = cfg["headersonly"]
def headersonly(self):
@@ -529,5 +224,5 @@ class Action:
self.conditions.match(envfrom=milter.mailfrom,
envto=[*milter.rcpts],
headers=milter.msg.items()):
- return self._func(milter=milter, pretend=self.pretend,
- logger=logger, **self._args)
+ return self._class.execute(
+ milter=milter, pretend=self.pretend, logger=logger)
diff --git a/pymodmilter/modifications.py b/pymodmilter/modifications.py
new file mode 100644
index 0000000..0f398af
--- /dev/null
+++ b/pymodmilter/modifications.py
@@ -0,0 +1,320 @@
+# PyMod-Milter is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# PyMod-Milter is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PyMod-Milter. If not, see .
+#
+
+__all__ = [
+ "AddHeader",
+ "ModHeader",
+ "DelHeader",
+ "AddDisclaimer",
+ "RewriteLinks"]
+
+import logging
+
+from base64 import b64encode
+from bs4 import BeautifulSoup
+from collections import defaultdict
+from copy import copy
+from email.message import MIMEPart
+
+from pymodmilter import replace_illegal_chars
+
+
+class AddHeader:
+ """Add a mail header field."""
+ def __init__(self, field, value):
+ self.field = field
+ self.value = value
+
+ def execute(self, milter, pretend=False,
+ logger=logging.getLogger(__name__)):
+ header = f"{self.field}: {self.value}"
+ if logger.getEffectiveLevel() == logging.DEBUG:
+ logger.debug(f"add_header: {header}")
+ else:
+ logger.info(f"add_header: {header[0:70]}")
+
+ milter.msg.add_header(self.field, self.value)
+
+ if not pretend:
+ milter.addheader(self.field, self.value)
+
+
+class ModHeader:
+ """Change the value of a mail header field."""
+ def __init__(self, field, value, search=None):
+ self.field = field
+ self.value = value
+ self.search = search
+
+ def execute(self, milter, pretend=False,
+ logger=logging.getLogger(__name__)):
+ idx = defaultdict(int)
+
+ for i, (field, value) in enumerate(milter.msg.items()):
+ field_lower = field.lower()
+ idx[field_lower] += 1
+
+ if not self.field.match(field):
+ continue
+
+ new_value = value
+ if self.search is not None:
+ new_value = self.search.sub(self.value, value).strip()
+ else:
+ new_value = self.value
+
+ if not new_value:
+ logger.warning(
+ "mod_header: resulting value is empty, "
+ "skip modification")
+ continue
+
+ if new_value == value:
+ continue
+
+ header = f"{field}: {value}"
+ new_header = f"{field}: {new_value}"
+
+ if logger.getEffectiveLevel() == logging.DEBUG:
+ logger.debug(f"mod_header: {header}: {new_header}")
+ else:
+ logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}")
+
+ milter.msg.replace_header(
+ field, replace_illegal_chars(new_value), idx=idx[field_lower])
+
+ if not pretend:
+ milter.chgheader(field, new_value, idx=idx[field_lower])
+
+
+class DelHeader:
+ """Delete a mail header field."""
+ def __init__(self, field, value=None):
+ self.field = field
+ self.value = value
+
+ def execute(self, milter, pretend=False,
+ logger=logging.getLogger(__name__)):
+ idx = defaultdict(int)
+
+ for field, value in milter.msg.items():
+ field_lower = field.lower()
+ idx[field_lower] += 1
+
+ if not self.field.match(field):
+ continue
+
+ if self.value is not None and not self.value.search(value):
+ continue
+
+ header = f"{field}: {value}"
+ if logger.getEffectiveLevel() == logging.DEBUG:
+ logger.debug(f"del_header: {header}")
+ else:
+ logger.info(f"del_header: {header[0:70]}")
+ milter.msg.remove_header(field, idx=idx[field_lower])
+
+ if not pretend:
+ milter.chgheader(field, "", idx=idx[field_lower])
+
+ idx[field_lower] -= 1
+
+
+def _get_body_content(msg, pref):
+ part = None
+ content = None
+ if not msg.is_multipart() and msg.get_content_type() == f"text/{pref}":
+ part = msg
+ else:
+ part = msg.get_body(preferencelist=(pref))
+
+ if part is not None:
+ content = part.get_content()
+
+ return (part, content)
+
+
+def _has_content_before_body_tag(soup):
+ s = copy(soup)
+ for element in s.find_all("head") + s.find_all("body"):
+ element.extract()
+
+ if len(s.text.strip()) > 0:
+ return True
+
+ return False
+
+
+def _inject_body(milter):
+ if not milter.msg.is_multipart():
+ milter.msg.make_mixed()
+
+ attachments = []
+ for attachment in milter.msg.iter_attachments():
+ if "content-disposition" not in attachment:
+ attachment["Content-Disposition"] = "attachment"
+ attachments.append(attachment)
+
+ milter.msg.clear_content()
+ milter.msg.set_content("")
+ milter.msg.add_alternative("", subtype="html")
+ milter.msg.make_mixed()
+
+ for attachment in attachments:
+ milter.msg.attach(attachment)
+
+
+def _wrap_message(milter):
+ attachment = MIMEPart()
+ attachment.set_content(milter.msg.as_bytes(),
+ maintype="plain", subtype="text",
+ disposition="attachment",
+ filename=f"{milter.qid}.eml",
+ params={"name": f"{milter.qid}.eml"})
+
+ milter.msg.clear_content()
+ milter.msg.set_content(
+ "Please see the original email attached.")
+ milter.msg.add_alternative(
+ "Please see the original email attached.",
+ subtype="html")
+ milter.msg.make_mixed()
+ milter.msg.attach(attachment)
+
+
+class AddDisclaimer:
+ """Append or prepend a disclaimer to the mail body."""
+ def __init__(self, text_template, html_template, action, error_policy):
+ self.text_template = text_template
+ self.html_template = html_template
+ self.action = action
+ self.error_policy = error_policy
+
+ def patch_message_body(self, milter, logger):
+ text_body, text_content = _get_body_content(milter.msg, "plain")
+ html_body, html_content = _get_body_content(milter.msg, "html")
+
+ if text_content is None and html_content is None:
+ raise RuntimeError("message does not contain any body part")
+
+ if text_content is not None:
+ logger.info(f"{self.action} text disclaimer")
+
+ if self.action == "prepend":
+ content = f"{self.text_template}{text_content}"
+ else:
+ content = f"{text_content}{self.text_template}"
+
+ text_body.set_content(
+ content.encode(), maintype="text", subtype="plain")
+ text_body.set_param("charset", "UTF-8", header="Content-Type")
+ del text_body["MIME-Version"]
+
+ if html_content is not None:
+ logger.info(f"{self.action} html disclaimer")
+
+ soup = BeautifulSoup(html_content, "html.parser")
+
+ body = soup.find('body')
+ if not body:
+ body = soup
+ elif _has_content_before_body_tag(soup):
+ body = soup
+
+ if self.action == "prepend":
+ body.insert(0, copy(self.html_template))
+ else:
+ body.append(self.html_template)
+
+ html_body.set_content(
+ str(soup).encode(), maintype="text", subtype="html")
+ html_body.set_param("charset", "UTF-8", header="Content-Type")
+ del html_body["MIME-Version"]
+
+ def execute(self, milter, pretend=False,
+ logger=logging.getLogger(__name__)):
+ old_headers = milter.msg.items()
+
+ try:
+ try:
+ self.patch_message_body(milter, logger)
+ except RuntimeError as e:
+ logger.info(f"{e}, inject empty plain and html body")
+ _inject_body(milter)
+ self.patch_message_body(milter, logger)
+
+ except Exception as e:
+ logger.warning(e)
+ if self.error_policy == "ignore":
+ logger.info(
+ "unable to add disclaimer to message body, "
+ "ignore error according to policy")
+ return
+ elif self.error_policy == "reject":
+ logger.info(
+ "unable to add disclaimer to message body, "
+ "reject message according to policy")
+ return [
+ ("reject", "Message rejected due to error")]
+
+ logger.info("wrap original message in a new message envelope")
+ try:
+ _wrap_message(milter)
+ self.patch_message_body(milter, logger)
+ except Exception as e:
+ logger.error(e)
+ raise Exception(
+ "unable to wrap message in a new message envelope, "
+ "give up ...")
+
+ if not pretend:
+ milter.update_headers(old_headers)
+ milter.replacebody()
+
+
+class RewriteLinks:
+ """Rewrite link targets in the mail html body."""
+ def __init__(self, repl):
+ self.repl = repl
+
+ def execute(self, milter, pretend=False,
+ logger=logging.getLogger(__name__)):
+ html_body, html_content = _get_body_content(milter.msg, "html")
+ if html_content is not None:
+ soup = BeautifulSoup(html_content, "html.parser")
+
+ rewritten = 0
+ for link in soup.find_all("a", href=True):
+ if not link["href"]:
+ continue
+
+ if "{URL_B64}" in self.repl:
+ url_b64 = b64encode(link["href"].encode()).decode()
+ target = self.repl.replace("{URL_B64}", url_b64)
+ else:
+ target = self.repl
+
+ link["href"] = target
+ rewritten += 1
+
+ if rewritten:
+ logger.info(f"rewrote {rewritten} link(s) in html body")
+
+ html_body.set_content(
+ str(soup).encode(), maintype="text", subtype="html")
+ html_body.set_param("charset", "UTF-8", header="Content-Type")
+ del html_body["MIME-Version"]
+
+ if not pretend:
+ milter.replacebody()
diff --git a/pymodmilter/storages.py b/pymodmilter/storages.py
new file mode 100644
index 0000000..8376cbd
--- /dev/null
+++ b/pymodmilter/storages.py
@@ -0,0 +1,226 @@
+# PyMod-Milter is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# PyMod-Milter is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PyMod-Milter. If not, see .
+#
+
+import json
+import logging
+import os
+
+from calendar import timegm
+from datetime import datetime
+from glob import glob
+from time import gmtime
+
+
+class BaseMailStorage(object):
+ "Mail storage base class"
+ def __init__(self):
+ return
+
+ def add(self, data, qid, mailfrom="", recipients=[]):
+ "Add email to storage."
+ return ("", "")
+
+ def find(self, mailfrom=None, recipients=None, older_than=None):
+ "Find emails in storage."
+ return
+
+ def get_metadata(self, storage_id):
+ "Return metadata of email in storage."
+ return
+
+ def delete(self, storage_id, recipients=None):
+ "Delete email from storage."
+ return
+
+ def get_mail(self, storage_id):
+ "Return email and metadata."
+ return
+
+
+class FileMailStorage(BaseMailStorage):
+ "Storage class to store mails on filesystem."
+ def __init__(self, directory, original=False):
+ super().__init__()
+ self.directory = directory
+ self.original = original
+ self._metadata_suffix = ".metadata"
+
+ def _save_datafile(self, storage_id, data):
+ datafile = os.path.join(self.directory, storage_id)
+ try:
+ with open(datafile, "wb") as f:
+ f.write(data)
+ except IOError as e:
+ raise RuntimeError(f"unable save data file: {e}")
+
+ return datafile
+
+ def _save_metafile(self, storage_id, metadata):
+ metafile = os.path.join(
+ self.directory, f"{storage_id}{self._metadata_suffix}")
+ try:
+ with open(metafile, "w") as f:
+ json.dump(metadata, f, indent=2)
+ except IOError as e:
+ raise RuntimeError(f"unable to save metadata file: {e}")
+
+ def _remove(self, storage_id):
+ datafile = os.path.join(self.directory, storage_id)
+ metafile = f"{datafile}{self._metadata_suffix}"
+
+ try:
+ os.remove(metafile)
+ except IOError as e:
+ raise RuntimeError(f"unable to remove metadata file: {e}")
+
+ try:
+ os.remove(datafile)
+ except IOError as e:
+ raise RuntimeError(f"unable to remove data file: {e}")
+
+ def add(self, data, qid, mailfrom="", recipients=[], subject=""):
+ "Add email to file storage and return storage id."
+ super().add(data, qid, mailfrom, recipients)
+ timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
+ storage_id = f"{timestamp}_{qid}"
+
+ # save mail
+ datafile = self._save_datafile(storage_id, data)
+
+ # save metadata
+ metadata = {
+ "mailfrom": mailfrom,
+ "recipients": recipients,
+ "subject": subject,
+ "timestamp": timegm(gmtime()),
+ "queue_id": qid}
+
+ try:
+ self._save_metafile(storage_id, metadata)
+ except RuntimeError as e:
+ os.remove(datafile)
+ raise e
+
+ return (storage_id, datafile)
+
+ def execute(self, milter, pretend=False,
+ logger=logging.getLogger(__name__)):
+ if self.original:
+ milter.fp.seek(0)
+ data = milter.fp.read
+ mailfrom = milter.mailfrom
+ recipients = list(milter.rcpts)
+ subject = ""
+ else:
+ data = milter.msg.as_bytes
+ mailfrom = milter.msginfo["mailfrom"]
+ recipients = list(milter.msginfo["rcpts"])
+ subject = milter.msg["subject"] or ""
+
+ storage_id, datafile = self.add(
+ data(), milter.qid, mailfrom, recipients, subject)
+ logger.info(f"stored message in file {datafile}")
+
+ def get_metadata(self, storage_id):
+ "Return metadata of email in storage."
+ super(FileMailStorage, self).get_metadata(storage_id)
+
+ metafile = os.path.join(
+ self.directory, f"{storage_id}{self._metadata_suffix}")
+ if not os.path.isfile(metafile):
+ raise RuntimeError(
+ f"invalid storage id '{storage_id}'")
+
+ try:
+ with open(metafile, "r") as f:
+ metadata = json.load(f)
+ except IOError as e:
+ raise RuntimeError(f"unable to read metadata file: {e}")
+ except json.JSONDecodeError as e:
+ raise RuntimeError(
+ f"invalid metafile '{metafile}': {e}")
+
+ return metadata
+
+ def find(self, mailfrom=None, recipients=None, older_than=None):
+ "Find emails in storage."
+ super(FileMailStorage, self).find(mailfrom, recipients, older_than)
+ if isinstance(mailfrom, str):
+ mailfrom = [mailfrom]
+ if isinstance(recipients, str):
+ recipients = [recipients]
+
+ emails = {}
+ metafiles = glob(os.path.join(
+ self.directory, f"*{self._metadata_suffix}"))
+ for metafile in metafiles:
+ if not os.path.isfile(metafile):
+ continue
+
+ storage_id = os.path.basename(
+ metafile[:-len(self._metadata_suffix)])
+ metadata = self.get_metadata(storage_id)
+ if older_than is not None:
+ if timegm(gmtime()) - metadata["date"] < (older_than * 86400):
+ continue
+
+ if mailfrom is not None:
+ if metadata["mailfrom"] not in mailfrom:
+ continue
+
+ if recipients is not None:
+ if len(recipients) == 1 and \
+ recipients[0] not in metadata["recipients"]:
+ continue
+ elif len(set(recipients + metadata["recipients"])) == \
+ len(recipients + metadata["recipients"]):
+ continue
+
+ emails[storage_id] = metadata
+
+ return emails
+
+ def delete(self, storage_id, recipients=None):
+ "Delete email from storage."
+ super(FileMailStorage, self).delete(storage_id, recipients)
+
+ try:
+ metadata = self.get_metadata(storage_id)
+ except RuntimeError as e:
+ raise RuntimeError(f"unable to delete email: {e}")
+
+ if not recipients:
+ self._remove(storage_id)
+ else:
+ if type(recipients) == str:
+ recipients = [recipients]
+ for recipient in recipients:
+ if recipient not in metadata["recipients"]:
+ raise RuntimeError(f"invalid recipient '{recipient}'")
+ metadata["recipients"].remove(recipient)
+ if not metadata["recipients"]:
+ self._remove(storage_id)
+ else:
+ self._save_metafile(storage_id, metadata)
+
+ def get_mail(self, storage_id):
+ super(FileMailStorage, self).get_mail(storage_id)
+
+ metadata = self.get_metadata(storage_id)
+ datafile = os.path.join(self.directory, storage_id)
+ try:
+ fp = open(datafile, "rb")
+ except IOError as e:
+ raise RuntimeError(f"unable to open email data file: {e}")
+ return (fp, metadata)