diff --git a/README.md b/README.md index 10027b4..bbce560 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Config options in **global** section: The socket used to communicate with the MTA. If it is not specified in the config, it has to be set as command line option. * **local_addrs** (optional) A list of hosts and network addresses which are considered local. It is used to for the condition option [local](#Conditions). - Default: **::1/128, 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16** + Default: **fe80::/64, ::1/128, 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16** * **loglevel** (optional) Set the log level. This option may be overriden by any rule or action object. Possible values are: * **error** @@ -115,6 +115,9 @@ Config options for **store** actions: * **storage_type** Storage type. Possible values are: * **file** + * **original** (optional) + Default: **false** + If set to true, store the message as received by the MTA instead of storing the current state of the message, that may was modified already by other actions. Config options for **file** storage: * **directory** diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index 58aeb38..8430f39 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -32,9 +32,10 @@ import json from Milter.utils import parse_addr from collections import defaultdict +from email import message_from_binary_file from email.header import Header -from email.parser import BytesFeedParser from email.policy import default as default_policy, SMTP +from io import BytesIO from netaddr import IPNetwork, AddrFormatError from pymodmilter.base import CustomLogger, BaseConfig, MilterMessage @@ -92,6 +93,7 @@ class ModifyMilterConfig(BaseConfig): "should be list of strings" else: local_addrs = [ + "fe80::/64", "::1/128", "127.0.0.0/8", "10.0.0.0/8", @@ -139,12 +141,6 @@ class ModifyMilter(Milter.Base): self.logger = logging.getLogger(__name__) self.logger.setLevel(ModifyMilter._loglevel) - # save rules, it must not change during runtime - self.rules = ModifyMilter._rules.copy() - - self.msg = None - self._replace_body = False - def addheader(self, field, value, idx=-1): value = replace_illegal_chars(Header(s=value).encode()) self.logger.debug(f"milter: addheader: {field}: {value}") @@ -181,28 +177,20 @@ class ModifyMilter(Milter.Base): self.addheader(field, value) def replacebody(self): - self._replace_body = True + self._replacebody = True def connect(self, IPname, family, hostaddr): try: if hostaddr is None: - self.logger.error("unable to proceed, host address is None") + self.logger.error(f"received invalid host address {hostaddr}, " + f"unable to proceed") return Milter.TEMPFAIL + self.IP = hostaddr[0] + self.port = hostaddr[1] self.logger.debug( - f"accepted milter connection from {hostaddr[0]} " - f"port {hostaddr[1]}") - - # remove rules which ignore this host - for rule in self.rules.copy(): - if rule.ignores(host=hostaddr[0]): - self.rules.remove(rule) - - if not self.rules: - self.logger.debug( - f"host {hostaddr[0]} is ignored by all rules, " - f"skip further processing") - return Milter.ACCEPT + f"accepted milter connection from {self.IP} " + f"port {self.port}") except Exception as e: self.logger.exception( f"an exception occured in connect method: {e}") @@ -210,20 +198,21 @@ class ModifyMilter(Milter.Base): return Milter.CONTINUE + def hello(self, heloname): + try: + self.heloname = heloname + self.logger.debug(f"received HELO name: {heloname}") + except Exception as e: + self.logger.exception( + f"an exception occured in hello method: {e}") + return Milter.TEMPFAIL + + return Milter.CONTINUE + def envfrom(self, mailfrom, *str): try: - mailfrom = "@".join(parse_addr(mailfrom)).lower() - for rule in self.rules.copy(): - if rule.ignores(envfrom=mailfrom): - self.rules.remove(rule) - - if not self.rules: - self.logger.debug( - f"envelope-from address {mailfrom} is ignored by " - f"all rules, skip further processing") - return Milter.ACCEPT - - self.recipients = set() + self.mailfrom = "@".join(parse_addr(mailfrom)).lower() + self.rcpts = set() except Exception as e: self.logger.exception( f"an exception occured in envfrom method: {e}") @@ -233,7 +222,7 @@ class ModifyMilter(Milter.Base): def envrcpt(self, to, *str): try: - self.recipients.add("@".join(parse_addr(to)).lower()) + self.rcpts.add("@".join(parse_addr(to)).lower()) except Exception as e: self.logger.exception( f"an exception occured in envrcpt method: {e}") @@ -243,32 +232,25 @@ class ModifyMilter(Milter.Base): def data(self): try: - for rule in self.rules.copy(): - if rule.ignores(envto=[*self.recipients]): - self.rules.remove(rule) - - if not self.rules: - self.logger.debug( - "envelope-to addresses are ignored by all rules, " - "skip further processing") - return Milter.ACCEPT - self.qid = self.getsymval('i') self.logger = CustomLogger(self.logger, {"qid": self.qid}) self.logger.debug("received queue-id from MTA") - self.fields = None - self.fields_bytes = None - self.body_data = None + self.rules = [] + self._headersonly = True + for rule in ModifyMilter._rules: + if not rule.ignores(host=self.IP, envfrom=self.mailfrom, + envto=[*self.rcpts]): + self.rules.append(rule) + if rule.need_body(): + self._headersonly = False - self._fp = BytesFeedParser( - _factory=MilterMessage, policy=default_policy) - self._keep_body = False - for rule in self.rules: - if rule.need_body(): - self._keep_body = True - break + if not self.rules: + self.logger.debug( + "message is ignored by all rules, skip further processing") + return Milter.ACCEPT + self.fp = BytesIO() except Exception as e: self.logger.exception( f"an exception occured in data method: {e}") @@ -282,7 +264,7 @@ class ModifyMilter(Milter.Base): field = field.encode("ascii", errors="replace") value = value.encode("ascii", errors="replace") - self._fp.feed(field + b": " + value + b"\r\n") + self.fp.write(field + b": " + value + b"\r\n") except Exception as e: self.logger.exception( f"an exception occured in header method: {e}") @@ -292,7 +274,7 @@ class ModifyMilter(Milter.Base): def eoh(self): try: - self._fp.feed(b"\r\n") + self.fp.write(b"\r\n") except Exception as e: self.logger.exception( f"an exception occured in eoh method: {e}") @@ -302,8 +284,8 @@ class ModifyMilter(Milter.Base): def body(self, chunk): try: - if self._keep_body: - self._fp.feed(chunk) + if not self._headersonly: + self.fp.write(chunk) except Exception as e: self.logger.exception( f"an exception occured in body method: {e}") @@ -313,7 +295,19 @@ class ModifyMilter(Milter.Base): def eom(self): try: - self.msg = self._fp.close() + self.fp.seek(0) + self.msg = message_from_binary_file( + self.fp, _class=MilterMessage, policy=default_policy) + + self.msg_info = defaultdict(str) + self.msg_info["ip"] = self.IP + self.msg_info["port"] = self.port + self.msg_info["heloname"] = self.heloname + self.msg_info["envfrom"] = self.mailfrom + self.msg_info["rcpts"] = self.rcpts + self.msg_info["qid"] = self.qid + + self._replacebody = False milter_action = None for rule in self.rules: milter_action = rule.execute(self) @@ -321,7 +315,7 @@ class ModifyMilter(Milter.Base): if milter_action is not None: break - if self._replace_body: + if self._replacebody: data = self.msg.as_bytes(policy=SMTP) body_pos = data.find(b"\r\n\r\n") + 4 self.logger.debug("milter: replacebody") diff --git a/pymodmilter/actions.py b/pymodmilter/actions.py index 7539be6..78d10c0 100644 --- a/pymodmilter/actions.py +++ b/pymodmilter/actions.py @@ -32,6 +32,7 @@ from collections import defaultdict from copy import copy from datetime import datetime from email.message import MIMEPart +from email.policy import SMTP from pymodmilter import CustomLogger, BaseConfig from pymodmilter.conditions import ConditionsConfig, Conditions @@ -204,7 +205,7 @@ def _patch_message_body(milter, action, text_template, html_template, logger): def _wrap_message(milter, logger): attachment = MIMEPart() - attachment.set_content(milter.msg.as_bytes(), + attachment.set_content(milter.msg.as_bytes(policy=SMTP), maintype="plain", subtype="text", disposition="attachment", filename=f"{milter.qid}.eml", @@ -317,7 +318,7 @@ def rewrite_links(milter, repl, pretend=False, milter.replacebody() -def store(milter, directory, pretend=False, +def store(milter, directory, original=False, pretend=False, logger=logging.getLogger(__name__)): timestamp = datetime.now().strftime("%Y%m%d%H%M%S") store_id = f"{timestamp}_{milter.qid}" @@ -326,7 +327,11 @@ def store(milter, directory, pretend=False, logger.info(f"store message in file {datafile}") try: with open(datafile, "wb") as fp: - fp.write(milter.msg.as_bytes()) + if original: + milter.fp.seek(0) + fp.write(milter.fp.read()) + else: + fp.write(milter.msg.as_bytes(policy=SMTP)) except IOError as e: raise RuntimeError(f"unable to store message: {e}") @@ -468,6 +473,10 @@ class ActionConfig(BaseConfig): f"{self['name']}: storage_type: invalid value, " \ f"should be string" self["storage_type"] = cfg["storage_type"] + + if "original" in cfg: + self.add_bool_arg(cfg, "original") + if self["storage_type"] == "file": self.add_string_arg(cfg, "directory") else: diff --git a/pymodmilter/docs/pymodmilter.conf.example b/pymodmilter/docs/pymodmilter.conf.example index 98967ea..e7e5d02 100644 --- a/pymodmilter/docs/pymodmilter.conf.example +++ b/pymodmilter/docs/pymodmilter.conf.example @@ -28,7 +28,7 @@ # Notes: A list of local hosts and networks. # Value: [ LIST ] # - "local_addrs": ["::1/128", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"], + "local_addrs": ["fe80::/64", "::1/128", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"], # Option: loglevel # Type: String @@ -206,7 +206,14 @@ # Type: String # Notes: Directory used to store e-mails. # Value: [ file ] - "directory": "/mnt/messages" + "directory": "/mnt/messages", + + # Option: original + # Type: Bool + # Notes: If set to true, store the message as received by the MTA instead of storing the current state + # of the message, that may was modified already by other actions. + # Value: [ true | false ] + "original": true } ] }