add dependency for pymilter version 1.5 or newer

Signed-off-by: Thomas Oettli <spacefreak@noop.ch>

MANIFEST.in

@@ -1,3 +1,3 @@
 include LICENSE README.md
-recursive-include pymodmilter/docs *
+recursive-include pyquarantine/docs *
-recursive-include pymodmilter/misc *
+recursive-include pyquarantine/misc *

README.md

@@ -1,134 +1,446 @@
-# pymodmilter
+# pyquarantine-milter
-A pymilter based sendmail/postfix pre-queue filter with the ability to add, remove and modify e-mail headers.  
+A pymilter based sendmail/postfix pre-queue filter with the ability to ...
-The project is currently in beta status, but it is already used in a productive enterprise environment which processes about a million e-mails per month.  
+* modify e-mail headers (add, modify, delete)
+* store e-mails
+* send e-mail notifications
+* append and prepend disclaimers to e-mail text parts
+* quarantine e-mails (store e-mail, optionally notify receivers)
 
-The basic idea is to define rules with conditions and actions which are processed when all conditions are true.
+It is useful in many cases due to its flexible configuration and the ability to handle any number of quarantines and modifications sequential and conditional. Storages and whitelists used by quarantines can be managed with the built-in CLI.  
+
+Addionally, pyquarantine-milter provides a sanitized, harmless version of the text parts of e-mails as template variable, which can be embedded in e-mail notifications. This makes it easier for users to decide, if a match is a false-positive or not.  
+It is also possible to use any metavariable as template variable (e.g. storage ID, envelope-from address, ...). This may be used to give your users the ability to release e-mails or whitelist the from-address for example. A webservice then releases the e-mail from the quarantine.  
+
+The project is currently in beta status, but it is already used in a productive enterprise environment that processes about a million e-mails per month.
 
 ## Dependencies
-Pymodmilter is depending on these python packages, but they are installed automatically if you are working with pip.
+pyquarantine is depending on these python packages, they are installed automatically if you are working with pip.
-* [pymilter](https://pythonhosted.org/pymilter/)
+* [jsonschema](https://github.com/Julian/jsonschema)
-* [netaddr](https://github.com/drkjam/netaddr/)
+* [pymilter](https://github.com/sdgathman/pymilter)
+* [netaddr](https://github.com/drkjam/netaddr)
+* [peewee](https://github.com/coleifer/peewee)
+* [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/)
 
 ## Installation
-* Install pymodmilter with pip and copy the example config file.
 ```sh
-pip install pymodmilter
+# install pyquarantine with pip.
-cp /etc/pymodmilter/pymodmilter.conf.example /etc/pymodmilter/pymodmilter.conf
+pip install pyquarantine
-```
-* Modify /etc/pymodmilter/pymodmilter.conf according to your needs.
 
-## Configuration options
+# install service files, default config and templates
-Pymodmilter uses a config file in JSON format. The config file has to be JSON valid with the exception of allowed comment lines starting with **#**. The options are described below.  
+pyquarantine-milter --install
-Rules and actions are processed in the given order.
+
+# copy default config file
+cp /etc/pyquarantine/pyquarantine.conf.default /etc/pyquarantine/pyquarantine.conf
+
+# Check the validity of the your config file.
+pyquarantine-milter -t
+```
+## Autostart
+The following init systems are supported.
+
+### systemd
+```sh
+# start the daemon at boot time
+systemctl enable pyquarantine-milter.service
+
+# start the daemon immediately
+systemctl start pyquarantine-milter.service
+```
+
+### OpenRC (Gentoo)
+```sh
+# start the daemon at boot time
+rc-update add pyquarantine-milter default
+
+# start the daemon immediately
+rc-service pyquarantine-milter start
+```
+
+## Configuration
+pyquarantine uses a config file in JSON format. It has to be JSON valid with the exception of allowed comment lines starting with **#**.  
+
+The basic idea is to configure rules that contain actions. Both rules and actions may have conditions. An example of using rules is separating incoming and outgoing e-mails using the local condition. Rules and actions are always processed in the given order.  
 
 ### Global
-Config options in **global** section:
+Global config options:
 * **socket** (optional)  
-  The socket used to communicate with the MTA. If it is not specified in the config, it has to be set as command line option.
+  Socket used to communicate with the MTA. If it is not specified in the config, it has to be set as command line option.
-* **local_addrs** (optional)  
+* **local_addrs** (optional, default: [fe80::/64, ::1/128, 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16])  
-  A list of hosts and network addresses which are considered local. It is used to for the condition option [local](#Conditions).  
+  List of hosts and network addresses which are considered local. It is used for the condition option [local](#Conditions).  
-  Default: **::1/128, 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16**
+* **loglevel**  (optional, default: "info")  
-* **loglevel**  (optional)
+  Set the log level. This option may be overriden by any rule or action object.  
-  Set the log level. This option may be overriden by any rule or action object. Possible values are:
+  Possible values:
   * **error**  
   * **warning**  
   * **info**  
   * **debug**  
-  Default: **info**
+* **pretend** (optional, default: false)  
-* **pretend** (optional)
+  Pretend actions, for test purposes. This option may be overriden by any rule or action object.  
-  Pretend actions, for test purposes. This option may be overriden by any rule or action object.
+* **rules**  
+  List of rule objects.
 
-### Rules
+### Rule
-Config options for **rule** objects:
+Config options for rule objects:
-* **name**  (optional)  
+* **name**  
   Name of the rule.  
-  Default: **Rule #n**
 * **actions**  
-  A list of action objects which are processed in the given order.
+  List of action objects.
 * **conditions** (optional)  
-  A list of conditions which all have to be true to process the actions.
+  See section [Conditions](#Conditions).
 * **loglevel** (optional)  
-  As described above in the [Global](#Global) section.
+  See section [Global](#Global).
 * **pretend** (optional)  
-  As described above in the [Global](#Global) section.
+  See section [Global](#Global).
 
-### Actions
+### Action
-Config options for **action** objects:
+Config options for action objects:
-* **name** (optional)  
+* **name**  
   Name of the action.
-  Default: **Action #n**
 * **type**  
-  Action type. Possible values are:
+  See section [Action types](#Action-types).
-  * **add_header**
+* **options**  
-  * **del_header**
+  Options depending on the action type, see section [Action types](#Action-types).
-  * **mod_header**
-  * **add_disclaimer**
-  * **store**
 * **conditions** (optional)  
-  A list of conditions which all have to be true to process the action.
+  See section [Conditions](#Conditions).
-* **pretend** (optional)  
-  Just pretend all actions of this rule, for test purposes.
 * **loglevel** (optional)  
-  As described above in the [Global](#Global) section.
+  See section [Global](#Global).
-
+* **pretend** (optional)  
-Config options for **add_header** actions:
+  See section [Global](#Global).
-  * **header**  
-    Name of the header.
-  * **value**  
-    Value of the header.
-
-Config options for **del_header** actions:
-  * **header**  
-    Regular expression to match against header names.
-  * **value** (optional)
-    Regular expression to match against the headers value.
-
-Config options for **mod_header** actions:
-  * **header**  
-    Regular expression to match against header names.
-  * **search** (optional)  
-    Regular expression to match against header values. You may use subgroups or named subgroups (python syntax) to include parts of the original value in the new value.
-  * **value**  
-    New value of the header.
-
-Config options for **add_disclaimer** actions:
-  * **action**  
-    Action to perform with the disclaimer. Possible values are:
-    * append
-    * prepend
-  * **html_file**  
-    Path to a file which contains the html representation of the disclaimer.
-  * **text_file**  
-    Path to a file which contains the text representation of the disclaimer.
-  * **error_policy** (optional)  
-    Set the error policy in case the disclaimer cannot be added (e.g. if no body part is present in the e-mail). Possible values are:
-    * **wrap**  
-      A new e-mail body is generated with the disclaimer as body and the original e-mail attached.
-    * **ignore**  
-      Ignore the error and do nothing.
-    * **reject**  
-      Reject the e-mail.
-    Default: **wrap**
-
-Config options for **store** actions:
-  * **storage_type**  
-    Storage type. Possible values are:
-    * **file**
-
-Config options for **file** storage:
-  * **directory**  
-  Directory used to store e-mails.
 
 ### Conditions
-Config options for **conditions** objects:
+Config options for conditions objects:
 * **local** (optional)  
-  If set to true, the rule is only executed for e-mails originating from addresses defined in local_addrs and vice versa.
+  Matches outgoing e-mails (sender address matches **local_addrs**) if set to **true** or matches incoming e-mails if set to **false**.
 * **hosts** (optional)  
-  A list of hosts and network addresses for which the rule should be executed.
+  Matches e-mails originating from the given list of hosts and network addresses.
 * **envfrom** (optional)  
-  A regular expression to match against the evenlope-from addresses for which the rule should be executed.
+  Matches e-mails for which the envelope-from address matches the given regular expression.
 * **envto** (optional)  
-  A regular expression to match against all evenlope-to addresses. All addresses must match to fulfill the condition.
+  Matches e-mails for which all envelope-to addresses match the given regular expression.
+* **headers** (optional)  
+  Matches e-mails for which all regular expressions in the given list are matching at least one e-mail header.
+* **whitelist** (optional)  
+  Matches e-mails for which the whitelist has no entry for the envelope-from and envelope-to address combination, see section [Whitelist](#Whitelist) for details.
+* **var** (optional)  
+  Matches e-mails for which a previous action or condition has set the given metavariable.
+* **metavar** (optional)  
+  Prefix for the name of metavariables which are possibly provided by the **envfrom**, **envto** or **headers** condition. Meta variables will be provided if the regular expressions contain named subgroups, see [python.re](https://docs.python.org/3/library/re.html) for details.  
+  If not set, no metavariables will be provided.
+
+### Whitelist
+Config options for whitelist objects:
+* **type**  
+  See section [Whitelist types](#Whitelist-types).
+
+### Action types
+Available action types:
+##### add_header
+Add new header.  
+Options:
+* **field**  
+  Name of the header.
+* **value**  
+  Value of the header.
+
+##### del_header
+Delete header(s).  
+Options:
+* **field**  
+  Regular expression to match against header names.
+* **value** (optional)  
+  Regular expression to match against the headers value.
+
+##### mod_header
+Modify header(s).  
+Options:
+* **field**  
+  Regular expression to match against header names.
+* **search** (optional)  
+  Regular expression to match against header values. You may use subgroups or named subgroups (python syntax) to include parts of the original value in the new value.
+* **value**  
+  New value of the header.
+
+##### add_disclaimer
+Append or prepend disclaimer to text and/or html body parts.  
+Options:
+* **action**  
+  Action to perform with the disclaimer.  
+  Possible values:
+  * append
+  * prepend
+* **html_template**  
+  Path to a file which contains the html representation of the disclaimer.
+* **text_template**  
+  Path to a file which contains the text representation of the disclaimer.
+* **error_policy** (optional, default: "wrap")  
+  Set the error policy in case the disclaimer cannot be added (e.g. if the html part cannot be parsed).  
+  Possible values:
+  * **wrap**  
+    A new e-mail body is generated with the disclaimer as body and the original e-mail attached.
+  * **ignore**  
+    Ignore the error and do nothing.
+  * **reject**  
+    Reject the e-mail.
+* **add_html_body** (optional, default: false)  
+  Generate a html body with the content of the text body if no html body is present.
+
+##### store
+Store e-mail.  
+Options:
+* **type**  
+  See section [Storage types](#Storage-types).
+* **original** (optional, default: false)  
+  If set to true, store the message as received by the MTA instead of storing the current state of the message, that may was modified already by other actions.
+* **metadata** (optional, default: false)  
+  Store metadata.
+* **metavar**  (optional)  
+  Prefix for the metavariable names. If not set, no metavariables will be provided.  
+  The storage provides the following metavariables:
+  * **ID** (the storage ID of the e-mail)  
+  * **DATAFILE** (path to the data file)  
+  * **METAFILE** (path to the meta file if **metadata** is set to **true**)  
+
+##### notify
+Send notification.  
+Options:
+* **type**  
+  See section [Notification types](#Notification-types).
+
+##### quarantine
+Quarantine e-mail.  
+Options:
+* **store**  
+  Options for e-mail storage, see action [store](#store).  
+  If the option **metadata** is not specificall set for this storage, it will be set to true.
+* **smtp_host**  
+  SMTP host used to release e-mails from quarantine.
+* **smtp_port**  
+  SMTP port used to release e-mails from quarantine.
+* **notify** (optional)  
+  Options for e-mail notifications, see action [notify](#notify).
+* **milter_action** (optional)  
+  Milter action to perform. If set, no further rules or actions will be processed.  
+  Please think carefully what you set here or your MTA may do something you do not want it to do.  
+  Possible values:
+  * **ACCEPT**  
+    Tell the MTA to continue processing the e-mail.
+  * **REJECT**  
+    Tell the MTA to reject the e-mail.
+  * **DISCARD**  
+    Tell the MTA to silently discard the e-mail.
+* **reject_reason** (optional, default: "Message rejected")  
+  Reject message sent to MTA if milter_action is set to reject.
+* **whitelist** (optional)  
+  Options for a whitelist object, see section [Whitelist](#Whitelist).
+
+### Storage types
+Available storage types:
+##### file
+File storage.  
+Options:
+* **directory**  
+  Directory used to store e-mails.
+* **metadata** (optional, default: false)  
+  Store metadata file.
+* **mode**  (optional, default: system default)  
+  File mode when new files are created.
+
+### Notification types
+Available notification types:
+##### email
+Generate an e-mail notification based on a template and send it to the original recipient.  
+Available template variables:
+* **{ENVELOPE_FROM}**  
+  Sender address received by the milter.
+* **{ENVELOPE_FROM_URL}**  
+  Like ENVELOPE_FROM, but URL encoded.
+* **{ENVELOPE_TO}**  
+  Recipient address of this notification.
+* **{ENVELOPE_TO_URL}**  
+  Like ENVELOPE_TO, but URL encoded.
+* **{FROM}**  
+  Value of the FROM header of the e-mail.
+* **{TO}**  
+  Value of the TO header of the e-mail.
+* **{SUBJECT}**  
+  Configured e-mail notification subject.
+* **{HTML_TEXT}**  
+  Sanitized version of the e-mail text part of the e-mail. Only harmless HTML tags and attributes are included. Images are optionally stripped or replaced with the image set by **repl_img** option.
+
+Additionally, every metavariable set by previous conditions or actions are also available as template variables. This is useful to include additional information (e.g. virus names, spam points, ...) within the notification.  
+
+Options:
+* **smtp_host**  
+  SMTP host used to send notifications.
+* **smtp_port**  
+  SMTP port used to send notifications.
+* **envelope_from**  
+  Envelope-From address.
+* **from_header**  
+  Value of the From header. You may use the template variable **{FROM}**.
+* **subject**  
+  Subject of the notification e-mail. You  may use the template variable **{SUBJECT}**.
+* **template**  
+  Path to the HTML template.  
+* **strip_imgs** (optional, default: false)  
+  Strip images from e-mail. This option superseeds **repl_img**.
+* **repl_img** (optional)  
+  Image used to replace all images in the e-mail HTML part.
+* **embed_imgs** (optional)  
+  List of images to embed into the notification e-mail. The Content-ID of each image will be set to the filename, so you can reference it from the e-mail template.
+
+### Whitelist types
+Available whitelist types:
+##### db
+Whitelist stored in database. The table is created automatically if it does not exist yet.  
+Options:
+* **connection**  
+  Database connection string, see [Peewee Playhouse Extension](https://docs.peewee-orm.com/en/latest/peewee/playhouse.html#db-url).
+* **table**  
+  Database table to use.
+
+### Integration with MTA
+For integration with Postfix, see [Postix Milter Readme](http://www.postfix.org/MILTER_README.html).  
+For integration with sendmail, see [Pymilter Sendmail Readme](https://pythonhosted.org/pymilter/milter_api/installation.html#config).
+
+## Examples
+Here are some config examples.
+
+### Virus and spam quarantine for incoming e-mails
+In this example it is assumed, that another milter (e.g. Amavisd or Rspamd) adds headers to spam and virus e-mails.
+```json
+{
+    "socket": "unix:/tmp/pyquarantine.sock",
+    "rules": [
+        {
+            "name": "inbound",
+            "conditions": {
+                "local": false
+            },
+            "actions": [
+                {
+                    "name": "virus",
+                    "type": "quarantine",
+                    "conditions": {
+                        "headers": ["^X-Virus: Yes"],
+                    },
+                    "options": {
+                        "store": {
+                            "type": "file",
+                            "directory": "/mnt/data/quarantine/virus",
+                        },
+                        "notify": {
+                            "type": "email",
+                            "smtp_host": "localhost",
+                            "smtp_port": 2525,
+                            "envelope_from": "notifications@example.com",
+                            "from_header": "{FROM}",
+                            "subject": "[VIRUS] {SUBJECT}",
+                            "template": "/etc/pyquarantine/templates/notification.template",
+                            "repl_img": "/etc/pyquarantine/templates/removed.png"
+                        },
+                        "smtp_host": "localhost",
+                        "smtp_port": 2525,
+                        "milter_action": "REJECT",
+                        "reject_reason": "Message rejected due to virus"
+                    }
+                }, {
+                    "name": "spam",
+                    "type": "quarantine",
+                    "conditions": {
+                        "headers": ["^X-Spam: Yes"]
+                    },
+                    "options": {
+                        "store": {
+                            "type": "file",
+                            "directory": "/mnt/data/quarantine/spam",
+                        },
+                        "notify": {
+                            "type": "email",
+                            "smtp_host": "localhost",
+                            "smtp_port": 2525,
+                            "envelope_from": "notifications@example.com",
+                            "from_header": "{FROM}",
+                            "subject": "[SPAM] {SUBJECT}",
+                            "template": "/etc/pyquarantine/templates/notification.template",
+                            "repl_img": "/etc/pyquarantine/templates/removed.png"
+                        },
+                        "smtp_host": "localhost",
+                        "smtp_port": 2525,
+                        "milter_action": "DISCARD"
+                    }
+                }
+            ]
+        }
+    ]
+}
+```
+### Mark subject of incoming e-mails and remove the mark from outgoing e-mails
+```json
+{
+    "socket": "unix:/tmp/pyquarantine.sock",
+    "rules": [
+        {
+            "name": "inbound",
+            "conditions": {
+                "local": false
+            },
+            "actions": [
+                {
+                    "name": "add_subject_prefix",
+                    "type": "mod_header",
+                    "options": {
+                        "field": "^(Subject|Thread-Topic)$",
+                        "search": "^(?P<subject>.*)",
+                        "value": "[EXTERNAL] \\g<subject>"
+                    }
+                }
+            ]
+        }, {
+            "name": "outbound",
+            "conditions": {
+                "local": true
+            },
+            "actions": [
+                {
+                    "name": "remove_subject_prefix",
+                    "type": "mod_header",
+                    "options": {
+                        "field": "^(Subject|Thread-Topic)$",
+                        "search": "^(?P<prefix>.*)\\[EXTERNAL\\] (?P<suffix>.*)$",
+                        "value": "\\g<prefix>\\g<suffix>"
+                    }
+                }
+            ]
+        }
+    ]
+}
+```
+### Store an exact copy of all incoming e-mails in directory
+```json
+{
+    "socket": "unix:/tmp/pyquarantine.sock",
+    "rules": [
+        {
+            "name": "inbound",
+            "conditions": {
+                "local": false
+            },
+            "actions": [
+                {
+                    "name": "store_original",
+                    "type": "store",
+                    "options": {
+                        "type": "file",
+                        "directory": "/mnt/data/incoming",
+                        "original": true,
+                    }
+                }
+            ]
+        }
+    ]
+}
+```
 
 ## Developer information
 Everyone who wants to improve or extend this project is very welcome.

distribution/gentoo/mail-filter/pyquarantine-milter/pyquarantine-milter-9999.ebuild

@@ -0,0 +1,52 @@
+# Copyright 2020 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=7
+PYTHON_COMPAT=( python3_{9..10} )
+DISTUTILS_USE_SETUPTOOLS=rdepend
+
+SCM=""
+if [ "${PV#9999}" != "${PV}" ] ; then
+	SCM="git-r3"
+	EGIT_REPO_URI="https://github.com/spacefreak86/${PN}"
+	EGIT_BRANCH="master"
+fi
+
+inherit ${SCM} distutils-r1 systemd
+
+DESCRIPTION="A pymilter based sendmail/postfix pre-queue filter."
+HOMEPAGE="https://github.com/spacefreak86/pyquarantine-milter"
+if [ "${PV#9999}" != "${PV}" ] ; then
+	SRC_URI=""
+	KEYWORDS=""
+	# Needed for tests
+	S="${WORKDIR}/${PN}"
+	EGIT_CHECKOUT_DIR="${S}"
+else
+	SRC_URI="https://github.com/spacefreak86/${PN}/archive/${PV}.tar.gz -> ${P}.tar.gz"
+	KEYWORDS="amd64 x86"
+fi
+
+LICENSE="GPL-3"
+SLOT="0"
+IUSE="+lxml systemd"
+
+RDEPEND="
+	dev-python/beautifulsoup4[${PYTHON_USEDEP}]
+	dev-python/jsonschema[${PYTHON_USEDEP}]
+	lxml? ( dev-python/lxml[${PYTHON_USEDEP}] )
+	dev-python/netaddr[${PYTHON_USEDEP}]
+	dev-python/peewee[${PYTHON_USEDEP}]
+	>=dev-python/pymilter-1.5[${PYTHON_USEDEP}]"
+
+python_install_all() {
+	distutils-r1_python_install_all
+	use systemd && systemd_dounit pyquarantine/misc/systemd/${PN}.service
+	newinitd pyquarantine/misc/openrc/${PN}.initd ${PN}
+	newconfd pyquarantine/misc/openrc/${PN}.confd ${PN}
+}
+
+pkg_postinst() {
+	elog "You will need to set up your /etc/pyquarantine/pyquarantine.conf file before"
+	elog "running pyquarantine-milter for the first time."
+}

distribution/pypi/build.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+set -x
+PYTHON=$(which python)
+
+script_dir=$(dirname "$(readlink -f -- "$BASH_SOURCE")")
+pkg_dir=$(realpath "${script_dir}"/../..)
+
+cd "${pkg_dir}"
+${PYTHON} setup.py clean
+${PYTHON} setup.py sdist bdist_wheel

distribution/pypi/distribute.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+TWINE=$(which twine)
+
+script_dir=$(dirname "$(readlink -f -- "$BASH_SOURCE")")
+pkg_dir=$(realpath "${script_dir}/../..")
+
+cd "${pkg_dir}/dist"
+ls -la
+msg="Select version to distribute (cancel with CTRL+C):"
+echo "${msg}"
+select version in $(find . -maxdepth 1 -type f -name "pyquarantine-*.*.*.tar.gz" -printf "%f\n" | sed "s#\.tar\.gz##g"); do
+  [ -n "${version}" ] && break
+  echo -e "\ninvalid choice\n\n${msg}"
+done
+${TWINE} upload "${version}"{.tar.gz,-*.whl}

pymodmilter/__init__.py

@@ -1,402 +0,0 @@
-# PyMod-Milter is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# PyMod-Milter is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with PyMod-Milter.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-__all__ = [
-    "actions",
-    "conditions",
-    "run",
-    "CustomLogger",
-    "Rule",
-    "ModifyMilter"]
-
-__version__ = "1.1.2"
-
-import Milter
-import logging
-import encodings
-
-from Milter.utils import parse_addr
-from email.message import MIMEPart
-from email.parser import BytesFeedParser
-from email.policy import default as default_policy
-
-from pymodmilter.conditions import Conditions
-
-########################################################
-#  monkey-patch pythons email library bug 27257,30988  #
-########################################################
-#
-# https://bugs.python.org/issue27257
-# https://bugs.python.org/issue30988
-#
-# fix: https://github.com/python/cpython/pull/15600
-
-import email._header_value_parser
-from email._header_value_parser import TokenList, NameAddr
-from email._header_value_parser import get_display_name, get_angle_addr
-from email._header_value_parser import get_cfws, errors
-from email._header_value_parser import CFWS_LEADER, PHRASE_ENDS
-
-
-class DisplayName(email._header_value_parser.DisplayName):
-    @property
-    def display_name(self):
-        res = TokenList(self)
-        if len(res) == 0:
-            return res.value
-        if res[0].token_type == 'cfws':
-            res.pop(0)
-        else:
-            if isinstance(res[0], TokenList) and \
-                    res[0][0].token_type == 'cfws':
-                res[0] = TokenList(res[0][1:])
-        if res[-1].token_type == 'cfws':
-            res.pop()
-        else:
-            if isinstance(res[-1], TokenList) and \
-                    res[-1][-1].token_type == 'cfws':
-                res[-1] = TokenList(res[-1][:-1])
-        return res.value
-
-
-def get_name_addr(value):
-    """ name-addr = [display-name] angle-addr
-
-    """
-    name_addr = NameAddr()
-    # Both the optional display name and the angle-addr can start with cfws.
-    leader = None
-    if value[0] in CFWS_LEADER:
-        leader, value = get_cfws(value)
-        if not value:
-            raise errors.HeaderParseError(
-                "expected name-addr but found '{}'".format(leader))
-    if value[0] != '<':
-        if value[0] in PHRASE_ENDS:
-            raise errors.HeaderParseError(
-                "expected name-addr but found '{}'".format(value))
-        token, value = get_display_name(value)
-        if not value:
-            raise errors.HeaderParseError(
-                "expected name-addr but found '{}'".format(token))
-        if leader is not None:
-            if isinstance(token[0], TokenList):
-                token[0][:0] = [leader]
-            else:
-                token[:0] = [leader]
-            leader = None
-        name_addr.append(token)
-    token, value = get_angle_addr(value)
-    if leader is not None:
-        token[:0] = [leader]
-    name_addr.append(token)
-    return name_addr, value
-
-
-setattr(email._header_value_parser, "DisplayName", DisplayName)
-setattr(email._header_value_parser, "get_name_addr", get_name_addr)
-
-################################################
-#  add charset alias for windows-874 encoding  #
-################################################
-
-aliases = encodings.aliases.aliases
-
-for alias in ["windows-874", "windows_874"]:
-    if alias not in aliases:
-        aliases[alias] = "cp874"
-
-setattr(encodings.aliases, "aliases", aliases)
-
-################################################
-
-
-class CustomLogger(logging.LoggerAdapter):
-    def process(self, msg, kwargs):
-        if "name" in self.extra:
-            msg = "{}: {}".format(self.extra["name"], msg)
-
-        if "qid" in self.extra:
-            msg = "{}: {}".format(self.extra["qid"], msg)
-
-        if self.logger.getEffectiveLevel() != logging.DEBUG:
-            msg = msg.replace("\n", "").replace("\r", "")
-
-        return msg, kwargs
-
-
-class Rule:
-    """
-    Rule to implement multiple actions on emails.
-    """
-
-    def __init__(self, name, local_addrs, conditions, actions, pretend=False,
-                 loglevel=logging.INFO):
-        logger = logging.getLogger(name)
-        self.logger = CustomLogger(logger, {"name": name})
-        self.logger.setLevel(loglevel)
-
-        if logger is None:
-            logger = logging.getLogger(__name__)
-
-        self.logger = CustomLogger(logger, {"name": name})
-        self.conditions = Conditions(
-            local_addrs=local_addrs,
-            args=conditions,
-            logger=self.logger)
-        self.actions = actions
-        self.pretend = pretend
-
-        self._need_body = False
-        for action in actions:
-            if action.need_body():
-                self._need_body = True
-                break
-
-    def need_body(self):
-        """Return the if this rule needs the message body."""
-        return self._need_body
-
-    def ignores(self, host=None, envfrom=None, envto=None):
-        args = {}
-
-        if host is not None:
-            args["host"] = host
-
-        if envfrom is not None:
-            args["envfrom"] = envfrom
-
-        if envto is not None:
-            args["envto"] = envto
-
-        if self.conditions.match(args):
-            for action in self.actions:
-                if action.conditions.match(args):
-                    return False
-
-        return True
-
-    def execute(self, milter, pretend=None):
-        """Execute all actions of this rule."""
-        if pretend is None:
-            pretend = self.pretend
-
-        for action in self.actions:
-            milter_action = action.execute(milter, pretend=pretend)
-            if milter_action is not None:
-                return milter_action
-
-
-class MilterMessage(MIMEPart):
-    def replace_header(self, _name, _value, occ=None):
-        _name = _name.lower()
-        counter = 0
-        for i, (k, v) in zip(range(len(self._headers)), self._headers):
-            if k.lower() == _name:
-                counter += 1
-                if not occ or counter == occ:
-                    self._headers[i] = self.policy.header_store_parse(
-                        k, _value)
-                    break
-
-        else:
-            raise KeyError(_name)
-
-    def remove_header(self, name, occ=None):
-        name = name.lower()
-        newheaders = []
-        counter = 0
-        for k, v in self._headers:
-            if k.lower() == name:
-                counter += 1
-                if counter != occ:
-                    newheaders.append((k, v))
-            else:
-                newheaders.append((k, v))
-
-        self._headers = newheaders
-
-
-class ModifyMilter(Milter.Base):
-    """ModifyMilter based on Milter.Base to implement milter communication"""
-
-    _rules = []
-    _loglevel = logging.INFO
-
-    @staticmethod
-    def set_rules(rules):
-        ModifyMilter._rules = rules
-
-    def set_loglevel(level):
-        ModifyMilter._loglevel = level
-
-    def __init__(self):
-        self.logger = logging.getLogger(__name__)
-        self.logger.setLevel(ModifyMilter._loglevel)
-
-        # save rules, it must not change during runtime
-        self.rules = ModifyMilter._rules.copy()
-
-    def connect(self, IPname, family, hostaddr):
-        try:
-            if hostaddr is None:
-                self.logger.error("unable to proceed, host address is None")
-                return Milter.TEMPFAIL
-
-            self.logger.debug(
-                f"accepted milter connection from {hostaddr[0]} "
-                f"port {hostaddr[1]}")
-
-            # remove rules which ignore this host
-            for rule in self.rules.copy():
-                if rule.ignores(host=hostaddr[0]):
-                    self.rules.remove(rule)
-
-            if not self.rules:
-                self.logger.debug(
-                    f"host {hostaddr[0]} is ignored by all rules, "
-                    f"skip further processing")
-                return Milter.ACCEPT
-        except Exception as e:
-            self.logger.exception(
-                f"an exception occured in connect method: {e}")
-            return Milter.TEMPFAIL
-
-        return Milter.CONTINUE
-
-    def envfrom(self, mailfrom, *str):
-        try:
-            mailfrom = "@".join(parse_addr(mailfrom)).lower()
-            for rule in self.rules.copy():
-                if rule.ignores(envfrom=mailfrom):
-                    self.rules.remove(rule)
-
-            if not self.rules:
-                self.logger.debug(
-                    f"envelope-from address {mailfrom} is ignored by "
-                    f"all rules, skip further processing")
-                return Milter.ACCEPT
-
-            self.recipients = set()
-        except Exception as e:
-            self.logger.exception(
-                f"an exception occured in envfrom method: {e}")
-            return Milter.TEMPFAIL
-
-        return Milter.CONTINUE
-
-    @Milter.noreply
-    def envrcpt(self, to, *str):
-        try:
-            self.recipients.add("@".join(parse_addr(to)).lower())
-        except Exception as e:
-            self.logger.exception(
-                f"an exception occured in envrcpt method: {e}")
-            return Milter.TEMPFAIL
-
-        return Milter.CONTINUE
-
-    def data(self):
-        try:
-            for rule in self.rules.copy():
-                if rule.ignores(envto=[*self.recipients]):
-                    self.rules.remove(rule)
-
-            if not self.rules:
-                self.logger.debug(
-                    "envelope-to addresses are ignored by all rules, "
-                    "skip further processing")
-                return Milter.ACCEPT
-
-            self.qid = self.getsymval('i')
-            self.logger = CustomLogger(self.logger, {"qid": self.qid})
-            self.logger.debug("received queue-id from MTA")
-
-            self.fields = None
-            self.fields_bytes = None
-            self.body_data = None
-
-            self._fp = BytesFeedParser(
-                _factory=MilterMessage, policy=default_policy)
-            self._keep_body = False
-            for rule in self.rules:
-                if rule.need_body():
-                    self._keep_body = True
-                    break
-
-        except Exception as e:
-            self.logger.exception(
-                f"an exception occured in data method: {e}")
-            return Milter.TEMPFAIL
-
-        return Milter.CONTINUE
-
-    def header(self, field, value):
-        try:
-            # remove surrogates
-            field = field.encode("ascii", errors="replace")
-            value = value.encode("ascii", errors="replace")
-
-            self._fp.feed(field + b": " + value + b"\r\n")
-        except Exception as e:
-            self.logger.exception(
-                f"an exception occured in header method: {e}")
-            return Milter.TEMPFAIL
-
-        return Milter.CONTINUE
-
-    def eoh(self):
-        try:
-            self._fp.feed(b"\r\n")
-        except Exception as e:
-            self.logger.exception(
-                f"an exception occured in eoh method: {e}")
-            return Milter.TEMPFAIL
-
-        return Milter.CONTINUE
-
-    def body(self, chunk):
-        try:
-            if self._keep_body:
-                self._fp.feed(chunk)
-        except Exception as e:
-            self.logger.exception(
-                f"an exception occured in body method: {e}")
-            return Milter.TEMPFAIL
-
-        return Milter.CONTINUE
-
-    def eom(self):
-        try:
-            self.msg = self._fp.close()
-            for rule in self.rules:
-                milter_action = rule.execute(self)
-
-                if milter_action is not None:
-                    if milter_action["action"] == "reject":
-                        self.setreply("554", "5.7.0", milter_action["reason"])
-                        return Milter.REJECT
-
-                    if milter_action["action"] == "accept":
-                        return Milter.ACCEPT
-
-                    if milter_action["action"] == "discard":
-                        return Milter.DISCARD
-
-        except Exception as e:
-            self.logger.exception(
-                f"an exception occured in eom method: {e}")
-            return Milter.TEMPFAIL
-
-        return Milter.ACCEPT

pymodmilter/actions.py

@@ -1,481 +0,0 @@
-# PyMod-Milter is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# PyMod-Milter is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with PyMod-Milter.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-import logging
-import os
-import re
-
-from bs4 import BeautifulSoup
-from collections import defaultdict
-from copy import copy
-from datetime import datetime
-from email.header import Header
-from email.message import MIMEPart
-from email.policy import SMTP
-
-from pymodmilter import CustomLogger, Conditions
-
-
-def _replace_illegal_chars(string):
-    """Replace illegal characters in header values."""
-    return string.replace(
-        "\x00", "").replace(
-        "\r", "").replace(
-        "\n", "")
-
-
-def _add_header(milter, field, value, idx=-1):
-    value = _replace_illegal_chars(
-        Header(s=value).encode())
-    milter.logger.debug(f"milter: addheader: {field}: {value}")
-    milter.addheader(field, value, idx)
-
-
-def add_header(milter, field, value, pretend=False,
-               logger=logging.getLogger(__name__)):
-    """Add a mail header field."""
-    header = f"{field}: {value}"
-    if logger.getEffectiveLevel() == logging.DEBUG:
-        logger.debug(f"add_header: {header}")
-    else:
-        logger.info(f"add_header: {header[0:70]}")
-
-    milter.msg.add_header(field, _replace_illegal_chars(value))
-
-    if pretend:
-        return
-
-    _add_header(milter, field, value)
-
-
-def _mod_header(milter, field, value, occ=1):
-    value = _replace_illegal_chars(
-        Header(s=value).encode())
-    milter.logger.debug(
-        f"milter: chgheader: {field}[{occ}]: {value}")
-    milter.chgheader(field, occ, value)
-
-
-def mod_header(milter, field, value, search=None, pretend=False,
-               logger=logging.getLogger(__name__)):
-    """Change the value of a mail header field."""
-    if isinstance(field, str):
-        field = re.compile(field, re.IGNORECASE)
-
-    if isinstance(search, str):
-        search = re.compile(search, re.MULTILINE + re.DOTALL + re.IGNORECASE)
-
-    occ = defaultdict(int)
-
-    for i, (f, v) in enumerate(milter.msg.items()):
-        f_lower = f.lower()
-        occ[f_lower] += 1
-
-        if not field.match(f):
-            continue
-
-        new_value = v
-        if search is not None:
-            new_value = search.sub(value, v).strip()
-        else:
-            new_value = value
-
-        if not new_value:
-            logger.warning(
-                "mod_header: resulting value is empty, "
-                "skip modification")
-            continue
-
-        if new_value == v:
-            continue
-
-        header = f"{f}: {v}"
-        new_header = f"{f}: {new_value}"
-
-        if logger.getEffectiveLevel() == logging.DEBUG:
-            logger.debug(f"mod_header: {header}: {new_header}")
-        else:
-            logger.info(f"mod_header: {header[0:70]}: {new_header[0:70]}")
-
-        milter.msg.replace_header(
-            f, _replace_illegal_chars(new_value), occ=occ[f_lower])
-
-        if pretend:
-            continue
-
-        _mod_header(milter, f, new_value, occ=occ[f_lower])
-
-
-def _del_header(milter, field, occ=1):
-    milter.logger.debug(
-        f"milter: delheader: {field}[{occ}]")
-    milter.chgheader(field, occ, "")
-
-
-def del_header(milter, field, value=None, pretend=False,
-               logger=logging.getLogger(__name__)):
-    """Delete a mail header field."""
-    if isinstance(field, str):
-        field = re.compile(field, re.IGNORECASE)
-
-    if isinstance(value, str):
-        value = re.compile(value, re.MULTILINE + re.DOTALL + re.IGNORECASE)
-
-    occ = defaultdict(int)
-
-    for f, v in milter.msg.items():
-        f_lower = f.lower()
-        occ[f_lower] += 1
-
-        if not field.match(f):
-            continue
-
-        if value is not None and not value.search(v):
-            continue
-
-        header = f"{f}: {v}"
-        if logger.getEffectiveLevel() == logging.DEBUG:
-            logger.debug(f"del_header: {header}")
-        else:
-            logger.info(f"del_header: {header[0:70]}")
-        milter.msg.remove_header(f, occ=occ[f_lower])
-
-        if not pretend:
-            _del_header(milter, f, occ=occ[f_lower])
-
-        occ[f_lower] -= 1
-
-
-def _serialize_msg(msg, logger):
-    if msg.is_multipart() and not msg["MIME-Version"]:
-        msg.add_header("MIME-Version", "1.0")
-
-    try:
-        logger.debug("serialize message as bytes")
-        data = msg.as_bytes(policy=SMTP)
-    except Exception as e:
-        logger.waring(
-            f"unable to serialize message as bytes: {e}")
-        try:
-            logger.warning("try to serialize message as string")
-            data = msg.as_string(policy=SMTP)
-            data = data.encode("ascii", errors="replace")
-        except Exception as e:
-            raise e
-
-    return data
-
-
-def _get_body_content(msg, pref):
-    part = None
-    content = None
-    if not msg.is_multipart() and msg.get_content_type() == f"text/{pref}":
-        part = msg
-    else:
-        part = msg.get_body(preferencelist=(pref))
-
-    if part is not None:
-        content = part.get_content()
-
-    return (part, content)
-
-
-def _has_content_before_body_tag(soup):
-    s = copy(soup)
-    for element in s.find_all("head") + s.find_all("body"):
-        element.extract()
-
-    if len(s.text.strip()) > 0:
-        return True
-
-    return False
-
-
-def _patch_message_body(milter, action, text, html, logger):
-    text_body, text_content = _get_body_content(milter.msg, "plain")
-    html_body, html_content = _get_body_content(milter.msg, "html")
-
-    if text_content is None and html_content is None:
-        raise RuntimeError("message does not contain any body part")
-
-    if text_content is not None:
-        logger.info(f"{action} text disclaimer")
-
-        if action == "prepend":
-            content = f"{text}{text_content}"
-        else:
-            content = f"{text_content}{text}"
-
-        text_body.set_content(
-            content.encode(), maintype="text", subtype="plain")
-        text_body.set_param("charset", "UTF-8", header="Content-Type")
-        del text_body["MIME-Version"]
-
-    if html_content is not None:
-        logger.info(f"{action} html disclaimer")
-
-        soup = BeautifulSoup(html_content, "html.parser")
-
-        body = soup.find('body')
-        if not body:
-            body = soup
-        elif _has_content_before_body_tag(soup):
-            body = soup
-
-        if action == "prepend":
-            body.insert(0, copy(html))
-        else:
-            body.append(html)
-
-        html_body.set_content(
-            str(soup).encode(), maintype="text", subtype="html")
-        html_body.set_param("charset", "UTF-8", header="Content-Type")
-        del html_body["MIME-Version"]
-
-
-def _update_body(milter, logger):
-    data = _serialize_msg(milter.msg, logger)
-    body_pos = data.find(b"\r\n\r\n") + 4
-    logger.debug("milter: replacebody")
-    milter.replacebody(data[body_pos:])
-    del data
-
-
-def _update_headers(milter, original_headers, logger):
-    if milter.msg.is_multipart() and not milter.msg["MIME-Version"]:
-        milter.msg.add_header("MIME-Version", "1.0")
-
-    # serialize the message object so it updates its internal strucure
-    milter.msg.as_bytes()
-
-    original_headers = [(f, f.lower(), v) for f, v in original_headers]
-    headers = [(f, f.lower(), v) for f, v in milter.msg.items()]
-
-    occ = defaultdict(int)
-    for field, field_lower, value in original_headers:
-        occ[field_lower] += 1
-        if (field, field_lower, value) not in headers:
-            _del_header(milter, field, occ=occ[field_lower])
-            occ[field] -= 1
-
-    for field, value in milter.msg.items():
-        field_lower = field.lower()
-        if (field, field_lower, value) not in original_headers:
-            _add_header(milter, field, value)
-
-
-def _wrap_message(milter, logger):
-    attachment = MIMEPart()
-    attachment.set_content(milter.msg.as_bytes(),
-                           maintype="plain", subtype="text",
-                           disposition="attachment",
-                           filename=f"{milter.qid}.eml",
-                           params={"name": f"{milter.qid}.eml"})
-
-    milter.msg.clear_content()
-    milter.msg.set_content(
-        "Please see the original email attached.")
-    milter.msg.add_alternative(
-        "<html><body>Please see the original email attached.</body></html>",
-        subtype="html")
-    milter.msg.make_mixed()
-    milter.msg.attach(attachment)
-
-
-def _inject_body(milter):
-    if not milter.msg.is_multipart():
-        milter.msg.make_mixed()
-
-    attachments = []
-    for attachment in milter.msg.iter_attachments():
-        if "content-disposition" not in attachment:
-            attachment["Content-Disposition"] = "attachment"
-        attachments.append(attachment)
-
-    milter.msg.clear_content()
-    milter.msg.set_content("")
-    milter.msg.add_alternative("", subtype="html")
-    milter.msg.make_mixed()
-
-    for attachment in attachments:
-        milter.msg.attach(attachment)
-
-
-def add_disclaimer(milter, text, html, action, policy, pretend=False,
-                   logger=logging.getLogger(__name__)):
-    """Append or prepend a disclaimer to the mail body."""
-    original_headers = milter.msg.items()
-
-    try:
-        try:
-            _patch_message_body(milter, action, text, html, logger)
-        except RuntimeError as e:
-            logger.info(f"{e}, inject empty plain and html body")
-            _inject_body(milter)
-            _patch_message_body(milter, action, text, html, logger)
-    except Exception as e:
-        logger.warning(e)
-        if policy == "ignore":
-            logger.info(
-                "unable to add disclaimer to message body, "
-                "ignore error according to policy")
-            return
-        elif policy == "reject":
-            logger.info(
-                "unable to add disclaimer to message body, "
-                "reject message according to policy")
-            return [
-                ("reject", "Message rejected due to error")]
-
-        logger.info("wrap original message in a new message envelope")
-        try:
-            _wrap_message(milter, logger)
-            _patch_message_body(milter, action, text, html, logger)
-        except Exception as e:
-            logger.error(e)
-            raise Exception(
-                "unable to wrap message in a new message envelope, "
-                "give up ...")
-
-    if pretend:
-        return
-
-    _update_headers(milter, original_headers, logger)
-    _update_body(milter, logger)
-
-
-def store(milter, directory, pretend=False,
-          logger=logging.getLogger(__name__)):
-    timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
-    store_id = f"{timestamp}_{milter.qid}"
-    datafile = os.path.join(directory, store_id)
-
-    logger.info(f"store message in file {datafile}")
-    try:
-        with open(datafile, "wb") as fp:
-            fp.write(milter.msg.as_bytes())
-    except IOError as e:
-        raise RuntimeError(f"unable to store message: {e}")
-
-
-class Action:
-    """Action to implement a pre-configured action to perform on e-mails."""
-    _need_body_map = {
-        "add_header": False,
-        "del_header": False,
-        "mod_header": False,
-        "add_disclaimer": True,
-        "store": True}
-
-    def __init__(self, name, local_addrs, conditions, action_type, args,
-                 loglevel=logging.INFO, pretend=False):
-        logger = logging.getLogger(name)
-        self.logger = CustomLogger(logger, {"name": name})
-        self.logger.setLevel(loglevel)
-
-        self.conditions = Conditions(
-            local_addrs=local_addrs,
-            args=conditions,
-            logger=self.logger)
-        self.pretend = pretend
-        self._args = {}
-
-        if action_type not in self._need_body_map:
-            raise RuntimeError(f"invalid action type '{action_type}'")
-        self._need_body = self._need_body_map[action_type]
-
-        try:
-            if action_type == "add_header":
-                self._func = add_header
-                self._args["field"] = args["header"]
-                self._args["value"] = args["value"]
-                if "idx" in args:
-                    self._args["idx"] = args["idx"]
-
-            elif action_type in ["mod_header", "del_header"]:
-                args["field"] = args["header"]
-                del args["header"]
-                regex_args = ["field"]
-
-                if action_type == "mod_header":
-                    self._func = mod_header
-                    self._args["value"] = args["value"]
-                    regex_args.append("search")
-                elif action_type == "del_header":
-                    self._func = del_header
-                    if "value" in args:
-                        regex_args.append("value")
-
-                for arg in regex_args:
-                    try:
-                        self._args[arg] = re.compile(
-                            args[arg],
-                            re.MULTILINE + re.DOTALL + re.IGNORECASE)
-                    except re.error as e:
-                        raise RuntimeError(
-                            f"unable to parse {arg} regex: {e}")
-
-            elif action_type == "add_disclaimer":
-                self._func = add_disclaimer
-                if args["action"] not in ["append", "prepend"]:
-                    raise RuntimeError(f"invalid action '{args['action']}'")
-
-                self._args["action"] = args["action"]
-
-                if args["error_policy"] not in ["wrap", "ignore", "reject"]:
-                    raise RuntimeError(f"invalid policy '{args['policy']}'")
-
-                self._args["policy"] = args["error_policy"]
-
-                try:
-                    with open(args["html_file"], "r") as f:
-                        html = BeautifulSoup(
-                            f.read(), "html.parser")
-                        body = html.find('body')
-                        if body:
-                            # just use content within the body tag if present
-                            html = body
-                        self._args["html"] = html
-                    with open(args["text_file"], "r") as f:
-                        self._args["text"] = f.read()
-                except IOError as e:
-                    raise RuntimeError(f"unable to read template: {e}")
-            elif action_type == "store":
-                self._func = store
-                if args["storage_type"] not in ["file"]:
-                    raise RuntimeError(
-                        "invalid storage_type 'args['storage_type']'")
-
-                if args["storage_type"] == "file":
-                    self._args["directory"] = args["directory"]
-            else:
-                raise RuntimeError(f"invalid action type: {action_type}")
-
-        except KeyError as e:
-            raise RuntimeError(
-                f"mandatory argument not found: {e}")
-
-    def need_body(self):
-        """Return the needs of this action."""
-        return self._need_body
-
-    def execute(self, milter, pretend=None):
-        """Execute configured action."""
-        if pretend is None:
-            pretend = self.pretend
-
-        logger = CustomLogger(self.logger, {"qid": milter.qid})
-
-        return self._func(milter=milter, pretend=pretend,
-                          logger=logger, **self._args)

pymodmilter/conditions.py

@@ -1,134 +0,0 @@
-# PyMod-Milter is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# PyMod-Milter is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with PyMod-Milter.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-import logging
-import re
-
-from netaddr import IPAddress, IPNetwork, AddrFormatError
-
-
-class Conditions:
-    """Conditions to implement conditions for rules and actions."""
-
-    def __init__(self, local_addrs, args, logger=None):
-        if logger is None:
-            logger = logging.getLogger(__name__)
-
-        self._local_addrs = []
-        self.logger = logger
-        self._args = {}
-
-        try:
-            for addr in local_addrs:
-                self._local_addrs.append(IPNetwork(addr))
-        except AddrFormatError as e:
-            raise RuntimeError(f"invalid address in local_addrs: {e}")
-
-        try:
-            if "local" in args:
-                logger.debug(f"condition: local = {args['local']}")
-                self._args["local"] = args["local"]
-
-            if "hosts" in args:
-                logger.debug(f"condition: hosts = {args['hosts']}")
-                self._args["hosts"] = []
-                try:
-                    for host in args["hosts"]:
-                        self._args["hosts"].append(IPNetwork(host))
-                except AddrFormatError as e:
-                    raise RuntimeError(f"invalid address in hosts: {e}")
-
-            if "envfrom" in args:
-                logger.debug(f"condition: envfrom = {args['envfrom']}")
-                try:
-                    self._args["envfrom"] = re.compile(
-                        args["envfrom"], re.IGNORECASE)
-                except re.error as e:
-                    raise RuntimeError(f"unable to parse envfrom regex: {e}")
-
-            if "envto" in args:
-                logger.debug(f"condition: envto = {args['envto']}")
-                try:
-                    self._args["envto"] = re.compile(
-                        args["envto"], re.IGNORECASE)
-                except re.error as e:
-                    raise RuntimeError(f"unable to parse envto regex: {e}")
-
-        except KeyError as e:
-            raise RuntimeError(
-                f"mandatory argument not found: {e}")
-
-    def match(self, args):
-        if "host" in args:
-            ip = IPAddress(args["host"])
-
-            if "local" in self._args:
-                is_local = False
-                for addr in self._local_addrs:
-                    if ip in addr:
-                        is_local = True
-                        break
-
-                if is_local != self._args["local"]:
-                    self.logger.debug(
-                        f"ignore host {args['host']}, "
-                        f"condition local does not match")
-                    return False
-
-                self.logger.debug(
-                    f"condition local matches for host {args['host']}")
-
-            if "hosts" in self._args:
-                found = False
-                for addr in self._args["hosts"]:
-                    if ip in addr:
-                        found = True
-                        break
-
-                if not found:
-                    self.logger.debug(
-                        f"ignore host {args['host']}, "
-                        f"condition hosts does not match")
-                    return False
-
-                self.logger.debug(
-                    f"condition hosts matches for host {args['host']}")
-
-        if "envfrom" in args and "envfrom" in self._args:
-            if not self._args["envfrom"].match(args["envfrom"]):
-                self.logger.debug(
-                    f"ignore envelope-from address {args['envfrom']}, "
-                    f"condition envfrom does not match")
-                return False
-
-            self.logger.debug(
-                f"condition envfrom matches for "
-                f"envelope-from address {args['envfrom']}")
-
-        if "envto" in args and "envto" in self._args:
-            if not isinstance(args["envto"], list):
-                args["envto"] = [args["envto"]]
-
-            for envto in args["envto"]:
-                if not self._args["envto"].match(envto):
-                    self.logger.debug(
-                        f"ignore envelope-to address {args['envto']}, "
-                        f"condition envto does not match")
-                    return False
-
-            self.logger.debug(
-                f"condition envto matches for "
-                f"envelope-to address {args['envto']}")
-
-        return True

pymodmilter/docs/pymodmilter.conf.example

@@ -1,214 +0,0 @@
-# This is an example /etc/pymodmilter.conf file.
-# Copy it into place before use.
-#
-# The file is in JSON format.
-#
-# The global option 'log' can be overriden per rule or per modification.
-#
-{
-    # Section: global
-    # Notes:   Global options.
-    #
-    "global": {
-        # Option: socket
-        # Type:   String
-        # Notes:  The socket used to communicate with the MTA.
-        #
-        #         Examples:
-        #         unix:/path/to/socket             a named pipe
-        #         inet:8899                        listen on ANY interface
-        #         inet:8899@localhost              listen on a specific interface
-        #         inet6:8899                       listen on ANY interface
-        #         inet6:8899@[2001:db8:1234::1]    listen on a specific interface
-        # Value:  [ SOCKET ]
-        "socket": "inet:8898@127.0.0.1",
-
-        # Option: local_addrs
-        # Type:   List
-        # Notes:  A list of local hosts and networks.
-        # Value:  [ LIST ]
-        #
-        "local_addrs": ["::1/128", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"],
-
-        # Option: loglevel
-        # Type:   String
-        # Notes:  Set loglevel for rules and actions.
-        # Value:  [ error | warning | info | debug ]
-        #
-        "loglevel": "info",
-
-        # Option: pretend
-        # Type:   Bool
-        # Notes:  Just pretend to do the actions, for test purposes.
-        # Value:  [ true | false ]
-        #
-        "pretend": true
-    },
-
-    # Section: rules
-    # Notes:   Rules and related actions.
-    #
-    "rules": [
-        {
-            # Option: name
-            # Type:   String
-            # Notes:  Name of the rule.
-            # Value:  [ NAME ]
-            #
-            "name": "myrule",
-
-            # Section: conditions
-            # Notes:   Optional conditions to process the rule.
-            #          If multiple conditions are set, they all
-            #          have to be true to process the rule.
-            #
-            "conditions": {
-                # Option: local
-                # Type:   Bool
-                # Notes:  Condition wheter the senders host address is listed in local_addrs.
-                # Value:  [ true | false ]
-                #
-                "local": false,
-
-                # Option: hosts
-                # Type:   String
-                # Notes:  Condition wheter the senders host address is listed in this list.
-                # Value: [ LIST ]
-                #
-                "hosts": [ "127.0.0.1" ],
-
-                # Option: envfrom
-                # Type:   String
-                # Notes:  Condition wheter the envelop-from address matches this regular expression.
-                # Value:  [ REGEX ]
-                #
-                "envfrom": "^.+@mypartner\\.com$",
-
-                # Option: envto
-                # Type:   String
-                # Notes:  Condition wheter the envelop-to address matches this regular expression.
-                # Value:  [ REGEX ]
-                #
-                "envto": "^postmaster@.+$"
-            },
-
-            # Section: actions
-            # Notes:   Actions of the rule.
-            #
-            "actions": [
-                {
-                    # Option: name
-                    # Type:   String
-                    # Notes:  Name of the modification.
-                    # Value:  [ NAME ]
-                    #
-                    "name": "add_test_header",
-
-                    # Option: type
-                    # Type:   String
-                    # Notes:  Type of the modification.
-                    # Value:  [ add_header | del_header | mod_header ]
-                    #
-                    "type": "add_header",
-
-                    # Option: header
-                    # Type:   String
-                    # Notes:  Name of the header.
-                    # Value:  [ NAME ]
-                    #
-                    "header": "X-Test-Header",
-
-                    # Option: value
-                    # Type:   String
-                    # Notes:  Value of the header.
-                    # Value:  [ VALUE ]
-                    #
-                    "value": "true"
-                }, {
-                    "name": "modify_subject",
-
-                    "type": "mod_header",
-
-                    # Option: header
-                    # Type:   String
-                    # Notes:  Regular expression to match against header lines (e.g. Subject: Test-Subject).
-                    # Value:  [ REGEX ]
-                    #
-                    "header": "^Subject$",
-
-                    # Option: search
-                    # Type:   String
-                    # Notes:  Regular expression to match against the headers value.
-                    # Values: [ VALUE ]
-                    #
-                    "search": "(?P<subject>.*)",
-
-                    # Option: value
-                    # Type:   String
-                    # Notes:  New value of the header.
-                    # Values: [ VALUE ]
-                    "value": "[EXTERNAL] \\g<subject>"
-                }, {
-                    "name": "delete_received_header",
-
-                    "type": "del_header",
-
-                    # Option: header
-                    # Type:   String
-                    # Notes:  Regular expression to match against header lines (e.g. Subject: Test-Subject).
-                    # Value:  [ REGEX ]
-                    #
-                    "header": "^Received$"
-                }, {
-                    "name": "add_disclaimer",
-
-                    "type": "add_disclaimer",
-
-                    # Option: action
-                    # Type:   String
-                    # Notes:  Action to perform with the disclaimer.
-                    # Value:  [ append | prepend ]
-                    #
-                    "action": "prepend",
-
-                    # Option: html_file
-                    # Type:   String
-                    # Notes:  Path to a file which contains the html representation of the disclaimer.
-                    # Value:  [ FILE_PATH ]
-                    #
-                    "html_file": "/etc/pymodmilter/templates/disclaimer_html.template",
-
-                    # Option: text_file
-                    # Type:   String
-                    # Notes:  Path to a file which contains the text representation of the disclaimer.
-                    # Value:  [ FILE_PATH ]
-                    #
-                    "text_file": "/etc/pymodmilter/templates/disclaimer_text.template",
-
-                    # Option: error_policy
-                    # Type:   String
-                    # Notes:  Set what should be done if the modification fails (e.g. no message body present).
-                    # Value:  [ wrap | ignore | reject ]
-                    #
-                    "error_policy": "wrap"
-                }, {
-                    "name": "store_message",
-
-                    "type": "store",
-
-                    # Option: storage_type
-                    # Type:   String
-                    # Notes:  The storage type used to store e-mails.
-                    # Value: [ file ]
-                    "storage_type": "file",
-
-                    # Option: directory
-                    # Type:   String
-                    # Notes:  Directory used to store e-mails.
-                    # Value: [ file ]
-                    "directory": "/mnt/messages"
-                }
-            ]
-        }
-    ]
-}

pymodmilter/run.py

@@ -1,270 +0,0 @@
-# PyMod-Milter is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# PyMod-Milter is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with PyMod-Milter.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-import Milter
-import argparse
-import logging
-import logging.handlers
-import sys
-
-from json import loads
-from re import sub
-
-from pymodmilter import Rule, ModifyMilter
-from pymodmilter import __version__ as version
-from pymodmilter.actions import Action
-
-
-def main():
-    "Run PyMod-Milter."
-    parser = argparse.ArgumentParser(
-        description="PyMod milter daemon",
-        formatter_class=lambda prog: argparse.HelpFormatter(
-            prog, max_help_position=45, width=140))
-
-    parser.add_argument(
-        "-c", "--config", help="Config file to read.",
-        default="/etc/pymodmilter/pymodmilter.conf")
-
-    parser.add_argument(
-        "-s",
-        "--socket",
-        help="Socket used to communicate with the MTA.",
-        default="")
-
-    parser.add_argument(
-        "-d",
-        "--debug",
-        help="Log debugging messages.",
-        action="store_true")
-
-    parser.add_argument(
-        "-t",
-        "--test",
-        help="Check configuration.",
-        action="store_true")
-
-    parser.add_argument(
-        "-v", "--version",
-        help="Print version.",
-        action="version",
-        version=f"%(prog)s ({version})")
-
-    args = parser.parse_args()
-
-    loglevels = {
-        "error": logging.ERROR,
-        "warning": logging.WARNING,
-        "info": logging.INFO,
-        "debug": logging.DEBUG
-    }
-
-    root_logger = logging.getLogger()
-    root_logger.setLevel(logging.DEBUG)
-
-    # setup console log
-    stdouthandler = logging.StreamHandler(sys.stdout)
-    stdouthandler.setFormatter(
-        logging.Formatter("%(asctime)s - %(levelname)s: %(message)s"))
-    root_logger.addHandler(stdouthandler)
-
-    # setup syslog
-    sysloghandler = logging.handlers.SysLogHandler(
-        address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL)
-    sysloghandler.setFormatter(
-        logging.Formatter("pymodmilter: %(message)s"))
-    root_logger.addHandler(sysloghandler)
-
-    logger = logging.getLogger(__name__)
-
-    if not args.debug:
-        logger.setLevel(logging.INFO)
-
-    try:
-        try:
-            with open(args.config, "r") as fh:
-                config = sub(r"(?m)^\s*#.*\n?", "", fh.read())
-                config = loads(config)
-        except Exception as e:
-            for num, line in enumerate(config.splitlines()):
-                logger.error(f"{num+1}: {line}")
-            raise RuntimeError(
-                f"unable to parse config file: {e}")
-
-        if "global" not in config:
-            config["global"] = {}
-
-        if args.debug:
-            loglevel = logging.DEBUG
-        else:
-            if "loglevel" not in config["global"]:
-                config["global"]["loglevel"] = "info"
-            loglevel = loglevels[config["global"]["loglevel"]]
-
-        logger.setLevel(loglevel)
-
-        logger.debug("prepar milter configuration")
-
-        if "pretend" not in config["global"]:
-            config["global"]["pretend"] = False
-
-        if args.socket:
-            socket = args.socket
-        elif "socket" in config["global"]:
-            socket = config["global"]["socket"]
-        else:
-            raise RuntimeError(
-                "listening socket is neither specified on the command line "
-                "nor in the configuration file")
-
-        if "local_addrs" in config["global"]:
-            local_addrs = config["global"]["local_addrs"]
-        else:
-            local_addrs = [
-                "::1/128",
-                "127.0.0.0/8",
-                "10.0.0.0/8",
-                "172.16.0.0/12",
-                "192.168.0.0/16"]
-
-        if "rules" not in config:
-            raise RuntimeError(
-                "mandatory config section 'rules' not found")
-
-        if not config["rules"]:
-            raise RuntimeError("no rules configured")
-
-        logger.debug("initialize rules ...")
-
-        rules = []
-        for rule_idx, rule in enumerate(config["rules"]):
-            if "name" in rule:
-                rule_name = rule["name"]
-            else:
-                rule_name = f"Rule #{rule_idx}"
-
-            logger.debug(f"prepare rule {rule_name} ...")
-
-            if "actions" not in rule:
-                raise RuntimeError(
-                    f"{rule_name}: mandatory config "
-                    f"section 'actions' not found")
-
-            if not rule["actions"]:
-                raise RuntimeError("{rule_name}: no actions configured")
-
-            if args.debug:
-                rule_loglevel = logging.DEBUG
-            elif "loglevel" in rule:
-                rule_loglevel = loglevels[rule["loglevel"]]
-            else:
-                rule_loglevel = loglevels[config["global"]["loglevel"]]
-
-            if "pretend" in rule:
-                rule_pretend = rule["pretend"]
-            else:
-                rule_pretend = config["global"]["pretend"]
-
-            actions = []
-            for action_idx, action in enumerate(rule["actions"]):
-                if "name" in action:
-                    action_name = f"{rule_name}: {action['name']}"
-                else:
-                    action_name = f"Action #{action_idx}"
-
-                if args.debug:
-                    action_loglevel = logging.DEBUG
-                elif "loglevel" in action:
-                    action_loglevel = loglevels[action["loglevel"]]
-                else:
-                    action_loglevel = rule_loglevel
-
-                if "pretend" in action:
-                    action_pretend = action["pretend"]
-                else:
-                    action_pretend = rule_pretend
-
-                if "type" not in action:
-                    raise RuntimeError(
-                        f"{rule_name}: {action_name}: mandatory config "
-                        f"section 'actions' not found")
-
-                if "conditions" not in action:
-                    action["conditions"] = {}
-
-                try:
-                    actions.append(
-                        Action(
-                            name=action_name,
-                            local_addrs=local_addrs,
-                            conditions=action["conditions"],
-                            action_type=action["type"],
-                            args=action,
-                            loglevel=action_loglevel,
-                            pretend=action_pretend))
-                except RuntimeError as e:
-                    logger.error(f"{action_name}: {e}")
-                    sys.exit(253)
-
-            if "conditions" not in rule:
-                rule["conditions"] = {}
-
-            try:
-                rules.append(
-                    Rule(
-                        name=rule_name,
-                        local_addrs=local_addrs,
-                        conditions=rule["conditions"],
-                        actions=actions,
-                        loglevel=rule_loglevel,
-                        pretend=rule_pretend))
-            except RuntimeError as e:
-                logger.error(f"{rule_name}: {e}")
-                sys.exit(254)
-
-    except RuntimeError as e:
-        logger.error(e)
-        sys.exit(255)
-
-    if args.test:
-        print("Configuration ok")
-        sys.exit(0)
-
-    # setup console log for runtime
-    formatter = logging.Formatter("%(asctime)s - %(levelname)s: %(message)s")
-    stdouthandler.setFormatter(formatter)
-    stdouthandler.setLevel(logging.DEBUG)
-
-    logger.info("pymodmilter starting")
-    ModifyMilter.set_rules(rules)
-    ModifyMilter.set_loglevel(loglevel)
-
-    # register milter factory class
-    Milter.factory = ModifyMilter
-    Milter.set_exception_policy(Milter.TEMPFAIL)
-
-    if args.debug:
-        Milter.setdbg(1)
-
-    rc = 0
-    try:
-        Milter.runmilter("pymodmilter", socketname=socket, timeout=30)
-    except Milter.milter.error as e:
-        logger.error(e)
-        rc = 255
-    sys.exit(rc)
-
-
-if __name__ == "__main__":
-    main()

pyquarantine/__init__.py

@@ -0,0 +1,336 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = [
+    "action",
+    "base",
+    "cli",
+    "conditions",
+    "config",
+    "mailer",
+    "modify",
+    "notify",
+    "rule",
+    "run",
+    "storage",
+    "whitelist",
+    "QuarantineMilter"]
+
+__version__ = "2.0.5"
+
+from pyquarantine import _runtime_patches
+
+import Milter
+import logging
+
+from Milter.utils import parse_addr
+from collections import defaultdict
+from copy import copy
+from email import message_from_binary_file
+from email.header import Header, decode_header, make_header
+from email.headerregistry import AddressHeader, _default_header_map
+from email.policy import SMTP
+from io import BytesIO
+from netaddr import IPNetwork, AddrFormatError
+
+from pyquarantine.base import CustomLogger, MilterMessage
+from pyquarantine.base import replace_illegal_chars
+from pyquarantine.rule import Rule
+
+
+class QuarantineMilter(Milter.Base):
+    """QuarantineMilter based on Milter.Base to implement
+       milter communication"""
+
+    _rules = []
+    _loglevel = logging.INFO
+    _addr_fields = [f for f, v in _default_header_map.items()
+                    if issubclass(v, AddressHeader)]
+
+    @staticmethod
+    def set_config(cfg, debug):
+        QuarantineMilter._loglevel = cfg.get_loglevel(debug)
+
+        try:
+            local_addrs = []
+            for addr in cfg["local_addrs"]:
+                local_addrs.append(IPNetwork(addr))
+        except AddrFormatError as e:
+            raise RuntimeError(e)
+
+        logger = logging.getLogger(__name__)
+        logger.setLevel(QuarantineMilter._loglevel)
+        for idx, rule_cfg in enumerate(cfg["rules"]):
+            rule = Rule(rule_cfg, local_addrs, debug)
+            logger.debug(rule)
+            QuarantineMilter._rules.append(rule)
+
+    def __init__(self):
+        self.logger = logging.getLogger(__name__)
+        self.logger.setLevel(QuarantineMilter._loglevel)
+
+    def addheader(self, field, value, idx=-1):
+        value = replace_illegal_chars(Header(s=value).encode())
+        self.logger.debug(f"addheader: {field}: {value}")
+        super().addheader(field, value, idx)
+
+    def chgheader(self, field, value, idx=1):
+        value = replace_illegal_chars(Header(s=value).encode())
+        if value:
+            self.logger.debug(f"chgheader: {field}[{idx}]: {value}")
+        else:
+            self.logger.debug(f"delheader: {field}[{idx}]")
+        super().chgheader(field, idx, value)
+
+    def msg_as_bytes(self):
+        try:
+            data = self.msg.as_bytes()
+        except Exception as e:
+            self.logger.warning(f"unable to serialize message as bytes: {e}")
+            try:
+                self.logger.warning("try to serialize as str and encode")
+                data = self.msg.as_string().encode(errors="replace")
+            except Exception as e:
+                self.logger.error(
+                    f"unable to serialize message, giving up: {e}")
+                raise e
+
+        return data
+
+    def update_headers(self, old_headers):
+        if self.msg.is_multipart() and not self.msg["MIME-Version"]:
+            self.msg.add_header("MIME-Version", "1.0")
+
+        # serialize the message object so it updates its internal strucure
+        self.msg_as_bytes()
+
+        headers = set(self.msg.items())
+        to_remove = list(set(old_headers) - headers)
+        to_add = list(headers - set(old_headers))
+
+        idx = defaultdict(int)
+        for field, value in old_headers:
+            field_lower = field.lower()
+            if (field, value) in to_remove:
+                self.chgheader(field, "", idx=idx[field_lower] + 1)
+                continue
+            idx[field_lower] += 1
+
+        for field, value in to_add:
+            self.addheader(field, value)
+
+    def replacebody(self):
+        self._body_changed = True
+
+    def _replacebody(self):
+        if not self._body_changed:
+            return
+        data = self.msg_as_bytes()
+        body_pos = data.find(b"\r\n\r\n") + 4
+        self.logger.debug("replace body")
+        super().replacebody(data[body_pos:])
+        del data
+
+    def delrcpt(self, rcpts):
+        "Remove recipient. May be called from eom callback only."
+        if not isinstance(rcpts, list):
+            rcpts = [rcpts]
+        for rcpt in rcpts:
+            self.logger.debug(f"delrcpt: {rcpt}")
+            self.msginfo["rcpts"].remove(rcpt)
+            super().delrcpt(rcpt)
+
+    def connect(self, IPname, family, hostaddr):
+        try:
+            if hostaddr is None:
+                self.logger.error(f"received invalid host address {hostaddr}, "
+                                  f"unable to proceed")
+                return Milter.TEMPFAIL
+
+            self.IP = hostaddr[0]
+            self.port = hostaddr[1]
+            self.logger.debug(
+                f"accepted milter connection from {self.IP} "
+                f"port {self.port}")
+
+            # pre-filter rules and actions by the host condition
+            # also check if the mail body is needed by any upcoming action.
+            self.rules = []
+            self._headersonly = True
+            for rule in QuarantineMilter._rules:
+                if rule.conditions is None or \
+                        rule.conditions.match_host(self.IP):
+                    actions = []
+                    for action in rule.actions:
+                        if action.conditions is None or \
+                                action.conditions.match_host(self.IP):
+                            actions.append(action)
+                            if not action.headersonly():
+                                self._headersonly = False
+
+                    if actions:
+                        # copy needed rules to preserve configured actions
+                        rule = copy(rule)
+                        rule.actions = actions
+                        self.rules.append(rule)
+
+            if not self.rules:
+                self.logger.debug(
+                    "host is ignored by all rules, skip further processing")
+                return Milter.ACCEPT
+
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in connect method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.CONTINUE
+
+    def hello(self, heloname):
+        try:
+            self.heloname = heloname
+            self.logger.debug(f"received HELO name: {heloname}")
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in hello method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.CONTINUE
+
+    @Milter.decode("replace")
+    def envfrom(self, mailfrom, *str):
+        try:
+            self.mailfrom = "@".join(parse_addr(mailfrom)).lower()
+            self.rcpts = set()
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in envfrom method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.CONTINUE
+
+    @Milter.decode("replace")
+    def envrcpt(self, to, *str):
+        try:
+            self.rcpts.add("@".join(parse_addr(to)).lower())
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in envrcpt method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.CONTINUE
+
+    def data(self):
+        try:
+            self.qid = self.getsymval('i')
+            self.logger = CustomLogger(
+                self.logger, {"qid": self.qid, "name": "milter"})
+            self.logger.debug("received queue-id from MTA")
+            self.fp = BytesIO()
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in data method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.CONTINUE
+
+    @Milter.decode("replace")
+    def header(self, field, value):
+        try:
+            # remove CR and LF from address fields, otherwise pythons
+            # email library throws an exception
+            if field.lower() in QuarantineMilter._addr_fields:
+                try:
+                    v = str(make_header(decode_header(value)))
+                except Exception as e:
+                    self.logger.error(
+                        f"unable to decode field '{field}': {e}")
+                else:
+                    if any(c in v for c in ["\r", "\n"]):
+                        v = v.replace("\r", "").replace("\n", "")
+                        value = Header(s=v).encode()
+
+            self.fp.write(field.encode() + b": " + value.encode() + b"\r\n")
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in header method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.CONTINUE
+
+    def eoh(self):
+        try:
+            self.fp.write(b"\r\n")
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in eoh method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.CONTINUE
+
+    def body(self, chunk):
+        try:
+            if not self._headersonly:
+                self.fp.write(chunk)
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in body method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.CONTINUE
+
+    def eom(self):
+        try:
+            # msg and msginfo contain the runtime data that
+            # is read/modified by actions
+            self.fp.seek(0)
+            self.msg = message_from_binary_file(
+                self.fp, _class=MilterMessage, policy=SMTP.clone(
+                    refold_source='none'))
+            self.msginfo = {
+                "mailfrom": self.mailfrom,
+                "rcpts": [*self.rcpts],
+                "vars": {}}
+
+            self._body_changed = False
+            milter_action = None
+            for rule in self.rules:
+                milter_action = rule.execute(self)
+                self.logger.debug(
+                    f"current template variables: {self.msginfo['vars']}")
+                if milter_action is not None:
+                    break
+                elif not self.msginfo["rcpts"]:
+                    milter_action = ("DISCARD", None)
+                    break
+
+            if milter_action is None:
+                self._replacebody()
+            else:
+                action, reason = milter_action
+                if action == "ACCEPT":
+                    self._replacebody()
+                    return Milter.ACCEPT
+                elif action == "REJECT":
+                    self.setreply("554", "5.7.0", reason)
+                    return Milter.REJECT
+                elif action == "DISCARD":
+                    return Milter.DISCARD
+
+        except Exception as e:
+            self.logger.exception(
+                f"an exception occured in eom method: {e}")
+            return Milter.TEMPFAIL
+
+        return Milter.ACCEPT

pyquarantine/_install.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+
+# pyinotifyd is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyinotifyd is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyinotifyd.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import filecmp
+import logging
+import os
+import shutil
+import sys
+
+
+SYSTEMD_PATHS = ["/lib/systemd/system", "/usr/lib/systemd/system"]
+OPENRC = "/sbin/openrc"
+
+
+def _systemd_files(pkg_dir, name):
+    for path in SYSTEMD_PATHS:
+        if os.path.isdir(path):
+            break
+
+    return [
+        (f"{pkg_dir}/misc/systemd/{name}-milter.service",
+            f"{path}/{name}-milter.service", True)]
+
+
+def _openrc_files(pkg_dir, name):
+    return [
+        (f"{pkg_dir}/misc/openrc/{name}-milter.initd", f"/etc/init.d/{name}-milter", True),
+        (f"{pkg_dir}/misc/openrc/{name}-milter.confd", f"/etc/conf.d/{name}-milter", False)]
+
+
+def _config_files(pkg_dir, name):
+    return [
+        (f"{pkg_dir}/misc/{name}.conf.default", f"/etc/{name}/{name}.conf.default", False),
+        (f"{pkg_dir}/misc/templates/removed.png", f"/etc/{name}/templates/removed.png", False),
+        (f"{pkg_dir}/misc/templates/disclaimer_html.template", f"/etc/{name}/templates/disclaimer_html.template", False),
+        (f"{pkg_dir}/misc/templates/disclaimer_text.template", f"/etc/{name}/templates/disclaimer_text.template", False),
+        (f"{pkg_dir}/misc/templates/notification.template", f"/etc/{name}/templates/notification.template", False)]
+
+
+def _install_files(files):
+    for src, dst, force in files:
+        if os.path.exists(dst):
+            if os.path.isdir(dst):
+                logging.error(
+                    " => unable to copy file, destination path is a directory")
+                continue
+            elif not force:
+                logging.info(f" => file {dst} already exists")
+                continue
+
+        try:
+            logging.info(f" => install file {dst}")
+            shutil.copy2(src, dst)
+        except Exception as e:
+            logging.error(f" => unable to install file {dst}: {e}")
+
+
+def _uninstall_files(files):
+    for src, dst, force in files:
+        if not os.path.isfile(dst):
+            continue
+
+        if not force and not filecmp.cmp(src, dst, shallow=True):
+            logging.warning(
+                f" => keep modified file {dst}, "
+                f"you have to remove it manually")
+            continue
+
+        try:
+            logging.info(f" => uninstall file {dst}")
+            os.remove(dst)
+        except Exception as e:
+            logging.error(f" => unable to uninstall file {dst}: {e}")
+
+
+def _create_dir(path):
+    if os.path.isdir(path):
+        logging.info(f" => directory {path} already exists")
+    else:
+        try:
+            logging.info(f" => create directory {path}")
+            os.mkdir(path)
+        except Exception as e:
+            logging.error(f" => unable to create directory {path}: {e}")
+            return False
+
+    return True
+
+
+def _delete_dir(path):
+    if os.path.isdir(path):
+        if not os.listdir(path):
+            try:
+                logging.info(f" => delete directory {path}")
+                os.rmdir(path)
+            except Exception as e:
+                logging.error(f" => unable to delete directory {path}: {e}")
+        else:
+            logging.warning(f" => keep non-empty directory {path}")
+
+
+def _check_root():
+    if os.getuid() != 0:
+        logging.error("you need to have root privileges, please try again")
+        return False
+
+    return True
+
+
+def _check_systemd():
+    for path in SYSTEMD_PATHS:
+        systemd = os.path.isdir(path)
+        if systemd:
+            break
+
+    if systemd:
+        logging.info("systemd detected")
+
+    return systemd
+
+
+def _check_openrc():
+    openrc = os.path.isfile(OPENRC) and os.access(OPENRC, os.X_OK)
+    if openrc:
+        logging.info("openrc detected")
+
+    return openrc
+
+
+def install(name):
+    if not _check_root():
+        sys.exit(2)
+
+    pkg_dir = os.path.dirname(__file__)
+
+    if _check_systemd():
+        _install_files(_systemd_files(pkg_dir, name))
+
+    if _check_openrc():
+        _install_files(_openrc_files(pkg_dir, name))
+
+    for d in [f"/etc/{name}", f"/etc/{name}/templates"]:
+        if not _create_dir(d):
+            logging.error(" => unable to create config dir, giving up ...")
+            sys.exit(3)
+    _install_files(_config_files(pkg_dir, name))
+
+    logging.info(f"{name} successfully installed")
+
+
+def uninstall(name):
+    if not _check_root():
+        sys.exit(2)
+
+    pkg_dir = os.path.dirname(__file__)
+
+    _uninstall_files(_systemd_files(pkg_dir, name))
+    _uninstall_files(_openrc_files(pkg_dir, name))
+    _uninstall_files(_config_files(pkg_dir, name))
+
+    _delete_dir(f"/etc/{name}/templates")
+    _delete_dir(f"/etc/{name}")
+
+    logging.info(f"{name} successfully uninstalled")

pyquarantine/_runtime_patches.py

@@ -0,0 +1,231 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from sys import version_info
+import encodings
+
+
+#####################################
+#    patch pythons email library    #
+#####################################
+#
+# https://bugs.python.org/issue27257
+# https://bugs.python.org/issue30988
+#
+# fix: https://github.com/python/cpython/pull/15600
+
+import email._header_value_parser
+from email._header_value_parser import TokenList, NameAddr
+from email._header_value_parser import get_display_name, get_angle_addr
+from email._header_value_parser import get_cfws, errors
+from email._header_value_parser import CFWS_LEADER, PHRASE_ENDS
+
+
+class DisplayName(email._header_value_parser.DisplayName):
+    @property
+    def display_name(self):
+        res = TokenList(self)
+        if len(res) == 0:
+            return res.value
+        if res[0].token_type == 'cfws':
+            res.pop(0)
+        else:
+            if isinstance(res[0], TokenList) and \
+                    res[0][0].token_type == 'cfws':
+                res[0] = TokenList(res[0][1:])
+        if res[-1].token_type == 'cfws':
+            res.pop()
+        else:
+            if isinstance(res[-1], TokenList) and \
+                    res[-1][-1].token_type == 'cfws':
+                res[-1] = TokenList(res[-1][:-1])
+        return res.value
+
+
+def get_name_addr(value):
+    """ name-addr = [display-name] angle-addr
+
+    """
+    name_addr = NameAddr()
+    # Both the optional display name and the angle-addr can start with cfws.
+    leader = None
+    if value[0] in CFWS_LEADER:
+        leader, value = get_cfws(value)
+        if not value:
+            raise errors.HeaderParseError(
+                "expected name-addr but found '{}'".format(leader))
+    if value[0] != '<':
+        if value[0] in PHRASE_ENDS:
+            raise errors.HeaderParseError(
+                "expected name-addr but found '{}'".format(value))
+        token, value = get_display_name(value)
+        if not value:
+            raise errors.HeaderParseError(
+                "expected name-addr but found '{}'".format(token))
+        if leader is not None:
+            if isinstance(token[0], TokenList):
+                token[0][:0] = [leader]
+            else:
+                token[:0] = [leader]
+            leader = None
+        name_addr.append(token)
+    token, value = get_angle_addr(value)
+    if leader is not None:
+        token[:0] = [leader]
+    name_addr.append(token)
+    return name_addr, value
+
+
+setattr(email._header_value_parser, "DisplayName", DisplayName)
+setattr(email._header_value_parser, "get_name_addr", get_name_addr)
+
+
+# https://bugs.python.org/issue42484
+#
+# fix: https://github.com/python/cpython/pull/24669
+
+from email._header_value_parser import DOT, ObsLocalPart, ValueTerminal, get_word
+
+
+def get_obs_local_part(value):
+    """ obs-local-part = word *("." word)
+    """
+    obs_local_part = ObsLocalPart()
+    last_non_ws_was_dot = False
+    while value and (value[0]=='\\' or value[0] not in PHRASE_ENDS):
+        if value[0] == '.':
+            if last_non_ws_was_dot:
+                obs_local_part.defects.append(errors.InvalidHeaderDefect(
+                    "invalid repeated '.'"))
+            obs_local_part.append(DOT)
+            last_non_ws_was_dot = True
+            value = value[1:]
+            continue
+        elif value[0]=='\\':
+            obs_local_part.append(ValueTerminal(value[0],
+                                                'misplaced-special'))
+            value = value[1:]
+            obs_local_part.defects.append(errors.InvalidHeaderDefect(
+                "'\\' character outside of quoted-string/ccontent"))
+            last_non_ws_was_dot = False
+            continue
+        if obs_local_part and obs_local_part[-1].token_type != 'dot':
+            obs_local_part.defects.append(errors.InvalidHeaderDefect(
+                "missing '.' between words"))
+        try:
+            token, value = get_word(value)
+            last_non_ws_was_dot = False
+        except errors.HeaderParseError:
+            if value[0] not in CFWS_LEADER:
+                raise
+            token, value = get_cfws(value)
+        obs_local_part.append(token)
+    if not obs_local_part:
+        return obs_local_part, value
+    if (obs_local_part[0].token_type == 'dot' or
+            obs_local_part[0].token_type=='cfws' and
+            obs_local_part[1].token_type=='dot'):
+        obs_local_part.defects.append(errors.InvalidHeaderDefect(
+            "Invalid leading '.' in local part"))
+    if (obs_local_part[-1].token_type == 'dot' or
+            obs_local_part[-1].token_type=='cfws' and
+            obs_local_part[-2].token_type=='dot'):
+        obs_local_part.defects.append(errors.InvalidHeaderDefect(
+            "Invalid trailing '.' in local part"))
+    if obs_local_part.defects:
+        obs_local_part.token_type = 'invalid-obs-local-part'
+    return obs_local_part, value
+
+
+setattr(email._header_value_parser, "get_obs_local_part", get_obs_local_part)
+
+
+#######################################
+#  add charset alias for windows-874  #
+#######################################
+#
+# https://bugs.python.org/issue17254
+#
+# fix: https://github.com/python/cpython/pull/10237
+
+aliases = encodings.aliases.aliases
+
+for alias in ["windows-874", "windows_874"]:
+    if alias not in aliases:
+        aliases[alias] = "cp874"
+
+setattr(encodings.aliases, "aliases", aliases)
+
+
+if version_info.major == 3 and version_info.minor < 10:
+    # https://bugs.python.org/issue30681
+    #
+    # fix: https://github.com/python/cpython/pull/22090
+
+    import email.errors
+    from email.errors import HeaderDefect
+
+
+    class InvalidDateDefect(HeaderDefect):
+        """Header has unparseable or invalid date"""
+
+
+    setattr(email.errors, "InvalidDateDefect", InvalidDateDefect)
+
+
+    import email.utils
+    from email.utils import _parsedate_tz
+    import datetime
+
+
+    def parsedate_to_datetime(data):
+        parsed_date_tz = _parsedate_tz(data)
+        if parsed_date_tz is None:
+            raise ValueError('Invalid date value or format "%s"' % str(data))
+        *dtuple, tz = parsed_date_tz
+        if tz is None:
+            return datetime.datetime(*dtuple[:6])
+        return datetime.datetime(*dtuple[:6],
+                tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
+
+
+    setattr(email.utils, "parsedate_to_datetime", parsedate_to_datetime)
+
+
+    import email.headerregistry
+    from email import utils, _header_value_parser as parser
+
+    @classmethod
+    def parse(cls, value, kwds):
+        if not value:
+            kwds['defects'].append(errors.HeaderMissingRequiredValue())
+            kwds['datetime'] = None
+            kwds['decoded'] = ''
+            kwds['parse_tree'] = parser.TokenList()
+            return
+        if isinstance(value, str):
+            kwds['decoded'] = value
+            try:
+                value = utils.parsedate_to_datetime(value)
+            except ValueError:
+                kwds['defects'].append(errors.InvalidDateDefect('Invalid date value or format'))
+                kwds['datetime'] = None
+                kwds['parse_tree'] = parser.TokenList()
+                return
+        kwds['datetime'] = value
+        kwds['decoded'] = utils.format_datetime(kwds['datetime'])
+        kwds['parse_tree'] = cls.value_parser(kwds['decoded'])
+
+
+    setattr(email.headerregistry.DateHeader, "parse", parse)

pyquarantine/action.py

@@ -0,0 +1,64 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = ["Action"]
+
+from pyquarantine import modify, notify, storage
+from pyquarantine.conditions import Conditions
+
+
+class Action:
+    """Action to implement a pre-configured action to perform on e-mails."""
+    ACTION_TYPES = {
+        "add_header": modify.Modify,
+        "mod_header": modify.Modify,
+        "del_header": modify.Modify,
+        "add_disclaimer": modify.Modify,
+        "rewrite_links": modify.Modify,
+        "store": storage.Store,
+        "notify": notify.Notify,
+        "quarantine": storage.Quarantine}
+
+    def __init__(self, cfg, local_addrs, debug):
+        self.cfg = cfg
+
+        self.conditions = cfg["conditions"] if "conditions" in cfg else None
+        if self.conditions is not None:
+            self.conditions["name"] = f"{cfg['name']}: conditions"
+            self.conditions["loglevel"] = cfg["loglevel"]
+            self.conditions = Conditions(self.conditions, local_addrs, debug)
+
+        action_type = cfg["type"]
+        self.action = self.ACTION_TYPES[action_type](
+            cfg, local_addrs, debug)
+
+    def __str__(self):
+        cfg = []
+        for key in ["name", "loglevel", "pretend", "type"]:
+            value = self.cfg[key]
+            cfg.append(f"{key}={value}")
+        if self.conditions is not None:
+            cfg.append(f"conditions={self.conditions}")
+        cfg.append(f"action={self.action}")
+        return "Action(" + ", ".join(cfg) + ")"
+
+    def headersonly(self):
+        """Return the needs of this action."""
+        return self.action._headersonly
+
+    def execute(self, milter):
+        """Execute configured action."""
+        if self.conditions is None or \
+                self.conditions.match(milter):
+            return self.action.execute(milter)

pyquarantine/base.py

@@ -0,0 +1,199 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = [
+    "CustomLogger",
+    "MilterMessage",
+    "replace_illegal_chars"]
+
+import logging
+
+from email.message import MIMEPart
+
+
+class CustomLogger(logging.LoggerAdapter):
+    def process(self, msg, kwargs):
+        if "name" in self.extra:
+            msg = f"{self.extra['name']}: {msg}"
+
+        if "qid" in self.extra:
+            msg = f"{self.extra['qid']}: {msg}"
+
+        if self.logger.getEffectiveLevel() != logging.DEBUG:
+            msg = msg.replace("\n", "").replace("\r", "")
+
+        return msg, kwargs
+
+
+class MilterMessage(MIMEPart):
+    def replace_header(self, _name, _value, idx=None):
+        _name = _name.lower()
+        counter = 0
+        for i, (k, v) in zip(range(len(self._headers)), self._headers):
+            if k.lower() == _name:
+                counter += 1
+                if not idx or counter == idx:
+                    self._headers[i] = self.policy.header_store_parse(
+                        k, _value)
+                    break
+
+        else:
+            raise KeyError(_name)
+
+    def remove_header(self, name, idx=None):
+        name = name.lower()
+        newheaders = []
+        counter = 0
+        for k, v in self._headers:
+            if k.lower() == name:
+                counter += 1
+                if counter != idx:
+                    newheaders.append((k, v))
+            else:
+                newheaders.append((k, v))
+
+        self._headers = newheaders
+
+    def _find_body_parent(self, part, preferencelist, parent=None):
+        if part.is_attachment():
+            return
+        maintype, subtype = part.get_content_type().split("/")
+        if maintype == "text":
+            if subtype in preferencelist:
+                yield(preferencelist.index(subtype), parent)
+            return
+        if maintype != "multipart" or not self.is_multipart():
+            return
+        if subtype != "related":
+            for subpart in part.iter_parts():
+                yield from self._find_body_parent(
+                    subpart, preferencelist, part)
+            return
+        if 'related' in preferencelist:
+            yield(preferencelist.index('related'), parent)
+        candidate = None
+        start = part.get_param('start')
+        if start:
+            for subpart in part.iter_parts():
+                if subpart['content-id'] == start:
+                    candidate = subpart
+                    break
+        if candidate is None:
+            subparts = part.get_payload()
+            candidate = subparts[0] if subparts else None
+        if candidate is not None:
+            yield from self._find_body_parent(candidate, preferencelist, part)
+
+    def get_body_parent(self, preferencelist=("related", "html", "plain")):
+        best_prio = len(preferencelist)
+        body_parent = None
+        for prio, parent in self._find_body_parent(self, preferencelist):
+            if prio < best_prio:
+                best_prio = prio
+                body_parent = parent
+                if prio == 0:
+                    break
+        return body_parent
+
+    def get_body_content(self, pref):
+        part = None
+        content = None
+        if not self.is_multipart() and \
+                self.get_content_type() == f"text/{pref}":
+            part = self
+        else:
+            part = self.get_body(preferencelist=(pref))
+
+        if part is not None:
+            content = part.get_content()
+
+        return (part, content)
+
+    def set_body(self, text_content=None, html_content=None):
+        parent = self.get_body_parent() or self
+        if "Content-Type" not in parent:
+            # set Content-Type header if not present, otherwise
+            # make_alternative and make_mixed skip the payload
+            parent["Content-Type"] = parent.get_content_type()
+
+        maintype, subtype = parent.get_content_type().split("/")
+        if not parent.is_multipart() or maintype != "multipart":
+            if maintype == "text" and subtype in ("html", "plain"):
+                parent.make_alternative()
+                maintype, subtype = ("multipart", "alternative")
+            else:
+                parent.make_mixed()
+                maintype, subtype = ("multipart", "mixed")
+
+        text_body = parent.get_body(preferencelist=("plain"))
+        html_body = parent.get_body(preferencelist=("html"))
+
+        if text_content is not None:
+            if text_body:
+                text_body.set_content(text_content)
+            else:
+                if not html_body or subtype == "alternative":
+                    inject_body_part(parent, text_content)
+                else:
+                    html_body.add_alternative(text_content)
+                text_body = parent.get_body(preferencelist=("plain"))
+
+        if html_content is not None:
+            if html_body:
+                html_body.set_content(html_content, subtype="html")
+            else:
+                if not text_body or subtype == "alternative":
+                    inject_body_part(parent, html_content, subtype="html")
+                else:
+                    text_body.add_alternative(html_content, subtype="html")
+
+
+def inject_body_part(part, content, subtype="plain"):
+    parts = []
+    text_body = None
+    text_content = None
+    if subtype == "html":
+        text_body, text_content = part.get_body_content("plain")
+
+    for p in part.iter_parts():
+        if text_body and p == text_body:
+            continue
+        parts.append(p)
+
+    boundary = part.get_boundary()
+    p_subtype = part.get_content_subtype()
+    part.clear_content()
+    if text_content != None:
+        part.set_content(text_content)
+        part.add_alternative(content, subtype=subtype)
+    else:
+        part.set_content(content, subtype=subtype)
+
+    if part.get_content_subtype() != p_subtype:
+        if p_subtype == "alternative":
+            part.make_alternative()
+        elif p_subtype == "related":
+            part.make_related()
+        else:
+            part.make_mixed()
+
+    if boundary:
+        part.set_boundary(boundary)
+    for p in parts:
+        part.attach(p)
+
+
+def replace_illegal_chars(string):
+    """Remove illegal characters from header values."""
+    return "".join(string.replace("\x00", "").splitlines())

pyquarantine/cli.py

@@ -0,0 +1,635 @@
+#!/usr/bin/env python
+#
+# PyQuarantine-Milter is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# PyQuarantine-Milter is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PyQuarantineMilter.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import argparse
+import json
+import logging
+import logging.handlers
+import sys
+import time
+
+from pyquarantine.config import get_milter_config, ActionConfig
+from pyquarantine.storage import Quarantine
+from pyquarantine import __version__ as version
+
+
+def _get_quarantine(quarantines, name, debug):
+    try:
+        quarantine = next((q for q in quarantines if q["name"] == name))
+    except StopIteration:
+        raise RuntimeError(f"invalid quarantine '{name}'")
+    return Quarantine(ActionConfig(quarantine), [], debug)
+
+
+def _get_notification(quarantines, name, debug):
+    notification = _get_quarantine(quarantines, name, debug).notification
+    if not notification:
+        raise RuntimeError(
+                "notification type is set to NONE")
+    return notification
+
+
+def _get_whitelist(quarantines, name, debug):
+    whitelist = _get_quarantine(quarantines, name, debug).whitelist
+    if not whitelist:
+        raise RuntimeError(
+                "whitelist type is set to NONE")
+    return whitelist
+
+
+def print_table(columns, rows):
+    if not rows:
+        return
+
+    column_lengths = []
+    column_formats = []
+
+    # iterate columns to display
+    for header, key in columns:
+        # get the length of the header string
+        lengths = [len(header)]
+        # get the length of the longest value
+        lengths.append(
+            len(str(max(rows, key=lambda x: len(str(x[key])))[key])))
+        # use the longer one
+        length = max(lengths)
+        column_lengths.append(length)
+        column_formats.append(f"{{:<{length}}}")
+
+    # define row format
+    row_format = " | ".join(column_formats)
+
+    # define header/body separator
+    separators = []
+    for length in column_lengths:
+        separators.append("-" * length)
+    separator = "-+-".join(separators)
+
+    # print header and separator
+    print(row_format.format(*[column[0] for column in columns]))
+    print(separator)
+
+    keys = [entry[1] for entry in columns]
+    # print rows
+    for entry in rows:
+        row = []
+        for key in keys:
+            row.append(entry[key])
+        print(row_format.format(*row))
+
+
+def list_quarantines(quarantines, args):
+    if args.batch:
+        print("\n".join([q["name"] for q in quarantines]))
+    else:
+        qlist = []
+        for q in quarantines:
+            cfg = q["options"]
+            storage_type = cfg["store"]["type"]
+
+            if "notify" in cfg:
+                notification_type = cfg["notify"]["type"]
+            else:
+                notification_type = "NONE"
+
+            if "whitelist" in cfg:
+                whitelist_type = cfg["whitelist"]["type"]
+            else:
+                whitelist_type = "NONE"
+
+            if "milter_action" in cfg:
+                milter_action = cfg["milter_action"]
+            else:
+                milter_action = "NONE"
+
+            qlist.append({
+                "name": q["name"],
+                "storage": storage_type,
+                "notification": notification_type,
+                "whitelist": whitelist_type,
+                "action": milter_action})
+
+        print_table(
+            [("Name", "name"),
+             ("Storage", "storage"),
+             ("Notification", "notification"),
+             ("Whitelist", "whitelist"),
+             ("Action", "action")],
+            qlist
+        )
+
+
+def list_quarantine_emails(quarantines, args):
+    storage = _get_quarantine(quarantines, args.quarantine, args.debug).storage
+
+    # find emails and transform some metadata values to strings
+    rows = []
+    emails = storage.find(
+        args.mailfrom, args.recipients, args.older_than)
+    for storage_id, metadata in emails.items():
+        row = emails[storage_id]
+        row["storage_id"] = storage_id
+        row["timestamp"] = time.strftime(
+            '%Y-%m-%d %H:%M:%S',
+            time.localtime(
+                metadata["timestamp"]))
+        row["mailfrom"] = metadata["mailfrom"]
+        row["recipient"] = metadata["recipients"].pop(0)
+        if "subject" not in emails[storage_id]:
+            emails[storage_id]["subject"] = ""
+        row["subject"] = emails[storage_id]["subject"][:60].strip()
+        rows.append(row)
+
+        if metadata["recipients"]:
+            row = {
+                "storage_id": "",
+                "timestamp": "",
+                "mailfrom": "",
+                "recipient": metadata["recipients"].pop(0),
+                "subject": ""
+            }
+            rows.append(row)
+
+    if args.batch:
+        # batch mode, print quarantine IDs, each on a new line
+        print("\n".join(emails.keys()))
+        return
+
+    if not emails:
+        print(f"quarantine '{args.quarantine}' is empty")
+        return
+
+    print_table(
+        [("Quarantine-ID", "storage_id"), ("When", "timestamp"),
+         ("From", "mailfrom"), ("Recipient(s)", "recipient"),
+         ("Subject", "subject")],
+        rows
+    )
+
+
+def list_whitelist(quarantines, args):
+    whitelist = _get_whitelist(quarantines, args.quarantine, args.debug)
+
+    # find whitelist entries
+    entries = whitelist.find(
+        mailfrom=args.mailfrom,
+        recipients=args.recipients,
+        older_than=args.older_than)
+    if not entries:
+        print(f"whitelist of quarantine '{args.quarantine}' is empty")
+        return
+
+    # transform some values to strings
+    for eid, entry in entries.items():
+        entries[eid]["permanent_str"] = str(entry["permanent"])
+        entries[eid]["created_str"] = entry["created"].strftime(
+            '%Y-%m-%d %H:%M:%S')
+        entries[eid]["last_used_str"] = entry["last_used"].strftime(
+            '%Y-%m-%d %H:%M:%S')
+
+    print_table(
+        [
+            ("ID", "id"), ("From", "mailfrom"), ("To", "recipient"),
+            ("Created", "created_str"), ("Last used", "last_used_str"),
+            ("Comment", "comment"), ("Permanent", "permanent_str")
+        ],
+        entries.values()
+    )
+
+
+def add_whitelist_entry(quarantines, args):
+    logger = logging.getLogger(__name__)
+    whitelist = _get_whitelist(quarantines, args.quarantine, args.debug)
+
+    # check existing entries
+    entries = whitelist.check(args.mailfrom, args.recipient, logger)
+    if entries:
+        # check if the exact entry exists already
+        for entry in entries.values():
+            if entry["mailfrom"] == args.mailfrom and \
+                    entry["recipient"] == args.recipient:
+                raise RuntimeError(
+                    "an entry with this from/to combination already exists")
+
+        if not args.force:
+            # the entry is already covered by others
+            for eid, entry in entries.items():
+                entries[eid]["permanent_str"] = str(entry["permanent"])
+                entries[eid]["created_str"] = entry["created"].strftime(
+                    '%Y-%m-%d %H:%M:%S')
+                entries[eid]["last_used_str"] = entry["last_used"].strftime(
+                    '%Y-%m-%d %H:%M:%S')
+            print_table(
+                [
+                    ("ID", "id"), ("From", "mailfrom"), ("To", "recipient"),
+                    ("Created", "created_str"), ("Last used", "last_used_str"),
+                    ("Comment", "comment"), ("Permanent", "permanent_str")
+                ],
+                entries.values()
+            )
+            print("")
+            raise RuntimeError(
+                "from/to combination is already covered by the entries above, "
+                "use --force to override.")
+
+    # add entry to whitelist
+    whitelist.add(args.mailfrom, args.recipient, args.comment, args.permanent)
+    print("whitelist entry added successfully")
+
+
+def delete_whitelist_entry(quarantines, args):
+    whitelist = _get_whitelist(quarantines, args.quarantine, args.debug)
+    whitelist.delete(args.whitelist_id)
+    print("whitelist entry deleted successfully")
+
+
+def notify(quarantines, args):
+    quarantine = _get_quarantine(quarantines, args.quarantine, args.debug)
+    quarantine.notify(args.quarantine_id, args.recipient)
+    print("notification sent successfully")
+
+
+def release(quarantines, args):
+    logger = logging.getLogger(__name__)
+    quarantine = _get_quarantine(quarantines, args.quarantine, args.debug)
+    rcpts = quarantine.release(args.quarantine_id, args.recipient)
+    rcpts = ", ".join(rcpts)
+    logger.info(
+        f"{args.quarantine}: released message with id {args.quarantine_id} "
+        f"for {rcpts}")
+
+
+def delete(quarantines, args):
+    storage = _get_quarantine(quarantines, args.quarantine, args.debug).storage
+    storage.delete(args.quarantine_id, args.recipient)
+    print("quarantined message deleted successfully")
+
+
+def get(quarantines, args):
+    storage = _get_quarantine(quarantines, args.quarantine, args.debug).storage
+    data = storage.get_mail_bytes(args.quarantine_id)
+    sys.stdout.buffer.write(data)
+
+
+def metadata(quarantines, args):
+    storage = _get_quarantine(quarantines, args.quarantine, args.debug).storage
+    metadata = storage.get_metadata(args.quarantine_id)
+    print(json.dumps(metadata))
+
+
+class StdErrFilter(logging.Filter):
+    def filter(self, rec):
+        return rec.levelno in (logging.ERROR, logging.WARNING)
+
+
+class StdOutFilter(logging.Filter):
+    def filter(self, rec):
+        return rec.levelno in (logging.DEBUG, logging.INFO)
+
+
+def main():
+    python_version = ".".join([str(v) for v in sys.version_info[0:3]])
+    python_version = f"{python_version}-{sys.version_info[3]}"
+
+    "PyQuarantine command-line interface."
+    # parse command line
+    def formatter_class(prog): return argparse.HelpFormatter(
+        prog, max_help_position=50, width=140)
+    parser = argparse.ArgumentParser(
+        description="PyQuarantine CLI",
+        formatter_class=formatter_class)
+    parser.add_argument(
+        "-c", "--config", help="Config file to read.",
+        default="/etc/pyquarantine/pyquarantine.conf")
+    parser.add_argument(
+        "-d", "--debug",
+        help="Log debugging messages.",
+        action="store_true")
+    parser.add_argument(
+        "-v", "--version",
+        help="Print version.",
+        action="version",
+        version=f"%(prog)s {version} (python {python_version})")
+    parser.set_defaults(syslog=False)
+    subparsers = parser.add_subparsers(
+        dest="command",
+        title="Commands")
+    subparsers.required = True
+
+    # list command
+    list_parser = subparsers.add_parser(
+        "list",
+        help="List available quarantines.",
+        formatter_class=formatter_class)
+    list_parser.add_argument(
+        "-b", "--batch",
+        help="Print results using only quarantine names, each on a new line.",
+        action="store_true")
+    list_parser.set_defaults(func=list_quarantines)
+
+    # quarantine command group
+    quar_parser = subparsers.add_parser(
+        "quarantine",
+        description="Manage quarantines.",
+        help="Manage quarantines.",
+        formatter_class=formatter_class)
+    quar_parser.add_argument(
+        "quarantine",
+        metavar="QUARANTINE",
+        help="Quarantine name.")
+    quar_subparsers = quar_parser.add_subparsers(
+        dest="command",
+        title="Quarantine commands")
+    quar_subparsers.required = True
+    # quarantine list command
+    quar_list_parser = quar_subparsers.add_parser(
+        "list",
+        description="List emails in quarantines.",
+        help="List emails in quarantine.",
+        formatter_class=formatter_class)
+    quar_list_parser.add_argument(
+        "-f", "--from",
+        dest="mailfrom",
+        help="Filter emails by from address.",
+        default=None,
+        nargs="+")
+    quar_list_parser.add_argument(
+        "-t", "--to",
+        dest="recipients",
+        help="Filter emails by recipient address.",
+        default=None,
+        nargs="+")
+    quar_list_parser.add_argument(
+        "-o", "--older-than",
+        dest="older_than",
+        help="Filter emails by age (days).",
+        default=None,
+        type=float)
+    quar_list_parser.add_argument(
+        "-b", "--batch",
+        help="Print results using only email quarantine IDs, "
+             "each on a new line.",
+        action="store_true")
+    quar_list_parser.set_defaults(func=list_quarantine_emails)
+    # quarantine notify command
+    quar_notify_parser = quar_subparsers.add_parser(
+        "notify",
+        description="Notify recipient about email in quarantine.",
+        help="Notify recipient about email in quarantine.",
+        formatter_class=formatter_class)
+    quar_notify_parser.add_argument(
+        "quarantine_id",
+        metavar="ID",
+        help="Quarantine ID.")
+    quar_notify_parser_grp = quar_notify_parser.add_mutually_exclusive_group(
+        required=True)
+    quar_notify_parser_grp.add_argument(
+        "-t", "--to",
+        dest="recipient",
+        help="Release email for one recipient address.")
+    quar_notify_parser_grp.add_argument(
+        "-a", "--all",
+        help="Release email for all recipients.",
+        action="store_true")
+    quar_notify_parser.set_defaults(func=notify)
+    # quarantine release command
+    quar_release_parser = quar_subparsers.add_parser(
+        "release",
+        description="Release email from quarantine.",
+        help="Release email from quarantine.",
+        formatter_class=formatter_class)
+    quar_release_parser.add_argument(
+        "quarantine_id",
+        metavar="ID",
+        help="Quarantine ID.")
+    quar_release_parser.add_argument(
+        "-n",
+        "--disable-syslog",
+        dest="syslog",
+        help="Disable syslog messages.",
+        action="store_false")
+    quar_release_parser_grp = quar_release_parser.add_mutually_exclusive_group(
+        required=True)
+    quar_release_parser_grp.add_argument(
+        "-t", "--to",
+        dest="recipient",
+        help="Release email for one recipient address.")
+    quar_release_parser_grp.add_argument(
+        "-a", "--all",
+        help="Release email for all recipients.",
+        action="store_true")
+    quar_release_parser.set_defaults(func=release)
+    # quarantine delete command
+    quar_delete_parser = quar_subparsers.add_parser(
+        "delete",
+        description="Delete email from quarantine.",
+        help="Delete email from quarantine.",
+        formatter_class=formatter_class)
+    quar_delete_parser.add_argument(
+        "quarantine_id",
+        metavar="ID",
+        help="Quarantine ID.")
+    quar_delete_parser.add_argument(
+        "-n", "--disable-syslog",
+        dest="syslog",
+        help="Disable syslog messages.",
+        action="store_false")
+    quar_delete_parser_grp = quar_delete_parser.add_mutually_exclusive_group(
+        required=True)
+    quar_delete_parser_grp.add_argument(
+        "-t", "--to",
+        dest="recipient",
+        help="Delete email for one recipient address.")
+    quar_delete_parser_grp.add_argument(
+        "-a", "--all",
+        help="Delete email for all recipients.",
+        action="store_true")
+    quar_delete_parser.set_defaults(func=delete)
+    # quarantine get command
+    quar_get_parser = quar_subparsers.add_parser(
+        "get",
+        description="Get email from quarantine.",
+        help="Get email from quarantine",
+        formatter_class=formatter_class)
+    quar_get_parser.add_argument(
+        "quarantine_id",
+        metavar="ID",
+        help="Quarantine ID.")
+    quar_get_parser.set_defaults(func=get)
+    # quarantine metadata command
+    quar_metadata_parser = quar_subparsers.add_parser(
+        "metadata",
+        description="Get metadata of email from quarantine.",
+        help="Get metadata of email from quarantine",
+        formatter_class=formatter_class)
+    quar_metadata_parser.add_argument(
+        "quarantine_id",
+        metavar="ID",
+        help="Quarantine ID.")
+    quar_metadata_parser.set_defaults(func=metadata)
+
+    # whitelist command group
+    whitelist_parser = subparsers.add_parser(
+        "whitelist",
+        description="Manage whitelists.",
+        help="Manage whitelists.",
+        formatter_class=formatter_class)
+    whitelist_parser.add_argument(
+        "quarantine",
+        metavar="QUARANTINE",
+        help="Quarantine name.")
+    whitelist_subparsers = whitelist_parser.add_subparsers(
+        dest="command",
+        title="Whitelist commands")
+    whitelist_subparsers.required = True
+    # whitelist list command
+    whitelist_list_parser = whitelist_subparsers.add_parser(
+        "list",
+        description="List whitelist entries.",
+        help="List whitelist entries.",
+        formatter_class=formatter_class)
+    whitelist_list_parser.add_argument(
+        "-f", "--from",
+        dest="mailfrom",
+        help="Filter entries by from address.",
+        default=None,
+        nargs="+")
+    whitelist_list_parser.add_argument(
+        "-t", "--to",
+        dest="recipients",
+        help="Filter entries by recipient address.",
+        default=None,
+        nargs="+")
+    whitelist_list_parser.add_argument(
+        "-o", "--older-than",
+        dest="older_than",
+        help="Filter emails by last used date (days).",
+        default=None,
+        type=float)
+    whitelist_list_parser.set_defaults(func=list_whitelist)
+    # whitelist add command
+    whitelist_add_parser = whitelist_subparsers.add_parser(
+        "add",
+        description="Add whitelist entry.",
+        help="Add whitelist entry.",
+        formatter_class=formatter_class)
+    whitelist_add_parser.add_argument(
+        "-f", "--from",
+        dest="mailfrom",
+        help="From address.",
+        required=True)
+    whitelist_add_parser.add_argument(
+        "-t", "--to",
+        dest="recipient",
+        help="Recipient address.",
+        required=True)
+    whitelist_add_parser.add_argument(
+        "-c", "--comment",
+        help="Comment.",
+        default="added by CLI")
+    whitelist_add_parser.add_argument(
+        "-p", "--permanent",
+        help="Add a permanent entry.",
+        action="store_true")
+    whitelist_add_parser.add_argument(
+        "--force",
+        help="Force adding an entry, "
+             "even if already covered by another entry.",
+        action="store_true")
+    whitelist_add_parser.set_defaults(func=add_whitelist_entry)
+    # whitelist delete command
+    whitelist_delete_parser = whitelist_subparsers.add_parser(
+        "delete",
+        description="Delete whitelist entry.",
+        help="Delete whitelist entry.",
+        formatter_class=formatter_class)
+    whitelist_delete_parser.add_argument(
+        "whitelist_id",
+        metavar="ID",
+        help="Whitelist ID.")
+    whitelist_delete_parser.set_defaults(func=delete_whitelist_entry)
+
+    args = parser.parse_args()
+
+    # setup logging
+    loglevel = logging.INFO
+    root_logger = logging.getLogger()
+    root_logger.setLevel(loglevel)
+
+    # setup console log
+    if args.debug:
+        formatter = logging.Formatter(
+            "%(levelname)s: [%(name)s] - %(message)s")
+    else:
+        formatter = logging.Formatter("%(levelname)s: %(message)s")
+    # stdout
+    stdouthandler = logging.StreamHandler(sys.stdout)
+    stdouthandler.setLevel(logging.DEBUG)
+    stdouthandler.setFormatter(formatter)
+    stdouthandler.addFilter(StdOutFilter())
+    root_logger.addHandler(stdouthandler)
+    # stderr
+    stderrhandler = logging.StreamHandler(sys.stderr)
+    stderrhandler.setLevel(logging.WARNING)
+    stderrhandler.setFormatter(formatter)
+    stderrhandler.addFilter(StdErrFilter())
+    root_logger.addHandler(stderrhandler)
+    logger = logging.getLogger(__name__)
+
+    try:
+        logger.debug("read milter configuration")
+        cfg = get_milter_config(args.config, raw=True)
+        if "rules" not in cfg or not cfg["rules"]:
+            raise RuntimeError("no rules configured")
+
+        for rule in cfg["rules"]:
+            if "actions" not in rule or not rule["actions"]:
+                raise RuntimeError(
+                    f"{rule['name']}: no actions configured")
+    except (RuntimeError, AssertionError) as e:
+        logger.error(f"config error: {e}")
+        sys.exit(255)
+
+    quarantines = []
+    for rule in cfg["rules"]:
+        for action in rule["actions"]:
+            if action["type"] == "quarantine":
+                quarantines.append(action)
+
+    if args.syslog:
+        # setup syslog
+        sysloghandler = logging.handlers.SysLogHandler(
+            address="/dev/log",
+            facility=logging.handlers.SysLogHandler.LOG_MAIL)
+        sysloghandler.setLevel(loglevel)
+        if args.debug:
+            formatter = logging.Formatter(
+                "pyquarantine: [%(name)s] [%(levelname)s] %(message)s")
+        else:
+            formatter = logging.Formatter("pyquarantine: %(message)s")
+        sysloghandler.setFormatter(formatter)
+        root_logger.addHandler(sysloghandler)
+
+    # call the commands function
+    try:
+        args.func(quarantines, args)
+    except RuntimeError as e:
+        logger.error(e)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

pyquarantine/conditions.py

@@ -0,0 +1,203 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = ["Conditions"]
+
+import logging
+import re
+
+from netaddr import IPAddress, IPNetwork, AddrFormatError
+from pyquarantine import CustomLogger
+from pyquarantine.whitelist import DatabaseWhitelist
+
+
+class Conditions:
+    """Conditions to implement conditions for rules and actions."""
+
+    def __init__(self, cfg, local_addrs, debug):
+        self.cfg = cfg
+        self.local_addrs = local_addrs
+
+        self.logger = logging.getLogger(cfg["name"])
+        self.logger.setLevel(cfg.get_loglevel(debug))
+
+        for arg in ("local", "hosts", "envfrom", "envto", "headers", "metavar",
+                    "var"):
+            if arg not in cfg:
+                setattr(self, arg, None)
+                continue
+
+            if arg == "hosts":
+                try:
+                    self.hosts = []
+                    for host in cfg["hosts"]:
+                        self.hosts.append(IPNetwork(host))
+                except AddrFormatError as e:
+                    raise RuntimeError(e)
+            elif arg in ("envfrom", "envto"):
+                try:
+                    setattr(self, arg, re.compile(
+                        cfg[arg], re.IGNORECASE))
+                except re.error as e:
+                    raise RuntimeError(e)
+            elif arg == "headers":
+                try:
+                    self.headers = []
+                    for header in cfg["headers"]:
+                        self.headers.append(re.compile(
+                            header, re.IGNORECASE + re.DOTALL + re.MULTILINE))
+                except re.error as e:
+                    raise RuntimeError(e)
+            else:
+                setattr(self, arg, cfg[arg])
+
+        self.whitelist = cfg["whitelist"] if "whitelist" in cfg else None
+        if self.whitelist is not None:
+            self.whitelist["name"] = f"{cfg['name']}: whitelist"
+            self.whitelist["loglevel"] = cfg["loglevel"]
+            if self.whitelist["type"] == "db":
+                self.whitelist = DatabaseWhitelist(self.whitelist, debug)
+            else:
+                raise RuntimeError("invalid whitelist type")
+
+    def __str__(self):
+        cfg = []
+        for arg in ("local", "hosts", "envfrom", "envto", "headers",
+                    "var", "metavar"):
+            if arg in self.cfg:
+                cfg.append(f"{arg}={self.cfg[arg]}")
+        if self.whitelist is not None:
+            cfg.append(f"whitelist={self.whitelist}")
+        return "Conditions(" + ", ".join(cfg) + ")"
+
+    def get_whitelist(self):
+        return self.whitelist
+
+    def match_host(self, host):
+        logger = CustomLogger(
+            self.logger, {"name": self.cfg["name"]})
+        ip = IPAddress(host)
+
+        if self.local is not None:
+            is_local = False
+            for addr in self.local_addrs:
+                if ip in addr:
+                    is_local = True
+                    break
+
+            if is_local != self.local:
+                logger.debug(
+                    f"ignore host {host}, "
+                    f"local does not match")
+                return False
+
+            logger.debug(
+                f"local matches for host {host}")
+
+        if self.hosts is not None:
+            found = False
+            for addr in self.hosts:
+                if ip in addr:
+                    found = True
+                    break
+
+            if not found:
+                logger.debug(
+                    f"ignore host {host}, "
+                    f"hosts does not match")
+                return False
+
+            logger.debug(
+                f"hosts matches for host {host}")
+
+        return True
+
+    def get_wl_rcpts(self, mailfrom, rcpts, logger):
+        if not self.whitelist:
+            return {}
+
+        wl_rcpts = []
+        for rcpt in rcpts:
+            if self.whitelist.check(mailfrom, rcpt, logger):
+                wl_rcpts.append(rcpt)
+
+        return wl_rcpts
+
+    def update_msginfo_from_match(self, milter, match):
+        if self.metavar is None:
+            return
+
+        named_subgroups = match.groupdict(default=None)
+        for group, value in named_subgroups.items():
+            if value is None:
+                continue
+            name = f"{self.metavar}_{group}"
+            milter.msginfo["vars"][name] = value
+
+    def match(self, milter):
+        logger = CustomLogger(
+            self.logger, {"qid": milter.qid, "name": self.cfg["name"]})
+
+        if self.envfrom is not None:
+            envfrom = milter.msginfo["mailfrom"]
+            if match := self.envfrom.match(envfrom):
+                logger.debug(
+                    f"envfrom matches for "
+                    f"envelope-from address {envfrom}")
+                self.update_msginfo_from_match(milter, match)
+            else:
+                logger.debug(
+                    f"ignore envelope-from address {envfrom}, "
+                    f"envfrom does not match")
+                return False
+
+        if self.envto is not None:
+            envto = milter.msginfo["rcpts"]
+            if not isinstance(envto, list):
+                envto = [envto]
+
+            for to in envto:
+                match = self.envto.match(to)
+                if not match:
+                    logger.debug(
+                        f"ignore envelope-to address {envto}, "
+                        f"envto does not match")
+                    return False
+
+            logger.debug(
+                f"envto matches for "
+                f"envelope-to address {envto}")
+            self.update_msginfo_from_match(milter, match)
+
+        if self.headers is not None:
+            headers = map(lambda h: f"{h[0]}: {h[1]}", milter.msg.items())
+            for hdr in self.headers:
+                matches = filter(None, map(lambda h: hdr.search(h), headers))
+                if match := next(matches, None):
+                    logger.debug(
+                        f"headers matches for "
+                        f"header: {match.string}")
+                    self.update_msginfo_from_match(milter, match)
+                    continue
+
+                logger.debug(
+                    "ignore message, "
+                    "headers does not match")
+                return False
+
+        if self.var is not None:
+            if self.var not in milter.msginfo["vars"]:
+                return False
+
+        return True

pyquarantine/config.py

@@ -0,0 +1,376 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = [
+    "BaseConfig",
+    "ConditionsConfig",
+    "AddHeaderConfig",
+    "ModHeaderConfig",
+    "DelHeaderConfig",
+    "AddDisclaimerConfig",
+    "RewriteLinksConfig",
+    "StoreConfig",
+    "NotifyConfig",
+    "WhitelistConfig",
+    "QuarantineConfig",
+    "ActionConfig",
+    "RuleConfig",
+    "QuarantineMilterConfig",
+    "get_milter_config"]
+
+import json
+import jsonschema
+import logging
+import re
+
+
+class BaseConfig:
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": [],
+        "additionalProperties": True,
+        "properties": {
+            "loglevel": {"type": "string", "default": "info"}}}
+
+    def __init__(self, config):
+        required = self.JSON_SCHEMA["required"]
+        properties = self.JSON_SCHEMA["properties"]
+        for p in properties.keys():
+            if p in required:
+                continue
+            elif p not in config and "default" in properties[p]:
+                config[p] = properties[p]["default"]
+        try:
+            jsonschema.validate(config, self.JSON_SCHEMA)
+        except jsonschema.exceptions.ValidationError as e:
+            raise RuntimeError(e)
+
+        self._config = config
+
+    def __getitem__(self, key):
+        return self._config[key]
+
+    def __setitem__(self, key, value):
+        self._config[key] = value
+
+    def __delitem__(self, key):
+        del self._config[key]
+
+    def __contains__(self, key):
+        return key in self._config
+
+    def keys(self):
+        return self._config.keys()
+
+    def items(self):
+        return self._config.items()
+
+    def get_loglevel(self, debug):
+        if debug:
+            level = logging.DEBUG
+        else:
+            level = getattr(logging, self["loglevel"].upper(), None)
+            assert isinstance(level, int), \
+                "loglevel: invalid value"
+        return level
+
+    def get_config(self):
+        return self._config
+
+
+class WhitelistConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["type"],
+        "additionalProperties": True,
+        "properties": {
+            "type": {"enum": ["db"]}},
+        "if": {"properties": {"type": {"const": "db"}}},
+        "then": {
+            "required": ["connection", "table"],
+            "additionalProperties": False,
+            "properties": {
+                "type": {"type": "string"},
+                "connection": {"type": "string"},
+                "table": {"type": "string"}}}}
+
+
+class ConditionsConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": [],
+        "additionalProperties": False,
+        "properties": {
+            "metavar": {"type": "string"},
+            "local": {"type": "boolean"},
+            "hosts": {"type": "array",
+                      "items": {"type": "string"}},
+            "envfrom": {"type": "string"},
+            "envto": {"type": "string"},
+            "headers": {"type": "array",
+                        "items": {"type": "string"}},
+            "var": {"type": "string"},
+            "whitelist": {"type": "object"}}}
+
+    def __init__(self, config, rec=True):
+        super().__init__(config)
+        if not rec:
+            return
+        if "whitelist" in self:
+            self["whitelist"] = WhitelistConfig(self["whitelist"])
+
+
+class AddHeaderConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["field", "value"],
+        "additionalProperties": False,
+        "properties": {
+            "field": {"type": "string"},
+            "value": {"type": "string"}}}
+
+
+class ModHeaderConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["field", "value"],
+        "additionalProperties": False,
+        "properties": {
+            "field": {"type": "string"},
+            "value": {"type": "string"},
+            "search": {"type": "string"}}}
+
+
+class DelHeaderConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["field"],
+        "additionalProperties": False,
+        "properties": {
+            "field": {"type": "string"},
+            "value": {"type": "string"}}}
+
+
+class AddDisclaimerConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["action", "html_template", "text_template"],
+        "additionalProperties": False,
+        "properties": {
+            "action": {"type": "string"},
+            "html_template": {"type": "string"},
+            "text_template": {"type": "string"},
+            "error_policy": {"type": "string", "default": "wrap"},
+            "add_html_body": {"type": "boolean", "default": False}}}
+
+
+class RewriteLinksConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["repl"],
+        "additionalProperties": False,
+        "properties": {
+            "repl": {"type": "string"}}}
+
+
+class StoreConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["type"],
+        "additionalProperties": True,
+        "properties": {
+            "type": {"enum": ["file"]}},
+        "if": {"properties": {"type": {"const": "file"}}},
+        "then": {
+            "required": ["directory"],
+            "additionalProperties": False,
+            "properties": {
+                "type": {"type": "string"},
+                "directory": {"type": "string"},
+                "mode": {"type": "string"},
+                "metavar": {"type": "string"},
+                "metadata": {"type": "boolean", "default": False},
+                "original": {"type": "boolean", "default": False}}}}
+
+
+class NotifyConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["type"],
+        "additionalProperties": True,
+        "properties": {
+            "type": {"enum": ["email"]}},
+        "if": {"properties": {"type": {"const": "email"}}},
+        "then": {
+            "required": ["smtp_host", "smtp_port", "envelope_from",
+                         "from_header", "subject", "template"],
+            "additionalProperties": False,
+            "properties": {
+                "type": {"type": "string"},
+                "smtp_host": {"type": "string"},
+                "smtp_port": {"type": "number"},
+                "envelope_from": {"type": "string"},
+                "from_header": {"type": "string"},
+                "subject": {"type": "string"},
+                "template": {"type": "string"},
+                "repl_img": {"type": "string"},
+                "strip_imgs": {"type": "boolean", "default": False},
+                "embed_imgs": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "default": []}}}}
+
+
+class QuarantineConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["store", "smtp_host", "smtp_port"],
+        "additionalProperties": False,
+        "properties": {
+            "name": {"type": "string"},
+            "notify": {"type": "object"},
+            "milter_action": {"type": "string"},
+            "reject_reason": {"type": "string"},
+            "whitelist": {"type": "object"},
+            "store": {"type": "object"},
+            "smtp_host": {"type": "string"},
+            "smtp_port": {"type": "number"}}}
+
+    def __init__(self, config, rec=True):
+        super().__init__(config)
+        if not rec:
+            return
+        if "metadata" not in self["store"]:
+            self["store"]["metadata"] = True
+        self["store"] = StoreConfig(self["store"])
+        if "notify" in self:
+            self["notify"] = NotifyConfig(self["notify"])
+        if "whitelist" in self:
+            self["whitelist"] = ConditionsConfig(
+                {"whitelist": self["whitelist"]}, rec)
+
+
+class ActionConfig(BaseConfig):
+    ACTION_TYPES = {
+        "add_header": AddHeaderConfig,
+        "mod_header": ModHeaderConfig,
+        "del_header": DelHeaderConfig,
+        "add_disclaimer": AddDisclaimerConfig,
+        "rewrite_links": RewriteLinksConfig,
+        "store": StoreConfig,
+        "notify": NotifyConfig,
+        "quarantine": QuarantineConfig}
+
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["name", "type", "options"],
+        "additionalProperties": False,
+        "properties": {
+            "name": {"type": "string"},
+            "loglevel": {"type": "string", "default": "info"},
+            "pretend": {"type": "boolean", "default": False},
+            "conditions": {"type": "object"},
+            "type": {"enum": list(ACTION_TYPES.keys())},
+            "options": {"type": "object"}}}
+
+    def __init__(self, config, rec=True):
+        super().__init__(config)
+        if not rec:
+            return
+        if "conditions" in self:
+            self["conditions"] = ConditionsConfig(self["conditions"])
+        self["action"] = self.ACTION_TYPES[self["type"]](self["options"])
+
+
+class RuleConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["name", "actions"],
+        "additionalProperties": False,
+        "properties": {
+            "name": {"type": "string"},
+            "loglevel": {"type": "string", "default": "info"},
+            "pretend": {"type": "boolean", "default": False},
+            "conditions": {"type": "object"},
+            "actions": {"type": "array"}}}
+
+    def __init__(self, config, rec=True):
+        super().__init__(config)
+        if not rec:
+            return
+        if "conditions" in self:
+            self["conditions"] = ConditionsConfig(self["conditions"])
+
+        actions = []
+        for idx, action in enumerate(self["actions"]):
+            if "loglevel" not in action:
+                action["loglevel"] = config["loglevel"]
+            if "pretend" not in action:
+                action["pretend"] = config["pretend"]
+            actions.append(ActionConfig(action, rec))
+        self["actions"] = actions
+
+
+class QuarantineMilterConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["rules"],
+        "additionalProperties": False,
+        "properties": {
+            "socket": {"type": "string"},
+            "local_addrs": {"type": "array",
+                            "items": {"type": "string"},
+                            "default": [
+                                "fe80::/64",
+                                "::1/128",
+                                "127.0.0.0/8",
+                                "10.0.0.0/8",
+                                "172.16.0.0/12",
+                                "192.168.0.0/16"]},
+            "loglevel": {"type": "string", "default": "info"},
+            "pretend": {"type": "boolean", "default": False},
+            "rules": {"type": "array"}}}
+
+    def __init__(self, config, rec=True):
+        super().__init__(config)
+        if not rec:
+            return
+        rules = []
+        for idx, rule in enumerate(self["rules"]):
+            if "loglevel" not in rule:
+                rule["loglevel"] = config["loglevel"]
+            if "pretend" not in rule:
+                rule["pretend"] = config["pretend"]
+            rules.append(RuleConfig(rule, rec))
+        self["rules"] = rules
+
+
+def get_milter_config(cfgfile, raw=False):
+    try:
+        with open(cfgfile, "r") as fh:
+            # remove lines with leading # (comments), they
+            # are not allowed in json
+            cfg = re.sub(r"(?m)^\s*#.*\n?", "", fh.read())
+    except IOError as e:
+        raise RuntimeError(f"unable to open/read config file: {e}")
+
+    try:
+        cfg = json.loads(cfg)
+    except json.JSONDecodeError as e:
+        cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())]
+        msg = "\n".join(cfg_text)
+        raise RuntimeError(f"{e}\n{msg}")
+    if raw:
+        return cfg
+    return QuarantineMilterConfig(cfg)

pyquarantine/mailer.py

@@ -0,0 +1,86 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import logging
+import smtplib
+
+from multiprocessing import Process, Queue
+
+
+logger = logging.getLogger(__name__)
+queue = Queue(maxsize=50)
+process = None
+
+
+def smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail):
+    s = smtplib.SMTP(host=smtp_host, port=smtp_port)
+    s.ehlo()
+    if s.has_extn("STARTTLS"):
+        s.starttls()
+        s.ehlo()
+    s.sendmail(mailfrom, [recipient], mail)
+    s.quit()
+
+
+def mailprocess():
+    "Mailer process to send emails asynchronously."
+    global logger
+    global queue
+
+    try:
+        while True:
+            m = queue.get()
+            if not m:
+                break
+
+            smtp_host, smtp_port, qid, mailfrom, recipient, mail, emailtype = m
+            try:
+                smtp_send(smtp_host, smtp_port, mailfrom, recipient, mail)
+            except Exception as e:
+                logger.error(
+                    f"{qid}: error while sending {emailtype} "
+                    f"to '{recipient}': {e}")
+            else:
+                logger.info(
+                    f"{qid}: successfully sent {emailtype} to: {recipient}")
+    except KeyboardInterrupt:
+        pass
+    logger.debug("mailer process terminated")
+
+
+def sendmail(smtp_host, smtp_port, qid, mailfrom, recipients, mail,
+             emailtype="email"):
+    "Send an email."
+    global logger
+    global process
+    global queue
+
+    if isinstance(recipients, str):
+        recipients = [recipients]
+
+    # start mailprocess if it is not started yet
+    if process is None:
+        process = Process(target=mailprocess)
+        process.daemon = True
+        logger.debug("starting mailer process")
+        process.start()
+
+    for recipient in recipients:
+        try:
+            queue.put(
+                (smtp_host, smtp_port, qid, mailfrom, recipient, mail,
+                 emailtype),
+                timeout=30)
+        except Queue.Full:
+            raise RuntimeError("email queue is full")

pyquarantine/misc/openrc/pyquarantine-milter.confd

@@ -1,8 +1,8 @@
-# /etc/conf.d/pymodmilter: config file for /etc/init.d/pymodmilter
+# /etc/conf.d/pyquarantine-milter: config file for /etc/init.d/pyquarantine-milter
 
 # Start the daemon as the user. You can optionally append a group name here also.
 # USER="daemon"
 # USER="daemon:nobody"
 
-# Optional parameters for pymodmilter
+# Optional parameters for pyquarantine-milter
 # MILTER_OPTS=""

pyquarantine/misc/openrc/pyquarantine-milter.initd

@@ -4,12 +4,12 @@ user=${USER:-daemon}
 milter_opts="${MILTER_OPTS:-}"
 
 pidfile="/run/${RC_SVCNAME}.pid"
-command="/usr/bin/pymodmilter"
+command="/usr/bin/pyquarantine-milter"
 command_args="${milter_opts}"
 command_background=true
-start_stop_daemon_args="--user ${user}"
+command_user="${user}"
-
 extra_commands="configtest"
+start_stop_daemon_args="--wait 500"
 
 depend() {
 	need net
@@ -40,7 +40,7 @@ start_pre() {
 }
 
 stop_pre() {
-	if [ "${RC_CMD}" != "restart" ]; then
+	if [ "${RC_CMD}" == "restart" ]; then
 		checkconfig || return $?
 	fi
 }

pyquarantine/misc/pyquarantine.conf.default

@@ -0,0 +1,33 @@
+# This is an example /etc/pyquarantine/pyquarantine.conf file.
+# Copy it into place and edit before use.
+#
+# The file is in JSON format but comments are allowed.
+#
+{
+    # Option: socket (optional)
+    # Notes:  The socket used to communicate with the MTA.
+    #
+    #         Examples:
+    #         unix:/path/to/socket             a named pipe
+    #         inet:8899                        listen on ANY interface
+    #         inet:8899@localhost              listen on a specific interface
+    #         inet6:8899                       listen on ANY interface
+    #         inet6:8899@[2001:db8:1234::1]    listen on a specific interface
+    #
+    "socket": "inet:8898@127.0.0.1",
+
+    # Option: local_addrs (optional)
+    # Notes:  A list of local hosts and networks.
+    #
+    #"local_addrs": ["fe80::/64", "::1/128", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"],
+
+    # Option: loglevel (optional)
+    # Notes:  The default loglevel.
+    #         
+    #"loglevel": "info",
+
+    # Option: rules
+    # Notes:  List of rules.
+    #
+    "rules": []
+}

pyquarantine/misc/systemd/pyquarantine-milter.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=pyquarantine-milter
+After=network.target
+
+[Service]
+Type=simple
+EnvironmentFile=/etc/conf.d/pyquarantine-milter
+ExecStart=/usr/bin/pyquarantine-milter $MILTER_OPTS
+User=mail
+Group=mail
+TimeoutStopSec=300
+
+[Install]
+WantedBy=multi-user.target

pyquarantine/misc/templates/notification.template

@@ -0,0 +1,29 @@
+<html>
+ <body>
+  <h1>Quarantine notification</h1>
+  <table>
+   <tr>
+    <td><b>Envelope-From:</b></td>
+    <td>{ENVELOPE_FROM}</td>
+   </tr>
+   <tr>
+    <td><b>From:</b></td>
+    <td>{FROM}</td>
+   </tr>
+   <tr>
+    <td><b>Envelope-To:</b></td>
+    <td>{ENVELOPE_TO}</td>
+   </tr>
+   <tr>
+    <td><b>To:</b></td>
+    <td>{TO}</td>
+   </tr>
+   <tr>
+    <td><b>Subject:</b></td>
+    <td>{SUBJECT}</td>
+   </tr>
+  </table><br/>
+  <h2>Preview of the original e-mail</h2>
+  {HTML_TEXT}
+ </body>
+</html>

pyquarantine/modify.py

@@ -0,0 +1,401 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = [
+    "AddHeader",
+    "ModHeader",
+    "DelHeader",
+    "AddDisclaimer",
+    "RewriteLinks",
+    "Modify"]
+
+import logging
+import re
+
+from base64 import b64encode
+from bs4 import BeautifulSoup
+from collections import defaultdict
+from copy import copy
+from email.message import MIMEPart
+from email.policy import SMTP
+from html import escape
+from urllib.parse import quote
+
+from pyquarantine import replace_illegal_chars
+from pyquarantine.base import CustomLogger
+
+
+class AddHeader:
+    """Add a mail header field."""
+    _headersonly = True
+
+    def __init__(self, field, value, pretend=False):
+        self.field = field
+        self.value = value
+        self.pretend = pretend
+
+    def execute(self, milter, logger):
+        header = f"{self.field}: {self.value}"
+        if logger.getEffectiveLevel() == logging.DEBUG:
+            logger.debug(f"add_header: {header}")
+        else:
+            logger.info(f"add_header: {header[0:70]}")
+
+        milter.msg.add_header(self.field, self.value)
+        if not self.pretend:
+            milter.addheader(self.field, self.value)
+
+
+class ModHeader:
+    """Change the value of a mail header field."""
+    _headersonly = True
+
+    def __init__(self, field, value, search=None, pretend=False):
+        try:
+            self.field = re.compile(field, re.IGNORECASE)
+            self.search = search
+            if self.search is not None:
+                self.search = re.compile(
+                    self.search, re.MULTILINE + re.DOTALL + re.IGNORECASE)
+
+        except re.error as e:
+            raise RuntimeError(e)
+
+        self.value = value
+        self.pretend = pretend
+
+    def execute(self, milter, logger):
+        idx = defaultdict(int)
+
+        for i, (field, value) in enumerate(milter.msg.items()):
+            field_lower = field.lower()
+            idx[field_lower] += 1
+
+            if not self.field.match(field):
+                continue
+
+            new_value = value
+            if self.search is not None:
+                new_value = self.search.sub(self.value, value).strip()
+            else:
+                new_value = self.value
+
+            if not new_value:
+                logger.warning(
+                    "mod_header: resulting value is empty, "
+                    "skip modification")
+                continue
+
+            if new_value == value:
+                continue
+
+            header = f"{field}: {value}"
+            new_header = f"{field}: {new_value}"
+
+            if logger.getEffectiveLevel() == logging.DEBUG:
+                logger.debug(f"mod_header: {header}: {new_header}")
+            else:
+                logger.info(
+                    f"mod_header: {header[0:70]}: {new_header[0:70]}")
+
+            milter.msg.replace_header(
+                field, replace_illegal_chars(new_value), idx=idx[field_lower])
+
+            if not self.pretend:
+                milter.chgheader(field, new_value, idx=idx[field_lower])
+
+
+class DelHeader:
+    """Delete a mail header field."""
+    _headersonly = True
+
+    def __init__(self, field, value=None, pretend=False):
+        try:
+            self.field = re.compile(field, re.IGNORECASE)
+            self.value = value
+            if self.value is not None:
+                self.value = re.compile(
+                    value, re.MULTILINE + re.DOTALL + re.IGNORECASE)
+        except re.error as e:
+            raise RuntimeError(e)
+
+        self.pretend = pretend
+
+    def execute(self, milter, logger):
+        idx = defaultdict(int)
+
+        for field, value in milter.msg.items():
+            field_lower = field.lower()
+            idx[field_lower] += 1
+
+            if not self.field.match(field):
+                continue
+
+            if self.value is not None and not self.value.search(value):
+                continue
+
+            header = f"{field}: {value}"
+            if logger.getEffectiveLevel() == logging.DEBUG:
+                logger.debug(f"del_header: {header}")
+            else:
+                logger.info(f"del_header: {header[0:70]}")
+            milter.msg.remove_header(field, idx=idx[field_lower])
+
+            if not self.pretend:
+                milter.chgheader(field, "", idx=idx[field_lower])
+
+            idx[field_lower] -= 1
+
+
+def _get_body_content(msg, pref):
+    part = None
+    content = None
+    if not msg.is_multipart() and msg.get_content_type() == f"text/{pref}":
+        part = msg
+    else:
+        part = msg.get_body(preferencelist=(pref))
+
+    if part is not None:
+        content = part.get_content()
+
+    return (part, content)
+
+
+def _has_content_before_body_tag(soup):
+    s = copy(soup)
+    for element in s.find_all("head") + s.find_all("body"):
+        element.extract()
+
+    if len(s.text.strip()) > 0:
+        return True
+
+    return False
+
+
+def _wrap_message(milter):
+    attachment = MIMEPart(policy=SMTP)
+    attachment.set_content(milter.msg_as_bytes(),
+                           maintype="plain", subtype="text",
+                           disposition="attachment",
+                           filename=f"{milter.qid}.eml",
+                           params={"name": f"{milter.qid}.eml"})
+
+    milter.msg.clear_content()
+    milter.msg.set_content(
+        "Please see the original email attached.")
+    milter.msg.add_alternative(
+        "<html><body>Please see the original email attached.</body></html>",
+        subtype="html")
+    milter.msg.make_mixed()
+    milter.msg.attach(attachment)
+
+
+class AddDisclaimer:
+    """Append or prepend a disclaimer to the mail body."""
+    _headersonly = False
+
+    def __init__(self, text_template, html_template, action, error_policy,
+                 add_html_body, pretend=False):
+        self.text_template_path = text_template
+        self.html_template_path = html_template
+        try:
+            with open(text_template, "r") as f:
+                self.text_template = f.read()
+
+            with open(html_template, "r") as f:
+                self.html_template = f.read()
+
+        except IOError as e:
+            raise RuntimeError(e)
+        self.action = action.lower()
+        assert self.action in ["prepend", "append"], \
+            f"invalid action '{action}'"
+        self.error_policy = error_policy.lower()
+        assert self.error_policy in ["ignore", "reject", "wrap"], \
+            f"invalid error_policy '{error_policy}'"
+        self.add_html_body = add_html_body
+        self.pretend = pretend
+
+    def patch_message_body(self, milter, logger):
+        text_body, text_content = milter.msg.get_body_content("plain")
+        html_body, html_content = milter.msg.get_body_content("html")
+
+        if text_content is None and html_content is None:
+            logger.info("message contains no body, inject it")
+            if self.add_html_body:
+                milter.msg.set_body("", "")
+                html_body, html_content = milter.msg.get_body_content("html")
+            else:
+                milter.msg.set_body("")
+            text_body, text_content = milter.msg.get_body_content("plain")
+
+        if html_content is None and self.add_html_body:
+            logger.info("inject html body based on plain body")
+            header = '<meta http-equiv="Content-Type" content="text/html; ' \
+                     'charset=utf-8">'
+            html_text = re.sub(r"^(.*)$", r"\1<br/>",
+                               escape(text_content, quote=False),
+                               flags=re.MULTILINE)
+            milter.msg.set_body(None, f"{header}{html_text}")
+            text_body, text_content = milter.msg.get_body_content("plain")
+            html_body, html_content = milter.msg.get_body_content("html")
+
+        variables = defaultdict(str, milter.msginfo["vars"])
+        variables["ENVELOPE_FROM"] = escape(
+            milter.msginfo["mailfrom"], quote=False)
+        variables["ENVELOPE_FROM_URL"] = escape(
+            quote(milter.msginfo["mailfrom"]), quote=False)
+
+        if text_content is not None:
+            logger.info(f"{self.action} text disclaimer")
+            text_template = self.text_template.format_map(variables)
+
+            if self.action == "prepend":
+                content = f"{text_template}{text_content}"
+            else:
+                content = f"{text_content}{text_template}"
+
+            text_body.set_content(
+                content.encode(errors="replace"),
+                maintype="text",
+                subtype="plain")
+            text_body.set_param("charset", "UTF-8", header="Content-Type")
+            del text_body["MIME-Version"]
+
+        if html_content is not None:
+            logger.info(f"{self.action} html disclaimer")
+
+            soup = BeautifulSoup(html_content, "html.parser")
+            body = soup.find('body')
+            if not body:
+                body = soup
+            elif _has_content_before_body_tag(soup):
+                body = soup
+
+            html_template = self.html_template.format_map(variables)
+            html_template = BeautifulSoup(html_template, "html.parser")
+            html_template = html_template.find("body") or html_template
+            if self.action == "prepend":
+                body.insert(0, html_template)
+            else:
+                body.append(html_template)
+
+            html_body.set_content(
+                str(soup).encode(errors="replace"),
+                maintype="text",
+                subtype="html")
+            html_body.set_param("charset", "UTF-8", header="Content-Type")
+            del html_body["MIME-Version"]
+
+    def execute(self, milter, logger):
+        old_headers = milter.msg.items()
+
+        try:
+            self.patch_message_body(milter, logger)
+        except Exception as e:
+            logger.warning(e)
+            if self.error_policy == "ignore":
+                logger.info(
+                    "unable to add disclaimer to message body, "
+                    "ignore error according to policy")
+                return
+            elif self.error_policy == "reject":
+                logger.info(
+                    "unable to add disclaimer to message body, "
+                    "reject message according to policy")
+                return [
+                    ("reject", "Message rejected due to error")]
+
+            logger.info("wrap original message in a new message envelope")
+            try:
+                _wrap_message(milter)
+                self.patch_message_body(milter, logger)
+            except Exception as e:
+                logger.error(e)
+                raise Exception(
+                    "unable to wrap message in a new message envelope, "
+                    "give up ...")
+
+        if not self.pretend:
+            milter.update_headers(old_headers)
+            milter.replacebody()
+
+
+class RewriteLinks:
+    """Rewrite link targets in the mail html body."""
+    _headersonly = False
+
+    def __init__(self, repl, pretend=False):
+        self.repl = repl
+        self.pretend = pretend
+
+    def execute(self, milter, logger):
+        html_body, html_content = _get_body_content(milter.msg, "html")
+        if html_content is not None:
+            soup = BeautifulSoup(html_content, "html.parser")
+
+            rewritten = 0
+            for link in soup.find_all("a", href=True):
+                if not link["href"]:
+                    continue
+
+                if "{URL_B64}" in self.repl:
+                    url_b64 = b64encode(link["href"].encode()).decode()
+                    target = self.repl.replace("{URL_B64}", url_b64)
+                else:
+                    target = self.repl
+
+                link["href"] = target
+                rewritten += 1
+
+            if rewritten:
+                logger.info(f"rewrote {rewritten} link(s) in html body")
+
+                html_body.set_content(
+                    str(soup).encode(), maintype="text", subtype="html")
+                html_body.set_param("charset", "UTF-8", header="Content-Type")
+                del html_body["MIME-Version"]
+
+                if not self.pretend:
+                    milter.replacebody()
+
+
+class Modify:
+    MODIFICATION_TYPES = {
+        "add_header": AddHeader,
+        "mod_header": ModHeader,
+        "del_header": DelHeader,
+        "add_disclaimer": AddDisclaimer,
+        "rewrite_links": RewriteLinks}
+
+    def __init__(self, cfg, local_addrs, debug):
+        self.cfg = cfg
+        self.logger = logging.getLogger(cfg["name"])
+        self.logger.setLevel(cfg.get_loglevel(debug))
+        cfg["options"]["pretend"] = cfg["pretend"]
+        self._modification = self.MODIFICATION_TYPES[cfg["type"]](
+            **cfg["options"])
+        self._headersonly = self._modification._headersonly
+
+    def __str__(self):
+        cfg = []
+        for key, value in self.cfg["options"].items():
+            cfg.append(f"{key}={value}")
+        class_name = type(self._modification).__name__
+        return f"{class_name}(" + ", ".join(cfg) + ")"
+
+    def execute(self, milter):
+        logger = CustomLogger(
+            self.logger, {"name": self.cfg["name"], "qid": milter.qid})
+        self._modification.execute(milter, logger)

pyquarantine/notify.py

@@ -0,0 +1,356 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = [
+    "BaseNotification",
+    "EMailNotification",
+    "Notify"]
+
+import email
+import logging
+import re
+
+from bs4 import BeautifulSoup
+from collections import defaultdict
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.mime.image import MIMEImage
+from html import escape
+from os.path import basename
+from urllib.parse import quote
+
+from pyquarantine.base import CustomLogger
+from pyquarantine import mailer
+
+
+class BaseNotification:
+    "Notification base class"
+    _headersonly = True
+
+    def __init__(self, pretend=False):
+        self.pretend = pretend
+
+    def execute(self, milter, logger):
+        return
+
+
+class EMailNotification(BaseNotification):
+    "Notification class to send notifications via mail."
+    _headersonly = False
+    _bad_tags = [
+        "applet",
+        "embed",
+        "frame",
+        "frameset",
+        "head",
+        "iframe",
+        "script",
+        "style"
+    ]
+    _good_tags = [
+        "a",
+        "b",
+        "br",
+        "center",
+        "div",
+        "font",
+        "h1",
+        "h2",
+        "h3",
+        "h4",
+        "h5",
+        "h6",
+        "i",
+        "img",
+        "li",
+        "p",
+        "pre",
+        "span",
+        "table",
+        "td",
+        "th",
+        "tr",
+        "tt",
+        "u",
+        "ul"
+    ]
+    _good_attributes = [
+        "align",
+        "alt",
+        "bgcolor",
+        "border",
+        "cellpadding",
+        "cellspacing",
+        "class",
+        "color",
+        "colspan",
+        "dir",
+        "face",
+        "headers",
+        "height",
+        "id",
+        "name",
+        "rowspan",
+        "size",
+        "src",
+        "style",
+        "title",
+        "type",
+        "valign",
+        "value",
+        "width"
+    ]
+
+    def __init__(self, smtp_host, smtp_port, envelope_from, from_header,
+                 subject, template, embed_imgs=[], repl_img=None,
+                 strip_imgs=False, parser_lib="lxml", pretend=False):
+        super().__init__(pretend)
+        self.smtp_host = smtp_host
+        self.smtp_port = smtp_port
+        self.mailfrom = envelope_from
+        self.from_header = from_header
+        self.subject = subject
+        try:
+            with open(template, "r") as fh:
+                self.template = fh.read()
+            self.embed_imgs = []
+            for img_path in embed_imgs:
+                with open(img_path, "rb") as fh:
+                    img = MIMEImage(fh.read())
+                filename = basename(img_path)
+                img.add_header("Content-ID", f"<{filename}>")
+                self.embed_imgs.append(img)
+            self.replacement_img = repl_img
+            self.strip_images = strip_imgs
+
+            if not strip_imgs and repl_img:
+                with open(repl_img, "rb") as fh:
+                    self.replacement_img = MIMEImage(fh.read())
+                self.replacement_img.add_header(
+                    "Content-ID", "<removed_for_security_reasons>")
+
+        except IOError as e:
+            raise RuntimeError(e)
+
+        self.parser_lib = parser_lib
+
+    def get_msg_body_soup(self, msg, logger):
+        "Extract and decode message body, return it as BeautifulSoup object."
+        # try to find the body part
+        logger.debug("trying to find message body")
+        try:
+            body = msg.get_body(preferencelist=("html", "plain"))
+        except Exception as e:
+            logger.error(
+                f"an error occured in email.message.EmailMessage.get_body: "
+                f"{e}")
+            body = None
+
+        if body:
+            charset = body.get_content_charset() or "utf-8"
+            content = body.get_payload(decode=True)
+            try:
+                content = content.decode(encoding=charset, errors="replace")
+            except LookupError:
+                logger.info(
+                    f"unknown encoding '{charset}', falling back to UTF-8")
+                content = content.decode("utf-8", errors="replace")
+            content_type = body.get_content_type()
+            if content_type == "text/plain":
+                # convert text/plain to text/html
+                logger.debug(
+                    f"content type is {content_type}, "
+                    f"converting to text/html")
+                content = re.sub(r"^(.*)$", r"\1<br/>",
+                                 escape(content, quote=False),
+                                 flags=re.MULTILINE)
+            else:
+                logger.debug(f"content type is {content_type}")
+        else:
+            logger.error("unable to find message body")
+            content = "ERROR: unable to find message body"
+
+        # create BeautifulSoup object
+        length = len(content)
+        logger.debug(
+            f"trying to create BeatufilSoup object with "
+            f"parser lib {self.parser_lib}, "
+            f"text length is {length} bytes")
+        soup = BeautifulSoup(content, self.parser_lib)
+        logger.debug("sucessfully created BeautifulSoup object")
+
+        return soup
+
+    def sanitize(self, soup, logger):
+        "Sanitize mail html text."
+        logger.debug("sanitize message text")
+
+        # completly remove bad elements
+        for element in soup(EMailNotification._bad_tags):
+            logger.debug(
+                f"removing dangerous tag '{element.name}' "
+                f"and its content")
+            element.extract()
+
+        # remove not whitelisted elements, but keep their content
+        for element in soup.find_all(True):
+            if element.name not in EMailNotification._good_tags:
+                logger.debug(
+                    f"removing tag '{element.name}', keep its content")
+                element.replaceWithChildren()
+
+        # remove not whitelisted attributes
+        for element in soup.find_all(True):
+            for attribute in list(element.attrs.keys()):
+                if attribute not in EMailNotification._good_attributes:
+                    if element.name == "a" and attribute == "href":
+                        logger.debug(
+                            f"setting attribute href to '#' "
+                            f"on tag '{element.name}'")
+                        element["href"] = "#"
+                    else:
+                        logger.debug(
+                            f"removing attribute '{attribute}' "
+                            f"from tag '{element.name}'")
+                        del(element.attrs[attribute])
+        return soup
+
+    def notify(self, msg, qid, mailfrom, recipients, logger,
+               template_vars={}, synchronous=False):
+        "Notify recipients via email."
+        # extract body from email
+        soup = self.get_msg_body_soup(msg, logger)
+
+        # replace picture sources
+        image_replaced = False
+        if self.strip_images:
+            logger.debug("looking for images to strip")
+            for element in soup("img"):
+                if "src" in element.attrs.keys():
+                    logger.debug(f"strip image: {element['src']}")
+                element.extract()
+        elif self.replacement_img:
+            logger.debug("looking for images to replace")
+            for element in soup("img"):
+                if "src" in element.attrs.keys():
+                    logger.debug(f"replacing image: {element['src']}")
+                element["src"] = "cid:removed_for_security_reasons"
+                image_replaced = True
+
+        # sanitize message text
+        sanitized_text = self.sanitize(soup, logger)
+        del soup
+
+        # send email notifications
+        for recipient in recipients:
+            logger.debug(
+                f"generating email notification for '{recipient}'")
+            logger.debug("parsing message template")
+
+            variables = defaultdict(str, template_vars)
+            variables["HTML_TEXT"] = sanitized_text
+            variables["ENVELOPE_FROM"] = escape(mailfrom, quote=False)
+            variables["ENVELOPE_FROM_URL"] = escape(
+                quote(mailfrom), quote=False)
+            variables["ENVELOPE_TO"] = escape(recipient, quote=False)
+            variables["ENVELOPE_TO_URL"] = escape(quote(recipient))
+
+            newmsg = MIMEMultipart('related')
+            if msg["from"] is not None:
+                newmsg["From"] = self.from_header.format_map(
+                    defaultdict(str, FROM=msg["from"]))
+                variables["FROM"] = escape(msg["from"], quote=False)
+            else:
+                newmsg["From"] = self.from_header.format_map(defaultdict(str))
+
+            if msg["to"] is not None:
+                newmsg["To"] = msg["to"]
+                variables["TO"] = escape(msg["to"], quote=False)
+            else:
+                newmsg["To"] = recipient
+
+            if msg["subject"] is not None:
+                newmsg["Subject"] = self.subject.format_map(
+                    defaultdict(str, SUBJECT=msg["subject"]))
+                variables["SUBJECT"] = escape(msg["subject"], quote=False)
+
+            newmsg["Date"] = email.utils.formatdate()
+
+            # parse template
+            htmltext = self.template.format_map(variables)
+            newmsg.attach(MIMEText(htmltext, "html", 'UTF-8'))
+
+            if image_replaced:
+                logger.debug("attaching notification_replacement_img")
+                newmsg.attach(self.replacement_img)
+
+            for img in self.embed_imgs:
+                logger.debug("attaching imgage")
+                newmsg.attach(img)
+
+            logger.debug(f"sending email notification to: {recipient}")
+            if synchronous:
+                try:
+                    mailer.smtp_send(self.smtp_host, self.smtp_port,
+                                     self.mailfrom, recipient,
+                                     newmsg.as_string())
+                except Exception as e:
+                    raise RuntimeError(
+                        f"error while sending email notification "
+                        f"to '{recipient}': {e}")
+            else:
+                mailer.sendmail(self.smtp_host, self.smtp_port, qid,
+                                self.mailfrom, recipient, newmsg.as_string(),
+                                "email notification")
+
+    def execute(self, milter, logger):
+        super().execute(milter, logger)
+
+        self.notify(msg=milter.msg, qid=milter.qid,
+                    mailfrom=milter.msginfo["mailfrom"],
+                    recipients=milter.msginfo["rcpts"],
+                    template_vars=milter.msginfo["vars"],
+                    logger=logger)
+
+
+class Notify:
+    NOTIFICATION_TYPES = {
+        "email": EMailNotification}
+
+    def __init__(self, cfg, local_addrs, debug):
+        self.cfg = cfg
+        self.logger = logging.getLogger(cfg["name"])
+        self.logger.setLevel(cfg.get_loglevel(debug))
+
+        nodification_type = cfg["options"]["type"]
+        del cfg["options"]["type"]
+        cfg["options"]["pretend"] = cfg["pretend"]
+        self._notification = self.NOTIFICATION_TYPES[nodification_type](
+            **cfg["options"])
+        self._headersonly = self._notification._headersonly
+
+    def __str__(self):
+        cfg = []
+        for key, value in self.cfg["options"].items():
+            cfg.append(f"{key}={value}")
+        class_name = type(self._notification).__name__
+        return f"{class_name}(" + ", ".join(cfg) + ")"
+
+    def get_notification(self):
+        return self._notification
+
+    def execute(self, milter):
+        logger = CustomLogger(
+            self.logger, {"name": self.cfg["name"], "qid": milter.qid})
+        self._notification.execute(milter, logger)

pyquarantine/rule.py

@@ -0,0 +1,59 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = ["Rule"]
+
+from pyquarantine.action import Action
+from pyquarantine.conditions import Conditions
+
+
+class Rule:
+    """
+    Rule to implement multiple actions on emails.
+    """
+    def __init__(self, cfg, local_addrs, debug):
+        self.cfg = cfg
+
+        self.conditions = cfg["conditions"] if "conditions" in cfg else None
+        if self.conditions is not None:
+            self.conditions["name"] = f"{cfg['name']}: condition"
+            self.conditions["loglevel"] = cfg["loglevel"]
+            self.conditions = Conditions(self.conditions, local_addrs, debug)
+
+        self.actions = []
+        for idx, action_cfg in enumerate(cfg["actions"]):
+            action_cfg["name"] = f"{cfg['name']}: {action_cfg['name']}"
+            self.actions.append(Action(action_cfg, local_addrs, debug))
+
+    def __str__(self):
+        cfg = []
+        for key in ["name", "loglevel", "pretend"]:
+            value = self.cfg[key]
+            cfg.append(f"{key}={value}")
+        if self.conditions is not None:
+            cfg.append(f"conditions={self.conditions}")
+        actions = []
+        for action in self.actions:
+            actions.append(str(action))
+        cfg.append("actions=[" + ", ".join(actions) + "]")
+        return "Rule(" + ", ".join(cfg) + ")"
+
+    def execute(self, milter):
+        """Execute all actions of this rule."""
+        if self.conditions is None or \
+                self.conditions.match(milter):
+            for action in self.actions:
+                milter_action = action.execute(milter)
+                if milter_action is not None:
+                    return milter_action

pyquarantine/run.py

@@ -0,0 +1,168 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = ["main"]
+
+import Milter
+import argparse
+import logging
+import logging.handlers
+import sys
+
+from pyquarantine._install import install, uninstall
+from pyquarantine import mailer
+from pyquarantine import QuarantineMilter
+from pyquarantine import __version__ as version
+from pyquarantine.config import get_milter_config
+
+
+def main():
+    python_version = ".".join([str(v) for v in sys.version_info[0:3]])
+    python_version = f"{python_version}-{sys.version_info[3]}"
+
+    "Run pyquarantine."
+    parser = argparse.ArgumentParser(
+        description="pyquarantine-milter daemon",
+        formatter_class=lambda prog: argparse.HelpFormatter(
+            prog, max_help_position=45, width=140))
+    parser.add_argument(
+        "-c", "--config", help="Config file to read.",
+        default="/etc/pyquarantine/pyquarantine.conf")
+    parser.add_argument(
+        "-s",
+        "--socket",
+        help="Socket used to communicate with the MTA.",
+        default="")
+    parser.add_argument(
+        "-d",
+        "--debug",
+        help="Log debugging messages.",
+        action="store_true")
+
+    exclusive = parser.add_mutually_exclusive_group()
+    exclusive.add_argument(
+        "-v", "--version",
+        help="Print version.",
+        action="version",
+        version=f"%(prog)s {version} (python {python_version})")
+    exclusive.add_argument(
+        "-i",
+        "--install",
+        help="install service files and config",
+        action="store_true")
+    exclusive.add_argument(
+        "-u",
+        "--uninstall",
+        help="uninstall service files and unmodified config",
+        action="store_true")
+    exclusive.add_argument(
+        "-t",
+        "--test",
+        help="Check configuration.",
+        action="store_true")
+
+    args = parser.parse_args()
+
+    root_logger = logging.getLogger()
+    root_logger.setLevel(logging.DEBUG)
+
+    # setup console log
+    stdouthandler = logging.StreamHandler(sys.stdout)
+    formatter = logging.Formatter("%(levelname)s: %(message)s")
+    stdouthandler.setFormatter(formatter)
+    root_logger.addHandler(stdouthandler)
+
+    logger = logging.getLogger(__name__)
+
+    if not args.debug:
+        logger.setLevel(logging.INFO)
+
+    name = "pyquarantine"
+    if args.install:
+        sys.exit(install(name))
+
+    if args.uninstall:
+        sys.exit(uninstall(name))
+
+    try:
+        logger.debug("read milter configuration")
+        cfg = get_milter_config(args.config)
+        logger.setLevel(cfg.get_loglevel(args.debug))
+
+        if args.socket:
+            socket = args.socket
+        elif "socket" in cfg:
+            socket = cfg["socket"]
+        else:
+            raise RuntimeError(
+                "listening socket is neither specified on the command line "
+                "nor in the configuration file")
+
+        if not cfg["rules"]:
+            raise RuntimeError("no rules configured")
+
+        for rule in cfg["rules"]:
+            if not rule["actions"]:
+                raise RuntimeError(
+                    f"{rule['name']}: no actions configured")
+        QuarantineMilter.set_config(cfg, args.debug)
+
+    except (RuntimeError, AssertionError) as e:
+        logger.error(f"config error: {e}")
+        sys.exit(255)
+
+    if args.test:
+        print("Configuration OK")
+        sys.exit(0)
+
+    # setup console log for runtime
+    formatter = logging.Formatter("%(asctime)s - %(levelname)s: %(message)s")
+    stdouthandler.setFormatter(formatter)
+    stdouthandler.setLevel(logging.DEBUG)
+
+    # setup syslog
+    sysloghandler = logging.handlers.SysLogHandler(
+        address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_MAIL)
+    sysloghandler.setFormatter(
+        logging.Formatter(f"{name}[%(process)d]: %(message)s"))
+    root_logger.addHandler(sysloghandler)
+
+    logger.info("milter starting")
+
+    # register milter factory class
+    Milter.factory = QuarantineMilter
+    Milter.set_exception_policy(Milter.TEMPFAIL)
+
+    if args.debug:
+        Milter.setdbg(1)
+
+    # increase the recursion limit so that BeautifulSoup can
+    # parse larger html content
+    sys.setrecursionlimit(4000)
+
+    rc = 0
+    try:
+        Milter.runmilter("pyquarantine", socketname=socket, timeout=600)
+    except Milter.milter.error as e:
+        logger.error(e)
+        rc = 255
+
+    mailer.queue.put(None)
+    logger.info("milter stopped")
+
+    sys.exit(rc)
+
+
+if __name__ == "__main__":
+    main()

pyquarantine/storage.py

@@ -0,0 +1,567 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = [
+    "BaseMailStorage",
+    "FileMailStorage",
+    "Store",
+    "Quarantine"]
+
+import json
+import logging
+import os
+
+from calendar import timegm
+from datetime import datetime
+from email import message_from_bytes
+from email.policy import SMTP
+from glob import glob
+from time import gmtime
+
+from pyquarantine import mailer
+from pyquarantine.base import CustomLogger, MilterMessage
+from pyquarantine.conditions import Conditions
+from pyquarantine.config import ActionConfig
+from pyquarantine.notify import Notify
+
+
+class BaseMailStorage:
+    "Mail storage base class"
+    _headersonly = True
+
+    def __init__(self, original=False, metadata=False, metavar=None,
+                 pretend=False):
+        self.original = original
+        self.metadata = metadata
+        self.metavar = metavar
+        self.pretend = False
+
+    def add(self, data, qid, mailfrom, recipients, subject, variables):
+        "Add message to storage."
+        return ("", "")
+
+    def execute(self, milter, logger):
+        return
+
+    def find(self, mailfrom=None, recipients=None, older_than=None):
+        "Find messages in storage."
+        return
+
+    def get_metadata(self, storage_id):
+        "Return metadata of message in storage."
+        return
+
+    def delete(self, storage_id, recipients=None):
+        "Delete message from storage."
+        return
+
+    def get_mail(self, storage_id):
+        "Return message and metadata."
+        return
+
+
+class FileMailStorage(BaseMailStorage):
+    "Storage class to store mails on filesystem."
+    _headersonly = False
+
+    def __init__(self, directory, original=False, metadata=False, metavar=None,
+                 mode=None, pretend=False):
+        super().__init__(original, metadata, metavar, pretend)
+        # check if directory exists and is writable
+        if not os.path.isdir(directory) or \
+                not os.access(directory, os.R_OK):
+            raise RuntimeError(
+                f"directory '{directory}' does not exist or is "
+                f"not readable")
+        self.directory = directory
+        try:
+            self.mode = int(mode, 8) if mode is not None else None
+            if self.mode is not None and self.mode > 511:
+                raise ValueError
+        except ValueError:
+            raise RuntimeError(f"invalid mode '{mode}'")
+
+        self._metadata_suffix = ".metadata"
+
+    def __str__(self):
+        cfg = []
+        cfg.append(f"metadata={self.metadata}")
+        cfg.append(f"metavar={self.metavar}")
+        cfg.append(f"pretend={self.pretend}")
+        cfg.append(f"directory={self.directory}")
+        cfg.append(f"original={self.original}")
+        return "FileMailStorage(" + ", ".join(cfg) + ")"
+
+    def get_storageid(self, qid):
+        timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
+        return f"{timestamp}_{qid}"
+
+    def _get_file_paths(self, storage_id):
+        datafile = os.path.join(self.directory, storage_id)
+        metafile = f"{datafile}{self._metadata_suffix}"
+        return metafile, datafile
+
+    def _save_datafile(self, datafile, data):
+        try:
+            if self.mode is None:
+                with open(datafile, "wb") as f:
+                    f.write(data)
+            else:
+                umask = os.umask(0)
+                with open(
+                        os.open(datafile,
+                                os.O_CREAT | os.O_WRONLY | os.O_TRUNC,
+                                self.mode),
+                        "wb") as f:
+                    f.write(data)
+                os.umask(umask)
+
+        except IOError as e:
+            raise RuntimeError(f"unable save data file: {e}")
+
+    def _save_metafile(self, metafile, metadata):
+        try:
+            if self.mode is None:
+                with open(metafile, "w") as f:
+                    json.dump(metadata, f, indent=2)
+            else:
+                umask = os.umask(0)
+                with open(
+                        os.open(metafile,
+                                os.O_CREAT | os.O_WRONLY | os.O_TRUNC,
+                                self.mode),
+                        "w") as f:
+                    json.dump(metadata, f, indent=2)
+                os.umask(umask)
+
+        except IOError as e:
+            raise RuntimeError(f"unable to save metadata file: {e}")
+
+    def _remove(self, storage_id):
+        metafile, datafile = self._get_file_paths(storage_id)
+
+        try:
+            if self.metadata:
+                os.remove(metafile)
+
+            os.remove(datafile)
+        except IOError as e:
+            raise RuntimeError(f"unable to remove file: {e}")
+
+    def add(self, data, qid, mailfrom, recipients, subject, variables, logger):
+        "Add message to file storage and return storage id."
+        super().add(data, qid, mailfrom, recipients, subject, variables)
+
+        storage_id = self.get_storageid(qid)
+        metafile, datafile = self._get_file_paths(storage_id)
+
+        if self.metavar:
+            variables[f"{self.metavar}_ID"] = storage_id
+            variables[f"{self.metavar}_DATAFILE"] = datafile
+            if self.metadata:
+                variables[f"{self.metavar}_METAFILE"] = metafile
+
+        if self.pretend:
+            return
+
+        # save mail
+        logger.debug(f"save message to {datafile}")
+        self._save_datafile(datafile, data)
+        logger.info(f"stored message with id {storage_id}")
+
+        if not self.metadata:
+            return storage_id, None, datafile
+
+        # save metadata
+        metadata = {
+            "mailfrom": mailfrom,
+            "recipients": recipients,
+            "subject": subject,
+            "timestamp": timegm(gmtime()),
+            "queue_id": qid,
+            "vars": variables}
+
+        try:
+            logger.debug(f"save metadata to {metafile}")
+            self._save_metafile(metafile, metadata)
+        except RuntimeError as e:
+            os.remove(datafile)
+            raise e
+
+    def execute(self, milter, logger):
+        if self.original:
+            milter.fp.seek(0)
+            data = milter.fp.read
+            mailfrom = milter.mailfrom
+            recipients = list(milter.rcpts)
+            # getting the subject is the only operation that needs any
+            # parsing of the message, catch all exceptions here
+            try:
+                subject = milter.msg["subject"] or ""
+            except Exception:
+                subject = ""
+        else:
+            data = milter.msg_as_bytes
+            mailfrom = milter.msginfo["mailfrom"]
+            recipients = list(milter.msginfo["rcpts"])
+            subject = milter.msg["subject"] or ""
+
+        self.add(data(), milter.qid, mailfrom, recipients, subject,
+                 milter.msginfo["vars"], logger)
+
+    def get_metadata(self, storage_id):
+        "Return metadata of message in storage."
+        super().get_metadata(storage_id)
+
+        if not self.metadata:
+            return None
+
+        metafile, _ = self._get_file_paths(storage_id)
+        if not os.path.isfile(metafile):
+            raise RuntimeError(
+                f"invalid storage id '{storage_id}'")
+
+        try:
+            with open(metafile, "r") as f:
+                metadata = json.load(f)
+        except IOError as e:
+            raise RuntimeError(f"unable to read metadata file: {e}")
+        except json.JSONDecodeError as e:
+            raise RuntimeError(
+                f"invalid metafile '{metafile}': {e}")
+
+        # convert metafile structure, this can be removed in the future
+        converted = False
+        if "subject" not in metadata:
+            try:
+                metadata["subject"] = metadata["headers"]["subject"]
+            except KeyError:
+                metadata["subject"] = ""
+            converted = True
+        if "timestamp" not in metadata:
+            try:
+                metadata["timestamp"] = metadata["date"]
+            except KeyError:
+                metadata["timestamp"] = 0
+            converted = True
+        if "vars" not in metadata:
+            try:
+                metadata["vars"] = metadata["named_subgroups"]
+            except KeyError:
+                metadata["vars"] = {}
+            converted = True
+        if "headers" in metadata:
+            del metadata["headers"]
+            converted = True
+        if "date" in metadata:
+            del metadata["date"]
+            converted = True
+        if "named_subgroups" in metadata:
+            del metadata["named_subgroups"]
+            converted = True
+        if "subgroups" in metadata:
+            del metadata["subgroups"]
+            converted = True
+        if converted:
+            self._save_metafile(metafile, metadata)
+
+        return metadata
+
+    def find(self, mailfrom=None, recipients=None, older_than=None):
+        "Find messages in storage."
+        super().find(mailfrom, recipients, older_than)
+        if isinstance(mailfrom, str):
+            mailfrom = [mailfrom]
+        if isinstance(recipients, str):
+            recipients = [recipients]
+
+        if not self.metadata:
+            return {}
+
+        msgs = {}
+        metafiles = glob(os.path.join(
+            self.directory, f"*{self._metadata_suffix}"))
+        for metafile in metafiles:
+            if not os.path.isfile(metafile):
+                continue
+
+            storage_id = os.path.basename(
+                metafile[:-len(self._metadata_suffix)])
+            metadata = self.get_metadata(storage_id)
+            if older_than is not None:
+                age = timegm(gmtime()) - metadata["timestamp"]
+                if age < (older_than * 86400):
+                    continue
+
+            if mailfrom is not None:
+                if metadata["mailfrom"] not in mailfrom:
+                    continue
+
+            if recipients is not None:
+                if len(recipients) == 1 and \
+                        recipients[0] not in metadata["recipients"]:
+                    continue
+                elif len(set(recipients + metadata["recipients"])) == \
+                        len(recipients + metadata["recipients"]):
+                    continue
+
+            msgs[storage_id] = metadata
+
+        return msgs
+
+    def delete(self, storage_id, recipients=None):
+        "Delete message from storage."
+        super().delete(storage_id, recipients)
+        if not recipients or not self.metadata:
+            self._remove(storage_id)
+            return
+
+        try:
+            metadata = self.get_metadata(storage_id)
+        except RuntimeError as e:
+            raise RuntimeError(f"unable to delete message: {e}")
+
+        metafile, _ = self._get_file_paths(storage_id)
+
+        if type(recipients) == str:
+            recipients = [recipients]
+
+        for recipient in recipients:
+            if recipient not in metadata["recipients"]:
+                raise RuntimeError(f"invalid recipient '{recipient}'")
+            metadata["recipients"].remove(recipient)
+            if not metadata["recipients"]:
+                self._remove(storage_id)
+            else:
+                self._save_metafile(metafile, metadata)
+
+    def get_mail_bytes(self, storage_id):
+        _, datafile = self._get_file_paths(storage_id)
+        try:
+            with open(datafile, "rb") as fh:
+                data = fh.read()
+        except IOError as e:
+            raise RuntimeError(f"unable to open data file: {e}")
+        return data
+
+    def get_mail(self, storage_id):
+        super().get_mail(storage_id)
+
+        metadata = self.get_metadata(storage_id)
+        msg = message_from_bytes(
+            self.get_mail_bytes(storage_id),
+            _class=MilterMessage,
+            policy=SMTP.clone(refold_source='none'))
+        return (metadata, msg)
+
+
+class Store:
+    STORAGE_TYPES = {
+        "file": FileMailStorage}
+
+    def __init__(self, cfg, local_addrs, debug):
+        self.cfg = cfg
+        self.logger = logging.getLogger(cfg["name"])
+        self.logger.setLevel(cfg.get_loglevel(debug))
+
+        storage_type = cfg["options"]["type"]
+        del cfg["options"]["type"]
+        cfg["options"]["pretend"] = cfg["pretend"]
+        self._storage = self.STORAGE_TYPES[storage_type](
+            **cfg["options"])
+        self._headersonly = self._storage._headersonly
+
+    def __str__(self):
+        cfg = []
+        for key, value in self.cfg["options"].items():
+            cfg.append(f"{key}={value}")
+        class_name = type(self._storage).__name__
+        return f"{class_name}(" + ", ".join(cfg) + ")"
+
+    def get_storage(self):
+        return self._storage
+
+    def execute(self, milter):
+        logger = CustomLogger(
+            self.logger, {"name": self.cfg["name"], "qid": milter.qid})
+        self._storage.execute(milter, logger)
+
+
+class Quarantine:
+    "Quarantine class."
+    _headersonly = False
+
+    def __init__(self, cfg, local_addrs, debug):
+        self.cfg = cfg
+        self.logger = logging.getLogger(cfg["name"])
+        self.logger.setLevel(cfg.get_loglevel(debug))
+
+        storage_cfg = ActionConfig({
+            "name": cfg["name"],
+            "loglevel": cfg["loglevel"],
+            "pretend": cfg["pretend"],
+            "type": "store",
+            "options": cfg["options"]["store"].get_config()})
+        self._storage = Store(storage_cfg, local_addrs, debug)
+
+        self.smtp_host = cfg["options"]["smtp_host"]
+        self.smtp_port = cfg["options"]["smtp_port"]
+
+        self._notification = None
+        if "notify" in cfg["options"]:
+            notify_cfg = ActionConfig({
+                "name": cfg["name"],
+                "loglevel": cfg["loglevel"],
+                "pretend": cfg["pretend"],
+                "type": "notify",
+                "options": cfg["options"]["notify"].get_config()})
+            self._notification = Notify(notify_cfg, local_addrs, debug)
+
+        self._whitelist = None
+        if "whitelist" in cfg["options"]:
+            whitelist_cfg = cfg["options"]["whitelist"]
+            whitelist_cfg["name"] = cfg["name"]
+            whitelist_cfg["loglevel"] = cfg["loglevel"]
+            self._whitelist = Conditions(
+                whitelist_cfg,
+                local_addrs=[],
+                debug=debug)
+
+        self._milter_action = None
+        if "milter_action" in cfg["options"]:
+            self._milter_action = cfg["options"]["milter_action"].upper()
+            assert self._milter_action in ["ACCEPT", "REJECT", "DISCARD"], \
+                f"invalid milter_action '{cfg['args']['milter_action']}'"
+
+        self._reason = None
+        if self._milter_action == "REJECT":
+            if "reject_reason" in cfg["options"]:
+                self._reason = cfg["options"]["reject_reason"]
+            else:
+                self._reason = "Message rejected"
+
+    def __str__(self):
+        cfg = []
+        cfg.append(f"store={str(self._storage)}")
+        if self._notification is not None:
+            cfg.append(f"notify={str(self._notification)}")
+        if self._whitelist is not None:
+            cfg.append(f"whitelist={str(self._whitelist)}")
+        for key in ["milter_action", "reject_reason"]:
+            if key not in self.cfg["options"]:
+                continue
+            value = self.cfg["options"][key]
+            cfg.append(f"{key}={value}")
+        class_name = type(self).__name__
+        return f"{class_name}(" + ", ".join(cfg) + ")"
+
+    @property
+    def name(self):
+        return self.cfg["name"]
+
+    @property
+    def storage(self):
+        return self._storage.get_storage()
+
+    @property
+    def notification(self):
+        if self._notification is None:
+            return None
+        return self._notification.get_notification()
+
+    @property
+    def whitelist(self):
+        if self._whitelist is None:
+            return None
+        return self._whitelist.get_whitelist()
+
+    @property
+    def milter_action(self):
+        return self._milter_action
+
+    def notify(self, storage_id, recipient=None):
+        "Notify recipient about message in storage."
+        if not self._notification:
+            raise RuntimeError(
+                "notification not defined, "
+                "unable to send notification")
+        metadata, msg = self.storage.get_mail(storage_id)
+
+        if recipient is not None:
+            if recipient not in metadata["recipients"]:
+                raise RuntimeError(f"invalid recipient '{recipient}'")
+            recipients = [recipient]
+        else:
+            recipients = metadata["recipients"]
+
+        self.notification.notify(msg, metadata["queue_id"],
+                                 metadata["mailfrom"], recipients,
+                                 self.logger, metadata["vars"],
+                                 synchronous=True)
+
+    def release(self, storage_id, recipients=None):
+        metadata, msg = self.storage.get_mail(storage_id)
+        if recipients and type(recipients) == str:
+            recipients = [recipients]
+        else:
+            recipients = metadata["recipients"]
+
+        for recipient in recipients:
+            if recipient not in metadata["recipients"]:
+                raise RuntimeError(f"invalid recipient '{recipient}'")
+            try:
+                mailer.smtp_send(
+                    self.smtp_host,
+                    self.smtp_port,
+                    metadata["mailfrom"],
+                    recipient,
+                    msg.as_string())
+
+            except Exception as e:
+                raise RuntimeError(
+                    f"error while sending message to '{recipient}': {e}")
+            self.storage.delete(storage_id, recipient)
+
+        return recipients
+
+    def execute(self, milter):
+        logger = CustomLogger(
+            self.logger, {"name": self.cfg["name"], "qid": milter.qid})
+        rcpts = milter.msginfo["rcpts"]
+        wl_rcpts = []
+        if self._whitelist:
+            wl_rcpts = self._whitelist.get_wl_rcpts(
+                milter.msginfo["mailfrom"], rcpts, logger)
+            if wl_rcpts:
+                logger.info(f"whitelisted recipients: {wl_rcpts}")
+            rcpts = [rcpt for rcpt in rcpts if rcpt not in wl_rcpts]
+            if not rcpts:
+                # all recipients whitelisted
+                return
+            milter.msginfo["rcpts"] = rcpts.copy()
+
+        if self._milter_action in ["REJECT", "DISCARD"]:
+            logger.info(f"quarantine message for {rcpts}")
+
+        self._storage.execute(milter)
+
+        if self._notification is not None:
+            self._notification.execute(milter)
+
+        milter.msginfo["rcpts"].extend(wl_rcpts)
+
+        if self._milter_action is not None:
+            milter.delrcpt(rcpts)
+            if not milter.msginfo["rcpts"]:
+                return (self._milter_action, self._reason)

pyquarantine/whitelist.py

@@ -0,0 +1,278 @@
+# pyquarantine is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyquarantine is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyquarantine.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = [
+    "DatabaseWhitelist",
+    "WhitelistBase"]
+
+import logging
+import peewee
+import re
+
+from datetime import datetime
+from playhouse.db_url import connect
+
+
+class WhitelistBase:
+    "Whitelist base class"
+    def __init__(self, cfg, debug):
+        self.cfg = cfg
+        self.logger = logging.getLogger(cfg["name"])
+        self.logger.setLevel(cfg.get_loglevel(debug))
+
+        peewee_logger = logging.getLogger("peewee")
+        peewee_logger.setLevel(cfg.get_loglevel(debug))
+
+        self.valid_entry_regex = re.compile(
+            r"^[a-zA-Z0-9_.=+-]*?(@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)?$")
+        self.batv_regex = re.compile(
+            r"^prvs=[0-9]{3}[0-9A-Fa-f]{6,7}=(?P<LEFT_PART>.+?)@")
+
+    @property
+    def type(self):
+        return "Base"
+
+    def remove_batv(self, addr):
+        return self.batv_regex.sub(r"\g<LEFT_PART>@", addr, count=1)
+
+    def check(self, mailfrom, recipient):
+        "Check if mailfrom/recipient combination is whitelisted."
+        return
+
+    def find(self, mailfrom=None, recipients=None, older_than=None):
+        "Find whitelist entries."
+        return
+
+    def add(self, mailfrom, recipient, comment, permanent):
+        "Add entry to whitelist."
+        # check if mailfrom and recipient are valid
+        if not self.valid_entry_regex.match(mailfrom):
+            raise RuntimeError("invalid from address")
+        if not self.valid_entry_regex.match(recipient):
+            raise RuntimeError("invalid recipient")
+        return
+
+    def delete(self, whitelist_id):
+        "Delete entry from whitelist."
+        return
+
+
+class WhitelistModel(peewee.Model):
+    mailfrom = peewee.CharField()
+    recipient = peewee.CharField()
+    created = peewee.DateTimeField(default=datetime.now)
+    last_used = peewee.DateTimeField(default=datetime.now)
+    comment = peewee.TextField(default="")
+    permanent = peewee.BooleanField(default=False)
+
+
+class Meta:
+    indexes = (
+        # trailing comma is mandatory if only one index should be created
+        (('mailfrom', 'recipient'), True),
+    )
+
+
+class DatabaseWhitelist(WhitelistBase):
+    "Whitelist class to store whitelist in a database"
+    whitelist_type = "db"
+    _db_connections = {}
+    _db_tables = {}
+
+    def __init__(self, cfg, debug):
+        super().__init__(cfg, debug)
+
+        tablename = cfg["table"]
+        connection_string = cfg["connection"]
+
+        if connection_string in DatabaseWhitelist._db_connections.keys():
+            db = DatabaseWhitelist._db_connections[connection_string]
+        else:
+            try:
+                # connect to database
+                conn = re.sub(
+                    r"(.*?://.*?):.*?(@.*)",
+                    r"\1:<PASSWORD>\2",
+                    connection_string)
+                self.logger.debug(
+                    f"connecting to database '{conn}'")
+                db = connect(connection_string)
+            except Exception as e:
+                raise RuntimeError(
+                    f"unable to connect to database: {e}")
+
+            DatabaseWhitelist._db_connections[connection_string] = db
+
+        # generate model meta class
+        self.meta = Meta
+        self.meta.database = db
+        self.meta.table_name = tablename
+        self.model = type(
+            f"WhitelistModel_{self.cfg['name']}",
+            (WhitelistModel,),
+            {"Meta": self.meta})
+
+        if connection_string not in DatabaseWhitelist._db_tables.keys():
+            DatabaseWhitelist._db_tables[connection_string] = []
+
+        if tablename not in DatabaseWhitelist._db_tables[connection_string]:
+            DatabaseWhitelist._db_tables[connection_string].append(tablename)
+            try:
+                db.create_tables([self.model])
+            except Exception as e:
+                raise RuntimeError(
+                    f"unable to initialize table '{tablename}': {e}")
+
+    def __str__(self):
+        cfg = []
+        for arg in ("connection", "table"):
+            if arg in self.cfg:
+                cfg.append(f"{arg}={self.cfg[arg]}")
+        return "DatabaseWhitelist(" + ", ".join(cfg) + ")"
+
+    def _entry_to_dict(self, entry):
+        result = {}
+        result[entry.id] = {
+            "id": entry.id,
+            "mailfrom": entry.mailfrom,
+            "recipient": entry.recipient,
+            "created": entry.created,
+            "last_used": entry.last_used,
+            "comment": entry.comment,
+            "permanent": entry.permanent
+        }
+        return result
+
+    def get_weight(self, entry):
+        value = 0
+        for address in [entry.mailfrom, entry.recipient]:
+            if address == "":
+                value += 2
+            elif address[0] == "@":
+                value += 1
+        return value
+
+    def check(self, mailfrom, recipient, logger):
+        # check if mailfrom/recipient combination is whitelisted
+        super().check(mailfrom, recipient)
+        mailfrom = self.remove_batv(mailfrom)
+        recipient = self.remove_batv(recipient)
+
+        # generate list of possible mailfroms
+        logger.debug(
+            f"query database for whitelist entries from <{mailfrom}> "
+            f"to <{recipient}>")
+        mailfroms = [""]
+        if "@" in mailfrom and not mailfrom.startswith("@"):
+            domain = mailfrom.split("@")[1]
+            mailfroms.append(f"@{domain}")
+        mailfroms.append(mailfrom)
+
+        # generate list of possible recipients
+        recipients = [""]
+        if "@" in recipient and not recipient.startswith("@"):
+            domain = recipient.split("@")[1]
+            recipients.append(f"@{domain}")
+        recipients.append(recipient)
+
+        # query the database
+        try:
+            entries = list(
+                self.model.select().where(
+                    self.model.mailfrom.in_(mailfroms),
+                    self.model.recipient.in_(recipients)))
+        except Exception as e:
+            raise RuntimeError(f"unable to query database: {e}")
+
+        if not entries:
+            # no whitelist entry found
+            return {}
+
+        if len(entries) > 1:
+            entries.sort(key=lambda x: self.get_weight(x), reverse=True)
+
+        # use entry with the highest weight
+        entry = entries[0]
+        entry.last_used = datetime.now()
+        entry.save()
+        result = {}
+        for entry in entries:
+            result.update(self._entry_to_dict(entry))
+
+        return result
+
+    def find(self, mailfrom=None, recipients=None, older_than=None):
+        "Find whitelist entries."
+        super().find(mailfrom, recipients, older_than)
+
+        if isinstance(mailfrom, str):
+            mailfrom = [mailfrom]
+        if isinstance(recipients, str):
+            recipients = [recipients]
+
+        entries = {}
+        try:
+            for entry in list(self.model.select()):
+                if older_than is not None:
+                    delta = (datetime.now() - entry.last_used).total_seconds()
+                    if delta < (older_than * 86400):
+                        continue
+
+                if mailfrom is not None:
+                    if entry.mailfrom not in mailfrom:
+                        continue
+
+                if recipients is not None:
+                    if entry.recipient not in recipients:
+                        continue
+
+                entries.update(self._entry_to_dict(entry))
+        except Exception as e:
+            raise RuntimeError(f"unable to query database: {e}")
+
+        return entries
+
+    def add(self, mailfrom, recipient, comment, permanent):
+        "Add entry to whitelist."
+        super().add(
+            mailfrom,
+            recipient,
+            comment,
+            permanent)
+
+        mailfrom = self.remove_batv(mailfrom)
+        recipient = self.remove_batv(recipient)
+
+        try:
+            self.model.create(
+                mailfrom=mailfrom,
+                recipient=recipient,
+                comment=comment,
+                permanent=permanent)
+        except Exception as e:
+            raise RuntimeError(f"unable to add entry to database: {e}")
+
+    def delete(self, whitelist_id):
+        "Delete entry from whitelist."
+        super().delete(whitelist_id)
+
+        try:
+            query = self.model.delete().where(self.model.id == whitelist_id)
+            deleted = query.execute()
+        except Exception as e:
+            raise RuntimeError(
+                f"unable to delete entry from database: {e}")
+
+        if deleted == 0:
+            raise RuntimeError("invalid whitelist id")

setup.cfg

@@ -1,2 +1,2 @@
 [metadata]
-version = attr: pymodmilter.__version__
+version = attr: pyquarantine.__version__

setup.py

@@ -4,22 +4,21 @@ def read_file(fname):
     with open(fname, 'r') as f:
         return f.read()
 
-
+setup(name = "pyquarantine",
-setup(name = "pymodmilter",
     author = "Thomas Oettli",
     author_email = "spacefreak@noop.ch",
     description = "A pymilter based sendmail/postfix pre-queue filter.",
     license = "GPL 3",
-    keywords = "header milter",
+    keywords = "header quarantine milter",
-    url = "https://github.com/spacefreak86/pymodmilter",
+    url = "https://github.com/spacefreak86/pyquarantine",
-    packages = ["pymodmilter"],
+    packages = ["pyquarantine"],
     long_description = read_file("README.md"),
     long_description_content_type = "text/markdown",
     classifiers = [
         #   3 - Alpha
         #   4 - Beta
         #   5 - Production/Stable
-        "Development Status :: 4 - Beta",
+        "Development Status :: 5 - Production/Stable",
         "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
         "Operating System :: OS Independent",
         "Programming Language :: Python",
@@ -29,23 +28,26 @@ setup(name = "pymodmilter",
     include_package_data = True,
     entry_points = {
         "console_scripts": [
-            "pymodmilter=pymodmilter.run:main"
+            "pyquarantine-milter=pyquarantine.run:main",
+            "pyquarantine=pyquarantine.cli:main",
         ]
     },
     data_files = [
         (
-            "/etc/pymodmilter",
+            "/etc/pyquarantine",
             [
-                "docs/pymodmilter.conf.example"
+                "pyquarantine/misc/pyquarantine.conf.default"
             ]
         ), (
-            "/etc/pymodmilter/templates",
+            "/etc/pyquarantine/templates",
             [
-                "docs/templates/disclaimer_html.template",
+                "pyquarantine/misc/templates/disclaimer_html.template",
-                "docs/templates/disclaimer_text.template"
+                "pyquarantine/misc/templates/disclaimer_text.template",
+                "pyquarantine/misc/templates/notification.template",
+                "pyquarantine/misc/templates/removed.png"
             ]
         )
     ],
-    install_requires = ["pymilter", "netaddr"],
+    install_requires = ["pymilter >= 1.5", "jsonschema", "netaddr", "beautifulsoup4[lxml]", "peewee"],
-    python_requires = ">=3.6"
+    python_requires = ">=3.9"
 )

test-pymodmilter

@@ -1,9 +0,0 @@
-#!/usr/bin/env python
-
-import sys
-import pymodmilter.run
-
-if __name__ == '__main__':
-    sys.exit(
-        pymodmilter.run.main()
-    )