From cd7e0688dc0b66844c87b4a2e244ca1b63b62133 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Thu, 30 Sep 2021 16:54:03 +0200 Subject: [PATCH] prepare for cli --- pymodmilter/__init__.py | 3 +- pymodmilter/cli.py | 628 ++++++++++++++++++++++++++++++++++++++++ pymodmilter/config.py | 24 +- pymodmilter/run.py | 30 +- 4 files changed, 658 insertions(+), 27 deletions(-) create mode 100644 pymodmilter/cli.py diff --git a/pymodmilter/__init__.py b/pymodmilter/__init__.py index aac9594..34c5d61 100644 --- a/pymodmilter/__init__.py +++ b/pymodmilter/__init__.py @@ -15,7 +15,9 @@ __all__ = [ "action", "base", + "cli", "conditions", + "config", "mailer", "modify", "notify", @@ -23,7 +25,6 @@ __all__ = [ "run", "storage", "whitelist", - "ModifyMilterConfig", "ModifyMilter"] __version__ = "1.2.0" diff --git a/pymodmilter/cli.py b/pymodmilter/cli.py new file mode 100644 index 0000000..2f7b8c8 --- /dev/null +++ b/pymodmilter/cli.py @@ -0,0 +1,628 @@ +#!/usr/bin/env python +# +# PyQuarantine-Milter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyQuarantine-Milter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with PyQuarantineMilter. If not, see . +# + +import argparse +import logging +import logging.handlers +import sys +import time + +from pymodmilter.config import get_milter_config +from pymodmilter import __version__ as version + + +def _get_quarantine(quarantines, name): + try: + quarantine = next((q for q in quarantines if q.name == name)) + except StopIteration: + raise RuntimeError("invalid quarantine 'name'") + return quarantine + + +def _get_storage(quarantines, name): + quarantine = _get_quarantine(quarantines, name) + storage = quarantine.get_storage() + if not storage: + raise RuntimeError( + "storage type is set to NONE") + return storage + + +def _get_notification(quarantines, name): + quarantine = _get_quarantine(quarantines, name) + notification = quarantine.get_notification() + if not notification: + raise RuntimeError( + "notification type is set to NONE") + return notification + + +def _get_whitelist(quarantines, name): + quarantine = _get_quarantine(quarantines, name) + whitelist = quarantine.get_whitelist() + if not whitelist: + raise RuntimeError( + "whitelist type is set to NONE") + return whitelist + + +def print_table(columns, rows): + if not rows: + return + + column_lengths = [] + column_formats = [] + + # iterate columns to display + for header, key in columns: + # get the length of the header string + lengths = [len(header)] + # get the length of the longest value + lengths.append( + len(str(max(rows, key=lambda x: len(str(x[key])))[key]))) + # use the longer one + length = max(lengths) + column_lengths.append(length) + column_formats.append(f"{{:<{length}}}") + + # define row format + row_format = " | ".join(column_formats) + + # define header/body separator + separators = [] + for length in column_lengths: + separators.append("-" * length) + separator = "-+-".join(separators) + + # print header and separator + print(row_format.format(*[column[0] for column in columns])) + print(separator) + + keys = [entry[1] for entry in columns] + # print rows + for entry in rows: + row = [] + for key in keys: + row.append(entry[key]) + print(row_format.format(*row)) + + +def list_quarantines(quarantines, args): + if args.batch: + print("\n".join([q.name for q in quarantines])) + else: + qlist = [] + for q in quarantines: + storage = q.get_storage() + if storage: + storage_type = q.get_storage().storage_type + else: + storage_type = "NONE" + + notification = q.get_notification() + if notification: + notification_type = q.get_notification().notification_type + else: + notification_type = "NONE" + + whitelist = q.get_whitelist() + if whitelist: + whitelist_type = q.get_whitelist().whitelist_type + else: + whitelist_type = "NONE" + + qlist.append({ + "name": q.name, + "storage": storage_type, + "notification": notification_type, + "whitelist": whitelist_type, + "action": q.action}) + print_table( + [("Name", "name"), + ("Storage", "storage"), + ("Notification", "notification"), + ("Whitelist", "whitelist"), + ("Action", "action")], + qlist + ) + + +def list_quarantine_emails(quarantines, args): + logger = logging.getLogger(__name__) + storage = _get_storage(quarantines, args.quarantine) + # find emails and transform some metadata values to strings + rows = [] + emails = storage.find( + args.mailfrom, args.recipients, args.older_than) + for storage_id, metadata in emails.items(): + row = emails[storage_id] + row["storage_id"] = storage_id + row["date"] = time.strftime( + '%Y-%m-%d %H:%M:%S', + time.localtime( + metadata["date"])) + row["mailfrom"] = metadata["mailfrom"] + row["recipient"] = metadata["recipients"].pop(0) + if "subject" not in emails[storage_id]["headers"].keys(): + emails[storage_id]["headers"]["subject"] = "" + row["subject"] = emails[storage_id]["headers"]["subject"][:60].strip() + rows.append(row) + + if metadata["recipients"]: + row = { + "storage_id": "", + "date": "", + "mailfrom": "", + "recipient": metadata["recipients"].pop(0), + "subject": "" + } + rows.append(row) + + if args.batch: + # batch mode, print quarantine IDs, each on a new line + print("\n".join(emails.keys())) + return + + if not emails: + logger.info(f"quarantine '{args.quarantine}' is empty") + print_table( + [("Quarantine-ID", "storage_id"), ("Date", "date"), + ("From", "mailfrom"), ("Recipient(s)", "recipient"), + ("Subject", "subject")], + rows + ) + + +def list_whitelist(quarantines, args): + logger = logging.getLogger(__name__) + whitelist = _get_whitelist(quarantines, args.quarantine) + + # find whitelist entries + entries = whitelist.find( + mailfrom=args.mailfrom, + recipients=args.recipients, + older_than=args.older_than) + if not entries: + logger.info( + f"whitelist of quarantine '{args.quarantine}' is empty") + return + + # transform some values to strings + for entry_id, entry in entries.items(): + entries[entry_id]["permanent_str"] = str(entry["permanent"]) + entries[entry_id]["created_str"] = entry["created"].strftime( + '%Y-%m-%d %H:%M:%S') + entries[entry_id]["last_used_str"] = entry["last_used"].strftime( + '%Y-%m-%d %H:%M:%S') + + print_table( + [ + ("ID", "id"), ("From", "mailfrom"), ("To", "recipient"), + ("Created", "created_str"), ("Last used", "last_used_str"), + ("Comment", "comment"), ("Permanent", "permanent_str") + ], + entries.values() + ) + + +def add_whitelist_entry(quarantines, args): + logger = logging.getLogger(__name__) + whitelist = _get_whitelist(quarantines, args.quarantine) + + # check existing entries + entries = whitelist.check(args.mailfrom, args.recipient) + if entries: + # check if the exact entry exists already + for entry in entries.values(): + if entry["mailfrom"] == args.mailfrom and \ + entry["recipient"] == args.recipient: + raise RuntimeError( + "an entry with this from/to combination already exists") + + if not args.force: + # the entry is already covered by others + for entry_id, entry in entries.items(): + entries[entry_id]["permanent_str"] = str(entry["permanent"]) + entries[entry_id]["created_str"] = entry["created"].strftime( + '%Y-%m-%d %H:%M:%S') + entries[entry_id]["last_used_str"] = entry["last_used"].strftime( + '%Y-%m-%d %H:%M:%S') + print_table( + [ + ("ID", "id"), ("From", "mailfrom"), ("To", "recipient"), + ("Created", "created_str"), ("Last used", "last_used_str"), + ("Comment", "comment"), ("Permanent", "permanent_str") + ], + entries.values() + ) + print("") + raise RuntimeError( + "from/to combination is already covered by the entries above, " + "use --force to override.") + + # add entry to whitelist + whitelist.add(args.mailfrom, args.recipient, args.comment, args.permanent) + logger.info("whitelist entry added successfully") + + +def delete_whitelist_entry(quarantines, args): + logger = logging.getLogger(__name__) + whitelist = _get_whitelist(quarantines, args.quarantine) + whitelist.delete(args.whitelist_id) + logger.info("whitelist entry deleted successfully") + + +def notify(quarantines, args): + logger = logging.getLogger(__name__) + quarantine = _get_quarantine(quarantines, args.quarantine) + quarantine.notify(args.quarantine_id, args.recipient) + logger.info("notification sent successfully") + + +def release(quarantines, args): + logger = logging.getLogger(__name__) + quarantine = _get_quarantine(quarantines, args.quarantine) + quarantine.release(args.quarantine_id, args.recipient) + logger.info("quarantined email released successfully") + + +def delete(quarantines, args): + logger = logging.getLogger(__name__) + storage = _get_storage(quarantines, args.quarantine) + storage.delete(args.quarantine_id, args.recipient) + logger.info("quarantined email deleted successfully") + + +def get(quarantines, args): + storage = _get_storage(quarantines, args.quarantine) + fp, _ = storage.get_mail(args.quarantine_id) + print(fp.read().decode()) + fp.close() + + +class StdErrFilter(logging.Filter): + def filter(self, rec): + return rec.levelno in (logging.ERROR, logging.WARNING) + + +class StdOutFilter(logging.Filter): + def filter(self, rec): + return rec.levelno in (logging.DEBUG, logging.INFO) + + +def main(): + python_version = ".".join([str(v) for v in sys.version_info[0:3]]) + python_version = f"{python_version}-{sys.version_info[3]}" + + "PyQuarantine command-line interface." + # parse command line + def formatter_class(prog): return argparse.HelpFormatter( + prog, max_help_position=50, width=140) + parser = argparse.ArgumentParser( + description="PyQuarantine CLI", + formatter_class=formatter_class) + parser.add_argument( + "-c", "--config", help="Config file to read.", + default="/etc/pymodmilter/pymodmilter.conf") + parser.add_argument( + "-d", "--debug", + help="Log debugging messages.", + action="store_true") + parser.add_argument( + "-v", "--version", + help="Print version.", + action="version", + version=f"%(prog)s {version} (python {python_version})") + parser.set_defaults(syslog=False) + subparsers = parser.add_subparsers( + dest="command", + title="Commands") + subparsers.required = True + + # list command + list_parser = subparsers.add_parser( + "list", + help="List available quarantines.", + formatter_class=formatter_class) + list_parser.add_argument( + "-b", "--batch", + help="Print results using only quarantine names, each on a new line.", + action="store_true") + list_parser.set_defaults(func=list_quarantines) + + # quarantine command group + quarantine_parser = subparsers.add_parser( + "quarantine", + description="Manage quarantines.", + help="Manage quarantines.", + formatter_class=formatter_class) + quarantine_parser.add_argument( + "quarantine", + metavar="QUARANTINE", + help="Quarantine name.") + quarantine_subparsers = quarantine_parser.add_subparsers( + dest="command", + title="Quarantine commands") + quarantine_subparsers.required = True + # quarantine list command + quarantine_list_parser = quarantine_subparsers.add_parser( + "list", + description="List emails in quarantines.", + help="List emails in quarantine.", + formatter_class=formatter_class) + quarantine_list_parser.add_argument( + "-f", "--from", + dest="mailfrom", + help="Filter emails by from address.", + default=None, + nargs="+") + quarantine_list_parser.add_argument( + "-t", "--to", + dest="recipients", + help="Filter emails by recipient address.", + default=None, + nargs="+") + quarantine_list_parser.add_argument( + "-o", "--older-than", + dest="older_than", + help="Filter emails by age (days).", + default=None, + type=float) + quarantine_list_parser.add_argument( + "-b", "--batch", + help="Print results using only email quarantine IDs, each on a new line.", + action="store_true") + quarantine_list_parser.set_defaults(func=list_quarantine_emails) + # quarantine notify command + quarantine_notify_parser = quarantine_subparsers.add_parser( + "notify", + description="Notify recipient about email in quarantine.", + help="Notify recipient about email in quarantine.", + formatter_class=formatter_class) + quarantine_notify_parser.add_argument( + "quarantine_id", + metavar="ID", + help="Quarantine ID.") + quarantine_notify_parser_group = quarantine_notify_parser.add_mutually_exclusive_group( + required=True) + quarantine_notify_parser_group.add_argument( + "-t", "--to", + dest="recipient", + help="Release email for one recipient address.") + quarantine_notify_parser_group.add_argument( + "-a", "--all", + help="Release email for all recipients.", + action="store_true") + quarantine_notify_parser.set_defaults(func=notify) + # quarantine release command + quarantine_release_parser = quarantine_subparsers.add_parser( + "release", + description="Release email from quarantine.", + help="Release email from quarantine.", + formatter_class=formatter_class) + quarantine_release_parser.add_argument( + "quarantine_id", + metavar="ID", + help="Quarantine ID.") + quarantine_release_parser.add_argument( + "-n", + "--disable-syslog", + dest="syslog", + help="Disable syslog messages.", + action="store_false") + quarantine_release_parser_group = quarantine_release_parser.add_mutually_exclusive_group( + required=True) + quarantine_release_parser_group.add_argument( + "-t", "--to", + dest="recipient", + help="Release email for one recipient address.") + quarantine_release_parser_group.add_argument( + "-a", "--all", + help="Release email for all recipients.", + action="store_true") + quarantine_release_parser.set_defaults(func=release) + # quarantine delete command + quarantine_delete_parser = quarantine_subparsers.add_parser( + "delete", + description="Delete email from quarantine.", + help="Delete email from quarantine.", + formatter_class=formatter_class) + quarantine_delete_parser.add_argument( + "quarantine_id", + metavar="ID", + help="Quarantine ID.") + quarantine_delete_parser.add_argument( + "-n", "--disable-syslog", + dest="syslog", + help="Disable syslog messages.", + action="store_false") + quarantine_delete_parser_group = quarantine_delete_parser.add_mutually_exclusive_group( + required=True) + quarantine_delete_parser_group.add_argument( + "-t", "--to", + dest="recipient", + help="Delete email for one recipient address.") + quarantine_delete_parser_group.add_argument( + "-a", "--all", + help="Delete email for all recipients.", + action="store_true") + quarantine_delete_parser.set_defaults(func=delete) + # quarantine get command + quarantine_get_parser = quarantine_subparsers.add_parser( + "get", + description="Get email from quarantine.", + help="Get email from quarantine", + formatter_class=formatter_class) + quarantine_get_parser.add_argument( + "quarantine_id", + metavar="ID", + help="Quarantine ID.") + quarantine_get_parser.set_defaults(func=get) + + # whitelist command group + whitelist_parser = subparsers.add_parser( + "whitelist", + description="Manage whitelists.", + help="Manage whitelists.", + formatter_class=formatter_class) + whitelist_parser.add_argument( + "quarantine", + metavar="QUARANTINE", + help="Quarantine name.") + whitelist_subparsers = whitelist_parser.add_subparsers( + dest="command", + title="Whitelist commands") + whitelist_subparsers.required = True + # whitelist list command + whitelist_list_parser = whitelist_subparsers.add_parser( + "list", + description="List whitelist entries.", + help="List whitelist entries.", + formatter_class=formatter_class) + whitelist_list_parser.add_argument( + "-f", "--from", + dest="mailfrom", + help="Filter entries by from address.", + default=None, + nargs="+") + whitelist_list_parser.add_argument( + "-t", "--to", + dest="recipients", + help="Filter entries by recipient address.", + default=None, + nargs="+") + whitelist_list_parser.add_argument( + "-o", "--older-than", + dest="older_than", + help="Filter emails by last used date (days).", + default=None, + type=float) + whitelist_list_parser.set_defaults(func=list_whitelist) + # whitelist add command + whitelist_add_parser = whitelist_subparsers.add_parser( + "add", + description="Add whitelist entry.", + help="Add whitelist entry.", + formatter_class=formatter_class) + whitelist_add_parser.add_argument( + "-f", "--from", + dest="mailfrom", + help="From address.", + required=True) + whitelist_add_parser.add_argument( + "-t", "--to", + dest="recipient", + help="Recipient address.", + required=True) + whitelist_add_parser.add_argument( + "-c", "--comment", + help="Comment.", + default="added by CLI") + whitelist_add_parser.add_argument( + "-p", "--permanent", + help="Add a permanent entry.", + action="store_true") + whitelist_add_parser.add_argument( + "--force", + help="Force adding an entry, even if already covered by another entry.", + action="store_true") + whitelist_add_parser.set_defaults(func=add_whitelist_entry) + # whitelist delete command + whitelist_delete_parser = whitelist_subparsers.add_parser( + "delete", + description="Delete whitelist entry.", + help="Delete whitelist entry.", + formatter_class=formatter_class) + whitelist_delete_parser.add_argument( + "whitelist_id", + metavar="ID", + help="Whitelist ID.") + whitelist_delete_parser.set_defaults(func=delete_whitelist_entry) + + args = parser.parse_args() + + # setup logging + loglevel = logging.INFO + root_logger = logging.getLogger() + root_logger.setLevel(loglevel) + + # setup console log + if args.debug: + formatter = logging.Formatter( + "%(levelname)s: [%(name)s] - %(message)s") + else: + formatter = logging.Formatter("%(levelname)s: %(message)s") + # stdout + stdouthandler = logging.StreamHandler(sys.stdout) + stdouthandler.setLevel(logging.DEBUG) + stdouthandler.setFormatter(formatter) + stdouthandler.addFilter(StdOutFilter()) + root_logger.addHandler(stdouthandler) + # stderr + stderrhandler = logging.StreamHandler(sys.stderr) + stderrhandler.setLevel(logging.WARNING) + stderrhandler.setFormatter(formatter) + stderrhandler.addFilter(StdErrFilter()) + root_logger.addHandler(stderrhandler) + logger = logging.getLogger(__name__) + + try: + logger.debug("read milter configuration") + cfg = get_milter_config(args.config) + if not cfg["rules"]: + raise RuntimeError("no rules configured") + + for rule in cfg["rules"]: + if not rule["actions"]: + raise RuntimeError( + f"{rule['name']}: no actions configured") + except (RuntimeError, AssertionError) as e: + logger.error(f"config error: {e}") + sys.exit(255) + + quarantines = [] + for rule in cfg["rules"]: + for action in rule["actions"]: + if action["type"] == "quarantine": + quarantines.append(action) + + print(quarantines) + sys.exit(0) + + if args.syslog: + # setup syslog + sysloghandler = logging.handlers.SysLogHandler( + address="/dev/log", + facility=logging.handlers.SysLogHandler.LOG_MAIL) + sysloghandler.setLevel(loglevel) + if args.debug: + formatter = logging.Formatter( + "pyquarantine: [%(name)s] [%(levelname)s] %(message)s") + else: + formatter = logging.Formatter("pyquarantine: %(message)s") + sysloghandler.setFormatter(formatter) + root_logger.addHandler(sysloghandler) + + # call the commands function + try: + args.func(cfg, args) + except RuntimeError as e: + logger.error(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/pymodmilter/config.py b/pymodmilter/config.py index 98b831d..6b6f336 100644 --- a/pymodmilter/config.py +++ b/pymodmilter/config.py @@ -26,10 +26,13 @@ __all__ = [ "QuarantineConfig", "ActionConfig", "RuleConfig", - "MilterConfig"] + "MilterConfig", + "get_milter_config"] +import json import jsonschema import logging +import re class BaseConfig: @@ -330,3 +333,22 @@ class MilterConfig(BaseConfig): for idx, rule in enumerate(self["rules"]): rules.append(RuleConfig(rule, rec)) self["rules"] = rules + + +def get_milter_config(cfgfile): + try: + with open(cfgfile, "r") as fh: + # remove lines with leading # (comments), they + # are not allowed in json + cfg = re.sub(r"(?m)^\s*#.*\n?", "", fh.read()) + except IOError as e: + raise RuntimeError(f"unable to open/read config file: {e}") + + try: + cfg = json.loads(cfg) + except json.JSONDecodeError as e: + cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())] + msg = "\n".join(cfg_text) + raise RuntimeError(f"{e}\n{msg}") + + return MilterConfig(cfg) diff --git a/pymodmilter/run.py b/pymodmilter/run.py index 3675809..22548b7 100644 --- a/pymodmilter/run.py +++ b/pymodmilter/run.py @@ -16,16 +16,14 @@ __all__ = ["main"] import Milter import argparse -import json import logging import logging.handlers -import re import sys from pymodmilter import mailer from pymodmilter import ModifyMilter from pymodmilter import __version__ as version -from pymodmilter.config import MilterConfig +from pymodmilter.config import get_milter_config def main(): @@ -84,30 +82,12 @@ def main(): try: logger.debug("read milter configuration") - - try: - with open(args.config, "r") as fh: - # remove lines with leading # (comments), they - # are not allowed in json - cfg = re.sub(r"(?m)^\s*#.*\n?", "", fh.read()) - except IOError as e: - raise RuntimeError(f"unable to open/read config file: {e}") - - try: - cfg = json.loads(cfg) - except json.JSONDecodeError as e: - cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())] - msg = "\n".join(cfg_text) - raise RuntimeError(f"{e}\n{msg}") - - cfg = MilterConfig(cfg) - - if not args.debug: - logger.setLevel(cfg.get_loglevel(args.debug)) + cfg = get_milter_config(args.config) + logger.setLevel(cfg.get_loglevel(args.debug)) if args.socket: socket = args.socket - elif cfg["socket"]: + elif "socket" in cfg: socket = cfg["socket"] else: raise RuntimeError( @@ -123,7 +103,7 @@ def main(): f"{rule['name']}: no actions configured") except (RuntimeError, AssertionError) as e: - logger.error(f"error in config file: {e}") + logger.error(f"config error: {e}") sys.exit(255) try: