From 4da1a0e9b30b9c270a9a98235d75b1b5575ce129 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Wed, 3 Jan 2024 19:54:34 +0100 Subject: [PATCH] change config structure --- pyquarantine/__init__.py | 2 +- pyquarantine/action.py | 3 +- pyquarantine/cli.py | 88 +++++++++++++++++------ pyquarantine/config.py | 146 ++++++++++++++++++++++++++++----------- pyquarantine/notify.py | 13 ++-- pyquarantine/rule.py | 5 +- pyquarantine/storage.py | 36 +++------- 7 files changed, 194 insertions(+), 99 deletions(-) diff --git a/pyquarantine/__init__.py b/pyquarantine/__init__.py index 380c8b1..2058612 100644 --- a/pyquarantine/__init__.py +++ b/pyquarantine/__init__.py @@ -27,7 +27,7 @@ __all__ = [ "list", "QuarantineMilter"] -__version__ = "2.0.8" +__version__ = "2.1.0" from pyquarantine import _runtime_patches diff --git a/pyquarantine/action.py b/pyquarantine/action.py index cb30822..bb03f23 100644 --- a/pyquarantine/action.py +++ b/pyquarantine/action.py @@ -39,8 +39,7 @@ class Action: self.conditions["loglevel"] = cfg["loglevel"] self.conditions = Conditions(self.conditions, local_addrs, debug) - action_type = cfg["type"] - self.action = self.ACTION_TYPES[action_type]( + self.action = self.ACTION_TYPES[cfg["type"]]( cfg, local_addrs, debug) def __str__(self): diff --git a/pyquarantine/cli.py b/pyquarantine/cli.py index f0a94fe..65c68e5 100644 --- a/pyquarantine/cli.py +++ b/pyquarantine/cli.py @@ -21,28 +21,31 @@ import logging.handlers import sys import time -from pyquarantine.config import get_milter_config, ActionConfig, ListConfig +from pyquarantine.action import Action +from pyquarantine.config import get_milter_config, ActionConfig, StorageConfig, NotificationConfig, ListConfig from pyquarantine.storage import Quarantine from pyquarantine.list import DatabaseList from pyquarantine import __version__ as version -def _get_quarantines(cfg): +def _get_quarantines(milter_cfg): quarantines = [] - for rule in cfg["rules"]: + for rule in milter_cfg["rules"]: for action in rule["actions"]: if action["type"] == "quarantine": quarantines.append(action) return quarantines -def _get_quarantine(cfg, name, debug): +def _get_quarantine(milter_cfg, name, debug): try: quarantine = next( - (q for q in _get_quarantines(cfg) if q["name"] == name)) + (q for q in _get_quarantines(milter_cfg) if q["name"] == name)) except StopIteration: raise RuntimeError(f"invalid quarantine '{name}'") - return Quarantine(ActionConfig(quarantine, cfg["lists"]), [], debug) + + cfg = ActionConfig(quarantine, milter_cfg) + return Quarantine(cfg, [], debug) def _get_notification(cfg, name, debug): @@ -55,7 +58,7 @@ def _get_notification(cfg, name, debug): def _get_list(cfg, name, debug): try: - list_cfg = ListConfig(cfg["lists"][name], {}) + list_cfg = cfg["lists"][name] except KeyError: raise RuntimeError(f"list '{name}' is not configured") @@ -107,7 +110,7 @@ def print_table(columns, rows): print(row_format.format(*row)) -def list_quarantines(cfg, args): +def llist(cfg, args): quarantines = _get_quarantines(cfg) if args.batch: print("\n".join([q["name"] for q in quarantines])) @@ -115,32 +118,33 @@ def list_quarantines(cfg, args): qlist = [] for q in quarantines: qcfg = q["options"] - storage_type = qcfg["store"]["type"] - if "notify" in cfg: - notification_type = qcfg["notify"]["type"] + if "notify" in qcfg: + notification = cfg["notifications"][qcfg["notify"]]["name"] else: - notification_type = "NONE" + notification = "NONE" - if "lists" in qcfg: - lists_type = qcfg["lists"] + if "allowlist" in qcfg: + allowlist = qcfg["allowlist"] else: - lists_type = "NONE" + allowlist = "NONE" if "milter_action" in qcfg: milter_action = qcfg["milter_action"] else: milter_action = "NONE" + storage_name = cfg["storages"][qcfg["store"]]["name"] + qlist.append({ "name": q["name"], - "storage": storage_type, - "notification": notification_type, - "lists": lists_type, + "storage": storage_name, + "notification": notification, + "lists": allowlist, "action": milter_action}) print_table( - [("Name", "name"), + [("Quarantine", "name"), ("Storage", "storage"), ("Notification", "notification"), ("Allowlist", "lists"), @@ -148,6 +152,48 @@ def list_quarantines(cfg, args): qlist ) + if "storages" in cfg: + storages = [] + for name, options in cfg["storages"].items(): + storages.append({ + "name": name, + "type": options["type"]}) + + print("\n") + print_table( + [("Storage", "name"), + ("Type", "type")], + storages + ) + + if "notifications" in cfg: + notifications = [] + for name, options in cfg["notifications"].items(): + notifications.append({ + "name": name, + "type": options["type"]}) + + print("\n") + print_table( + [("Notification", "name"), + ("Type", "type")], + notifications + ) + + if "lists" in cfg: + lst_list = [] + for name, options in cfg["lists"].items(): + lst_list.append({ + "name": name, + "type": options["type"]}) + + print("\n") + print_table( + [("List", "name"), + ("Type", "type")], + lst_list + ) + def list_quarantine_emails(cfg, args): storage = _get_quarantine(cfg, args.quarantine, args.debug).storage @@ -364,7 +410,7 @@ def main(): "-b", "--batch", help="Print results using only quarantine names, each on a new line.", action="store_true") - list_parser.set_defaults(func=list_quarantines) + list_parser.set_defaults(func=llist) # quarantine command group quar_parser = subparsers.add_parser( @@ -638,7 +684,7 @@ def main(): try: logger.debug("read milter configuration") - cfg = get_milter_config(args.config, raw=True) + cfg = get_milter_config(args.config, rec=False) if "rules" not in cfg or not cfg["rules"]: raise RuntimeError("no rules configured") diff --git a/pyquarantine/config.py b/pyquarantine/config.py index 123815f..8b94717 100644 --- a/pyquarantine/config.py +++ b/pyquarantine/config.py @@ -20,7 +20,9 @@ __all__ = [ "DelHeaderConfig", "AddDisclaimerConfig", "RewriteLinksConfig", + "StorageConfig", "StoreConfig", + "NotificationConfig", "NotifyConfig", "ListConfig", "QuarantineConfig", @@ -43,7 +45,7 @@ class BaseConfig: "properties": { "loglevel": {"type": "string", "default": "info"}}} - def __init__(self, config, lists): + def __init__(self, config, *args, **kwargs): required = self.JSON_SCHEMA["required"] properties = self.JSON_SCHEMA["properties"] for p in properties.keys(): @@ -126,15 +128,13 @@ class ConditionsConfig(BaseConfig): "list": {"type": "string"}}} def __init__(self, config, lists, rec=True): - super().__init__(config, lists) + super().__init__(config) if "list" in self: lst = self["list"] try: self["list"] = lists[lst] except KeyError: - raise RuntimeError(f"list '{lst}' is not configured") - if not rec: - return + raise RuntimeError(f"list '{lst}' not found in config") class AddHeaderConfig(BaseConfig): @@ -190,7 +190,7 @@ class RewriteLinksConfig(BaseConfig): "repl": {"type": "string"}}} -class StoreConfig(BaseConfig): +class StorageConfig(BaseConfig): JSON_SCHEMA = { "type": "object", "required": ["type"], @@ -203,6 +203,7 @@ class StoreConfig(BaseConfig): "additionalProperties": False, "properties": { "type": {"type": "string"}, + "name": {"type": "string"}, "directory": {"type": "string"}, "mode": {"type": "string"}, "metavar": {"type": "string"}, @@ -210,7 +211,16 @@ class StoreConfig(BaseConfig): "original": {"type": "boolean", "default": False}}}} -class NotifyConfig(BaseConfig): +class StoreConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["storage"], + "additionalProperties": False, + "properties": { + "storage": {"type": "string"}}} + + +class NotificationConfig(BaseConfig): JSON_SCHEMA = { "type": "object", "required": ["type"], @@ -224,6 +234,7 @@ class NotifyConfig(BaseConfig): "additionalProperties": False, "properties": { "type": {"type": "string"}, + "name": {"type": "string"}, "smtp_host": {"type": "string"}, "smtp_port": {"type": "number"}, "envelope_from": {"type": "string"}, @@ -238,6 +249,15 @@ class NotifyConfig(BaseConfig): "default": []}}}} +class NotifyConfig(BaseConfig): + JSON_SCHEMA = { + "type": "object", + "required": ["notification"], + "additionalProperties": False, + "properties": { + "notification": {"type": "string"}}} + + class QuarantineConfig(BaseConfig): JSON_SCHEMA = { "type": "object", @@ -245,29 +265,38 @@ class QuarantineConfig(BaseConfig): "additionalProperties": False, "properties": { "name": {"type": "string"}, - "notify": {"type": "object"}, + "notify": {"type": "string"}, "milter_action": {"type": "string"}, "reject_reason": {"type": "string"}, "allowlist": {"type": "string"}, - "store": {"type": "object"}, + "store": {"type": "string"}, "smtp_host": {"type": "string"}, "smtp_port": {"type": "number"}}} - def __init__(self, config, lists, rec=True): - super().__init__(config, lists) - if not rec: - return + def __init__(self, config, milter_config, rec=True): + super().__init__(config) + storage = self["store"] + try: + self["store"] = milter_config["storages"][storage] + except KeyError: + raise RuntimeError(f"storage '{storage}' not found") if "metadata" not in self["store"]: self["store"]["metadata"] = True - self["store"] = StoreConfig(self["store"], lists) if "notify" in self: - self["notify"] = NotifyConfig(self["notify"], lists) + notify = self["notify"] + try: + self["notify"] = milter_config["notifications"][notify] + except KeyError: + raise RuntimeError(f"notification '{notify}' not found") if "allowlist" in self: allowlist = self["allowlist"] try: - self["allowlist"] = lists[allowlist] + self["allowlist"] = milter_config["lists"][allowlist] except KeyError: - raise RuntimeError(f"list '{allowlist}' is not configured") + raise RuntimeError(f"list '{allowlist}' not found") + + if not rec: + return class ActionConfig(BaseConfig): @@ -293,14 +322,31 @@ class ActionConfig(BaseConfig): "type": {"enum": list(ACTION_TYPES.keys())}, "options": {"type": "object"}}} - def __init__(self, config, lists, rec=True): - super().__init__(config, lists) + def __init__(self, config, milter_config, rec=True): + super().__init__(config) if not rec: return + lists = milter_config["lists"] if "conditions" in self: self["conditions"] = ConditionsConfig(self["conditions"], lists) - self["action"] = self.ACTION_TYPES[self["type"]]( - self["options"], lists) + + if self["type"] == "store": + storage = StoreConfig(self["options"])["storage"] + try: + self["action"] = milter_config["storages"][storage] + except KeyError: + raise RuntimeError(f"storage '{storage}' not found") + + elif self["type"] == "notify": + notify = NotifyConfig(self["options"])["notification"] + try: + self["action"] = milter_config["notifications"][notify] + except KeyError: + raise RuntimeError(f"notification '{notify}' not found") + + else: + self["action"] = self.ACTION_TYPES[self["type"]]( + self["options"], milter_config) class RuleConfig(BaseConfig): @@ -315,10 +361,11 @@ class RuleConfig(BaseConfig): "conditions": {"type": "object"}, "actions": {"type": "array"}}} - def __init__(self, config, lists, rec=True): - super().__init__(config, lists) + def __init__(self, config, milter_config, rec=True): + super().__init__(config) if not rec: return + lists = milter_config["lists"] if "conditions" in self: self["conditions"] = ConditionsConfig(self["conditions"], lists) @@ -328,7 +375,7 @@ class RuleConfig(BaseConfig): action["loglevel"] = config["loglevel"] if "pretend" not in action: action["pretend"] = config["pretend"] - actions.append(ActionConfig(action, lists, rec)) + actions.append(ActionConfig(action, milter_config, rec)) self["actions"] = actions @@ -350,29 +397,52 @@ class QuarantineMilterConfig(BaseConfig): "192.168.0.0/16"]}, "loglevel": {"type": "string", "default": "info"}, "pretend": {"type": "boolean", "default": False}, - "rules": {"type": "array"}, - "lists": {"type": "array", - "default": []}}} + "lists": { + "type": "object", + "patternProperties": {"^(.+)$": {"type": "object"}}, + "additionalProperties": False, + "default": {}}, + "storages": { + "type": "object", + "patternProperties": {"^(.+)$": {"type": "object"}}, + "additionalProperties": False, + "default": {}}, + "notifications": { + "type": "object", + "patternProperties": {"^(.+)$": {"type": "object"}}, + "additionalProperties": False, + "default": {}}, + "rules": {"type": "array"}}} def __init__(self, config, rec=True): - super().__init__(config, {}) + super().__init__(config) + for param in ["lists", "storages", "notifications"]: + for name, cfg in self[param].items(): + if "name" not in cfg: + cfg["name"] = name + for name, cfg in self["lists"].items(): + self["lists"][name] = ListConfig(cfg) + + for name, cfg in self["storages"].items(): + self["storages"][name] = StorageConfig(cfg) + + for name, cfg in self["notifications"].items(): + self["notifications"][name] = NotificationConfig(cfg) + if not rec: return - lists = {} - for lst in self["lists"]: - lists[lst["name"]] = ListConfig(lst, rec) - self["lists"] = lists + rules = [] for rule in self["rules"]: if "loglevel" not in rule: rule["loglevel"] = config["loglevel"] if "pretend" not in rule: rule["pretend"] = config["pretend"] - rules.append(RuleConfig(rule, lists, rec)) + rules.append(RuleConfig(rule, self, rec)) self["rules"] = rules -def get_milter_config(cfgfile, raw=False): +def get_milter_config(cfgfile, rec=True): try: with open(cfgfile, "r") as fh: # remove lines with leading # (comments), they @@ -387,10 +457,4 @@ def get_milter_config(cfgfile, raw=False): cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())] msg = "\n".join(cfg_text) raise RuntimeError(f"{e}\n{msg}") - if raw: - lists = {} - for lst in cfg["lists"]: - lists[lst["name"]] = lst - cfg["lists"] = lists - return cfg - return QuarantineMilterConfig(cfg) + return QuarantineMilterConfig(cfg, rec) diff --git a/pyquarantine/notify.py b/pyquarantine/notify.py index 586b457..1464c3d 100644 --- a/pyquarantine/notify.py +++ b/pyquarantine/notify.py @@ -331,18 +331,17 @@ class Notify: def __init__(self, cfg, local_addrs, debug): self.cfg = cfg self.logger = logging.getLogger(cfg["name"]) + del cfg["name"] self.logger.setLevel(cfg.get_loglevel(debug)) - - nodification_type = cfg["options"]["type"] - del cfg["options"]["type"] - cfg["options"]["pretend"] = cfg["pretend"] - self._notification = self.NOTIFICATION_TYPES[nodification_type]( - **cfg["options"]) + del cfg["loglevel"] + nodification_type = cfg["type"] + del cfg["type"] + self._notification = self.NOTIFICATION_TYPES[nodification_type](**cfg) self._headersonly = self._notification._headersonly def __str__(self): cfg = [] - for key, value in self.cfg["options"].items(): + for key, value in self.cfg.items(): cfg.append(f"{key}={value}") class_name = type(self._notification).__name__ return f"{class_name}(" + ", ".join(cfg) + ")" diff --git a/pyquarantine/rule.py b/pyquarantine/rule.py index 35cc54d..d2a22ab 100644 --- a/pyquarantine/rule.py +++ b/pyquarantine/rule.py @@ -46,8 +46,9 @@ class Rule: actions = [] for action in self.actions: actions.append(str(action)) - cfg.append("actions=[" + ", ".join(actions) + "]") - return "Rule(" + ", ".join(cfg) + ")" + cfg.append("actions=[\n " + + ",\n ".join(actions) + "\n ]") + return "Rule(\n " + ",\n ".join(cfg) + "\n)" def execute(self, milter): """Execute all actions of this rule.""" diff --git a/pyquarantine/storage.py b/pyquarantine/storage.py index 0185d3b..965726b 100644 --- a/pyquarantine/storage.py +++ b/pyquarantine/storage.py @@ -31,7 +31,6 @@ from time import gmtime from pyquarantine import mailer from pyquarantine.base import CustomLogger, MilterMessage -from pyquarantine.config import ActionConfig from pyquarantine.list import DatabaseList from pyquarantine.notify import Notify @@ -373,18 +372,17 @@ class Store: def __init__(self, cfg, local_addrs, debug): self.cfg = cfg self.logger = logging.getLogger(cfg["name"]) + del cfg["name"] self.logger.setLevel(cfg.get_loglevel(debug)) - - storage_type = cfg["options"]["type"] - del cfg["options"]["type"] - cfg["options"]["pretend"] = cfg["pretend"] - self._storage = self.STORAGE_TYPES[storage_type]( - **cfg["options"]) + del cfg["loglevel"] + storage_type = cfg["type"] + del cfg["type"] + self._storage = self.STORAGE_TYPES[storage_type](**cfg) self._headersonly = self._storage._headersonly def __str__(self): cfg = [] - for key, value in self.cfg["options"].items(): + for key, value in self.cfg.items(): cfg.append(f"{key}={value}") class_name = type(self._storage).__name__ return f"{class_name}(" + ", ".join(cfg) + ")" @@ -407,29 +405,17 @@ class Quarantine: self.logger = logging.getLogger(cfg["name"]) self.logger.setLevel(cfg.get_loglevel(debug)) - storage_cfg = ActionConfig( - { - "name": cfg["name"], - "loglevel": cfg["loglevel"], - "pretend": cfg["pretend"], - "type": "store", - "options": cfg["options"]["store"].get_config()}, - {}) - self._storage = Store(storage_cfg, local_addrs, debug) + cfg["options"]["store"]["loglevel"] = cfg["loglevel"] + self._storage = Store(cfg["options"]["store"], local_addrs, debug) self.smtp_host = cfg["options"]["smtp_host"] self.smtp_port = cfg["options"]["smtp_port"] self._notification = None if "notify" in cfg["options"]: - notify_cfg = ActionConfig({ - "name": cfg["name"], - "loglevel": cfg["loglevel"], - "pretend": cfg["pretend"], - "type": "notify", - "options": cfg["options"]["notify"].get_config()}, - {}) - self._notification = Notify(notify_cfg, local_addrs, debug) + cfg["options"]["notify"]["loglevel"] = cfg["loglevel"] + self._notification = Notify( + cfg["options"]["notify"], local_addrs, debug) self._allowlist = None if "allowlist" in cfg["options"]: