change config structure

2024-01-03 19:54:34 +01:00
parent 5d21b81530
commit 4da1a0e9b3
7 changed files with 194 additions and 99 deletions
--- a/pyquarantine/init.py
+++ b/pyquarantine/init.py
@@ -27,7 +27,7 @@ __all__ = [
    "list",
    "QuarantineMilter"]

-__version__ = "2.0.8"
+__version__ = "2.1.0"

 from pyquarantine import _runtime_patches

--- a/pyquarantine/action.py
+++ b/pyquarantine/action.py
@@ -39,8 +39,7 @@ class Action:
            self.conditions["loglevel"] = cfg["loglevel"]
            self.conditions = Conditions(self.conditions, local_addrs, debug)

-        action_type = cfg["type"]
-        self.action = self.ACTION_TYPES[action_type](
+        self.action = self.ACTION_TYPES[cfg["type"]](
            cfg, local_addrs, debug)

    def __str__(self):
--- a/pyquarantine/cli.py
+++ b/pyquarantine/cli.py
@@ -21,28 +21,31 @@ import logging.handlers
 import sys
 import time

-from pyquarantine.config import get_milter_config, ActionConfig, ListConfig
+from pyquarantine.action import Action
+from pyquarantine.config import get_milter_config, ActionConfig, StorageConfig, NotificationConfig, ListConfig
 from pyquarantine.storage import Quarantine
 from pyquarantine.list import DatabaseList
 from pyquarantine import __version__ as version


-def _get_quarantines(cfg):
+def _get_quarantines(milter_cfg):
    quarantines = []
-    for rule in cfg["rules"]:
+    for rule in milter_cfg["rules"]:
        for action in rule["actions"]:
            if action["type"] == "quarantine":
                quarantines.append(action)
    return quarantines


-def _get_quarantine(cfg, name, debug):
+def _get_quarantine(milter_cfg, name, debug):
    try:
        quarantine = next(
-            (q for q in _get_quarantines(cfg) if q["name"] == name))
+            (q for q in _get_quarantines(milter_cfg) if q["name"] == name))
    except StopIteration:
        raise RuntimeError(f"invalid quarantine '{name}'")
-    return Quarantine(ActionConfig(quarantine, cfg["lists"]), [], debug)
+
+    cfg = ActionConfig(quarantine, milter_cfg)
+    return Quarantine(cfg, [], debug)


 def _get_notification(cfg, name, debug):
@@ -55,7 +58,7 @@ def _get_notification(cfg, name, debug):

 def _get_list(cfg, name, debug):
    try:
-        list_cfg = ListConfig(cfg["lists"][name], {})
+        list_cfg = cfg["lists"][name]
    except KeyError:
        raise RuntimeError(f"list '{name}' is not configured")

@@ -107,7 +110,7 @@ def print_table(columns, rows):
        print(row_format.format(*row))


-def list_quarantines(cfg, args):
+def llist(cfg, args):
    quarantines = _get_quarantines(cfg)
    if args.batch:
        print("\n".join([q["name"] for q in quarantines]))
@@ -115,32 +118,33 @@ def list_quarantines(cfg, args):
        qlist = []
        for q in quarantines:
            qcfg = q["options"]
-            storage_type = qcfg["store"]["type"]

-            if "notify" in cfg:
-                notification_type = qcfg["notify"]["type"]
+            if "notify" in qcfg:
+                notification = cfg["notifications"][qcfg["notify"]]["name"]
            else:
-                notification_type = "NONE"
+                notification = "NONE"

-            if "lists" in qcfg:
-                lists_type = qcfg["lists"]
+            if "allowlist" in qcfg:
+                allowlist = qcfg["allowlist"]
            else:
-                lists_type = "NONE"
+                allowlist = "NONE"

            if "milter_action" in qcfg:
                milter_action = qcfg["milter_action"]
            else:
                milter_action = "NONE"

+            storage_name = cfg["storages"][qcfg["store"]]["name"]
+
            qlist.append({
                "name": q["name"],
-                "storage": storage_type,
-                "notification": notification_type,
-                "lists": lists_type,
+                "storage": storage_name,
+                "notification": notification,
+                "lists": allowlist,
                "action": milter_action})

        print_table(
-            [("Name", "name"),
+            [("Quarantine", "name"),
             ("Storage", "storage"),
             ("Notification", "notification"),
             ("Allowlist", "lists"),
@@ -148,6 +152,48 @@ def list_quarantines(cfg, args):
            qlist
        )

+        if "storages" in cfg:
+            storages = []
+            for name, options in cfg["storages"].items():
+                storages.append({
+                    "name": name,
+                    "type": options["type"]})
+
+            print("\n")
+            print_table(
+                [("Storage", "name"),
+                 ("Type", "type")],
+                storages
+            )
+
+        if "notifications" in cfg:
+            notifications = []
+            for name, options in cfg["notifications"].items():
+                notifications.append({
+                    "name": name,
+                    "type": options["type"]})
+
+            print("\n")
+            print_table(
+                [("Notification", "name"),
+                 ("Type", "type")],
+                notifications
+            )
+
+        if "lists" in cfg:
+            lst_list = []
+            for name, options in cfg["lists"].items():
+                lst_list.append({
+                    "name": name,
+                    "type": options["type"]})
+
+            print("\n")
+            print_table(
+                [("List", "name"),
+                 ("Type", "type")],
+                lst_list
+            )
+

 def list_quarantine_emails(cfg, args):
    storage = _get_quarantine(cfg, args.quarantine, args.debug).storage
@@ -364,7 +410,7 @@ def main():
        "-b", "--batch",
        help="Print results using only quarantine names, each on a new line.",
        action="store_true")
-    list_parser.set_defaults(func=list_quarantines)
+    list_parser.set_defaults(func=llist)

    # quarantine command group
    quar_parser = subparsers.add_parser(
@@ -638,7 +684,7 @@ def main():

    try:
        logger.debug("read milter configuration")
-        cfg = get_milter_config(args.config, raw=True)
+        cfg = get_milter_config(args.config, rec=False)

        if "rules" not in cfg or not cfg["rules"]:
            raise RuntimeError("no rules configured")
--- a/pyquarantine/config.py
+++ b/pyquarantine/config.py
@@ -20,7 +20,9 @@ __all__ = [
    "DelHeaderConfig",
    "AddDisclaimerConfig",
    "RewriteLinksConfig",
+    "StorageConfig",
    "StoreConfig",
+    "NotificationConfig",
    "NotifyConfig",
    "ListConfig",
    "QuarantineConfig",
@@ -43,7 +45,7 @@ class BaseConfig:
        "properties": {
            "loglevel": {"type": "string", "default": "info"}}}

-    def __init__(self, config, lists):
+    def __init__(self, config, *args, **kwargs):
        required = self.JSON_SCHEMA["required"]
        properties = self.JSON_SCHEMA["properties"]
        for p in properties.keys():
@@ -126,15 +128,13 @@ class ConditionsConfig(BaseConfig):
            "list": {"type": "string"}}}

    def __init__(self, config, lists, rec=True):
-        super().__init__(config, lists)
+        super().__init__(config)
        if "list" in self:
            lst = self["list"]
            try:
                self["list"] = lists[lst]
            except KeyError:
-                raise RuntimeError(f"list '{lst}' is not configured")
-        if not rec:
-            return
+                raise RuntimeError(f"list '{lst}' not found in config")


 class AddHeaderConfig(BaseConfig):
@@ -190,7 +190,7 @@ class RewriteLinksConfig(BaseConfig):
            "repl": {"type": "string"}}}


-class StoreConfig(BaseConfig):
+class StorageConfig(BaseConfig):
    JSON_SCHEMA = {
        "type": "object",
        "required": ["type"],
@@ -203,6 +203,7 @@ class StoreConfig(BaseConfig):
            "additionalProperties": False,
            "properties": {
                "type": {"type": "string"},
+                "name": {"type": "string"},
                "directory": {"type": "string"},
                "mode": {"type": "string"},
                "metavar": {"type": "string"},
@@ -210,7 +211,16 @@ class StoreConfig(BaseConfig):
                "original": {"type": "boolean", "default": False}}}}


-class NotifyConfig(BaseConfig):
+class StoreConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["storage"],
+        "additionalProperties": False,
+        "properties": {
+            "storage": {"type": "string"}}}
+
+
+class NotificationConfig(BaseConfig):
    JSON_SCHEMA = {
        "type": "object",
        "required": ["type"],
@@ -224,6 +234,7 @@ class NotifyConfig(BaseConfig):
            "additionalProperties": False,
            "properties": {
                "type": {"type": "string"},
+                "name": {"type": "string"},
                "smtp_host": {"type": "string"},
                "smtp_port": {"type": "number"},
                "envelope_from": {"type": "string"},
@@ -238,6 +249,15 @@ class NotifyConfig(BaseConfig):
                    "default": []}}}}


+class NotifyConfig(BaseConfig):
+    JSON_SCHEMA = {
+        "type": "object",
+        "required": ["notification"],
+        "additionalProperties": False,
+        "properties": {
+            "notification": {"type": "string"}}}
+
+
 class QuarantineConfig(BaseConfig):
    JSON_SCHEMA = {
        "type": "object",
@@ -245,29 +265,38 @@ class QuarantineConfig(BaseConfig):
        "additionalProperties": False,
        "properties": {
            "name": {"type": "string"},
-            "notify": {"type": "object"},
+            "notify": {"type": "string"},
            "milter_action": {"type": "string"},
            "reject_reason": {"type": "string"},
            "allowlist": {"type": "string"},
-            "store": {"type": "object"},
+            "store": {"type": "string"},
            "smtp_host": {"type": "string"},
            "smtp_port": {"type": "number"}}}

-    def __init__(self, config, lists, rec=True):
-        super().__init__(config, lists)
-        if not rec:
-            return
+    def __init__(self, config, milter_config, rec=True):
+        super().__init__(config)
+        storage = self["store"]
+        try:
+            self["store"] = milter_config["storages"][storage]
+        except KeyError:
+            raise RuntimeError(f"storage '{storage}' not found")
        if "metadata" not in self["store"]:
            self["store"]["metadata"] = True
-        self["store"] = StoreConfig(self["store"], lists)
        if "notify" in self:
-            self["notify"] = NotifyConfig(self["notify"], lists)
+            notify = self["notify"]
+            try:
+                self["notify"] = milter_config["notifications"][notify]
+            except KeyError:
+                raise RuntimeError(f"notification '{notify}' not found")
        if "allowlist" in self:
            allowlist = self["allowlist"]
            try:
-                self["allowlist"] = lists[allowlist]
+                self["allowlist"] = milter_config["lists"][allowlist]
            except KeyError:
-                raise RuntimeError(f"list '{allowlist}' is not configured")
+                raise RuntimeError(f"list '{allowlist}' not found")
+
+        if not rec:
+            return


 class ActionConfig(BaseConfig):
@@ -293,14 +322,31 @@ class ActionConfig(BaseConfig):
            "type": {"enum": list(ACTION_TYPES.keys())},
            "options": {"type": "object"}}}

-    def __init__(self, config, lists, rec=True):
-        super().__init__(config, lists)
+    def __init__(self, config, milter_config, rec=True):
+        super().__init__(config)
        if not rec:
            return
+        lists = milter_config["lists"]
        if "conditions" in self:
            self["conditions"] = ConditionsConfig(self["conditions"], lists)
-        self["action"] = self.ACTION_TYPES[self["type"]](
-            self["options"], lists)
+
+        if self["type"] == "store":
+            storage = StoreConfig(self["options"])["storage"]
+            try:
+                self["action"] = milter_config["storages"][storage]
+            except KeyError:
+                raise RuntimeError(f"storage '{storage}' not found")
+
+        elif self["type"] == "notify":
+            notify = NotifyConfig(self["options"])["notification"]
+            try:
+                self["action"] = milter_config["notifications"][notify]
+            except KeyError:
+                raise RuntimeError(f"notification '{notify}' not found")
+
+        else:
+            self["action"] = self.ACTION_TYPES[self["type"]](
+                self["options"], milter_config)


 class RuleConfig(BaseConfig):
@@ -315,10 +361,11 @@ class RuleConfig(BaseConfig):
            "conditions": {"type": "object"},
            "actions": {"type": "array"}}}

-    def __init__(self, config, lists, rec=True):
-        super().__init__(config, lists)
+    def __init__(self, config, milter_config, rec=True):
+        super().__init__(config)
        if not rec:
            return
+        lists = milter_config["lists"]
        if "conditions" in self:
            self["conditions"] = ConditionsConfig(self["conditions"], lists)

@@ -328,7 +375,7 @@ class RuleConfig(BaseConfig):
                action["loglevel"] = config["loglevel"]
            if "pretend" not in action:
                action["pretend"] = config["pretend"]
-            actions.append(ActionConfig(action, lists, rec))
+            actions.append(ActionConfig(action, milter_config, rec))
        self["actions"] = actions


@@ -350,29 +397,52 @@ class QuarantineMilterConfig(BaseConfig):
                                "192.168.0.0/16"]},
            "loglevel": {"type": "string", "default": "info"},
            "pretend": {"type": "boolean", "default": False},
-            "rules": {"type": "array"},
-            "lists": {"type": "array",
-                      "default": []}}}
+            "lists": {
+                "type": "object",
+                "patternProperties": {"^(.+)$": {"type": "object"}},
+                "additionalProperties": False,
+                "default": {}},
+            "storages": {
+                "type": "object",
+                "patternProperties": {"^(.+)$": {"type": "object"}},
+                "additionalProperties": False,
+                "default": {}},
+            "notifications": {
+                "type": "object",
+                "patternProperties": {"^(.+)$": {"type": "object"}},
+                "additionalProperties": False,
+                "default": {}},
+            "rules": {"type": "array"}}}

    def __init__(self, config, rec=True):
-        super().__init__(config, {})
+        super().__init__(config)
+        for param in ["lists", "storages", "notifications"]:
+            for name, cfg in self[param].items():
+                if "name" not in cfg:
+                    cfg["name"] = name
+        for name, cfg in self["lists"].items():
+            self["lists"][name] = ListConfig(cfg)
+
+        for name, cfg in self["storages"].items():
+            self["storages"][name] = StorageConfig(cfg)
+
+        for name, cfg in self["notifications"].items():
+            self["notifications"][name] = NotificationConfig(cfg)
+
        if not rec:
            return
-        lists = {}
-        for lst in self["lists"]:
-            lists[lst["name"]] = ListConfig(lst, rec)
-        self["lists"] = lists
+
        rules = []
        for rule in self["rules"]:
            if "loglevel" not in rule:
                rule["loglevel"] = config["loglevel"]
            if "pretend" not in rule:
                rule["pretend"] = config["pretend"]
-            rules.append(RuleConfig(rule, lists, rec))
+            rules.append(RuleConfig(rule, self, rec))
        self["rules"] = rules


-def get_milter_config(cfgfile, raw=False):
+def get_milter_config(cfgfile, rec=True):
    try:
        with open(cfgfile, "r") as fh:
            # remove lines with leading # (comments), they
@@ -387,10 +457,4 @@ def get_milter_config(cfgfile, raw=False):
        cfg_text = [f"{n+1}: {l}" for n, l in enumerate(cfg.splitlines())]
        msg = "\n".join(cfg_text)
        raise RuntimeError(f"{e}\n{msg}")
-    if raw:
-        lists = {}
-        for lst in cfg["lists"]:
-            lists[lst["name"]] = lst
-        cfg["lists"] = lists
-        return cfg
-    return QuarantineMilterConfig(cfg)
+    return QuarantineMilterConfig(cfg, rec)
--- a/pyquarantine/notify.py
+++ b/pyquarantine/notify.py
@@ -331,18 +331,17 @@ class Notify:
    def __init__(self, cfg, local_addrs, debug):
        self.cfg = cfg
        self.logger = logging.getLogger(cfg["name"])
+        del cfg["name"]
        self.logger.setLevel(cfg.get_loglevel(debug))
-
-        nodification_type = cfg["options"]["type"]
-        del cfg["options"]["type"]
-        cfg["options"]["pretend"] = cfg["pretend"]
-        self._notification = self.NOTIFICATION_TYPES[nodification_type](
-            **cfg["options"])
+        del cfg["loglevel"]
+        nodification_type = cfg["type"]
+        del cfg["type"]
+        self._notification = self.NOTIFICATION_TYPES[nodification_type](**cfg)
        self._headersonly = self._notification._headersonly

    def __str__(self):
        cfg = []
-        for key, value in self.cfg["options"].items():
+        for key, value in self.cfg.items():
            cfg.append(f"{key}={value}")
        class_name = type(self._notification).__name__
        return f"{class_name}(" + ", ".join(cfg) + ")"
--- a/pyquarantine/rule.py
+++ b/pyquarantine/rule.py
@@ -46,8 +46,9 @@ class Rule:
        actions = []
        for action in self.actions:
            actions.append(str(action))
-        cfg.append("actions=[" + ", ".join(actions) + "]")
-        return "Rule(" + ", ".join(cfg) + ")"
+        cfg.append("actions=[\n        " +
+                   ",\n        ".join(actions) + "\n    ]")
+        return "Rule(\n    " + ",\n    ".join(cfg) + "\n)"

    def execute(self, milter):
        """Execute all actions of this rule."""
--- a/pyquarantine/storage.py
+++ b/pyquarantine/storage.py
@@ -31,7 +31,6 @@ from time import gmtime

 from pyquarantine import mailer
 from pyquarantine.base import CustomLogger, MilterMessage
-from pyquarantine.config import ActionConfig
 from pyquarantine.list import DatabaseList
 from pyquarantine.notify import Notify

@@ -373,18 +372,17 @@ class Store:
    def __init__(self, cfg, local_addrs, debug):
        self.cfg = cfg
        self.logger = logging.getLogger(cfg["name"])
+        del cfg["name"]
        self.logger.setLevel(cfg.get_loglevel(debug))
-
-        storage_type = cfg["options"]["type"]
-        del cfg["options"]["type"]
-        cfg["options"]["pretend"] = cfg["pretend"]
-        self._storage = self.STORAGE_TYPES[storage_type](
-            **cfg["options"])
+        del cfg["loglevel"]
+        storage_type = cfg["type"]
+        del cfg["type"]
+        self._storage = self.STORAGE_TYPES[storage_type](**cfg)
        self._headersonly = self._storage._headersonly

    def __str__(self):
        cfg = []
-        for key, value in self.cfg["options"].items():
+        for key, value in self.cfg.items():
            cfg.append(f"{key}={value}")
        class_name = type(self._storage).__name__
        return f"{class_name}(" + ", ".join(cfg) + ")"
@@ -407,29 +405,17 @@ class Quarantine:
        self.logger = logging.getLogger(cfg["name"])
        self.logger.setLevel(cfg.get_loglevel(debug))

-        storage_cfg = ActionConfig(
-            {
-                "name": cfg["name"],
-                "loglevel": cfg["loglevel"],
-                "pretend": cfg["pretend"],
-                "type": "store",
-                "options": cfg["options"]["store"].get_config()},
-            {})
-        self._storage = Store(storage_cfg, local_addrs, debug)
+        cfg["options"]["store"]["loglevel"] = cfg["loglevel"]
+        self._storage = Store(cfg["options"]["store"], local_addrs, debug)

        self.smtp_host = cfg["options"]["smtp_host"]
        self.smtp_port = cfg["options"]["smtp_port"]

        self._notification = None
        if "notify" in cfg["options"]:
-            notify_cfg = ActionConfig({
-                    "name": cfg["name"],
-                    "loglevel": cfg["loglevel"],
-                    "pretend": cfg["pretend"],
-                    "type": "notify",
-                    "options": cfg["options"]["notify"].get_config()},
-                {})
-            self._notification = Notify(notify_cfg, local_addrs, debug)
+            cfg["options"]["notify"]["loglevel"] = cfg["loglevel"]
+            self._notification = Notify(
+                cfg["options"]["notify"], local_addrs, debug)

        self._allowlist = None
        if "allowlist" in cfg["options"]: