From bccd9e27444815fc3ec7d134070d064639c19892 Mon Sep 17 00:00:00 2001 From: Thomas Oettli Date: Fri, 6 Nov 2020 00:17:59 +0100 Subject: [PATCH] turn FileManager to FileManagerScheduler --- .todo.swp | Bin 12288 -> 0 bytes pyinotifyd/__init__.py | 1 - pyinotifyd/filemanager.py | 193 -------------------------------------- pyinotifyd/scheduler.py | 188 +++++++++++++++++++++++++++++++++++++ todo | 0 5 files changed, 188 insertions(+), 194 deletions(-) delete mode 100644 .todo.swp delete mode 100755 pyinotifyd/filemanager.py delete mode 100644 todo diff --git a/.todo.swp b/.todo.swp deleted file mode 100644 index e53906f7c92e6cdad933c0c0933389ad25c9bdb0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI%!Aiq07zgmLf=6}e3q(C{ogK1W^eA}f#W}&#mMv);3{6>6WETa&H}MI)dJ!MM zlkc$CFddx-Vbg2JKk)I9=F8XScMBc2Z!SaDYX#&+gXnqpWjH$h^xkh9WNc!}$A67U zl1I622HozW%gXYHvY{`y<$kQJpUqTitcqsBw?_QyO)Fbt0Rj+MCs5d^+1nX(f|GVx zc6@Zm4)zD@c3=$x5P$##AOHafKmY;|*f0T`dGzScv{jvJyUO0LTuqT6009U<00Izz z00bZa0SG_<0uZR702f40dql4;%K!h%`~Td{zg&E{cyqB>ui^Rd$q;}51Rwwb2tWV= z5P$##AOL}S3iOl03Ui;*mDI*v->-Ro*PkR(6k1MO+QbwV+Nw-Twk*x*LTMTDl*e+) K&d%9SJ$(Zq1V2mw diff --git a/pyinotifyd/__init__.py b/pyinotifyd/__init__.py index 8a0da4c..f051433 100755 --- a/pyinotifyd/__init__.py +++ b/pyinotifyd/__init__.py @@ -195,7 +195,6 @@ def main(): try: config = {} exec(f"from {myname}.scheduler import *", config) - exec(f"from {myname}.filemanager import *", config) with open(args.config, "r") as c: exec(c.read(), globals(), config) daemon = config[f"{myname}"] diff --git a/pyinotifyd/filemanager.py b/pyinotifyd/filemanager.py deleted file mode 100755 index 02952c7..0000000 --- a/pyinotifyd/filemanager.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 - -# pyinotifyd is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# pyinotifyd is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with pyinotifyd. If not, see . -# - -import logging -import os -import re -import shutil - - -class Rule: - valid_actions = ["copy", "move", "delete"] - - def __init__(self, action, src_re, dst_re="", auto_create=False, - dirmode=None, filemode=None, user=None, group=None, - rec=False): - assert action in self.valid_actions, \ - f"action: expected [{Rule.valid_actions.join(', ')}], got{action}" - self.action = action - - self.src_re = re.compile(src_re) - - assert isinstance(dst_re, str), \ - f"dst_re: expected {type('')}, got {type(dst_re)}" - self.dst_re = dst_re - - assert isinstance(auto_create, bool), \ - f"auto_create: expected {type(bool)}, got {type(auto_create)}" - self.auto_create = auto_create - - if dirmode is not None: - assert isinstance(dirmode, int), \ - f"dirmode: expected {type(int)}, got {type(dirmode)}" - self.dirmode = dirmode - - if filemode is not None: - assert isinstance(filemode, int), \ - f"filemode: expected {type(int)}, got {type(filemode)}" - self.filemode = filemode - - if user is not None: - assert isinstance(user, str), \ - f"user: expected {type('')}, got {type(user)}" - self.user = user - - if group is not None: - assert isinstance(group, str), \ - f"group: expected {type('')}, got {type(group)}" - self.group = group - - assert isinstance(rec, bool), \ - f"rec: expected {type(bool)}, got {type(rec)}" - self.rec = rec - - -class FileManager: - def __init__(self, rules, logname="filemgr"): - if not isinstance(rules, list): - rules = [rules] - - for rule in rules: - assert isinstance(rule, Rule), \ - f"rules: expected {type(Rule)}, got {type(rule)}" - - self._rules = rules - self._log = logging.getLogger((logname or __name__)) - - def add_rule(self, *args, **kwargs): - self._rules.append(Rule(*args, **kwargs)) - - async def _chmod_and_chown(self, path, mode, chown, task_id): - if mode is not None: - self._log.debug(f"{task_id}: chmod {oct(mode)} '{path}'") - os.chmod(path, mode) - - if chown is not None: - changes = "" - if chown[0] is not None: - changes = chown[0] - - if chown[1] is not None: - changes = f"{changes}:{chown[1]}" - - self._log.debug(f"{task_id}: chown {changes} '{path}'") - shutil.chown(path, *chown) - - async def _set_mode_and_owner(self, path, rule, task_id): - if (rule.user is rule.group is None): - chown = None - else: - chown = (rule.user, rule.group) - - work_on_dirs = not (rule.dirmode is chown is None) - work_on_files = not (rule.filemode is chown is None) - - if os.path.isdir(path): - await self._chmod_and_chown(path, rule.dirmode, chown, task_id) - if work_on_dirs or work_on_files: - for root, dirs, files in os.walk(path): - if work_on_dirs: - for p in [os.path.join(root, d) for d in dirs]: - await self._chmod_and_chown( - p, rule.dirmode, chown, task_id) - - if work_on_files: - for p in [os.path.join(root, f) for f in files]: - await self._chmod_and_chown( - p, rule.filemode, chown, task_id) - else: - await self._chmod_and_chown(path, rule.filemode, chown, task_id) - - async def task(self, event, task_id): - path = event.pathname - match = None - for rule in self._rules: - match = rule.src_re.match(path) - if match: - break - - if not match: - self._log.debug( - f"{task_id}: path '{path}' matches no rule in ruleset") - return - - try: - if rule.action in ["copy", "move"]: - dst = rule.src_re.sub(rule.dst_re, path) - if not dst: - raise RuntimeError( - f"{task_id}: unable to {rule.action} '{path}', " - f"resulting destination path is empty") - - if os.path.exists(dst): - raise RuntimeError( - f"{task_id}: unable to move file from '{path} " - f"to '{dst}', dstination path exists already") - - dst_dir = os.path.dirname(dst) - if not os.path.isdir(dst_dir) and rule.auto_create: - self._log.info( - f"{task_id}: create directory '{dst_dir}'") - first_subdir = dst_dir - while not os.path.isdir(first_subdir): - parent = os.path.dirname(first_subdir) - if not os.path.isdir(parent): - first_subdir = parent - else: - break - os.makedirs(dst_dir) - await self._set_mode_and_owner(first_subdir, rule, task_id) - - self._log.info( - f"{task_id}: {rule.action} '{path}' to '{dst}'") - if rule.action == "copy": - if os.path.isdir(path): - shutil.copytree(path, dst) - else: - shutil.copy2(path, dst) - - else: - os.rename(path, dst) - - await self._set_mode_and_owner(dst, rule, task_id) - - elif rule.action == "delete": - self._log.info( - f"{task_id}: {rule.action} '{path}'") - if os.path.isdir(path): - if rule.rec: - shutil.rmtree(path) - else: - shutil.rmdir(path) - - else: - os.remove(path) - - except RuntimeError as e: - self._log.error(f"{task_id}: {e}") - - except Exception as e: - self._log.exception(f"{task_id}: {e}") diff --git a/pyinotifyd/scheduler.py b/pyinotifyd/scheduler.py index 1898479..2ce6f0f 100755 --- a/pyinotifyd/scheduler.py +++ b/pyinotifyd/scheduler.py @@ -14,6 +14,10 @@ import asyncio import logging +import os +import re +import shutil + from shlex import quote as shell_quote from uuid import uuid4 @@ -130,6 +134,9 @@ class TaskScheduler: task.cancel() del self._tasks[path] + def log(self, event): + self._log.info(f"LOG: received {event}") + class ShellScheduler(TaskScheduler): def __init__(self, cmd, task=None, *args, **kwargs): @@ -154,3 +161,184 @@ class ShellScheduler(TaskScheduler): self._log.info(f"{task_id}: execute shell command: {cmd}") proc = await asyncio.create_subprocess_shell(cmd) await proc.communicate() + + +class FileManagerRule: + valid_actions = ["copy", "move", "delete"] + + def __init__(self, action, src_re, dst_re="", auto_create=False, + dirmode=None, filemode=None, user=None, group=None, + rec=False): + valid = f"{', '.join(FileManagerRule.valid_actions)}" + assert action in self.valid_actions, \ + f"action: expected [{valid}], got{action}" + self.action = action + + self.src_re = re.compile(src_re) + + assert isinstance(dst_re, str), \ + f"dst_re: expected {type('')}, got {type(dst_re)}" + self.dst_re = dst_re + + assert isinstance(auto_create, bool), \ + f"auto_create: expected {type(bool)}, got {type(auto_create)}" + self.auto_create = auto_create + + if dirmode is not None: + assert isinstance(dirmode, int), \ + f"dirmode: expected {type(int)}, got {type(dirmode)}" + self.dirmode = dirmode + + if filemode is not None: + assert isinstance(filemode, int), \ + f"filemode: expected {type(int)}, got {type(filemode)}" + self.filemode = filemode + + if user is not None: + assert isinstance(user, str), \ + f"user: expected {type('')}, got {type(user)}" + self.user = user + + if group is not None: + assert isinstance(group, str), \ + f"group: expected {type('')}, got {type(group)}" + self.group = group + + assert isinstance(rec, bool), \ + f"rec: expected {type(bool)}, got {type(rec)}" + self.rec = rec + + +class FileManagerScheduler(TaskScheduler): + def __init__(self, rules, task=None, *args, **kwargs): + if not isinstance(rules, list): + rules = [rules] + + for rule in rules: + assert isinstance(rule, FileManagerRule), \ + f"rules: expected {type(FileManagerRule)}, got {type(rule)}" + self._rules = rules + + super().__init__(*args, task=self.task, **kwargs) + + async def _chmod_and_chown(self, path, mode, chown, task_id): + if mode is not None: + self._log.debug(f"{task_id}: chmod {oct(mode)} '{path}'") + os.chmod(path, mode) + + if chown is not None: + changes = "" + if chown[0] is not None: + changes = chown[0] + + if chown[1] is not None: + changes = f"{changes}:{chown[1]}" + + self._log.debug(f"{task_id}: chown {changes} '{path}'") + shutil.chown(path, *chown) + + async def _set_mode_and_owner(self, path, rule, task_id): + if (rule.user is rule.group is None): + chown = None + else: + chown = (rule.user, rule.group) + + work_on_dirs = not (rule.dirmode is chown is None) + work_on_files = not (rule.filemode is chown is None) + + if os.path.isdir(path): + await self._chmod_and_chown(path, rule.dirmode, chown, task_id) + if work_on_dirs or work_on_files: + for root, dirs, files in os.walk(path): + if work_on_dirs: + for p in [os.path.join(root, d) for d in dirs]: + await self._chmod_and_chown( + p, rule.dirmode, chown, task_id) + + if work_on_files: + for p in [os.path.join(root, f) for f in files]: + await self._chmod_and_chown( + p, rule.filemode, chown, task_id) + else: + await self._chmod_and_chown(path, rule.filemode, chown, task_id) + + def _get_rule_by_event(self, event): + rule = None + for r in self._rules: + if r.src_re.match(event.pathname): + rule = r + break + + return rule + + def schedule(self, event): + if self._get_rule_by_event(event): + super().schedule(event) + else: + self._log.debug( + f"no rule in ruleset matches path '{event.pathname}'") + + async def task(self, event, task_id): + path = event.pathname + rule = self._get_rule_by_event(event) + + if not rule: + return + + try: + if rule.action in ["copy", "move"]: + dst = rule.src_re.sub(rule.dst_re, path) + if not dst: + raise RuntimeError( + f"{task_id}: unable to {rule.action} '{path}', " + f"resulting destination path is empty") + + if os.path.exists(dst): + raise RuntimeError( + f"{task_id}: unable to move file from '{path} " + f"to '{dst}', dstination path exists already") + + dst_dir = os.path.dirname(dst) + if not os.path.isdir(dst_dir) and rule.auto_create: + self._log.info( + f"{task_id}: create directory '{dst_dir}'") + first_subdir = dst_dir + while not os.path.isdir(first_subdir): + parent = os.path.dirname(first_subdir) + if not os.path.isdir(parent): + first_subdir = parent + else: + break + os.makedirs(dst_dir) + await self._set_mode_and_owner(first_subdir, rule, task_id) + + self._log.info( + f"{task_id}: {rule.action} '{path}' to '{dst}'") + if rule.action == "copy": + if os.path.isdir(path): + shutil.copytree(path, dst) + else: + shutil.copy2(path, dst) + + else: + os.rename(path, dst) + + await self._set_mode_and_owner(dst, rule, task_id) + + elif rule.action == "delete": + self._log.info( + f"{task_id}: {rule.action} '{path}'") + if os.path.isdir(path): + if rule.rec: + shutil.rmtree(path) + else: + shutil.rmdir(path) + + else: + os.remove(path) + + except RuntimeError as e: + self._log.error(f"{task_id}: {e}") + + except Exception as e: + self._log.exception(f"{task_id}: {e}") diff --git a/todo b/todo deleted file mode 100644 index e69de29..0000000