diff --git a/MANIFEST.in b/MANIFEST.in index ca4758c..84821af 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include LICENSE README.md -recursive-include docs * -recursive-include misc * +recursive-include pyinotifyd/docs * +recursive-include pyinotifyd/misc * diff --git a/README.md b/README.md index a6b036e..dc433b8 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ A rule holds an *action* (move, copy or delete) and a regular expression *src_re If the action is copy or move, the destination path *dst_re* is mandatory and if *action* is delete and *rec* is set to True, non-empty directories will be deleted recursively. With *auto_create* set to True, possibly missing subdirectories in *dst_re* are created automatically. Regex subgroups or named-subgroups may be used in *src_re* and *dst_re*. Set the mode of moved/copied files/directories with *filemode* and *dirmode*. Ownership of moved/copied files/directories is set with *user* and *group*. Mode and ownership is also set to automatically created subdirectories. -Use *logname* in log messages. +Log messages with *logname*. ```python rule = Rule( action="move", @@ -53,7 +53,7 @@ pyinotifyd has different schedulers to schedule tasks with an optional delay. Th #### TaskScheduler TaskScheduler schedules *task* with an optional *delay* in seconds. Use the *files* and *dirs* arguments to schedule tasks only for files and/or directories. -Use *logname* in log messages. All arguments except for *task* are optional. +Log messages with *logname*. All arguments except for *task* are optional. ```python s = TaskScheduler( task=task, @@ -109,13 +109,47 @@ watch = Watch( auto_add=False) ``` -### PyinotifydConfig -pyinotifyd expects an instance of PyinotifydConfig named **pyinotifyd_config** that holds its config options. The options are a list of *watches*, the *loglevel* (see https://docs.python.org/3/library/logging.html#levels) and the *shutdown_timeout*. pyinotifyd will wait *shutdown_timeout* seconds for pending tasks to complete during shutdown. +### Pyinotifyd +pyinotifyd expects an instance of Pyinotifyd named **pyinotifyd** defined in the config file. The options are a list of *watches* and the *shutdown_timeout*. pyinotifyd will wait *shutdown_timeout* seconds for pending tasks to complete during shutdown. Log messages with *logname*. ```python -pyinotifyd_config = PyinotifydConfig( +pyinotifyd = Pyinotifyd( watches=[watch], - loglevel=logging.INFO, - shutdown_timeout=30) + shutdown_timeout=30, + logname="Pyinotifyd") +``` + +### Logging +Pythons [logging](https://docs.python.org/3/howto/logging.html) framework is used to log messages (see https://docs.python.org/3/howto/logging.html). + +Configure the global loglevel. This is the default: +```python +logging.getLogger().setLevel(logging.WARNING) +``` +It is possible to configure the loglevel per log name. This is an example for logname **TaskScheduler**: +```python +logging.getLogger("TaskScheduler").setLevel(logging.INFO) +``` + +#### Syslog +Add this to your config file to send log messages to a local syslog server. +```python +# send log messages to the Unix socket of the syslog server. +syslog = logging.handlers.SysLogHandler( + address="/dev/log") + +# set the log format of syslog messages +log_format = "pyinotifyd/%(name)s: %(message)s" +syslog.setFormatter( + logging.Formatter(formatter) + +# set the log level for syslog messages +syslog.setLevel(logging.INFO) + +# enable syslog +logging.getLogger().addHandler(syslog) + +# or enable syslog just for TaskScheduler +logging.getLogger("TaskManager").addHandler(syslog) ``` ### Autostart @@ -151,7 +185,6 @@ watch = Watch( pyinotifyd_config = PyinotifydConfig( watches=[watch], - loglevel=logging.INFO, shutdown_timeout=5) ``` @@ -173,7 +206,6 @@ watch = Watch( pyinotifyd_config = PyinotifydConfig( watches=[watch], - loglevel=logging.INFO, shutdown_timeout=5) ``` @@ -227,6 +259,5 @@ watch = Watch( # otherwise pending tasks may get cancelled during shutdown. pyinotifyd_config = PyinotifydConfig( watches=[watch], - loglevel=logging.INFO, shutdown_timeout=35) ``` diff --git a/pyinotifyd.py b/pyinotifyd.py deleted file mode 100755 index 2c6f288..0000000 --- a/pyinotifyd.py +++ /dev/null @@ -1,576 +0,0 @@ -#!/usr/bin/env python3 - -# pyinotifyd is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# pyinotifyd is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with pyinotifyd. If not, see . -# - -import argparse -import asyncio -import logging -import logging.handlers -import os -import pyinotify -import re -import shutil -import signal -import sys - -from shlex import quote as shell_quote -from uuid import uuid4 - -__version__ = "0.0.1" - - -class Task: - def __init__(self, event, delay, task_id, task, callback=None, - logname=None): - self._event = event - self._path = event.pathname - self._delay = delay - self._task_id = task_id - self._job = task - self._callback = callback - - self._task = None - self._log = logging.getLogger((logname or self.__class__.__name__)) - - async def _start(self): - if self._delay > 0: - await asyncio.sleep(self._delay) - - if self._callback is not None: - self._callback(self._event) - - self._task = None - - self._log.info(f"execute task {self._task_id}") - await asyncio.shield(self._job(self._event, self._task_id)) - self._log.info(f"task {self._task_id} finished") - - def start(self): - if self._task is None: - self._task = asyncio.create_task(self._start()) - - def cancel(self): - if self._task is not None: - self._task.cancel() - self._task = None - - def restart(self): - self.cancel() - self.start() - - -class TaskList: - def __init__(self, tasks=[]): - if not isinstance(tasks, list): - tasks = [tasks] - - self._tasks = tasks - - def add(self, task): - self._tasks.append(task) - - def remove(self, task): - self._tasks.remove(task) - - def execute(self, event): - for task in self._tasks: - task(event) - - -class TaskScheduler: - def __init__(self, task, files, dirs, delay=0, logname=None): - assert callable(task), \ - f"task: expected callable, got {type(task)}" - self._task = task - - assert isinstance(delay, int), \ - f"delay: expected {type(int)}, got {type(delay)}" - self._delay = delay - - assert isinstance(files, bool), \ - f"files: expected {type(bool)}, got {type(files)}" - self._files = files - - assert isinstance(dirs, bool), \ - f"dirs: expected {type(bool)}, got {type(dirs)}" - self._dirs = dirs - - self._tasks = {} - self._logname = (logname or self.__class__.__name__) - self._log = logging.getLogger(self._logname) - - def _task_started(self, event): - path = event.pathname - if path in self._tasks: - del self._tasks[path] - - def schedule(self, event): - self._log.debug(f"received {event}") - - if (not event.dir and not self._files) or \ - (event.dir and not self._dirs): - return - - path = event.pathname - maskname = event.maskname.split("|", 1)[0] - - if path in self._tasks: - task = self._tasks[path] - self._log.info( - f"received event {maskname} on '{path}', " - f"re-schedule task {task.task_id} (delay={self._delay}s)") - task.restart() - else: - task_id = str(uuid4()) - self._log.info( - f"received event {maskname} on '{path}', " - f"schedule task {task_id} (delay={self._delay}s)") - task = Task( - event, self._delay, task_id, self._task, - callback=self._task_started, logname=self._logname) - self._tasks[path] = task - task.start() - - def cancel(self, event): - self._log.debug(f"received {event}") - - path = event.pathname - maskname = event.maskname.split("|", 1)[0] - if path in self._tasks: - task = self._tasks[path] - self._log.info( - f"received event {maskname} on '{path}', " - f"cancel scheduled task {task.task_id}") - task.cancel() - del self._tasks[path] - - -class ShellScheduler(TaskScheduler): - def __init__(self, cmd, task=None, logname=None, *args, **kwargs): - assert isinstance(cmd, str), \ - f"cmd: expected {type('')}, got {type(cmd)}" - self._cmd = cmd - - logname = (logname or self.__class__.__name__) - super().__init__(*args, task=self.task, logname=logname, **kwargs) - - async def task(self, event, task_id): - maskname = event.maskname.split("|", 1)[0] - - if hasattr(event, "src_pathname"): - src_pathname = event.src_pathname - else: - src_pathname = "" - - cmd = self._cmd.replace("{maskname}", shell_quote(maskname)).replace( - "{pathname}", shell_quote(event.pathname)).replace( - "{src_pathname}", shell_quote(src_pathname)) - - self._log.info(f"{task_id}: execute shell command: {cmd}") - proc = await asyncio.create_subprocess_shell(cmd) - await proc.communicate() - - -class EventMap: - flags = { - **pyinotify.EventsCodes.OP_FLAGS, - **pyinotify.EventsCodes.EVENT_FLAGS} - - def __init__(self, event_map=None, default_task=None): - self._map = {} - - if default_task is not None: - assert callable(default_task), \ - f"default_task: expected callable, got {type(default_task)}" - for flag in EventMap.flags: - self.set(flag, default_task) - - if event_map is not None: - assert isinstance(event_map, dict), \ - f"event_map: expected {type(dict)}, got {type(event_map)}" - for flag, task in event_map.items(): - self.set(flag, task) - - def items(self): - return self._map.items() - - def set(self, flag, values): - assert flag in EventMap.flags, \ - f"event_map: invalid flag: {flag}" - if values is not None: - if not isinstance(values, list): - values = [values] - - for value in values: - assert callable(value), \ - f"event_map: {flag}: expected callable, got {type(value)}" - - self._map[flag] = values - elif flag in self._map: - del self._map[flag] - - -class Watch: - def __init__(self, path, event_map, rec=False, auto_add=False): - assert isinstance(path, str), \ - f"path: expected {type('')}, got {type(path)}" - self.path = path - - if isinstance(event_map, EventMap): - self.event_map = event_map - elif isinstance(event_map, dict): - self.event_map = EventMap(event_map) - else: - raise AssertionError( - f"event_map: expected {type(EventMap)} or {type(dict)}, " - f"got {type(event_map)}") - - assert isinstance(rec, bool), \ - f"rec: expected {type(bool)}, got {type(rec)}" - self.rec = rec - - assert isinstance(auto_add, bool), \ - f"auto_add: expected {type(bool)}, got {type(auto_add)}" - self.auto_add = auto_add - - def event_notifier(self, wm, loop): - handler = pyinotify.ProcessEvent() - mask = False - for flag, values in self.event_map.items(): - setattr(handler, f"process_{flag}", TaskList(values).execute) - if mask: - mask = mask | EventMap.flags[flag] - else: - mask = EventMap.flags[flag] - - wm.add_watch( - self.path, mask, rec=self.rec, auto_add=self.auto_add, - do_glob=True) - - return pyinotify.AsyncioNotifier(wm, loop, default_proc_fun=handler) - - -class Rule: - valid_actions = ["copy", "move", "delete"] - - def __init__(self, action, src_re, dst_re="", auto_create=False, - dirmode=None, filemode=None, user=None, group=None, - rec=False): - assert action in self.valid_actions, \ - f"action: expected [{Rule.valid_actions.join(', ')}], got{action}" - self.action = action - - self.src_re = re.compile(src_re) - - assert isinstance(dst_re, str), \ - f"dst_re: expected {type('')}, got {type(dst_re)}" - self.dst_re = dst_re - - assert isinstance(auto_create, bool), \ - f"auto_create: expected {type(bool)}, got {type(auto_create)}" - self.auto_create = auto_create - - if dirmode is not None: - assert isinstance(dirmode, int), \ - f"dirmode: expected {type(int)}, got {type(dirmode)}" - self.dirmode = dirmode - - if filemode is not None: - assert isinstance(filemode, int), \ - f"filemode: expected {type(int)}, got {type(filemode)}" - self.filemode = filemode - - if user is not None: - assert isinstance(user, str), \ - f"user: expected {type('')}, got {type(user)}" - self.user = user - - if group is not None: - assert isinstance(group, str), \ - f"group: expected {type('')}, got {type(group)}" - self.group = group - - assert isinstance(rec, bool), \ - f"rec: expected {type(bool)}, got {type(rec)}" - self.rec = rec - - -class FileManager: - def __init__(self, rules, logname=None): - if not isinstance(rules, list): - rules = [rules] - - for rule in rules: - assert isinstance(rule, Rule), \ - f"rules: expected {type(Rule)}, got {type(rule)}" - - self._rules = rules - self._log = logging.getLogger((logname or self.__class__.__name__)) - - def add_rule(self, *args, **kwargs): - self._rules.append(Rule(*args, **kwargs)) - - async def _chmod_and_chown(self, path, mode, chown, task_id): - if mode is not None: - self._log.debug(f"{task_id}: chmod {oct(mode)} '{path}'") - os.chmod(path, mode) - - if chown is not None: - changes = "" - if chown[0] is not None: - changes = chown[0] - - if chown[1] is not None: - changes = f"{changes}:{chown[1]}" - - self._log.debug(f"{task_id}: chown {changes} '{path}'") - shutil.chown(path, *chown) - - async def _set_mode_and_owner(self, path, rule, task_id): - if (rule.user is rule.group is None): - chown = None - else: - chown = (rule.user, rule.group) - - work_on_dirs = not (rule.dirmode is chown is None) - work_on_files = not (rule.filemode is chown is None) - - if os.path.isdir(path): - await self._chmod_and_chown(path, rule.dirmode, chown, task_id) - if work_on_dirs or work_on_files: - for root, dirs, files in os.walk(path): - if work_on_dirs: - for p in [os.path.join(root, d) for d in dirs]: - await self._chmod_and_chown( - p, rule.dirmode, chown, task_id) - - if work_on_files: - for p in [os.path.join(root, f) for f in files]: - await self._chmod_and_chown( - p, rule.filemode, chown, task_id) - else: - await self._chmod_and_chown(path, rule.filemode, chown, task_id) - - async def task(self, event, task_id): - path = event.pathname - match = None - for rule in self._rules: - match = rule.src_re.match(path) - if match: - break - - if not match: - self._log.debug( - f"{task_id}: path '{path}' matches no rule in ruleset") - return - - try: - if rule.action in ["copy", "move"]: - dst = rule.src_re.sub(rule.dst_re, path) - if not dst: - raise RuntimeError( - f"{task_id}: unable to {rule.action} '{path}', " - f"resulting destination path is empty") - - if os.path.exists(dst): - raise RuntimeError( - f"{task_id}: unable to move file from '{path} " - f"to '{dst}', dstination path exists already") - - dst_dir = os.path.dirname(dst) - if not os.path.isdir(dst_dir) and rule.auto_create: - self._log.info( - f"{task_id}: create directory '{dst_dir}'") - first_subdir = dst_dir - while not os.path.isdir(first_subdir): - parent = os.path.dirname(first_subdir) - if not os.path.isdir(parent): - first_subdir = parent - else: - break - os.makedirs(dst_dir) - await self._set_mode_and_owner(first_subdir, rule, task_id) - - self._log.info( - f"{task_id}: {rule.action} '{path}' to '{dst}'") - if rule.action == "copy": - if os.path.isdir(path): - shutil.copytree(path, dst) - else: - shutil.copy2(path, dst) - - else: - os.rename(path, dst) - - await self._set_mode_and_owner(dst, rule, task_id) - - elif rule.action == "delete": - self._log.info( - f"{task_id}: {rule.action} '{path}'") - if os.path.isdir(path): - if rule.rec: - shutil.rmtree(path) - else: - shutil.rmdir(path) - - else: - os.remove(path) - - except RuntimeError as e: - self._log.error(f"{task_id}: {e}") - - except Exception as e: - self._log.exception(f"{task_id}: {e}") - - -class PyinotifydConfig: - def __init__(self, watches=[], loglevel=logging.INFO, shutdown_timeout=30): - if not isinstance(watches, list): - watches = [watches] - - self.set_watches(watches) - - assert isinstance(loglevel, int), \ - f"loglevel: expected {type(int)}, got {type(loglevel)}" - self.loglevel = loglevel - - assert isinstance(shutdown_timeout, int), \ - f"shutdown_timeout: expected {type(int)}, " \ - f"got {type(shutdown_timeout)}" - self.shutdown_timeout = shutdown_timeout - - def add_watch(self, *args, **kwargs): - self.watches.append(Watch(*args, **kwargs)) - - def set_watches(self, watches): - self.watches = [] - for watch in watches: - assert isinstance(watch, Watch), \ - f"watches: expected {type(Watch)}, got {type(watch)}" - - -async def shutdown(signame, notifiers, logname, timeout=30): - log = logging.getLogger(logname) - - log.info(f"got signal {signame}, shutdown ...") - for notifier in notifiers: - notifier.stop() - - pending = [t for t in asyncio.all_tasks() - if t is not asyncio.current_task()] - if len(pending) > 0: - log.info( - f"graceful shutdown, waiting {timeout}s " - f"for remaining tasks to complete") - try: - future = asyncio.gather(*pending) - await asyncio.wait_for(future, timeout) - except asyncio.TimeoutError: - log.warning( - "forcefully terminate remaining tasks") - future.cancel() - future.exception() - - log.info("shutdown") - asyncio.get_event_loop().stop() - - -def main(): - myname = "pyinotifyd" - - parser = argparse.ArgumentParser( - description=myname, - formatter_class=lambda prog: argparse.HelpFormatter( - prog, max_help_position=45, width=140)) - parser.add_argument( - "-c", - "--config", - help=f"path to config file (defaults to /etc/{myname}/config.py)", - default=f"/etc/{myname}/config.py") - parser.add_argument( - "-d", - "--debug", - help="log debugging messages", - action="store_true") - parser.add_argument( - "-e", - "--events", - help="show event types and exit", - action="store_true") - parser.add_argument( - "-v", - "--version", - help="show version and exit", - action="store_true") - args = parser.parse_args() - - if args.version: - print(f"{myname} ({__version__})") - sys.exit(0) - elif args.events: - types = "\n".join(EventMap.flags.keys()) - print(types) - sys.exit(0) - - log = logging.getLogger(myname) - - try: - cfg = {} - with open(args.config, "r") as c: - exec(c.read(), globals(), cfg) - cfg = cfg[f"{myname}_config"] - assert isinstance(cfg, PyinotifydConfig), \ - f"{myname}_config: expected {type(PyinotifydConfig)}, " \ - f"got {type(cfg)}" - except Exception as e: - log.exception(f"error in config file: {e}") - sys.exit(1) - - console = logging.StreamHandler() - formatter = logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s - %(message)s") - console.setFormatter(formatter) - - if args.debug: - loglevel = logging.DEBUG - else: - loglevel = cfg.loglevel - - root_logger = logging.getLogger() - root_logger.setLevel(loglevel) - root_logger.addHandler(console) - - wm = pyinotify.WatchManager() - loop = asyncio.get_event_loop() - notifiers = [] - for watch in cfg.watches: - log.info(f"start watching '{watch.path}'") - notifiers.append(watch.event_notifier(wm, loop)) - - for signame in ["SIGINT", "SIGTERM"]: - loop.add_signal_handler( - getattr(signal, signame), - lambda: asyncio.ensure_future( - shutdown( - signame, notifiers, myname, timeout=cfg.shutdown_timeout))) - - loop.run_forever() - loop.close() - - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/pyinotifyd/__init__.py b/pyinotifyd/__init__.py new file mode 100755 index 0000000..67b94ab --- /dev/null +++ b/pyinotifyd/__init__.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 + +# pyinotifyd is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyinotifyd is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyinotifyd. If not, see . +# + +__all__ = [ + "Pyinotifyd", + "filemanager", + "scheduler", + "watch"] + +import argparse +import asyncio +import logging +import logging.handlers +import pyinotify +import signal +import sys + +from pyinotifyd.version import __version__ as version +from pyinotifyd.watch import Watch, EventMap + + +class Pyinotifyd: + def __init__(self, watches=[], shutdown_timeout=30, logname=None): + self.set_watches(watches) + self.set_shutdown_timeout(shutdown_timeout) + logname = (logname or "daemon") + self._log = logging.getLogger(logname) + self._loop = asyncio.get_event_loop() + self._notifiers = [] + self._wm = pyinotify.WatchManager() + + def set_watches(self, watches): + if not isinstance(watches, list): + watches = [watches] + + for watch in watches: + assert isinstance(watch, Watch), \ + f"watches: expected {type(Watch)}, got {type(watch)}" + + self._watches = watches + + def add_watch(self, *args, **kwargs): + self._watches.append(Watch(*args, **kwargs)) + + def set_shutdown_timeout(self, timeout): + assert isinstance(timeout, int), \ + f"timeout: expected {type(int)}, " \ + f"got {type(timeout)}" + self._shutdown_timeout = timeout + + def start(self, loop=None): + assert len(self._watches) > 0, \ + "pyinotifyd: unable to start, no watches set" + if not loop: + loop = self._loop + + for watch in self._watches: + self._log.info(f"start watching '{watch.path}' for inotify events") + self._notifiers.append(watch.event_notifier(self._wm, loop)) + + def stop(self): + self._log.info("stop watching for inotify events") + for notifier in self._notifiers: + notifier.stop() + + self._notifiers = [] + return self._shutdown_timeout + + +async def _shutdown(signame, daemon, log): + log.info(f"got signal {signame}, graceful shutdown") + timeout = daemon.stop() + pending = [t for t in asyncio.all_tasks() + if t is not asyncio.current_task()] + if len(pending) > 0: + log.info( + f"waiting {timeout}s for remaining tasks to complete") + try: + future = asyncio.gather(*pending) + await asyncio.wait_for(future, timeout) + except asyncio.TimeoutError: + log.warning("forcefully terminate remaining tasks") + future.cancel() + future.exception() + + log.info("shutdown complete") + asyncio.get_event_loop().stop() + + +def main(): + myname = "pyinotifyd" + + parser = argparse.ArgumentParser( + description=myname, + formatter_class=lambda prog: argparse.HelpFormatter( + prog, max_help_position=45, width=140)) + parser.add_argument( + "-c", + "--config", + help=f"path to config file (defaults to /etc/{myname}/config.py)", + default=f"/etc/{myname}/config.py") + parser.add_argument( + "-d", + "--debug", + help="log debugging messages", + action="store_true") + parser.add_argument( + "-e", + "--events", + help="show event types and exit", + action="store_true") + parser.add_argument( + "-v", + "--version", + help="show version and exit", + action="store_true") + args = parser.parse_args() + + if args.version: + print(f"{myname} ({version})") + sys.exit(0) + elif args.events: + types = "\n".join(EventMap.flags.keys()) + print(types) + sys.exit(0) + + if args.debug: + loglevel = logging.DEBUG + else: + loglevel = logging.INFO + + root_logger = logging.getLogger() + root_logger.setLevel(loglevel) + + ch = logging.StreamHandler() + formatter = logging.Formatter("%(levelname)s - %(message)s") + ch.setFormatter(formatter) + root_logger.addHandler(ch) + + try: + config = {} + exec(f"from {myname}.scheduler import *", config) + exec(f"from {myname}.filemanager import *", config) + with open(args.config, "r") as c: + exec(c.read(), globals(), config) + daemon = config[f"{myname}"] + assert isinstance(daemon, Pyinotifyd), \ + f"{myname}: expected {type(Pyinotifyd)}, " \ + f"got {type(daemon)}" + except Exception as e: + logging.exception(f"config file '{args.config}': {e}") + sys.exit(1) + + if args.debug: + root_logger.setLevel(loglevel) + formatter = logging.Formatter( + f"%(asctime)s - {myname}/%(name)s - %(levelname)s - %(message)s") + ch.setFormatter(formatter) + + log = logging.getLogger(myname) + loop = asyncio.get_event_loop() + for signame in ["SIGINT", "SIGTERM"]: + loop.add_signal_handler( + getattr(signal, signame), + lambda: asyncio.ensure_future( + _shutdown(signame, daemon, log))) + + daemon.start() + loop.run_forever() + loop.close() + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/docs/config.py b/pyinotifyd/docs/config.py.example similarity index 89% rename from docs/config.py rename to pyinotifyd/docs/config.py.example index 0cd4c29..dc2364d 100644 --- a/docs/config.py +++ b/pyinotifyd/docs/config.py.example @@ -74,11 +74,10 @@ # auto_add=True) -############################### -# Example pyinotifyd config # -############################### +######################## +# Example pyinotifyd # +######################## -#pyinotifyd_config = PyinotifydConfig( +#pyinotifyd_config = Pyinotifyd( # watches=[watch], -# loglevel=logging.INFO, -# shutdown_timeout=30) +# loglevel=logging.INFO) diff --git a/pyinotifyd/filemanager.py b/pyinotifyd/filemanager.py new file mode 100755 index 0000000..43ef988 --- /dev/null +++ b/pyinotifyd/filemanager.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 + +# pyinotifyd is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyinotifyd is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyinotifyd. If not, see . +# + +import logging +import os +import re +import shutil + + +class Rule: + valid_actions = ["copy", "move", "delete"] + + def __init__(self, action, src_re, dst_re="", auto_create=False, + dirmode=None, filemode=None, user=None, group=None, + rec=False): + assert action in self.valid_actions, \ + f"action: expected [{Rule.valid_actions.join(', ')}], got{action}" + self.action = action + + self.src_re = re.compile(src_re) + + assert isinstance(dst_re, str), \ + f"dst_re: expected {type('')}, got {type(dst_re)}" + self.dst_re = dst_re + + assert isinstance(auto_create, bool), \ + f"auto_create: expected {type(bool)}, got {type(auto_create)}" + self.auto_create = auto_create + + if dirmode is not None: + assert isinstance(dirmode, int), \ + f"dirmode: expected {type(int)}, got {type(dirmode)}" + self.dirmode = dirmode + + if filemode is not None: + assert isinstance(filemode, int), \ + f"filemode: expected {type(int)}, got {type(filemode)}" + self.filemode = filemode + + if user is not None: + assert isinstance(user, str), \ + f"user: expected {type('')}, got {type(user)}" + self.user = user + + if group is not None: + assert isinstance(group, str), \ + f"group: expected {type('')}, got {type(group)}" + self.group = group + + assert isinstance(rec, bool), \ + f"rec: expected {type(bool)}, got {type(rec)}" + self.rec = rec + + +class FileManager: + def __init__(self, rules, logname=None): + if not isinstance(rules, list): + rules = [rules] + + for rule in rules: + assert isinstance(rule, Rule), \ + f"rules: expected {type(Rule)}, got {type(rule)}" + + self._rules = rules + self._log = logging.getLogger((logname or __name__)) + + def add_rule(self, *args, **kwargs): + self._rules.append(Rule(*args, **kwargs)) + + async def _chmod_and_chown(self, path, mode, chown, task_id): + if mode is not None: + self._log.debug(f"{task_id}: chmod {oct(mode)} '{path}'") + os.chmod(path, mode) + + if chown is not None: + changes = "" + if chown[0] is not None: + changes = chown[0] + + if chown[1] is not None: + changes = f"{changes}:{chown[1]}" + + self._log.debug(f"{task_id}: chown {changes} '{path}'") + shutil.chown(path, *chown) + + async def _set_mode_and_owner(self, path, rule, task_id): + if (rule.user is rule.group is None): + chown = None + else: + chown = (rule.user, rule.group) + + work_on_dirs = not (rule.dirmode is chown is None) + work_on_files = not (rule.filemode is chown is None) + + if os.path.isdir(path): + await self._chmod_and_chown(path, rule.dirmode, chown, task_id) + if work_on_dirs or work_on_files: + for root, dirs, files in os.walk(path): + if work_on_dirs: + for p in [os.path.join(root, d) for d in dirs]: + await self._chmod_and_chown( + p, rule.dirmode, chown, task_id) + + if work_on_files: + for p in [os.path.join(root, f) for f in files]: + await self._chmod_and_chown( + p, rule.filemode, chown, task_id) + else: + await self._chmod_and_chown(path, rule.filemode, chown, task_id) + + async def task(self, event, task_id): + path = event.pathname + match = None + for rule in self._rules: + match = rule.src_re.match(path) + if match: + break + + if not match: + self._log.debug( + f"{task_id}: path '{path}' matches no rule in ruleset") + return + + try: + if rule.action in ["copy", "move"]: + dst = rule.src_re.sub(rule.dst_re, path) + if not dst: + raise RuntimeError( + f"{task_id}: unable to {rule.action} '{path}', " + f"resulting destination path is empty") + + if os.path.exists(dst): + raise RuntimeError( + f"{task_id}: unable to move file from '{path} " + f"to '{dst}', dstination path exists already") + + dst_dir = os.path.dirname(dst) + if not os.path.isdir(dst_dir) and rule.auto_create: + self._log.info( + f"{task_id}: create directory '{dst_dir}'") + first_subdir = dst_dir + while not os.path.isdir(first_subdir): + parent = os.path.dirname(first_subdir) + if not os.path.isdir(parent): + first_subdir = parent + else: + break + os.makedirs(dst_dir) + await self._set_mode_and_owner(first_subdir, rule, task_id) + + self._log.info( + f"{task_id}: {rule.action} '{path}' to '{dst}'") + if rule.action == "copy": + if os.path.isdir(path): + shutil.copytree(path, dst) + else: + shutil.copy2(path, dst) + + else: + os.rename(path, dst) + + await self._set_mode_and_owner(dst, rule, task_id) + + elif rule.action == "delete": + self._log.info( + f"{task_id}: {rule.action} '{path}'") + if os.path.isdir(path): + if rule.rec: + shutil.rmtree(path) + else: + shutil.rmdir(path) + + else: + os.remove(path) + + except RuntimeError as e: + self._log.error(f"{task_id}: {e}") + + except Exception as e: + self._log.exception(f"{task_id}: {e}") diff --git a/misc/pyinotifyd.service b/pyinotifyd/misc/pyinotifyd.service similarity index 100% rename from misc/pyinotifyd.service rename to pyinotifyd/misc/pyinotifyd.service diff --git a/pyinotifyd/scheduler.py b/pyinotifyd/scheduler.py new file mode 100755 index 0000000..347116a --- /dev/null +++ b/pyinotifyd/scheduler.py @@ -0,0 +1,152 @@ +# pyinotifyd is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyinotifyd is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyinotifyd. If not, see . +# + +import asyncio +import logging +from shlex import quote as shell_quote +from uuid import uuid4 + + +class _Task: + def __init__(self, event, delay, task_id, task, callback=None, + logname=None): + self._event = event + self._path = event.pathname + self._delay = delay + self._task_id = task_id + self._job = task + self._callback = callback + + self._task = None + self._log = logging.getLogger((logname or __name__)) + + async def _start(self): + if self._delay > 0: + await asyncio.sleep(self._delay) + + if self._callback is not None: + self._callback(self._event) + + self._task = None + + self._log.info(f"execute task {self._task_id}") + await asyncio.shield(self._job(self._event, self._task_id)) + self._log.info(f"task {self._task_id} finished") + + def start(self): + if self._task is None: + self._task = asyncio.create_task(self._start()) + + def cancel(self): + if self._task is not None: + self._task.cancel() + self._task = None + + def restart(self): + self.cancel() + self.start() + + +class TaskScheduler: + def __init__(self, task, files, dirs, delay=0, logname=None): + assert callable(task), \ + f"task: expected callable, got {type(task)}" + self._task = task + + assert isinstance(delay, int), \ + f"delay: expected {type(int)}, got {type(delay)}" + self._delay = delay + + assert isinstance(files, bool), \ + f"files: expected {type(bool)}, got {type(files)}" + self._files = files + + assert isinstance(dirs, bool), \ + f"dirs: expected {type(bool)}, got {type(dirs)}" + self._dirs = dirs + + self._tasks = {} + self._logname = (logname or __name__) + self._log = logging.getLogger(self._logname) + + def _task_started(self, event): + path = event.pathname + if path in self._tasks: + del self._tasks[path] + + def schedule(self, event): + self._log.debug(f"received {event}") + + if (not event.dir and not self._files) or \ + (event.dir and not self._dirs): + return + + path = event.pathname + maskname = event.maskname.split("|", 1)[0] + + if path in self._tasks: + task = self._tasks[path] + self._log.info( + f"received event {maskname} on '{path}', " + f"re-schedule task {task.task_id} (delay={self._delay}s)") + task.restart() + else: + task_id = str(uuid4()) + self._log.info( + f"received event {maskname} on '{path}', " + f"schedule task {task_id} (delay={self._delay}s)") + task = _Task( + event, self._delay, task_id, self._task, + callback=self._task_started, logname=self._logname) + self._tasks[path] = task + task.start() + + def cancel(self, event): + self._log.debug(f"received {event}") + + path = event.pathname + maskname = event.maskname.split("|", 1)[0] + if path in self._tasks: + task = self._tasks[path] + self._log.info( + f"received event {maskname} on '{path}', " + f"cancel scheduled task {task.task_id}") + task.cancel() + del self._tasks[path] + + +class ShellScheduler(TaskScheduler): + def __init__(self, cmd, task=None, logname=None, *args, **kwargs): + assert isinstance(cmd, str), \ + f"cmd: expected {type('')}, got {type(cmd)}" + self._cmd = cmd + + logname = (logname or __name__) + super().__init__(*args, task=self.task, logname=logname, **kwargs) + + async def task(self, event, task_id): + maskname = event.maskname.split("|", 1)[0] + + if hasattr(event, "src_pathname"): + src_pathname = event.src_pathname + else: + src_pathname = "" + + cmd = self._cmd.replace("{maskname}", shell_quote(maskname)).replace( + "{pathname}", shell_quote(event.pathname)).replace( + "{src_pathname}", shell_quote(src_pathname)) + + self._log.info(f"{task_id}: execute shell command: {cmd}") + proc = await asyncio.create_subprocess_shell(cmd) + await proc.communicate() diff --git a/pyinotifyd/version.py b/pyinotifyd/version.py new file mode 100755 index 0000000..f102a9c --- /dev/null +++ b/pyinotifyd/version.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/pyinotifyd/watch.py b/pyinotifyd/watch.py new file mode 100755 index 0000000..ff99149 --- /dev/null +++ b/pyinotifyd/watch.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +# pyinotifyd is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyinotifyd is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyinotifyd. If not, see . +# + +import asyncio +import pyinotify + + +class EventMap: + flags = { + **pyinotify.EventsCodes.OP_FLAGS, + **pyinotify.EventsCodes.EVENT_FLAGS} + + def __init__(self, event_map=None, default_task=None): + self._map = {} + + if default_task is not None: + assert callable(default_task), \ + f"default_task: expected callable, got {type(default_task)}" + for flag in EventMap.flags: + self.set(flag, default_task) + + if event_map is not None: + assert isinstance(event_map, dict), \ + f"event_map: expected {type(dict)}, got {type(event_map)}" + for flag, task in event_map.items(): + self.set(flag, task) + + def items(self): + return self._map.items() + + def set(self, flag, values): + assert flag in EventMap.flags, \ + f"event_map: invalid flag: {flag}" + if values is not None: + if not isinstance(values, list): + values = [values] + + for value in values: + assert callable(value), \ + f"event_map: {flag}: expected callable, got {type(value)}" + + self._map[flag] = values + elif flag in self._map: + del self._map[flag] + + +class _TaskList: + def __init__(self, tasks=[]): + if not isinstance(tasks, list): + tasks = [tasks] + + self._tasks = tasks + + def add(self, task): + self._tasks.append(task) + + def remove(self, task): + self._tasks.remove(task) + + def execute(self, event): + for task in self._tasks: + task(event) + + +class Watch: + def __init__(self, path, event_map, rec=False, auto_add=False): + assert isinstance(path, str), \ + f"path: expected {type('')}, got {type(path)}" + self.path = path + + if isinstance(event_map, EventMap): + self.event_map = event_map + elif isinstance(event_map, dict): + self.event_map = EventMap(event_map) + else: + raise AssertionError( + f"event_map: expected {type(EventMap)} or {type(dict)}, " + f"got {type(event_map)}") + + assert isinstance(rec, bool), \ + f"rec: expected {type(bool)}, got {type(rec)}" + self.rec = rec + + assert isinstance(auto_add, bool), \ + f"auto_add: expected {type(bool)}, got {type(auto_add)}" + self.auto_add = auto_add + + def event_notifier(self, wm, loop=asyncio.get_event_loop()): + handler = pyinotify.ProcessEvent() + mask = False + for flag, values in self.event_map.items(): + setattr(handler, f"process_{flag}", _TaskList(values).execute) + if mask: + mask = mask | EventMap.flags[flag] + else: + mask = EventMap.flags[flag] + + wm.add_watch( + self.path, mask, rec=self.rec, auto_add=self.auto_add, + do_glob=True) + + return pyinotify.AsyncioNotifier(wm, loop, default_proc_fun=handler) diff --git a/setup.cfg b/setup.cfg index 3401800..19b5cef 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ [metadata] -version = attr: pyinotifyd.__version__ +version = attr: pyinotifyd.version.__version__ diff --git a/setup.py b/setup.py index 26477b3..356fa41 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup(name = "pyinotifyd", license = "GPL 3", keywords = "inotify daemon", url = "https://github.com/spacefreak86/pyinotifyd", - py_modules = ["pyinotifyd"], + packages = ["pyinotifyd"], long_description = read_file("README.md"), long_description_content_type = "text/markdown", classifiers = [ @@ -31,10 +31,6 @@ setup(name = "pyinotifyd", "pyinotifyd=pyinotifyd:main" ] }, - data_files = [ - ("/etc/pyinotifyd", ["docs/config.py"]), - ("/usr/lib/systemd/system", ["misc/pyinotifyd.service"]) - ], install_requires = ["pyinotify"], python_requires = ">=3.7" ) diff --git a/test-pyinotifyd b/test-pyinotifyd new file mode 100755 index 0000000..d2d5eb4 --- /dev/null +++ b/test-pyinotifyd @@ -0,0 +1,9 @@ +#!/usr/bin/env python + +import sys +import pyinotifyd + +if __name__ == '__main__': + sys.exit( + pyinotifyd.main() + )