spacefreak/pyinotifyd
1
0
Fork 0
Code Issues Pull Requests Releases Activity

make it a package and improve logging

Browse Source
This commit is contained in:
Thomas Oettli
2020-11-04 23:34:34 +01:00
parent 6feaf9d2b2
commit ad97547568
13 changed files with 709 additions and 600 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
MANIFEST.in
View File

@@ -1,3 +1,3 @@
include LICENSE README.md include LICENSE README.md
recursive-include docs * recursive-include pyinotifyd/docs *
recursive-include misc * recursive-include pyinotifyd/misc *

51
README.md
View File

@@ -29,7 +29,7 @@ A rule holds an *action* (move, copy or delete) and a regular expression *src_re
If the action is copy or move, the destination path *dst_re* is mandatory and if *action* is delete and *rec* is set to True, non-empty directories will be deleted recursively. If the action is copy or move, the destination path *dst_re* is mandatory and if *action* is delete and *rec* is set to True, non-empty directories will be deleted recursively.
With *auto_create* set to True, possibly missing subdirectories in *dst_re* are created automatically. Regex subgroups or named-subgroups may be used in *src_re* and *dst_re*. With *auto_create* set to True, possibly missing subdirectories in *dst_re* are created automatically. Regex subgroups or named-subgroups may be used in *src_re* and *dst_re*.
Set the mode of moved/copied files/directories with *filemode* and *dirmode*. Ownership of moved/copied files/directories is set with *user* and *group*. Mode and ownership is also set to automatically created subdirectories. Set the mode of moved/copied files/directories with *filemode* and *dirmode*. Ownership of moved/copied files/directories is set with *user* and *group*. Mode and ownership is also set to automatically created subdirectories.
Use *logname* in log messages. Log messages with *logname*.
```python ```python
rule = Rule( rule = Rule(
action="move", action="move",
@@ -53,7 +53,7 @@ pyinotifyd has different schedulers to schedule tasks with an optional delay. Th
#### TaskScheduler #### TaskScheduler
TaskScheduler schedules *task* with an optional *delay* in seconds. Use the *files* and *dirs* arguments to schedule tasks only for files and/or directories. TaskScheduler schedules *task* with an optional *delay* in seconds. Use the *files* and *dirs* arguments to schedule tasks only for files and/or directories.
Use *logname* in log messages. All arguments except for *task* are optional. Log messages with *logname*. All arguments except for *task* are optional.
```python ```python
s = TaskScheduler( s = TaskScheduler(
task=task, task=task,
@@ -109,13 +109,47 @@ watch = Watch(
auto_add=False) auto_add=False)
``` ```
### PyinotifydConfig ### Pyinotifyd
pyinotifyd expects an instance of PyinotifydConfig named **pyinotifyd_config** that holds its config options. The options are a list of *watches*, the *loglevel* (see https://docs.python.org/3/library/logging.html#levels) and the *shutdown_timeout*. pyinotifyd will wait *shutdown_timeout* seconds for pending tasks to complete during shutdown. pyinotifyd expects an instance of Pyinotifyd named **pyinotifyd** defined in the config file. The options are a list of *watches* and the *shutdown_timeout*. pyinotifyd will wait *shutdown_timeout* seconds for pending tasks to complete during shutdown. Log messages with *logname*.
```python ```python
pyinotifyd_config = PyinotifydConfig( pyinotifyd = Pyinotifyd(
watches=[watch], watches=[watch],
loglevel=logging.INFO, shutdown_timeout=30,
shutdown_timeout=30) logname="Pyinotifyd")
```
### Logging
Pythons [logging](https://docs.python.org/3/howto/logging.html) framework is used to log messages (see https://docs.python.org/3/howto/logging.html).
Configure the global loglevel. This is the default:
```python
logging.getLogger().setLevel(logging.WARNING)
```
It is possible to configure the loglevel per log name. This is an example for logname **TaskScheduler**:
```python
logging.getLogger("TaskScheduler").setLevel(logging.INFO)
```
#### Syslog
Add this to your config file to send log messages to a local syslog server.
```python
# send log messages to the Unix socket of the syslog server.
syslog = logging.handlers.SysLogHandler(
address="/dev/log")
# set the log format of syslog messages
log_format = "pyinotifyd/%(name)s: %(message)s"
syslog.setFormatter(
logging.Formatter(formatter)
# set the log level for syslog messages
syslog.setLevel(logging.INFO)
# enable syslog
logging.getLogger().addHandler(syslog)
# or enable syslog just for TaskScheduler
logging.getLogger("TaskManager").addHandler(syslog)
``` ```
### Autostart ### Autostart
@@ -151,7 +185,6 @@ watch = Watch(
pyinotifyd_config = PyinotifydConfig( pyinotifyd_config = PyinotifydConfig(
watches=[watch], watches=[watch],
loglevel=logging.INFO,
shutdown_timeout=5) shutdown_timeout=5)
``` ```
@@ -173,7 +206,6 @@ watch = Watch(
pyinotifyd_config = PyinotifydConfig( pyinotifyd_config = PyinotifydConfig(
watches=[watch], watches=[watch],
loglevel=logging.INFO,
shutdown_timeout=5) shutdown_timeout=5)
``` ```
@@ -227,6 +259,5 @@ watch = Watch(
# otherwise pending tasks may get cancelled during shutdown. # otherwise pending tasks may get cancelled during shutdown.
pyinotifyd_config = PyinotifydConfig( pyinotifyd_config = PyinotifydConfig(
watches=[watch], watches=[watch],
loglevel=logging.INFO,
shutdown_timeout=35) shutdown_timeout=35)
``` ```

576
pyinotifyd.py
View File

@@ -1,576 +0,0 @@
#!/usr/bin/env python3
# pyinotifyd is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# pyinotifyd is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with pyinotifyd. If not, see <http://www.gnu.org/licenses/>.
#
import argparse
import asyncio
import logging
import logging.handlers
import os
import pyinotify
import re
import shutil
import signal
import sys
from shlex import quote as shell_quote
from uuid import uuid4
__version__ = "0.0.1"
class Task:
def __init__(self, event, delay, task_id, task, callback=None,
logname=None):
self._event = event
self._path = event.pathname
self._delay = delay
self._task_id = task_id
self._job = task
self._callback = callback
self._task = None
self._log = logging.getLogger((logname or self.__class__.__name__))
async def _start(self):
if self._delay > 0:
await asyncio.sleep(self._delay)
if self._callback is not None:
self._callback(self._event)
self._task = None
self._log.info(f"execute task {self._task_id}")
await asyncio.shield(self._job(self._event, self._task_id))
self._log.info(f"task {self._task_id} finished")
def start(self):
if self._task is None:
self._task = asyncio.create_task(self._start())
def cancel(self):
if self._task is not None:
self._task.cancel()
self._task = None
def restart(self):
self.cancel()
self.start()
class TaskList:
def __init__(self, tasks=[]):
if not isinstance(tasks, list):
tasks = [tasks]
self._tasks = tasks
def add(self, task):
self._tasks.append(task)
def remove(self, task):
self._tasks.remove(task)
def execute(self, event):
for task in self._tasks:
task(event)
class TaskScheduler:
def __init__(self, task, files, dirs, delay=0, logname=None):
assert callable(task), \
f"task: expected callable, got {type(task)}"
self._task = task
assert isinstance(delay, int), \
f"delay: expected {type(int)}, got {type(delay)}"
self._delay = delay
assert isinstance(files, bool), \
f"files: expected {type(bool)}, got {type(files)}"
self._files = files
assert isinstance(dirs, bool), \
f"dirs: expected {type(bool)}, got {type(dirs)}"
self._dirs = dirs
self._tasks = {}
self._logname = (logname or self.__class__.__name__)
self._log = logging.getLogger(self._logname)
def _task_started(self, event):
path = event.pathname
if path in self._tasks:
del self._tasks[path]
def schedule(self, event):
self._log.debug(f"received {event}")
if (not event.dir and not self._files) or \
(event.dir and not self._dirs):
return
path = event.pathname
maskname = event.maskname.split("|", 1)[0]
if path in self._tasks:
task = self._tasks[path]
self._log.info(
f"received event {maskname} on '{path}', "
f"re-schedule task {task.task_id} (delay={self._delay}s)")
task.restart()
else:
task_id = str(uuid4())
self._log.info(
f"received event {maskname} on '{path}', "
f"schedule task {task_id} (delay={self._delay}s)")
task = Task(
event, self._delay, task_id, self._task,
callback=self._task_started, logname=self._logname)
self._tasks[path] = task
task.start()
def cancel(self, event):
self._log.debug(f"received {event}")
path = event.pathname
maskname = event.maskname.split("|", 1)[0]
if path in self._tasks:
task = self._tasks[path]
self._log.info(
f"received event {maskname} on '{path}', "
f"cancel scheduled task {task.task_id}")
task.cancel()
del self._tasks[path]
class ShellScheduler(TaskScheduler):
def __init__(self, cmd, task=None, logname=None, *args, **kwargs):
assert isinstance(cmd, str), \
f"cmd: expected {type('')}, got {type(cmd)}"
self._cmd = cmd
logname = (logname or self.__class__.__name__)
super().__init__(*args, task=self.task, logname=logname, **kwargs)
async def task(self, event, task_id):
maskname = event.maskname.split("|", 1)[0]
if hasattr(event, "src_pathname"):
src_pathname = event.src_pathname
else:
src_pathname = ""
cmd = self._cmd.replace("{maskname}", shell_quote(maskname)).replace(
"{pathname}", shell_quote(event.pathname)).replace(
"{src_pathname}", shell_quote(src_pathname))
self._log.info(f"{task_id}: execute shell command: {cmd}")
proc = await asyncio.create_subprocess_shell(cmd)
await proc.communicate()
class EventMap:
flags = {
**pyinotify.EventsCodes.OP_FLAGS,
**pyinotify.EventsCodes.EVENT_FLAGS}
def __init__(self, event_map=None, default_task=None):
self._map = {}
if default_task is not None:
assert callable(default_task), \
f"default_task: expected callable, got {type(default_task)}"
for flag in EventMap.flags:
self.set(flag, default_task)
if event_map is not None:
assert isinstance(event_map, dict), \
f"event_map: expected {type(dict)}, got {type(event_map)}"
for flag, task in event_map.items():
self.set(flag, task)
def items(self):
return self._map.items()
def set(self, flag, values):
assert flag in EventMap.flags, \
f"event_map: invalid flag: {flag}"
if values is not None:
if not isinstance(values, list):
values = [values]
for value in values:
assert callable(value), \
f"event_map: {flag}: expected callable, got {type(value)}"
self._map[flag] = values
elif flag in self._map:
del self._map[flag]
class Watch:
def __init__(self, path, event_map, rec=False, auto_add=False):
assert isinstance(path, str), \
f"path: expected {type('')}, got {type(path)}"
self.path = path
if isinstance(event_map, EventMap):
self.event_map = event_map
elif isinstance(event_map, dict):
self.event_map = EventMap(event_map)
else:
raise AssertionError(
f"event_map: expected {type(EventMap)} or {type(dict)}, "
f"got {type(event_map)}")
assert isinstance(rec, bool), \
f"rec: expected {type(bool)}, got {type(rec)}"
self.rec = rec
assert isinstance(auto_add, bool), \
f"auto_add: expected {type(bool)}, got {type(auto_add)}"
self.auto_add = auto_add
def event_notifier(self, wm, loop):
handler = pyinotify.ProcessEvent()
mask = False
for flag, values in self.event_map.items():
setattr(handler, f"process_{flag}", TaskList(values).execute)
if mask:
mask = mask | EventMap.flags[flag]
else:
mask = EventMap.flags[flag]
wm.add_watch(
self.path, mask, rec=self.rec, auto_add=self.auto_add,
do_glob=True)
return pyinotify.AsyncioNotifier(wm, loop, default_proc_fun=handler)
class Rule:
valid_actions = ["copy", "move", "delete"]
def __init__(self, action, src_re, dst_re="", auto_create=False,
dirmode=None, filemode=None, user=None, group=None,
rec=False):
assert action in self.valid_actions, \
f"action: expected [{Rule.valid_actions.join(', ')}], got{action}"
self.action = action
self.src_re = re.compile(src_re)
assert isinstance(dst_re, str), \
f"dst_re: expected {type('')}, got {type(dst_re)}"
self.dst_re = dst_re
assert isinstance(auto_create, bool), \
f"auto_create: expected {type(bool)}, got {type(auto_create)}"
self.auto_create = auto_create
if dirmode is not None:
assert isinstance(dirmode, int), \
f"dirmode: expected {type(int)}, got {type(dirmode)}"
self.dirmode = dirmode
if filemode is not None:
assert isinstance(filemode, int), \
f"filemode: expected {type(int)}, got {type(filemode)}"
self.filemode = filemode
if user is not None:
assert isinstance(user, str), \
f"user: expected {type('')}, got {type(user)}"
self.user = user
if group is not None:
assert isinstance(group, str), \
f"group: expected {type('')}, got {type(group)}"
self.group = group
assert isinstance(rec, bool), \
f"rec: expected {type(bool)}, got {type(rec)}"
self.rec = rec
class FileManager:
def __init__(self, rules, logname=None):
if not isinstance(rules, list):
rules = [rules]
for rule in rules:
assert isinstance(rule, Rule), \
f"rules: expected {type(Rule)}, got {type(rule)}"
self._rules = rules
self._log = logging.getLogger((logname or self.__class__.__name__))
def add_rule(self, *args, **kwargs):
self._rules.append(Rule(*args, **kwargs))
async def _chmod_and_chown(self, path, mode, chown, task_id):
if mode is not None:
self._log.debug(f"{task_id}: chmod {oct(mode)} '{path}'")
os.chmod(path, mode)
if chown is not None:
changes = ""
if chown[0] is not None:
changes = chown[0]
if chown[1] is not None:
changes = f"{changes}:{chown[1]}"
self._log.debug(f"{task_id}: chown {changes} '{path}'")
shutil.chown(path, *chown)
async def _set_mode_and_owner(self, path, rule, task_id):
if (rule.user is rule.group is None):
chown = None
else:
chown = (rule.user, rule.group)
work_on_dirs = not (rule.dirmode is chown is None)
work_on_files = not (rule.filemode is chown is None)
if os.path.isdir(path):
await self._chmod_and_chown(path, rule.dirmode, chown, task_id)
if work_on_dirs or work_on_files:
for root, dirs, files in os.walk(path):
if work_on_dirs:
for p in [os.path.join(root, d) for d in dirs]:
await self._chmod_and_chown(
p, rule.dirmode, chown, task_id)
if work_on_files:
for p in [os.path.join(root, f) for f in files]:
await self._chmod_and_chown(
p, rule.filemode, chown, task_id)
else:
await self._chmod_and_chown(path, rule.filemode, chown, task_id)
async def task(self, event, task_id):
path = event.pathname
match = None
for rule in self._rules:
match = rule.src_re.match(path)
if match:
break
if not match:
self._log.debug(
f"{task_id}: path '{path}' matches no rule in ruleset")
return
try:
if rule.action in ["copy", "move"]:
dst = rule.src_re.sub(rule.dst_re, path)
if not dst:
raise RuntimeError(
f"{task_id}: unable to {rule.action} '{path}', "
f"resulting destination path is empty")
if os.path.exists(dst):
raise RuntimeError(
f"{task_id}: unable to move file from '{path} "
f"to '{dst}', dstination path exists already")
dst_dir = os.path.dirname(dst)
if not os.path.isdir(dst_dir) and rule.auto_create:
self._log.info(
f"{task_id}: create directory '{dst_dir}'")
first_subdir = dst_dir
while not os.path.isdir(first_subdir):
parent = os.path.dirname(first_subdir)
if not os.path.isdir(parent):
first_subdir = parent
else:
break
os.makedirs(dst_dir)
await self._set_mode_and_owner(first_subdir, rule, task_id)
self._log.info(
f"{task_id}: {rule.action} '{path}' to '{dst}'")
if rule.action == "copy":
if os.path.isdir(path):
shutil.copytree(path, dst)
else:
shutil.copy2(path, dst)
else:
os.rename(path, dst)
await self._set_mode_and_owner(dst, rule, task_id)
elif rule.action == "delete":
self._log.info(
f"{task_id}: {rule.action} '{path}'")
if os.path.isdir(path):
if rule.rec:
shutil.rmtree(path)
else:
shutil.rmdir(path)
else:
os.remove(path)
except RuntimeError as e:
self._log.error(f"{task_id}: {e}")
except Exception as e:
self._log.exception(f"{task_id}: {e}")
class PyinotifydConfig:
def __init__(self, watches=[], loglevel=logging.INFO, shutdown_timeout=30):
if not isinstance(watches, list):
watches = [watches]
self.set_watches(watches)
assert isinstance(loglevel, int), \
f"loglevel: expected {type(int)}, got {type(loglevel)}"
self.loglevel = loglevel
assert isinstance(shutdown_timeout, int), \
f"shutdown_timeout: expected {type(int)}, " \
f"got {type(shutdown_timeout)}"
self.shutdown_timeout = shutdown_timeout
def add_watch(self, *args, **kwargs):
self.watches.append(Watch(*args, **kwargs))
def set_watches(self, watches):
self.watches = []
for watch in watches:
assert isinstance(watch, Watch), \
f"watches: expected {type(Watch)}, got {type(watch)}"
async def shutdown(signame, notifiers, logname, timeout=30):
log = logging.getLogger(logname)
log.info(f"got signal {signame}, shutdown ...")
for notifier in notifiers:
notifier.stop()
pending = [t for t in asyncio.all_tasks()
if t is not asyncio.current_task()]
if len(pending) > 0:
log.info(
f"graceful shutdown, waiting {timeout}s "
f"for remaining tasks to complete")
try:
future = asyncio.gather(*pending)
await asyncio.wait_for(future, timeout)
except asyncio.TimeoutError:
log.warning(
"forcefully terminate remaining tasks")
future.cancel()
future.exception()
log.info("shutdown")
asyncio.get_event_loop().stop()
def main():
myname = "pyinotifyd"
parser = argparse.ArgumentParser(
description=myname,
formatter_class=lambda prog: argparse.HelpFormatter(
prog, max_help_position=45, width=140))
parser.add_argument(
"-c",
"--config",
help=f"path to config file (defaults to /etc/{myname}/config.py)",
default=f"/etc/{myname}/config.py")
parser.add_argument(
"-d",
"--debug",
help="log debugging messages",
action="store_true")
parser.add_argument(
"-e",
"--events",
help="show event types and exit",
action="store_true")
parser.add_argument(
"-v",
"--version",
help="show version and exit",
action="store_true")
args = parser.parse_args()
if args.version:
print(f"{myname} ({__version__})")
sys.exit(0)
elif args.events:
types = "\n".join(EventMap.flags.keys())
print(types)
sys.exit(0)
log = logging.getLogger(myname)
try:
cfg = {}
with open(args.config, "r") as c:
exec(c.read(), globals(), cfg)
cfg = cfg[f"{myname}_config"]
assert isinstance(cfg, PyinotifydConfig), \
f"{myname}_config: expected {type(PyinotifydConfig)}, " \
f"got {type(cfg)}"
except Exception as e:
log.exception(f"error in config file: {e}")
sys.exit(1)
console = logging.StreamHandler()
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s")
console.setFormatter(formatter)
if args.debug:
loglevel = logging.DEBUG
else:
loglevel = cfg.loglevel
root_logger = logging.getLogger()
root_logger.setLevel(loglevel)
root_logger.addHandler(console)
wm = pyinotify.WatchManager()
loop = asyncio.get_event_loop()
notifiers = []
for watch in cfg.watches:
log.info(f"start watching '{watch.path}'")
notifiers.append(watch.event_notifier(wm, loop))
for signame in ["SIGINT", "SIGTERM"]:
loop.add_signal_handler(
getattr(signal, signame),
lambda: asyncio.ensure_future(
shutdown(
signame, notifiers, myname, timeout=cfg.shutdown_timeout)))
loop.run_forever()
loop.close()
sys.exit(0)
if __name__ == "__main__":
main()

189
pyinotifyd/__init__.py Executable file
View File

@@ -0,0 +1,189 @@
#!/usr/bin/env python3
# pyinotifyd is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# pyinotifyd is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with pyinotifyd. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = [
"Pyinotifyd",
"filemanager",
"scheduler",
"watch"]
import argparse
import asyncio
import logging
import logging.handlers
import pyinotify
import signal
import sys
from pyinotifyd.version import __version__ as version
from pyinotifyd.watch import Watch, EventMap
class Pyinotifyd:
def __init__(self, watches=[], shutdown_timeout=30, logname=None):
self.set_watches(watches)
self.set_shutdown_timeout(shutdown_timeout)
logname = (logname or "daemon")
self._log = logging.getLogger(logname)
self._loop = asyncio.get_event_loop()
self._notifiers = []
self._wm = pyinotify.WatchManager()
def set_watches(self, watches):
if not isinstance(watches, list):
watches = [watches]
for watch in watches:
assert isinstance(watch, Watch), \
f"watches: expected {type(Watch)}, got {type(watch)}"
self._watches = watches
def add_watch(self, *args, **kwargs):
self._watches.append(Watch(*args, **kwargs))
def set_shutdown_timeout(self, timeout):
assert isinstance(timeout, int), \
f"timeout: expected {type(int)}, " \
f"got {type(timeout)}"
self._shutdown_timeout = timeout
def start(self, loop=None):
assert len(self._watches) > 0, \
"pyinotifyd: unable to start, no watches set"
if not loop:
loop = self._loop
for watch in self._watches:
self._log.info(f"start watching '{watch.path}' for inotify events")
self._notifiers.append(watch.event_notifier(self._wm, loop))
def stop(self):
self._log.info("stop watching for inotify events")
for notifier in self._notifiers:
notifier.stop()
self._notifiers = []
return self._shutdown_timeout
async def _shutdown(signame, daemon, log):
log.info(f"got signal {signame}, graceful shutdown")
timeout = daemon.stop()
pending = [t for t in asyncio.all_tasks()
if t is not asyncio.current_task()]
if len(pending) > 0:
log.info(
f"waiting {timeout}s for remaining tasks to complete")
try:
future = asyncio.gather(*pending)
await asyncio.wait_for(future, timeout)
except asyncio.TimeoutError:
log.warning("forcefully terminate remaining tasks")
future.cancel()
future.exception()
log.info("shutdown complete")
asyncio.get_event_loop().stop()
def main():
myname = "pyinotifyd"
parser = argparse.ArgumentParser(
description=myname,
formatter_class=lambda prog: argparse.HelpFormatter(
prog, max_help_position=45, width=140))
parser.add_argument(
"-c",
"--config",
help=f"path to config file (defaults to /etc/{myname}/config.py)",
default=f"/etc/{myname}/config.py")
parser.add_argument(
"-d",
"--debug",
help="log debugging messages",
action="store_true")
parser.add_argument(
"-e",
"--events",
help="show event types and exit",
action="store_true")
parser.add_argument(
"-v",
"--version",
help="show version and exit",
action="store_true")
args = parser.parse_args()
if args.version:
print(f"{myname} ({version})")
sys.exit(0)
elif args.events:
types = "\n".join(EventMap.flags.keys())
print(types)
sys.exit(0)
if args.debug:
loglevel = logging.DEBUG
else:
loglevel = logging.INFO
root_logger = logging.getLogger()
root_logger.setLevel(loglevel)
ch = logging.StreamHandler()
formatter = logging.Formatter("%(levelname)s - %(message)s")
ch.setFormatter(formatter)
root_logger.addHandler(ch)
try:
config = {}
exec(f"from {myname}.scheduler import *", config)
exec(f"from {myname}.filemanager import *", config)
with open(args.config, "r") as c:
exec(c.read(), globals(), config)
daemon = config[f"{myname}"]
assert isinstance(daemon, Pyinotifyd), \
f"{myname}: expected {type(Pyinotifyd)}, " \
f"got {type(daemon)}"
except Exception as e:
logging.exception(f"config file '{args.config}': {e}")
sys.exit(1)
if args.debug:
root_logger.setLevel(loglevel)
formatter = logging.Formatter(
f"%(asctime)s - {myname}/%(name)s - %(levelname)s - %(message)s")
ch.setFormatter(formatter)
log = logging.getLogger(myname)
loop = asyncio.get_event_loop()
for signame in ["SIGINT", "SIGTERM"]:
loop.add_signal_handler(
getattr(signal, signame),
lambda: asyncio.ensure_future(
_shutdown(signame, daemon, log)))
daemon.start()
loop.run_forever()
loop.close()
sys.exit(0)
if __name__ == "__main__":
main()

11
docs/config.py → pyinotifyd/docs/config.py.example
View File

@@ -74,11 +74,10 @@
# auto_add=True) # auto_add=True)
############################### ########################
# Example pyinotifyd config # # Example pyinotifyd #
############################### ########################
#pyinotifyd_config = PyinotifydConfig( #pyinotifyd_config = Pyinotifyd(
# watches=[watch], # watches=[watch],
# loglevel=logging.INFO, # loglevel=logging.INFO)
# shutdown_timeout=30)

193
pyinotifyd/filemanager.py Executable file
View File

@@ -0,0 +1,193 @@
#!/usr/bin/env python3
# pyinotifyd is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# pyinotifyd is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with pyinotifyd. If not, see <http://www.gnu.org/licenses/>.
#
import logging
import os
import re
import shutil
class Rule:
valid_actions = ["copy", "move", "delete"]
def __init__(self, action, src_re, dst_re="", auto_create=False,
dirmode=None, filemode=None, user=None, group=None,
rec=False):
assert action in self.valid_actions, \
f"action: expected [{Rule.valid_actions.join(', ')}], got{action}"
self.action = action
self.src_re = re.compile(src_re)
assert isinstance(dst_re, str), \
f"dst_re: expected {type('')}, got {type(dst_re)}"
self.dst_re = dst_re
assert isinstance(auto_create, bool), \
f"auto_create: expected {type(bool)}, got {type(auto_create)}"
self.auto_create = auto_create
if dirmode is not None:
assert isinstance(dirmode, int), \
f"dirmode: expected {type(int)}, got {type(dirmode)}"
self.dirmode = dirmode
if filemode is not None:
assert isinstance(filemode, int), \
f"filemode: expected {type(int)}, got {type(filemode)}"
self.filemode = filemode
if user is not None:
assert isinstance(user, str), \
f"user: expected {type('')}, got {type(user)}"
self.user = user
if group is not None:
assert isinstance(group, str), \
f"group: expected {type('')}, got {type(group)}"
self.group = group
assert isinstance(rec, bool), \
f"rec: expected {type(bool)}, got {type(rec)}"
self.rec = rec
class FileManager:
def __init__(self, rules, logname=None):
if not isinstance(rules, list):
rules = [rules]
for rule in rules:
assert isinstance(rule, Rule), \
f"rules: expected {type(Rule)}, got {type(rule)}"
self._rules = rules
self._log = logging.getLogger((logname or __name__))
def add_rule(self, *args, **kwargs):
self._rules.append(Rule(*args, **kwargs))
async def _chmod_and_chown(self, path, mode, chown, task_id):
if mode is not None:
self._log.debug(f"{task_id}: chmod {oct(mode)} '{path}'")
os.chmod(path, mode)
if chown is not None:
changes = ""
if chown[0] is not None:
changes = chown[0]
if chown[1] is not None:
changes = f"{changes}:{chown[1]}"
self._log.debug(f"{task_id}: chown {changes} '{path}'")
shutil.chown(path, *chown)
async def _set_mode_and_owner(self, path, rule, task_id):
if (rule.user is rule.group is None):
chown = None
else:
chown = (rule.user, rule.group)
work_on_dirs = not (rule.dirmode is chown is None)
work_on_files = not (rule.filemode is chown is None)
if os.path.isdir(path):
await self._chmod_and_chown(path, rule.dirmode, chown, task_id)
if work_on_dirs or work_on_files:
for root, dirs, files in os.walk(path):
if work_on_dirs:
for p in [os.path.join(root, d) for d in dirs]:
await self._chmod_and_chown(
p, rule.dirmode, chown, task_id)
if work_on_files:
for p in [os.path.join(root, f) for f in files]:
await self._chmod_and_chown(
p, rule.filemode, chown, task_id)
else:
await self._chmod_and_chown(path, rule.filemode, chown, task_id)
async def task(self, event, task_id):
path = event.pathname
match = None
for rule in self._rules:
match = rule.src_re.match(path)
if match:
break
if not match:
self._log.debug(
f"{task_id}: path '{path}' matches no rule in ruleset")
return
try:
if rule.action in ["copy", "move"]:
dst = rule.src_re.sub(rule.dst_re, path)
if not dst:
raise RuntimeError(
f"{task_id}: unable to {rule.action} '{path}', "
f"resulting destination path is empty")
if os.path.exists(dst):
raise RuntimeError(
f"{task_id}: unable to move file from '{path} "
f"to '{dst}', dstination path exists already")
dst_dir = os.path.dirname(dst)
if not os.path.isdir(dst_dir) and rule.auto_create:
self._log.info(
f"{task_id}: create directory '{dst_dir}'")
first_subdir = dst_dir
while not os.path.isdir(first_subdir):
parent = os.path.dirname(first_subdir)
if not os.path.isdir(parent):
first_subdir = parent
else:
break
os.makedirs(dst_dir)
await self._set_mode_and_owner(first_subdir, rule, task_id)
self._log.info(
f"{task_id}: {rule.action} '{path}' to '{dst}'")
if rule.action == "copy":
if os.path.isdir(path):
shutil.copytree(path, dst)
else:
shutil.copy2(path, dst)
else:
os.rename(path, dst)
await self._set_mode_and_owner(dst, rule, task_id)
elif rule.action == "delete":
self._log.info(
f"{task_id}: {rule.action} '{path}'")
if os.path.isdir(path):
if rule.rec:
shutil.rmtree(path)
else:
shutil.rmdir(path)
else:
os.remove(path)
except RuntimeError as e:
self._log.error(f"{task_id}: {e}")
except Exception as e:
self._log.exception(f"{task_id}: {e}")

0
misc/pyinotifyd.service → pyinotifyd/misc/pyinotifyd.service
View File

152
pyinotifyd/scheduler.py Executable file
View File

@@ -0,0 +1,152 @@
# pyinotifyd is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# pyinotifyd is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with pyinotifyd. If not, see <http://www.gnu.org/licenses/>.
#
import asyncio
import logging
from shlex import quote as shell_quote
from uuid import uuid4
class _Task:
def __init__(self, event, delay, task_id, task, callback=None,
logname=None):
self._event = event
self._path = event.pathname
self._delay = delay
self._task_id = task_id
self._job = task
self._callback = callback
self._task = None
self._log = logging.getLogger((logname or __name__))
async def _start(self):
if self._delay > 0:
await asyncio.sleep(self._delay)
if self._callback is not None:
self._callback(self._event)
self._task = None
self._log.info(f"execute task {self._task_id}")
await asyncio.shield(self._job(self._event, self._task_id))
self._log.info(f"task {self._task_id} finished")
def start(self):
if self._task is None:
self._task = asyncio.create_task(self._start())
def cancel(self):
if self._task is not None:
self._task.cancel()
self._task = None
def restart(self):
self.cancel()
self.start()
class TaskScheduler:
def __init__(self, task, files, dirs, delay=0, logname=None):
assert callable(task), \
f"task: expected callable, got {type(task)}"
self._task = task
assert isinstance(delay, int), \
f"delay: expected {type(int)}, got {type(delay)}"
self._delay = delay
assert isinstance(files, bool), \
f"files: expected {type(bool)}, got {type(files)}"
self._files = files
assert isinstance(dirs, bool), \
f"dirs: expected {type(bool)}, got {type(dirs)}"
self._dirs = dirs
self._tasks = {}
self._logname = (logname or __name__)
self._log = logging.getLogger(self._logname)
def _task_started(self, event):
path = event.pathname
if path in self._tasks:
del self._tasks[path]
def schedule(self, event):
self._log.debug(f"received {event}")
if (not event.dir and not self._files) or \
(event.dir and not self._dirs):
return
path = event.pathname
maskname = event.maskname.split("|", 1)[0]
if path in self._tasks:
task = self._tasks[path]
self._log.info(
f"received event {maskname} on '{path}', "
f"re-schedule task {task.task_id} (delay={self._delay}s)")
task.restart()
else:
task_id = str(uuid4())
self._log.info(
f"received event {maskname} on '{path}', "
f"schedule task {task_id} (delay={self._delay}s)")
task = _Task(
event, self._delay, task_id, self._task,
callback=self._task_started, logname=self._logname)
self._tasks[path] = task
task.start()
def cancel(self, event):
self._log.debug(f"received {event}")
path = event.pathname
maskname = event.maskname.split("|", 1)[0]
if path in self._tasks:
task = self._tasks[path]
self._log.info(
f"received event {maskname} on '{path}', "
f"cancel scheduled task {task.task_id}")
task.cancel()
del self._tasks[path]
class ShellScheduler(TaskScheduler):
def __init__(self, cmd, task=None, logname=None, *args, **kwargs):
assert isinstance(cmd, str), \
f"cmd: expected {type('')}, got {type(cmd)}"
self._cmd = cmd
logname = (logname or __name__)
super().__init__(*args, task=self.task, logname=logname, **kwargs)
async def task(self, event, task_id):
maskname = event.maskname.split("|", 1)[0]
if hasattr(event, "src_pathname"):
src_pathname = event.src_pathname
else:
src_pathname = ""
cmd = self._cmd.replace("{maskname}", shell_quote(maskname)).replace(
"{pathname}", shell_quote(event.pathname)).replace(
"{src_pathname}", shell_quote(src_pathname))
self._log.info(f"{task_id}: execute shell command: {cmd}")
proc = await asyncio.create_subprocess_shell(cmd)
await proc.communicate()

1
pyinotifyd/version.py Executable file
View File

@@ -0,0 +1 @@
__version__ = "0.0.1"

115
pyinotifyd/watch.py Executable file
View File

@@ -0,0 +1,115 @@
#!/usr/bin/env python3
# pyinotifyd is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# pyinotifyd is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with pyinotifyd. If not, see <http://www.gnu.org/licenses/>.
#
import asyncio
import pyinotify
class EventMap:
flags = {
**pyinotify.EventsCodes.OP_FLAGS,
**pyinotify.EventsCodes.EVENT_FLAGS}
def __init__(self, event_map=None, default_task=None):
self._map = {}
if default_task is not None:
assert callable(default_task), \
f"default_task: expected callable, got {type(default_task)}"
for flag in EventMap.flags:
self.set(flag, default_task)
if event_map is not None:
assert isinstance(event_map, dict), \
f"event_map: expected {type(dict)}, got {type(event_map)}"
for flag, task in event_map.items():
self.set(flag, task)
def items(self):
return self._map.items()
def set(self, flag, values):
assert flag in EventMap.flags, \
f"event_map: invalid flag: {flag}"
if values is not None:
if not isinstance(values, list):
values = [values]
for value in values:
assert callable(value), \
f"event_map: {flag}: expected callable, got {type(value)}"
self._map[flag] = values
elif flag in self._map:
del self._map[flag]
class _TaskList:
def __init__(self, tasks=[]):
if not isinstance(tasks, list):
tasks = [tasks]
self._tasks = tasks
def add(self, task):
self._tasks.append(task)
def remove(self, task):
self._tasks.remove(task)
def execute(self, event):
for task in self._tasks:
task(event)
class Watch:
def __init__(self, path, event_map, rec=False, auto_add=False):
assert isinstance(path, str), \
f"path: expected {type('')}, got {type(path)}"
self.path = path
if isinstance(event_map, EventMap):
self.event_map = event_map
elif isinstance(event_map, dict):
self.event_map = EventMap(event_map)
else:
raise AssertionError(
f"event_map: expected {type(EventMap)} or {type(dict)}, "
f"got {type(event_map)}")
assert isinstance(rec, bool), \
f"rec: expected {type(bool)}, got {type(rec)}"
self.rec = rec
assert isinstance(auto_add, bool), \
f"auto_add: expected {type(bool)}, got {type(auto_add)}"
self.auto_add = auto_add
def event_notifier(self, wm, loop=asyncio.get_event_loop()):
handler = pyinotify.ProcessEvent()
mask = False
for flag, values in self.event_map.items():
setattr(handler, f"process_{flag}", _TaskList(values).execute)
if mask:
mask = mask | EventMap.flags[flag]
else:
mask = EventMap.flags[flag]
wm.add_watch(
self.path, mask, rec=self.rec, auto_add=self.auto_add,
do_glob=True)
return pyinotify.AsyncioNotifier(wm, loop, default_proc_fun=handler)

2
setup.cfg
View File

@@ -1,2 +1,2 @@
[metadata] [metadata]
version = attr: pyinotifyd.__version__ version = attr: pyinotifyd.version.__version__

6
setup.py
View File

@@ -11,7 +11,7 @@ setup(name = "pyinotifyd",
license = "GPL 3", license = "GPL 3",
keywords = "inotify daemon", keywords = "inotify daemon",
url = "https://github.com/spacefreak86/pyinotifyd", url = "https://github.com/spacefreak86/pyinotifyd",
py_modules = ["pyinotifyd"], packages = ["pyinotifyd"],
long_description = read_file("README.md"), long_description = read_file("README.md"),
long_description_content_type = "text/markdown", long_description_content_type = "text/markdown",
classifiers = [ classifiers = [
@@ -31,10 +31,6 @@ setup(name = "pyinotifyd",
"pyinotifyd=pyinotifyd:main" "pyinotifyd=pyinotifyd:main"
] ]
}, },
data_files = [
("/etc/pyinotifyd", ["docs/config.py"]),
("/usr/lib/systemd/system", ["misc/pyinotifyd.service"])
],
install_requires = ["pyinotify"], install_requires = ["pyinotify"],
python_requires = ">=3.7" python_requires = ">=3.7"
) )

9
test-pyinotifyd Executable file
View File

@@ -0,0 +1,9 @@
#!/usr/bin/env python
import sys
import pyinotifyd
if __name__ == '__main__':
sys.exit(
pyinotifyd.main()
)