increase timeout when reading data from gathering functions

README.md

@@ -1,3 +1,23 @@
 # snmpd-oid-daemon
 
 A customizable daemon written in Bash to provide custom OIDs to snmpd.
+
+# Requirements
+snmpd-oid-daemon runs with Bash version 4.3 or later.
+
+# Installation
+* Copy **snmpd-oid-daemon.sh** to your system where it is accessible by snmpd (e.g. to /usr/local/bin).
+* Edit **snmpd.conf** and add the following line, replace OID with your custom base OID (e.g. .1.3.6.1.4.1.8072.9999.9999).
+```
+pass_persist OID /PATH/TO/snmpd-oid-agent.sh --base-oid OID
+```
+* Restart snmpd.
+
+# Data gathering functions
+* Take a look at the existing functions to learn which ones already exist and how they are implemented.
+* Implement your own function or overwrite an existing one in the overload-script if neccessary.
+* Overwrite the global array DATA_FUNCS in the overload-script to enable/disable functions or change their refresh delay.
+
+# Known issues
+* The interface to snmpd does not support type Gauge64. One way arround this is to use String instead and convert to int64 on the receiving end.
+* The function 'gather_filesum_data' in its current form requires snmpd to run as root which is not always the case. Overload the function (--overload-script) if this is an issue in your case.

snmpd-oid-daemon.sh

@@ -23,7 +23,7 @@ function usage() {
 Usage: $SCRIPT [-b BASE_OID] [-d] [-m FILE] [-n] [-h]
 
 Mandatory arguments to long options are mandatory for short options too.
-  -b, --base=BASE_OID        base OID to operate on, default is '${BASE_OID}'
+  -b, --base=BASE_OID        base OID to operate on, default is '$BASE_OID'
   -d, --debug                enable debug output
   -h, --help                 display this help and exit
   -m, --debug-marker=FILE    debug logs will enabled or disabled during runtime
@@ -31,7 +31,7 @@ Mandatory arguments to long options are mandatory for short options too.
   -o, --overload-script=FILE source file to add or overload data gathering functions
   -n, --no-log               disable logging
   -t, --tag                  mark every line to be logged with the specified tag,
-                             default is '${LOG_TAG}'
+                             default is '$LOG_TAG'
 EOF
 }
 
@@ -177,20 +177,16 @@ function set_oid_list {
   if (( ${#COL_TYPES[@]} == 1 )); then
     for row_decl in "${DATA[@]}"; do
       declare -a row=$(strip_declaration <<<"$row_decl")
-      echo $base_oid.$row_id
-      echo ${COL_TYPES[0]}
-      echo ${row[0]//[$'\n'$'\r']/}
+      set_oid "$base_oid.$row_id" "${COL_TYPES[0]}" "${row[0]//[$'\n'$'\r']/}"
       ((row_id++))
     done
   else
     for row_decl in "${DATA[@]}"; do
       local -a row=$(strip_declaration <<<"$row_decl")
-      col_id=${col_start_idx}
+      col_id=$col_start_idx
       type_id=0
       for value in "${row[@]}"; do
-        echo $base_oid.$col_id.$row_id
-        echo ${COL_TYPES[$type_id]}
-        echo ${value//[$'\n'$'\r']/}
+        set_oid "$base_oid.$col_id.$row_id" "${COL_TYPES[$type_id]}" "${value//[$'\n'$'\r']/}"
         ((col_id++))
         ((type_id++))
       done
@@ -219,7 +215,6 @@ function submit_oids() {
     echo -n "UPDATE "
     declare -p oids | strip_declaration
   fi
-  echo ENDOFDATA
   return 0
 }
 
@@ -345,7 +340,7 @@ if [ -f "$OVERLOAD_SCRIPT" -a -r "$OVERLOAD_SCRIPT" ]; then
   echo "source $OVERLOAD_SCRIPT" >&$LOG
   source "$OVERLOAD_SCRIPT"
 else
-  echo "overload script '$OVERLOAD_SCRIPT' does not exist or is not readable"
+  echo "overload script '$OVERLOAD_SCRIPT' does not exist or is not readable" >&$DEBUGLOG
 fi
 
 
@@ -365,12 +360,15 @@ declare -A OIDTYPES
 function clear_cached_oid() {
   local base_oid=$1
   local oid
+  local count=0
   for oid in ${!OIDDATA[@]}; do
     if [[ $oid == $base_oid.* ]]; then
       unset OIDDATA[$oid]
       unset OIDTYPES[$oid]
+      ((count++))
     fi
   done
+  echo "cache: removed $count OIDs" >&$DEBUGLOG
   return 0
 }
 
@@ -405,7 +403,7 @@ function update_oid_cache() {
         break
       ;;
       *)
-        echo "cache: received invalid line" >&$DEBUGLOG
+        echo "cache: received invalid line: $line" >&2
       ;;
     esac
   done
@@ -443,8 +441,7 @@ function return_oid() {
 # Main logic of the daemon.
 #
 function main() {
-  local cmd line oid req next
-  local -a args
+  local buf cmd oid req next
 
   echo "waiting for all data gathering functions to return data" >&$LOG
   update_oid_cache true
@@ -455,28 +452,22 @@ function main() {
       update_oid_cache
     done
     read -r -t 1 -u $STDIN buf
-    local rc=$?
-    if (( rc > 128 )); then
-      line+=$buf
-      continue
-    elif (( rc == 0 )); then
-      line+=$buf
+    rc=$?
+    if (( rc == 0 )); then
+      cmd+=${buf}
+    elif (( rc > 128 )); then
+      # read timed out
+      [ -z "$buf" ] && continue
+      echo "< $buf (partial line)" >&$DEBUGLOG
+      cmd+=${buf}
+      # to work around a bug in Bash prior to version 5.3, check if $cmd contains a complete command first before continuing reading
+      # -> bug report: https://lists.gnu.org/archive/html/bug-bash/2024-10/msg00005.html
+      # -> bug fix:    https://git.savannah.gnu.org/cgit/bash.git/diff/builtins/read.def?h=devel&id=3ed028ccec871bc8d3b198c1681374b1e37df7cd
+      [[ "${cmd,,}" =~ ^(ping|set|get|getnext)$ ]] || continue
     else
       exit 255
     fi
-    echo "< $line" >&$DEBUGLOG
-    if [ -z $cmd ]; then
-      cmd=$line
-      args=()
-    elif [ -z $line ]; then
-      cmd=""
-      args=()
-      snmp_echo NONE
-      continue
-    else
-      args+=("$line")
-    fi
-    line=""
+    echo "< $cmd" >&$DEBUGLOG
 
     case "${cmd,,}" in
       ping)
@@ -485,14 +476,21 @@ function main() {
       ;;
       set)
         # we need to args here, 'oid' and 'type_and_value'
-        (( ${#args[@]} < 2 )) && continue
         cmd=""
+        read -r -u $STDIN buf
+        echo "< $buf" >&$DEBUGLOG
+        read -r -u $STDIN buf
+        echo "< $buf" >&$DEBUGLOG
         snmp_echo not-writable
       ;;
       get)
-        (( ${#args[@]} < 1 )) && continue
         cmd=""
-        oid=${args[0]}
+        read -r -u $STDIN oid
+        echo "< $oid" >&$DEBUGLOG
+        if [ -z "$oid" ]; then
+          echo "received empty oid" >&2
+          snmp_echo NONE
+        fi
         req_from_oid $oid req || continue
         if [[ ! -v OIDDATA[$req] ]]; then
           echo "$oid not found" >&$DEBUGLOG
@@ -502,9 +500,13 @@ function main() {
         return_oid "$req"
       ;;
       getnext)
-        (( ${#args[@]} < 1 )) && continue
         cmd=""
-        oid=${args[0]}
+        read -r -u $STDIN oid
+        echo "< $oid" >&$DEBUGLOG
+        if [ -z "$oid" ]; then
+          echo "received empty oid" >&2
+          snmp_echo NONE
+        fi
         req_from_oid $oid req || continue
         next=$(printf "%s\n" ${!OIDDATA[@]} $req | sort -V | grep -A1 -E "^$req\$" | tail -n 1)
         echo "evaluated next candidate: [requested: '$req', next: '$next']" >&$DEBUGLOG
@@ -520,8 +522,8 @@ function main() {
         break
       ;;
       *)
-        echo "invalid command '$cmd'" >&$LOG
-        cmd=""
+        echo "invalid command '$cmd', exiting ..." >&2
+        break
       ;;
     esac
   done
@@ -548,26 +550,77 @@ exec {STDIN}<&0
 # Start main in a sub-shell and create a writable fd to it.
 echo "daemon starting (PID: $$)" >&$LOG
 exec {DATAIN}> >(main)
-pid=$!
+main_pid=$!
 
 trap "echo daemon stopped >&$LOG" EXIT
 
 declare -A timetable
+declare -A fdtable
+declare -A pidtable
 first_run=true
 while :; do
   # Check if main is still alive and exit otherwise.
-  ps -p $pid > /dev/null || break
+  ps -p $main_pid >/dev/null || break
 
-  [ -v EPOCHSECONDS ] && now=$EPOCHSECONDS || now=$(data +%s)
+  [ -v EPOCHSECONDS ] && now=$EPOCHSECONDS || now=$(date +%s)
   for func in "${!DATA_FUNCS[@]}"; do
-    if (( now >= ${timetable[$func]:-0} )); then
+    next_update=${timetable[$func]:-$now}
+    if (( now >= next_update )); then
       delay=${DATA_FUNCS[$func]}
-      next_update=$((now + delay))
-      timetable[$func]=$next_update
       $first_run && echo "starting $func (refresh every $delay seconds)" >&$LOG
-      echo "executing $func, scheduled next refresh at $(date -d @$next_update)" >&$DEBUGLOG
-      # execute data gathering function and pipe its output to main
-      $func >&$DATAIN
+      fd=${fdtable[$func]:--1}
+      if (( fd == -1 )); then
+        exec {data}< <($func </dev/null)
+        pid=$!
+        echo "gather: executed $func (PID $pid, FD $data)" >&$DEBUGLOG
+        fdtable[$func]=$data
+        pidtable[$func]=$pid
+      else
+        pid=${pidtable[$func]}
+        echo "gather: skip executing $func, it is still running (PID $pid, FD $fd)" >&2
+      fi
+      ((next_update+=delay))
+      timetable[$func]=$next_update
+      echo "gathe: scheduled next execution of $func at $(date -d @$next_update)" >&$DEBUGLOG
+    fi
+  done
+
+  for func in "${!fdtable[@]}"; do
+    fd=${fdtable[$func]}
+    (( fd == -1 )) && continue
+    if ! $first_run; then
+      read -t 0 -u $fd
+      (( $? == 0 )) || continue
+    fi
+
+    data=$(timeout 5 cat <&$fd)
+    rc=$?
+    eval "exec $fd>&-"
+    fdtable[$func]=-1
+
+    pid=${pidtable[$func]}
+    if (( rc == 124 )); then
+      echo "gather: timeout receiving data from $func (PID $pid, FD $fd), sending SIGTERM" >&2
+      kill -SIGTERM $pid
+      sleep 1
+      if ps -p $pid >/dev/null; then
+        echo "gather: unable to terminate $func (PID $pid, FD $fd), sending SIGKILL" >&2
+        kill -SIGKILL $pid
+      fi
+      continue
+    fi
+
+    wait $pid &>/dev/null
+    rc=$?
+    # the wait function in older Bash versions prior to 5.1 always returns 127 if the
+    # sub-process already exited at this point
+    (( rc == 127 )) && rc=0
+
+    echo "gather: $func (PID $pid, FD $fd) exited with rc = $rc" >&$DEBUGLOG
+    if (( rc == 0 )) && [ -n "$data" ]; then
+      echo "gather: sending data to cache" >&$DEBUGLOG
+      echo "$data" 1>&$DATAIN
+      echo "ENDOFDATA" >&$DATAIN
     fi
   done