(require 'cl) ;;; to start daemon, use: ;;; emacs --batch -l /home/dto/e/monitor.el -f monitor-start-daemon ;;; General monitoring setup (defvar monitor-base-directory "/home/dto/monitor" "Directory where monitoring data are stored.") (defvar monitor-configuration-file "monitor.conf.el") (defvar monitor-profiles nil "Association list, mapping profile names to profile definitions.") ;;; Getting data from agents with SNMP (defun monitor-snmp-get (host community-string oid) (let ((output (with-temp-buffer (call-process "snmpget" nil t t "-Of" "-OQ" "-v1" host "-c" community-string oid) (buffer-substring-no-properties (point-min) (point-max))))) (string-match "^\\(.*\\) = \\(.*\\)$" output) (match-string 2 output))) (defun monitor-read-configuration () (load-file (concat (file-name-as-directory monitor-base-directory) monitor-configuration-file))) ;;; Creating and updating RRD databases (defvar monitor-rrdtool-program "/usr/local/bin/rrdtool") (defun monitor-rrd-filename (profile-name) (concat (file-name-as-directory monitor-base-directory) profile-name ".rrd")) (defun monitor-rrd-create (profile) "Accept a profile and create an appropriate RRD database." (let* ((profile-name (car profile)) (profile-plist (cdr profile)) (rrd-file (monitor-rrd-filename (car profile))) (rrd-args nil) (rrd-line nil)) (destructuring-bind (&key sysname label host community-string sources) profile-plist ;; begin building rrd-args (setf rrd-args (list "create" rrd-file)) ;; define data sources (dolist (source sources) (destructuring-bind (&key oid label type heartbeat color) source (setf rrd-line (format "DS:%s:%s:%s:U:U" label type heartbeat)) (setf rrd-args (append rrd-args (list rrd-line))))) ;; define archives (setf rrd-args (append rrd-args (list "RRA:AVERAGE:0.5:1:600" "RRA:AVERAGE:0.5:6:700" "RRA:AVERAGE:0.5:24:775" "RRA:AVERAGE:0.5:288:750")))) ;; ;; now call rrdtool (let ((retval (eval `(call-process monitor-rrdtool-program nil "*rrd-create*" nil ,@rrd-args)))) (if (not (equal 0 retval)) (error "Error creating RRD database.") (message "Created RRD database %s with args %S." rrd-file rrd-args)) ))) (defun monitor-rrd-update (data-file data-sources data-values) (let* ((rrd-args (list "update" data-file "-t")) (rrd-template "") (rrd-values "")) ;; ;; build template (setf rrd-template (getf (car data-sources) :label)) (dolist (source (cdr data-sources)) (setf rrd-template (concat rrd-template (format ":%s" (getf source :label))))) ;; ;; build values (setf rrd-values "N") (dolist (value data-values) (setf rrd-values (concat rrd-values (format ":%s" value)))) ;; ;; assemble final arg list (setf rrd-args (append rrd-args (list rrd-template) (list rrd-values))) ;; ;; call rrdtrool (let ((retval (eval `(call-process monitor-rrdtool-program nil "*rrd-update*" nil ,@rrd-args)))) (if (not (equal 0 retval)) (error "Error updating RRD database: %S" rrd-args) (message "Updated RRD database with args %S" rrd-args))))) ;;; Processing sets of monitor profiles (defun monitor-create-databases () (dolist (profile monitor-profiles) (let ((rrd-file (monitor-rrd-filename (car profile)))) (when (not (file-exists-p rrd-file)) (monitor-rrd-create profile))))) (defun monitor-update-databases () (dolist (profile monitor-profiles) (destructuring-bind (&key sysname label host community-string sources) (cdr profile) (let ((rrd-file (monitor-rrd-filename (car profile))) (values nil)) ;; retrieve data value for each source (setf values (mapcar (lambda (source) (monitor-snmp-get host community-string (getf source :oid))) sources)) ;; update rrd database with values (monitor-rrd-update rrd-file sources values))))) ;;; Generating graphs (defun monitor-graph-filename (profile-name) (concat (file-name-as-directory monitor-base-directory) profile-name ".png")) (defun monitor-create-graph (profile) (let* ((profile-name (car profile)) (profile-plist (cdr profile)) (rrd-file (monitor-rrd-filename profile-name)) (rrd-args nil)) (destructuring-bind (&key sysname label host community-string sources) profile-plist ;; begin building rrd-args (setf rrd-args (list "graph" (monitor-graph-filename profile-name) "--title" (concat sysname "::" label))) ;; define data sources (dolist (source sources) (destructuring-bind (&key oid label type heartbeat color) source (setf rrd-args (append rrd-args (list (format "DEF:%s=%s:%s:AVERAGE" label rrd-file label)))))) ;; create graph lines and captions (dolist (source sources) (destructuring-bind (&key oid label type heartbeat color) source (setf rrd-args (append rrd-args (list (format "LINE2:%s%s:%s" label color label) (format "GPRINT:%s:AVERAGE:(avg=%%.0lf" label) (format "GPRINT:%s:MIN:min=%%.0lf" label) (format "GPRINT:%s:MAX:max=%%.0lf)" label)))))) ;; ;; now call rrdtool grapher (let ((retval (eval `(call-process monitor-rrdtool-program nil "*rrd-graph*" nil ,@rrd-args)))) (if (not (equal 0 retval)) (error "Error creating graph file from %s" rrd-file) (message "Created graph file from %s with args %S" rrd-file rrd-args)))))) (defun monitor-update-graphs () (dolist (profile monitor-profiles) (monitor-create-graph profile))) ;;; Viewing graphs (defun monitor-view-graphs () (interactive) (let ((monitor-buffer (get-buffer-create "*Monitor*"))) (with-current-buffer monitor-buffer (clear-image-cache) (delete-region (point-min) (point-max)) (dolist (profile monitor-profiles) (let ((image (create-image (monitor-graph-filename (car profile)) 'png nil))) (insert-image image) (insert "\n")))))) ;;; Running as a daemon (defun monitor-poll () (message "--- monitor-poll called at %s" (current-time-string)) (monitor-read-configuration) (monitor-create-databases) (monitor-update-databases) (monitor-update-graphs)) (defvar monitor-daemon-timer nil) (defun monitor-start-daemon () ;; do initial poll (monitor-poll) ;; start daemon (setf monitor-daemon-timer (run-at-time t 300 'monitor-poll)) ;; wait forever (while t (sit-for 30))) ;;; Automatically generating monitor profiles (defun monitor-generate-storage-profile (host community-string index) (let ((descr (monitor-snmp-get host community-string (concat ".iso.org.dod.internet.mgmt.mib-2.host.hrStorage.hrStorageTable.hrStorageEntry.hrStorageDescr." index))) (sysname (monitor-snmp-get host community-string ".iso.org.dod.internet.mgmt.mib-2.system.sysName.0"))) `(,(concat sysname "_Storage_" index) :sysname ,sysname :label ,descr :host ,host :community-string ,community-string :sources ((:oid ,(concat ".iso.org.dod.internet.mgmt.mib-2.host.hrStorage.hrStorageTable.hrStorageEntry.hrStorageUsed." index) :label "used" :type "GAUGE" :heartbeat 600 :color "#FF0000") (:oid ,(concat ".iso.org.dod.internet.mgmt.mib-2.host.hrStorage.hrStorageTable.hrStorageEntry.hrStorageSize." index) :label "size" :type "GAUGE" :heartbeat 600 :color "#0000FF"))))) (defun monitor-generate-interface-profile (host community-string index) (let ((descr (monitor-snmp-get host community-string (concat ".iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifDescr." index))) (sysname (monitor-snmp-get host community-string ".iso.org.dod.internet.mgmt.mib-2.system.sysName.0"))) `(,(concat sysname "_Interface_" index) :sysname ,sysname :label ,descr :host ,host :community-string ,community-string :sources ((:oid ,(concat ".iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifInOctets." index) :label "in" :type "COUNTER" :heartbeat 600 :color "#FF0000") (:oid ,(concat ".iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifOutOctets." index) :label "out" :type "COUNTER" :heartbeat 600 :color "#00FF00"))))) (defun monitor-generate-system-profile (host community-string) (let ((sysname (monitor-snmp-get host community-string ".iso.org.dod.internet.mgmt.mib-2.system.sysName.0"))) `(,(concat sysname "_System") :sysname ,sysname :label "system" :host ,host :community-string ,community-string :sources ((:oid ".iso.org.dod.internet.mgmt.mib-2.host.hrSystem.hrSystemProcesses.0" :label "processes" :type "GAUGE" :heartbeat 600 :color "#FF0000") (:oid ".iso.org.dod.internet.private.enterprises.ucdavis.laTable.laEntry.laLoadInt.1" :label "load1" :type "GAUGE" :heartbeat 600 :color "#0000FF"))))) ; ;(monitor-create-databases) ;(monitor-update-databases) ;(monitor-update-graphs) ;(monitor-view-graphs) (provide 'monitor)