160 lines
5.7 KiB
Bash
Executable File
160 lines
5.7 KiB
Bash
Executable File
#!/bin/sh
|
||
# Purpose {{{
|
||
## If Xymon server says that a service is in error on a remote host, try to restart this service.
|
||
## 1. Create a ssh keyring for xymon user {{{
|
||
# sudo mkdir -p -- /var/lib/xymon/.ssh/
|
||
# sudo ssh-keygen -f /var/lib/xymon/.ssh/id_rsa -N '' -q
|
||
# sudo chown -R xymon:xymon /var/lib/xymon/.ssh/
|
||
## }}}
|
||
## 2. Remote user {{{
|
||
# Ensure to have the ${REMOTE_SSH_USER} available on remote hosts and allowed to connect with SSH.
|
||
# Restrict the SSH access to a single SSH key from the Xymon server IP (~${REMOTE_SSH_USER}/.ssh/authorized_keys) :
|
||
## from="IP.SRV.XYM.ON" ssh-rsa AAAAA…
|
||
# Allow sudo commands to restart services (/etc/sudoers.d/xymon-ssh) :
|
||
## xymon-ssh ALL=(root:root) NOPASSWD: /bin/systemctl restart *
|
||
## }}}
|
||
|
||
# }}}
|
||
# Vars {{{
|
||
DEBUG=1
|
||
|
||
LOCAL_SSH_USER="xymon"
|
||
REMOTE_SSH_USER="xymon-ssh"
|
||
|
||
temp_dir=$(mktemp -d -t xymon-libs-alert-XXXXXX.tmp)
|
||
debug_stdout="${temp_dir}/debug.stdout"
|
||
debug_stderr="${temp_dir}/debug.stderr"
|
||
service_list="${temp_dir}/services.error.list"
|
||
# }}}
|
||
|
||
# Create log files
|
||
touch "${debug_stdout}" "${debug_stderr}"
|
||
|
||
# Manage only libs probe {{{
|
||
if [ "${BBSVCNAME}" = "libs" ]; then
|
||
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : ${BBHOSTNAME} — ${BBSVCNAME} error" >> "${debug_stdout}"
|
||
else
|
||
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : ${BBHOSTNAME} — ${BBSVCNAME} probe is not managed." >> "${debug_stderr}"
|
||
[ "${DEBUG}" -eq "0" ] || rm -rf -- "${temp_dir}"
|
||
exit 0
|
||
fi
|
||
# }}}
|
||
|
||
# Check if host need to reboot {{{
|
||
if echo "${BBALPHAMSG}" | grep -qE "\\&yellow Machine should be reboot.*" ; then
|
||
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : Test kernel — The host need to be rebooted." >> "${debug_stdout}"
|
||
fi
|
||
# }}}
|
||
|
||
# Check if a service need to restart
|
||
if echo "${BBALPHAMSG}" | grep -qE "\\&yellow The following processes.*" ; then
|
||
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : Test service — Some services need to be restarted." >> "${debug_stdout}"
|
||
# Get the list of binaries path
|
||
echo "${BBALPHAMSG}" | sed -n 's/^ \(\/.*\) (.*)/\1/p' > "${service_list}"
|
||
|
||
while IFS= read -r bin_path; do
|
||
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : while bin_path loop — ${bin_path} use old libs." >> "${debug_stdout}"
|
||
# Match binaries path and services name {{{
|
||
case "${bin_path}" in
|
||
# Apache2
|
||
'/usr/sbin/apache2' )
|
||
service_name="apache2"
|
||
;;
|
||
# Arpwatch
|
||
'/usr/sbin/arpwatch' )
|
||
service_name="arpwatch"
|
||
;;
|
||
# Dbus
|
||
'/usr/bin/dbus-daemon' )
|
||
service_name="dbus"
|
||
;;
|
||
# lvmetad − LVM metadata cache daemon
|
||
'/sbin/lvmetad' )
|
||
service_name="lvm2-lvmetad"
|
||
;;
|
||
# Netdata
|
||
'/usr/sbin/netdata' )
|
||
service_name="netdata"
|
||
;;
|
||
# blkmapd − pNFS block layout mapping daemon
|
||
'/usr/sbin/blkmapd' )
|
||
service_name="nfs-utils"
|
||
;;
|
||
# Mumble-server
|
||
'/usr/sbin/murmurd' )
|
||
service_name="mumble-server"
|
||
;;
|
||
# Nginx
|
||
'/usr/sbin/nginx' )
|
||
service_name="nginx"
|
||
;;
|
||
# Ntp
|
||
'/usr/sbin/ntpd' )
|
||
service_name="ntp"
|
||
;;
|
||
# Nslcd
|
||
'/usr/sbin/nslcd' )
|
||
service_name="nslcd"
|
||
;;
|
||
# PHP-FPM 7.0
|
||
'/usr/sbin/php-fpm7.0' )
|
||
service_name="php7.0-fpm"
|
||
;;
|
||
# PHP-FPM 7.3
|
||
'/usr/sbin/php-fpm7.3' )
|
||
service_name="php7.3-fpm"
|
||
;;
|
||
# Postfix
|
||
'/usr/lib/postfix/sbin/pickup' | '/usr/lib/postfix/qmgr' | '/usr/lib/postfix/sbin/tlsmgr' | '/usr/lib/postfix/sbin/qmgr' )
|
||
service_name="postfix"
|
||
;;
|
||
# Rdnssd
|
||
'/sbin/rdnssd' )
|
||
service_name="rdnssd"
|
||
;;
|
||
# Systemd-journald
|
||
'/lib/systemd/systemd-journald' )
|
||
service_name="systemd-journald"
|
||
;;
|
||
# Systemd-logind
|
||
'/lib/systemd/systemd-logind' )
|
||
service_name="systemd-logind"
|
||
;;
|
||
# default
|
||
* )
|
||
service_name="NOT.MANAGED"
|
||
;;
|
||
esac
|
||
# }}}
|
||
|
||
# Restart service if needed {{{
|
||
if [ "${service_name}" != "NOT.MANAGED" ] && [ "${service_name}" != "${previous_service_name}" ]; then
|
||
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : while bin_path loop — ${service_name} need to be restarted." >> "${debug_stdout}"
|
||
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : while bin_path loop — ssh -n -o StrictHostKeyChecking=no ${REMOTE_SSH_USER}@${BBHOSTNAME} sudo systemctl restart ${service_name}.service" >> "${debug_stdout}"
|
||
ssh -n -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "${REMOTE_SSH_USER}"@"${BBHOSTNAME}" "sudo systemctl restart ${service_name}.service" >> "${debug_stdout}" 2>> "${debug_stderr}"
|
||
previous_service_name="${service_name}"
|
||
else
|
||
if [ "${service_name}" = "NOT.MANAGED" ]; then
|
||
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : while bin_path loop — service for ${bin_path} is not managed." >> "${debug_stdout}"
|
||
else
|
||
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : while bin_path loop — ${service_name} was already restarted." >> "${debug_stdout}"
|
||
fi
|
||
fi
|
||
# }}}
|
||
done < "${service_list}"
|
||
|
||
# Also restart xymon-client service {{{
|
||
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : Test service — xymon-client also need to be restarted." >> "${debug_stdout}"
|
||
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : Test service — ssh -n -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${REMOTE_SSH_USER}@${BBHOSTNAME} sudo systemctl restart xymon-client.service" >> "${debug_stdout}"
|
||
ssh -n -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "${REMOTE_SSH_USER}"@"${BBHOSTNAME}" "sudo systemctl restart xymon-client.service" >> "${debug_stdout}" 2>> "${debug_stderr}"
|
||
# }}}
|
||
fi
|
||
|
||
# Remove empty error file
|
||
[ -s "${debug_stderr}" ] || rm -f "${debug_stderr}"
|
||
|
||
# Remove temp_dir if DEBUG is disable
|
||
[ "${DEBUG}" -eq "0" ] || rm -rf -- "${temp_dir}"
|
||
|
||
exit 0
|