Difference between revisions of "Local monitoring and automatic recovery"

Latest revision as of 16:26, 25 January 2018

Since some time in late 2017, the KoBo Toolbox installation at kobo.msf.org developed availability problem, as one of hosts in KoBo, KoboCAT at https://kc.msf.org becomes unresponsive, rendering many other functions of the site (not residing on kc) unaccessible.

As a temporary solution, the mechanism of automatic recovery was implemented locally on host server gatitux, in the following manner:

$ nano /etc/cron.d/check-docker

MAILTO=support@mreznisistemi.rs,dr.watson@brussels.msf.org
MAILFROM=Kobo-OCB
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/usr/lib/nagios/plugins/

*/2 * * * *     root    kobocat-restart -q

$ nano /usr/local/sbin/kobocat-restart

#!/bin/bash
# kobocat-restart bash script # Jan 2018 © mreznisistemi.rs
#
if [ "$1" == "-h" ] || [ "$1" == "--help" ]; then
        echo
        echo "restarts kobocat service if kc.msf.org fails to output user's landing page, i.e. returns HTTP code other than 200"
        echo
        echo "usage:   " `basename $0` "[-q|-qq]"
        echo "params:   -q    quiet, output status info only when kobocat is not OK; suitable for cron"
        echo "          -qq   very quiet, suppress all info except error message when kobocat is not OK"
        echo "returns:  0 for no action (HTTP 200), 1 for kobocat service attempted restart"
        echo
        exit 0
fi

UPTRESHOLD=300
read -d. < /proc/uptime UPTIME
if [ $UPTIME -lt $UPTRESHOLD ];  then
        date
        (>&2 echo "kobo.msf.org: server up for less than $UPTRESHOLD seconds, will check again.")
        exit 0
fi

if [ "$1" == "-q" ] || [ "$1" == "-qq" ]; then Q=" -q"; else Q=""; fi
kobocat-test $Q
if [ $? -ne 200 ]; then
        date
        (>&2 echo "kobo.msf.org: service KOBOCAT not accessible for user msfuser at https://kc.msf.org")
        (>&2 echo "kobo.msf.org: attempting recovery")
        if [ "$1" != "-qq" ]; then
                echo
                echo " ▇▇▇▇▇▇ docker-compose ps ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ ▇▇▇▇▇▇▇▇▇▇▇▇▇ "
                /usr/local/bin/docker-compose --no-ansi -f /home/ate/kobo-docker/docker-compose.yml ps
                echo
                echo " ▇▇▇▇▇▇ docker-compose top ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ ▇▇▇▇▇▇▇▇▇▇▇▇ "
                /usr/local/bin/docker-compose --no-ansi -f /home/ate/kobo-docker/docker-compose.yml top
                echo
                echo " ▇▇▇▇▇▇ docker-compose logs ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ ▇▇▇▇▇▇▇▇▇▇ "
                /usr/local/bin/docker-compose --no-ansi -f /home/ate/kobo-docker/docker-compose.yml logs
                echo
                echo " ▇▇▇▇▇▇ docker-compose restart kobocat ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ ▇▇▇▇▇▇▇▇▇ "
                (>&2 /usr/local/bin/docker-compose --no-ansi -f /home/ate/kobo-docker/docker-compose.yml restart kobocat)
                echo
                echo " * automated warning message from kobo.msf.org * contact support@mreznisistemi.rs *"
        fi
        exit 1
fi
if [ "$1" != "-q" ] && [ "$1" != "-qq" ]; then
        echo "kobo.msf.org : service KOBOCAT responding HTTP 200 - OK"
fi
exit 0

$ sudo nano sudo more /usr/local/sbin/kobocat-test

#!/bin/bash
# start bash script # Jan 2018 © mreznisistemi.rs
#
# tests kobocat service by opening test user's landing page on kc.msf.org, after authenticating with Django
# returns and prints HTTP code (200 for OK)
#
# usage:  kobocat-test [-q]
#
# params:   -q  quiet

LOGIN_URL='https://kobo.msf.org/accounts/login/?next=/'
LOGIN_USER='msfuser'
LOGIN_PASS='msfOCB1971'
COOKIES=cookies.txt
CURL_BIN="curl -s -c $COOKIES -b $COOKIES -e $LOGIN_URL"

[ "$1" != "-q" ] && echo -n "Django Auth  : get csrftoken ... "
$CURL_BIN $LOGIN_URL > /dev/null
if [ ! -f $COOKIES ]; then
        [ "$1" != "-q" ] && echo " unexpected error 600 - no cookies ... aborting."
        exit 600;
fi
DJANGO_TOKEN="csrfmiddlewaretoken=$(grep csrftoken $COOKIES | sed 's/^.*csrftoken\s*//')"

[ "$1" != "-q" ] && echo "perform login as $LOGIN_USER ..."
$CURL_BIN \
    -d "$DJANGO_TOKEN&username=$LOGIN_USER&password=$LOGIN_PASS" \
    -X POST $LOGIN_URL

[ "$1" != "-q" ] && echo -n "KoboCAT      : open landing page for $LOGIN_USER: HTTP "
HTTPCODE=`$CURL_BIN -o /dev/null -sw '%{response_code}' https://kc.msf.org/$LOGIN_USER/`
[ "$1" != "-q" ] && echo $HTTPCODE

[ "$1" != "-q" ] && echo    "Django Auth  : logout ..."
rm $COOKIES >/dev/null 2>&1
exit $HTTPCODE