Thursday, February 23, 2012

Top in Batch Mode, and Expect Script to monitor ESX logs

Unix Top:

0,15,30,45 * * * * /home/nutanix/serviceability/top_pprof.sh > /dev/null 2>&1

nutanix@NTNX-Ctrl-VM-3-NTNX:~$ more serviceability/top_pprof.sh
#!/bin/sh
if [ ! -d /home/nutanix/data/logs/top ]; then
mkdir /home/nutanix/data/logs/top
fi
m=`date "+%d%m"`
dod=`date "+%H%M"`
if [ ! -d /home/nutanix/data/logs/top/$m ]; then
mkdir /home/nutanix/data/logs/top/$m
fi
top -b -n1 > /home/nutanix/data/logs/top/$m/$dod.top
#TOP IN BATCH MODE
mem=`grep stargate /home/nutanix/data/logs/top/$m/$dod.top | awk '{print $10}'|sort -r | head -1| cut -d"." -f1`
if [ ${mem} -gt 55 ]; then
# IF MEM USAGE is more than 55.
curl http://localhost:2009 2>/dev/null |html2text -ascii -width 600 >/home/nutanix/data/logs/top/$m/$dod.stargatepage
curl http://localhost:2009/h/pprof/growth >/home/nutanix/data/logs/top/$m/$dod.pprof 2>/dev/null
curl http://localhost:2009/h/pprof/heapstats >/home/nutanix/data/logs/top/$m/$dod.pprof.heap 2>/dev/null
/home/nutanix/toolchain/x86_64-unknown-linux-gnu/1.1/bin/pprof -pdf /home/nutanix/bin/stargate http://localhost:2009/h/pprof/growth > /home/nutanix/data/logs/top/$m/$dod.pprof.pdf 2>/dev/null
fi

Monitor ESX logs from webpage:
ns:~/public_html/ATLAS$ ~/scripts/check_vcenter_disconnect.sh >> ~/public_html/ATLAS/index.html &

nutanix@uranus:~/public_html/ATLAS$ cat /data/home/nutanix/scripts/check_vcenter_disconnect.sh
#!/bin/sh
while true
do
dddd=`date -u +%Y-%m-%dT%H`
echo $dddd
for seq in `seq 1 4`
do
echo "atlas$seq"
~/scripts/atlas$seq |grep $dddd
done
sleep 1800 <<<< I did not use Cron, because we might forget to turn it off, altleast here when uranus rebooted, it is gone.
done

I could have create single expect script..
nutanix@uranus:~/public_html/ATLAS$ cat /data/home/nutanix/scripts/atlas*
#!/usr/bin/expect
spawn ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@172.
expect "word:"
send "a\r"
expect "#"
send "gunzip /var/run/log/vpxa*.gz\r"
expect "#"
send "egrep stolen /var/run/log/vpxa*\r"
expect "#"
send "egrep -i dropping /var/run/log/vpx*\r"
expect "#"
send "exit\r"

atlas2:
#!/usr/bin/expect
spawn ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@172
expect "word:"
send "\r"
expect "#"
send "gunzip /var/run/log/vpxa*.gz\r"
expect "#"
send "egrep stolen /var/run/log/vpxa*\r"
expect "#"
send "egrep -i dropping /var/run/log/vpx*\r"
expect "#"
send "exit\r"

atlas3:
#!/usr/bin/expect
spawn ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@172.
expect "word:"
send "a\r"
expect "#"
send "gunzip /var/run/log/vpxa*.gz\r"
expect "#"
send "egrep stolen /var/run/log/vpxa*\r"
expect "#"
send "egrep -i dropping /var/run/log/vpx*\r"
expect "#"

atlas4:
send "exit\r"
#!/usr/bin/expect
spawn ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@17
expect "word:"
send "apple4u2$\r"
expect "#"
send "gunzip /var/run/log/vpxa*.gz\r"
expect "#"
send "egrep stolen /var/run/log/vpxa*\r"
expect "#"
send "egrep -i dropping /var/run/log/vpx*\r"
expect "#"