使用shell脚本进行服务器系统监控——进程监控

#!/bin/ksh

typeset -u RUN_PRE_EVENT # Force to UPPERCASE
typeset -u RUN_STARTUP_EVENT # Force to UPPERCASE
typeset -u RUN_POST_EVENT # force to UPPERCASE

RUN_PRE_EVENT='N' # A 'Y' will execute, anything else will not
RUN_STARTUP_EVENT='Y' # A 'Y' will execute, anything else will not
RUN_POST_EVENT='Y' # A 'Y' will execute, anything else will not

LOGFILE="/tmp/proc_status.log"
[[ ! -s $LOGFILE ]] && touch $LOGFILE

SCRIPT_NAME=$(basename $0)
TTY=$(tty)
INTERVAL="1" # Seconds between sampling
JOBS=

####################################################
############# DEFINE FUNCTIONS HERE ################
####################################################

usage ()
{
echo "/n/n/t*****USAGE ERROR*****"
echo "/n/nUSAGE: $SCRIPT_NAME seconds process"
echo "/nWill monitor the specified process for the"
echo "specified number of seconds."
echo "/nUSAGE: $SCRIPT_NAME [-s|-S seconds] [-m|-M minutes]"
echo " [-h|-H hours] [-d|-D days] [-p|-P process]/n"
echo "/nWill monitor the specified process for number of"
echo "seconds specified within -s seconds, -m minutes,"
echo "-h hours and -d days. Any combination of command"
echo "switches can be used./n"
echo "/nEXAMPLE: $SCRIPT_NAME 300 dtcalc"
echo "/n/nEXAMPLE: $SCRIPT_NAME -m 5 -p dtcalc"
echo "/nBoth examples will monitor the dtcalc process"
echo "for 5 minutes. Can specify days, hours, minutes"
echo "and seconds, using -d, -h, -m and -s/n/n"
}

####################################################

trap_exit ()
{
# set -x # Uncommant to debug this function
# Log an ending time for process monitoring
echo "INTERRUPT: Program Received an Interrupt...EXITING..." > $TTY
echo "INTERRUPT: Program Received an Interrupt...EXITING..." >> $LOGFILE
TIMESTAMP=$(date +%D@%T) # Get a new time stamp...
echo "MON_STOPPED: Monitoring for $PROCESS ended ==> $TIMESTAMP/n" /
>> $TTY
echo "MON_STOPPED: Monitoring for $PROCESS ended ==> $TIMESTAMP/n" /
>> $LOGFILE
echo "LOGFILE: All Events are Logged ==> $LOGFILE /n" > $TTY

# Kill all functions
JOBS=$(jobs -p)
if [[ ! -z $JOBS && $JOBS != '' && $JOBS != '0' ]]
then
kill $(jobs -p) 2>/dev/null 1>&2
fi
return 2
}

####################################################

pre_event_script ()
{
# Put anything that you want to execute BEFORE the
# monitored process STARTS in this function

: # No-OP - Needed as a place holder for an empty function
# Comment Out the Above colon, ':'

PRE_RC=$?
return $PRE_RC
}

####################################################
startup_event_script ()
{
# Put anything that you want to execute WHEN, or AS, the
# monitored process STARTS in this function

: # No-OP - Needed as a place holder for an empty function
# Comment Out the Above colon, ':'

STARTUP_RC=$?
return $STARTUP_RC
}

####################################################

post_event_script ()
{
# Put anything that you want to execute AFTER the
# monitored process ENDS in this function

: # No-OP - Need as a place holder for an empty function
# Comment Out the Above colon, ':'

POST_RC=$?
return $POST_RC
}

####################################################

test_string ()
{
if (( $# != 1 ))
then
echo 'ERROR'
return
fi

C_STRING=$1

case $C_STRING in

+([0-9])) echo 'POS_INT' # Integer >= 0
;;
+([-0-9])) echo 'NEG_INT' # Integer < 0
;;
+([a-z])) echo 'LOW_CASE' # lower case text
;;
+([A-Z])) echo 'UP_CASE' # UPPER case text
;;
+([a-z]|[A-Z])) echo 'MIX_CASE' # MIxed CAse text
;;
*) echo 'UNKNOWN' # Anything else
esac
}

####################################################

proc_watch ()
{
# set -x # Uncomment to debug this function

while : # Loop Forever!!
do
case $RUN in
'Y')
# This will run the startup_event_script, which is a function

if [[ $RUN_STARTUP_EVENT = 'Y' ]]
then
echo "STARTUP EVENT: Executing Startup Event Script..." > $TTY
echo "STARTUP EVENT: Executing Startup Event Script..." >> $LOGFILE

startup_event_script # USER DEFINED FUNCTION!!!
RC=$?
if (( "RC" == 0 ))
then
echo "SUCCESS: Startup Event Script Completed RC - ${RC}"/
> $TTY
echo "SUCCESS: Startup Event Script Completed RC - ${RC}"/
>> $LOGFILE

else
echo "FAILURE: Startup Event Script FAILED RC - ${RC}"/
> $TTY
echo "FAILURE: Startup Event Script FAILED RC - ${RC}"/
>> $LOGFILE

fi
fi

integer PROC_COUNT='-1' # Reset the Counters
integer LAST_COUNT='-1'

# Loop until the process(es) end(s)

until (( "PROC_COUNT" == 0 ))
do
# This function is a Co-Process. $BREAK checks to see if
# "Program Interrupt" has taken place. If so BREAK will
# be 'Y' and we exit both the loop and function.

read BREAK
if [[ $BREAK = 'Y' ]]
then
return 3
fi
PROC_COUNT=$(ps -ef | grep -v "grep $PROCESS" /
| grep -v $SCRIPT_NAME /
| grep $PROCESS| wc -l) >/dev/null 2>&1

if (( "LAST_COUNT" > 0 && "LAST_COUNT" != "PROC_COUNT" ))
then
# The Process Count has Changed...
TIMESTAMP=$(date +%D@%T)
# Get a list of the PID of all of the processes
PID_LIST=$(ps -ef | grep -v "grep $PROCESS" /
| grep -v $SCRIPT_NAME /
| grep $PROCESS | awk '{print $2}')

echo "PROCESS COUNT: $PROC_COUNT $PROCESS/
Processes Running ==> $TIMESTAMP" >> $LOGFILE &
echo "PROCESS COUNT: $PROC_COUNT $PROCESS/
Processes Running ==> $TIMESTAMP" > $TTY

echo ACTIVE PIDS: $PID_LIST >> $LOGFILE &
echo ACTIVE PIDS: $PID_LIST > $TTY
fi
LAST_COUNT=$PROC_COUNT
sleep $INTERVAL # Needed to reduce CPU load!
done

RUN='N' # Turn the RUN Flag Off

TIMESTAMP=$(date +%D@%T)
echo "ENDING PROCESS: $PROCESS END time ==>/
$TIMESTAMP" >> $LOGFILE &
echo "ENDING PROCESS: $PROCESS END time ==>/
$TIMESTAMP" > $TTY

# This will run the post_event_script, which is a function

if [[ $RUN_POST_EVENT = 'Y' ]]
then
echo "POST EVENT: Executing Post Event Script..."/
> $TTY
echo "POST EVENT: Executing Post Event Script..."/
>> $LOGFILE

post_event_script # USER DEFINED FUNCTION!!!
RC=$?
if (( "RC" == 0 ))
then
echo "SUCCESS: Post Event Script Completed RC -/
${RC}" > $TTY
echo "SUCCESS: Post Event Script Completed RC - ${RC}"/
>> $LOGFILE
else
echo "FAILURE: Post Event Script FAILED RC - ${RC}"/
> $TTY
echo "FAILURE: Post Event Script FAILED RC - ${RC}"/
>> $LOGFILE
fi
fi
;;

'N')
# This will run the pre_event_script, which is a function

if [[ $RUN_PRE_EVENT = 'Y' ]]
then
echo "PRE EVENT: Executing Pre Event Script..." > $TTY
echo "PRE EVENT: Executing Pre Event Script..." >> $LOGFILE

pre_event_script # USER DEFINED FUNCTION!!!
RC=$?
if (( "RC" == 0 ))
then
echo "SUCCESS: Pre Event Script Completed RC - ${RC}"/
> $TTY
echo "SUCCESS: Pre Event Script Completed RC - ${RC}"/
>> $LOGFILE
else
echo "FAILURE: Pre Event Script FAILED RC - ${RC}"/
> $TTY
echo "FAILURE: Pre Event Script FAILED RC - ${RC}"/
>> $LOGFILE
fi
fi

echo "WAITING: Waiting for $PROCESS to startup...Monitoring..."

integer PROC_COUNT='-1' # Initialize to a fake value

# Loop until at least one process starts

until (( "PROC_COUNT" > 0 ))
do
# This is a Co-Process. This checks to see if a "Program
# Interrupt" has taken place. If so BREAK will be 'Y' and
# we exit both the loop and function

read BREAK
if [[ $BREAK = 'Y' ]]
then
return 3
fi
PROC_COUNT=$(ps -ef | grep -v "grep $PROCESS" /
| grep -v $SCRIPT_NAME | grep $PROCESS | wc -l) /
>/dev/null 2>&1

sleep $INTERVAL # Needed to reduce CPU load!
done

RUN='Y' # Turn the RUN Flag On

TIMESTAMP=$(date +%D@%T)

PID_LIST=$(ps -ef | grep -v "grep $PROCESS" /
| grep -v $SCRIPT_NAME /
| grep $PROCESS | awk '{print $2}')

if (( "PROC_COUNT" == 1 ))
then
echo "START PROCESS: $PROCESS START time ==>/
$TIMESTAMP" >> $LOGFILE &
echo ACTIVE PIDS: $PID_LIST >> $LOGFILE &
echo "START PROCESS: $PROCESS START time ==>/
$TIMESTAMP" > $TTY
echo ACTIVE PIDS: $PID_LIST > $TTY
elif (( "PROC_COUNT" > 1 ))
then
echo "START PROCESS: $PROC_COUNT $PROCESS /
Processes Started: START time ==> $TIMESTAMP" >> $LOGFILE &
echo ACTIVE PIDS: $PID_LIST >> $LOGFILE &
echo "START PROCESS: $PROC_COUNT $PROCESS /
Processes Started: START time ==> $TIMESTAMP" > $TTY
echo ACTIVE PIDS: $PID_LIST > $TTY
fi
;;
esac
done
}

####################################################
############## START OF MAIN #######################
####################################################

### SET A TRAP ####

trap 'BREAK='Y';print -p $BREAK 2>/dev/null;trap_exit/
2>/dev/null;exit 0' 1 2 3 15

BREAK='N' # The BREAK variable is used in the co-process proc_watch
PROCESS=
PROCESS_ID=$$
integer TOTAL_SECONDS=0

# Check commnand line arguments

if (( $# > 10 || $# < 2 ))
then
usage
exit 1
fi

# Check to see if only the seconds and a process are
# the only arguments

if [[ ($# -eq 2) && ($1 != -*) && ($2 != -*) ]]
then
NUM_TEST=$(test_string $1)
if [[ "$NUM_TEST" = 'POS_INT' ]]
then
TOTAL_SECONDS=$1
PROCESS=$2
else
usage
exit 1
fi
else
# Since getopts does not care what arguments it gets lets
# do a quick sanity check to make sure that we only have
# between 2 and 10 arguments and the first one must start
# with a -* (hyphen and anything), else usage error

case "$#" in
[2-10]) if [[ $1 != -* ]]; then
usage; exit 1
fi
;;
esac

HOURS=0 # Initialize all to zero
MINUTES=0
SECS=0
DAYS=0

# Use getopts to parse the command line arguments

# For each $OPTARG for DAYS, HOURS, MINUTES and DAYS check to see
# that each one is an integer by using the check_string function

while getopts ":h:H:m:M:s:S:d:D:P:p:" TIMED 2>/dev/null
do
case $TIMED in
h|H) [[ $(test_string $OPTARG) != 'POS_INT' ]] && usage && exit 1
(( HOURS = $OPTARG * 3600 )) # 3600 seconds per hour
;;
m|H) [[ $(test_string $OPTARG) != 'POS_INT' ]] && usage && exit 1
(( MINUTES = $OPTARG * 60 )) # 60 seconds per minute
;;
s|S) [[ $(test_string $OPTARG) != 'POS_INT' ]] && usage && exit 1
SECS="$OPTARG" # seconds are seconds
;;
d|D) [[ $(test_string $OPTARG) != 'POS_INT' ]] && usage && exit 1
(( DAYS = $OPTARG * 86400 )) # 86400 seconds per day
;;
p|P) PROCESS=$OPTARG # process can be anything
;;
/?) usage # USAGE ERROR
exit 1
;;
:) usage
exit 1
;;
*) usage
exit 1
;;
esac
done
fi

# We need to make sure that we have a process - sanity check

if [[ -z "$PROCESS" || "$PROCESS" = '' ]]
then
usage
exit 1
fi

# Check to see that TOTAL_SECONDS was not previously set

if (( TOTAL_SECONDS == 0 ))
then
# Add everything together if anything is > 0

if [[ $SECS -gt 0 || $MINUTES -gt 0 || $HOURS -gt 0 /
|| $DAYS -gt 0 ]]
then
(( TOTAL_SECONDS = SECS + MINUTES + HOURS + DAYS ))
fi
SECONDS_LEFT=$TOTAL_SECONDS # Set the countdown varaible...

fi

# Last Sanity Check!

if (( TOTAL_SECONDS <= 0 )) || [ -z $PROCESS ]
then
# Either There are No Seconds to Count or the
# $PROCESS Variable is Null...USAGE ERROR...

usage
exit 1
fi

echo "/nCurrently running $PROCESS processes:/n" > $TTY
ps -ef | grep -v "grep $PROCESS" | grep -v $SCRIPT_NAME /
| grep $PROCESS > $TTY

PROC_RC=$? # Get the initial state of the monitored function

echo >$TTY # Send a blank line to the screen

(( PROC_RC != 0 )) && echo "/nThere are no $PROCESS processes running/n"

if (( PROC_RC == 0 )) # The Target Process(es) is/are running...
then
RUN='Y' # Set the RUN flag to true, or yes.

integer PROC_COUNT # Strips out the "padding" for display

PROC_COUNT=$(ps -ef | grep -v "grep $PROCESS" | grep -v /
$SCRIPT_NAME | grep $PROCESS | wc -l) >/dev/null 2>&1
if (( PROC_COUNT == 1 ))
then
echo "The $PROCESS process is currently running...Monitoring.../n"
elif (( PROC_COUNT > 1 ))
then
print "There are $PROC_COUNT $PROCESS processes currently running/
...Monitoring.../n"
fi
else
echo "The $PROCESS process is not currently/ running...monitoring..."
RUN='N' # Set the RUN flag to false, or no.
fi

TIMESTAMP=$(date +%D@%T) # Time that this script started monitoring

# Get a list of the currently active process IDs

PID_LIST=$(ps -ef | grep -v "grep $PROCESS" /
| grep -v $SCRIPT_NAME /
| grep $PROCESS | awk '{print $2}')

echo "MON_STARTED: Monitoring for $PROCESS began ==> $TIMESTAMP" /
| tee -a $LOGFILE
echo ACTIVE PIDS: $PID_LIST | tee -a $LOGFILE

proc_watch |& # Create a Background Co-Process!!
WATCH_PID=$! # Get the process of the last background job!

# Start the Count Down!

integer SECONDS_LEFT=$TOTAL_SECONDS

while (( SECONDS_LEFT > 0 ))
do
print -p $BREAK 2>/dev/null

(( SECONDS_LEFT = SECONDS_LEFT - 1 ))
sleep 1 # Seconds Countdown
done

# Finished - Normal Timeout Exit...

TIMESTAMP=$(date +%D@%T) # Get a new time stamp...
echo "MON_STOPPED: Monitoring for $PROCESS ended ==> $TIMESTAMP/n" /
| tee -a $LOGFILE

echo "LOGFILE: All Events are Logged ==> $LOGFILE /n"

# Tell the proc_watch function to break out of the loop and die

BREAK='Y'
print -p $BREAK 2>/dev/null

kill $WATCH_PID 2>/dev/null

exit 0

# End of Script