#!/bin/csh -f

switch (`uname -s``uname -r`)
case SunOS5*:
    set HOST = /usr/local/bin/host
	breaksw
case Linux[23]*:
    set HOST = /usr/bin/host
	breaksw
default:
	exec echo ${0}: does not run on `uname -s` `uname -r`
endsw

while($#argv)
	switch ("$1")
	case -d:
		set DEBUG
#		shift
#		breaksw
# fall through to verbose
	case -v:
		set VERBOSE
		shift
		breaksw
	case -date:		# -date <yymmdd>
		set DATE = $2
		shift ; shift
		breaksw
	case -q:		# quiet -- just report if things are "abnormal"
		set QUIET
		shift
		breaksw
	case -h:		# tell last successful dump if latest wasn't clean
		set DOHIST
		shift
		breaksw
	case -lsfile:		# append last successful dump to specified file
		set LSFILE = $2
		shift ; shift
		breaksw
	case -hint:		# give a hint as to how to kill a hung backup
		set HINT
		shift
		breaksw
	case -log:		# just look at specified log file
		set LOGS = $2
		shift ; shift
		breaksw
	case -gw:		# grep for "rsync warning"
		set GW
		shift
		breaksw
	default:
	    if (! $?THOST) then
		set THOST = $1
		shift
		breaksw
	    endif
		set DQ = '"'
		exec echo ${0}: "What do I do with $DQ$1$DQ?"
	endsw
end

set BASE = `/bin/basename $0`
set TMP = /tmp/$BASE.$$
set TODAY = `date +%y%m%d`

if ($?DEBUG) echo "  --- logdir ---"
set LOGDIR = /rsync-backup/logs
if ($?THOST) then
    set LOGDIR = /backup/logs/$THOST
endif

if (! $?DATE) then
    if ($?DEBUG) echo "    setting date"
    if ($?DEBUG) echo `/bin/ls -t $LOGDIR | grep 'log$'` #| head -1 | sed 's;\..*;;'`
    if ($?DEBUG) echo ""
#    set DATE = `/bin/ls -t $LOGDIR | grep log | head -1 | sed 's;\..*;;'`
    set DATE = `/bin/ls -t $LOGDIR | grep 'log$' | head -1 | sed 's;\..*;;'`
    if ($?DEBUG) echo "    done setting date"

    set FOR
    if ($?THOST) then
	set FOR = "for $THOST"
    endif
    if ("$DATE" == "") exec echo ${0}: no log files found $FOR

# determine date dump should have run; 7/16/12
    set SCHEDTIME = `crontab -l | sed 's;#.*;;' | grep "^[ 	]*[0-9][0-9]*[ 	][ 	]*[0-9][0-9]*[ 	][ 	]*\*[ 	][ 	]*\*.*[ 	][ 	]*${THOST}[ 	][ 	]*" | awk '{print $1,$2}'`
    if ($#SCHEDTIME != 2) then
	crontab -l | sed 's;#.*;;' | grep "^[ 	]*[0-9][0-9]*[ 	][ 	]*\*/[0-9][0-9]*[ 	][ 	]*\*[ 	][ 	]*\*.*[ 	][ 	]*${THOST}[ 	][ 	]*" | grep . > /dev/null
	if ($status) echo Could not determine scheduled time for dump of $THOST
	set SCHEDTIME = ( 0 0 )
    endif
#    set CURRTIME = `date "+%M %H"`
    set CURRTIME = `date "+%M %H" | sed 's;0\([0-9]\);\1;g'`
#    echo SCHEDTIME = $SCHEDTIME\; CURRTIME = $CURRTIME
    if (($SCHEDTIME[2] < $CURRTIME[2]) || ($SCHEDTIME[2] == $CURRTIME[2] && $SCHEDTIME[1] < $CURRTIME[1])) then
	set TODAY = `date +%y%m%d`
	set RTODAY = $TODAY
    else
	set TODAY = `date --date=-1day +%y%m%d`
# Allow "real" today to go without comment too; 6/24/13
	set RTODAY = `date +%y%m%d`
    endif
    if ($DATE != $TODAY && $DATE != $RTODAY) then
	echo "    ************************************************"
	echo "    * Log file(s) are from $DATE; today is $TODAY *"
	echo "    ************************************************"
	echo ""
    endif

endif

if ($?DEBUG) echo "  --- logs ---"

if (! $?LOGS) set LOGS = `/bin/ls $LOGDIR | grep $DATE`

if ("$LOGS" == "") exec echo "    No logs exist from $DATE"

if ($?QUIET) then
    foreach LOG ( $LOGS )
	if ($?VERBOSE) echo `date +%T` checking $LOG
	set CAT = /bin/cat
#	echo $LOG | grep '\.log$' > /dev/null
# FAILED log files are not gzipped either; 4/20/17
	echo $LOG | egrep '\.log$|\.FAILED$' > /dev/null
	if ($status) set CAT = /bin/zcat
#	awk -f ~watrous/summarize-rsync-backup-log.awk $LOGDIR/$LOG | \
	$CAT $LOGDIR/$LOG | \
	    awk -f ~watrous/summarize-rsync-backup-log.awk | \
	    egrep -v "^/|Filesystem|Files.*Elapsed" > /dev/null
	set STATUS = $status
# Add a clue for hung dumps; 5/11/10
# Clue: To kill off hung backup, kill rsync process below PID listed
# in /var/run/<hostname>.backup.  Then confirm that the dump directory
# in /backup/backups/<hostname> is renamed to .FAILED.  Once this is
# confirmed, you can reschedule thw backup (or let the next one take
# place as already scheduled).
#    grep "still running" $LOGDIR/$LOG > /dev/null
#    if (! $status) then
#	echo ""
#echo "    To kill off hung dump, kill rdist process under process listed in"
#echo "    /var/run/<hostname>.backup.  Confirm backup directory in echo"
#echo "    /backup/backup/<hostname> is renamed to .FAILED.  Then reschedule"
#echo "    backup."
#	echo ""
#    endif
# end addition
	if ($?VERBOSE) echo "    status = $STATUS"
	if (! $STATUS) then
	    if ($?DOHIST) set NEEDHIST
	    goto dosum
	endif
    end
    if ($?VERBOSE) echo `date +%T` Nothing unusual found
    set NOPING
    if ($?THOST) goto checkactive
#    exit
    goto cleanup
endif

dosum:
if ($?DEBUG) echo "  --- dosum ---"

foreach LOG ( $LOGS )

    set CAT = /bin/cat
#    echo $LOG | grep '\.log$' > /dev/null
# FAILED log files are not gzipped either; 4/20/17
    echo $LOG | egrep '\.log$|\.FAILED$' > /dev/null
    if ($status) set CAT = /bin/zcat

    set SPACES
    if ($#LOGS > 1 || $?VERBOSE) then
	echo ${LOG}:
	echo ""
	set SPACES = "    "
    endif

#    awk -f ~watrous/summarize-rsync-backup-log.awk $LOGDIR/$LOG | sed "s;.;$SPACES&;"
#    awk -f ~watrous/summarize-rsync-backup-log.awk $LOGDIR/$LOG | sed "s;.;$SPACES&;" | tee $TMP
    set SINCE
#    grep "still running" $LOGDIR/$LOG > /dev/null
    $CAT $LOGDIR/$LOG | \
	grep "still running" > /dev/null
    if (! $status && -e /var/run/$THOST.backup) then
	set SINCE = `/bin/ls -l /var/run/$THOST.backup | awk '{print $6,$7}'`
	set NOW = `date`
	if ("$SINCE[1]" == $NOW[2] && "$SINCE[2]" == $NOW[3]) set SINCE = `/bin/ls -l /var/run/$THOST.backup | awk '{print $6,$7,$8}'`
	if ("$SINCE" != "") set SINCE = ( "(since" $SINCE")" )
    endif
#    awk -f ~watrous/summarize-rsync-backup-log.awk $LOGDIR/$LOG | sed -e "s;.;$SPACES&;" -e "s;still running;& $SINCE;" | tee $TMP
     $CAT $LOGDIR/$LOG | \
	awk -f ~watrous/summarize-rsync-backup-log.awk | \
	sed -e "s;.;$SPACES&;" -e "s;still running;& $SINCE;" | \
	tee $TMP
    egrep -v "^/|Filesystem|Files.*Elapsed" $TMP > /dev/null
    set STATUS = $status
    if (! $STATUS) then
	if ($?DOHIST) then
	    if ($?DEBUG) echo set NEEDHIST
	    set NEEDHIST
	endif
    endif

# Add a clue for hung dumps; 5/19/10
# Clue: To kill off hung backup, kill rsync process below PID listed
# in /var/run/<hostname>.backup.  Then confirm that the dump directory
# in /backup/backups/<hostname> is renamed to .FAILED.  Once this is
# confirmed, you can reschedule thw backup (or let the next one take
# place as already scheduled).
    grep "still running" $LOGDIR/$LOG > /dev/null
    if (! $status && ($?THOST || $?HINT)) then
	if (-e /var/run/$THOST.backup && $LOG == $LOGS[$#LOGS]) then
	    echo ""
echo "    To kill off hung dump, kill rdist process under process listed in"
#echo "    /var/run/<hostname>.backup.  Confirm backup directory in"
#echo "    /backup/backup/<hostname> is renamed to .FAILED.  Then reschedule"
#echo "    /var/run/$THOST.backup.  Confirm backup directory in"
echo "    backup.cs:/var/run/$THOST.backup.  Confirm backup directory in"
echo "    /backup/backup/$THOST is renamed to .FAILED.  Then reschedule"
echo "    backup."
	    echo ""
	else
	    echo ""
echo "    This backup no longer seems to be running."
	    echo ""
	    set DOADMIN
	endif
    endif
    grep "Can't connect to backup client" $LOGDIR/$LOG > /dev/null
    if (! $status && ($?THOST || $?HINT)) then
	if (-e /var/run/$THOST.backup && $LOG == $LOGS[$#LOGS]) then
	    echo ""
echo "    /var/run/$THOST.backup still exists.  Next scheduled backup will fail"
echo "    because this one appears to still be running."
	endif
    endif

# end addition

    if ($#LOGS > 1 ) echo ""

end

if (! $?NEEDHIST) then
    if ($?THOST) goto checkactive
#    exit
    goto cleanup
endif

if ($?DEBUG) echo "  --- dosum (post loop) ---"

if ($?VERBOSE) echo `date +%T` checking for last successful dump
set THISMON = `date +%y%m`
#set LOGS = `/bin/ls -t $LOGDIR | grep $THISMON`
set LOGS = `/bin/ls -t $LOGDIR | grep '.log$'`

foreach LOG ( $LOGS )
    if ($?VERBOSE) echo `date +%T` checking $LOG
    set CAT = /bin/cat
#    echo $LOG | grep '\.log$' > /dev/null
# FAILED log files are not gzipped either; 4/20/17
    echo $LOG | egrep '\.log$|\.FAILED$' > /dev/null
    if ($status) set CAT = /bin/zcat
#    awk -f ~watrous/summarize-rsync-backup-log.awk $LOGDIR/$LOG | \
    $CAT $LOGDIR/$LOG | \
	awk -f ~watrous/summarize-rsync-backup-log.awk | \
	egrep -v "^/|Filesystem|Files.*Elapsed" > /dev/null
    set STATUS = $status
# Add a clue for hung dumps; 5/11/10
#    grep "still running" $LOGDIR/$LOG > /dev/null
#    if (! $status) then
#	echo ""
#echo "    To kill off hung dump, kill rdist process under process listed in"
#echo "    /var/run/<hostname>.backup.  Confirm backup directory in echo"
#echo "    /backup/backup/<hostname> is renamed to .FAILED.  Then reschedule"
#echo "    backup."
#	echo ""
#e   endif
# end addition
# Don't count empty log file as one with no errors; 2/15/13
    if (-z $LOGDIR/$LOG) then
# Be verbose about this until I see it
	echo ""
	echo "    $LOGDIR/$LOG is zero length."
	echo "    Not counting it as good dump."
	set STATUS = 0
    endif
    if ($?VERBOSE) echo "    status = $STATUS"
    if ($STATUS) then
	echo ""
#	echo "    Last successful dump was" `echo $LOG | sed 's;\.\([0-9]*\)\..*; at \1;'`
#	echo "    Last dump with no errors was" `echo $LOG | sed 's;\.\([0-9]*\)\..*; at \1;'`
	set ODATE = `echo $LOG | sed 's;\..*;;'`
# Julian date could be 008; daw, 1/8/12
	set OJDATE = `date --date=$ODATE +%j | sed 's;^[0]*;;'`
	set JDATE = `date +%j | sed 's;^[0]*;;'`
	if ($JDATE < $OJDATE) then
# wrapped year end.  add number of days last year...
	    set LASTDAYLY = `echo $ODATE | sed 's;....$;1231;'`
	    set NODAYSLY = `date +%j --date=$LASTDAYLY`
	    @ JDATE += $NODAYSLY
	endif
	@ DAYS = $JDATE - $OJDATE
if ($?DEBUG) echo OJDATE = $OJDATE\; JDATE = $JDATE\; DAYS = $DAYS
	set DAYSAGO = "(1 day ago)"
	if ($DAYS > 1) set DAYSAGO = "($DAYS days ago)"
	if ($?LSFILE) then
	    echo $LOG | sed "s;\(.*\).log;/backup/backups/$THOST/\1;" >> $LSFILE
	endif
	echo "    Last dump with no errors was" `echo $LOG | sed 's;\.\([0-9]*\)\..*; at \1;'` $DAYSAGO
	set DOADMIN
	if ($?THOST) goto checkactive
#	exit
	goto cleanup
    endif
end

set DOADMIN			# need to notify admin even if last successful dump does not exist

if ($?VERBOSE) echo `date +%T` No successful dumps found this month

checkactive:
if ($?DEBUG) echo "  --- checkactive ---"

if ($?THOST) then
    /bin/ps -eaf | grep "/usr/local/rsync-backup/backup.pl.* $THOST" > /dev/null
    if (! $status && $DATE == $TODAY) then
#	set CSTR = `/bin/ps -eaf | grep "/[r]sync .*/$THOST/" | sed -e 's;.*[0-9][0-9][0-9][0-9][0-9][0-9]\.[0-9][0-9][0-9][0-9];;' -e 's;/$;;' | sort -u`
	set CSTR = `/bin/ps -eaf | grep "/[r]sync .*/$THOST/" | grep "[0-9][0-9][0-9][0-9][0-9][0-9]\.[0-9][0-9][0-9][0-9]" | sed -e 's;.*[0-9][0-9][0-9][0-9][0-9][0-9]\.[0-9][0-9][0-9][0-9];;' -e 's;/$;;' | sort -u`
	set SINCE = `/bin/ps -eaf | grep "/[r]sync .*/$THOST/" | awk '{print $5}' | head -1`
	if ("$CSTR" != "" && ! $?QUIET) echo $CSTR is currently being backed up for $THOST "(since $SINCE)"
    endif
    if (! $?NOPING) then
    /bin/ping -c1 $THOST > /dev/null
	if ($status) then
	    echo "    $THOST appears to be down -- does not answer ping"
	endif
    endif
endif

if ( $?DOADMIN) then
    echo ""
    set RP = `$HOST -t txt $THOST | grep RP: | sed -e 's;.*RP:;;' -e 's; .*;;' -e 's;\.$;;' -e 's;\.;@;;'`
    if ("$RP" != "") echo "   " The responsible person for $THOST is $RP
    if (-e   /backup/config/$THOST/backup.conf) then
	sed '#.*;;' /backup/config/$THOST/backup.conf | \
	    grep admin_email | \
	    sed "s;admin_email[ ]*=[ ]*;    The administrative contact for $THOST is ;"
    else
	echo "   " /backup/config/$THOST/backup.conf does not exist
    endif
endif

cleanup:

if ($?GW) grep "rsync warning" $LOGDIR/$LOG

/bin/rm -f $TMP