#!/bin/bash

# This is dotCOMmies backup script. Its nothing uber-fansy, really its just an
# rsync wrapper. Its quite thurougly tested and account for silly things that
# can take place.
#	* "safe" multiday inremental backup using rsyncs hardlink functionality
#	* Deletion of source data will interrupt backup.
#	* Overmount of source data will also interrupt backup
#	* All errors are reported via email.
#	* Since most errors can be ignored for a couple of cycles and some are self
#	  curing one can set threshold for how many failures should happend before
#	  email is sent
#	* "running out of space" notification

# There are several assumptions made by this script:
#   * It's intended to use rsync over SSH not the daemon. The are benefits to
#     using the daemon but it also complicates some things. If you want to use
#     the daemon to sync then all you need to do is mody the rsync line.
#   * It's intended for incremental backup. Using it for plain syncing is
#     stupid, just use plain rsync without any wrappers.
#   * Rsync can do push and pull sycronization. This script relies on pull
#     because it make configuration management so much simpler. There are some
#     other reasons beyond the scope of this discussion.

# Installing
# The first thing you need to do is to ensure 
#   * Configuring SSH:
#     * Add a key 
#The documentation has been interrupted by a coffe break. BBL

#########CONFIG HERE############
#Fallback configuration, triggers special exit routine.
remoteMachine="myLaptop"
remotePath=""
backUpDescriptor="NONE"
backUpPath="/srv/backup/dot-lap/"
backUpNumber=7
sshKeyPath="/home/rsync/.ssh/id_rsa"
remoteUser="rsync"
emailNotify=1
emailNotifyThreshold=3	#number of errors in backup before email is sent
emailFromAddr="me@example.net"
emailToAddr="me@example.net"
emailServer="smtp.example.net"
#####SANE OPTIONS BELOW##########
deleteFiles=""			#set to --delete to delete old files
diskUsageMax=90
remoteOvemountProtection=1
#########CONFIG ENDS#############

#todo:
#former todo:
#	overmount protection, we check for existance of .backUpThis in $remotePath
#	Notify of disk usage
#	Notify admins
#	Check for 0MB/0files $remotePath
#	uncycle if rsync fails
#	lock files (if $backUpDescriptor.toRM exists then abort execution)

backUpDescriptor="$remoteMachine.$backUpDescriptor"
logFile="$backUpPath/log.$backUpDescriptor"
failCntFile="$backUpPath/fail.$backUpDescriptor"

function notifyOfError {
	if [ $emailNotify -eq 1 ]; then
		if ! [ -e $failCntFile ]; then
			echo "0" > $failCntFile
		fi
		failCount=`cat $failCntFile`
		failCount=$((failCount+1))
		echo "$failCount" > $failCntFile
		if [ $failCount -ge $emailNotifyThreshold ]; then
			sendEmail -s $emailServer -f $emailFromAddr -t $emailToAddr -u "BACKUP ERROR on $HOSTNAME for $remoteMachine " -m "Hello\nThis is the automatic backup script. I've encountered a problem while backing up $remoteMachine:$remotePath to $HOSTNAME:$backUpPath for the last $failCount attempts.\n\nThe last error message reads:\n $1 \n\nPlease help me.\n\nBest Regards\n--backup.sh" 2>&1 >> $logFile 
		fi
	fi
}

#Is the script configured?
if [ $remoteMachine = "NONE" -o $remotePath = "NONE" -o $backUpDescriptor = "NONE" ]; then 
	#Script is not configured send email and exit
	emailNotifyThreshold=0	#we override threshold for immediate notification of missconfiguration
	notifyOfError "The script has not been configured for the machine it is running on. Please make sure the hostname is matches one of the options. This is a fatal error."
	exit
fi

#save date for log
date=`date +%Y%m%d%H%M%S`

#enter the back up directory
cd $backUpPath

#does backUpDescriptor.toRM exist?
if [ -e $backUpDescriptor.toRM ]; then
	#Looks like the last process did not complete. Cancel execution
	echo "$date $backUpDescriptor.toRM exists. aborting execution" >> $logFile
	notifyOfError "'Lock file' exits. Aborting backup. This is usually caused by a backup that is running longer than the backup-cycle. Unless this error re-occurs it can be ignored."
	exit
fi

echo " " >> $logFile
echo "%<-----------------------------------------------------------------------------" >> $logFile
echo "BACKING UP $backUpDescriptor (`date`)" >> $logFile

#make all missing directories
count=0
madeDir=0
while [ $count -le $backUpNumber ]; do
	if ! [ -a $backUpDescriptor.$count ]; then
		mkdir $backUpDescriptor.$count
		madeDir=1
		echo "directory missing, making $backUpPath/$backUpDescriptor.$count" >> $logFile
	fi
	count=$((count+1))
done
if [ $madeDir -eq 1 ]; then
	emailNotifyThreshold=0	#we override threshold for immediate notification of potential nastiness 
	notifyOfError "Recreated missing directories. This should usually only happend after install or some changes. Check log file for more details." 
fi

#move oldest backup to temp dir
mv $backUpDescriptor.$backUpNumber $backUpDescriptor.toRM

#Cycle backups up one 
count=$backUpNumber;
while [ $count -gt 0 ]; do
	mv $backUpDescriptor.$(($count-1)) $backUpDescriptor.$count
	count=$((count-1))
done

echo "rsync -az --filter=':e /.rsyncExcl' --stats $deleteFiles --link-dest=../$backUpDescriptor.1/ -e'ssh -i $sshKeyPath' $remoteUser@$remoteMachine:$remotePath $backUpDescriptor.0/ 2>&1 >> $logFile" >> $logFile
rsync -az --filter=':e /.rsyncExcl' --stats $deleteFiles --link-dest=../$backUpDescriptor.1/ -e"ssh -i $sshKeyPath" $remoteUser@$remoteMachine:$remotePath $backUpDescriptor.0/ 2>&1 >> $logFile
commandOk=$?

#save to log.
echo "rsync code: $commandOk " >> $logFile

if [ $commandOk == 24 ]; then
	commandOk=0;
	echo "Ignoring 'vanished files' warning" >> $logFile
fi

#is backup empty?
if [ `ls -1A $backUpDescriptor.0|wc -l` -le 0 ]; then
	#if commandOk is non null there is no point overriding it. 
	if [ $commandOk == 0 ]; then
		#setting commandOk to bogus non-null, this forces uncycling
		#Admins are notified of this through next test of commandOk
		commandOk=1337
	fi
fi

if [ $remoteOvemountProtection -eq 1 ]; then
	#check for magic file called .backUpThis as a safeguard that we'r backing up the right thing. Maybe dir got overmounted?
	if ! [ -e "$backUpDescriptor.0/.backUpThis" ]; then
		#if commandOk is non null there is no point overriding it. 
		if [ $commandOk == 0 ]; then
			#setting commandOk to bogus non-null, this forces uncycling
			#Admins are notified of this through next test of commandOk
			commandOk=1338
		fi
	fi
fi

#Was command successful?
if [ $commandOk == 0 ]; then
	#rsync returned 0. This is good. 
	#Delete old backup
	rm -rf $backUpDescriptor.toRM
	echo "0" > $failCntFile
else
	#rsync didn't return 0. THIS IS BAD.
	echo "RSYNC FAILED ($commandOk). UNCYCLING." >> $logFile
	
	#if new backup is not empty, rename it else, rm it
	if [ `ls -1A $backUpDescriptor.0|wc -l` -gt 0 ]; then
		echo "RSYNC DID SYNC SOMETHING. STORED TO: $backUpDescriptor.FAIL.$date PLEASE RESOLVE AND RM " >> $logFile
		notifyOfError "Backup failed (exit code: $commandOk). Something was synced, and has been stored to $backUpDescriptor.FAIL.$date. Please resolve." 
		mv $backUpDescriptor.0 $backUpDescriptor.FAIL.$date	
	else
		echo "RSYNC DID NOT SYNC ANYTHING. REMOVING ATTEMP." >> $logFile
		notifyOfError "Backup failed (exit code: $commandOk). Nothing was synced." 
		rm -rf $backUpDescriptor.0
	fi;
	
	#uncycle
	#dirs=$((backUpNumber-1));
	dirs=$((backUpNumber));
	count=0;
	while [ $count -lt $dirs ]; do
		echo "mv $backUpDescriptor.$(($count+1))  $backUpDescriptor.$count"
		mv $backUpDescriptor.$(($count+1))  $backUpDescriptor.$count
		count=$(($count+1))
	done
	
	#recover oldest directory.
	mv $backUpDescriptor.toRM $backUpDescriptor.$backUpNumber
fi

diskUsage=`df $backUpPath|tail -n 1|awk '{print $5}'`
diskUsage=${diskUsage:0:${#diskUsage}-1}

if [ $diskUsage -gt $diskUsageMax ]; then
	emailNotifyThreshold=0	#we override threshold for immediate notification of potential nastiness 
	notifyOfError "Disk usage is $diskUsage%, I've been configured to notify when it exceeds $diskUsageMax%." 
fi

echo "DISK USAGE: $diskUsage" >> $logFile
echo "BACKUP SCRIPT FINISHED (`date`)" >> $logFile
echo "%<-----------------------------------------------------------------------------" >> $logFile
echo " " >> $logFile