bare/bare.sh

#!/bin/bash
VALIDCOMMANDS="go ls init repos stats diff"

RCFILE=${HOME}/.backup/config
AUTHFILE=${HOME}/.backup/auth
REPOFILE=${HOME}/.backup/repos
LOGFILE=/var/log/backup.log

RESTIC=/usr/local/bin/restic
RCLONE=/usr/local/bin/rclone
RSYNC=/usr/local/bin/rsync
USMB=/usr/local/bin/usmb

#RCLONEOPTS="--cache-chunk-no-memory --buffer-size=10M --progress"
RCLONEOPTS="--progress --buffer-size 10M --cache-chunk-no-memory"

SPEED=""
CRONMODE=0
CONNECTIONS=""
LOG=/dev/stdout
DATE=/bin/date
DOALL=0
REPOSTOBACKUP=""

function log() {
	local now
	now=`${DATE} +%Y/%m/%d-%H:%M:%S`
	echo "${now} <${mode}> $*" >>${LOG}
}

function usage-repo() {
	echo "Should be of this format:"
	echo "   reponame1:/path/to/files1/"
	echo "   reponame2:/path/to/files2/"
	echo "   ..."
	echo "   reponameX:/path/to/filesX/"
}
function usage-rc() {
	echo "Should be of this format:"
	echo "   export B2_ACCOUNT_ID=xxx"
	echo "   export B2_ACCOUNT_KEY=xxx"
	echo "   export B2_APP_ID=xxx"
	echo "   export B2_APP_KEY=xxx"
	echo "   export RESTIC_PASSWORD=xxx"
	echo "   export RSYNC_SERVER=xxx"
	echo "   export RSYNC_USER=backups"
	echo "   export RSYNC_DIR=/home/backups/backups"
	echo "   export RSYNC_OPTIONS=-Pavz"
	echo "   # optional:"
	echo "   #export USMB_PREFIX=/DataVolume/shares"
	echo "   #export NFS_SERVER=nfs.yourdomain.com"
	echo "   #export NFS_SERVER_BASE=/remote/nfs/share/base"
	echo "   #export NFS_PREFIX=/local/nfs/mountpoint"
}

function usage() {
	echo "usage:  $0 command reponame"
	echo ""
	echo "       -a         Run on all repos defined as 'auto'"
	echo "       -h         Show this usage text"
	echo "       -s num     Limit speed to 'num' Mbps (default: unlimited)"
	echo "       -x num     Use 'num' simultaneous connections (default: 20)"
	echo "       -c         Cron mode - log to ${LOGFILE}"
	echo "       -t         Test mode - dump what would be done then exit."
	echo ""
	echo "Valid commands are:"
	echo "   $VALIDCOMMANDS"
	echo ""
	echo "${RCFILE} should look like this:"
	usage-rc
	echo ""
	echo "${REPOFILE} should look like this:"
	usage-repo
	echo ""
}
function getdatapath() {
	local DATAPATH REPO x
	REPO="$1"

	DATAPATH=""
	for x in ${REPODEFS[@]}; do
		match="^${REPO}:"
		if [[ $x =~ $match ]]; then
			DATAPATH=`echo $x | sed -e s/${match}//`
		fi
	done
	DATAPATH=`echo $DATAPATH | sed -e s/:.*//`
	echo "$DATAPATH"
}
function getmode() {
	checktag "$1" rclone
	if [[ $? -eq 0 ]]; then
		mode=rclone
	else
		checktag "$1" rsync
		if [[ $? -eq 0 ]]; then
			mode=rsync
		else
			mode=restic
		fi
	fi
}

function checktag() { # return 0 if tag matches
	local reponame lookfor x match repodef
	reponame="$1"
	lookfor="$2"

	# if we were just given a repo name, look up the def
	if [[ $reponame  =~ ^.*:.*$ ]]; then
		repodef="$reponame"
	else
		repodef=""
		for x in ${REPODEFS[@]}; do
			match="^${reponame}:"
			if [[ $x =~ $match ]]; then
				repodef="$x"
			fi
		done
	fi


	if [[ $repodef =~ ^.*:.*:.*${lookfor}.*$ ]]; then
		return 0
	fi
	return 1
}

# Handle args
ARGS="acdhs:tx:"

while getopts "$ARGS" i; do
	case "$i" in
		a)
			DOALL=1
			;;
		h)
			usage;
                        exit 1
			;;
		s)
			SPEED="$OPTARG";
			;;
		x)
			CONNECTIONS="$OPTARG";
			;;
		c)
			CRONMODE=1;
			LOG=${LOGFILE}
			;;
		t)
			TESTMODE=1
			;;
		*)
			echo "ERROR: invalid argument:  $i";
			usage;
			;;
	esac
done
shift $((OPTIND - 1))

if ! [ -e ${RCFILE} ]; then
	echo "Error - can't find ${RCFILE}"
	echo ""
	usage-rc
	exit 1
fi
if ! [ -e ${REPOFILE} ]; then
	echo "Error - can't find ${REPOFILE}"
	echo ""
	usage-repo
	exit 1
fi

. ${RCFILE}


if [[ -z $B2_APP_ID ]]; then
	echo "Error - \$B2_APP_ID not set."
	exit 1
fi
if [[ -z $B2_APP_KEY ]]; then
	echo "Error - \$B2_APP_KEY not set."
	exit 1
fi

# Convert speed from Mbps to KBps
if ! [[ -z $SPEED ]]; then
	KSPEED=$(echo "scale=2; $SPEED * 125" | bc)
else
	KSPEED=""
fi


# Get list of defined repos
idx=0
for f in `cat $REPOFILE`; do
	REPODEFS[$idx]="$f"
	if [[ $DOALL -eq 1 ]]; then
		checktag "$f" auto
		if [[ $? -eq 0 ]]; then
			REPOSTOBACKUP="$REPOSTOBACKUP `echo $f | sed -e 's/:.*//'`"
		fi
	fi
	idx=$((idx + 1))
done

if ! [[ $VALIDCOMMANDS == *"$CMD"* ]]; then
	echo "Error - invalid command $CMD.  Should one of:  $VALIDCOMMANDS"
	exit 1
fi

CMD="$1"
shift 1

if [[ -z $REPOSTOBACKUP ]]; then
	REPOSTOBACKUP="$*"
fi

if [[ $CMD == "repos" ]]; then
	format="%-20s%-40s%-10s\n"
	echo "Repos defined in ${RCFILE}:"
	echo ""
	printf "$format" "Repo"  "Path"  "Tags"
	for x in ${REPODEFS[@]}; do

		IFS=':' read -ra tok <<< "$x"
		thisrepo=${tok[0]}
		thispath=${tok[1]}
		if [[ -z ${tok[2]} ]]; then
			tags="n/a"
		else
			tags=`echo "${tok[2]}" | sed -e 's/,/ /'`
		fi

		printf "$format" "$thisrepo"  "$thispath"  "$tags"
	done
	exit 0
fi

if [[ -z $REPOSTOBACKUP ]]; then
	usage
	exit 1
fi

# Validate repos
for f in $REPOSTOBACKUP; do
	DATAPATH=$(getdatapath $f)
	getmode "$f"

	checktag "$f" usmb
	if [[ $? -eq 0 ]]; then
		if [[ -z $USMB_PREFIX ]]; then
			log "Error:  repo '$REPO' has usmb tag but \$USMB_PREFIX not defined."
			exit 1
		fi
		if ! [[ $DATAPATH == *${USMB_PREFIX}* ]]; then
			log "Error:  path '$DATAPATH' of repo '$REPO' does not contain '$USMB_PREFIX'."
			exit 1
		fi
		USMBNAME=`echo "${DATAPATH}" | sed -e "s,${USMB_PREFIX}/\(.*\).*,\1,"`
		USEUSMB=1
	else
		USMBNAME=""
		USEUSMB=0
	fi

	checktag "$f" nfs
	if [[ $? -eq 0 ]]; then
		if [[ -z $NFS_SERVER ]]; then
			log "Error:  repo '$REPO' has nfs tag but \$NFS_SERVER not defined."
			exit 1
		fi
		if [[ -z $NFS_SERVER_BASE ]]; then
			log "Error:  repo '$REPO' has nfs tag but \$NFS_SERVER_BASE not defined."
			exit 1
		fi
		if ! [[ $DATAPATH == *${NFS_PREFIX}* ]]; then
			log "Error:  path '$DATAPATH' of repo '$REPO' does not contain '$NFS_PREFIX'."
			exit 1
		fi
		NFSNAME=`echo "${DATAPATH}" | sed -e "s,${NFS_PREFIX}/\(.*\).*,\1,"`
		NFSPATH="${NFS_SERVER}:${NFS_SERVER_BASE}/${NFSNAME}"
		USENFS=1
	else
		NFSNAME=""
		USENFS=0
	fi

	if [[ -z $DATAPATH ]]; then
		log "can't find matching repo for $f - make sure it is listed in ${REPOFILE}"
		exit 1
	fi
 
	if [[ $USEUSMB -eq 1 ]]; then
		mount | grep -q ${DATAPATH}
		if [[ $? -ne 0 ]]; then
			# try mounting it.
			${USMB} ${USMBNAME} >/dev/null 
			if [[ $? -eq 0 ]]; then
				log "Mount usmb volume '$USMBNAME':  success"
			else
				log "Mount usmb volume '$USMBNAME':  FAILED"
				exit 1
			fi
		fi
	elif [[ $USENFS -eq 1 ]]; then
		mount | grep -q ${DATAPATH}
		if [[ $? -ne 0 ]]; then
			# try mounting it.
			mount -t nfs ${NFSPATH} ${DATAPATH} >/dev/null 
			if [[ $? -eq 0 ]]; then
				log "Mount nfs volume '$NFSPATH' to $DATAPATH:  success"
			else
				log "Mount nfs volume '$NFSPATH' to $DATAPATH:  FAILED"
				exit 1
			fi
		fi
	fi

	if ! [[ -e ${DATAPATH} ]]; then
		log "Error:  ${DATAPATH} doesn't exist"
		exit 1
	fi

	count=`ls ${DATAPATH} | wc -l`
	if [ $count -le 2 ]; then
		log "Error:  ${DATAPATH} exists but appears to be empty" 
		exit 1
	fi

done

if [[ $TESTMODE -eq 1 ]]; then
	echo "Would have run '$CMD' on these repos:"
	for f in $REPOSTOBACKUP; do
		DATAPATH=$(getdatapath $f)
		echo "   $f -> $DATAPATH"
	done
	exit 0
fi

for f in $REPOSTOBACKUP; do
	REPO=${f}
	DATAPATH=$(getdatapath $REPO)
	export RESTIC_REPOSITORY="b2:nethack-${REPO}"
	export RCLONE_REPOSITORY="remote:nethack-${REPO}"
	export RSYNC_FULLDIR="${RSYNC_DIR}/${REPO}"
	export RSYNC_REPOSITORY="${RSYNC_USER}@${RSYNC_SERVER}:${RSYNC_FULLDIR}"


	getmode "$f"


	if [[ $mode == "restic" ]]; then
		if [[ -z $KSPEED ]]; then
			SPEEDOPTS=""
		else
			SPEEDOPTS="--limit-upload $KSPEED"
		fi

		if [[ -z $CONNECTIONS ]]; then
			CONNECTIONSOPTS=""
		else
			CONNECTIONSOPTS="-o b2.connections=${CONNECTIONS}"
		fi

                AUTHOPTS="-p ${AUTHFILE}"
	elif [[ $mode == "rsync" ]]; then
		for f in RSYNC_SERVER RSYNC_USER RSYNC_DIR RSYNC_OPTIONS ]]; do
			eval val='$'$f
			if [[ -z $val ]]; then
				echo "Error - \$$f not set.  Please update ${RCFILE}."
				echo ""
				usage-rc
				exit 1
			fi
		done

		# rsync
		if [[ -z $KSPEED ]]; then
			SPEEDOPTS=""
		else
			SPEEDOPTS="--bwlimit=${KSPEED}k"
		fi
		CONNECTIONSOPTS=""
	else
		# rclone
		if [[ -z $KSPEED ]]; then
			SPEEDOPTS=""
		else
			SPEEDOPTS="--bwlimit=${KSPEED}k"
		fi

		if [[ -z $CONNECTIONS ]]; then
			CONNECTIONSOPTS=""
		else
			CONNECTIONSOPTS="--transfers=${CONNECTIONS}"
		fi
	fi

	log "Starting '$CMD' on repo '$REPO' [$DATAPATH]"


	if [[ $mode == "restic" ]]; then
		if [[ $CMD == "go" ]]; then
			${RESTIC} backup $AUTHOPTS $CONNECTIONSOPTS $SPEEDOPTS $DATAPATH 2>&1 >> ${LOG}
			rv=$?
		elif [[ $CMD == "ls" ]]; then
			${RESTIC} snapshots $AUTHOPTS $CONNECTIONSOPTS 2>&1 >> ${LOG}
			rv=$?
		elif [[ $CMD == "stats" ]]; then
			${RESTIC} stats $AUTHOPTS $CONNECTIONSOPTS 2>&1 >> ${LOG}
			rv=$?
		elif [[ $CMD == "diff" ]]; then
			log "Error:  diff not supported in restic."
			rv=1
		elif [[ $CMD == "init" ]]; then
			# check whether it already exists first!
			${RESTIC} stats $AUTHOPTS $CONNECTIONSOPTS >/dev/null 2>&1
			if [ $? -eq 0 ]; then
				log "ERROR:  Repo ${REPO} already exists.  Aborting."
			else
				${RESTIC} init $AUTHOPTS $CONNECTIONSOPTS >/dev/null 2>&1
			fi
		else
			log "Error:  invalid command $CMD"
			rv=0
		fi
	elif [[ $mode == "rsync" ]]; then
		REMOTE=""
		if [[ $CMD == "go" ]]; then
			#echo run ${RSYNC} ${RSYNC_OPTIONS} $DATAPATH/ ${RSYNC_REPOSITORY} 2>&1 >> ${LOG}
			#exit 1
			${RSYNC} ${RSYNC_OPTIONS} $DATAPATH/ ${RSYNC_REPOSITORY} 2>&1 >> ${LOG}
			rv=$?
		elif [[ $CMD == "ls" ]]; then
			ssh ${RSYNC_USER}@${RSYNC_SERVER} ls ${RSYNC_FULLDIR} 2>&1 >> ${LOG}
			rv=$?
		elif [[ $CMD == "stats" ]]; then
			ssh ${RSYNC_USER}@${RSYNC_SERVER} du -hs ${RSYNC_FULLDIR} 2>&1 >> ${LOG}
			rv=$?
		elif [[ $CMD == "diff" ]]; then
			${RSYNC} ${RSYNC_OPTIONS} -n $DATAPATH/ ${RSYNC_REPOSITORY} 2>&1 >> ${LOG}
			rv=$?
		elif [[ $CMD == "init" ]]; then
			# check whether it already exists first!
			ssh ${RSYNC_USER}@${RSYNC_SERVER} ls -d ${RSYNC_FULLDIR} 2>&1 >> ${LOG}
			if [ $? -eq 0 ]; then
				log "ERROR:  Repo ${REPO} already exists.  Aborting."
			else
				ssh ${RSYNC_USER}@${RSYNC_SERVER} mkdir -p ${RSYNC_FULLDIR} 2>&1 >>${LOG}
				if [ $? -eq 0 ]; then
					log "Created directory ${RSYNC_FULLDIR}..."
				else
					log "Creation of ${RSYNC_FULLDIR} failed."
				fi
			fi
		else
			log "Error:  invalid command $CMD"
			rv=0
		fi
	else
		# rclone
		REMOTE=""
		if [[ $CMD == "go" ]]; then
			${RCLONE} sync $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $DATAPATH $RCLONE_REPOSITORY 2>&1 >> ${LOG}
			rv=$?
		elif [[ $CMD == "ls" ]]; then
			${RCLONE} ls $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
			rv=$?
		elif [[ $CMD == "stats" ]]; then
			${RCLONE} size $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
			rv=$?
		elif [[ $CMD == "diff" ]]; then
			${RCLONE} check $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
			rv=$?
		elif [[ $CMD == "init" ]]; then
			# check whether it already exists first!
			${RCLONE} size $RCLONEOPTS $RCLONE_REPOSITORY >/dev/null 2>&1
			if [ $? -eq 0 ]; then
				log "ERROR:  Repo ${REPO} already exists.  Aborting."
			else
				# does 'remote' exist?
				${RCLONE} listremotes | grep -q remote: >/dev/null 2>&1
				rv=$?
				if [ $rv -ne 0 ]; then
					log "Rclone remote doesn't exist - creating it..."
					${RCLONE} $RCLONEOPTS config create remote b2 account $B2_APP_ID key $B2_APP_KEY  >>${LOG}
					rv=$?

					if [ $rv -ne 0 ]; then
						log "Rclone remote init failed."
						exit 1
					fi

				fi

				if [ $rv -eq 0 ]; then
					log "Creating B2 bucket for $RCLONE_REPOSITORY..."
					# create the bucket
					${RCLONE} $RCLONEOPTS mkdir $RCLONE_REPOSITORY >>${LOG}
				else
					log "Rclone bucket creation of $RCLONE_REPOSITORY failed."
				fi
			fi
		else
			log "Error:  invalid command $CMD"
			rv=0
		fi
	fi

	if [[ $rv -eq 0 ]]; then
		if [[ $USEUSMB -eq 1 ]]; then
			log "Unmounting usmb volume '$USMBNAME'"
			usmb -u $USMBNAME
		elif [[ $USENFS -eq 1 ]]; then
			log "Unmounting nfs volume '$NFSPATH' from $DATAPATH"
			umount $DATAPATH
		fi
	else
		log "Error: '$CMD' on repo '$REPO' failed"
	fi

	log "Finished '$CMD' on repo '$REPO'"
done