bare/bare.sh

598 lines
14 KiB
Bash
Executable File

#!/bin/bash
VALIDCOMMANDS="go ls init repos stats diff forget"
RCFILE=${HOME}/.backup/config
AUTHFILE=${HOME}/.backup/auth
REPOFILE=${HOME}/.backup/repos
LOGFILE=/var/log/backup.log
RESTIC=/usr/local/bin/restic
RCLONE=/usr/local/bin/rclone
RSYNC=/usr/local/bin/rsync
USMB=/usr/local/bin/usmb
#RCLONEOPTS="--cache-chunk-no-memory --buffer-size=10M --progress"
RCLONEOPTS="--progress --buffer-size 10M --cache-chunk-no-memory"
SPEED=""
CRONMODE=0
CONNECTIONS=""
LOG=/dev/stdout
DATE=/bin/date
DOALL=0
REPOSTOBACKUP=""
function log() {
local now
now=`${DATE} +%Y/%m/%d-%H:%M:%S`
echo "${now} <${mode}> $*" >>${LOG}
}
function usage-repo() {
echo " reponame1:/path/to/files1/:tag1,tag2,...,tagX"
echo " reponame2:/path/to/files2/:tag1,tag2,...,tagX"
echo " ..."
echo " reponameX:/path/to/filesX/:tag1,tag2,...,tagX"
echo ""
echo "Backup method tags:"
echo " restic Backup using restic to Backblaze B2 bucket \$B2_BUCKET_PREFIX-<reponame>"
echo " rclone Backup using rclone to Backblaze B2 bucket \$B2_BUCKET_PREFIX-<reponame>"
echo " rsync Backup using rsync to \$RSYNC_USER@\$RSYNC_SERVER:\$RSYNC_DIR/<reponame>."
echo ""
echo "Pre-backup commands to access local files:"
echo " nfs Mount \$NFS_SERVER:\$NFS_SERVER_BASE/<sharename> to <repopath>"
echo " before running. Sharename determined by stripping \$NFS_PREFIX"
echo " from start of repo path."
echo " usmb Run 'usmb reponame' to mount \$USMB_PREFIX/<sharename> before running."
echo " Sharename determined by stripping \$NFS_PREFIX from start of <repopath>."
}
function usage-rc() {
echo " export B2_ACCOUNT_ID=xxx"
echo " export B2_ACCOUNT_KEY=xxx"
echo " export B2_APP_ID=xxx"
echo " export B2_APP_KEY=xxx"
echo " export B2_BUCKET_PREFIX=xxx"
echo " export RESTIC_PASSWORD=xxx"
echo " export RSYNC_SERVER=xxx"
echo " export RSYNC_USER=backups"
echo " export RSYNC_DIR=/home/backups/backups"
echo " export RSYNC_OPTIONS=-Pavz"
echo " export KEEPDAYS=31 # days to keep restic backups for"
echo " # optional:"
echo " #export USMB_PREFIX=/DataVolume/shares"
echo " #export NFS_SERVER=nfs.yourdomain.com"
echo " #export NFS_SERVER_BASE=/remote/nfs/share/base"
echo " #export NFS_PREFIX=/local/nfs/mountpoint"
}
function usage() {
echo "usage: $0 command reponame"
echo ""
echo " -a Run on all repos defined as 'auto'"
echo " -h Show this usage text"
echo " -s num Limit speed to 'num' Mbps (default: unlimited)"
echo " -x num Use 'num' simultaneous connections (default: 20)"
echo " -c Cron mode - log to ${LOGFILE}"
echo " -t Test mode - dump what would be done then exit."
echo ""
echo "Valid commands are:"
echo " $VALIDCOMMANDS"
echo ""
echo "${RCFILE} should look like this:"
usage-rc
echo ""
echo "${REPOFILE} should look like this:"
usage-repo
echo ""
}
function getdatapath() {
local DATAPATH REPO x
REPO="$1"
DATAPATH=""
for x in ${REPODEFS[@]}; do
match="^${REPO}:"
if [[ $x =~ $match ]]; then
DATAPATH=`echo $x | sed -e s/${match}//`
fi
done
DATAPATH=`echo $DATAPATH | sed -e s/:.*//`
echo "$DATAPATH"
}
function getmode() {
checktag "$1" rclone
if [[ $? -eq 0 ]]; then
mode=rclone
else
checktag "$1" rsync
if [[ $? -eq 0 ]]; then
mode=rsync
else
mode=restic
fi
fi
}
function checktag() { # return 0 if tag matches
local reponame lookfor x match repodef
reponame="$1"
lookfor="$2"
# if we were just given a repo name, look up the def
if [[ $reponame =~ ^.*:.*$ ]]; then
repodef="$reponame"
else
repodef=""
for x in ${REPODEFS[@]}; do
match="^${reponame}:"
if [[ $x =~ $match ]]; then
repodef="$x"
fi
done
fi
if [[ $repodef =~ ^.*:.*:.*${lookfor}.*$ ]]; then
return 0
fi
return 1
}
# Handle args
ARGS="acdhs:tx:"
while getopts "$ARGS" i; do
case "$i" in
a)
DOALL=1
;;
h)
usage;
exit 1
;;
s)
SPEED="$OPTARG";
;;
x)
CONNECTIONS="$OPTARG";
;;
c)
CRONMODE=1;
LOG=${LOGFILE}
;;
t)
TESTMODE=1
;;
*)
echo "ERROR: invalid argument: $i";
usage;
;;
esac
done
shift $((OPTIND - 1))
if ! [ -e ${RCFILE} ]; then
echo "Error - can't find ${RCFILE}"
echo ""
usage-rc
exit 1
fi
if ! [ -e ${REPOFILE} ]; then
echo "Error - can't find ${REPOFILE}"
echo ""
usage-repo
exit 1
fi
. ${RCFILE}
if [[ -z $B2_APP_ID ]]; then
echo "Error - \$B2_APP_ID not set."
exit 1
fi
if [[ -z $B2_APP_KEY ]]; then
echo "Error - \$B2_APP_KEY not set."
exit 1
fi
if [[ -z $KEEPFOR ]]; then
echo "Error - \$KEEPFOR not set."
exit 1
fi
# Convert speed from Mbps to KBps
if ! [[ -z $SPEED ]]; then
KSPEED=$(echo "scale=2; $SPEED * 125" | bc)
else
KSPEED=""
fi
# Get list of defined repos
idx=0
for f in `cat $REPOFILE`; do
REPODEFS[$idx]="$f"
if [[ $DOALL -eq 1 ]]; then
checktag "$f" auto
if [[ $? -eq 0 ]]; then
REPOSTOBACKUP="$REPOSTOBACKUP `echo $f | sed -e 's/:.*//'`"
fi
fi
idx=$((idx + 1))
done
if ! [[ $VALIDCOMMANDS == *"$CMD"* ]]; then
echo "Error - invalid command $CMD. Should one of: $VALIDCOMMANDS"
exit 1
fi
CMD="$1"
shift 1
if [[ -z $REPOSTOBACKUP ]]; then
REPOSTOBACKUP="$*"
fi
if [[ $CMD == "repos" ]]; then
format="%-20s%-40s%-10s\n"
echo "Repos defined in ${RCFILE}:"
echo ""
printf "$format" "Repo" "Path" "Tags"
for x in ${REPODEFS[@]}; do
IFS=':' read -ra tok <<< "$x"
thisrepo=${tok[0]}
thispath=${tok[1]}
if [[ -z ${tok[2]} ]]; then
tags="n/a"
else
tags=`echo "${tok[2]}" | sed -e 's/,/ /g'`
fi
printf "$format" "$thisrepo" "$thispath" "$tags"
done
exit 0
fi
if [[ -z $REPOSTOBACKUP ]]; then
usage
exit 1
fi
# Validate repos
for f in $REPOSTOBACKUP; do
DATAPATH=$(getdatapath $f)
getmode "$f"
checktag "$f" usmb
if [[ $? -eq 0 ]]; then
if [[ -z $USMB_PREFIX ]]; then
log "Error: repo '$REPO' has usmb tag but \$USMB_PREFIX not defined."
exit 1
fi
if ! [[ $DATAPATH == *${USMB_PREFIX}* ]]; then
log "Error: path '$DATAPATH' of repo '$REPO' does not contain '$USMB_PREFIX'."
exit 1
fi
USMBNAME=`echo "${DATAPATH}" | sed -e "s,${USMB_PREFIX}/\(.*\).*,\1,"`
USEUSMB=1
else
USMBNAME=""
USEUSMB=0
fi
checktag "$f" nfs
if [[ $? -eq 0 ]]; then
if [[ -z $NFS_SERVER ]]; then
log "Error: repo '$REPO' has nfs tag but \$NFS_SERVER not defined."
exit 1
fi
if [[ -z $NFS_SERVER_BASE ]]; then
log "Error: repo '$REPO' has nfs tag but \$NFS_SERVER_BASE not defined."
exit 1
fi
if ! [[ $DATAPATH == *${NFS_PREFIX}* ]]; then
log "Error: path '$DATAPATH' of repo '$REPO' does not contain '$NFS_PREFIX'."
exit 1
fi
NFSNAME=`echo "${DATAPATH}" | sed -e "s,${NFS_PREFIX}/\(.*\).*,\1,"`
NFSPATH="${NFS_SERVER}:${NFS_SERVER_BASE}/${NFSNAME}"
USENFS=1
else
NFSNAME=""
USENFS=0
fi
if [[ -z $DATAPATH ]]; then
log "can't find matching repo for $f - make sure it is listed in ${REPOFILE}"
exit 1
fi
if [[ $CMD == "go" ]]; then
NEEDMOUNT=1
elif [[ $CMD == "diff" ]]; then
NEEDMOUNT=1
else
NEEDMOUNT=0
fi
if [[ $NEEDMOUNT -eq 1 ]]; then
if [[ $USEUSMB -eq 1 ]]; then
mount | grep -q ${DATAPATH}
if [[ $? -ne 0 ]]; then
# try mounting it.
${USMB} ${USMBNAME} >/dev/null
if [[ $? -eq 0 ]]; then
log "Mount usmb volume '$USMBNAME': success"
else
log "Mount usmb volume '$USMBNAME': FAILED"
exit 1
fi
fi
elif [[ $USENFS -eq 1 ]]; then
mount | grep -q ${DATAPATH}
if [[ $? -ne 0 ]]; then
# try mounting it.
mkdir -p $DATAPATH
mount -t nfs ${NFSPATH} ${DATAPATH} >/dev/null
if [[ $? -eq 0 ]]; then
log "Mount nfs volume '$NFSPATH' to $DATAPATH: success"
else
log "Mount nfs volume '$NFSPATH' to $DATAPATH: FAILED"
exit 1
fi
fi
fi
if ! [[ -e ${DATAPATH} ]]; then
log "Error: ${DATAPATH} doesn't exist"
exit 1
fi
count=`ls ${DATAPATH} | wc -l`
if [ $count -le 2 ]; then
log "Error: ${DATAPATH} exists but appears to be empty"
exit 1
fi
fi
done
if [[ $TESTMODE -eq 1 ]]; then
echo "Would have run '$CMD' on these repos:"
for f in $REPOSTOBACKUP; do
DATAPATH=$(getdatapath $f)
echo " $f -> $DATAPATH"
done
exit 0
fi
for f in $REPOSTOBACKUP; do
REPO=${f}
DATAPATH=$(getdatapath $REPO)
checktag "$f" usmb
if [[ $? -eq 0 ]]; then
USMBNAME=`echo "${DATAPATH}" | sed -e "s,${USMB_PREFIX}/\(.*\).*,\1,"`
USEUSMB=1
else
USMBNAME=""
USEUSMB=0
fi
checktag "$f" nfs
if [[ $? -eq 0 ]]; then
NFSNAME=`echo "${DATAPATH}" | sed -e "s,${NFS_PREFIX}/\(.*\).*,\1,"`
NFSPATH="${NFS_SERVER}:${NFS_SERVER_BASE}/${NFSNAME}"
USENFS=1
else
NFSNAME=""
USENFS=0
fi
export RESTIC_REPOSITORY="b2:${B2_BUCKET_PREFIX}-${REPO}"
export RCLONE_REPOSITORY="remote:${B2_BUCKET_PREFIX}-${REPO}"
export RSYNC_FULLDIR="${RSYNC_DIR}/${REPO}"
export RSYNC_REPOSITORY="${RSYNC_USER}@${RSYNC_SERVER}:${RSYNC_FULLDIR}"
if [[ $CMD == "go" ]]; then
NEEDMOUNT=1
elif [[ $CMD == "diff" ]]; then
NEEDMOUNT=1
else
NEEDMOUNT=0
fi
getmode "$f"
if [[ $mode == "restic" ]]; then
MYREPO="restic: $RESTIC_REPOSITORY"
if [[ -z $KSPEED ]]; then
SPEEDOPTS=""
else
SPEEDOPTS="--limit-upload $KSPEED"
fi
if [[ -z $CONNECTIONS ]]; then
CONNECTIONSOPTS=""
else
CONNECTIONSOPTS="-o b2.connections=${CONNECTIONS}"
fi
AUTHOPTS="-p ${AUTHFILE}"
elif [[ $mode == "rsync" ]]; then
# rsync
MYREPO="rsync: ${RSYNC_USER}@${RSYNC_SERVER}:${RSYNC_FULLDIR}"
for f in RSYNC_SERVER RSYNC_USER RSYNC_DIR RSYNC_OPTIONS ]]; do
eval val='$'$f
if [[ -z $val ]]; then
echo "Error - \$$f not set. Please update ${RCFILE}."
echo ""
usage-rc
exit 1
fi
done
if [[ -z $KSPEED ]]; then
SPEEDOPTS=""
else
SPEEDOPTS="--bwlimit=${KSPEED}k"
fi
CONNECTIONSOPTS=""
else
# rclone
MYREPO="rclone: $RCLONE_REPOSITORY"
if [[ -z $KSPEED ]]; then
SPEEDOPTS=""
else
SPEEDOPTS="--bwlimit=${KSPEED}k"
fi
if [[ -z $CONNECTIONS ]]; then
CONNECTIONSOPTS=""
else
CONNECTIONSOPTS="--transfers=${CONNECTIONS}"
fi
fi
text="Starting '$CMD' on repo '$REPO' [$DATAPATH <-> $MYREPO]"
log "$text"
if [[ $mode == "restic" ]]; then
if [[ $CMD == "go" ]]; then
if [[ $CRONMODE -eq 1 ]]; then
${RESTIC} backup $AUTHOPTS $CONNECTIONSOPTS $SPEEDOPTS $DATAPATH 2>&1 | egrep "^(Added|processed|snapshot)" >> ${LOG}
else
${RESTIC} backup $AUTHOPTS $CONNECTIONSOPTS $SPEEDOPTS $DATAPATH 2>&1 >> ${LOG}
fi
rv=$?
elif [[ $CMD == "ls" ]]; then
${RESTIC} snapshots $AUTHOPTS $CONNECTIONSOPTS 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "stats" ]]; then
${RESTIC} stats $AUTHOPTS $CONNECTIONSOPTS 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "diff" ]]; then
log "Error: diff not supported in restic."
rv=1
elif [[ $CMD == "init" ]]; then
# check whether it already exists first!
${RESTIC} stats $AUTHOPTS $CONNECTIONSOPTS >/dev/null 2>&1
if [ $? -eq 0 ]; then
log "ERROR: Repo ${REPO} already exists. Aborting."
else
${RESTIC} init $AUTHOPTS $CONNECTIONSOPTS >/dev/null 2>&1
fi
elif [[ $CMD == "forget" ]]; then
# age out stuff after xxx days
FORGETOPTS="-d ${KEEPDAYS} --prune"
${RESTIC} forget $AUTHOPTS $CONNECTIONSOPTS $FORGETOPTS 2>&1 >> ${LOG}
else
log "Error: invalid command $CMD"
rv=0
fi
elif [[ $mode == "rsync" ]]; then
REMOTE=""
if [[ $CMD == "go" ]]; then
#echo run ${RSYNC} ${RSYNC_OPTIONS} $DATAPATH/ ${RSYNC_REPOSITORY} 2>&1 >> ${LOG}
#exit 1
${RSYNC} ${RSYNC_OPTIONS} $DATAPATH/ ${RSYNC_REPOSITORY} 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "ls" ]]; then
ssh ${RSYNC_USER}@${RSYNC_SERVER} ls ${RSYNC_FULLDIR} 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "stats" ]]; then
echo "Size of remote data for ${REPO}:"
ssh ${RSYNC_USER}@${RSYNC_SERVER} du -hs ${RSYNC_FULLDIR} 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "diff" ]]; then
${RSYNC} ${RSYNC_OPTIONS} -n $DATAPATH/ ${RSYNC_REPOSITORY} 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "init" ]]; then
# check whether it already exists first!
ssh ${RSYNC_USER}@${RSYNC_SERVER} ls -d ${RSYNC_FULLDIR} 2>&1 >> ${LOG}
if [ $? -eq 0 ]; then
log "ERROR: Repo ${REPO} already exists. Aborting."
else
ssh ${RSYNC_USER}@${RSYNC_SERVER} mkdir -p ${RSYNC_FULLDIR} 2>&1 >>${LOG}
if [ $? -eq 0 ]; then
log "Created directory ${RSYNC_FULLDIR}..."
else
log "Creation of ${RSYNC_FULLDIR} failed."
fi
fi
elif [[ $CMD == "forget" ]]; then
log "Error: forget not supported for rsync mode."
else
log "Error: invalid command $CMD"
rv=0
fi
else
# rclone
REMOTE=""
if [[ $CMD == "go" ]]; then
${RCLONE} sync $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $DATAPATH $RCLONE_REPOSITORY 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "ls" ]]; then
${RCLONE} ls $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "stats" ]]; then
${RCLONE} size $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "diff" ]]; then
${RCLONE} check $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "init" ]]; then
# check whether it already exists first!
${RCLONE} size $RCLONEOPTS $RCLONE_REPOSITORY >/dev/null 2>&1
if [ $? -eq 0 ]; then
log "ERROR: Repo ${REPO} already exists. Aborting."
else
# does 'remote' exist?
${RCLONE} listremotes | grep -q remote: >/dev/null 2>&1
rv=$?
if [ $rv -ne 0 ]; then
log "Rclone remote doesn't exist - creating it..."
${RCLONE} $RCLONEOPTS config create remote b2 account $B2_APP_ID key $B2_APP_KEY >>${LOG}
rv=$?
if [ $rv -ne 0 ]; then
log "Rclone remote init failed."
exit 1
fi
fi
if [ $rv -eq 0 ]; then
log "Creating B2 bucket for $RCLONE_REPOSITORY..."
# create the bucket
${RCLONE} $RCLONEOPTS mkdir $RCLONE_REPOSITORY >>${LOG}
else
log "Rclone bucket creation of $RCLONE_REPOSITORY failed."
fi
fi
elif [[ $CMD == "forget" ]]; then
log "Error: forget not supported for rclone mode."
else
log "Error: invalid command $CMD"
rv=0
fi
fi
if [[ $rv -eq 0 ]]; then
if [[ $NEEDMOUNT -eq 1 ]]; then
if [[ $USEUSMB -eq 1 ]]; then
log "Unmounting usmb volume '$USMBNAME'"
usmb -u $USMBNAME
elif [[ $USENFS -eq 1 ]]; then
log "Unmounting nfs volume '$NFSPATH' from $DATAPATH"
umount $DATAPATH
fi
fi
else
log "Error: '$CMD' on repo '$REPO' failed"
fi
log "Finished '$CMD' on repo '$REPO'"
done