Removed s3 mode.

Added rsync mode.
-s options is now in Mbps.
This commit is contained in:
Rob Pearce 2019-03-17 14:33:52 +11:00
parent 66b3c450b0
commit e6cd761452
1 changed files with 91 additions and 17 deletions

106
bare.sh
View File

@ -8,10 +8,15 @@ LOGFILE=/var/log/backup.log
RESTIC=/usr/local/bin/restic
RCLONE=/usr/local/bin/rclone
RSYNC=/usr/local/bin/rsync
#RCLONEOPTS="--cache-chunk-no-memory --buffer-size=10M --progress"
RCLONEOPTS="--progress --buffer-size 10M --cache-chunk-no-memory"
SPEED=""
CRONMODE=0
CONNECTIONS=20
CONNECTIONS=""
LOG=/dev/stdout
DATE=/bin/date
DOALL=0
@ -37,6 +42,10 @@ function usage-rc() {
echo " export B2_APP_ID=xxx"
echo " export B2_APP_KEY=xxx"
echo " export RESTIC_PASSWORD=xxx"
echo " export RSYNC_SERVER=xxx"
echo " export RSYNC_USER=backups"
echo " export RSYNC_DIR=/home/backups/backups"
echo " export RSYNC_OPTIONS=-Pavz"
}
function usage() {
@ -44,7 +53,7 @@ function usage() {
echo ""
echo " -a Run on all repos defined as 'auto'"
echo " -h Show this usage text"
echo " -s num Limit speed to 'num' kBps (default: unlimited)"
echo " -s num Limit speed to 'num' Mbps (default: unlimited)"
echo " -x num Use 'num' simultaneous connections (default: 20)"
echo " -c Cron mode - log to ${LOGFILE}"
echo " -t Test mode - dump what would be done then exit."
@ -156,6 +165,12 @@ if [[ -z $B2_APP_KEY ]]; then
exit 1
fi
# Convert speed from Mbps to KBps
if ! [[ -z $SPEED ]]; then
KSPEED=$(echo "scale=2; $SPEED * 125" | bc)
else
KSPEED=""
fi
# Get list of defined repos
@ -243,22 +258,31 @@ fi
for f in $REPOSTOBACKUP; do
REPO=${f}
DATAPATH=$(getdatapath $REPO)
export RESTIC_REPOSITORY="b2:nethack-${REPO}"
export RCLONE_REPOSITORY="remote:nethack-${REPO}"
export RSYNC_FULLDIR="${RSYNC_DIR}/${REPO}"
export RSYNC_REPOSITORY="${RSYNC_USER}@${RSYNC_SERVER}:${RSYNC_FULLDIR}"
checktag "$f" rclone
if [[ $? -eq 0 ]]; then
mode=rclone
else
checktag "$f" rsync
if [[ $? -eq 0 ]]; then
mode=rsync
else
mode=restic
fi
fi
if [[ $mode == "restic" ]]; then
if [[ -z $SPEED ]]; then
if [[ -z $KSPEED ]]; then
SPEEDOPTS=""
else
SPEEDOPTS="--limit-upload $SPEED"
SPEEDOPTS="--limit-upload $KSPEED"
fi
if [[ -z $CONNECTIONS ]]; then
@ -268,12 +292,30 @@ for f in $REPOSTOBACKUP; do
fi
AUTHOPTS="-p ${AUTHFILE}"
else
# rclone
if [[ -z $SPEED ]]; then
elif [[ $mode == "rsync" ]]; then
for f in RSYNC_SERVER RSYNC_USER RSYNC_DIR RSYNC_OPTIONS ]]; do
eval val='$'$f
if [[ -z $val ]]; then
echo "Error - \$$f not set. Please update ${RCFILE}."
echo ""
usage-rc
exit 1
fi
done
# rsync
if [[ -z $KSPEED ]]; then
SPEEDOPTS=""
else
SPEEDOPTS="--bwlimit=${SPEED}k"
SPEEDOPTS="--bwlimit=${KSPEED}k"
fi
CONNECTIONSOPTS=""
else
# rclone
if [[ -z $KSPEED ]]; then
SPEEDOPTS=""
else
SPEEDOPTS="--bwlimit=${KSPEED}k"
fi
if [[ -z $CONNECTIONS ]]; then
@ -311,24 +353,55 @@ for f in $REPOSTOBACKUP; do
log "Error: invalid command $CMD"
rv=0
fi
elif [[ $mode == "rsync" ]]; then
REMOTE=""
if [[ $CMD == "go" ]]; then
${RSYNC} ${RSYNC_OPTIONS} $DATAPATH/ ${RSYNC_REPOSITORY} 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "ls" ]]; then
ssh ${RSYNC_USER}@${RSYNC_SERVER} ls ${RSYNC_FULLDIR} 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "stats" ]]; then
ssh ${RSYNC_USER}@${RSYNC_SERVER} du -hs ${RSYNC_FULLDIR} 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "diff" ]]; then
${RSYNC} ${RSYNC_OPTIONS} -n $DATAPATH/ ${RSYNC_REPOSITORY} 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "init" ]]; then
# check whether it already exists first!
ssh ${RSYNC_USER}@${RSYNC_SERVER} ls -d ${RSYNC_FULLDIR} 2>&1 >> ${LOG}
if [ $? -eq 0 ]; then
log "ERROR: Repo ${REPO} already exists. Aborting."
else
ssh ${RSYNC_USER}@${RSYNC_SERVER} mkdir -p ${RSYNC_FULLDIR} 2>&1 >>${LOG}
if [ $? -eq 0 ]; then
log "Created directory ${RSYNC_FULLDIR}..."
else
log "Creation of ${RSYNC_FULLDIR} failed."
fi
fi
else
log "Error: invalid command $CMD"
rv=0
fi
else
# rclone
REMOTE=""
if [[ $CMD == "go" ]]; then
${RCLONE} sync $CONNECTIONSOPTS $SPEEDOPTS $DATAPATH $RCLONE_REPOSITORY 2>&1 >> ${LOG}
${RCLONE} sync $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $DATAPATH $RCLONE_REPOSITORY 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "ls" ]]; then
${RCLONE} ls $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
${RCLONE} ls $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "stats" ]]; then
${RCLONE} size $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
${RCLONE} size $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "diff" ]]; then
${RCLONE} check $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
${RCLONE} check $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
rv=$?
elif [[ $CMD == "init" ]]; then
# check whether it already exists first!
${RCLONE} size $RCLONE_REPOSITORY >/dev/null 2>&1
${RCLONE} size $RCLONEOPTS $RCLONE_REPOSITORY >/dev/null 2>&1
if [ $? -eq 0 ]; then
log "ERROR: Repo ${REPO} already exists. Aborting."
else
@ -337,8 +410,9 @@ for f in $REPOSTOBACKUP; do
rv=$?
if [ $rv -ne 0 ]; then
log "Rclone remote doesn't exist - creating it..."
${RCLONE} config create remote b2 account $B2_APP_ID key $B2_APP_KEY >>${LOG}
${RCLONE} $RCLONEOPTS config create remote b2 account $B2_APP_ID key $B2_APP_KEY >>${LOG}
rv=$?
if [ $rv -ne 0 ]; then
log "Rclone remote init failed."
exit 1
@ -349,7 +423,7 @@ for f in $REPOSTOBACKUP; do
if [ $rv -eq 0 ]; then
log "Creating B2 bucket for $RCLONE_REPOSITORY..."
# create the bucket
${RCLONE} mkdir $RCLONE_REPOSITORY >>${LOG}
${RCLONE} $RCLONEOPTS mkdir $RCLONE_REPOSITORY >>${LOG}
else
log "Rclone bucket creation of $RCLONE_REPOSITORY failed."
fi