diff --git a/bare.sh b/bare.sh index 7a4cb6a..1426108 100755 --- a/bare.sh +++ b/bare.sh @@ -8,10 +8,15 @@ LOGFILE=/var/log/backup.log RESTIC=/usr/local/bin/restic RCLONE=/usr/local/bin/rclone +RSYNC=/usr/local/bin/rsync + + +#RCLONEOPTS="--cache-chunk-no-memory --buffer-size=10M --progress" +RCLONEOPTS="--progress --buffer-size 10M --cache-chunk-no-memory" SPEED="" CRONMODE=0 -CONNECTIONS=20 +CONNECTIONS="" LOG=/dev/stdout DATE=/bin/date DOALL=0 @@ -37,6 +42,10 @@ function usage-rc() { echo " export B2_APP_ID=xxx" echo " export B2_APP_KEY=xxx" echo " export RESTIC_PASSWORD=xxx" + echo " export RSYNC_SERVER=xxx" + echo " export RSYNC_USER=backups" + echo " export RSYNC_DIR=/home/backups/backups" + echo " export RSYNC_OPTIONS=-Pavz" } function usage() { @@ -44,7 +53,7 @@ function usage() { echo "" echo " -a Run on all repos defined as 'auto'" echo " -h Show this usage text" - echo " -s num Limit speed to 'num' kBps (default: unlimited)" + echo " -s num Limit speed to 'num' Mbps (default: unlimited)" echo " -x num Use 'num' simultaneous connections (default: 20)" echo " -c Cron mode - log to ${LOGFILE}" echo " -t Test mode - dump what would be done then exit." @@ -156,6 +165,12 @@ if [[ -z $B2_APP_KEY ]]; then exit 1 fi +# Convert speed from Mbps to KBps +if ! [[ -z $SPEED ]]; then + KSPEED=$(echo "scale=2; $SPEED * 125" | bc) +else + KSPEED="" +fi # Get list of defined repos @@ -243,22 +258,31 @@ fi for f in $REPOSTOBACKUP; do REPO=${f} DATAPATH=$(getdatapath $REPO) - export RESTIC_REPOSITORY="b2:nethack-${REPO}" export RCLONE_REPOSITORY="remote:nethack-${REPO}" + export RSYNC_FULLDIR="${RSYNC_DIR}/${REPO}" + export RSYNC_REPOSITORY="${RSYNC_USER}@${RSYNC_SERVER}:${RSYNC_FULLDIR}" + checktag "$f" rclone if [[ $? -eq 0 ]]; then mode=rclone else - mode=restic + checktag "$f" rsync + if [[ $? -eq 0 ]]; then + mode=rsync + else + mode=restic + fi fi + + if [[ $mode == "restic" ]]; then - if [[ -z $SPEED ]]; then + if [[ -z $KSPEED ]]; then SPEEDOPTS="" else - SPEEDOPTS="--limit-upload $SPEED" + SPEEDOPTS="--limit-upload $KSPEED" fi if [[ -z $CONNECTIONS ]]; then @@ -268,12 +292,30 @@ for f in $REPOSTOBACKUP; do fi AUTHOPTS="-p ${AUTHFILE}" - else - # rclone - if [[ -z $SPEED ]]; then + elif [[ $mode == "rsync" ]]; then + for f in RSYNC_SERVER RSYNC_USER RSYNC_DIR RSYNC_OPTIONS ]]; do + eval val='$'$f + if [[ -z $val ]]; then + echo "Error - \$$f not set. Please update ${RCFILE}." + echo "" + usage-rc + exit 1 + fi + done + + # rsync + if [[ -z $KSPEED ]]; then SPEEDOPTS="" else - SPEEDOPTS="--bwlimit=${SPEED}k" + SPEEDOPTS="--bwlimit=${KSPEED}k" + fi + CONNECTIONSOPTS="" + else + # rclone + if [[ -z $KSPEED ]]; then + SPEEDOPTS="" + else + SPEEDOPTS="--bwlimit=${KSPEED}k" fi if [[ -z $CONNECTIONS ]]; then @@ -311,24 +353,55 @@ for f in $REPOSTOBACKUP; do log "Error: invalid command $CMD" rv=0 fi + elif [[ $mode == "rsync" ]]; then + REMOTE="" + if [[ $CMD == "go" ]]; then + ${RSYNC} ${RSYNC_OPTIONS} $DATAPATH/ ${RSYNC_REPOSITORY} 2>&1 >> ${LOG} + rv=$? + elif [[ $CMD == "ls" ]]; then + ssh ${RSYNC_USER}@${RSYNC_SERVER} ls ${RSYNC_FULLDIR} 2>&1 >> ${LOG} + rv=$? + elif [[ $CMD == "stats" ]]; then + ssh ${RSYNC_USER}@${RSYNC_SERVER} du -hs ${RSYNC_FULLDIR} 2>&1 >> ${LOG} + rv=$? + elif [[ $CMD == "diff" ]]; then + ${RSYNC} ${RSYNC_OPTIONS} -n $DATAPATH/ ${RSYNC_REPOSITORY} 2>&1 >> ${LOG} + rv=$? + elif [[ $CMD == "init" ]]; then + # check whether it already exists first! + ssh ${RSYNC_USER}@${RSYNC_SERVER} ls -d ${RSYNC_FULLDIR} 2>&1 >> ${LOG} + if [ $? -eq 0 ]; then + log "ERROR: Repo ${REPO} already exists. Aborting." + else + ssh ${RSYNC_USER}@${RSYNC_SERVER} mkdir -p ${RSYNC_FULLDIR} 2>&1 >>${LOG} + if [ $? -eq 0 ]; then + log "Created directory ${RSYNC_FULLDIR}..." + else + log "Creation of ${RSYNC_FULLDIR} failed." + fi + fi + else + log "Error: invalid command $CMD" + rv=0 + fi else # rclone REMOTE="" if [[ $CMD == "go" ]]; then - ${RCLONE} sync $CONNECTIONSOPTS $SPEEDOPTS $DATAPATH $RCLONE_REPOSITORY 2>&1 >> ${LOG} + ${RCLONE} sync $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $DATAPATH $RCLONE_REPOSITORY 2>&1 >> ${LOG} rv=$? elif [[ $CMD == "ls" ]]; then - ${RCLONE} ls $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG} + ${RCLONE} ls $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG} rv=$? elif [[ $CMD == "stats" ]]; then - ${RCLONE} size $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG} + ${RCLONE} size $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG} rv=$? elif [[ $CMD == "diff" ]]; then - ${RCLONE} check $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG} + ${RCLONE} check $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG} rv=$? elif [[ $CMD == "init" ]]; then # check whether it already exists first! - ${RCLONE} size $RCLONE_REPOSITORY >/dev/null 2>&1 + ${RCLONE} size $RCLONEOPTS $RCLONE_REPOSITORY >/dev/null 2>&1 if [ $? -eq 0 ]; then log "ERROR: Repo ${REPO} already exists. Aborting." else @@ -337,8 +410,9 @@ for f in $REPOSTOBACKUP; do rv=$? if [ $rv -ne 0 ]; then log "Rclone remote doesn't exist - creating it..." - ${RCLONE} config create remote b2 account $B2_APP_ID key $B2_APP_KEY >>${LOG} + ${RCLONE} $RCLONEOPTS config create remote b2 account $B2_APP_ID key $B2_APP_KEY >>${LOG} rv=$? + if [ $rv -ne 0 ]; then log "Rclone remote init failed." exit 1 @@ -349,7 +423,7 @@ for f in $REPOSTOBACKUP; do if [ $rv -eq 0 ]; then log "Creating B2 bucket for $RCLONE_REPOSITORY..." # create the bucket - ${RCLONE} mkdir $RCLONE_REPOSITORY >>${LOG} + ${RCLONE} $RCLONEOPTS mkdir $RCLONE_REPOSITORY >>${LOG} else log "Rclone bucket creation of $RCLONE_REPOSITORY failed." fi