888 lines
21 KiB
Bash
Executable File
888 lines
21 KiB
Bash
Executable File
#!/bin/bash
|
|
VALIDCOMMANDS="go ls init repos stats diff forget"
|
|
|
|
RCFILE=${HOME}/.backup/config
|
|
DEF_AUTHFILE=${HOME}/.backup/auth
|
|
REPOFILE=${HOME}/.backup/repos
|
|
STATFILE=${HOME}/.backup/stats
|
|
LOGFILE=/var/log/backup.log
|
|
|
|
RESTIC=/usr/local/bin/restic
|
|
RCLONE=/usr/local/bin/rclone
|
|
RSYNC=/usr/local/bin/rsync
|
|
USMB=/usr/local/bin/usmb
|
|
|
|
#RCLONEOPTS="--cache-chunk-no-memory --buffer-size=10M --progress"
|
|
RCLONEOPTS="--progress --buffer-size 10M --cache-chunk-no-memory"
|
|
|
|
SPEED=""
|
|
CRONMODE=0
|
|
CONNECTIONS=""
|
|
LOG=/dev/stdout
|
|
DATE=/bin/date
|
|
DOALL=0
|
|
REPOSTOBACKUP=""
|
|
TESTMODE=0
|
|
VERBOSE=0
|
|
|
|
function log() {
|
|
local now
|
|
now=`${DATE} +%Y/%m/%d-%H:%M:%S`
|
|
echo "${now} <${mode}> $*" >>${LOG}
|
|
}
|
|
|
|
function updatestats() {
|
|
local repo cmd stamp code
|
|
repo=$1
|
|
cmd=$2
|
|
code=$3
|
|
stamp=`date +%s`
|
|
if [[ -f $STATFILE ]]; then
|
|
sed -i "/${repo}:${cmd}/d" $STATFILE
|
|
fi
|
|
echo "${repo}:${cmd}:${code}:${stamp}" >> $STATFILE
|
|
}
|
|
|
|
function usage-repo() {
|
|
echo " reponame1;/path/to/files1/;tag1,tag2,...,tagX;repo_url;repo_passfile;repo_excludefile"
|
|
echo " reponame2;/path/to/files2/;tag1,tag2,...,tagX;repo_url;repo_passfile;repo_excludefile"
|
|
echo " ..."
|
|
echo " reponameX;/path/to/filesX/;tag1,tag2,...,tagX;repo_url;repo_passfile;repo_excludefile"
|
|
echo ""
|
|
echo " Default password file is: $DEF_AUTHFILE"
|
|
echo ""
|
|
echo "Backup method tags:"
|
|
echo " restic Backup using restic (default repo is Backblaze B2 bucket \$B2_BUCKET_PREFIX-<reponame>)"
|
|
echo " rclone Backup using rclone (default repo is Backblaze B2 bucket \$B2_BUCKET_PREFIX-<reponame>)"
|
|
echo " rsync Backup using rsync (defualt repo is \$DEF_RSYNC_USER@\$DEF_RSYNC_SERVER:\$DEF_RSYNC_DIR/<reponame>)"
|
|
echo ""
|
|
echo "Pre-backup commands to access local files:"
|
|
echo " nfs Mount \$DEF_NFS_SERVER:\$DEF_NFS_SERVER_BASE/<sharename> to <repopath>"
|
|
echo " before running. Sharename determined by stripping \$NFS_PREFIX"
|
|
echo " from start of repo path."
|
|
echo " usmb Run 'usmb reponame' to mount \$USMB_PREFIX/<sharename> before running."
|
|
echo " Sharename determined by stripping \$NFS_PREFIX from start of <repopath>."
|
|
echo " localmount Run 'mount reponame' to mount <path_to_files> before running."
|
|
}
|
|
function usage-rc() {
|
|
echo " export B2_ACCOUNT_ID=xxx # Backblaze B2 credentials"
|
|
echo " export B2_ACCOUNT_KEY=xxx # Backblaze B2 credentials"
|
|
echo " export B2_APP_ID=xxx # Backblaze B2 bucket details"
|
|
echo " export B2_APP_KEY=xxx # Backblaze B2 bucket auth"
|
|
echo " export B2_BUCKET_PREFIX=xxx # Prefix to append to create unique B2 bucket name"
|
|
echo ""
|
|
echo " export RESTIC_KEEPDAYS=31 # Days to keep restic backups for"
|
|
echo " export RESTICOPTS=\"--one-file-system\" # Extra args to pass to restic"
|
|
echo ""
|
|
echo " export DEF_RSYNC_SERVER=xxx # Server used for repos with 'rsync' tag"
|
|
echo " export DEF_RSYNC_USER=backups # Username for rsync server"
|
|
echo " export DEF_RSYNC_DIR=/home/backups/backups # Remote directory on rsync server"
|
|
echo " export RSYNC_OPTIONS=-Pavz # Options to pass to rsync"
|
|
echo ""
|
|
echo " export USMB_PREFIX=/DataVolume/shares # Strip this from repo path to get USMB share name"
|
|
echo ""
|
|
echo " export DEF_NFS_SERVER=nfs.yourdomain.com # NFS server where local files for 'nfs' repos are found"
|
|
echo " export DEF_NFS_SERVER_BASE=/remote/nfs/share/base # Path on local NFS server to prefix to repo names"
|
|
echo " export NFS_PREFIX=/local/nfs/mountpoint # Where to mount NFS exports locally before backing them up"
|
|
}
|
|
|
|
function usage() {
|
|
echo "usage: $0 command reponame"
|
|
echo ""
|
|
echo " -a Run on all repos defined as 'auto'"
|
|
echo " -h Show this usage text"
|
|
echo " -s num Limit speed to 'num' Mbps (default: unlimited)"
|
|
echo " -x num Use 'num' simultaneous connections (default: 20)"
|
|
echo " -c Cron mode - log to ${LOGFILE}"
|
|
echo " -t Test mode - dump what would be done then exit."
|
|
echo ""
|
|
echo "Valid commands are:"
|
|
echo " $VALIDCOMMANDS"
|
|
echo ""
|
|
echo "${RCFILE} should look like this:"
|
|
usage-rc
|
|
echo ""
|
|
echo "${REPOFILE} should look like this:"
|
|
usage-repo
|
|
echo ""
|
|
}
|
|
function getdatapath() {
|
|
local DATAPATH REPO x
|
|
REPO="$1"
|
|
|
|
DATAPATH=""
|
|
for x in ${REPODEFS[@]}; do
|
|
match="^${REPO}\;"
|
|
if [[ $x =~ $match ]]; then
|
|
DATAPATH=`echo $x | sed -e s/${match}//`
|
|
fi
|
|
done
|
|
DATAPATH=`echo $DATAPATH | sed -e s/\;.*//`
|
|
echo "$DATAPATH"
|
|
}
|
|
function getmode() {
|
|
local mode
|
|
checktag "$1" rclone
|
|
if [[ $? -eq 0 ]]; then
|
|
mode=rclone
|
|
else
|
|
checktag "$1" rsync
|
|
if [[ $? -eq 0 ]]; then
|
|
mode=rsync
|
|
else
|
|
mode=restic
|
|
fi
|
|
fi
|
|
echo "$mode"
|
|
}
|
|
|
|
function getrepoexcludefile() { # output path
|
|
local res rv
|
|
res=`getrepotok $1 6`
|
|
rv=$?
|
|
echo "$res"
|
|
return $rv
|
|
}
|
|
|
|
function getrepopath() { # output path
|
|
local res rv mode def repobase
|
|
res=`getrepotok $1 4`
|
|
rv=$?
|
|
|
|
if [[ -z $res ]]; then
|
|
mode=`getmode $1`
|
|
repobase=`getreponame $1`
|
|
if [[ $mode == "restic" ]]; then
|
|
def="b2:${B2_BUCKET_PREFIX}-${repobase}"
|
|
elif [[ $mode == "rclone" ]]; then
|
|
def="remote:${B2_BUCKET_PREFIX}-${repobase}"
|
|
elif [[ $mode == "rsync" ]]; then
|
|
def="${DEF_RSYNC_USER}@${DEF_RSYNC_SERVER}:${DEF_RSYNC_DIR}/$repobase"
|
|
else
|
|
def="(no default repo for mode $mode)"
|
|
fi
|
|
res="$def"
|
|
fi
|
|
|
|
echo "$res"
|
|
return $rv
|
|
}
|
|
|
|
function getrepopassfile() { # output path
|
|
local res rv
|
|
res=`getrepotok $1 5`
|
|
rv=$?
|
|
echo "$res"
|
|
return $rv
|
|
}
|
|
|
|
function getrepotags() { # output path
|
|
local res rv
|
|
res=`getrepotok $1 3`
|
|
rv=$?
|
|
echo "$res"
|
|
return $rv
|
|
}
|
|
|
|
function getrepotok() { # output given token from repo def
|
|
local def res rv code idx
|
|
def=`getrepodef "$1"`
|
|
idx=$2
|
|
code="{ print \$${idx} }"
|
|
res=`echo "$def" | awk -F';' "$code"`
|
|
echo "$res"
|
|
if [[ -z $res ]]; then
|
|
rv=1
|
|
else
|
|
rv=0
|
|
fi
|
|
return $rv
|
|
}
|
|
|
|
function getreponame() {
|
|
local reponame
|
|
|
|
reponame="$1"
|
|
|
|
if [[ $reponame =~ ^.*\;.*$ ]]; then
|
|
# if we were given a repo def, return just the name
|
|
reponame=`echo "$reponame" | sed -e 's/\;.*//g'`
|
|
fi
|
|
echo "$reponame"
|
|
}
|
|
|
|
function getrepodef() {
|
|
local reponame match repodef x
|
|
|
|
reponame="$1"
|
|
|
|
if [[ $reponame =~ ^.*\;.*$ ]]; then
|
|
repodef="$reponame"
|
|
else
|
|
# if we were just given a repo name, look up the def
|
|
repodef=""
|
|
for x in ${REPODEFS[@]}; do
|
|
match="^${reponame};"
|
|
if [[ $x =~ $match ]]; then
|
|
repodef="$x"
|
|
fi
|
|
done
|
|
fi
|
|
echo "$repodef"
|
|
}
|
|
|
|
|
|
function checktag() { # return 0 if tag matches
|
|
local reponame lookfor justtags
|
|
|
|
reponame="$1"
|
|
lookfor="$2"
|
|
|
|
justtags=`getrepotags "$reponame"`
|
|
|
|
if [[ $justtags =~ ^.*${lookfor}.*$ ]]; then
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
function do_umount() {
|
|
if [[ $USEUSMB -eq 1 ]]; then
|
|
log "Unmounting usmb volume '$USMBNAME'"
|
|
usmb -u $USMBNAME
|
|
elif [[ $USENFS -eq 1 ]]; then
|
|
log "Unmounting nfs volume '$NFSPATH' from $DATAPATH"
|
|
umount $DATAPATH
|
|
elif [[ $USELOCALMOUNT -eq 1 ]]; then
|
|
log "Unmounting dir $DATAPATH"
|
|
umount $DATAPATH
|
|
fi
|
|
}
|
|
|
|
# Handle args
|
|
ARGS="acdhs:tvx:"
|
|
|
|
while getopts "$ARGS" i; do
|
|
case "$i" in
|
|
a)
|
|
DOALL=1
|
|
;;
|
|
h)
|
|
usage;
|
|
exit 1
|
|
;;
|
|
s)
|
|
SPEED="$OPTARG";
|
|
;;
|
|
x)
|
|
CONNECTIONS="$OPTARG";
|
|
;;
|
|
c)
|
|
CRONMODE=1;
|
|
LOG=${LOGFILE}
|
|
;;
|
|
t)
|
|
TESTMODE=1
|
|
;;
|
|
v)
|
|
VERBOSE=1
|
|
;;
|
|
*)
|
|
echo "ERROR: invalid argument: $i";
|
|
usage;
|
|
;;
|
|
esac
|
|
done
|
|
shift $((OPTIND - 1))
|
|
|
|
if ! [ -e ${RCFILE} ]; then
|
|
echo "Error - can't find ${RCFILE}"
|
|
echo ""
|
|
usage-rc
|
|
exit 1
|
|
fi
|
|
if ! [ -e ${REPOFILE} ]; then
|
|
echo "Error - can't find ${REPOFILE}"
|
|
echo ""
|
|
usage-repo
|
|
exit 1
|
|
fi
|
|
which bc >/dev/null 2>&1
|
|
if [ $? -ne 0 ]; then
|
|
echo "Error - can't find bc in path"
|
|
echo ""
|
|
exit 1
|
|
fi
|
|
|
|
. ${RCFILE}
|
|
|
|
if [[ -z $RESTIC_KEEPDAYS ]]; then
|
|
echo "Error - \$RESTIC_KEEPDAYS not set."
|
|
exit 1
|
|
fi
|
|
|
|
# Convert speed from Mbps to KBps
|
|
if ! [[ -z $SPEED ]]; then
|
|
KSPEED=$(echo "scale=2; $SPEED * 125" | bc)
|
|
else
|
|
KSPEED=""
|
|
fi
|
|
|
|
|
|
# Get list of defined repos
|
|
idx=0
|
|
for f in `cat $REPOFILE`; do
|
|
REPODEFS[$idx]="$f"
|
|
|
|
mode=`getmode "$f"`
|
|
if [[ $mode == "restic" ]]; then
|
|
thisrpath=`getrepopath $f`
|
|
if [[ ${thisrpath,,} == *b2* ]]; then
|
|
if [[ -z $B2_APP_ID ]]; then
|
|
echo "Error - \$B2_APP_ID not set."
|
|
exit 1
|
|
fi
|
|
if [[ -z $B2_APP_KEY ]]; then
|
|
echo "Error - \$B2_APP_KEY not set."
|
|
exit 1
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
if [[ $DOALL -eq 1 ]]; then
|
|
checktag "$f" auto
|
|
if [[ $? -eq 0 ]]; then
|
|
REPOSTOBACKUP="$REPOSTOBACKUP `echo $f | sed -e 's/\;.*//'`"
|
|
fi
|
|
fi
|
|
idx=$((idx + 1))
|
|
done
|
|
|
|
if ! [[ $VALIDCOMMANDS == *"$CMD"* ]]; then
|
|
echo "Error - invalid command $CMD. Should one of: $VALIDCOMMANDS"
|
|
exit 1
|
|
fi
|
|
|
|
CMD="$1"
|
|
shift 1
|
|
|
|
if [[ -z $REPOSTOBACKUP ]]; then
|
|
REPOSTOBACKUP="$*"
|
|
fi
|
|
|
|
if [[ $CMD == "repos" ]]; then
|
|
format="%-10s%-20s%-20s%s\n"
|
|
echo "Repos defined in ${RCFILE}:"
|
|
echo ""
|
|
printf "$format" "Repo" "Path" "Tags" "Repo path"
|
|
for x in ${REPODEFS[@]}; do
|
|
|
|
IFS=';' read -ra tok <<< "$x"
|
|
thisrepo=${tok[0]}
|
|
thispath=${tok[1]}
|
|
if [[ -z ${tok[2]} ]]; then
|
|
tags="n/a"
|
|
else
|
|
tags=`echo "${tok[2]}" | sed -e 's/,/ /g'`
|
|
fi
|
|
thisrpath=`getrepopath $thisrepo`
|
|
[[ -z $thisrpath ]] && thisrpath="(default)"
|
|
|
|
printf "$format" "$thisrepo" "$thispath" "$tags" "$thisrpath"
|
|
done
|
|
exit 0
|
|
fi
|
|
|
|
if [[ -z $REPOSTOBACKUP ]]; then
|
|
usage
|
|
exit 1
|
|
fi
|
|
|
|
# Validate repos
|
|
GOODREPOS=""
|
|
for f in $REPOSTOBACKUP; do
|
|
REPO=${f}
|
|
DATAPATH=$(getdatapath $f)
|
|
mode=`getmode "$REPO"`
|
|
|
|
checktag "$f" usmb
|
|
if [[ $? -eq 0 ]]; then
|
|
if [[ -z $USMB_PREFIX ]]; then
|
|
USMBNAME=$f
|
|
else
|
|
if ! [[ $DATAPATH == *${USMB_PREFIX}* ]]; then
|
|
log "Error: path '$DATAPATH' of repo '$REPO' does not contain smb prefix '$USMB_PREFIX'."
|
|
continue
|
|
fi
|
|
USMBNAME=`echo "${DATAPATH}" | sed -e "s,${USMB_PREFIX}/\(.*\).*,\1,"`
|
|
fi
|
|
USEUSMB=1
|
|
else
|
|
USMBNAME=""
|
|
USEUSMB=0
|
|
fi
|
|
|
|
checktag "$f" nfs
|
|
if [[ $? -eq 0 ]]; then
|
|
if [[ -z $DEF_NFS_SERVER ]]; then
|
|
log "Error: repo '$REPO' has nfs tag but \$DEF_NFS_SERVER not defined."
|
|
continue
|
|
fi
|
|
if [[ -z $DEF_NFS_SERVER_BASE ]]; then
|
|
log "Error: repo '$REPO' has nfs tag but \$DEF_NFS_SERVER_BASE not defined."
|
|
continue
|
|
fi
|
|
if ! [[ $DATAPATH == *${NFS_PREFIX}* ]]; then
|
|
log "Error: path '$DATAPATH' of repo '$REPO' does not contain nfs prefix '$NFS_PREFIX'."
|
|
continue
|
|
fi
|
|
NFSNAME=`echo "${DATAPATH}" | sed -e "s,${NFS_PREFIX}/\(.*\).*,\1,"`
|
|
NFSPATH="${DEF_NFS_SERVER}:${DEF_NFS_SERVER_BASE}/${NFSNAME}"
|
|
USENFS=1
|
|
else
|
|
NFSNAME=""
|
|
USENFS=0
|
|
fi
|
|
checktag "$f" localmount
|
|
if [[ $? -eq 0 ]]; then
|
|
grep -q $DTAPATH /etc/fstab
|
|
if [[ $? -ne 0 ]]; then
|
|
log "Error: repo '$REPO' has localmount tag but $DATAPATH not in fstab."
|
|
continue
|
|
fi
|
|
USELOCALMOUNT=1
|
|
else
|
|
USELOCALMOUNT=0
|
|
fi
|
|
|
|
if [[ -z $DATAPATH ]]; then
|
|
log "can't find matching repo for $f - make sure it is listed in ${REPOFILE}"
|
|
continue
|
|
fi
|
|
|
|
if [[ $CMD == "go" ]]; then
|
|
NEEDMOUNT=1
|
|
elif [[ $CMD == "diff" ]]; then
|
|
NEEDMOUNT=1
|
|
else
|
|
NEEDMOUNT=0
|
|
fi
|
|
|
|
if [[ $NEEDMOUNT -eq 1 ]]; then
|
|
if [[ $USEUSMB -eq 1 ]]; then
|
|
mount | grep -q ${DATAPATH}
|
|
if [[ $? -ne 0 ]]; then
|
|
# try mounting it.
|
|
${USMB} ${USMBNAME} >/dev/null
|
|
if [[ $? -eq 0 ]]; then
|
|
log "Mount usmb volume '$USMBNAME': success"
|
|
else
|
|
log "Mount usmb volume '$USMBNAME': FAILED"
|
|
continue
|
|
fi
|
|
fi
|
|
elif [[ $USENFS -eq 1 ]]; then
|
|
mount | grep -q ${DATAPATH}
|
|
if [[ $? -ne 0 ]]; then
|
|
# try mounting it.
|
|
mkdir -p $DATAPATH
|
|
mount_nfs -R 1 ${NFSPATH} ${DATAPATH} >/dev/null
|
|
if [[ $? -eq 0 ]]; then
|
|
log "Mount nfs volume '$NFSPATH' to $DATAPATH: success"
|
|
else
|
|
log "Mount nfs volume '$NFSPATH' to $DATAPATH: FAILED"
|
|
continue
|
|
fi
|
|
fi
|
|
elif [[ $USELOCALMOUNT -eq 1 ]]; then
|
|
mount | grep -q ${DATAPATH}
|
|
if [[ $? -ne 0 ]]; then
|
|
# try mounting it.
|
|
mkdir -p $DATAPATH
|
|
mount ${DATAPATH} >/dev/null
|
|
if [[ $? -eq 0 ]]; then
|
|
log "Mount $DATAPATH: success"
|
|
else
|
|
log "Mount $DATAPATH: FAILED"
|
|
continue
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
if ! [[ -e ${DATAPATH} ]]; then
|
|
log "Error: ${DATAPATH} doesn't exist"
|
|
continue
|
|
fi
|
|
|
|
count=`ls ${DATAPATH} | wc -l`
|
|
if [ $count -le 2 ]; then
|
|
log "Error: ${DATAPATH} exists but appears to be empty"
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
if [[ -z $GOODREPOS ]]; then
|
|
GOODREPOS="$f"
|
|
else
|
|
GOODREPOS="$GOODREPOS $f"
|
|
fi
|
|
done
|
|
|
|
REPOSTOBACKUP="$GOODREPOS"
|
|
if [[ -z $REPOSTOBACKUP ]]; then
|
|
log "Error: errors found with all repos. Aborting."
|
|
exit 1
|
|
fi
|
|
|
|
if [[ $TESTMODE -eq 1 ]]; then
|
|
echo "Would have run '$CMD' on these repos:"
|
|
for f in $REPOSTOBACKUP; do
|
|
DATAPATH=$(getdatapath $f)
|
|
echo " $f -> $DATAPATH"
|
|
if [[ $NEEDMOUNT -eq 1 ]]; then
|
|
do_umount
|
|
fi
|
|
done
|
|
exit 0
|
|
fi
|
|
|
|
errcount=0 # used for 'check' mode
|
|
warncount=0 # used for 'check' mode
|
|
errtext=""
|
|
for f in $REPOSTOBACKUP; do
|
|
REPO=${f}
|
|
DATAPATH=$(getdatapath $REPO)
|
|
checktag "$f" usmb
|
|
if [[ $? -eq 0 ]]; then
|
|
if [[ -z $USMB_PREFIX ]]; then
|
|
USMBNAME=$f
|
|
else
|
|
USMBNAME=`echo "${DATAPATH}" | sed -e "s,${USMB_PREFIX}/\(.*\).*,\1,"`
|
|
fi
|
|
USEUSMB=1
|
|
else
|
|
USMBNAME=""
|
|
USEUSMB=0
|
|
fi
|
|
|
|
checktag "$f" nfs
|
|
if [[ $? -eq 0 ]]; then
|
|
NFSNAME=`echo "${DATAPATH}" | sed -e "s,${NFS_PREFIX}/\(.*\).*,\1,"`
|
|
NFSPATH="${DEF_NFS_SERVER}:${DEF_NFS_SERVER_BASE}/${NFSNAME}"
|
|
USENFS=1
|
|
else
|
|
NFSNAME=""
|
|
USENFS=0
|
|
fi
|
|
|
|
checktag "$f" localmount
|
|
if [[ $? -eq 0 ]]; then
|
|
USELOCALMOUNT=1
|
|
else
|
|
USELOCALMOUNT=0
|
|
fi
|
|
|
|
|
|
export REPO_PATH=`getrepopath "$f"`
|
|
if [[ $? -eq 0 ]]; then
|
|
USINGDEFAULT=0
|
|
else
|
|
USINGDEFAULT=1
|
|
fi
|
|
|
|
if [[ $CMD == "go" ]]; then
|
|
NEEDMOUNT=1
|
|
elif [[ $CMD == "diff" ]]; then
|
|
NEEDMOUNT=1
|
|
else
|
|
NEEDMOUNT=0
|
|
fi
|
|
|
|
|
|
mode=`getmode "$f"`
|
|
|
|
SPEEDOPTS=""
|
|
AUTHOPTS=""
|
|
OTHEROPTS=""
|
|
CONNECTIONOPTS=""
|
|
if [[ $mode == "restic" ]]; then
|
|
MYREPO="restic: $REPO_PATH"
|
|
export RESTIC_REPOSITORY="$REPO_PATH"
|
|
export RESTIC_PASSWORD_FILE=`getrepopassfile "$f"`
|
|
if [[ ! -z $RESTIC_PASSWORD_FILE ]]; then
|
|
unset RESTIC_PASSWORD
|
|
AUTHOPTS="-p ${RESTIC_PASSWORD_FILE}"
|
|
else
|
|
AUTHOPTS="-p ${DEF_AUTHFILE}"
|
|
fi
|
|
if [[ -z $KSPEED ]]; then
|
|
SPEEDOPTS=""
|
|
else
|
|
SPEEDOPTS="--limit-upload $KSPEED"
|
|
fi
|
|
|
|
if [[ -z $CONNECTIONS ]]; then
|
|
CONNECTIONSOPTS=""
|
|
else
|
|
CONNECTIONSOPTS="-o b2.connections=${CONNECTIONS}"
|
|
fi
|
|
if [[ $VERBOSE -eq 1 ]]; then
|
|
OTHEROPTS="$OTHEROPTS -v"
|
|
fi
|
|
|
|
export RESTIC_EXCLUDEFILE=`getrepoexcludefile "$f"`
|
|
if [[ ! -z $RESTIC_PASSWORD_FILE ]]; then
|
|
OTHEROPTS="$OTHEROPTS --exclude-file=${RESTIC_EXCLUDEFILE}"
|
|
fi
|
|
OTHEROPTS="$OTHEROPTS $RESTICOPTS"
|
|
elif [[ $mode == "rsync" ]]; then
|
|
# rsync
|
|
MYREPO="rsync: $REPO_PATH"
|
|
export RSYNC_REPOSITORY="$REPO_PATH"
|
|
export RSYNC_FULLDIR="${DEF_RSYNC_DIR}/${REPO}"
|
|
if [[ $USINGDEFAULT -eq 1 ]]; then
|
|
list="DEF_RSYNC_SERVER DEF_RSYNC_USER DEF_RSYNC_DIR RSYNC_OPTIONS"
|
|
else
|
|
list="RSYNC_OPTIONS"
|
|
fi
|
|
|
|
for xx in $list; do
|
|
eval val='$'$xx
|
|
if [[ -z $val ]]; then
|
|
echo "Error - \$$xx not set. Please update ${RCFILE}."
|
|
echo ""
|
|
usage-rc
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
if [[ -z $KSPEED ]]; then
|
|
SPEEDOPTS=""
|
|
else
|
|
SPEEDOPTS="--bwlimit=${KSPEED}k"
|
|
fi
|
|
CONNECTIONSOPTS=""
|
|
else
|
|
# rclone
|
|
MYREPO="rclone: $REPO_PATH"
|
|
export RCLONE_REPOSITORY="$REPO_PATH"
|
|
if [[ -z $KSPEED ]]; then
|
|
SPEEDOPTS=""
|
|
else
|
|
SPEEDOPTS="--bwlimit=${KSPEED}k"
|
|
fi
|
|
|
|
if [[ -z $CONNECTIONS ]]; then
|
|
CONNECTIONSOPTS=""
|
|
else
|
|
CONNECTIONSOPTS="--transfers=${CONNECTIONS}"
|
|
fi
|
|
fi
|
|
|
|
|
|
if [[ ! $CMD == "check" ]]; then
|
|
text="Starting '$CMD' on repo '$REPO' [$DATAPATH <-> $MYREPO]"
|
|
log "$text"
|
|
fi
|
|
if [[ $CMD == "check" ]]; then
|
|
line=`egrep "^$REPO:go:" $STATFILE 2>/dev/null`
|
|
if [[ $? -eq 0 ]]; then
|
|
# Get result
|
|
code=`echo "$line" | cut -d: -f3`
|
|
stamp=`echo "$line" | cut -d: -f4`
|
|
date --version 2>&1 | grep -q GNU
|
|
if [[ $? -eq 0 ]]; then
|
|
humanstamp=`date --date="@$stamp"`
|
|
else
|
|
humanstamp=`date -r $stamp`
|
|
fi
|
|
now=`date +%s`
|
|
age=$( echo "$now - $stamp" | bc )
|
|
oneday=$( echo "24 * 60 * 60" | bc)
|
|
if [[ $code -eq 0 ]]; then
|
|
if [[ $age -le $oneday ]]; then
|
|
echo "OK: Last backup for '$REPO' succeeded on $humanstamp."
|
|
rv=0
|
|
else
|
|
echo "CRITICAL: Last backup for '$REPO' succeeded on $humanstamp (age $age not in last 24 hours)."
|
|
errcount=$(( $errcount + 1))
|
|
rv=2
|
|
fi
|
|
else
|
|
echo "CRITICAL: Last backup for '$REPO' failed on $humanstamp."
|
|
rv=2
|
|
errcount=$(( $errcount + 1))
|
|
fi
|
|
else
|
|
echo "WARNING: No history found for repo '$REPO'"
|
|
rv=1
|
|
warncount=$(( $warncount + 1))
|
|
fi
|
|
elif [[ $mode == "restic" ]]; then
|
|
if [[ $CMD == "go" ]]; then
|
|
if [[ $CRONMODE -eq 1 ]]; then
|
|
${RESTIC} backup $AUTHOPTS $CONNECTIONSOPTS $SPEEDOPTS $OTHEROPTS $DATAPATH 2>&1 | egrep "^(Added|processed|snapshot)" >> ${LOG}
|
|
else
|
|
${RESTIC} backup $AUTHOPTS $CONNECTIONSOPTS $SPEEDOPTS $OTHEROPTS $DATAPATH 2>&1 >> ${LOG}
|
|
fi
|
|
rv=$?
|
|
elif [[ $CMD == "ls" ]]; then
|
|
${RESTIC} snapshots $AUTHOPTS $CONNECTIONSOPTS 2>&1 >> ${LOG}
|
|
rv=$?
|
|
elif [[ $CMD == "stats" ]]; then
|
|
${RESTIC} stats $AUTHOPTS $CONNECTIONSOPTS 2>&1 >> ${LOG}
|
|
rv=$?
|
|
elif [[ $CMD == "diff" ]]; then
|
|
log "Error: diff not supported in restic."
|
|
rv=1
|
|
elif [[ $CMD == "init" ]]; then
|
|
# check whether it already exists first!
|
|
${RESTIC} stats $AUTHOPTS $CONNECTIONSOPTS >/dev/null 2>&1
|
|
if [ $? -eq 0 ]; then
|
|
log "ERROR: Repo ${REPO} already exists. Aborting."
|
|
rv=1
|
|
else
|
|
cmdtorun="${RESTIC} init $AUTHOPTS $CONNECTIONSOPTS"
|
|
if [[ $VERBOSE -eq 1 ]]; then
|
|
|
|
|
|
|
|
log "export RESTIC_REPOSITORY=\"$REPO_PATH\""
|
|
log "export RESTIC_PASSWORD_FILE=`getrepopassfile \"$f\"`"
|
|
log "will run: [${cmdtorun}]"
|
|
fi
|
|
errtext=`${RESTIC} init $AUTHOPTS $CONNECTIONSOPTS 2>&1`
|
|
rv=$?
|
|
fi
|
|
elif [[ $CMD == "forget" ]]; then
|
|
# age out stuff after RESTIC_KEEPDAYS days
|
|
FORGETOPTS="-d ${RESTIC_KEEPDAYS}"
|
|
${RESTIC} forget $AUTHOPTS $CONNECTIONSOPTS $FORGETOPTS 2>&1 | awk '($1 == "Applying") { print } ($2 == "snapshots") { show=0 } (show == 1 && $1 != "ID" && NF > 1) { print "Removing: " $0 } ($1 == "remove") { show=1 }' >> ${LOG}
|
|
rv=$?
|
|
${RESTIC} prune $AUTHOPTS 2>&1 | egrep "^(repository contain|found|will|remove)" >> ${LOG}
|
|
else
|
|
log "Error: invalid command $CMD"
|
|
rv=1
|
|
fi
|
|
elif [[ $mode == "rsync" ]]; then
|
|
REMOTE=""
|
|
if [[ $CMD == "go" ]]; then
|
|
#echo run ${RSYNC} ${RSYNC_OPTIONS} $DATAPATH/ ${RSYNC_REPOSITORY} 2>&1 >> ${LOG}
|
|
#exit 1
|
|
${RSYNC} ${RSYNC_OPTIONS} $DATAPATH/ ${RSYNC_REPOSITORY} 2>&1 >> ${LOG}
|
|
rv=$?
|
|
elif [[ $CMD == "ls" ]]; then
|
|
ssh ${DEF_RSYNC_USER}@${DEF_RSYNC_SERVER} ls ${RSYNC_FULLDIR} 2>&1 >> ${LOG}
|
|
rv=$?
|
|
elif [[ $CMD == "stats" ]]; then
|
|
echo "Size of remote data for ${REPO}:"
|
|
ssh ${DEF_RSYNC_USER}@${DEF_RSYNC_SERVER} du -hs ${RSYNC_FULLDIR} 2>&1 >> ${LOG}
|
|
rv=$?
|
|
elif [[ $CMD == "diff" ]]; then
|
|
${RSYNC} ${RSYNC_OPTIONS} -n $DATAPATH/ ${RSYNC_REPOSITORY} 2>&1 >> ${LOG}
|
|
rv=$?
|
|
elif [[ $CMD == "init" ]]; then
|
|
# check whether it already exists first!
|
|
ssh ${DEF_RSYNC_USER}@${DEF_RSYNC_SERVER} ls -d ${RSYNC_FULLDIR} 2>&1 >> ${LOG}
|
|
if [ $? -eq 0 ]; then
|
|
log "ERROR: Repo ${REPO} already exists. Aborting."
|
|
else
|
|
ssh ${DEF_RSYNC_USER}@${DEF_RSYNC_SERVER} mkdir -p ${RSYNC_FULLDIR} 2>&1 >>${LOG}
|
|
if [ $? -eq 0 ]; then
|
|
log "Created directory ${RSYNC_FULLDIR}..."
|
|
else
|
|
log "Creation of ${RSYNC_FULLDIR} failed."
|
|
fi
|
|
fi
|
|
elif [[ $CMD == "forget" ]]; then
|
|
log "Error: forget not supported for rsync mode."
|
|
rv=1
|
|
else
|
|
log "Error: invalid command $CMD"
|
|
rv=1
|
|
fi
|
|
else
|
|
# rclone
|
|
REMOTE=""
|
|
if [[ $CMD == "go" ]]; then
|
|
${RCLONE} sync $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $OTHEROPTS $DATAPATH $RCLONE_REPOSITORY 2>&1 >> ${LOG}
|
|
rv=$?
|
|
elif [[ $CMD == "ls" ]]; then
|
|
${RCLONE} ls $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $OTHEROPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
|
|
rv=$?
|
|
elif [[ $CMD == "stats" ]]; then
|
|
${RCLONE} size $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $OTHEROPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
|
|
rv=$?
|
|
elif [[ $CMD == "diff" ]]; then
|
|
${RCLONE} check $RCLONEOPTS $CONNECTIONSOPTS $SPEEDOPTS $OTHEROPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
|
|
rv=$?
|
|
elif [[ $CMD == "init" ]]; then
|
|
# check whether it already exists first!
|
|
${RCLONE} size $RCLONEOPTS $RCLONE_REPOSITORY >/dev/null 2>&1
|
|
if [ $? -eq 0 ]; then
|
|
log "ERROR: Repo ${REPO} already exists. Aborting."
|
|
else
|
|
# does 'remote' exist?
|
|
${RCLONE} listremotes | grep -q remote: >/dev/null 2>&1
|
|
rv=$?
|
|
if [ $rv -ne 0 ]; then
|
|
log "Rclone remote doesn't exist - creating it..."
|
|
${RCLONE} $RCLONEOPTS config create remote b2 account $B2_APP_ID key $B2_APP_KEY >>${LOG}
|
|
rv=$?
|
|
|
|
if [ $rv -ne 0 ]; then
|
|
log "Rclone remote init failed."
|
|
exit 1
|
|
fi
|
|
|
|
fi
|
|
|
|
if [ $rv -eq 0 ]; then
|
|
log "Creating B2 bucket for $RCLONE_REPOSITORY..."
|
|
# create the bucket
|
|
${RCLONE} $RCLONEOPTS mkdir $RCLONE_REPOSITORY >>${LOG}
|
|
else
|
|
log "Rclone bucket creation of $RCLONE_REPOSITORY failed."
|
|
fi
|
|
fi
|
|
elif [[ $CMD == "forget" ]]; then
|
|
log "Error: forget not supported for rclone mode."
|
|
rv=1
|
|
else
|
|
log "Error: invalid command $CMD"
|
|
rv=1
|
|
fi
|
|
fi
|
|
|
|
if [[ ! $CMD == "check" ]]; then
|
|
# Record result
|
|
updatestats $f $CMD $rv
|
|
fi
|
|
|
|
if [[ $rv -eq 0 ]]; then
|
|
if [[ $NEEDMOUNT -eq 1 ]]; then
|
|
do_umount
|
|
fi
|
|
elif [[ ! $CMD == "check" ]]; then
|
|
log "Error: '$CMD' on repo '$REPO' failed"
|
|
if [[ ! -z $errtext ]]; then
|
|
log "$errtext"
|
|
fi
|
|
fi
|
|
|
|
if [[ ! $CMD == "check" ]]; then
|
|
log "Finished '$CMD' on repo '$REPO'"
|
|
fi
|
|
done
|
|
|
|
if [[ $CMD == "check" ]]; then
|
|
if [[ $errcount -ge 1 ]]; then
|
|
rv=2
|
|
elif [[ $warncount -ge 1 ]]; then
|
|
rv=1
|
|
else
|
|
rv=0
|
|
fi
|
|
exit $rv
|
|
fi
|