369 lines
7.5 KiB
Bash
Executable File
369 lines
7.5 KiB
Bash
Executable File
#!/bin/bash
|
|
VALIDCOMMANDS="go ls init repos stats diff"
|
|
|
|
RCFILE=${HOME}/.backup/config
|
|
AUTHFILE=${HOME}/.backup/auth
|
|
REPOFILE=${HOME}/.backup/repos
|
|
LOGFILE=/var/log/backup.log
|
|
|
|
RESTIC=/usr/local/bin/restic
|
|
RCLONE=/usr/local/bin/rclone
|
|
|
|
SPEED=""
|
|
CRONMODE=0
|
|
CONNECTIONS=20
|
|
LOG=/dev/stdout
|
|
DATE=/bin/date
|
|
DOALL=0
|
|
REPOSTOBACKUP=""
|
|
|
|
function log() {
|
|
local now
|
|
now=`${DATE} +%Y/%m/%d-%H:%M:%S`
|
|
echo "${now} <${mode}> $*" >>${LOG}
|
|
}
|
|
|
|
function usage-repo() {
|
|
echo "Should be of this format:"
|
|
echo " reponame1:/path/to/files1/"
|
|
echo " reponame2:/path/to/files2/"
|
|
echo " ..."
|
|
echo " reponameX:/path/to/filesX/"
|
|
}
|
|
function usage-rc() {
|
|
echo "Should be of this format:"
|
|
echo " export B2_ACCOUNT_ID=xxx"
|
|
echo " export B2_ACCOUNT_KEY=xxx"
|
|
echo " export B2_APP_ID=xxx"
|
|
echo " export B2_APP_KEY=xxx"
|
|
echo " export RESTIC_PASSWORD=xxx"
|
|
}
|
|
|
|
function usage() {
|
|
echo "usage: $0 command reponame"
|
|
echo ""
|
|
echo " -a Run on all repos defined as 'auto'"
|
|
echo " -h Show this usage text"
|
|
echo " -s num Limit speed to 'num' kBps (default: unlimited)"
|
|
echo " -x num Use 'num' simultaneous connections (default: 20)"
|
|
echo " -c Cron mode - log to ${LOGFILE}"
|
|
echo " -t Test mode - dump what would be done then exit."
|
|
echo ""
|
|
echo "Valid commands are:"
|
|
echo " $VALIDCOMMANDS"
|
|
echo ""
|
|
echo "${RCFILE} should look like this:"
|
|
usage-rc
|
|
echo ""
|
|
echo "${REPOFILE} should look like this:"
|
|
usage-repo
|
|
echo ""
|
|
}
|
|
function getdatapath() {
|
|
local DATAPATH REPO x
|
|
REPO="$1"
|
|
|
|
DATAPATH=""
|
|
for x in ${REPODEFS[@]}; do
|
|
match="^${REPO}:"
|
|
if [[ $x =~ $match ]]; then
|
|
DATAPATH=`echo $x | sed -e s/${match}//`
|
|
fi
|
|
done
|
|
DATAPATH=`echo $DATAPATH | sed -e s/:.*//`
|
|
echo "$DATAPATH"
|
|
}
|
|
function checktag() { # return 0 if tag matches
|
|
local reponame lookfor x match repodef
|
|
reponame="$1"
|
|
lookfor="$2"
|
|
|
|
# if we were just given a repo name, look up the def
|
|
if [[ $reponame =~ ^.*:.*$ ]]; then
|
|
repodef="$reponame"
|
|
else
|
|
repodef=""
|
|
for x in ${REPODEFS[@]}; do
|
|
match="^${reponame}:"
|
|
if [[ $x =~ $match ]]; then
|
|
repodef="$x"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
|
|
if [[ $repodef =~ ^.*:.*:.*${lookfor}.*$ ]]; then
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
# Handle args
|
|
ARGS="acdhs:tu:x:"
|
|
|
|
while getopts "$ARGS" i; do
|
|
case "$i" in
|
|
a)
|
|
DOALL=1
|
|
;;
|
|
h)
|
|
usage;
|
|
exit 1
|
|
;;
|
|
s)
|
|
SPEED="$OPTARG";
|
|
;;
|
|
x)
|
|
CONNECTIONS="$OPTARG";
|
|
;;
|
|
c)
|
|
CRONMODE=1;
|
|
LOG=${LOGFILE}
|
|
;;
|
|
t)
|
|
TESTMODE=1
|
|
;;
|
|
*)
|
|
echo "ERROR: invalid argument: $i";
|
|
usage;
|
|
;;
|
|
esac
|
|
done
|
|
shift $((OPTIND - 1))
|
|
|
|
if ! [ -e ${RCFILE} ]; then
|
|
echo "Error - can't find ${RCFILE}"
|
|
echo ""
|
|
usage-rc
|
|
exit 1
|
|
fi
|
|
if ! [ -e ${REPOFILE} ]; then
|
|
echo "Error - can't find ${REPOFILE}"
|
|
echo ""
|
|
usage-repo
|
|
exit 1
|
|
fi
|
|
|
|
. ${RCFILE}
|
|
|
|
|
|
if [[ -z $B2_APP_ID ]]; then
|
|
echo "Error - \$B2_APP_ID not set."
|
|
exit 1
|
|
fi
|
|
if [[ -z $B2_APP_KEY ]]; then
|
|
echo "Error - \$B2_APP_KEY not set."
|
|
exit 1
|
|
fi
|
|
|
|
|
|
|
|
# Get list of defined repos
|
|
idx=0
|
|
for f in `cat $REPOFILE`; do
|
|
REPODEFS[$idx]="$f"
|
|
if [[ $DOALL -eq 1 ]]; then
|
|
checktag "$f" auto
|
|
if [[ $? -eq 0 ]]; then
|
|
REPOSTOBACKUP="$REPOSTOBACKUP `echo $f | sed -e 's/:.*//'`"
|
|
fi
|
|
fi
|
|
idx=$((idx + 1))
|
|
done
|
|
|
|
if ! [[ $VALIDCOMMANDS == *"$CMD"* ]]; then
|
|
echo "Error - invalid command $CMD. Should one of: $VALIDCOMMANDS"
|
|
exit 1
|
|
fi
|
|
|
|
CMD="$1"
|
|
shift 1
|
|
|
|
if [[ -z $REPOSTOBACKUP ]]; then
|
|
REPOSTOBACKUP="$*"
|
|
fi
|
|
|
|
if [[ $CMD == "repos" ]]; then
|
|
format="%-20s%-40s%-10s\n"
|
|
echo "Repos defined in ${RCFILE}:"
|
|
echo ""
|
|
printf "$format" "Repo" "Path" "Tags"
|
|
for x in ${REPODEFS[@]}; do
|
|
|
|
IFS=':' read -ra tok <<< "$x"
|
|
thisrepo=${tok[0]}
|
|
thispath=${tok[1]}
|
|
if [[ -z ${tok[2]} ]]; then
|
|
tags="n/a"
|
|
else
|
|
tags=`echo "${tok[2]}" | sed -e 's/,/ /'`
|
|
fi
|
|
|
|
printf "$format" "$thisrepo" "$thispath" "$tags"
|
|
done
|
|
exit 0
|
|
fi
|
|
|
|
if [[ -z $REPOSTOBACKUP ]]; then
|
|
usage
|
|
exit 1
|
|
fi
|
|
|
|
# Validate repos
|
|
for f in $REPOSTOBACKUP; do
|
|
DATAPATH=$(getdatapath $f)
|
|
|
|
if [[ -z $DATAPATH ]]; then
|
|
log "can't find matching repo for $f - make sure it is listed in ${RCFILE}"
|
|
exit 1
|
|
fi
|
|
|
|
if ! [[ -e ${DATAPATH} ]]; then
|
|
log "Error: ${DATAPATH} doesn't exist"
|
|
exit 1
|
|
fi
|
|
|
|
count=`ls ${DATAPATH} | wc -l`
|
|
if [ $count -le 2 ]; then
|
|
log "Error: ${DATAPATH} exists but appears to be empty"
|
|
exit 1
|
|
fi
|
|
|
|
done
|
|
|
|
if [[ $TESTMODE -eq 1 ]]; then
|
|
echo "Would have run '$CMD' on these repos:"
|
|
for f in $REPOSTOBACKUP; do
|
|
DATAPATH=$(getdatapath $f)
|
|
echo " $f -> $DATAPATH"
|
|
done
|
|
exit 0
|
|
fi
|
|
|
|
for f in $REPOSTOBACKUP; do
|
|
REPO=${f}
|
|
DATAPATH=$(getdatapath $REPO)
|
|
|
|
export RESTIC_REPOSITORY="b2:nethack-${REPO}"
|
|
export RCLONE_REPOSITORY="remote:nethack-${REPO}"
|
|
|
|
checktag "$f" rclone
|
|
if [[ $? -eq 0 ]]; then
|
|
mode=rclone
|
|
else
|
|
mode=restic
|
|
fi
|
|
|
|
if [[ $mode == "restic" ]]; then
|
|
if [[ -z $SPEED ]]; then
|
|
SPEEDOPTS=""
|
|
else
|
|
SPEEDOPTS="--limit-upload $SPEED"
|
|
fi
|
|
|
|
if [[ -z $CONNECTIONS ]]; then
|
|
CONNECTIONSOPTS=""
|
|
else
|
|
CONNECTIONSOPTS="-o b2.connections=${CONNECTIONS}"
|
|
fi
|
|
|
|
AUTHOPTS="-p ${AUTHFILE}"
|
|
else
|
|
# rclone
|
|
if [[ -z $SPEED ]]; then
|
|
SPEEDOPTS=""
|
|
else
|
|
SPEEDOPTS="--bwlimit=${SPEED}k"
|
|
fi
|
|
|
|
if [[ -z $CONNECTIONS ]]; then
|
|
CONNECTIONSOPTS=""
|
|
else
|
|
CONNECTIONSOPTS="--transfers=${CONNECTIONS}"
|
|
fi
|
|
fi
|
|
|
|
log "Starting '$CMD' on repo '$REPO' [$DATAPATH]"
|
|
|
|
|
|
if [[ $mode == "restic" ]]; then
|
|
if [[ $CMD == "go" ]]; then
|
|
${RESTIC} backup $AUTHOPTS $CONNECTIONSOPTS $SPEEDOPTS $DATAPATH 2>&1 >> ${LOG}
|
|
rv=$?
|
|
elif [[ $CMD == "ls" ]]; then
|
|
${RESTIC} snapshots $AUTHOPTS $CONNECTIONSOPTS 2>&1 >> ${LOG}
|
|
rv=$?
|
|
elif [[ $CMD == "stats" ]]; then
|
|
${RESTIC} stats $AUTHOPTS $CONNECTIONSOPTS 2>&1 >> ${LOG}
|
|
rv=$?
|
|
elif [[ $CMD == "diff" ]]; then
|
|
log "Error: diff not supported in restic."
|
|
rv=1
|
|
elif [[ $CMD == "init" ]]; then
|
|
# check whether it already exists first!
|
|
${RESTIC} stats $AUTHOPTS $CONNECTIONSOPTS >/dev/null 2>&1
|
|
if [ $? -eq 0 ]; then
|
|
log "ERROR: Repo ${REPO} already exists. Aborting."
|
|
else
|
|
${RESTIC} init $AUTHOPTS $CONNECTIONSOPTS >/dev/null 2>&1
|
|
fi
|
|
else
|
|
log "Error: invalid command $CMD"
|
|
rv=0
|
|
fi
|
|
else
|
|
# rclone
|
|
REMOTE=""
|
|
if [[ $CMD == "go" ]]; then
|
|
${RCLONE} sync $CONNECTIONSOPTS $SPEEDOPTS $DATAPATH $RCLONE_REPOSITORY 2>&1 >> ${LOG}
|
|
rv=$?
|
|
elif [[ $CMD == "ls" ]]; then
|
|
${RCLONE} ls $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
|
|
rv=$?
|
|
elif [[ $CMD == "stats" ]]; then
|
|
${RCLONE} size $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
|
|
rv=$?
|
|
elif [[ $CMD == "diff" ]]; then
|
|
${RCLONE} check $CONNECTIONSOPTS $SPEEDOPTS $RCLONE_REPOSITORY 2>&1 >> ${LOG}
|
|
rv=$?
|
|
elif [[ $CMD == "init" ]]; then
|
|
# check whether it already exists first!
|
|
${RCLONE} size $RCLONE_REPOSITORY >/dev/null 2>&1
|
|
if [ $? -eq 0 ]; then
|
|
log "ERROR: Repo ${REPO} already exists. Aborting."
|
|
else
|
|
# does 'remote' exist?
|
|
${RCLONE} listremotes | grep -q remote: >/dev/null 2>&1
|
|
rv=$?
|
|
if [ $rv -ne 0 ]; then
|
|
log "Rclone remote doesn't exist - creating it..."
|
|
${RCLONE} config create remote b2 account $B2_APP_ID key $B2_APP_KEY >>${LOG}
|
|
rv=$?
|
|
if [ $rv -ne 0 ]; then
|
|
log "Rclone remote init failed."
|
|
exit 1
|
|
fi
|
|
|
|
fi
|
|
|
|
if [ $rv -eq 0 ]; then
|
|
log "Creating B2 bucket for $RCLONE_REPOSITORY..."
|
|
# create the bucket
|
|
${RCLONE} mkdir $RCLONE_REPOSITORY >>${LOG}
|
|
else
|
|
log "Rclone bucket creation of $RCLONE_REPOSITORY failed."
|
|
fi
|
|
fi
|
|
else
|
|
log "Error: invalid command $CMD"
|
|
rv=0
|
|
fi
|
|
fi
|
|
|
|
if [[ $rv -ne 0 ]]; then
|
|
echo "Error: command failed."
|
|
fi
|
|
|
|
log "Finished '$CMD' on repo '$REPO'"
|
|
done
|