#! /bin/bash -E

## @id $Id$
##       1         2         3         4         5         6         7         8
## 45678901234567890123456789012345678901234567890123456789012345678901234567890

#set -x

# Simple setup of a list of hosts (internal and external):
#
# setup-backup dev.swisssign.com dev0001 dev0002 dev0004 devmac0001
#
# What the setup script really does is:
#
# HOSTS="dev_swisssign_com dev0001 dev0002 dev0004 devmac0001"
# for period in daily weekly monthly; do
#   cd /etc/cron.$period;
#   for h in $HOSTS; do
#     sudo ln -s $0 backup-$h;
#   done;
# done
 
################################################################################
# Configuration (read from @sysconfdir@/backup.conf and ~/.backup
# or host specific backup from backup-HOST.conf and .backup-HOST
################################################################################

# disable: dont' start backup if a certain file exists
DISABLE=${DISABLE:-/var/run/nobackup}
if [ -e ${DISABLE} ]; then
    return
fi

# get host name from filename (name files with _ instead of .)
# e.g.
#   $0 (this file name) is backup-dev_marc_waeckerlin_org
#   $HOST becomes dev.marc.waeckerlin.org
# you can test for $HOST in /etc/backup.conf or ~/.backup.conf
HOST=${HOST:-$(basename ${0##*/backup-} .sh | sed 's/_/./g')}

# Get defaults from a file /etc/backup.conf or ~/.backup.conf
# file is bash syntax and shall define variables, i.e.
#
#   EMAIL="me@freemail.com"
#   NUM=2
#   TARGET="/media/backup"
#   MORE_ARGS="--exclude atheismus.ch_error_log --exclude marc_error_log"
#
prefix="@prefix@"
test -e "@sysconfdir@/backup.conf" && . "@sysconfdir@/backup.conf"
test -e ~/.backup && . ~/.backup
test -e "@sysconfdir@/backup-$HOST.conf" && . "@sysconfdir@/backup-$HOST.conf"
test -e ~/".backup-$HOST" && . ~/".backup-$HOST"

# E-Mail to report errors
EMAIL=${EMAIL:-""}

# Number of backups to keep per type
NUM=${NUM:-5}

# Target to backup to
TARGET=${TARGET:-"/var/backup"}

# The following variables are detected automatically
# and setup with good defaulte
# You rarely need to change them

# get name of host to backup from file name
# e.g. /var/cron/backup-my-host-com backups from url my.host.com
# if your hostnaim contains dashes '-', then you're lost
# the keyword 'backup-generic' defaults to this host here
if [ "$HOST" = "generic" -o -z "$HOST" ]; then
    HOST=$HOSTNAME
    HOSTADDR=${HOSTADDR:-""}
    BACKUP=${BACKUP:-""}
else
    BACKUP=${BACKUP:-"-e ssh"}
    HOSTADDR=${HOSTADDR:-"${HOST}:"}
fi
SOURCES=${SOURCES:-$(ssh $HOST mount | awk '/^\// && !/'${TARGET//\//\\\/}'/ && $2=="on" && $6!~/bind/ {print $3}')}
LOGFILE=${LOGFILE:-"/var/log/backup.log"}
LOCK=${LOCK:-${TARGET}/${HOST}.lock}
TMPFILE=${TMPFILE:-/tmp/${0##*/}.$$}
RSYNC_CMD=${RSYNC_CMD:-"nice -n 19 ionice -c 2 -n 7 rsync"}
LOCAL_RSYNC_CMD=${LOCAL_RSYNC_CMD:-$RSYNC_CMD}
REMOTE_RSYNC_CMD=${REMOTE_RSYNC_CMD:-$RSYNC_CMD}
ARGS=${ARGS:-"-aqx --delete"}
ARGS="$ARGS --rsync-path='${REMOTE_RSYNC_CMD}' $BACKUP --exclude ${TARGET} ${MORE_ARGS}"
################################################################################

function error() {
    if test -f "${TMPFILE}"; then
        if test -n "$EMAIL"; then
            mail -s "Backup ERROR: $0:$1 ret:$2" "${EMAIL}" < "${TMPFILE}"
        fi
        rm "${TMPFILE}" 2> /dev/null || true
        echo "$(date) $0 $$ ******** Backup ERROR: $0:$1 ret:$2" >> "${LOGFILE}"
    else
        if test -n "$EMAIL"; then
            echo "Aborted without logfile $TMPFILE" | \
                mail -s "Backup ERROR: $0:$1 ret:$2" "${EMAIL}"
        fi
        echo "$(date) $0 $$ ******** Backup ERROR: $0:$1 ret:$2" >> "${LOGFILE}"
    fi
    exit 1
}

trap 'error ${LINENO} $?' ERR

umask 022
export PATH=/sbin:/bin:/usr/sbin:/usr/bin

# check rsync capabilities
RSYNC_VERSION=$(rsync --version | sed -n 's,.*rsync  version \([0-9.]*\).*,\1,p' | awk -F. '{printf "%04d%04d", $1, $2}')
if [ $RSYNC_VERSION -ge 00020006 ]; then
    HAS_MULTIDEST=1
else
    HAS_MULTIDEST=0
fi

(

echo "$(date) $0 $$ ########################################################"

# Prüfen, ob Server erreichbar ist
if ! ping -q -w 5 -c 1 $HOST 2> /dev/null > /dev/null; then
    echo  "$(date) $0 Server $HOST ist nicht erreichbar!" 1>&2
    error
    exit 2
else
    echo "$(date) $0 $$ Server $HOST is reachable"
fi

# locking
echo "$(date) $0 $$ ******** Aquire lock for $HOST from $0"
flock -x 200
echo "$(date) $0 $$ ******** Got lock for $HOST from $0"
 
## Prüfen, ob Sicherungsmedium vorhanden ist
if [ "$TARGET" != "${TARGET#/media/}" ]; then
    if ! (mount | grep "$TARGET" 2> /dev/null > /dev/null); then
        echo  "$(date) $0 Die Partition $TARGET ist nicht eingebunden!" 1>&2
        error
        exit 2
    fi
else
    if ! test -d "${TARGET}"; then
        echo  "$(date) $0 Der Sicherungspfad $TARGET ist nicht vorhanden!" 1>&2
        error
        exit 2
    fi
fi

echo "$(date) $0 $$ Backup ${HOST} to ${TARGET}"

FULL=$(LANG= df -P ${TARGET} | awk '$1 !~ /^File/ {print $5}' | tr -d '%' | tail -1)
echo "$(date) $0 $$ Füllgrad: ${FULL}%"
if [ "$FULL" -gt 80 ]; then
    if [ "$FULL" -gt 98 ]; then
        echo  "$(date) $0 **** Alarm! **** ${TARGET} ist $FULL% voll" 1>&2
        error
        exit 2
    fi
    echo "$(date) $0 $$ **** Warnung! **** ${TARGET} ist $FULL% voll"
fi

# Sicherungstyp eruieren
case "$(pwd)/$0" in
    *hourly*) TYPE="hourly";;
    *daily*) TYPE="daily";;
    *weekly*) TYPE="weekly";;
    *monthly*) TYPE="monthly";;
    *) TYPE="manual";;
esac

# 3. Was ist die Referenz?
#
# Die Sicherung macht Hardlinks entweder zu hourly.1 oder wenn der nicht
# existiert zu daily.1 oder wenn der nicht existiert zu manual.1
#
# Die Idee ist, immer zum aktuellsten zu verlinken. Auf einem System
# mit regelmässigem Backup ist das gestern, auf einem System mit
# Backup von Hand, ist das das letzte manuelle. (Beispiel: Mein Server
# hat ein tägliches Backup, mein Notebook ein manuelles: Automatisch
# wenn ich es am USB angehängt habe.)
#
# Prüfung daily oder manual?
#   a) gibt es den hourly.1? -> hourly
#   b) gibt es den daily.1? -> daily
#   c) gibt es manual.1? -> manual
#   d) sonst              -> Was ich bin...
if [ -e ${TARGET}/${HOST}-hourly.1 ]; then
  echo "$(date) $0 $$ **** Found an hourly backup for reference"
  REFTYPE=hourly
elif [ -e ${TARGET}/${HOST}-daily.1 ]; then
  echo "$(date) $0 $$ **** Found an daily backup for reference"
  REFTYPE=daily
elif [ -e ${TARGET}/${HOST}-manual.1 ]; then
  echo "$(date) $0 $$ **** Found an manual backup for reference"
  REFTYPE=manual
else
  echo "$(date) $0 $$ **** No reference found, try $TYPE"
  REFTYPE=$TYPE
fi

REFNUM=1
echo "$(date) $0 $$ **** SOURCES:" $SOURCES

# 4b. Sonst aktuelle Sicherungskopie gegen neueste Sicherung anlegen
for S in ${SOURCES}; do
    SRC=${S%/}/
    
    # Detect possible references, prefered (from above) first
    REFERENCES=
    for dest in ${TARGET}/${HOST}-${REFTYPE}.${REFNUM}${SRC} \
                ${TARGET}/${HOST}-${REFTYPE}.[0-9]*${SRC} \
                ${TARGET}/${HOST}-{hourly,daily,weekly,monthly}.${REFNUM}${SRC} \
                ${TARGET}/*-${REFTYPE}.${REFNUM}${SRC}; do
        if [ -d ${dest} -a "${REFERENCES//$dest/}" = "${REFERENCES}" ]; then
            REFERENCES="${REFERENCES} $dest"
            if [ $HAS_MULTIDEST -eq 0 ]; then
                break;
            fi
        fi
    done
    DESTINATIONS="${REFERENCES// / --link-dest=}"
    echo "$(date) $0 $$ **** REFERENCES: ${REFERENCES}"
    if [ -d ${TARGET}/${HOST}-${TYPE}.new${SRC} ]; then
        rm -r ${TARGET}/${HOST}-${TYPE}.new${SRC}
    fi
    CMD="${LOCAL_RSYNC_CMD} ${ARGS} ${DESTINATIONS} ${HOSTADDR}${SRC%/}/ ${TARGET}/${HOST}-${TYPE}.new${SRC%/}/"
    echo "$(date) $0 $$ **** BACKUP: ${CMD}"
    bash -c "$CMD" || true
    RES=${PIPESTATUS[0]}
    if [ $RES -eq 0 ]; then
        echo "$(date) $0 $$ **** SUCCESS: ${CMD}"
    elif [ $RES -eq 24 ]; then
        echo "$(date) $0 $$ **** WARNING: ${CMD}"
        mail -s "Backup Warning: $0" "${EMAIL}" < "${TMPFILE}"
    else
        echo  "$(date) $0 **** ERROR $RES: ${CMD}" 1>&2
        error
        exit 2
    fi
done
touch ${TARGET}/${HOST}-${TYPE}.new/backup-timestamp
 
# 5. Altern
if [ -d "${TARGET}/${HOST}-${TYPE}.1" ]; then
# 1. Älteste Sicherungskopie löschen
    if [ -d "${TARGET}/${HOST}-${TYPE}.${NUM}" ]; then
        echo "$(date) $0 $$ **** REMOVE: ${TARGET}/${HOST}-${TYPE}.${NUM}"
        rm -rf "${TARGET}/${HOST}-${TYPE}.${NUM}"
    fi
# 2. Bestehende Sicherungskopien um eine Stufe altern lassen
    while [ $((--NUM)) -gt 0 ]; do
        if [ -d "${TARGET}/${HOST}-${TYPE}.${NUM}" ]; then
            echo "$(date) $0 $$ **** MOVE: ${TARGET}/${HOST}-${TYPE}.${NUM}"
            mv ${TARGET}/${HOST}-${TYPE}.${NUM} \
                ${TARGET}/${HOST}-${TYPE}.$((NUM+1)) 2> /dev/null
        fi
    done
fi

# 6. move xxx.new to xxx.1
mv ${TARGET}/${HOST}-${TYPE}.new ${TARGET}/${HOST}-${TYPE}.1
 
echo "$(date) $0 $$ ############################################### Finished"
 
) 200> "${LOCK}" 2>&1 | tee "${TMPFILE}" >> "${LOGFILE}"

rm "${TMPFILE}" 2> /dev/null || true