add-repository

#!/bin/bash
# Add a CVMFS repository to a dual-hosted stratum 1.
# This can be run on either host.
# Written by Dave Dykstra 20 June 2014

EXTRAKEYS=""
. /etc/cvmfs/hastratum1.conf

refreshcredentials()
{
    # large repos can take a long time, so refresh kerberos credentials
    #  if that is being used
    if [ ! -f /etc/krb5.keytab ]; then return; fi 
    export KRB5CCNAME=FILE:/tmp/krb5cc_root_hastratum1
    PATH=$PATH:/usr/krb5/bin:/usr/kerberos/bin kinit -k host/`uname -n`
}

usage()
{
    echo "Usage: add-repository [-h|-H] fqrn {continue|http://stratum-zero-fqdn:port [sourcefqrn]}" >&2
    echo "  -h only creates the repository on this half of an HA pair" >&2
    echo "  -H is like -h but only creates if data already present" >&2
    echo "     either on the current host or the partner host" >&2
    echo "  If second required parameter is "continue", continue from a failed snapshot" >&2
    exit 1
}

HALFONLY=false
ONLYIFDATA=false
if [ "$1" = "-h" ]; then
    HALFONLY=true
    shift
elif [ "$1" = "-H" ]; then
    HALFONLY=true
    ONLYIFDATA=true
    shift
fi

if [ $# != 2 ] && [ $# != 3 ]; then
    usage
fi

CREATEREPO=true
case "$2" in
    http://*);;
    continue)
	CREATEREPO=false;;
    *) usage;;
esac

if [ "`id -u`" != 0 ]; then
    echo "Not running as root" >&2
    exit 1
fi

REPO="$1"
URL="$2"
SOURCEREPO="$3"

SHORTREPO=""
case "$REPO" in
    *.cern.ch|*.opensciencegrid.org)
	SHORTREPO="${REPO%%.*}"
	;;
esac

DOMAIN="${REPO#*.}"
if [ -f /etc/cvmfs/keys/$REPO.pub ]; then
    KEYS="/etc/cvmfs/keys/$REPO.pub"
elif [ -d /etc/cvmfs/keys/$DOMAIN ]; then
    KEYS="/etc/cvmfs/keys/$DOMAIN"
    KEYS="`echo /etc/cvmfs/keys/$DOMAIN/*.pub|tr ' ' ':'`"
elif [ -f /etc/cvmfs/keys/$DOMAIN.pub ]; then
    KEYS="`echo /etc/cvmfs/keys/*$DOMAIN.pub|tr ' ' ':'`"
elif [ -n "$EXTRAKEYS" ]; then
    echo "Note: $DOMAIN.pub not found, but \$EXTRAKEYS is set, so continuing" >&2
else
    echo "$DOMAIN.pub not found in /etc/cvmfs/keys, aborting" >&2
    exit 1
fi

if [ -n "$EXTRAKEYS" ]; then
    case "$KEYS" in
	*"$EXTRAKEYS"*) ;;
	"") KEYS="$EXTRAKEYS" ;;
	*)  KEYS="$KEYS:$EXTRAKEYS" ;;
    esac
fi

THISHOST="`uname -n`"
case $THISHOST in
    $HOST1) OTHERHOST=$HOST2;;
    $HOST2) OTHERHOST=$HOST1;;
    *) echo "Not running on $HOST1 or $HOST2" >&2; exit 1;;
esac

HTTPPORT=${HTTPPORT:-8081}

REPOSTORE=$STORAGE/$REPO

set -e

refreshcredentials

if $CREATEREPO; then
    if [ -d /etc/cvmfs/repositories.d/$REPO ]; then
        echo "/etc/cvmfs/repositories.d/$REPO already exists" >&2
        exit 1
    fi
    if [ -d "$REPOSTORE" ]; then
        if [ -d "$REPOSTORE.ins" ]; then
            echo "Both $REPOSTORE and $REPOSTORE.ins exist" >&2
            exit 1
        fi
        # Old data left from previous install, save it for later
        echo "Saving $REPOSTORE to $REPOSTORE.ins"
        mv $REPOSTORE $REPOSTORE.ins
    elif $ONLYIFDATA; then
        # This is used from manage-replicas to only add a repo config on one
        #  half if some data already existed.  If data does not yet exist on
        #  either half, wait for the master's manage-replicas to add it.
        if ! ssh $OTHERHOST test -f $REPOSTORE/.cvmfspublished; then
            echo "$REPOSTORE not present here and $REPOSTORE/.cvmfspublished not on $OTHERHOST, skipping" >&2
            exit 0
        fi
        echo "Read snapshot from $OTHERHOST"
        export CVMFS_SERVER_OVERRIDE=http://$OTHERHOST:$HTTPPORT/cvmfs/$REPO/stage
    fi
else
    if [ ! -d "$REPOSTORE" ]; then
	echo "$REPOSTORE does not exist, cannot continue" >&2
	exit 1
    fi
    if [ ! -f "$REPOSTORE/stage/.cvmfs_master_replica" ]; then
	echo "Creation of $REPO didn't finish, cannot continue" >&2
	exit 1
    fi
    if ! $HALFONLY && ! ssh $OTHERHOST test -f $REPOSTORE/stage/.cvmfs_master_replica; then
	echo "Creation of $REPO on $OTHERHOST didn't finish, cannot continue" >&2
	exit 1
    fi
    if [ -f "$REPOSTORE/.cvmfspublished" ]; then
	echo "Initial snapshot already done for $REPO, cannot continue" >&2
	exit 1
    fi
fi

set -x

if $CREATEREPO; then
    if ! $HALFONLY; then
	# this is first to allow remove-repository to run on either host even
	#  if this command is aborted
	ssh $OTHERHOST mkdir -p $REPOSTORE
    fi

    mkdir -p $REPOSTORE/data $REPOSTORE/stage
    ln -sf ../data $REPOSTORE/stage/data

    if [ -n "$SHORTREPO" ]; then
	ln -sf $REPO $STORAGE/$SHORTREPO
	if ! $HALFONLY; then
	    ssh $OTHERHOST ln -sf $REPO $STORAGE/$SHORTREPO
	fi
    fi

    mkdir -p /srv/cvmfs
    ln -sf $REPOSTORE/stage /srv/cvmfs/$REPO
    if [ ! -L /srv/cvmfs/info ]; then
        rm -rf /srv/cvmfs/info
        mkdir -p $STORAGE/info
        ln -sf $STORAGE/info /srv/cvmfs
    fi
    if ! if [ -z "$SOURCEREPO" ]; then
            cvmfs_server add-replica -p -o root $URL/cvmfs/$REPO $KEYS
        else
            cvmfs_server add-replica -p -o root -n $REPO $URL/cvmfs/$SOURCEREPO $KEYS
        fi; then
        set +x
        echo "add-replica failed, removing repository and $REPOSTORE"
        cvmfs_server rmfs -f $REPO || true
        rm -rf $REPOSTORE
        if [ -d $REPOSTORE.ins ]; then
          echo "and restoring $REPOSTORE.ins to $REPOSTORE"
          mv $REPOSTORE.ins $REPOSTORE
        fi
        exit 1
    fi
    if [ -L /var/spool/cvmfs/$REPO/tmp ]; then
	# change this link so repos can be moved to different $SRV directories
	rm -f /var/spool/cvmfs/$REPO/tmp
	ln -sf $STORAGE/$REPO/data/txn /var/spool/cvmfs/$REPO/tmp
    fi
    if [ "$SRV" != "/srv/cvmfs" ]; then
	mkdir -p $SRV
	mv /srv/cvmfs/$REPO $SRV
	sed -i "s,/srv/cvmfs/,$SRV/,g" /etc/cvmfs/repositories.d/$REPO/server.conf
    fi

    sed -i -e 's,^CVMFS_STRATUM1=\(.*\),CVMFS_STRATUM1=\1/stage,' \
      -e 's,^CVMFS_STRATUM0=\(.*\),CVMFS_STRATUM0=${CVMFS_SERVER_OVERRIDE:-\1},' \
      /etc/cvmfs/repositories.d/$REPO/server.conf
    if ! $HALFONLY; then
	rsync -a --delete $REPOSTORE/ $OTHERHOST:$REPOSTORE
	rsync -a /etc/cvmfs/repositories.d/$REPO $OTHERHOST:/etc/cvmfs/repositories.d
	rsync -a /var/spool/cvmfs/$REPO $OTHERHOST:/var/spool/cvmfs
	ssh $OTHERHOST mkdir -p $SRV
        rsync -a $STORAGE/info $OTHERHOST:$STORAGE
	rsync -a $SRV/$REPO $OTHERHOST:$SRV
        rsync -a /srv/cvmfs/info $OTHERHOST:/srv/cvmfs
    fi

    # these have to be last because of "continue" test above
    touch $REPOSTORE/stage/.cvmfs_master_replica
    if ! $HALFONLY; then
	ssh $OTHERHOST touch $REPOSTORE/stage/.cvmfs_master_replica
    fi
fi

if [ -d $REPOSTORE.ins ]; then
    echo "Restoring $REPOSTORE.ins to $REPOSTORE"
    rm -rf $REPOSTORE
    mv $REPOSTORE.ins $REPOSTORE
    if [ -f "$REPOSTORE/stage/.cvmfsreflog" ]; then
        echo "Removing $REPOSTORE/stage/.cvmfsreflog"
        rm $REPOSTORE/stage/.cvmfsreflog
    fi
    if $HALFONLY && [ -f "$REPOSTORE/stage/.cvmfspublished" ]; then
	# Initial snapshot done, assume the old data is reasonably up to date.
	# Any that it is behind will be caught up at next update from master.
	exit
    fi
fi

# note snapshot in progress for monitoring
ISSHOTTING=$REPOSTORE/.cvmfs_is_snapshotting
date >$ISSHOTTING
DONECMD="rm -f $ISSHOTTING"
if ! $HALFONLY; then
    scp $ISSHOTTING $OTHERHOST:$ISSHOTTING
    DONECMD="$DONECMD; refreshcredentials; ssh $OTHERHOST rm -f $ISSHOTTING"
fi
trap "$DONECMD" 0

# do initial snapshot
cvmfs_server snapshot -t $REPO

if $HALFONLY; then
    # let the next update from master do the rest
    exit
fi

# Holding this lock prevents manage-replicas from trying to continue this
# repo again and prevents another snapshot from happening
LOCKFILE=/var/spool/cvmfs/$REPO/is_updating.lock
exec 9<>$LOCKFILE
if ! flock -n 9; then
    echo "Another process is updating this repo, exiting" >&2
    exit 1
fi
DONECMD="rm -f $LOCKFILE; $DONECMD"
trap "$DONECMD" 0

date
refreshcredentials

# comment out because rsync gets better throughput
#ssh $OTHERHOST CVMFS_SERVER_OVERRIDE=http://$THISHOST:$HTTPPORT/cvmfs/$REPO/stage cvmfs_server snapshot -t $REPO

# ignore differences in modification time
rsync -a --size-only $REPOSTORE/ $OTHERHOST:$REPOSTORE

refreshcredentials

if [ -f /var/spool/cvmfs/$REPO/reflog.chksum ]; then
    rsync -a /var/spool/cvmfs/$REPO/reflog.chksum $OTHERHOST:/var/spool/cvmfs/$REPO
fi

EXCLUDES=" --exclude .cvmfs_master_replica --exclude data --exclude stage"
rsync -a $EXCLUDES $REPOSTORE/stage/ $REPOSTORE/
rsync -a $EXCLUDES $REPOSTORE/stage/ $OTHERHOST:$REPOSTORE/