Merge pull request #413 from github/lildude/optimise-mysql-es-restores

Optimise hookshot and audit log backups and restores and MySQL restores

bin/ghe-restore

@@ -261,9 +261,7 @@ if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then
 fi
 
 echo "Restoring MySQL database ..."
-bm_start "ghe-import-mysql"
-gzip -dc "$GHE_RESTORE_SNAPSHOT_PATH/mysql.sql.gz" | ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-mysql'
-bm_end "ghe-import-mysql"
+ghe-restore-mysql "$GHE_HOSTNAME" 1>&3
 
 echo "Restoring Redis database ..."
 bm_start "ghe-import-redis"

share/github-backup-utils/ghe-backup-es-audit-log

@@ -48,7 +48,7 @@ for index in $indices; do
     ln $GHE_DATA_DIR/current/audit-log/$index_name.gz $GHE_SNAPSHOT_DIR/audit-log/$index_name.gz
     ln $GHE_DATA_DIR/current/audit-log/$index_name.gz.size $GHE_SNAPSHOT_DIR/audit-log/$index_name.gz.size
   else
-    ghe-ssh "$host" "/usr/local/share/enterprise/ghe-es-dump-json \"http://localhost:9201/$index_name\"" | gzip > $GHE_SNAPSHOT_DIR/audit-log/$index_name.gz
+    echo "/usr/local/share/enterprise/ghe-es-dump-json \"http://localhost:9201/$index_name\" | gzip" | ghe-ssh "$host" -- /bin/bash > $GHE_SNAPSHOT_DIR/audit-log/$index_name.gz
     echo $index_size > $GHE_SNAPSHOT_DIR/audit-log/$index_name.gz.size
   fi
 done

share/github-backup-utils/ghe-backup-es-hookshot

@@ -41,7 +41,7 @@ for index in $indices; do
     ln $GHE_DATA_DIR/current/hookshot/$index_name.gz $GHE_SNAPSHOT_DIR/hookshot/$index_name.gz
     ln $GHE_DATA_DIR/current/hookshot/$index_name.gz.size $GHE_SNAPSHOT_DIR/hookshot/$index_name.gz.size
   else
-    ghe-ssh "$host" "/usr/local/share/enterprise/ghe-es-dump-json \"http://localhost:9201/$index_name\"" | gzip > $GHE_SNAPSHOT_DIR/hookshot/$index_name.gz
+    echo "/usr/local/share/enterprise/ghe-es-dump-json \"http://localhost:9201/$index_name\" | gzip" | ghe-ssh "$host" -- /bin/bash > $GHE_SNAPSHOT_DIR/hookshot/$index_name.gz
     echo $index_size > $GHE_SNAPSHOT_DIR/hookshot/$index_name.gz.size
   fi
 done

share/github-backup-utils/ghe-restore-es-audit-log

@@ -43,22 +43,32 @@ fi
 # Only restore indices that don't exist and the last two months' indices.
 for index in $indices; do
   if ! ghe-ssh "$GHE_HOSTNAME" "curl -f -s -XGET http://localhost:9201/$index > /dev/null" || [[ $index =~ $last_month ]] || [[ $index =~ $current_month ]]; then
-    if $CLUSTER || [ -n "$configured" ]; then
-      ghe_verbose "* Restoring $index"
-      gzip -dc $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/audit-log/$index.gz | ghe-ssh "$GHE_HOSTNAME" "/usr/local/share/enterprise/ghe-es-load-json 'http://localhost:9201/$index'" 1>&3
-    else
-      echo "$index.gz" >> $tmp_list
-    fi
+    echo "$index.gz" >> $tmp_list
   fi
 done
 
 if [ -s "$tmp_list" ]; then
+  ghe-ssh "$GHE_HOSTNAME" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore'" 1>&3
+  ghe-ssh "$GHE_HOSTNAME" -- "sudo chown elasticsearch:elasticsearch '$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore'" 1>&3
+
   ghe-rsync -avz --delete \
     -e "ghe-ssh -p $(ssh_port_part "$GHE_HOSTNAME")" \
     --rsync-path="sudo -u elasticsearch rsync" \
     --files-from=$tmp_list \
     "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/audit-log/" \
     "$(ssh_host_part "$GHE_HOSTNAME"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore/" 1>&3
+
+  if $CLUSTER || [ -n "$configured" ]; then
+    for index in $(cat $tmp_list | sed 's/\.gz$//g'); do
+      ghe_verbose "* Restoring $index"
+      echo "export PATH=\$PATH:/usr/local/share/enterprise && gzip -dc $GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore/$index | ghe-es-load-json 'http://localhost:9201/$index'" |
+      ghe-ssh "$GHE_HOSTNAME" -- /bin/bash 1>&3
+    done
+
+    ghe-ssh "$GHE_HOSTNAME" -- "sudo rm $GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore/*.gz" 1>&3
+  fi
+
   rm $tmp_list
 fi
+
 bm_end "$(basename $0)"

share/github-backup-utils/ghe-restore-es-hookshot

@@ -30,22 +30,31 @@ fi
 
 for index in $indices; do
   if [ -z "$last_index" ] || ! [ $index \< $last_index ]; then
-    if $CLUSTER || [ -n "$configured" ]; then
-      ghe_verbose "* Restoring $index"
-      gzip -dc $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/hookshot/$index.gz | ghe-ssh "$GHE_HOSTNAME" "/usr/local/share/enterprise/ghe-es-load-json 'http://localhost:9201/$index'" 1>&3
-    else
-      echo "$index.gz" >> $tmp_list
-    fi
+    echo "$index.gz" >> $tmp_list
   fi
 done
 
 if [ -s "$tmp_list" ]; then
+  ghe-ssh "$GHE_HOSTNAME" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore'" 1>&3
+  ghe-ssh "$GHE_HOSTNAME" -- "sudo chown elasticsearch:elasticsearch '$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore'" 1>&3
+
   ghe-rsync -avz --delete \
     -e "ghe-ssh -p $(ssh_port_part "$GHE_HOSTNAME")" \
     --rsync-path="sudo -u elasticsearch rsync" \
     --files-from=$tmp_list \
     "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/hookshot/" \
     "$(ssh_host_part "$GHE_HOSTNAME"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore/" 1>&3
+
+  if $CLUSTER || [ -n "$configured" ]; then
+    for index in $(cat $tmp_list | sed 's/\.gz$//g'); do
+      ghe_verbose "* Restoring $index"
+      echo "export PATH=\$PATH:/usr/local/share/enterprise && gzip -dc $GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore/$index | ghe-es-load-json 'http://localhost:9201/$index'" |
+      ghe-ssh "$GHE_HOSTNAME" -- /bin/bash 1>&3
+    done
+
+    ghe-ssh "$GHE_HOSTNAME" -- "sudo rm $GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore/*.gz" 1>&3
+  fi
+
   rm $tmp_list
 fi
 

share/github-backup-utils/ghe-restore-mysql

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-restore-mysql <host>
+#/ Restore MySQL backup to a GitHub instance.
+#/
+#/ Note: This script typically isn't called directly. It's invoked by the
+#/ ghe-restore command when the rsync strategy is used.
+set -e
+
+# Bring in the backup configuration
+# shellcheck source=share/github-backup-utils/ghe-backup-config
+. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
+
+# Show usage and bail with no arguments
+[ -z "$*" ] && print_usage
+
+bm_start "$(basename $0)"
+
+# Grab host arg
+GHE_HOSTNAME="$1"
+
+# Perform a host-check and establish the remote version in GHE_REMOTE_VERSION.
+ghe_remote_version_required "$GHE_HOSTNAME"
+
+# The snapshot to restore should be set by the ghe-restore command but this lets
+# us run this script directly.
+: ${GHE_RESTORE_SNAPSHOT:=current}
+
+# The directory holding the snapshot to restore
+snapshot_dir="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"
+
+cleanup() {
+  ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz"
+}
+trap 'cleanup' INT TERM EXIT
+
+ghe-ssh "$GHE_HOSTNAME" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/tmp'" 1>&3
+
+# Transfer MySQL data from the snapshot to the GitHub instance.
+cat $snapshot_dir/mysql.sql.gz | ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz >/dev/null 2>&1"
+
+# Import the database
+echo "gunzip -cd $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | ghe-import-mysql" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash 1>&3
+
+
+bm_end "$(basename $0)"