grid

#!/bin/bash -e

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# This script will download, setup, start, and stop servers for Kafka, YARN, and ZooKeeper,
# as well as downloading, building and locally publishing Samza

if [ -z "$JAVA_HOME" ]; then
  if [ -x /usr/libexec/java_home ]; then
    export JAVA_HOME="$(/usr/libexec/java_home)"
  else
    echo "JAVA_HOME not set. Exiting."
    exit 1
  fi
fi

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
BASE_DIR=/opt/samza
DEPLOY_ROOT_DIR=$BASE_DIR/deploy
DOWNLOAD_CACHE_DIR=$HOME/.samza/download
COMMAND=$1
SYSTEM=$2



# Version are in gradle properties
SAMZA_VERSION=
SAMZA_TAG=
SCALA_VERSION=
KAFKA_VERSION=
ZK_VERSION=
HADOOP_VERSION=
source $SCRIPT_DIR/../gradle.properties

# File and URL
KAFKA_PACKAGE="kafka_${SCALA_VERSION}-${KAFKA_VERSION}"
DOWNLOAD_KAFKA="https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/${KAFKA_PACKAGE}.tgz"
YARN_PACKAGE="hadoop-${HADOOP_VERSION}"
DOWNLOAD_YARN="https://archive.apache.org/dist/hadoop/common/${YARN_PACKAGE}/${YARN_PACKAGE}.tar.gz"
ZOOKEEPER_PACKAGE="zookeeper-${ZK_VERSION}"
DOWNLOAD_ZOOKEEPER="https://archive.apache.org/dist/zookeeper/${ZOOKEEPER_PACKAGE}/${ZOOKEEPER_PACKAGE}.tar.gz"

if [ "${SAMZA_VERSION}" == "1.4.0" ]; then
  SAMZA_TOOL_PACKAGE="samza-tools-1.3.1" # 1.4.1 not available
  DOWNLOAD_SAMZA_TOOL="https://downloads.apache.org/samza/1.3.1/samza-tools_${SCALA_VERSION}-1.3.1.tgz"
else
  SAMZA_TOOL_PACKAGE="samza-tools_${SCALA_VERSION}-${SAMZA_VERSION}"
  DOWNLOAD_SAMZA_TOOL="https://downloads.apache.org/samza/${SAMZA_VERSION}/${SAMZA_TOOL_PACKAGE}.tgz"
fi


SERVICE_WAIT_TIMEOUT_SEC=20
ZOOKEEPER_PORT=2181
RESOURCEMANAGER_PORT=8032
NODEMANAGER_PORT=8042
KAFKA_PORT=9092

bootstrap() {
  echo "Bootstrapping the system..."
  stop_all
  rm -rf "$DEPLOY_ROOT_DIR"
  mkdir "$DEPLOY_ROOT_DIR"
  install_all
  start_all
  exit 0
}

standalone() {
  echo "Setting up the system..."
  stop_all
  rm -rf "$DEPLOY_ROOT_DIR"
  mkdir "$DEPLOY_ROOT_DIR"
  install_all_without_yarn
  start_all_without_yarn
  exit 0
}

install_all() {
  $SCRIPT_DIR/grid install samza
  $SCRIPT_DIR/grid install zookeeper
  $SCRIPT_DIR/grid install yarn
  $SCRIPT_DIR/grid install kafka
}

install_all_without_yarn() {
  $SCRIPT_DIR/grid install samza
  $SCRIPT_DIR/grid install zookeeper
  $SCRIPT_DIR/grid install kafka
}

install_samza() {
  echo "Building samza ${SAMZA_TAG} from master..."
  mkdir -p "$DEPLOY_ROOT_DIR"
  if [ -d "$DEPLOY_ROOT_DIR/samza/.git" ]; then
    pushd "$DEPLOY_ROOT_DIR/samza"
    git fetch origin
    git reset --hard origin/master
  else
    mkdir -p $DEPLOY_ROOT_DIR
    pushd $DEPLOY_ROOT_DIR
    git clone https://gitbox.apache.org/repos/asf/samza.git
    cd samza
  fi
  git checkout tags/${SAMZA_TAG}
  # ./gradlew clean build -x test
  ./gradlew -PscalaSuffix=${SCALA_VERSION} clean publishToMavenLocal
  popd
}

install_zookeeper() {
  mkdir -p "$DEPLOY_ROOT_DIR"
  install zookeeper $DOWNLOAD_ZOOKEEPER $ZOOKEEPER_PACKAGE
  cp "$DEPLOY_ROOT_DIR/zookeeper/conf/zoo_sample.cfg" "$DEPLOY_ROOT_DIR/zookeeper/conf/zoo.cfg"
}

# https://samza.apache.org/learn/tutorials/0.14/samza-tools.html
install_samza_tool() {
  mkdir -p "$DEPLOY_ROOT_DIR"
  install samzatool $DOWNLOAD_SAMZA_TOOL $SAMZA_TOOL_PACKAGE
}

install_yarn() {
  mkdir -p "$DEPLOY_ROOT_DIR"
  install yarn $DOWNLOAD_YARN $YARN_PACKAGE
  HADOOP_CONF_DIR="${DEPLOY_ROOT_DIR}/yarn/conf"
  mkdir -p $HADOOP_CONF_DIR
  cp "$DEPLOY_ROOT_DIR/yarn/etc/hadoop/yarn-site.xml" "${HADOOP_CONF_DIR}/yarn-site.xml"
  if [ ! -f "$HOME/.samza/conf/yarn-site.xml" ]; then
    mkdir -p "$HOME/.samza/conf"
    cp "${HADOOP_CONF_DIR}/yarn-site.xml" "$HOME/.samza/conf/yarn-site.xml"
  fi
}

install_kafka() {
  mkdir -p "$DEPLOY_ROOT_DIR"
  install kafka $DOWNLOAD_KAFKA $KAFKA_PACKAGE
  # have to use SIGTERM since nohup on appears to ignore SIGINT
  # and Kafka switched to SIGINT in KAFKA-1031.
  sed -i.bak 's/SIGINT/SIGTERM/g' $DEPLOY_ROOT_DIR/kafka/bin/kafka-server-stop.sh
  # in order to simplify the wikipedia-stats example job, set topic to have just 1 partition by default
  sed -i.bak 's/^num\.partitions *=.*/num.partitions=1/' $DEPLOY_ROOT_DIR/kafka/config/server.properties
}

install() {
  DESTINATION_DIR="$DEPLOY_ROOT_DIR/$1"
  DOWNLOAD_URL=$2
  PACKAGE_DIR="$DOWNLOAD_CACHE_DIR/$3"
  PACKAGE_FILE="$DOWNLOAD_CACHE_DIR/$(basename $DOWNLOAD_URL)"
  if [ -f "$PACKAGE_FILE" ]; then
    echo "Using previously downloaded file $PACKAGE_FILE"
  else
    echo "Downloading $(basename $DOWNLOAD_URL)..."
    mkdir -p $DOWNLOAD_CACHE_DIR
    curl "$DOWNLOAD_URL" > "${PACKAGE_FILE}.tmp"
    mv "${PACKAGE_FILE}.tmp" "$PACKAGE_FILE"
  fi
  rm -rf "$DESTINATION_DIR" "$PACKAGE_DIR"
  tar -xf "$PACKAGE_FILE" -C $DOWNLOAD_CACHE_DIR
  mv "$PACKAGE_DIR" "$DESTINATION_DIR"
}

start_all() {
  $SCRIPT_DIR/grid start zookeeper
  $SCRIPT_DIR/grid start yarn
  $SCRIPT_DIR/grid start kafka
}

start_all_without_yarn() {
  $SCRIPT_DIR/grid start zookeeper
  $SCRIPT_DIR/grid start kafka
}

start_zookeeper() {
  if [ -f $DEPLOY_ROOT_DIR/$SYSTEM/bin/zkServer.sh ]; then
    cd $DEPLOY_ROOT_DIR/$SYSTEM
    bin/zkServer.sh start
    wait_for_service "zookeeper" $ZOOKEEPER_PORT
    cd - > /dev/null
  else
    echo 'Zookeeper is not installed. Run: bin/grid install zookeeper'
  fi
}

start_yarn() {
  if [ -f $DEPLOY_ROOT_DIR/$SYSTEM/sbin/yarn-daemon.sh ]; then
    $DEPLOY_ROOT_DIR/$SYSTEM/sbin/yarn-daemon.sh start resourcemanager
    wait_for_service "resourcemanager" $RESOURCEMANAGER_PORT
    $DEPLOY_ROOT_DIR/$SYSTEM/sbin/yarn-daemon.sh start nodemanager
    wait_for_service "nodemanager" $NODEMANAGER_PORT
  else
    echo 'YARN is not installed. Run: bin/grid install yarn'
  fi
}

start_kafka() {
  if [ -f $DEPLOY_ROOT_DIR/$SYSTEM/bin/kafka-server-start.sh ]; then
    mkdir -p $DEPLOY_ROOT_DIR/$SYSTEM/logs
    cd $DEPLOY_ROOT_DIR/$SYSTEM
    nohup bin/kafka-server-start.sh config/server.properties > logs/kafka.log 2>&1 &
    cd - > /dev/null
    wait_for_service "kafka" $KAFKA_PORT
  else
    echo 'Kafka is not installed. Run: bin/grid install kafka'
  fi
}

wait_for_service() {
  local SERVICE_NAME=$1
  local PORT=$2
  echo "Waiting for $SERVICE_NAME to start..."
  local CURRENT_WAIT_TIME=0

  while [[ $(echo | nc -w1 localhost $PORT >/dev/null 2>&1 ;echo $?) -ne 0 ]]; do
      printf '.'
      sleep 1
      if [ $((++CURRENT_WAIT_TIME)) -eq $SERVICE_WAIT_TIMEOUT_SEC ]; then
        printf "\nError: timed out while waiting for $SERVICE_NAME to start.\n"
        exit 1
      fi
  done
  printf '\n'
  echo "$SERVICE_NAME has started";
}

stop_all() {
  $SCRIPT_DIR/grid stop kafka
  $SCRIPT_DIR/grid stop yarn
  $SCRIPT_DIR/grid stop zookeeper
}

stop_zookeeper() {
  if [ -f $DEPLOY_ROOT_DIR/$SYSTEM/bin/zkServer.sh ]; then
    cd $DEPLOY_ROOT_DIR/$SYSTEM
    bin/zkServer.sh stop
    cd - > /dev/null
  else
    echo 'Zookeeper is not installed. Run: bin/grid install zookeeper'
  fi
}

stop_yarn() {
  if [ -f $DEPLOY_ROOT_DIR/$SYSTEM/sbin/yarn-daemon.sh ]; then
    $DEPLOY_ROOT_DIR/$SYSTEM/sbin/yarn-daemon.sh stop resourcemanager
    $DEPLOY_ROOT_DIR/$SYSTEM/sbin/yarn-daemon.sh stop nodemanager
  else
    echo 'YARN is not installed. Run: bin/grid install yarn'
  fi
}

stop_kafka() {
  if [ -f $DEPLOY_ROOT_DIR/$SYSTEM/bin/kafka-server-stop.sh ]; then
    cd $DEPLOY_ROOT_DIR/$SYSTEM
    bin/kafka-server-stop.sh || true # tolerate nonzero exit status if Kafka isn't running
    cd - > /dev/null
  else
    echo 'Kafka is not installed. Run: bin/grid install kafka'
  fi
}

# Check arguments
if [ "$COMMAND" == "bootstrap" ] && test -z "$SYSTEM"; then
  bootstrap
  exit 0
elif [ "$COMMAND" == "standalone" ] && test -z "$SYSTEM"; then
  standalone
  exit 0
elif (test -z "$COMMAND" && test -z "$SYSTEM") \
  || ( [ "$COMMAND" == "help" ] || test -z "$COMMAND" || test -z "$SYSTEM"); then
  echo
  echo "  Usage.."
  echo
  echo "  $ grid"
  echo "  $ grid bootstrap"
  echo "  $ grid standalone"
  echo "  $ grid install [yarn|kafka|zookeeper|samza|samza_tool|all]"
  echo "  $ grid start [yarn|kafka|zookeeper|all]"
  echo "  $ grid stop [yarn|kafka|zookeeper|all]"
  echo
  exit 1
else
  echo "EXECUTING: $COMMAND $SYSTEM"

  "$COMMAND"_"$SYSTEM"
fi