Merge pull request #1 from alexjbush/livy-0.5-zeppelin-0.8

Livy 0.5 zeppelin 0.8
alexjbush · Aug 16, 2018 · caf6141 · caf6141
2 parents fefc6dd + f6896ec
commit caf6141
Show file tree

Hide file tree

Showing 11 changed files with 501 additions and 315 deletions.
diff --git a/build.sh b/build.sh
@@ -3,26 +3,31 @@ set -x
 set -e
 
 CM_EXT_BRANCH=cm5-5.12.0
-LIVY_URL=http://apache.mirror.anlx.net/incubator/livy/0.4.0-incubating/livy-0.4.0-incubating-bin.zip
-LIVY_MD5="0818685b9bc196de2bef9f0dd0e008b3"
-LIVY_VERSION=0.4.0
-
-ZEPPELIN_URL=http://apache.mirror.anlx.net/zeppelin/zeppelin-0.7.3/zeppelin-0.7.3-bin-all.tgz
-ZEPPELIN_MD5="6f84f5581f59838b632a75071a2157cc"
-ZEPPELIN_VERSION=0.7.3
+LIVY_URL=http://apache.mirror.anlx.net/incubator/livy/0.5.0-incubating/livy-0.5.0-incubating-bin.zip
+LIVY_MD5="cc9dc5518e8c178808707eaa68b7672a"
+LIVY_VERSION=0.5.0
 
+ZEPPELIN_URL=http://apache.mirror.anlx.net/zeppelin/zeppelin-0.8.0/zeppelin-0.8.0-bin-all.tgz
+ZEPPELIN_MD5="d87a285c0640ed02ad74bea219525364"
+ZEPPELIN_VERSION=0.8.0
 
+livy_service_name="LIVY"
+livy_service_name_lower="$( echo $livy_service_name | tr '[:upper:]' '[:lower:]' )"
 livy_archive="$( basename $LIVY_URL )"
 livy_folder="$( basename $livy_archive .zip )"
-livy_parcel_folder="LIVY-${LIVY_VERSION}"
+livy_parcel_folder="${livy_service_name}-${LIVY_VERSION}"
 livy_parcel_name="$livy_parcel_folder-el7.parcel"
 livy_built_folder="${livy_parcel_folder}_build"
+livy_csd_build_folder="livy_csd_build"
 
+zeppelin_service_name="ZEPPELIN"
+zeppelin_service_name_lower="$( echo $zeppelin_service_name | tr '[:upper:]' '[:lower:]' )"
 zeppelin_archive="$( basename $ZEPPELIN_URL )"
 zeppelin_folder="$( basename $zeppelin_archive .tgz )"
-zeppelin_parcel_folder="ZEPPELIN-${ZEPPELIN_VERSION}"
+zeppelin_parcel_folder="${zeppelin_service_name}-${ZEPPELIN_VERSION}"
 zeppelin_parcel_name="$zeppelin_parcel_folder-el7.parcel"
 zeppelin_built_folder="${zeppelin_parcel_folder}_build"
+zeppelin_csd_build_folder="zeppelin_csd_build"
 
 function build_cm_ext {
 
@@ -76,6 +81,8 @@ function build_livy_parcel {
   fi
   cp -r livy-parcel-src/meta $livy_parcel_folder
   sed -i -e "s/%VERSION%/$LIVY_VERSION/" ./$livy_parcel_folder/meta/parcel.json
+  sed -i -e "s/%SERVICENAME%/$livy_service_name/" ./$livy_parcel_folder/meta/parcel.json
+  sed -i -e "s/%SERVICENAMELOWER%/$livy_service_name_lower/" ./$livy_parcel_folder/meta/parcel.json
   java -jar cm_ext/validator/target/validator.jar -d ./$livy_parcel_folder
   mkdir -p $livy_built_folder
   tar zcvhf ./$livy_built_folder/$livy_parcel_name $livy_parcel_folder --owner=root --group=root
@@ -93,6 +100,9 @@ function build_zeppelin_parcel {
   fi
   cp -r zeppelin-parcel-src/meta $zeppelin_parcel_folder
   sed -i -e "s/%VERSION%/$ZEPPELIN_VERSION/" ./$zeppelin_parcel_folder/meta/parcel.json
+  sed -i -e "s/%SERVICENAME%/$zeppelin_service_name/" ./$zeppelin_parcel_folder/meta/parcel.json
+  sed -i -e "s/%SERVICENAMELOWER%/$zeppelin_service_name_lower/" ./$zeppelin_parcel_folder/meta/parcel.json
+  sed -i -e "s/%LIVYSERVICENAME%/$livy_service_name/" ./$zeppelin_parcel_folder/meta/parcel.json
   java -jar cm_ext/validator/target/validator.jar -d ./$zeppelin_parcel_folder
   mkdir -p $zeppelin_built_folder
   tar zcvhf ./$zeppelin_built_folder/$zeppelin_parcel_name $zeppelin_parcel_folder --owner=root --group=root
@@ -101,46 +111,39 @@ function build_zeppelin_parcel {
 }
 
 function build_livy_csd {
-  JARNAME=LIVY-${LIVY_VERSION}.jar
+  JARNAME=${livy_service_name}-${LIVY_VERSION}.jar
   if [ -f "$JARNAME" ]; then
     return
   fi
-  java -jar cm_ext/validator/target/validator.jar -s ./livy-csd-src/descriptor/service.sdl -l "SPARK_ON_YARN SPARK2_ON_YARN"
+  rm -rf ${livy_csd_build_folder}
+  cp -rf ./livy-csd-src ${livy_csd_build_folder}
+  sed -i -e "s/%SERVICENAME%/$livy_service_name/" ${livy_csd_build_folder}/descriptor/service.sdl
+  sed -i -e "s/%SERVICENAMELOWER%/$livy_service_name_lower/" ${livy_csd_build_folder}/descriptor/service.sdl
+  sed -i -e "s/%SERVICENAMELOWER%/$livy_service_name_lower/" ${livy_csd_build_folder}/scripts/control.sh
+  java -jar cm_ext/validator/target/validator.jar -s ${livy_csd_build_folder}/descriptor/service.sdl -l "SPARK_ON_YARN SPARK2_ON_YARN"
 
-  jar -cvf ./$JARNAME -C ./livy-csd-src .
+  jar -cvf ./$JARNAME -C ${livy_csd_build_folder} .
 }
 
 function build_zeppelin_csd {
-  JARNAME=ZEPPELIN-${ZEPPELIN_VERSION}.jar
+  JARNAME=${zeppelin_service_name}-${ZEPPELIN_VERSION}.jar
   if [ -f "$JARNAME" ]; then
     return
   fi
-  java -jar cm_ext/validator/target/validator.jar -s ./zeppelin-csd-src/descriptor/service.sdl -l "LIVY"
-
-  jar -cvf ./$JARNAME -C ./zeppelin-csd-src .
+  rm -rf ${zeppelin_csd_build_folder}
+  cp -rf ./zeppelin-csd-src ${zeppelin_csd_build_folder}
+  sed -i -e "s/%SERVICENAME%/$zeppelin_service_name/" ${zeppelin_csd_build_folder}/descriptor/service.sdl
+  sed -i -e "s/%SERVICENAMELOWER%/$zeppelin_service_name_lower/" ${zeppelin_csd_build_folder}/descriptor/service.sdl
+  sed -i -e "s/%LIVYSERVICENAME%/$livy_service_name/" ${zeppelin_csd_build_folder}/descriptor/service.sdl
+  sed -i -e "s/%LIVYSERVICENAMELOWER%/$livy_service_name_lower/" ${zeppelin_csd_build_folder}/descriptor/service.sdl
+  sed -i -e "s/%SERVICENAMELOWER%/$zeppelin_service_name_lower/" ${zeppelin_csd_build_folder}/scripts/control.py
+  sed -i -e "s/%LIVYSERVICENAMELOWER%/$livy_service_name_lower/" ${zeppelin_csd_build_folder}/scripts/control.py
+  java -jar cm_ext/validator/target/validator.jar -s ${zeppelin_csd_build_folder}/descriptor/service.sdl -l "${livy_service_name}"
+
+  jar -cvf ./$JARNAME -C ${zeppelin_csd_build_folder} .
 }
 
 case $1 in
-clean)
-  if [ -d cm_ext ]; then
-    rm -rf cm_ext
-  fi
-  if [ -d $livy_folder ]; then
-    rm -rf $livy_folder
-  fi
-  if [ -f $livy_archive ]; then
-    rm -rf $livy_archive
-  fi
-  if [ -d $livy_parcel_folder ]; then
-    rm -rf $livy_parcel_folder
-  fi
-  if [ -d $livy_built_folder ]; then
-    rm -rf $livy_built_folder
-  fi
-  if [ -f "LIVY-${LIVY_VERSION}.jar" ]; then
-    rm -rf "LIVY-${LIVY_VERSION}.jar"
-  fi
-  ;;
 parcel)
   build_cm_ext
   build_livy_parcel
@@ -151,6 +154,6 @@ csd)
   build_zeppelin_csd
   ;;
 *)
-  echo "Usage: $0 [parcel|csd|clean]"
+  echo "Usage: $0 [parcel|csd]"
   ;;
 esac
diff --git a/livy-csd-src/descriptor/service.sdl b/livy-csd-src/descriptor/service.sdl
@@ -1,8 +1,8 @@
 {
-  "name": "LIVY",
-  "label": "Livy",
+  "name": "%SERVICENAME%",
+  "label": "%SERVICENAME%",
   "description": "Apache Livy is a REST Service for Apache Spark. This service runs a Livy REST service.",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "compatibility": {
     "cdhVersion": {
       "min": "5.11.0"
@@ -17,12 +17,12 @@
   "icon": "images/icon.png",
   "parcel": {
     "requiredTags": [
-      "livy",
+      "%SERVICENAMELOWER%",
       "spark"
     ],
     "optionalTags": [
       "spark2",
-      "livy-plugin"
+      "%SERVICENAMELOWER%-plugin"
     ]
   },
   "serviceDependencies": [
@@ -61,10 +61,10 @@
     ]
   },
   "parameters": [],
-  "rolesWithExternalLinks" : ["LIVY_REST_SERVER"],
+  "rolesWithExternalLinks" : ["%SERVICENAME%_REST_SERVER"],
   "roles": [
     {
-      "name": "LIVY_REST_SERVER",
+      "name": "%SERVICENAME%_REST_SERVER",
       "label": "Livy REST Server",
       "pluralLabel": "Livy REST Servers",
       "jvmBased": true,
@@ -84,8 +84,8 @@
           "LIVY_SPARK_VERSION": "${livy_spark_version}",
           "SSL_ENABLED": "${ssl_enabled}",
           "KEYSTORE_LOCATION": "${ssl_server_keystore_location}",
-          "KEYSTORE_PASSWORD": "${ssl_server_keystore_keypassword}",
-          "KEYSTORE_KEYPASSWORD": "${ssl_server_keystore_password}",
+          "KEYSTORE_PASSWORD": "${ssl_server_keystore_password}",
+          "KEYSTORE_KEYPASSWORD": "${ssl_server_keystore_keypassword}",
           "IVY_DATA_DIR": "${ivy_data_dir}"
         }
       },
@@ -107,9 +107,9 @@
         "maxInstances": 2
       },
       "logging": {
-        "configFilename": "livy-conf/log4j.properties",
-        "dir": "/var/log/livy",
-        "filename": "livy-rest-server-${host}.log",
+        "configFilename": "%SERVICENAMELOWER%-conf/log4j.properties",
+        "dir": "/var/log/%SERVICENAMELOWER%",
+        "filename": "%SERVICENAMELOWER%-rest-server-${host}.log",
         "modifiable": true,
         "loggingType": "log4j"
       },
@@ -209,7 +209,7 @@
           "required": "false",
           "type": "path",
           "pathType": "localDataDir",
-          "default": "/var/local/livy/ivy",
+          "default": "/var/local/%SERVICENAMELOWER%/ivy",
           "required": "true",
           "minLength": 1,
           "configurableInWizard": true
@@ -263,7 +263,7 @@
       "configWriter": {
         "generators": [
           {
-            "filename": "livy-conf/livy.conf",
+            "filename": "%SERVICENAMELOWER%-conf/livy.conf",
             "configFormat": "properties",
             "includedParams": [
               "livy_server_port",
@@ -277,32 +277,32 @@
         ],
         "auxConfigGenerators": [
           {
-            "filename": "livy-conf/livy-env.sh",
+            "filename": "%SERVICENAMELOWER%-conf/livy-env.sh",
             "sourceFilename": "aux/client/livy-env.sh"
           }
         ],
         "peerConfigGenerators" : [
           {
-            "filename": "livy-conf/server.properties",
+            "filename": "%SERVICENAMELOWER%-conf/server.properties",
             "params": ["livy_server_port", "ssl_enabled", "livy_spark_version" ],
-            "roleName": "LIVY_REST_SERVER"
+            "roleName": "%SERVICENAME%_REST_SERVER"
           }
         ]
       }
     }
   ],
   "gateway" : {
     "alternatives" : {
-      "name" : "livy-conf",
+      "name" : "%SERVICENAMELOWER%-conf",
       "priority" : 51,
-      "linkRoot" : "/etc/livy"
+      "linkRoot" : "/etc/%SERVICENAMELOWER%"
     },
     "configWriter" : {
        "peerConfigGenerators" : [
           {
-            "filename": "livy-conf/server.properties",
+            "filename": "%SERVICENAMELOWER%-conf/server.properties",
             "params": ["livy_server_port", "ssl_enabled", "livy_spark_version" ],
-            "roleName": "LIVY_REST_SERVER"
+            "roleName": "%SERVICENAME%_REST_SERVER"
           }
         ]
      }

diff --git a/livy-csd-src/scripts/control.sh b/livy-csd-src/scripts/control.sh
@@ -57,9 +57,9 @@ case $1 in
     fi
     # Copy hive-site to hadoop dir
     # Set Livy conf
-    export LIVY_CONF_DIR="$CONF_DIR/livy-conf"
+    export LIVY_CONF_DIR="$CONF_DIR/%SERVICENAMELOWER%-conf"
     if [ ! -d "$LIVY_CONF_DIR" ]; then
-      log "Could not find livy-conf directory at $LIVY_CONF_DIR"
+      log "Could not find %SERVICENAMELOWER%-conf directory at $LIVY_CONF_DIR"
       exit 3
     fi
     # Update Livy conf for Kerberos and ssl
@@ -72,16 +72,16 @@ case $1 in
     if [ "$SSL_ENABLED" == "true" ]; then
        echo "livy.keystore=$KEYSTORE_LOCATION" >> "$CONF_FILE"
        echo "livy.keystore.password=$KEYSTORE_PASSWORD" >> "$CONF_FILE"
-       echo "livy.keystore.keypassword=$KEYSTORE_KEYPASSWORD" >> "$CONF_FILE"
+       echo "livy.key-password=$KEYSTORE_KEYPASSWORD" >> "$CONF_FILE"
     fi
     if [ "$LIVY_PRINCIPAL" != "" ]; then
        echo "livy.server.launch.kerberos.principal=$LIVY_PRINCIPAL" >> "$CONF_FILE"
-       echo "livy.server.launch.kerberos.keytab=livy.keytab" >> "$CONF_FILE"
+       echo "livy.server.launch.kerberos.keytab=%SERVICENAMELOWER%.keytab" >> "$CONF_FILE"
        #SPNEGO config
        if [ "$ENABLE_SPNEGO" = "true" ] && [ -n "$SPNEGO_PRINCIPAL" ]; then
          echo "livy.server.auth.type=kerberos" >> "$CONF_FILE"
          echo "livy.server.auth.kerberos.principal=$SPNEGO_PRINCIPAL" >> "$CONF_FILE"
-         echo "livy.server.auth.kerberos.keytab=livy.keytab" >> "$CONF_FILE"
+         echo "livy.server.auth.kerberos.keytab=%SERVICENAMELOWER%.keytab" >> "$CONF_FILE"
          echo "livy.superusers=$LIVY_SUPERUSERS" >> "$CONF_FILE"
          if [ "$ENABLE_ACCESS_CONTROL" == "true" ]; then
            echo "livy.server.access-control.enabled=true" >> "$CONF_FILE"

diff --git a/livy-parcel-src/meta/parcel.json b/livy-parcel-src/meta/parcel.json
@@ -1,11 +1,11 @@
 {
   "schema_version": 1,
-  "name": "LIVY",
+  "name": "%SERVICENAME%",
   "version": "%VERSION%",
   "setActiveSymlink": true,
 
   "provides": [
-    "livy"
+    "%SERVICENAMELOWER%"
   ],
 
   "depends": "CDH",

diff --git a/zeppelin-csd-src/aux/client/interpreter.json b/zeppelin-csd-src/aux/client/interpreter.json
@@ -1,7 +1,5 @@
 {
   "interpreterSettings": {
-{{SPARK_CONFIG_spark}}
-{{SPARK_CONFIG_spark2}}
     "2CZ9EX8ZX": {
       "id": "2CZ9EX8ZX",
       "name": "md",
@@ -36,8 +34,6 @@
   },
   "interpreterBindings": {
     "2CYUMWD38": [
-      {{INTERPRETER_BINDING_spark}}
-      {{INTERPRETER_BINDING_spark2}}
       "2CZ9EX8ZX"
     ]
   },