diff --git a/build.sh b/build.sh index 10d8f3b..1adb8a9 100644 --- a/build.sh +++ b/build.sh @@ -3,26 +3,31 @@ set -x set -e CM_EXT_BRANCH=cm5-5.12.0 -LIVY_URL=http://apache.mirror.anlx.net/incubator/livy/0.4.0-incubating/livy-0.4.0-incubating-bin.zip -LIVY_MD5="0818685b9bc196de2bef9f0dd0e008b3" -LIVY_VERSION=0.4.0 - -ZEPPELIN_URL=http://apache.mirror.anlx.net/zeppelin/zeppelin-0.7.3/zeppelin-0.7.3-bin-all.tgz -ZEPPELIN_MD5="6f84f5581f59838b632a75071a2157cc" -ZEPPELIN_VERSION=0.7.3 +LIVY_URL=http://apache.mirror.anlx.net/incubator/livy/0.5.0-incubating/livy-0.5.0-incubating-bin.zip +LIVY_MD5="cc9dc5518e8c178808707eaa68b7672a" +LIVY_VERSION=0.5.0 +ZEPPELIN_URL=http://apache.mirror.anlx.net/zeppelin/zeppelin-0.8.0/zeppelin-0.8.0-bin-all.tgz +ZEPPELIN_MD5="d87a285c0640ed02ad74bea219525364" +ZEPPELIN_VERSION=0.8.0 +livy_service_name="LIVY" +livy_service_name_lower="$( echo $livy_service_name | tr '[:upper:]' '[:lower:]' )" livy_archive="$( basename $LIVY_URL )" livy_folder="$( basename $livy_archive .zip )" -livy_parcel_folder="LIVY-${LIVY_VERSION}" +livy_parcel_folder="${livy_service_name}-${LIVY_VERSION}" livy_parcel_name="$livy_parcel_folder-el7.parcel" livy_built_folder="${livy_parcel_folder}_build" +livy_csd_build_folder="livy_csd_build" +zeppelin_service_name="ZEPPELIN" +zeppelin_service_name_lower="$( echo $zeppelin_service_name | tr '[:upper:]' '[:lower:]' )" zeppelin_archive="$( basename $ZEPPELIN_URL )" zeppelin_folder="$( basename $zeppelin_archive .tgz )" -zeppelin_parcel_folder="ZEPPELIN-${ZEPPELIN_VERSION}" +zeppelin_parcel_folder="${zeppelin_service_name}-${ZEPPELIN_VERSION}" zeppelin_parcel_name="$zeppelin_parcel_folder-el7.parcel" zeppelin_built_folder="${zeppelin_parcel_folder}_build" +zeppelin_csd_build_folder="zeppelin_csd_build" function build_cm_ext { @@ -76,6 +81,8 @@ function build_livy_parcel { fi cp -r livy-parcel-src/meta $livy_parcel_folder sed -i -e "s/%VERSION%/$LIVY_VERSION/" ./$livy_parcel_folder/meta/parcel.json + sed -i -e "s/%SERVICENAME%/$livy_service_name/" ./$livy_parcel_folder/meta/parcel.json + sed -i -e "s/%SERVICENAMELOWER%/$livy_service_name_lower/" ./$livy_parcel_folder/meta/parcel.json java -jar cm_ext/validator/target/validator.jar -d ./$livy_parcel_folder mkdir -p $livy_built_folder tar zcvhf ./$livy_built_folder/$livy_parcel_name $livy_parcel_folder --owner=root --group=root @@ -93,6 +100,9 @@ function build_zeppelin_parcel { fi cp -r zeppelin-parcel-src/meta $zeppelin_parcel_folder sed -i -e "s/%VERSION%/$ZEPPELIN_VERSION/" ./$zeppelin_parcel_folder/meta/parcel.json + sed -i -e "s/%SERVICENAME%/$zeppelin_service_name/" ./$zeppelin_parcel_folder/meta/parcel.json + sed -i -e "s/%SERVICENAMELOWER%/$zeppelin_service_name_lower/" ./$zeppelin_parcel_folder/meta/parcel.json + sed -i -e "s/%LIVYSERVICENAME%/$livy_service_name/" ./$zeppelin_parcel_folder/meta/parcel.json java -jar cm_ext/validator/target/validator.jar -d ./$zeppelin_parcel_folder mkdir -p $zeppelin_built_folder tar zcvhf ./$zeppelin_built_folder/$zeppelin_parcel_name $zeppelin_parcel_folder --owner=root --group=root @@ -101,46 +111,39 @@ function build_zeppelin_parcel { } function build_livy_csd { - JARNAME=LIVY-${LIVY_VERSION}.jar + JARNAME=${livy_service_name}-${LIVY_VERSION}.jar if [ -f "$JARNAME" ]; then return fi - java -jar cm_ext/validator/target/validator.jar -s ./livy-csd-src/descriptor/service.sdl -l "SPARK_ON_YARN SPARK2_ON_YARN" + rm -rf ${livy_csd_build_folder} + cp -rf ./livy-csd-src ${livy_csd_build_folder} + sed -i -e "s/%SERVICENAME%/$livy_service_name/" ${livy_csd_build_folder}/descriptor/service.sdl + sed -i -e "s/%SERVICENAMELOWER%/$livy_service_name_lower/" ${livy_csd_build_folder}/descriptor/service.sdl + sed -i -e "s/%SERVICENAMELOWER%/$livy_service_name_lower/" ${livy_csd_build_folder}/scripts/control.sh + java -jar cm_ext/validator/target/validator.jar -s ${livy_csd_build_folder}/descriptor/service.sdl -l "SPARK_ON_YARN SPARK2_ON_YARN" - jar -cvf ./$JARNAME -C ./livy-csd-src . + jar -cvf ./$JARNAME -C ${livy_csd_build_folder} . } function build_zeppelin_csd { - JARNAME=ZEPPELIN-${ZEPPELIN_VERSION}.jar + JARNAME=${zeppelin_service_name}-${ZEPPELIN_VERSION}.jar if [ -f "$JARNAME" ]; then return fi - java -jar cm_ext/validator/target/validator.jar -s ./zeppelin-csd-src/descriptor/service.sdl -l "LIVY" - - jar -cvf ./$JARNAME -C ./zeppelin-csd-src . + rm -rf ${zeppelin_csd_build_folder} + cp -rf ./zeppelin-csd-src ${zeppelin_csd_build_folder} + sed -i -e "s/%SERVICENAME%/$zeppelin_service_name/" ${zeppelin_csd_build_folder}/descriptor/service.sdl + sed -i -e "s/%SERVICENAMELOWER%/$zeppelin_service_name_lower/" ${zeppelin_csd_build_folder}/descriptor/service.sdl + sed -i -e "s/%LIVYSERVICENAME%/$livy_service_name/" ${zeppelin_csd_build_folder}/descriptor/service.sdl + sed -i -e "s/%LIVYSERVICENAMELOWER%/$livy_service_name_lower/" ${zeppelin_csd_build_folder}/descriptor/service.sdl + sed -i -e "s/%SERVICENAMELOWER%/$zeppelin_service_name_lower/" ${zeppelin_csd_build_folder}/scripts/control.py + sed -i -e "s/%LIVYSERVICENAMELOWER%/$livy_service_name_lower/" ${zeppelin_csd_build_folder}/scripts/control.py + java -jar cm_ext/validator/target/validator.jar -s ${zeppelin_csd_build_folder}/descriptor/service.sdl -l "${livy_service_name}" + + jar -cvf ./$JARNAME -C ${zeppelin_csd_build_folder} . } case $1 in -clean) - if [ -d cm_ext ]; then - rm -rf cm_ext - fi - if [ -d $livy_folder ]; then - rm -rf $livy_folder - fi - if [ -f $livy_archive ]; then - rm -rf $livy_archive - fi - if [ -d $livy_parcel_folder ]; then - rm -rf $livy_parcel_folder - fi - if [ -d $livy_built_folder ]; then - rm -rf $livy_built_folder - fi - if [ -f "LIVY-${LIVY_VERSION}.jar" ]; then - rm -rf "LIVY-${LIVY_VERSION}.jar" - fi - ;; parcel) build_cm_ext build_livy_parcel @@ -151,6 +154,6 @@ csd) build_zeppelin_csd ;; *) - echo "Usage: $0 [parcel|csd|clean]" + echo "Usage: $0 [parcel|csd]" ;; esac diff --git a/livy-csd-src/descriptor/service.sdl b/livy-csd-src/descriptor/service.sdl index d7779d1..15433f8 100644 --- a/livy-csd-src/descriptor/service.sdl +++ b/livy-csd-src/descriptor/service.sdl @@ -1,8 +1,8 @@ { - "name": "LIVY", - "label": "Livy", + "name": "%SERVICENAME%", + "label": "%SERVICENAME%", "description": "Apache Livy is a REST Service for Apache Spark. This service runs a Livy REST service.", - "version": "0.4.0", + "version": "0.5.0", "compatibility": { "cdhVersion": { "min": "5.11.0" @@ -17,12 +17,12 @@ "icon": "images/icon.png", "parcel": { "requiredTags": [ - "livy", + "%SERVICENAMELOWER%", "spark" ], "optionalTags": [ "spark2", - "livy-plugin" + "%SERVICENAMELOWER%-plugin" ] }, "serviceDependencies": [ @@ -61,10 +61,10 @@ ] }, "parameters": [], - "rolesWithExternalLinks" : ["LIVY_REST_SERVER"], + "rolesWithExternalLinks" : ["%SERVICENAME%_REST_SERVER"], "roles": [ { - "name": "LIVY_REST_SERVER", + "name": "%SERVICENAME%_REST_SERVER", "label": "Livy REST Server", "pluralLabel": "Livy REST Servers", "jvmBased": true, @@ -84,8 +84,8 @@ "LIVY_SPARK_VERSION": "${livy_spark_version}", "SSL_ENABLED": "${ssl_enabled}", "KEYSTORE_LOCATION": "${ssl_server_keystore_location}", - "KEYSTORE_PASSWORD": "${ssl_server_keystore_keypassword}", - "KEYSTORE_KEYPASSWORD": "${ssl_server_keystore_password}", + "KEYSTORE_PASSWORD": "${ssl_server_keystore_password}", + "KEYSTORE_KEYPASSWORD": "${ssl_server_keystore_keypassword}", "IVY_DATA_DIR": "${ivy_data_dir}" } }, @@ -107,9 +107,9 @@ "maxInstances": 2 }, "logging": { - "configFilename": "livy-conf/log4j.properties", - "dir": "/var/log/livy", - "filename": "livy-rest-server-${host}.log", + "configFilename": "%SERVICENAMELOWER%-conf/log4j.properties", + "dir": "/var/log/%SERVICENAMELOWER%", + "filename": "%SERVICENAMELOWER%-rest-server-${host}.log", "modifiable": true, "loggingType": "log4j" }, @@ -209,7 +209,7 @@ "required": "false", "type": "path", "pathType": "localDataDir", - "default": "/var/local/livy/ivy", + "default": "/var/local/%SERVICENAMELOWER%/ivy", "required": "true", "minLength": 1, "configurableInWizard": true @@ -263,7 +263,7 @@ "configWriter": { "generators": [ { - "filename": "livy-conf/livy.conf", + "filename": "%SERVICENAMELOWER%-conf/livy.conf", "configFormat": "properties", "includedParams": [ "livy_server_port", @@ -277,15 +277,15 @@ ], "auxConfigGenerators": [ { - "filename": "livy-conf/livy-env.sh", + "filename": "%SERVICENAMELOWER%-conf/livy-env.sh", "sourceFilename": "aux/client/livy-env.sh" } ], "peerConfigGenerators" : [ { - "filename": "livy-conf/server.properties", + "filename": "%SERVICENAMELOWER%-conf/server.properties", "params": ["livy_server_port", "ssl_enabled", "livy_spark_version" ], - "roleName": "LIVY_REST_SERVER" + "roleName": "%SERVICENAME%_REST_SERVER" } ] } @@ -293,16 +293,16 @@ ], "gateway" : { "alternatives" : { - "name" : "livy-conf", + "name" : "%SERVICENAMELOWER%-conf", "priority" : 51, - "linkRoot" : "/etc/livy" + "linkRoot" : "/etc/%SERVICENAMELOWER%" }, "configWriter" : { "peerConfigGenerators" : [ { - "filename": "livy-conf/server.properties", + "filename": "%SERVICENAMELOWER%-conf/server.properties", "params": ["livy_server_port", "ssl_enabled", "livy_spark_version" ], - "roleName": "LIVY_REST_SERVER" + "roleName": "%SERVICENAME%_REST_SERVER" } ] } diff --git a/livy-csd-src/scripts/control.sh b/livy-csd-src/scripts/control.sh index 59bc4d9..ef5bfe3 100644 --- a/livy-csd-src/scripts/control.sh +++ b/livy-csd-src/scripts/control.sh @@ -57,9 +57,9 @@ case $1 in fi # Copy hive-site to hadoop dir # Set Livy conf - export LIVY_CONF_DIR="$CONF_DIR/livy-conf" + export LIVY_CONF_DIR="$CONF_DIR/%SERVICENAMELOWER%-conf" if [ ! -d "$LIVY_CONF_DIR" ]; then - log "Could not find livy-conf directory at $LIVY_CONF_DIR" + log "Could not find %SERVICENAMELOWER%-conf directory at $LIVY_CONF_DIR" exit 3 fi # Update Livy conf for Kerberos and ssl @@ -72,16 +72,16 @@ case $1 in if [ "$SSL_ENABLED" == "true" ]; then echo "livy.keystore=$KEYSTORE_LOCATION" >> "$CONF_FILE" echo "livy.keystore.password=$KEYSTORE_PASSWORD" >> "$CONF_FILE" - echo "livy.keystore.keypassword=$KEYSTORE_KEYPASSWORD" >> "$CONF_FILE" + echo "livy.key-password=$KEYSTORE_KEYPASSWORD" >> "$CONF_FILE" fi if [ "$LIVY_PRINCIPAL" != "" ]; then echo "livy.server.launch.kerberos.principal=$LIVY_PRINCIPAL" >> "$CONF_FILE" - echo "livy.server.launch.kerberos.keytab=livy.keytab" >> "$CONF_FILE" + echo "livy.server.launch.kerberos.keytab=%SERVICENAMELOWER%.keytab" >> "$CONF_FILE" #SPNEGO config if [ "$ENABLE_SPNEGO" = "true" ] && [ -n "$SPNEGO_PRINCIPAL" ]; then echo "livy.server.auth.type=kerberos" >> "$CONF_FILE" echo "livy.server.auth.kerberos.principal=$SPNEGO_PRINCIPAL" >> "$CONF_FILE" - echo "livy.server.auth.kerberos.keytab=livy.keytab" >> "$CONF_FILE" + echo "livy.server.auth.kerberos.keytab=%SERVICENAMELOWER%.keytab" >> "$CONF_FILE" echo "livy.superusers=$LIVY_SUPERUSERS" >> "$CONF_FILE" if [ "$ENABLE_ACCESS_CONTROL" == "true" ]; then echo "livy.server.access-control.enabled=true" >> "$CONF_FILE" diff --git a/livy-parcel-src/meta/parcel.json b/livy-parcel-src/meta/parcel.json index d524430..a04c700 100644 --- a/livy-parcel-src/meta/parcel.json +++ b/livy-parcel-src/meta/parcel.json @@ -1,11 +1,11 @@ { "schema_version": 1, - "name": "LIVY", + "name": "%SERVICENAME%", "version": "%VERSION%", "setActiveSymlink": true, "provides": [ - "livy" + "%SERVICENAMELOWER%" ], "depends": "CDH", diff --git a/zeppelin-csd-src/aux/client/interpreter.json b/zeppelin-csd-src/aux/client/interpreter.json index f2a4ef4..401aa49 100644 --- a/zeppelin-csd-src/aux/client/interpreter.json +++ b/zeppelin-csd-src/aux/client/interpreter.json @@ -1,7 +1,5 @@ { "interpreterSettings": { -{{SPARK_CONFIG_spark}} -{{SPARK_CONFIG_spark2}} "2CZ9EX8ZX": { "id": "2CZ9EX8ZX", "name": "md", @@ -36,8 +34,6 @@ }, "interpreterBindings": { "2CYUMWD38": [ - {{INTERPRETER_BINDING_spark}} - {{INTERPRETER_BINDING_spark2}} "2CZ9EX8ZX" ] }, diff --git a/zeppelin-csd-src/aux/client/interpreter.livy.json b/zeppelin-csd-src/aux/client/interpreter.livy.json index 64536af..5b90f20 100644 --- a/zeppelin-csd-src/aux/client/interpreter.livy.json +++ b/zeppelin-csd-src/aux/client/interpreter.livy.json @@ -1,89 +1,86 @@ - "{{INTERPRETER_ID}}": { - "id": "{{INTERPRETER_ID}}", - "name": "{{INTERPRETER_NAME}}", - "group": "livy", - "properties": { - "zeppelin.livy.pull_status.interval.millis": "1000", - "livy.spark.executor.memory": "{{EXECUTOR_MEMORY}}", - "livy.spark.executor.extraJavaOptions": "{{EXECUTOR_EXTRA_OPTIONS}}", - "zeppelin.livy.session.create_timeout": "120", - "zeppelin.livy.principal": "{{LIVY_PRINCIPAL}}", - "zeppelin.livy.spark.sql.maxResult": "1000", - "zeppelin.interpreter.localRepo": "{{DATA_DIR}}/local-repo/{{INTERPRETER_ID}}", - "zeppelin.livy.keytab": "{{LIVY_KEYTAB}}", - "zeppelin.interpreter.output.limit": "102400", - "zeppelin.livy.concurrentSQL": "false", - "livy.spark.executor.cores": "", - "zeppelin.livy.displayAppInfo": "true", - "zeppelin.livy.url": "{{LIVY_URL}}", - "livy.spark.dynamicAllocation.minExecutors": "", - "livy.spark.driver.cores": "", - "livy.spark.jars.packages": "{{LIVY_SPARK_JARS_PACKAGES}}", - "livy.spark.jars": "{{LIVY_SPARK_JARS}}", - "livy.spark.dynamicAllocation.enabled": "", - "livy.spark.executor.instances": "", - "livy.spark.dynamicAllocation.cachedExecutorIdleTimeout": "", - "livy.spark.dynamicAllocation.maxExecutors": "{{MAX_EXECUTORS}}", - "livy.spark.dynamicAllocation.initialExecutors": "", - "livy.spark.driver.memory": "{{DRIVER_MEMORY}}" - }, - "status": "READY", - "interpreterGroup": [ - { - "name": "spark", - "class": "org.apache.zeppelin.livy.LivySparkInterpreter", - "defaultInterpreter": true, - "editor": { - "language": "scala", - "editOnDblClick": false - } - }, - { - "name": "sql", - "class": "org.apache.zeppelin.livy.LivySparkSQLInterpreter", - "defaultInterpreter": false, - "editor": { - "language": "sql", - "editOnDblClick": false - } - }, - { - "name": "pyspark", - "class": "org.apache.zeppelin.livy.LivyPySparkInterpreter", - "defaultInterpreter": false, - "editor": { - "language": "python", - "editOnDblClick": false - } - }, - { - "name": "pyspark3", - "class": "org.apache.zeppelin.livy.LivyPySpark3Interpreter", - "defaultInterpreter": false, - "editor": { - "language": "python", - "editOnDblClick": false - } - }, - { - "name": "sparkr", - "class": "org.apache.zeppelin.livy.LivySparkRInterpreter", - "defaultInterpreter": false, - "editor": { - "language": "r", - "editOnDblClick": false - } - } - ], - "dependencies": [], - "option": { - "remote": true, - "port": -1, - "perNote": "{{INTERPRETER_FOR_NOTE}}", - "perUser": "{{INTERPRETER_FOR_USER}}", - "isExistingProcess": false, - "setPermission": false, - "users": [], - "isUserImpersonate": false +{ + "id": "", + "name": "", + "group": "livy", + "properties": { + "zeppelin.livy.pull_status.interval.millis": "1000", + "zeppelin.livy.session.create_timeout": "120", + "zeppelin.livy.principal": "", + "zeppelin.livy.spark.sql.maxResult": "1000", + "zeppelin.livy.keytab": "", + "zeppelin.interpreter.output.limit": "102400", + "zeppelin.livy.concurrentSQL": "false", + "livy.spark.executor.cores": "", + "zeppelin.livy.displayAppInfo": "true", + "livy.spark.dynamicAllocation.minExecutors": "", + "livy.spark.driver.cores": "", + "livy.spark.jars.packages": "", + "livy.spark.jars": "", + "livy.spark.dynamicAllocation.enabled": "", + "livy.spark.executor.instances": "", + "livy.spark.dynamicAllocation.cachedExecutorIdleTimeout": "", + "livy.spark.dynamicAllocation.maxExecutors": "", + "livy.spark.dynamicAllocation.initialExecutors": "", + "livy.spark.driver.memory": "", + "livy.spark.executor.memory": "" + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "spark", + "class": "org.apache.zeppelin.livy.LivySparkInterpreter", + "defaultInterpreter": true, + "editor": { + "language": "scala", + "editOnDblClick": false } }, + { + "name": "sql", + "class": "org.apache.zeppelin.livy.LivySparkSQLInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sql", + "editOnDblClick": false + } + }, + { + "name": "pyspark", + "class": "org.apache.zeppelin.livy.LivyPySparkInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "python", + "editOnDblClick": false + } + }, + { + "name": "pyspark3", + "class": "org.apache.zeppelin.livy.LivyPySpark3Interpreter", + "defaultInterpreter": false, + "editor": { + "language": "python", + "editOnDblClick": false + } + }, + { + "name": "sparkr", + "class": "org.apache.zeppelin.livy.LivySparkRInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "r", + "editOnDblClick": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "perNote": "shared", + "perUser": "shared", + "isExistingProcess": false, + "setPermission": false, + "users": [], + "isUserImpersonate": false + } +} diff --git a/zeppelin-csd-src/descriptor/service.sdl b/zeppelin-csd-src/descriptor/service.sdl index 5c50b1d..d8dad60 100644 --- a/zeppelin-csd-src/descriptor/service.sdl +++ b/zeppelin-csd-src/descriptor/service.sdl @@ -1,8 +1,8 @@ { - "name": "ZEPPELIN", - "label": "ZEPPELIN", + "name": "%SERVICENAME%", + "label": "%SERVICENAME%", "description": "Apache Zeppelin is a web-based notebook that enables interactive data analytics. You can make beautiful data-driven, interactive and collaborative documents with SQL, Scala and more.", - "version": "0.7.3", + "version": "0.8.0", "compatibility": { "cdhVersion": { "min": "5.11.0" @@ -17,8 +17,8 @@ "icon": "images/icon.png", "parcel": { "requiredTags": [ - "zeppelin", - "livy" + "%SERVICENAMELOWER%", + "%LIVYSERVICENAMELOWER%" ], "optionalTags": [ "zeppelin-plugin" @@ -26,7 +26,7 @@ }, "serviceDependencies": [ { - "name": "LIVY", + "name": "%LIVYSERVICENAME%", "required": "true" } ], @@ -48,34 +48,49 @@ ] }, "parameters": [], - "rolesWithExternalLinks" : ["ZEPPELIN_SERVER"], + "rolesWithExternalLinks" : ["%SERVICENAME%_SERVER"], "roles": [ { - "name": "ZEPPELIN_SERVER", + "name": "%SERVICENAME%_SERVER", "label": "Zeppelin Server", "pluralLabel": "Zeppelin Servers", "jvmBased": true, "startRunner": { - "program": "scripts/control.sh", + "program": "scripts/control.py", "args": [ "start" ], "environmentVariables": { - "DRIVER_MEMORY": "${driver_memory}", - "EXECUTOR_MEMORY": "${executor_memory}", - "EXECUTOR_EXTRA_OPTIONS": "${executor_extra_options}", - "MAX_EXECUTORS": "${max_executors}", "ZEPPELIN_LOG_DIR": "${log_dir}", "INTERPRETER_FOR_NOTE": "${interpreter_for_note}", "INTERPRETER_FOR_USER": "${interpreter_for_user}", "ZEPPELIN_MEMORY": "${zeppelin_max_heapsize}", - "ZEPPELIN_INTERPRETER_LIST": "${zeppelin_interpreter_list}", "ZEPPELIN_TRUSTSTORE": "${ssl_client_truststore_location}", - "ZEPPELIN_TRUSTSTORE_PASSWORD": "${ssl_client_truststore_password}", "ZEPPELIN_SHIRO_ENABLED": "${zeppelin_shiro_enabled}", "ZEPPELIN_DATA_DIR": "${zeppelin_data_dir}", - "LIVY_SPARK_JARS_PACKAGES": "${livy_spark_jars_packages}", - "LIVY_SPARK_JARS": "${livy_spark_jars}" + "ZEPPELIN_CONF_FS_DIR": "${zeppelin_conf_dir}", + "ZEPPELIN_NOTEBOOK_DIR": "${zeppelin_notebook_dir}", + "ZEPPELIN_EXTRA_JAVA_OPTIONS": "${zeppelin_java_options}" + } + }, + "stopRunner": { + "runner" : { + "program": "scripts/control.py", + "args": [ + "stop" + ], + "environmentVariables": { + "ZEPPELIN_LOG_DIR": "${log_dir}", + "INTERPRETER_FOR_NOTE": "${interpreter_for_note}", + "INTERPRETER_FOR_USER": "${interpreter_for_user}", + "ZEPPELIN_MEMORY": "${zeppelin_max_heapsize}", + "ZEPPELIN_TRUSTSTORE": "${ssl_client_truststore_location}", + "ZEPPELIN_SHIRO_ENABLED": "${zeppelin_shiro_enabled}", + "ZEPPELIN_DATA_DIR": "${zeppelin_data_dir}", + "ZEPPELIN_CONF_FS_DIR": "${zeppelin_conf_dir}", + "ZEPPELIN_NOTEBOOK_DIR": "${zeppelin_notebook_dir}", + "ZEPPELIN_EXTRA_JAVA_OPTIONS": "${zeppelin_java_options}" + } } }, "kerberosPrincipals": [ @@ -91,9 +106,9 @@ "maxInstances": 1 }, "logging": { - "configFilename": "zeppelin-conf/log4j.properties", - "dir": "/var/log/zeppelin", - "filename": "zeppelin-server-${host}.log", + "configFilename": "%SERVICENAMELOWER%-conf/log4j.properties", + "dir": "/var/log/%SERVICENAMELOWER%", + "filename": "%SERVICENAMELOWER%-server-${host}.log", "modifiable": true, "loggingType": "log4j" }, @@ -125,10 +140,18 @@ "type": "boolean", "default": true }, + { + "name": "zeppelin_java_options", + "label": "Zeppelin Java Options", + "description": "Extra java options given as ZEPPELIN_JAVA_OPTS.", + "required": "false", + "type": "string" + }, { "name": "executor_memory", "label": "Spark Executor Memory", "description": "Executor memory per worker instance. ex) 512m, 32g.", + "configName": "livy.spark.executor.memory", "required": "false", "type": "string", "default": "4g" @@ -137,6 +160,7 @@ "name": "driver_memory", "label": "Spark Driver Memory", "description": "Driver memory. ex) 512m, 32g.", + "configName": "livy.spark.driver.memory", "required": "false", "type": "string", "default": "4g" @@ -145,6 +169,7 @@ "name": "max_executors", "label": "Max Spark Executors", "description": "Upper bound for the number of executors (per interpreter instance).", + "configName": "livy.spark.dynamicAllocation.maxExecutors", "required": "false", "type": "string", "default": "" @@ -153,6 +178,7 @@ "name": "executor_extra_options", "label": "Spark Executor Extra Java Options", "description": "A string of extra JVM options to pass to executors. For instance, GC settings or other logging.", + "configName": "livy.spark.executor.extraJavaOptions", "required": "false", "type": "string", "default": "" @@ -174,18 +200,6 @@ "type": "boolean", "default": false }, - { - "name": "zeppelin_interpreter_list", - "label": "Zeppelin Interpreter List", - "description": "List of Zeppelin interpreters to enable. Allowed values are: alluxio, angular, beam, bigquery, cassandra, elasticsearch, file, flink, hbase, ignite, jdbc, kylin, lens, livy, md, pig, postgresql, python, scio, shell.", - "required": "false", - "type": "string_array", - "separator": " ", - "default": [ - "livy", - "md" - ] - }, { "name": "interpreter_for_note", "label": "Interpreter for notes", @@ -213,11 +227,50 @@ { "name": "zeppelin_data_dir", "label": "Zeppelin Data Directory", - "description": "The root data directory where the notebook and jetty temporary directories are stored.", - "required": "false", + "description": "The root data directory where the Zeppelin temporary directories are stored.", "type": "path", "pathType": "localDataDir", - "default": "/var/local/zeppelin/data", + "default": "/var/local/%SERVICENAMELOWER%/data", + "required": "true", + "minLength": 1, + "configurableInWizard": true + }, + { + "name": "zeppelin_conf_dir", + "label": "Zeppelin Conf Directory", + "description": "The configuration directory where Zeppelin stores the interpreter.json, notebook-authorization.json and credential.json files.", + "type": "uri", + "configName": "zeppelin.config.fs.dir", + "allowedSchemes": [ + "hdfs", + "abfs", + "adl", + "file", + "s3a", + "wasbs", + "wasb" + ], + "default": "file:///var/local/%SERVICENAMELOWER%/conf", + "required": "true", + "minLength": 1, + "configurableInWizard": true + }, + { + "name": "zeppelin_notebook_dir", + "label": "Zeppelin Notebook Directory", + "description": "The root data directory where the notebooks are stored.", + "configName": "zeppelin.notebook.dir", + "allowedSchemes": [ + "hdfs", + "abfs", + "adl", + "file", + "s3a", + "wasbs", + "wasb" + ], + "default": "file:///var/local/%SERVICENAMELOWER%/data/notebook", + "type": "uri", "required": "true", "minLength": 1, "configurableInWizard": true @@ -236,6 +289,7 @@ "name": "livy_spark_jars_packages", "label": "Livy Spark Maven Jars", "description": "List of packages to add to the Livy interpreter. The format for the entries should be groupId:artifactId:version.", + "configName": "livy.spark.jars.packages", "required": "false", "type": "string_array", "default": [] @@ -244,10 +298,16 @@ "name": "livy_spark_jars", "label": "Livy Spark Jars", "description": "List of HDFS paths of Jars to add to the Livy classpath.", + "configName": "livy.spark.jars", "type": "uri_array", "default": [], "allowedSchemes": [ - "hdfs" + "hdfs", + "adl", + "file", + "s3a", + "wasbs", + "wasb" ] } ], @@ -274,13 +334,15 @@ "configWriter": { "generators": [ { - "filename": "zeppelin-conf/zeppelin-site.xml", + "filename": "%SERVICENAMELOWER%-conf/zeppelin-site.xml", "configFormat": "hadoop_xml", "includedParams": [ "zeppelin_server_port", "zeppelin_server_ssl_port", "zeppelin_anonymous_allowed", "zeppelin_notebook_public", + "zeppelin_notebook_dir", + "zeppelin_conf_dir", "ssl_enabled", "ssl_server_keystore_location", "ssl_server_keystore_password", @@ -300,10 +362,6 @@ "key" : "zeppelin.war.tempdir", "value" : "${zeppelin_data_dir}/webapps" }, - { - "key" : "zeppelin.notebook.dir", - "value" : "${zeppelin_data_dir}/notebook" - }, { "key" : "zeppelin.dep.localrepo", "value" : "${zeppelin_data_dir}/local-repo" @@ -311,25 +369,45 @@ { "key" : "zeppelin.interpreter.localRepo", "value" : "${zeppelin_data_dir}/local-repo" + }, + { + "key" : "zeppelin.notebook.storage", + "value" : "org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo" + }, + { + "key" : "zeppelin.config.storage.class", + "value" : "org.apache.zeppelin.storage.FileSystemConfigStorage" } ] + }, + { + "filename": "%SERVICENAMELOWER%-conf/livy.properties", + "configFormat": "properties", + "includedParams": [ + "executor_memory", + "driver_memory", + "max_executors", + "executor_extra_options", + "livy_spark_jars_packages", + "livy_spark_jars" + ] } ], "auxConfigGenerators": [ { - "filename": "zeppelin-conf/zeppelin-env.sh", + "filename": "%SERVICENAMELOWER%-conf/zeppelin-env.sh", "sourceFilename": "aux/client/zeppelin-env.sh" }, { - "filename": "zeppelin-conf/shiro.ini", + "filename": "%SERVICENAMELOWER%-conf/shiro.ini", "sourceFilename": "aux/client/shiro.ini" }, { - "filename": "zeppelin-conf/interpreter.livy.json", + "filename": "%SERVICENAMELOWER%-conf/interpreter.livy.json", "sourceFilename": "aux/client/interpreter.livy.json" }, { - "filename": "zeppelin-conf/interpreter.json", + "filename": "%SERVICENAMELOWER%-conf/interpreter.json", "sourceFilename": "aux/client/interpreter.json" } ] diff --git a/zeppelin-csd-src/scripts/control.py b/zeppelin-csd-src/scripts/control.py new file mode 100644 index 0000000..ff27312 --- /dev/null +++ b/zeppelin-csd-src/scripts/control.py @@ -0,0 +1,204 @@ +#!/usr/bin/python +import ConfigParser +import StringIO +import json +import sys, datetime, os + +import shutil + + +def log(msg): + msg_with_ts = "%s: %s" % (datetime.datetime.now(), msg) + print >> sys.stderr, msg_with_ts + print msg_with_ts + + +def error_if_missing(path): + if not os.path.exists(path): + log("ERROR: Path does not exist: %s" % path) + sys.exit(1) + + +def properties_string_to_dict(prop_string): + ini_str = '[dummy]\n' + prop_string + ini_fp = StringIO.StringIO(ini_str) + config = ConfigParser.RawConfigParser() + config.optionxform = str + config.readfp(ini_fp) + return {k: v for k, v in config.items("dummy")} + + +def read_file(filename): + with open(filename, 'r') as openfile: + return openfile.read() + + +def get_livy_details(prop_string, spark_service_name): + maybe_hostname = [line.split(":")[0] for line in prop_string.splitlines() if + line.endswith("spark.version=%s" % spark_service_name)] + if len(maybe_hostname) == 0: + return None + hostname = maybe_hostname[0] + res = {} + for line in prop_string.splitlines(): + if line.startswith(hostname): + trimmed = line.split(":")[1] + kv = trimmed.split("=") + res[kv[0]] = kv[1] + res["livy.server.hostname"] = hostname + return res + + +def get_livy_url(livy_dict): + if livy_dict["livy.ssl"] == "true": + if "ZEPPELIN_TRUSTSTORE" not in os.environ.keys(): + log("A truststore must be specified since Livy is using SSL") + sys.exit(1) + else: + livy_protocol = "https" + else: + livy_protocol = "http" + return "%s://%s:%s" % (livy_protocol, livy_dict["livy.server.hostname"], livy_dict['livy.server.port']) + + +def generate_livy_conf_struct(spark_version, interpreter_id, interpreter_name, livy_server_properties, interpreter_json, + interpreter_properties_string): + livy_dict = get_livy_details(livy_server_properties, spark_version) + if livy_dict is None: + return None + livy_url = get_livy_url(livy_dict) + conf = json.loads(interpreter_json) + conf["option"]["perNote"] = os.environ["INTERPRETER_FOR_NOTE"] + conf["option"]["perUser"] = os.environ["INTERPRETER_FOR_USER"] + conf["id"] = interpreter_id + conf["name"] = interpreter_name + conf["properties"]["zeppelin.livy.url"] = livy_url + conf["properties"]["zeppelin.interpreter.localRepo"] = "%s/local-repo/%s" % (os.environ["ZEPPELIN_DATA_DIR"], interpreter_id) + if "ZEPPELIN_PRINCIPAL" in os.environ.keys(): + conf["properties"]["zeppelin.livy.principal"] = os.environ["ZEPPELIN_PRINCIPAL"] + conf["properties"]["zeppelin.livy.keytab"] = "%SERVICENAMELOWER%.keytab" + props = properties_string_to_dict(interpreter_properties_string) + for k, v in props.iteritems(): + conf["properties"][k] = v + return conf + + +def merge_livy_confs_into_interpreter(interpreter_string, livy_interpreters): + interpreter = json.loads(interpreter_string) + for livy in livy_interpreters: + id = livy["id"] + interpreter["interpreterSettings"][id] = livy + interpreter["interpreterBindings"]["2CYUMWD38"].append(id) + return interpreter + + +def run_command(command): + log("Running command: %s" % command) + rt = os.system(command) + if rt != 0: + log("Non-zero error code running command: %s" % command) + sys.exit(1) + + +def base_conf(): + log("Detected CDH_VERSION of [%s]" % os.environ["CDH_VERSION"]) + + # Set java path + if "JAVA_HOME" in os.environ.keys(): + log("JAVA_HOME added to path as %s" % os.environ["JAVA_HOME"]) + os.environ["PATH"] = "%s:%s" % (os.environ["JAVA_HOME"], os.environ["PATH"]) + else: + log("JAVA_HOME not set") + + # Zeppelin java opts + os.environ["ZEPPELIN_JAVA_OPTS"] = "%s -Xms%s -Xmx%s" % ( + os.environ["ZEPPELIN_EXTRA_JAVA_OPTIONS"], + os.environ["ZEPPELIN_MEMORY"], os.environ["ZEPPELIN_MEMORY"]) + + # Check and set various conf directories + base_conf_dir = os.environ["CONF_DIR"] + zeppelin_conf_dir = "%s/%SERVICENAMELOWER%-conf" % base_conf_dir + error_if_missing(zeppelin_conf_dir) + os.environ["ZEPPELIN_CONF_DIR"] = zeppelin_conf_dir + os.environ["ZEPPELIN_PID_DIR"] = "%s/run" % os.environ["ZEPPELIN_LOG_DIR"] + os.environ["ZEPPELIN_IDENT_STRING"] = "zeppelin" + return (base_conf_dir, zeppelin_conf_dir) + +def start(): + log("Attempting to start") + + # Base config + (base_conf_dir, zeppelin_conf_dir) = base_conf() + + #Start specific conf + livy_conf_dir = "%s/%LIVYSERVICENAMELOWER%-conf" % base_conf_dir + error_if_missing(livy_conf_dir) + livy_conf_file = "%s/server.properties" % livy_conf_dir + error_if_missing(livy_conf_file) + server_props_string = read_file(livy_conf_file) + livy_interpreter_json_filename = "%s/interpreter.livy.json" % zeppelin_conf_dir + error_if_missing(livy_interpreter_json_filename) + livy_interpreter_json_string = read_file(livy_interpreter_json_filename) + livy_properties_filename = "%s/livy.properties" % zeppelin_conf_dir + error_if_missing(livy_properties_filename) + livy_properties_string = read_file(livy_properties_filename) + + livy_conf_struct = generate_livy_conf_struct("spark", "2CYCWRZPP", "livy", server_props_string, + livy_interpreter_json_string, livy_properties_string) + livy2_conf_struct = generate_livy_conf_struct("spark2", "2CYCWDZPP", "livy2", server_props_string, + livy_interpreter_json_string, livy_properties_string) + + livy_interpreters = [] + if livy_conf_struct is not None: + livy_interpreters.append(livy_conf_struct) + if livy2_conf_struct is not None: + livy_interpreters.append(livy2_conf_struct) + + interpreter_json_filename = "%s/interpreter.json" % zeppelin_conf_dir + error_if_missing(interpreter_json_filename) + interpreter_string = read_file(interpreter_json_filename) + + interpreter_struct = merge_livy_confs_into_interpreter(interpreter_string, livy_interpreters) + completed_filename = "%s/interpreter.json" % zeppelin_conf_dir + with open(completed_filename, 'w+') as f: + json.dump(interpreter_struct, f) + + run_command("hdfs dfs -mkdir -p %s" % (os.environ["ZEPPELIN_CONF_FS_DIR"])) + + run_command("hdfs dfs -put -f %s %s" % (completed_filename, os.environ["ZEPPELIN_CONF_FS_DIR"])) + + run_command("hdfs dfs -mkdir -p %s" % (os.environ["ZEPPELIN_NOTEBOOK_DIR"])) + + shiro_filename = "%s/shiro.ini" % zeppelin_conf_dir + error_if_missing(interpreter_json_filename) + if os.environ["ZEPPELIN_SHIRO_ENABLED"] == "false": + shutil.move(shiro_filename, "%s.template" % shiro_filename) + + #Run zeppelin, this seems to leak threads, look for a way to clean up properly + run_command("%s/bin/zeppelin-daemon.sh --config %s start" % (os.environ["ZEPPELIN_HOME"], os.environ["ZEPPELIN_CONF_DIR"])) + + #Wait for PID to stop + run_command("tail --pid=$( cat %s/zeppelin-zeppelin-*.pid) -f /dev/null" % os.environ["ZEPPELIN_PID_DIR"]) + return + +def stop(): + log("Attempting to stop") + + # Base config + (base_conf_dir, zeppelin_conf_dir) = base_conf() + + run_command("%s/bin/zeppelin-daemon.sh --config %s stop" % (os.environ["ZEPPELIN_HOME"], os.environ["ZEPPELIN_CONF_DIR"])) + return + +if __name__ == '__main__': + log("Running Zeppelin CSD control script...") + + if len(sys.argv) < 2: + log("No argument given") + sys.exit(1) + elif sys.argv[1] == "start": + start() + elif sys.argv[1] == "stop": + stop() + else: + log("Don't understand [%s]" % sys.argv[1]) diff --git a/zeppelin-csd-src/scripts/control.sh b/zeppelin-csd-src/scripts/control.sh deleted file mode 100644 index 6405425..0000000 --- a/zeppelin-csd-src/scripts/control.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash - -set -ex - -function log { - timestamp=$(date) - echo "$timestamp: $1" #stdout - echo "$timestamp: $1" 1>&2; #stderr -} - -log "Running Zeppelin CSD control script..." -log "Detected CDH_VERSION of [$CDH_VERSION]" -log "Got command as $1" - -case $1 in - (start) - # Set java path - if [ -n "$JAVA_HOME" ]; then - log "JAVA_HOME added to path as $JAVA_HOME" - export PATH=$JAVA_HOME/bin:$PATH - else - log "JAVA_HOME not set" - fi - # Set Zeppelin conf - export ZEPPELIN_CONF_DIR="$CONF_DIR/zeppelin-conf" - if [ ! -d "$ZEPPELIN_CONF_DIR" ]; then - log "Could not find zeppelin-conf directory at $ZEPPELIN_CONF_DIR" - exit 3 - fi - # Set Livy conf - export LIVY_CONF_DIR="$CONF_DIR/livy-conf" - if [ ! -d "$LIVY_CONF_DIR" ]; then - log "Could not find livy-conf directory at $LIVY_CONF_DIR" - exit 3 - fi - # Get livy url - LIVY_CONF_FILE="$LIVY_CONF_DIR/server.properties" - if [ ! -f "$LIVY_CONF_FILE" ]; then - log "Cannot find livy config at $LIVY_CONF_FILE" - exit 3 - fi - - LIVY_INTERPRETER_CONF="$ZEPPELIN_CONF_DIR/interpreter.json" - function get_livy_url { - LIVY_HOSTNAME="$( grep "$1\$" "$LIVY_CONF_FILE" | sed 's#^\([^:]\+\):.*#\1#g' | head -1 )" - if [ -z "$LIVY_HOSTNAME" ]; then - # Spark version not found - return - fi - LIVY_SSL_ENABLED="$( grep "${LIVY_HOSTNAME}:livy\.ssl" "$LIVY_CONF_FILE" | sed 's#^.*livy\.ssl=\(.*\)#\1#g' | head -1 )" - if [ "$LIVY_SSL_ENABLED" == "true" ]; then - if [ -z "$ZEPPELIN_TRUSTSTORE" ]; then - log "A truststore must be specified since Livy is using SSL" - exit 6 - fi - LIVY_PROTOCOL=https - else - LIVY_PROTOCOL=http - fi - LIVY_PORT="$( grep "${LIVY_HOSTNAME}:livy\.server\.port" "$LIVY_CONF_FILE" | sed 's#^.*livy\.server\.port=\(.*\)#\1#g' | head -1 )" - LIVY_URL="${LIVY_PROTOCOL}://${LIVY_HOSTNAME}:${LIVY_PORT}" - echo "$LIVY_URL" - return - } - function spark_interpreter { - SPARK_VERSION="$1" #spark or spark2 - INTERPRETER_ID="$2" # some ID like 2CZ9EX8ZX - INTERPRETER_NAME="$3" # some name like livy or livy2 - LIVY_URL="$( get_livy_url "$SPARK_VERSION" )" - if [ -z "$LIVY_URL" ]; then - log "No endpoint for $SPARK_VERSION, skipping interpreter setup" - sed -i "s#{{INTERPRETER_BINDING_$SPARK_VERSION}}##g" "$LIVY_INTERPRETER_CONF" - sed -i "s#{{SPARK_CONFIG_$SPARK_VERSION}}##g" "$LIVY_INTERPRETER_CONF" - else - LIVY_TEMP_CONF="$ZEPPELIN_CONF_DIR/interpreter.livy.json.$SPARK_VERSION" - cp "$ZEPPELIN_CONF_DIR/interpreter.livy.json" "$LIVY_TEMP_CONF" - sed -i "s#{{EXECUTOR_MEMORY}}#$EXECUTOR_MEMORY#g" "$LIVY_TEMP_CONF" - sed -i "s#{{EXECUTOR_EXTRA_OPTIONS}}#$EXECUTOR_EXTRA_OPTIONS#g" "$LIVY_TEMP_CONF" - sed -i "s#{{MAX_EXECUTORS}}#$MAX_EXECUTORS#g" "$LIVY_TEMP_CONF" - sed -i "s#{{DRIVER_MEMORY}}#$DRIVER_MEMORY#g" "$LIVY_TEMP_CONF" - sed -i "s#{{LIVY_URL}}#$LIVY_URL#g" "$LIVY_TEMP_CONF" - sed -i "s#{{LIVY_PRINCIPAL}}#$ZEPPELIN_PRINCIPAL#g" "$LIVY_TEMP_CONF" - sed -i "s#{{LIVY_KEYTAB}}#zeppelin.keytab#g" "$LIVY_TEMP_CONF" - sed -i "s#{{DATA_DIR}}#$ZEPPELIN_DATA_DIR#g" "$LIVY_TEMP_CONF" - sed -i "s#{{INTERPRETER_NAME}}#$INTERPRETER_NAME#g" "$LIVY_TEMP_CONF" - sed -i "s#{{INTERPRETER_ID}}#$INTERPRETER_ID#g" "$LIVY_TEMP_CONF" - sed -i "s#{{LIVY_SPARK_JARS_PACKAGES}}#$LIVY_SPARK_JARS_PACKAGES#g" "$LIVY_TEMP_CONF" - sed -i "s#{{LIVY_SPARK_JARS}}#$LIVY_SPARK_JARS#g" "$LIVY_TEMP_CONF" - sed -e "/{{SPARK_CONFIG_$SPARK_VERSION}}/ {" -e "r $LIVY_TEMP_CONF" -e 'd' -e '}' -i "$LIVY_INTERPRETER_CONF" - sed -i "s#{{INTERPRETER_BINDING_$SPARK_VERSION}}#\"$INTERPRETER_ID\",#g" "$LIVY_INTERPRETER_CONF" - fi - } - spark_interpreter "spark" "2CYCWRZPP" "livy" - spark_interpreter "spark2" "2CYCWDZPP" "livy2" - sed -i "s#{{INTERPRETER_FOR_NOTE}}#$INTERPRETER_FOR_NOTE#g" "$LIVY_INTERPRETER_CONF" - sed -i "s#{{INTERPRETER_FOR_USER}}#$INTERPRETER_FOR_USER#g" "$LIVY_INTERPRETER_CONF" - SHIRO_CONF="$ZEPPELIN_CONF_DIR/shiro.ini" - if [ "$ZEPPELIN_SHIRO_ENABLED" == "false" ]; then - mv "$SHIRO_CONF" "${SHIRO_CONF}.template" - fi - #Add link to interpreter permissions so it is maintained between restarts - ln -s "${ZEPPELIN_DATA_DIR}/notebook-authorization.json" "${ZEPPELIN_CONF_DIR}/notebook-authorization.json" - - log "Starting the Zeppelin server" - exec env ZEPPELIN_JAVA_OPTS="-Xms$ZEPPELIN_MEMORY -Xmx$ZEPPELIN_MEMORY" $ZEPPELIN_HOME/bin/zeppelin.sh - ;; - (*) - echo "Don't understand [$1]" - exit 1 - ;; -esac diff --git a/zeppelin-csd-src/scripts/test_control.py b/zeppelin-csd-src/scripts/test_control.py new file mode 100644 index 0000000..464d083 --- /dev/null +++ b/zeppelin-csd-src/scripts/test_control.py @@ -0,0 +1,19 @@ +import unittest +import tempfile +import control + + +class TestControl(unittest.TestCase): + def test_parse_server_properties(self): + prop = """host.name1:livy.server.port=8998 +host.name1:livy.ssl=false +host.name1:spark.version=spark2 +host.name2:spark.version=spark +""" + parsed = control.get_livy_details(prop, "spark2") + self.assertEqual(parsed, {'livy.server.hostname': 'host.name1', 'livy.server.port': '8998', 'livy.ssl': 'false', + 'spark.version': 'spark2'}) + + +if __name__ == '__main__': + unittest.main() diff --git a/zeppelin-parcel-src/meta/parcel.json b/zeppelin-parcel-src/meta/parcel.json index c7ee6bf..e204984 100644 --- a/zeppelin-parcel-src/meta/parcel.json +++ b/zeppelin-parcel-src/meta/parcel.json @@ -1,14 +1,14 @@ { "schema_version": 1, - "name": "ZEPPELIN", + "name": "%SERVICENAME%", "version": "%VERSION%", "setActiveSymlink": true, "provides": [ - "zeppelin" + "%SERVICENAMELOWER%" ], - "depends": "LIVY", + "depends": "%LIVYSERVICENAME%", "replaces": "", "conflicts": "",