add dependency

.gitignore

@@ -67,4 +67,15 @@ submarine-workbench/workbench-web/.env.*.local
 submarine-workbench/workbench-web/npm-debug.log*
 submarine-workbench/workbench-web/yarn-debug.log*
 submarine-workbench/workbench-web/yarn-error.log*
+submarine-workbench/workbench-web-ng/node
+submarine-workbench/workbench-web-ng/node_modules
+submarine-workbench/workbench-web-ng/dist
+submarine-workbench/workbench-web-ng/package-lock.json
+submarine-workbench/workbench-web-ng/npm-debug.log*
 submarine-test/e2e/Driver
+
+# interpreter temp files
+derby.log
+submarine-workbench/interpreter/spark-interpreter/metastore_db/
+spark-1.*-bin-hadoop*
+.spark-dist

.travis.yml

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-sudo: false
+sudo: required
 
 before_cache:
   - sudo chown -R travis:travis $HOME/.m2
@@ -22,8 +22,10 @@ cache:
   apt: true
   directories:
     - ${HOME}/.m2
-    -  submarine-workbench/workbench-web/node
-    -  submarine-workbench/workbench-web/node_modules
+    - submarine-workbench/workbench-web/node
+    - submarine-workbench/workbench-web/node_modules
+    - submarine-workbench/workbench-web-ng/node
+    - submarine-workbench/workbench-web-ng/node_modules
 
 addons:
   apt:
@@ -38,19 +40,26 @@ services:
 
 env:
   global:
-    # submarine core does not required by workbench-server integration tests
     # If you need to compile Phadoop-3.1 or Phadoop-3.2, you need to add `!submarine-server/server-submitter/submitter-yarnservice` in EXCLUDE_SUBMARINE
-    - EXCLUDE_SUBMARINE="!submarine-all,!submarine-client,!submarine-commons,!submarine-commons/commons-runtime,!submarine-dist,!submarine-server/server-submitter/submitter-yarn"
-    - EXCLUDE_WORKBENCH="!submarine-workbench,!submarine-workbench/workbench-web,!submarine-workbench/workbench-server"
+    - EXCLUDE_SUBMARINE="!submarine-all,!submarine-client,!submarine-commons,!submarine-commons/commons-runtime,!submarine-dist,!submarine-server/server-submitter/submitter-yarn,!submarine-server/server-core"
+    - EXCLUDE_WORKBENCH="!submarine-workbench,!submarine-workbench/workbench-web"
     - EXCLUDE_INTERPRETER="!submarine-workbench/interpreter,!submarine-workbench/interpreter/interpreter-engine,!submarine-workbench/interpreter/python-interpreter,!submarine-workbench/interpreter/spark-interpreter""
     - EXCLUDE_SUBMODULE_TONY="!submodules/tony,!submodules/tony/tony-mini,!submodules/tony/tony-core,!submodules/tony/tony-proxy,!submodules/tony/tony-portal,!submodules/tony/tony-azkaban,!submodules/tony/tony-cli"
+    - EXCLUDE_K8S="!submarine-server/server-submitter/submitter-k8s"
 
 before_install:
+  # maven 3.6.1 (3.6.2 build tony failed!!!)
+  - echo "Download Maven 3.6.1"
+  - wget https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz
+  - tar zxvf apache-maven-3.6.1-bin.tar.gz || travis_terminate 1
+  - export M2_HOME=$PWD/apache-maven-3.6.1
+  - export PATH=$M2_HOME/bin:$PATH
+  # mysql
   - sudo service mysql restart
-  - mysql -e "create database submarineDB_test;"
+  - mysql -e "create database submarine_test;"
   - mysql -e "CREATE USER 'submarine_test'@'%' IDENTIFIED BY 'password_test';"
-  - mysql -e "GRANT ALL PRIVILEGES ON * . * TO 'submarine_test'@'%';"
-  - mysql -e "use submarineDB_test; source ./docs/database/submarine.sql; show tables;"
+  - mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'submarine_test'@'%';"
+  - mysql -e "use submarine_test; source ./docs/database/submarine.sql; show tables;"
   - ./dev-support/travis/install_external_dependencies.sh
 
 matrix:
@@ -65,25 +74,25 @@ matrix:
     - language: java
       jdk: "openjdk8"
       dist: xenial
-      env: NAME="Build hadoop-2.7" PROFILE="-Phadoop-2.7" BUILD_FLAG="clean package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am" MODULES="-pl \"${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},!submarine-dist\"" TEST_PROJECTS=""
+      env: NAME="Build hadoop-2.7" PROFILE="-Phadoop-2.7" BUILD_FLAG="clean package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am" MODULES="-pl \"${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},!submarine-dist\"" TEST_PROJECTS=""
 
     # Build hadoop-2.9(default)
     - language: java
       jdk: "openjdk8"
       dist: xenial
-      env: NAME="Build hadoop-2.9" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am" MODULES="-pl \"${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},!submarine-dist\"" TEST_PROJECTS=""
+      env: NAME="Build hadoop-2.9" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am" MODULES="-pl \"${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},!submarine-dist\"" TEST_PROJECTS=""
 
     # Build hadoop-3.1
     - language: java
       jdk: "openjdk8"
       dist: xenial
-      env: NAME="Build hadoop-3.1" PROFILE="-Phadoop-3.1" BUILD_FLAG="clean package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am" MODULES="-pl \"${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},!submarine-dist\"" TEST_PROJECTS=""
+      env: NAME="Build hadoop-3.1" PROFILE="-Phadoop-3.1" BUILD_FLAG="clean package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am" MODULES="-pl \"${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},!submarine-dist\"" TEST_PROJECTS=""
 
     # Build hadoop-3.2
     - language: java
       jdk: "openjdk8"
       dist: xenial
-      env: NAME="Build hadoop-3.2" PROFILE="-Phadoop-3.2" BUILD_FLAG="clean package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am" MODULES="-pl \"${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},!submarine-dist\"" TEST_PROJECTS=""
+      env: NAME="Build hadoop-3.2" PROFILE="-Phadoop-3.2" BUILD_FLAG="clean package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am" MODULES="-pl \"${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},!submarine-dist\"" TEST_PROJECTS=""
 
     # Build workbench-web
     - language: node_js
@@ -98,17 +107,11 @@ matrix:
         - yarn run lint --no-fix && yarn run build
       env: NAME="Build workbench-web"
 
-    # Test workbench-server
-    - language: java
-      jdk: "openjdk8"
-      dist: xenial
-      env: NAME="Test workbench-server" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_SUBMARINE},${EXCLUDE_INTERPRETER}" TEST_MODULES="-pl submarine-workbench/workbench-server" TEST_PROJECTS=""
-
     # Test workbench-web
     - language: java
       jdk: "openjdk8"
       dist: xenial
-      env: NAME="Test workbench-web" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_SUBMARINE},${EXCLUDE_INTERPRETER}" TEST_MODULES="-pl submarine-workbench/workbench-web" TEST_PROJECTS=""
+      env: NAME="Test workbench-web" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_K8S},${EXCLUDE_SUBMARINE},${EXCLUDE_INTERPRETER}" TEST_MODULES="-pl submarine-workbench/workbench-web" TEST_PROJECTS=""
 
     # Test test-e2e
     - language: java
@@ -120,7 +123,7 @@ matrix:
     - language: java
       jdk: "openjdk8"
       dist: xenial
-      env: NAME="Test interpreter" PYTHON="3" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_SUBMARINE},${EXCLUDE_WORKBENCH},${EXCLUDE_SUBMODULE_TONY}" TEST_MODULES="-pl $(echo ${EXCLUDE_INTERPRETER} | sed 's/!//g')" TEST_PROJECTS=""
+      env: NAME="Test interpreter" PYTHON="3" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_K8S},${EXCLUDE_SUBMARINE},${EXCLUDE_WORKBENCH},${EXCLUDE_SUBMODULE_TONY}" TEST_MODULES="-pl $(echo ${EXCLUDE_INTERPRETER} | sed 's/!//g')" TEST_PROJECTS=""
 
     # Test submarine-sdk
     - language: python
@@ -137,7 +140,58 @@ matrix:
     - language: java
       jdk: "openjdk8"
       dist: xenial
-      env: NAME="Test submarine distribution" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="" TEST_MODULES="" TEST_PROJECTS=""
+      env: NAME="Test submarine distribution" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_K8S}" TEST_MODULES="-pl ${EXCLUDE_K8S}" TEST_PROJECTS=""
+
+    # Test submarine web-ng
+    - language: node_js
+      node_js:
+        - "10"
+      before_install:
+        - cd submarine-workbench/workbench-web-ng
+      before_script:
+        npm install
+      addons:
+        apt:
+          sources:
+            - google-chrome
+          packages:
+            - google-chrome-stable
+      script:
+        - npm run test -- --no-watch --no-progress --browsers=ChromeHeadlessCI
+        - npm run e2e -- --protractor-config=e2e/protractor-ci.conf.js
+      env: NAME="Build workbench-web-ng"
+
+    # Test submarine-server
+    - language: java
+      jdk: "openjdk8"
+      dist: xenial
+      env: NAME="Test submarine-server" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER}" TEST_MODULES="-pl submarine-server/server-core" TEST_PROJECTS=""
+
+    - name: Submarine on Kubernetes
+      dist: xenial
+      services: docker
+      language: java
+      jdk: openjdk8
+      env: BUILD_FLAG="clean package install -DskipTests -am" TEST_FLAG="test -am" MODULES="-pl org.apache.submarine:submitter-k8s" TEST_MODULES="-pl org.apache.submarine:submitter-k8s" TEST_PROJECTS=""
+      before_install:
+        # deploy Kubernetes cluster
+        - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
+        - chmod +x kubectl
+        - sudo mv kubectl /usr/bin
+        - docker run -d --name kube --privileged -p 8443:8443 -p 10080:10080 bsycorp/kind:latest-1.15
+        - until curl -s --fail http://127.0.0.1:10080/kubernetes-ready; do
+            sleep 1;
+          done
+        - echo "Kubernetes ready - run tests!"
+        - mkdir $HOME/.kube
+        - curl http://127.0.0.1:10080/config > $HOME/.kube/config
+        - export KUBECONFIG=$HOME/.kube/config
+        - echo $KUBECONFIG
+        - kubectl config set clusters.kubernetes.server https://127.0.0.1:8443
+        - kubectl get nodes
+        - kubectl create namespace kubeflow
+        - kubectl apply -f ./dev-support/travis/tf-operator/crd_v1.yaml
+        - kubectl apply -f ./dev-support/travis/tf-operator/tfevent-volume/.
 
 install:
   - mvn --version

bin/common.sh

@@ -42,27 +42,27 @@ if [[ -f "${SUBMARINE_CONF_DIR}/submarine-env.sh" ]]; then
   . "${SUBMARINE_CONF_DIR}/submarine-env.sh"
 fi
 
-WORKBENCH_CLASSPATH+=":${SUBMARINE_CONF_DIR}"
+SUBMARINE_SERVER_CLASSPATH+=":${SUBMARINE_CONF_DIR}"
 
 function add_each_jar_in_dir(){
   if [[ -d "${1}" ]]; then
     for jar in $(find -L "${1}" -maxdepth 1 -name '*jar'); do
-      WORKBENCH_CLASSPATH="$jar:$WORKBENCH_CLASSPATH"
+      SUBMARINE_SERVER_CLASSPATH="$jar:$SUBMARINE_SERVER_CLASSPATH"
     done
   fi
 }
 
 function add_each_jar_in_dir_recursive(){
   if [[ -d "${1}" ]]; then
     for jar in $(find -L "${1}" -type f -name '*jar'); do
-      WORKBENCH_CLASSPATH="$jar:$WORKBENCH_CLASSPATH"
+      SUBMARINE_SERVER_CLASSPATH="$jar:$SUBMARINE_SERVER_CLASSPATH"
     done
   fi
 }
 
 function add_jar_in_dir(){
   if [[ -d "${1}" ]]; then
-    WORKBENCH_CLASSPATH="${1}/*:${WORKBENCH_CLASSPATH}"
+    SUBMARINE_SERVER_CLASSPATH="${1}/*:${SUBMARINE_SERVER_CLASSPATH}"
   fi
 }
 
@@ -74,11 +74,11 @@ function download_mysql_jdbc_jar(){
     MYSQL_JAR_URL="https://repo1.maven.org/maven2/mysql/mysql-connector-java/${MYSQL_VERSION}/mysql-connector-java-${MYSQL_VERSION}.jar"
   fi
   echo "Downloading mysql jdbc jar from ${MYSQL_JAR_URL}."
-  wget ${MYSQL_JAR_URL} -P "${BIN}/../workbench/lib"
-  echo "Mysql jdbc jar is downloaded and put in the path of workbench/lib."
+  wget ${MYSQL_JAR_URL} -P "${BIN}/../lib"
+  echo "Mysql jdbc jar is downloaded and put in the path of submarine/lib."
 }
 
-JAVA_OPTS+=" ${WORKBENCH_JAVA_OPTS} -Dfile.encoding=UTF-8 ${WORKBENCH_MEM}"
+JAVA_OPTS+=" ${SUBMARINE_SERVER_JAVA_OPTS} -Dfile.encoding=UTF-8 ${SUBMARINE_SERVER_MEM}"
 JAVA_OPTS+=" -Dlog4j.configuration=file://${SUBMARINE_CONF_DIR}/log4j.properties"
 export JAVA_OPTS
 

bin/workbench-daemon.sh → bin/submarine-daemon.sh

@@ -19,7 +19,7 @@
 # description: Start and stop daemon script for.
 #
 
-USAGE="-e Usage: workbench-daemon.sh {start|stop|restart|status}"
+USAGE="-e Usage: submarine-daemon.sh {start|stop|restart|status}"
 
 if [ -L ${BASH_SOURCE-$0} ]; then
   BIN=$(dirname $(readlink "${BASH_SOURCE-$0}"))
@@ -33,13 +33,12 @@ GET_MYSQL_JAR=false
 
 cd ${BIN}/>/dev/null
 
-WORKBENCH_NAME="Submarine Workbench"
-WORKBENCH_LOGFILE="${SUBMARINE_LOG_DIR}/workbench.log"
-WORKBENCH_MAIN=org.apache.submarine.server.WorkbenchServer
-JAVA_OPTS+=" -Dworkbench.log.file=${WORKBENCH_LOGFILE}"
+SUBMARINE_SERVER_NAME="Submarine Server"
+SUBMARINE_SERVER_LOGFILE="${SUBMARINE_LOG_DIR}/submarine.log"
+SUBMARINE_SERVER_MAIN=org.apache.submarine.server.SubmarineServer
+JAVA_OPTS+=" -Dsubmarine.log.file=${SUBMARINE_SERVER_LOGFILE}"
 
-add_jar_in_dir "${BIN}/../workbench"
-add_jar_in_dir "${BIN}/../workbench/lib"
+add_jar_in_dir "${BIN}/../lib"
 
 function initialize_default_directories() {
   if [[ ! -d "${SUBMARINE_LOG_DIR}" ]]; then
@@ -48,26 +47,26 @@ function initialize_default_directories() {
   fi
 }
 
-function found_workbench_server_pid() {
-  process='WorkbenchServer';
+function found_submarine_server_pid() {
+  process='SubmarineServer';
   RUNNING_PIDS=$(ps x | grep ${process} | grep -v grep | awk '{print $1}');
 
   if [[ -z "${RUNNING_PIDS}" ]]; then
     return
   fi
 
   if ! kill -0 ${RUNNING_PIDS} > /dev/null 2>&1; then
-    echo "${WORKBENCH_NAME} running but process is dead"
+    echo "${SUBMARINE_SERVER_NAME} running but process is dead"
   fi
 
   echo "${RUNNING_PIDS}"
 }
 
-function wait_for_workbench_to_die() {
+function wait_for_submarine_server_to_die() {
   local pid
   local count
 
-  pid=`found_workbench_server_pid`
+  pid=`found_submarine_server_pid`
   timeout=10
   count=0
   timeoutTime=$(date "+%s")
@@ -98,9 +97,9 @@ function check_jdbc_jar() {
       if [[ ${GET_MYSQL_JAR} = true ]]; then
         download_mysql_jdbc_jar
       else
-        echo -e "\\033[31mError: There is no mysql jdbc jar in workbench/lib.\\033[0m"
-        echo -e "\\033[31mPlease download a mysql jdbc jar and put it under workbench/lib manually.\\033[0m"
-        echo -e "\\033[31mOr add a parameter getMysqlJar, like this:\n./bin/workbench-daemon.sh start getMysqlJar\\033[0m"
+        echo -e "\\033[31mError: There is no mysql jdbc jar in lib.\\033[0m"
+        echo -e "\\033[31mPlease download a mysql jdbc jar and put it under lib manually.\\033[0m"
+        echo -e "\\033[31mOr add a parameter getMysqlJar, like this:\n./bin/submarine-daemon.sh start getMysqlJar\\033[0m"
         echo -e "\\033[31mIt would download mysql jdbc jar automatically.\\033[0m"
         exit 1
       fi
@@ -111,53 +110,53 @@ function check_jdbc_jar() {
 function start() {
   local pid
 
-  pid=`found_workbench_server_pid`
+  pid=`found_submarine_server_pid`
   if [[ ! -z "$pid" && "$pid" != 0 ]]; then
-    echo "${WORKBENCH_NAME}:${pid} is already running"
+    echo "${SUBMARINE_SERVER_NAME}:${pid} is already running"
     return 0;
   fi
 
-  check_jdbc_jar "${BIN}/../workbench/lib"
-  
+  check_jdbc_jar "${BIN}/../lib"
+
   initialize_default_directories
 
-  echo "WORKBENCH_CLASSPATH: ${WORKBENCH_CLASSPATH}" >> "${WORKBENCH_LOGFILE}"
+  echo "SUBMARINE_SERVER_CLASSPATH: ${SUBMARINE_SERVER_CLASSPATH}" >> "${SUBMARINE_SERVER_LOGFILE}"
 
-  nohup $JAVA_RUNNER $JAVA_OPTS -cp $WORKBENCH_CLASSPATH $WORKBENCH_MAIN >> "${WORKBENCH_LOGFILE}" 2>&1 < /dev/null &
+  nohup $JAVA_RUNNER $JAVA_OPTS -cp $SUBMARINE_SERVER_CLASSPATH $SUBMARINE_SERVER_MAIN >> "${SUBMARINE_SERVER_LOGFILE}" 2>&1 < /dev/null &
   pid=$!
   if [[ ! -z "${pid}" ]]; then
-    echo "${WORKBENCH_NAME} start"
+    echo "${SUBMARINE_SERVER_NAME} start"
     return 1;
   fi
 }
 
 function stop() {
   local pid
-  pid=`found_workbench_server_pid`
+  pid=`found_submarine_server_pid`
 
   if [[ -z "$pid" ]]; then
-    echo "${WORKBENCH_NAME} is not running"
+    echo "${SUBMARINE_SERVER_NAME} is not running"
     return 0;
   else
     # submarine workbench daemon kill
-    wait_for_workbench_to_die
-    echo "${WORKBENCH_NAME} stop"
+    wait_for_submarine_server_to_die
+    echo "${SUBMARINE_SERVER_NAME} stop"
   fi
 }
 
-function find_workbench_process() {
+function find_submarine_server_process() {
   local pid
-  pid=`found_workbench_server_pid`
+  pid=`found_submarine_server_pid`
 
   if [[ -z "$pid" ]]; then
-    echo "${WORKBENCH_NAME} is not running"
+    echo "${SUBMARINE_SERVER_NAME} is not running"
     return 1
   else
     if ! kill -0 ${pid} > /dev/null 2>&1; then
-      echo "${WORKBENCH_NAME} running but process is dead"
+      echo "${SUBMARINE_SERVER_NAME} running but process is dead"
       return 1
     else
-      echo "${WORKBENCH_NAME} is running"
+      echo "${SUBMARINE_SERVER_NAME} is running"
     fi
   fi
 }
@@ -174,12 +173,12 @@ case "${1}" in
     stop
     ;;
   restart)
-    echo "${WORKBENCH_NAME} is restarting" >> "${WORKBENCH_LOGFILE}"
+    echo "${SUBMARINE_SERVER_NAME} is restarting" >> "${SUBMARINE_SERVER_LOGFILE}"
     stop
     start
     ;;
   status)
-    find_workbench_process
+    find_submarine_server_process
     ;;
   *)
     echo ${USAGE}