feat: source config_hdfs.sh automatically before starting onebox

apache · Dec 4, 2023 · bb39cd9 · bb39cd9
1 parent 50563ea
commit bb39cd9
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 8 deletions.
diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml
@@ -252,7 +252,6 @@ jobs:
         run: |
           export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
           ulimit -s unlimited
-          . ./scripts/config_hdfs.sh
           ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }}
 
   build_ASAN:
@@ -392,7 +391,6 @@ jobs:
         run: |
           export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
           ulimit -s unlimited
-          . ./scripts/config_hdfs.sh
           ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }}
 
 # TODO(yingchun): Build and test UBSAN version would cost a very long time, we will run these tests
@@ -534,7 +532,6 @@ jobs:
 #        run: |
 #          export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
 #          ulimit -s unlimited
-#          . ./scripts/config_hdfs.sh
 #          ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }}
 
   build_with_jemalloc:
@@ -647,7 +644,6 @@ jobs:
         run: |
           export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
           ulimit -s unlimited
-          . ./scripts/config_hdfs.sh
           ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }}
 
   build_pegasus_on_macos:

diff --git a/run.sh b/run.sh
@@ -784,7 +784,7 @@ function run_start_onebox()
         exit 1
     fi
 
-    echo "HDFS_SERVICE_ARGS $HDFS_SERVICE_ARGS"
+    source "${ROOT}"/scripts/config_hdfs.sh
     if [ $USE_PRODUCT_CONFIG == "true" ]; then
         [ -z "${CONFIG_FILE}" ] && CONFIG_FILE=${ROOT}/src/server/config.ini
         [ ! -f "${CONFIG_FILE}" ] && { echo "${CONFIG_FILE} is not exist"; exit 1; }
@@ -1028,6 +1028,7 @@ function run_start_onebox_instance()
         esac
         shift
     done
+    source "${ROOT}"/scripts/config_hdfs.sh
     if [ $META_ID = "0" -a $REPLICA_ID = "0" -a $COLLECTOR_ID = "0" ]; then
         echo "ERROR: no meta_id or replica_id or collector set"
         exit 1

diff --git a/scripts/config_hdfs.sh b/scripts/config_hdfs.sh
@@ -19,6 +19,8 @@
 
 set -e
 
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+ROOT=$(dirname "${SCRIPT_DIR}")
 # This file should be sourced to set up LD_LIBRARY_PATH and CLASSPATH to
 # run Pegasus binaries which use libhdfs in the context of a dev environment.
 
@@ -39,11 +41,19 @@ fi
 JAVA_JVM_LIBRARY_DIR=$(dirname $(find "${JAVA_HOME}/" -name libjvm.so  | head -1))
 export LD_LIBRARY_PATH=${JAVA_JVM_LIBRARY_DIR}:$LD_LIBRARY_PATH
 
+if [ ! -d "$HADOOP_HOME" ]; then
+  PEGASUS_HADOOP_HOME=${ROOT}/hadoop-bin
+  if [ ! -d "$PEGASUS_HADOOP_HOME" ]; then
+    "${SCRIPT_DIR}"/download_hadoop.sh hadoop-bin
+  fi
+
+  # Set the HADOOP_HOME to the pegasus's hadoop directory.
+  export HADOOP_HOME="${PEGASUS_HADOOP_HOME}"
+  echo "set HADOOP_HOME to ${PEGASUS_HADOOP_HOME}"
+fi
+
 # Set CLASSPATH to all the Hadoop jars needed to run Hadoop itself as well as
 # the right configuration directory containing core-site.xml or hdfs-site.xml.
-PEGASUS_HADOOP_HOME=`pwd`/hadoop-bin
-# Prefer the HADOOP_HOME set in the environment, but use the pegasus's hadoop dir otherwise.
-export HADOOP_HOME="${HADOOP_HOME:-${PEGASUS_HADOOP_HOME}}"
 if [ ! -d "$HADOOP_HOME/etc/hadoop" ] || [ ! -d "$HADOOP_HOME/share/hadoop" ]; then
   echo "HADOOP_HOME must be set to the location of your Hadoop jars and core-site.xml."
   return 1