diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index 1f5af4ee2d..d780f683e8 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -252,7 +252,6 @@ jobs: run: | export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server ulimit -s unlimited - . ./scripts/config_hdfs.sh ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }} build_ASAN: @@ -392,7 +391,6 @@ jobs: run: | export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server ulimit -s unlimited - . ./scripts/config_hdfs.sh ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }} # TODO(yingchun): Build and test UBSAN version would cost a very long time, we will run these tests @@ -534,7 +532,6 @@ jobs: # run: | # export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server # ulimit -s unlimited -# . ./scripts/config_hdfs.sh # ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }} build_with_jemalloc: @@ -647,7 +644,6 @@ jobs: run: | export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server ulimit -s unlimited - . ./scripts/config_hdfs.sh ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }} build_pegasus_on_macos: diff --git a/run.sh b/run.sh index 3c86f669ca..dd81061022 100755 --- a/run.sh +++ b/run.sh @@ -784,7 +784,7 @@ function run_start_onebox() exit 1 fi - echo "HDFS_SERVICE_ARGS $HDFS_SERVICE_ARGS" + source "${ROOT}"/scripts/config_hdfs.sh if [ $USE_PRODUCT_CONFIG == "true" ]; then [ -z "${CONFIG_FILE}" ] && CONFIG_FILE=${ROOT}/src/server/config.ini [ ! -f "${CONFIG_FILE}" ] && { echo "${CONFIG_FILE} is not exist"; exit 1; } @@ -1028,6 +1028,7 @@ function run_start_onebox_instance() esac shift done + source "${ROOT}"/scripts/config_hdfs.sh if [ $META_ID = "0" -a $REPLICA_ID = "0" -a $COLLECTOR_ID = "0" ]; then echo "ERROR: no meta_id or replica_id or collector set" exit 1 diff --git a/scripts/config_hdfs.sh b/scripts/config_hdfs.sh index caa1cb342e..d9c900be3c 100755 --- a/scripts/config_hdfs.sh +++ b/scripts/config_hdfs.sh @@ -19,6 +19,8 @@ set -e +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ROOT=$(dirname "${SCRIPT_DIR}") # This file should be sourced to set up LD_LIBRARY_PATH and CLASSPATH to # run Pegasus binaries which use libhdfs in the context of a dev environment. @@ -39,11 +41,19 @@ fi JAVA_JVM_LIBRARY_DIR=$(dirname $(find "${JAVA_HOME}/" -name libjvm.so | head -1)) export LD_LIBRARY_PATH=${JAVA_JVM_LIBRARY_DIR}:$LD_LIBRARY_PATH +if [ ! -d "$HADOOP_HOME" ]; then + PEGASUS_HADOOP_HOME=${ROOT}/hadoop-bin + if [ ! -d "$PEGASUS_HADOOP_HOME" ]; then + "${SCRIPT_DIR}"/download_hadoop.sh hadoop-bin + fi + + # Set the HADOOP_HOME to the pegasus's hadoop directory. + export HADOOP_HOME="${PEGASUS_HADOOP_HOME}" + echo "set HADOOP_HOME to ${PEGASUS_HADOOP_HOME}" +fi + # Set CLASSPATH to all the Hadoop jars needed to run Hadoop itself as well as # the right configuration directory containing core-site.xml or hdfs-site.xml. -PEGASUS_HADOOP_HOME=`pwd`/hadoop-bin -# Prefer the HADOOP_HOME set in the environment, but use the pegasus's hadoop dir otherwise. -export HADOOP_HOME="${HADOOP_HOME:-${PEGASUS_HADOOP_HOME}}" if [ ! -d "$HADOOP_HOME/etc/hadoop" ] || [ ! -d "$HADOOP_HOME/share/hadoop" ]; then echo "HADOOP_HOME must be set to the location of your Hadoop jars and core-site.xml." return 1