[Write stall] CD pipeline integration (#2817)

* update e2e tests in cd pipeline

* update steps

* add some installation

* move it appropriate place

* small fix

* small fix

* replacing od type

* adding sudo while running docker

* adding sudo while running docker

* test on kokoro

tools/cd_scripts/e2e_test.sh

@@ -142,6 +142,7 @@ git checkout $(sed -n 2p ~/details.txt) |& tee -a ~/logs.txt
 set +e
 # Test directory arrays
 TEST_DIR_PARALLEL=(
+  "monitoring"
   "local_file"
   "log_rotation"
   "mounting"
@@ -285,6 +286,10 @@ function run_e2e_tests_for_hns_bucket(){
    return 0
 }
 
+function run_e2e_tests_for_emulator() {
+  ./tools/integration_tests/emulator_tests/emulator_tests.sh true > ~/logs-emulator.txt
+}
+
 function gather_test_logs() {
   readarray -t test_logs_array < "$TEST_LOGS_FILE"
   rm "$TEST_LOGS_FILE"
@@ -313,6 +318,12 @@ echo "Running integration tests for FLAT bucket..."
 run_e2e_tests_for_flat_bucket &
 e2e_tests_flat_bucket_pid=$!
 
+run_e2e_tests_for_emulator &
+e2e_tests_emulator_pid=$!
+
+wait $e2e_tests_emulator_pid
+e2e_tests_emulator_status=$?
+
 wait $e2e_tests_flat_bucket_pid
 e2e_tests_flat_bucket_status=$?
 
@@ -338,4 +349,14 @@ else
     gsutil cp success-hns.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/
 fi
 gsutil cp ~/logs-hns.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/
+
+if [ $e2e_tests_emulator_status != 0 ];
+then
+    echo "Test failures detected in emulator based tests." &>> ~/logs-emulator.txt
+else
+    touch success-emulator.txt
+    gsutil cp success-emulator.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/
+fi
+
+gsutil cp ~/logs-emulator.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/
 '

tools/integration_tests/emulator_tests/emulator_tests.sh

@@ -19,8 +19,8 @@ set -eo pipefail
 # Display commands being run
 set -x
 
-architecture=$(dpkg --print-architecture)
-if [ $architecture == "arm64" ];then
+uname=$(uname -m)
+if [ $uname == "aarch64" ];then
   # TODO: Remove this when we have an ARM64 image for the storage test bench.(b/384388821)
   echo "These tests will not run for arm64 machine..."
   exit 0
@@ -39,6 +39,38 @@ if [ "$minor_ver" -lt "$min_minor_ver" ]; then
     exit 0
 fi
 
+# Install dependencies
+# Ubuntu/Debian based machine.
+if [ -f /etc/debian_version ]; then
+  if grep -q "Ubuntu" /etc/os-release; then
+    os="ubuntu"
+  elif grep -q "Debian" /etc/os-release; then
+    os="debian"
+  fi
+
+  sudo apt-get update
+  sudo apt-get install -y ca-certificates curl
+  sudo install -m 0755 -d /etc/apt/keyrings
+  sudo curl -fsSL https://download.docker.com/linux/${os}/gpg -o /etc/apt/keyrings/docker.asc
+  sudo chmod a+r /etc/apt/keyrings/docker.asc
+  # Add the repository to Apt sources:
+  echo \
+    "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/${os} \
+    $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+    sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+  sudo apt-get update
+  sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+  sudo apt-get install -y lsof
+# RHEL/CentOS based machine.
+elif [ -f /etc/redhat-release ]; then
+    sudo dnf -y install dnf-plugins-core
+    sudo dnf config-manager --add-repo https://download.docker.com/linux/rhel/docker-ce.repo
+    sudo dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+    sudo usermod -aG docker $USER
+    sudo systemctl start docker
+    sudo yum -y install lsof
+fi
+
 export STORAGE_EMULATOR_HOST="http://localhost:9000"
 
 DEFAULT_IMAGE_NAME='gcr.io/cloud-devrel-public-resources/storage-testbench'
@@ -54,17 +86,17 @@ CONTAINER_NAME=storage_testbench
 DOCKER_NETWORK="--net=host"
 
 # Get the docker image for the testbench
-docker pull $DOCKER_IMAGE
+sudo docker pull $DOCKER_IMAGE
 
 # Start the testbench
-docker run --name $CONTAINER_NAME --rm -d $DOCKER_NETWORK $DOCKER_IMAGE
+sudo docker run --name $CONTAINER_NAME --rm -d $DOCKER_NETWORK $DOCKER_IMAGE
 echo "Running the Cloud Storage testbench: $STORAGE_EMULATOR_HOST"
 sleep 5
 
 # Stop the testbench & cleanup environment variables
 function cleanup() {
     echo "Cleanup testbench"
-    docker stop $CONTAINER_NAME
+    sudo docker stop $CONTAINER_NAME
     unset STORAGE_EMULATOR_HOST;
 }
 trap cleanup EXIT