Merge pull request #80 from rakshithakamath94/job-code_compilation

Update the job code_compilation

Author: rakshithakamath94

build_scripts/code-compilation/client.sh

@@ -32,7 +32,14 @@ do
     cd /mnt/nfs
     if [ $once -eq 0 ]
     then
-        yum -y install bison flex cmake gcc-c++ libacl-devel krb5-devel dbus-devel libnfsidmap-devel libwbclient-devel libcap-devel libblkid-devel rpm-build redhat-rpm-config
+        if [ "${CENTOS_VERSION}" == "7" ]; then
+          yum -y install bison flex cmake gcc-c++ libacl-devel krb5-devel dbus-devel libnfsidmap-devel libwbclient-devel libcap-devel libblkid-devel rpm-build redhat-rpm-config userspace-rcu-devel userspace-rcu libnsl2-devel libcephfs-devel
+        elif [ "${CENTOS_VERSION}" == "8s" ]; then
+          yum -y install bison flex cmake gcc-c++ libacl-devel krb5-devel dbus-devel libcap-devel libblkid-devel rpm-build redhat-rpm-config glusterfs-api
+          yum -y --enablerepo=powertools install libnfsidmap-devel libwbclient-devel userspace-rcu-devel userspace-rcu libnsl2-devel libcephfs-devel
+        else
+          echo "Check the parameter $CENTOS_VERSION"
+        fi
         git clone --depth=1 https://review.gerrithub.io/ffilz/nfs-ganesha
     fi
     cd nfs-ganesha

build_scripts/common/basic-gluster.sh

@@ -41,7 +41,7 @@ systemctl start rpcbind
 echo 'TODO: this is BAD, needs a fix in the selinux-policy'
 setenforce 0
 
-systemctl stop firewalld
+systemctl stop firewalld || true
 
 if [ -n "${YUM_REPO}" ]
 then
@@ -69,13 +69,14 @@ else
 	GIT_URL="https://${GERRIT_HOST}/${GERRIT_PROJECT}"
 
         BASE_PACKAGES="git bison flex cmake gcc-c++ libacl-devel krb5-devel dbus-devel rpm-build redhat-rpm-config"
+        BUILDREQUIRES_EXTRA="libnsl2-devel libnfsidmap-devel libwbclient-devel libcephfs-devel userspace-rcu-devel"
         if [ "${CENTOS_VERSION}" = "7" ]; then
             yum -y install libgfapi-devel
             yum -y install ${BASE_PACKAGES} libnfsidmap-devel libwbclient-devel libcap-devel libblkid-devel userspace-rcu-devel userspace-rcu
-        elif [ "${CENTOS_VERSION}" = "8-stream" ]; then
-            yum -y install libgfapi-devel
-            yum -y install ${BASE_PACKAGES}
-            yum -y --enablerepo=powertools install libnfsidmap-devel libwbclient-devel libcap-devel libblkid-devel userspace-rcu-devel
+        elif [ "${CENTOS_VERSION}" = "8s" ]; then
+            yum install -y ${BASE_PACKAGES} libacl-devel libblkid-devel libcap-devel redhat-rpm-config rpm-build libgfapi-devel xfsprogs-devel python2-devel
+            yum install --enablerepo=powertools -y ${BUILDREQUIRES_EXTRA}
+            yum -y install selinux-policy-devel sqlite
         fi
 
 	git init "${GIT_REPO}"
@@ -107,7 +108,7 @@ else
 		ntirpc_version=$(rpm -q --qf '%{VERSION}-%{RELEASE}' -p ${rpm_arch}/libntirpc-devel*.rpm)
 		ntirpc_rpm=${rpm_arch}/libntirpc-${ntirpc_version}.${rpm_arch}.rpm
 	fi
-	yum -y install ${ntirpc_rpm} ${rpm_arch}/nfs-ganesha-{,gluster-}${ganesha_version}.${rpm_arch}.rpm
+	yum -y install {x86_64,noarch}/*.rpm
 
 	# start nfs-ganesha service with an empty configuration
 	echo "NFSv4 { Graceless = true; }" > /etc/ganesha/ganesha.conf
@@ -122,7 +123,12 @@ else
 fi
 
 # create and start gluster volume
-yum -y install glusterfs-server
+if [ "${CENTOS_VERSION}" = "7" ]; then
+  yum -y install glusterfs-server
+elif [ "${CENTOS_VERSION}" = "8s" ]; then
+  yum -y install --enablerepo=powertools glusterfs-server
+fi
+
 systemctl start glusterd
 mkdir -p /bricks/${GLUSTER_VOLUME}
 gluster volume create ${GLUSTER_VOLUME} \
@@ -142,7 +148,7 @@ gluster vol status
 
 # TODO: open only the ports needed?
 # disable the firewall, otherwise the client can not connect
-systemctl stop firewalld || service iptables stop
+systemctl stop firewalld || service iptables stop || true
 
 mount -t glusterfs $(hostname --fqdn):/${GLUSTER_VOLUME} /mnt
 mount

jobs/nfs-ganesha_code_compilation.yml renamed to jobs/code_compilation.yml

@@ -3,7 +3,7 @@
     node: cico-workspace
     description: 'Run code compilation for nfs-ganesha against the latest build of Ganesha with FSAL_GLUSTER.'
     project-type: freestyle
-    concurrent: false
+    concurrent: true
     allow-manual-triggers: true
 
     scm:
@@ -31,9 +31,11 @@
         option: 'code_compilation'
 
     builders:
+    - shell: !include-raw: scripts/common/get-node.sh
     - shell: !include-raw: scripts/common.sh
 
     wrappers:
+    - gerrithub_key
     - cleanup-ws
 
     publishers:

jobs/macros.yml

@@ -104,8 +104,8 @@
     parameters:
     - choice:
         choices:
+        - "8s"
         - "7"
-        - "6"
         description: 'CentOS version to run the tests on (server + client)'
         name: CENTOS_VERSION
     - choice:
@@ -135,4 +135,4 @@
             - log-text: Build was aborted
               operator: OR
           script:
-            !include-raw: ../build_scripts/common/return-node.sh
+            !include-raw: scripts/common/return-node.sh

jobs/scripts/code_compilation/basic-gluster-duffy.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+SERVER_TEST_SCRIPT=${SERVER_TEST_SCRIPT}
+CLIENT_TEST_SCRIPT=${CLIENT_TEST_SCRIPT}
+
+server_env="export GERRIT_HOST='${GERRIT_HOST}'"
+server_env+=" GERRIT_PROJECT='${GERRIT_PROJECT}'"
+server_env+=" GERRIT_REFSPEC='${GERRIT_REFSPEC}'"
+server_env+=" YUM_REPO='${YUM_REPO}'"
+
+if [ $CENTOS_VERSION ]; then 
+    server_env+=" CENTOS_VERSION='${CENTOS_VERSION}'"
+fi
+
+if [ "$JOB_NAME" == "nfs_ganesha_iozone_vfs" ] || [ "$JOB_NAME" == "nfs_ganesha_iozone_vfs_minmdcache" ]; then
+    server_env+=" VFS_VOLUME='${EXPORT}'"
+else
+    server_env+=" GLUSTER_VOLUME='${EXPORT}'"
+fi
+
+SERVER_IP=$(cat $WORKSPACE/hosts | sed -n '1p')
+CLIENT_IP=$(cat $WORKSPACE/hosts | sed -n '2p')
+
+echo $server_env > $WORKSPACE/SERVER_ENV.txt
+
+SSH_OPTIONS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
+
+#add the export with environment to ~/.bashrc
+scp ${SSH_OPTIONS} "$WORKSPACE/SERVER_ENV.txt" "root@${SERVER_IP}:./SERVER_ENV.txt"
+
+ssh -t ${SSH_OPTIONS} root@${SERVER_IP} "tee -a ~/.bashrc < ./SERVER_ENV.txt"
+
+scp ${SSH_OPTIONS} ${SERVER_TEST_SCRIPT} root@${SERVER_IP}:./$(basename ${SERVER_TEST_SCRIPT})
+
+ssh -t ${SSH_OPTIONS} root@${SERVER_IP} "bash ./$(basename ${SERVER_TEST_SCRIPT})"
+RETURN_CODE=$?
+
+if [ $RETURN_CODE == 0 ]; then
+    client_env="export SERVER='${SERVER_IP}'"
+    client_env+=" EXPORT='/${EXPORT}'"
+    client_env+=" CENTOS_VERSION='${CENTOS_VERSION}'"
+
+    echo $client_env > $WORKSPACE/CLIENT_ENV.txt
+
+    scp ${SSH_OPTIONS} "$WORKSPACE/CLIENT_ENV.txt" "root@${CLIENT_IP}:./CLIENT_ENV.txt"
+
+    ssh -t ${SSH_OPTIONS} root@$CLIENT_IP 'tee -a ~/.bashrc < ./CLIENT_ENV.txt'
+
+    scp ${SSH_OPTIONS} ${CLIENT_TEST_SCRIPT} root@${CLIENT_IP}:./$(basename ${CLIENT_TEST_SCRIPT})
+
+    ssh -t ${SSH_OPTIONS} root@${CLIENT_IP} "bash ./$(basename ${CLIENT_TEST_SCRIPT})"
+    RETURN_CODE_CLIENT=$?
+
+    exit $RETURN_CODE_CLIENT
+else
+    echo "The SERVER script failed!"
+    exit $RETURN_CODE
+fi

jobs/scripts/common.sh

@@ -1,13 +1,18 @@
-export CENTOS_VERSION=7
-export CENTOS_ARCH=x86_64
+export CENTOS_VERSION=${CENTOS_VERSION}
+export CENTOS_ARCH=${CENTOS_ARCH}
 export GERRIT_HOST=${GERRIT_HOST}
 export GERRIT_PROJECT=${GERRIT_PROJECT}
 export GERRIT_REFSPEC=${GERRIT_REFSPEC}
 export LAST_TRIGGERED_JOB_NAME=$JOB_NAME
 export BUILD_NUMBER=${BUILD_NUMBER}
 
-python $WORKSPACE/ci-tests/build_scripts/common/basic-gluster-duffy.py
-RET=$?
+if [ "$JOB_NAME" == "nfs_ganesha_dbench" ]; then 
+  bash $WORKSPACE/ci-tests/jobs/scripts/dbench/basic-gluster-duffy.sh
+  RET=$?
+elif [ "$JOB_NAME" == "nfs_ganesha_code_compilation" ]; then
+  bash $WORKSPACE/ci-tests/jobs/scripts/code_compilation/basic-gluster-duffy.sh
+  RET=$?
+fi
 
 JOB_OUTPUT="${JENKINS_URL}/job/${LAST_TRIGGERED_JOB_NAME}/${BUILD_NUMBER}/console"
 

jobs/scripts/common/get-node.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set +x
+
+mkdir -p ~/.config
+
+cat > ~/.config/duffy <<EOF
+client:
+  url: https://duffy.ci.centos.org/api/v1
+  auth:
+    name: nfs-ganesha
+    key: ${CICO_API_KEY}
+EOF
+
+readarray -t POOLS < <(duffy client list-pools | jq -r '.pools[].name')
+
+LIST_POOLS=()
+for i in "${POOLS[@]}"
+do
+        if [[ $i =~ ${CENTOS_VERSION}(s)*-x86_64 ]]; then
+                LIST_POOLS+=($i)
+        fi
+done
+
+if [ "$JOB_NAME" == "nfs-ganesha_trigger-fsal_cephfs" ] || [ "$JOB_NAME" == "nfs-ganesha_trigger-fsal_gluster" ] || [ "$JOB_NAME" == "nfs-ganesha_trigger-fsal_rgw" ]; then
+    node_count=1
+else
+    node_count=2
+fi
+
+
+for my_pool in ${LIST_POOLS[@]};
+do
+        if [[ $(duffy client show-pool $my_pool | jq -r '.pool.levels.ready') -gt 1 ]]; then
+                SESSION=$(duffy client request-session pool="${my_pool}",quantity=$node_count)
+                echo "${SESSION}" | jq -r '.session.nodes[].ipaddr' > "${WORKSPACE}"/hosts
+                echo "${SESSION}" | jq -r '.session.id' > "${WORKSPACE}"/session_id
+                break
+        fi
+
+        sleep 60
+        echo -n "."
+done
+
+if [ -z "${SESSION}" ]; then
+        echo "Failed to reserve node"
+        exit 1
+fi

jobs/scripts/common/return-node.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# A script that releases nodes from session ids
+
+set +x
+
+SESSION_ID=$(cat "${WORKSPACE}"/session_id)
+
+duffy client retire-session "${SESSION_ID}" > /dev/null