diff --git a/.env b/.env index 1eb5d8f2..edb3c425 100644 --- a/.env +++ b/.env @@ -71,7 +71,7 @@ CEPH_SHA=latest CEPH_DEVEL_MGR_PATH=../ceph # Atom -ATOM_SHA=be72d1c50eb7d0c5ad9274795c1b38b3ee261878 +ATOM_SHA=af604da197c260403c5cefccc2665c6758e558d5 # Demo settings RBD_POOL=rbd diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 9c629cee..76cde126 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -737,7 +737,7 @@ jobs: strategy: fail-fast: false matrix: - test: ["sanity", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify", "ceph_status"] + test: ["sanity", "ns_lb_change", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify", "ceph_status"] runs-on: ubuntu-latest env: HUGEPAGES: 1024 # 4 spdk instances diff --git a/.gitmodules b/.gitmodules index d39e8591..f2611fc3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "spdk"] path = spdk url = https://github.com/ceph/spdk.git - branch = ceph-nvmeof-v24.05 + branch = ceph-nvmeof-v24.05 diff --git a/spdk b/spdk index 0d9430fa..6140a67f 160000 --- a/spdk +++ b/spdk @@ -1 +1 @@ -Subproject commit 0d9430fab9b368761b13dc73a65b0ed13d6c6bc6 +Subproject commit 6140a67ffec9d231a103577ea139c082a6e470ff diff --git a/tests/atom/clusterBuildTestsRun.sh b/tests/atom/clusterBuildTestsRun.sh index 9b5c6ed9..4cf5d863 100755 --- a/tests/atom/clusterBuildTestsRun.sh +++ b/tests/atom/clusterBuildTestsRun.sh @@ -38,7 +38,7 @@ sudo docker run \ --rmNvmeofDaemon \ --gitHubActionDeployment \ --dontUseMTLS \ - --skipLbalancingTest \ + --skiplbTest \ --journalctlToConsole \ --dontPowerOffCloudVMs noKey noKey \ --multiIBMCloudServers_m2 diff --git a/tests/ha/ns_lb_change.sh b/tests/ha/ns_lb_change.sh new file mode 100755 index 00000000..077c9394 --- /dev/null +++ b/tests/ha/ns_lb_change.sh @@ -0,0 +1,121 @@ +#!/bin/bash +set -xe +# See +# - https://github.com/spdk/spdk/blob/master/doc/jsonrpc.md +# - https://spdk.io/doc/nvmf_multipath_howto.html + +GW1_NAME=$(docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /1/ {print $1}') +GW2_NAME=$(docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /2/ {print $1}') + +ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$GW1_NAME")" +ip2="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$GW2_NAME")" + +calc_written_bytes_in_sec() +{ + IP=$1 + num_bytes=$(docker compose run --rm nvmeof-cli --server-address $IP --server-port 5500 --output stdio --format json namespace get_io_stats -n nqn.2016-06.io.spdk:cnode1 --nsid 1 | jq '.bytes_written'| sed 's/[^0-9]*//g'); + + sleep 1; + num_bytes1=$(docker compose run --rm nvmeof-cli --server-address $IP --server-port 5500 --output stdio --format json namespace get_io_stats -n nqn.2016-06.io.spdk:cnode1 --nsid 1 | jq '.bytes_written'| sed 's/[^0-9]*//g'); + + res=$(expr $num_bytes1 - $num_bytes ); + #echo "Bytes written in sec: $res"; + if [ "$res" -gt 0 ]; then + # limit values to boolean for simplify futher analysis + res=1; + else + res=0; + fi; + echo "$res"; +} + + +echo -n "ℹ️ Starting bdevperf container" +docker compose up -d bdevperf +sleep 10 +echo "ℹ️ bdevperf start up logs" +make logs SVC=bdevperf +BDEVPERF_SOCKET=/tmp/bdevperf.sock +NVMEOF_DISC_PORT=8009 + + +echo "ℹ️ Using discovery service in gateway $GW1 ip $ip" +rpc="/usr/libexec/spdk/scripts/rpc.py" +echo "ℹ️ bdevperf bdev_nvme_set_options" +make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_set_options -r -1" +echo "ℹ️ bdevperf start discovery ip: $ip port: $NVMEOF_DISC_PORT" +make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_start_discovery -b Nvme0 -t tcp -a $ip -s $NVMEOF_DISC_PORT -f ipv4 -w" +echo "ℹ️ bdevperf bdev_nvme_get_discovery_info" +make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_get_discovery_info" +echo "ℹ️ bdevperf perform_tests" +eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_TEST_DURATION | tr -d '\n\r' ) + +timeout=$(expr $BDEVPERF_TEST_DURATION \* 2) + +echo $timeout +bdevperf="/usr/libexec/spdk/scripts/bdevperf.py" +echo "run io test" +make exec SVC=bdevperf OPTS=-T CMD="$bdevperf -v -t $timeout -s $BDEVPERF_SOCKET perform_tests" & +#test write ios for the ns1 + +( +sleep 8; + +lb_group=1; +docker compose run -T --rm nvmeof-cli --server-address $ip --server-port 5500 namespace change_load_balancing_group -n nqn.2016-06.io.spdk:cnode1 --nsid 1 --load-balancing-group $lb_group ; + + +priv_res1=$(calc_written_bytes_in_sec $ip) ; + +echo "ℹ️ written bytes through $ip $priv_res1 "; + +priv_res2=$(calc_written_bytes_in_sec $ip2); + +echo "ℹ️ written bytes through $ip2 $priv_res2 "; + + +for i in $(seq 6); do + if [ $lb_group -eq 1 ]; then + lb_group=2 + else + lb_group=1 + fi; + + echo "ℹ️ ℹ️ Change lb group of ns 1 to $lb_group :" ; + docker compose run -T --rm nvmeof-cli --server-address $ip --server-port 5500 namespace change_load_balancing_group -n nqn.2016-06.io.spdk:cnode1 --nsid 1 --load-balancing-group $lb_group ; + sleep 4; + + res1=$(calc_written_bytes_in_sec $ip) ; + + echo "ℹ️ written bytes through $ip ?: $res1"; + + res2=$(calc_written_bytes_in_sec $ip2) ; + + echo "ℹ️ written bytes through $ip2 ?: $res2 "; + echo "ℹ️ ℹ️ ℹ️ DEBUG iteration $i : priv_res1 and res1 : $priv_res1 , $res1 , priv_res2 and res2 : $priv_res2 , $res2 "; + + #check that io is switched each iteration to different Gateway + if [ $res1 -eq $res2 ]; then + echo " ℹ️ ℹ️ ℹ️ res1 and res2 : $res1 $res2 "; + exit 1 #both eq 0 - no traffic at all + fi; + + if [ $res1 -ne $priv_res1 ] && [ $res2 -ne $priv_res2 ]; then + echo " ℹ️ ℹ️ Valid traffic results"; + else + echo "ℹ️ ℹ️ ℹ️ Not valid checks !!! : priv_res1 and res1 : $priv_res1 $res1 , priv_res2 and res2 : $priv_res2 $res2 "; + exit 1; + fi; + + priv_res1=$res1; + priv_res2=$res2; + +done; + + +echo "wait for join"; + +) & + +wait +exit 0