[backport] CI/CD changes to PR and CNI Release Test Pipelines (#2940)

* ci: Add node restart to cniv2 E2E (#2901) * ci: add node restart to cniv2 E2E * chore: change directory after scale * chore: fixup nodepool * chore: pipeline cleanup * ci: cleanup test resources * chore: address comments * [FIX] ci: Update AzCLI task to v2 (#2922) * ci: Update AzCLI task to v2 * ci: add scriptType: "bash" * ci: Shift E2E cluster creation (#2862) * ci: shift cluster creation * chore: change depends on for release test
Azure · Aug 21, 2024 · a6fc0e3 · a6fc0e3
1 parent 3d3634d
commit a6fc0e3
Show file tree

Hide file tree

Showing 40 changed files with 453 additions and 189 deletions.
diff --git a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml
@@ -28,6 +28,7 @@ stages:
       commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
     dependsOn:
       - setup
+      - build_images
     displayName: "Create Cluster - ${{ parameters.clusterName }}"
     jobs:
       - job: create_aks_cluster_with_${{ parameters.name }}
@@ -68,7 +69,7 @@ stages:
         - job: deploy_cilium_components
           displayName: Deploy Cilium
           steps:
-            - task: AzureCLI@1
+            - task: AzureCLI@2
               displayName: "Install Cilium, CNS, and ip-masq-agent"
               inputs:
                 azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
@@ -105,7 +106,7 @@ stages:
                   kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-agent/files
                   kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-operator/files
 
-                  envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-agent/templates/daemonset.yaml | kubectl apply -f -
+                    envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-agent/templates/daemonset.yaml | kubectl apply -f -
                   envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-operator/templates/deployment.yaml | kubectl apply -f -
                   kubectl get po -owide -A
 
@@ -116,7 +117,7 @@ stages:
         - job: deploy_cilium_components
           displayName: Deploy Cilium with Hubble
           steps:
-            - task: AzureCLI@1
+            - task: AzureCLI@2
               displayName: "Install Cilium, CNS, and ip-masq-agent"
               inputs:
                 azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
@@ -197,7 +198,7 @@ stages:
         dependsOn: restart_cns
         steps:
           - template: ../../templates/cilium-cli.yaml
-          - task: AzureCLI@1
+          - task: AzureCLI@2
             inputs:
               azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
               scriptLocation: "inlineScript"

diff --git a/.pipelines/cni/cilium/cilium-scale-test.yaml b/.pipelines/cni/cilium/cilium-scale-test.yaml
@@ -9,7 +9,7 @@ stages:
         pool:
           name: "$(BUILD_POOL_NAME_DEFAULT)"
         steps:
-          - task: AzureCLI@1
+          - task: AzureCLI@2
             inputs:
               azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
               scriptLocation: "inlineScript"
@@ -27,12 +27,12 @@ stages:
                 kubectl apply -f test/integration/manifests/cilium/cilium-operator
                 echo "Keep CNS version up to date, grabbing pipeline parameter"
                 CNS_IMAGE=${CNS_IMAGE}
-                sed -i '/containers:/{n;n;s/\(image\).*/\1: '"${CNS_IMAGE//\//\\/}"'/}' test/integration/manifests/cns/daemonset.yaml 
-                kubectl apply -f test/integration/manifests/cns/daemonset.yaml 
+                sed -i '/containers:/{n;n;s/\(image\).*/\1: '"${CNS_IMAGE//\//\\/}"'/}' test/integration/manifests/cns/daemonset.yaml
+                kubectl apply -f test/integration/manifests/cns/daemonset.yaml
                 for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
                   make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
                 done
-                kubectl get node 
+                kubectl get node
                 kubectl get pod -A
             name: "UpdateCiliumandCNSVersion"
             displayName: "Update Cilium and CNS Version"
@@ -43,7 +43,7 @@ stages:
         pool:
           name: "$(BUILD_POOL_NAME_DEFAULT)"
         steps:
-          - task: AzureCLI@1
+          - task: AzureCLI@2
             inputs:
               azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
               scriptLocation: "inlineScript"
@@ -63,7 +63,7 @@ stages:
         pool:
           name: "$(BUILD_POOL_NAME_DEFAULT)"
         steps:
-          - task: AzureCLI@1
+          - task: AzureCLI@2
             inputs:
               azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
               scriptLocation: "inlineScript"
@@ -72,7 +72,7 @@ stages:
               inlineScript: |
                 echo "Set node label scale-test=true and connectivity-test=true for testing"
                 az aks get-credentials --resource-group ${CLUSTER} --name ${CLUSTER}
-                cd test/scale 
+                cd test/scale
                 chmod +x label-nodes.sh
                 ./label-nodes.sh
             name: "LabelNodes"
@@ -84,7 +84,7 @@ stages:
         pool:
           name: "$(BUILD_POOL_NAME_DEFAULT)"
         steps:
-          - task: AzureCLI@1
+          - task: AzureCLI@2
             inputs:
               azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
               scriptLocation: "inlineScript"
@@ -95,14 +95,14 @@ stages:
                 echo "collect cpu and memory usage before scaling for network policies"
                 mkdir test1_1_netpol_cpu_and_mem_before
                 cd test1_1_netpol_cpu_and_mem_before
-                echo "running k top node" 
+                echo "running k top node"
                 kubectl top node >> "node_before_netpol_scale.log"
                 echo "running k top pod"
                 kubectl top pod -A | grep cilium >> "pod_before_netpol_scale.log"
                 echo "Logs will be available as a build artifact"
                 ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test1_1_netpol_cpu_and_mem_before/
                 echo $ARTIFACT_DIR
-                sudo rm -rf $ARTIFACT_DIR 
+                sudo rm -rf $ARTIFACT_DIR
                 sudo mkdir $ARTIFACT_DIR
                 cd ..
                 sudo cp ./test1_1_netpol_cpu_and_mem_before/* $ARTIFACT_DIR
@@ -113,7 +113,7 @@ stages:
                 echo "collect cpu and mem results after scaling"
                 mkdir test1_2_netpol_cpu_and_mem_scale
                 cd test1_2_netpol_cpu_and_mem_scale
-                echo "running k top node" 
+                echo "running k top node"
                 kubectl top node >> "node_netpol_scale.log"
                 echo "running k top pod"
                 kubectl top pod -A | grep cilium >> "pod_netpol_scale.log"
@@ -148,7 +148,7 @@ stages:
           name: "$(BUILD_POOL_NAME_DEFAULT)"
         timeoutInMinutes: 120
         steps:
-          - task: AzureCLI@1
+          - task: AzureCLI@2
             inputs:
               azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
               scriptLocation: "inlineScript"
@@ -163,14 +163,14 @@ stages:
                 echo "collect cpu and mem results after connectivity tests"
                 mkdir test1_3_netpol_cpu_and_mem_after
                 cd test1_3_netpol_cpu_and_mem_after
-                echo "running k top node" 
+                echo "running k top node"
                 kubectl top node >> "node_after_netpol_tests.log"
                 echo "running k top pod"
                 kubectl top pod -A | grep cilium >> "pod_after_netpol_tests.log"
                 echo "Logs will be available as a build artifact"
                 ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test1_3_netpol_cpu_and_mem_after/
                 echo $ARTIFACT_DIR
-                sudo rm -rf $ARTIFACT_DIR 
+                sudo rm -rf $ARTIFACT_DIR
                 sudo mkdir $ARTIFACT_DIR
                 cd ..
                 sudo cp ./test1_3_netpol_cpu_and_mem_after/* $ARTIFACT_DIR
@@ -190,7 +190,7 @@ stages:
         pool:
           name: "$(BUILD_POOL_NAME_DEFAULT)"
         steps:
-          - task: AzureCLI@1
+          - task: AzureCLI@2
             inputs:
               azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
               scriptLocation: "inlineScript"
@@ -201,14 +201,14 @@ stages:
                 echo "collect cpu and mem results before scale for lb tests"
                 mkdir test2_1_lb_cpu_and_mem_before
                 cd test2_1_lb_cpu_and_mem_before
-                echo "running k top node" 
+                echo "running k top node"
                 kubectl top node >> "node_before_lb_scale.log"
                 echo "running k top pod"
                 kubectl top pod -A | grep cilium >> "pod_before_lb_scale.log"
                 echo "Logs will be available as a build artifact"
                 ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test2_1_lb_cpu_and_mem_before/
                 echo $ARTIFACT_DIR
-                sudo rm -rf $ARTIFACT_DIR 
+                sudo rm -rf $ARTIFACT_DIR
                 sudo mkdir $ARTIFACT_DIR
                 cd ..
                 sudo cp ./test2_1_lb_cpu_and_mem_before/* $ARTIFACT_DIR
@@ -218,7 +218,7 @@ stages:
                 echo "collect cpu and mem results after scaling"
                 mkdir test2_2_lb_cpu_and_mem_scale
                 cd test2_2_lb_cpu_and_mem_scale
-                echo "running k top node" 
+                echo "running k top node"
                 kubectl top node >> "node_lb_scale.log"
                 echo "running k top pod"
                 kubectl top pod -A | grep cilium >> "pod_lb_scale.log"
@@ -252,7 +252,7 @@ stages:
         pool:
           name: "$(BUILD_POOL_NAME_DEFAULT)"
         steps:
-          - task: AzureCLI@1
+          - task: AzureCLI@2
             inputs:
               azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
               scriptLocation: "inlineScript"
@@ -274,7 +274,7 @@ stages:
                 cd ..
                 mkdir test2_3_lb_cpu_and_mem_after
                 cd test2_3_lb_cpu_and_mem_after
-                echo "running k top node" 
+                echo "running k top node"
                 kubectl top node >> "node_after_lb_tests.log"
                 echo "running k top pod"
                 kubectl top pod -A | grep cilium >> "pod_after_lb_tests.log"
@@ -312,7 +312,7 @@ stages:
         pool:
           name: "$(BUILD_POOL_NAME_DEFAULT)"
         steps:
-          - task: AzureCLI@1
+          - task: AzureCLI@2
             inputs:
               azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
               scriptLocation: "inlineScript"
@@ -328,7 +328,7 @@ stages:
                 echo "collect cpu and mem results after netperf tests"
                 mkdir test3_netperf_cpu_and_mem
                 cd test3_netperf_cpu_and_mem
-                echo "running k top node" 
+                echo "running k top node"
                 kubectl top node >> "node_netperf.log"
                 echo "running k top pod"
                 kubectl top pod -A | grep cilium >> "pod_netperf.log"
@@ -337,8 +337,8 @@ stages:
                 ARTIFACT_DIR2=$(Build.ArtifactStagingDirectory)/test3_netperf/
                 echo $ARTIFACT_DIR
                 echo $ARTIFACT_DIR2
-                sudo rm -rf $ARTIFACT_DIR 
-                sudo rm -rf $ARTIFACT_DIR2 
+                sudo rm -rf $ARTIFACT_DIR
+                sudo rm -rf $ARTIFACT_DIR2
                 sudo mkdir $ARTIFACT_DIR
                 sudo mkdir $ARTIFACT_DIR2
                 cd ..
@@ -367,7 +367,7 @@ stages:
         pool:
           name: "$(BUILD_POOL_NAME_DEFAULT)"
         steps:
-          - task: AzureCLI@1
+          - task: AzureCLI@2
             inputs:
               azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
               scriptLocation: "inlineScript"
@@ -389,7 +389,7 @@ stages:
         pool:
           name: "$(BUILD_POOL_NAME_DEFAULT)"
         steps:
-        - task: AzureCLI@1
+        - task: AzureCLI@2
           inputs:
             azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
             scriptLocation: "inlineScript"

diff --git a/.pipelines/cni/cilium/nightly-release-test.yml b/.pipelines/cni/cilium/nightly-release-test.yml
@@ -35,17 +35,19 @@ stages:
               make docker-operator-generic-image
             name: BuildCiliumImage
             displayName: "Build Cilium Image"
-          - task: AzureCLI@1
+          - task: AzureCLI@2
             displayName: "Login"
             inputs:
               azureSubscription: $(ACR_ARM_SERVICE_CONNECTION)
               scriptLocation: "inlineScript"
+              scriptType: "bash"
               inlineScript: |
                 az acr login -n $(ACR)
-          - task: AzureCLI@1
+          - task: AzureCLI@2
             inputs:
               azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
               scriptLocation: "inlineScript"
+              scriptType: "bash"
               inlineScript: |
                 set -ex
                 docker tag quay.io/cilium/cilium:$(DOCKER_IMAGE_TAG) $(ACR).azurecr.io/cilium/cilium:$(DOCKER_IMAGE_TAG)
@@ -54,11 +56,12 @@ stages:
                 docker push $(ACR).azurecr.io/cilium/operator-generic:$(DOCKER_IMAGE_TAG)
             name: "PushCiliumImage"
             displayName: "Push Cilium Image"
-          - task: AzureCLI@1
+          - task: AzureCLI@2
             displayName: "Logout"
             inputs:
               azureSubscription: $(ACR_ARM_SERVICE_CONNECTION)
               scriptLocation: "inlineScript"
+              scriptType: "bash"
               inlineScript: |
                 docker logout
 

diff --git a/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml b/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml
@@ -14,7 +14,7 @@ jobs:
     pool:
       name: $(BUILD_POOL_NAME_DEFAULT)
     steps:
-      - task: AzureCLI@1
+      - task: AzureCLI@2
         inputs:
           azureSubscription: ${{ parameters.sub }}
           scriptLocation: "inlineScript"

diff --git a/.pipelines/cni/load-test-templates/create-cluster-template.yaml b/.pipelines/cni/load-test-templates/create-cluster-template.yaml
@@ -10,7 +10,7 @@ parameters:
   nodeCountWin: 2
 
 steps:
-  - task: AzureCLI@1
+  - task: AzureCLI@2
     inputs:
       azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
       scriptLocation: "inlineScript"

diff --git a/.pipelines/cni/load-test-templates/pod-deployment-template.yaml b/.pipelines/cni/load-test-templates/pod-deployment-template.yaml
@@ -8,7 +8,7 @@ parameters:
   jobName: "deploy_pods"
 
 steps:
-  - task: AzureCLI@1
+  - task: AzureCLI@2
     displayName: "Pod Deployment"
     inputs:
       azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)

diff --git a/.pipelines/cni/load-test-templates/restart-cns-template.yaml b/.pipelines/cni/load-test-templates/restart-cns-template.yaml
@@ -7,7 +7,7 @@ parameters:
   jobName: "restart_cns"
 
 steps:
-  - task: AzureCLI@1
+  - task: AzureCLI@2
     inputs:
       azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
       scriptLocation: "inlineScript"

diff --git a/.pipelines/cni/load-test-templates/restart-hns-template.yaml b/.pipelines/cni/load-test-templates/restart-hns-template.yaml
@@ -3,7 +3,7 @@ parameters:
   cni: "cniv1"
 
 steps:
-  - task: AzureCLI@1
+  - task: AzureCLI@2
     inputs:
       azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
       scriptLocation: "inlineScript"

diff --git a/.pipelines/cni/load-test-templates/restart-node-template.yaml b/.pipelines/cni/load-test-templates/restart-node-template.yaml
@@ -6,7 +6,7 @@ parameters:
   region: ""
 
 steps:
-  - task: AzureCLI@1
+  - task: AzureCLI@2
     inputs:
       azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
       scriptLocation: "inlineScript"

diff --git a/.pipelines/cni/load-test-templates/validate-state-template.yaml b/.pipelines/cni/load-test-templates/validate-state-template.yaml
@@ -5,7 +5,7 @@ parameters:
   cni: "cilium"
 
 steps:
-  - task: AzureCLI@1
+  - task: AzureCLI@2
     inputs:
       azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
       scriptLocation: "inlineScript"

diff --git a/.pipelines/cni/lsg/kernel-upgrade-template.yaml b/.pipelines/cni/lsg/kernel-upgrade-template.yaml
@@ -2,7 +2,7 @@ parameters:
   clusterName: ""
 
 steps:
-  - task: AzureCLI@1
+  - task: AzureCLI@2
     inputs:
       azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
       scriptLocation: "inlineScript"