ci: Add node restart to cniv2 E2E (#2901)

* ci: add node restart to cniv2 E2E * chore: change directory after scale * chore: fixup nodepool * chore: pipeline cleanup * ci: cleanup test resources * chore: address comments
Azure · Aug 20, 2024 · e1f5aba · e1f5aba
1 parent 6b92514
commit e1f5aba
Show file tree

Hide file tree

Showing 14 changed files with 347 additions and 93 deletions.
diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
@@ -59,6 +59,7 @@ stages:
             parameters:
               name: ${{ parameters.name }}
               clusterName: ${{ parameters.clusterName }}-$(commitID)
+              scaleup: 100
 
       - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
         parameters:

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -1,6 +1,7 @@
 parameters:
   name: ""
   clusterName: ""
+  scaleup: ""
 
 steps:
   - bash: |
@@ -47,6 +48,42 @@ steps:
     name: "aksswifte2e"
     displayName: "Run AKS Swift E2E"
 
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        set -e
+        kubectl get po -owide -A
+        clusterName=${{ parameters.clusterName }}
+        echo "Restarting nodes"
+        for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
+          make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
+        done
+    displayName: "Restart Nodes"
+
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        cd test/integration/load
+
+        # Scale Cluster Up/Down to confirm functioning CNS
+        ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
+        kubectl get pods -owide -A
+
+        cd ../../..
+        echo "Validating Node Restart"
+        make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cniv2
+        kubectl delete ns load-test
+    displayName: "Validate Node Restart"
+    retryCountOnTaskFailure: 3
+
   - script: |
       echo "Run wireserver and metadata connectivity Tests"
       bash test/network/wireserver_metadata_test.sh

diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml
@@ -45,6 +45,7 @@ stages:
     jobs:
       - job: ${{ parameters.name }}_linux
         displayName: Azure CNI Overlay Test Suite | Linux - (${{ parameters.name }})
+        timeoutInMinutes: 120
         pool:
           name: $(BUILD_POOL_NAME_DEFAULT)
           demands:
@@ -56,10 +57,39 @@ stages:
               name: ${{ parameters.name }}
               clusterName: ${{ parameters.clusterName }}-$(commitID)
               os: linux
+              scaleup: 100
+
+      - job: windows_nodepool
+        displayName: Add Windows Nodepool
+        dependsOn: ${{ parameters.name }}_linux
+        pool:
+          name: $(BUILD_POOL_NAME_DEFAULT)
+          demands:
+          - agent.os -equals Linux
+          - Role -equals $(CUSTOM_E2E_ROLE)
+        steps:
+          - task: AzureCLI@2
+            inputs:
+              azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+              scriptLocation: "inlineScript"
+              scriptType: "bash"
+              addSpnToEnvironment: true
+              inlineScript: |
+                set -e
+                make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
+                make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}-$(commitID) VM_SIZE_WIN=${{ parameters.vmSize }}
+                echo "Windows node are successfully added to v4 Overlay Cluster"
+                kubectl cluster-info
+                kubectl get node -owide
+                kubectl get po -owide -A
+            name: "Add_Windows_Node"
+            displayName: "Add windows node on v4 overlay cluster"
+
 
       - job: ${{ parameters.name }}_windows
         displayName: Azure CNI Overlay Test Suite | Windows - (${{ parameters.name }})
-        dependsOn: ${{ parameters.name }}_linux
+        timeoutInMinutes: 120
+        dependsOn: windows_nodepool
         pool:
           name: $(BUILD_POOL_NAME_DEFAULT)
           demands:
@@ -71,7 +101,7 @@ stages:
               name: ${{ parameters.name }}
               clusterName: ${{ parameters.clusterName }}-$(commitID)
               os: windows
-              vmSizeWin: ${{ parameters.vmSize }} # Matching linux vmSize
+              scaleup: 50
 
       - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
         parameters:

diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml
@@ -2,6 +2,7 @@ parameters:
   name: ""
   clusterName: ""
   os: ""
+  scaleup: ""
 
 steps:
   - bash: |
@@ -64,30 +65,18 @@ steps:
         addSpnToEnvironment: true
         inlineScript: |
           cd test/integration/load
-          clusterName=${{ parameters.clusterName }}
-          make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
-          make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
+
+          # Scale Cluster Up/Down to confirm functioning CNS
+          ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
           kubectl get pods -owide -A
+
+          cd ../../..
           echo "Validating Node Restart"
-          CNI_TYPE=cniv2 RESTART_CASE=true go test -timeout 30m -tags load -run ^TestValidateState$
+          make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cniv2
+          kubectl delete ns load-test
       displayName: "Validate Node Restart"
       retryCountOnTaskFailure: 3
 
-    - script: |
-        echo "validate pod IP assignment before CNS restart"
-        kubectl get pod -owide -A
-        echo "validate pod state before CNS restarts"
-        cd test/integration/load
-        CNI_TYPE=cniv2 go test -timeout 30m -tags load -run ^TestValidateState$
-        kubectl rollout restart ds azure-cns -n kube-system
-        kubectl rollout status ds azure-cns -n kube-system
-        kubectl get pod -owide -A
-        echo "validate pods after CNS restart"
-        CNI_TYPE=cniv2 go test -timeout 30m -tags load -run ^TestValidateState$
-      name: "restartCNS_ValidatePodState"
-      displayName: "Restart CNS and validate pod state"
-      retryCountOnTaskFailure: 3
-
     - script: |
         echo "Run wireserver and metadata connectivity Tests"
         bash test/network/wireserver_metadata_test.sh
@@ -96,22 +85,6 @@ steps:
       displayName: "Run Wireserver and Metadata Connectivity Tests"
 
   - ${{ if eq(parameters.os, 'windows') }}:
-    - task: AzureCLI@2
-      inputs:
-        azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
-        scriptLocation: "inlineScript"
-        scriptType: "bash"
-        addSpnToEnvironment: true
-        inlineScript: |
-          set -e
-          make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }}
-          echo "Windows node are successfully added to v4 Overlay Cluster"
-          kubectl cluster-info
-          kubectl get node -owide
-          kubectl get po -owide -A
-      name: "Add_Windows_Node"
-      displayName: "Add windows node on v4 overlay cluster"
-
     - script: |
         nodeList=`kubectl get node -owide | grep Windows | awk '{print $1}'`
         for node in $nodeList; do
@@ -132,3 +105,39 @@ steps:
       name: "WindowsV4OverlayDatapathTests"
       displayName: "Windows v4Overlay Datapath Tests"
       retryCountOnTaskFailure: 3
+
+    - task: AzureCLI@1
+      inputs:
+        azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+        scriptLocation: "inlineScript"
+        scriptType: "bash"
+        addSpnToEnvironment: true
+        inlineScript: |
+          set -e
+          kubectl get po -owide -A
+          clusterName=${{ parameters.clusterName }}
+          echo "Restarting nodes"
+          for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
+            make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
+          done
+      displayName: "Restart Nodes"
+
+    - task: AzureCLI@1
+      inputs:
+        azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+        scriptLocation: "inlineScript"
+        scriptType: "bash"
+        addSpnToEnvironment: true
+        inlineScript: |
+          cd test/integration/load
+
+          # Scale Cluster Up/Down to confirm functioning CNS
+          ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=windows go test -count 1 -timeout 30m -tags load -run ^TestLoad$
+          kubectl get pods -owide -A
+
+          cd ../../..
+          echo "Validating Node Restart"
+          make test-validate-state OS_TYPE=windows RESTART_CASE=true CNI_TYPE=cniv2
+          kubectl delete ns load-test
+      displayName: "Validate Node Restart"
+      retryCountOnTaskFailure: 3
diff --git a/...ines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml b/...ines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml
@@ -55,6 +55,7 @@ stages:
             parameters:
               name: ${{ parameters.name }}
               clusterName: ${{ parameters.clusterName }}-$(commitID)
+              scaleup: 100
 
       - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
         parameters:

diff --git a/...nes/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml b/...nes/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml
@@ -1,6 +1,7 @@
 parameters:
   name: ""
   clusterName: ""
+  scaleup: ""
 
 steps:
 
@@ -68,6 +69,42 @@ steps:
     name: "CiliumStatus"
     displayName: "Cilium Status"
 
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        set -e
+        kubectl get po -owide -A
+        clusterName=${{ parameters.clusterName }}
+        echo "Restarting nodes"
+        for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
+          make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
+        done
+    displayName: "Restart Nodes"
+
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        cd test/integration/load
+
+        # Scale Cluster Up/Down to confirm functioning CNS
+        ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
+        kubectl get pods -owide -A
+
+        cd ../../..
+        echo "Validating Node Restart"
+        make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cilium_dualstack
+        kubectl delete ns load-test
+    displayName: "Validate Node Restart"
+    retryCountOnTaskFailure: 3
+
   - script: |
       echo "Run Cilium Connectivity Tests"
       cilium status
@@ -109,7 +146,7 @@ steps:
     retryCountOnTaskFailure: 3
     name: "WireserverMetadataConnectivityTests"
     displayName: "Run Wireserver and Metadata Connectivity Tests"
-  
+
   - script: |
       cd hack/scripts
       chmod +x async-delete-test.sh

diff --git a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml
@@ -59,6 +59,7 @@ stages:
               name: ${{ parameters.name }}
               clusterName: ${{ parameters.clusterName }}-$(commitID)
               testHubble: ${{ parameters.testHubble }}
+              scaleup: 100
 
       - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
         parameters:

diff --git a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml
@@ -2,6 +2,7 @@ parameters:
   name: ""
   clusterName: ""
   testHubble: false
+  scaleup: ""
 
 steps:
   - bash: |
@@ -72,6 +73,42 @@ steps:
     name: "CiliumStatus"
     displayName: "Cilium Status"
 
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        set -e
+        kubectl get po -owide -A
+        clusterName=${{ parameters.clusterName }}
+        echo "Restarting nodes"
+        for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
+          make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
+        done
+    displayName: "Restart Nodes"
+
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        cd test/integration/load
+
+        # Scale Cluster Up/Down to confirm functioning CNS
+        ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
+        kubectl get pods -owide -A
+
+        cd ../../..
+        echo "Validating Node Restart"
+        make test-validate-state OS_TYPE=linux RESTART_CASE=true
+        kubectl delete ns load-test
+    displayName: "Validate Node Restart"
+    retryCountOnTaskFailure: 3
+
   - script: |
       echo "Run Cilium Connectivity Tests"
       cilium status
@@ -147,7 +184,7 @@ steps:
     retryCountOnTaskFailure: 3
     name: "WireserverMetadataConnectivityTests"
     displayName: "Run Wireserver and Metadata Connectivity Tests"
-  
+
   - script: |
       cd hack/scripts
       chmod +x async-delete-test.sh

diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml
@@ -57,6 +57,7 @@ stages:
             parameters:
               name: ${{ parameters.name }}
               clusterName: ${{ parameters.clusterName }}-$(commitID)
+              scaleup: 100
 
       - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
         parameters: