From 6b0ae1abd4982126afc1c41c559ac04e1113bb24 Mon Sep 17 00:00:00 2001 From: John Payne <89417863+jpayne3506@users.noreply.github.com> Date: Fri, 16 Aug 2024 14:47:17 -0700 Subject: [PATCH] ci: Add node restart to cniv2 E2E (#2901) * ci: add node restart to cniv2 E2E * chore: change directory after scale * chore: fixup nodepool * chore: pipeline cleanup * ci: cleanup test resources * chore: address comments --- .../aks-swift/e2e-job-template.yaml | 1 + .../aks-swift/e2e-step-template.yaml | 37 +++++++++ .../azure-cni-overlay-e2e-job-template.yaml | 34 +++++++- .../azure-cni-overlay-e2e-step-template.yaml | 79 ++++++++++-------- ...ium-dualstackoverlay-e2e-job-template.yaml | 1 + ...um-dualstackoverlay-e2e-step-template.yaml | 37 +++++++++ .../cilium-overlay-e2e-job-template.yaml | 1 + .../cilium-overlay-e2e-step-template.yaml | 37 +++++++++ .../cilium-overlay-e2e-job-template.yaml | 1 + .../cilium-overlay-e2e-step-template.yaml | 38 +++++++++ .../cilium/cilium-e2e-job-template.yaml | 1 + .../cilium/cilium-e2e-step-template.yaml | 50 ++++++++--- .../dualstackoverlay-e2e-job-template.yaml | 33 +++++++- .../dualstackoverlay-e2e-step-template.yaml | 82 ++++++++++--------- 14 files changed, 343 insertions(+), 89 deletions(-) diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml index 50257112d95..126b56d7661 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml @@ -59,6 +59,7 @@ stages: parameters: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) + scaleup: 100 - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml index f94eb2906ab..1e5a923d4fd 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml @@ -1,6 +1,7 @@ parameters: name: "" clusterName: "" + scaleup: "" steps: - bash: | @@ -47,6 +48,42 @@ steps: name: "aksswifte2e" displayName: "Run AKS Swift E2E" + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + kubectl get po -owide -A + clusterName=${{ parameters.clusterName }} + echo "Restarting nodes" + for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} + done + displayName: "Restart Nodes" + + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + cd test/integration/load + + # Scale Cluster Up/Down to confirm functioning CNS + ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + kubectl get pods -owide -A + + cd ../../.. + echo "Validating Node Restart" + make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cniv2 + kubectl delete ns load-test + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3 + - script: | echo "Run wireserver and metadata connectivity Tests" bash test/network/wireserver_metadata_test.sh diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml index 626606c499f..2bc03a47268 100644 --- a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml @@ -45,6 +45,7 @@ stages: jobs: - job: ${{ parameters.name }}_linux displayName: Azure CNI Overlay Test Suite | Linux - (${{ parameters.name }}) + timeoutInMinutes: 120 pool: name: $(BUILD_POOL_NAME_DEFAULT) demands: @@ -56,10 +57,39 @@ stages: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) os: linux + scaleup: 100 + + - job: windows_nodepool + displayName: Add Windows Nodepool + dependsOn: ${{ parameters.name }}_linux + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + demands: + - agent.os -equals Linux + - Role -equals $(CUSTOM_E2E_ROLE) + steps: + - task: AzureCLI@2 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) + make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}-$(commitID) VM_SIZE_WIN=${{ parameters.vmSize }} + echo "Windows node are successfully added to v4 Overlay Cluster" + kubectl cluster-info + kubectl get node -owide + kubectl get po -owide -A + name: "Add_Windows_Node" + displayName: "Add windows node on v4 overlay cluster" + - job: ${{ parameters.name }}_windows displayName: Azure CNI Overlay Test Suite | Windows - (${{ parameters.name }}) - dependsOn: ${{ parameters.name }}_linux + timeoutInMinutes: 120 + dependsOn: windows_nodepool pool: name: $(BUILD_POOL_NAME_DEFAULT) demands: @@ -71,7 +101,7 @@ stages: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) os: windows - vmSizeWin: ${{ parameters.vmSize }} # Matching linux vmSize + scaleup: 50 - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml index 2f8ce064583..633c427e62e 100644 --- a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml @@ -2,6 +2,7 @@ parameters: name: "" clusterName: "" os: "" + scaleup: "" steps: - bash: | @@ -64,30 +65,18 @@ steps: addSpnToEnvironment: true inlineScript: | cd test/integration/load - clusterName=${{ parameters.clusterName }} - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName} - make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) + + # Scale Cluster Up/Down to confirm functioning CNS + ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$ kubectl get pods -owide -A + + cd ../../.. echo "Validating Node Restart" - CNI_TYPE=cniv2 RESTART_CASE=true go test -timeout 30m -tags load -run ^TestValidateState$ + make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cniv2 + kubectl delete ns load-test displayName: "Validate Node Restart" retryCountOnTaskFailure: 3 - - script: | - echo "validate pod IP assignment before CNS restart" - kubectl get pod -owide -A - echo "validate pod state before CNS restarts" - cd test/integration/load - CNI_TYPE=cniv2 go test -timeout 30m -tags load -run ^TestValidateState$ - kubectl rollout restart ds azure-cns -n kube-system - kubectl rollout status ds azure-cns -n kube-system - kubectl get pod -owide -A - echo "validate pods after CNS restart" - CNI_TYPE=cniv2 go test -timeout 30m -tags load -run ^TestValidateState$ - name: "restartCNS_ValidatePodState" - displayName: "Restart CNS and validate pod state" - retryCountOnTaskFailure: 3 - - script: | echo "Run wireserver and metadata connectivity Tests" bash test/network/wireserver_metadata_test.sh @@ -96,22 +85,6 @@ steps: displayName: "Run Wireserver and Metadata Connectivity Tests" - ${{ if eq(parameters.os, 'windows') }}: - - task: AzureCLI@2 - inputs: - azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }} - echo "Windows node are successfully added to v4 Overlay Cluster" - kubectl cluster-info - kubectl get node -owide - kubectl get po -owide -A - name: "Add_Windows_Node" - displayName: "Add windows node on v4 overlay cluster" - - script: | nodeList=`kubectl get node -owide | grep Windows | awk '{print $1}'` for node in $nodeList; do @@ -132,3 +105,39 @@ steps: name: "WindowsV4OverlayDatapathTests" displayName: "Windows v4Overlay Datapath Tests" retryCountOnTaskFailure: 3 + + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + kubectl get po -owide -A + clusterName=${{ parameters.clusterName }} + echo "Restarting nodes" + for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} + done + displayName: "Restart Nodes" + + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + cd test/integration/load + + # Scale Cluster Up/Down to confirm functioning CNS + ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=windows go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + kubectl get pods -owide -A + + cd ../../.. + echo "Validating Node Restart" + make test-validate-state OS_TYPE=windows RESTART_CASE=true CNI_TYPE=cniv2 + kubectl delete ns load-test + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3 diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml index bf8cb35b2a8..fc8ece1a3af 100644 --- a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml @@ -55,6 +55,7 @@ stages: parameters: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) + scaleup: 100 - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml index 1362cc2b67f..68ffec9dd83 100644 --- a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml @@ -1,6 +1,7 @@ parameters: name: "" clusterName: "" + scaleup: "" steps: @@ -68,6 +69,42 @@ steps: name: "CiliumStatus" displayName: "Cilium Status" + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + kubectl get po -owide -A + clusterName=${{ parameters.clusterName }} + echo "Restarting nodes" + for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} + done + displayName: "Restart Nodes" + + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + cd test/integration/load + + # Scale Cluster Up/Down to confirm functioning CNS + ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + kubectl get pods -owide -A + + cd ../../.. + echo "Validating Node Restart" + make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cilium_dualstack + kubectl delete ns load-test + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3 + - script: | echo "Run Cilium Connectivity Tests" cilium status diff --git a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml index 01ab0823ebc..7753baa716f 100644 --- a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml @@ -59,6 +59,7 @@ stages: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) testHubble: ${{ parameters.testHubble }} + scaleup: 100 - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: diff --git a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml index 9cb731f0887..50a5515226c 100644 --- a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml @@ -2,6 +2,7 @@ parameters: name: "" clusterName: "" testHubble: false + scaleup: "" steps: - bash: | @@ -72,6 +73,42 @@ steps: name: "CiliumStatus" displayName: "Cilium Status" + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + kubectl get po -owide -A + clusterName=${{ parameters.clusterName }} + echo "Restarting nodes" + for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} + done + displayName: "Restart Nodes" + + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + cd test/integration/load + + # Scale Cluster Up/Down to confirm functioning CNS + ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + kubectl get pods -owide -A + + cd ../../.. + echo "Validating Node Restart" + make test-validate-state OS_TYPE=linux RESTART_CASE=true + kubectl delete ns load-test + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3 + - script: | echo "Run Cilium Connectivity Tests" cilium status diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml index 4a0a840b94b..17c5245dac6 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml @@ -57,6 +57,7 @@ stages: parameters: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) + scaleup: 100 - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml index cde9c2c46e7..1942c00c576 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml @@ -2,6 +2,8 @@ parameters: name: "" clusterName: "" testHubble: false + scaleup: "" + steps: - bash: | @@ -93,6 +95,42 @@ steps: name: "CiliumStatus" displayName: "Cilium Status" + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + kubectl get po -owide -A + clusterName=${{ parameters.clusterName }} + echo "Restarting nodes" + for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} + done + displayName: "Restart Nodes" + + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + cd test/integration/load + + # Scale Cluster Up/Down to confirm functioning CNS + ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + kubectl get pods -owide -A + + cd ../../.. + echo "Validating Node Restart" + make test-validate-state OS_TYPE=linux RESTART_CASE=true + kubectl delete ns load-test + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3 + - script: | echo "Run Cilium Connectivity Tests" cilium status diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml index 5af6a31ab9e..b5ce2502593 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml @@ -60,6 +60,7 @@ stages: parameters: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) + scaleup: 100 - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml index 67f0d71fbf7..55975585569 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml @@ -1,6 +1,7 @@ parameters: name: "" clusterName: "" + scaleup: "" steps: - bash: | @@ -72,6 +73,42 @@ steps: name: "CiliumStatus" displayName: "Cilium Status" + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + kubectl get po -owide -A + clusterName=${{ parameters.clusterName }} + echo "Restarting nodes" + for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} + done + displayName: "Restart Nodes" + + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + cd test/integration/load + + # Scale Cluster Up/Down to confirm functioning CNS + ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + kubectl get pods -owide -A + + cd ../../.. + echo "Validating Node Restart" + make test-validate-state OS_TYPE=linux RESTART_CASE=true + kubectl delete ns load-test + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3 + - script: | echo "Run Cilium Connectivity Tests" cilium status @@ -94,19 +131,6 @@ steps: name: "validatePods" displayName: "Validate Pods" - - script: | - echo "validate pod IP assignment before CNS restart" - kubectl get pod -owide -A - make test-validate-state - echo "restart CNS" - kubectl rollout restart ds azure-cns -n kube-system - kubectl rollout status ds azure-cns -n kube-system - kubectl get pod -owide -A - echo "validate pods after CNS restart" - make test-validate-state - name: "restartCNS" - displayName: "Restart CNS and validate pods" - - script: | echo "Run wireserver and metadata connectivity Tests" bash test/network/wireserver_metadata_test.sh diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml index 0093dd8e483..8d4daa617e0 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml @@ -46,6 +46,7 @@ stages: jobs: - job: ${{ parameters.name }}_linux displayName: DualStack Overlay Test Suite | Linux - (${{ parameters.name }}) + timeoutInMinutes: 120 pool: name: $(BUILD_POOL_NAME_DEFAULT) demands: @@ -57,10 +58,38 @@ stages: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) os: linux + scaleup: 100 + + - job: windows_nodepool + displayName: Add Windows Nodepool + dependsOn: ${{ parameters.name }}_linux + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + demands: + - agent.os -equals Linux + - Role -equals $(CUSTOM_E2E_ROLE) + steps: + - task: AzureCLI@2 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) + make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}-$(commitID) VM_SIZE_WIN=${{ parameters.vmSize }} + echo "Windows nodes have been successfully added to DualStack Overlay Cluster" + kubectl cluster-info + kubectl get node -owide + kubectl get po -owide -A + name: "Add_Windows_Node" + displayName: "Add windows node" - job: ${{ parameters.name }}_windows displayName: DualStack Overlay Test Suite | Windows - (${{ parameters.name }}) - dependsOn: ${{ parameters.name }}_linux + timeoutInMinutes: 120 + dependsOn: windows_nodepool pool: name: $(BUILD_POOL_NAME_DEFAULT) demands: @@ -72,7 +101,7 @@ stages: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) os: windows - vmSizeWin: ${{ parameters.vmSize }} # Matching linux vmSize + scaleup: 50 - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index c63e68dceeb..9dcbe186db4 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -3,6 +3,7 @@ parameters: clusterName: "" cni: "dualstack" os: "" + scaleup: "" steps: - bash: | @@ -37,7 +38,7 @@ steps: kubectl cluster-info kubectl get node kubectl get po -owide -A - sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true VALIDATE_DUALSTACK=true CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) CLEANUP=true + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=dualstack VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true VALIDATE_DUALSTACK=true CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) CLEANUP=true retryCountOnTaskFailure: 3 name: "integrationTest" displayName: "Run CNS Integration Tests on AKS DualStack Overlay" @@ -77,47 +78,19 @@ steps: addSpnToEnvironment: true inlineScript: | cd test/integration/load - clusterName=${{ parameters.clusterName }} - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName} - make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) - kubectl get pods -A + + # Scale Cluster Up/Down to confirm functioning CNS + ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + kubectl get pods -owide -A + + cd ../../.. echo "Validating Node Restart" - CNI_TYPE=dualstack RESTART_CASE=true go test -timeout 30m -tags load -run ^TestValidateState$ + make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=dualstack + kubectl delete ns load-test displayName: "Validate Node Restart" retryCountOnTaskFailure: 3 - - script: | - echo "validate pod IP assignment before CNS restart" - kubectl get pod -owide -A - echo "validate pod state before CNS restarts" - cd test/integration/load - CNI_TYPE=dualstack go test -timeout 30m -tags load -run ^TestValidateState$ - kubectl rollout restart ds azure-cns -n kube-system - kubectl rollout status ds azure-cns -n kube-system - kubectl get pod -owide -A - echo "validate pods after CNS restarts" - CNI_TYPE=dualstack go test -timeout 30m -tags load -run ^TestValidateState$ - name: "restartCNS_ValidatePodState" - displayName: "Restart CNS and Validate Pod State" - retryCountOnTaskFailure: 3 - - ${{ if eq(parameters.os, 'windows') }}: - - task: AzureCLI@2 - inputs: - azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }} - echo "Windows nodes have been successfully added to DualStack Overlay Cluster" - kubectl cluster-info - kubectl get node -owide - kubectl get po -owide -A - name: "Add_Windows_Node" - displayName: "Add windows node" - - script: | nodeList=`kubectl get node -owide | grep Windows | awk '{print $1}'` for node in $nodeList; do @@ -138,3 +111,38 @@ steps: name: "WindowsDualStackOverlayDatapathTests" displayName: "Windows DualStack Overlay Datapath Tests" retryCountOnTaskFailure: 3 + + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + clusterName=${{ parameters.clusterName }} + echo "Restarting nodes" + for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) VMSS_NAME=${val} + done + displayName: "Restart Nodes" + + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + cd test/integration/load + + # Scale Cluster Up/Down to confirm functioning CNS + ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=windows go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + kubectl get pods -owide -A + + cd ../../.. + echo "Validating Node Restart" + make test-validate-state OS_TYPE=windows RESTART_CASE=true CNI_TYPE=cniv2 + kubectl delete ns load-test + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3