Skip to content

ci: Add node restart to cniv2 E2E #2901

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Aug 17, 2024
+347 −93
Merged

ci: Add node restart to cniv2 E2E #2901

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
11d6c82
ci: add node restart to cniv2 E2E
jpayne3506 Aug 7, 2024
92d88a9
chore: change directory after scale
jpayne3506 Aug 9, 2024
da5c191
chore: fixup nodepool
jpayne3506 Aug 9, 2024
cc07553
chore: pipeline cleanup
jpayne3506 Aug 11, 2024
443cde8
ci: cleanup test resources
jpayne3506 Aug 14, 2024
c7d4068
chore: address comments
jpayne3506 Aug 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .pipelines/singletenancy/aks-swift/e2e-job-template.yaml
View file
Original file line number Diff line number Diff line change
@@ -59,6 +59,7 @@ stages:
parameters:
name: ${{ parameters.name }}
clusterName: ${{ parameters.clusterName }}-$(commitID)
scaleup: 100

- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
parameters:
37 changes: 37 additions & 0 deletions .pipelines/singletenancy/aks-swift/e2e-step-template.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
parameters:
name: ""
clusterName: ""
scaleup: ""

steps:
- bash: |
@@ -47,6 +48,42 @@ steps:
name: "aksswifte2e"
displayName: "Run AKS Swift E2E"

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -e
kubectl get po -owide -A
clusterName=${{ parameters.clusterName }}
echo "Restarting nodes"
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
done
displayName: "Restart Nodes"

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
cd test/integration/load

# Scale Cluster Up/Down to confirm functioning CNS
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
kubectl get pods -owide -A

cd ../../..
echo "Validating Node Restart"
make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cniv2
kubectl delete ns load-test
displayName: "Validate Node Restart"
retryCountOnTaskFailure: 3

- script: |
echo "Run wireserver and metadata connectivity Tests"
bash test/network/wireserver_metadata_test.sh
34 changes: 32 additions & 2 deletions .pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml
View file
Original file line number Diff line number Diff line change
@@ -45,6 +45,7 @@ stages:
jobs:
- job: ${{ parameters.name }}_linux
displayName: Azure CNI Overlay Test Suite | Linux - (${{ parameters.name }})
timeoutInMinutes: 120
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
@@ -56,10 +57,39 @@ stages:
name: ${{ parameters.name }}
clusterName: ${{ parameters.clusterName }}-$(commitID)
os: linux
scaleup: 100

- job: windows_nodepool
displayName: Add Windows Nodepool
dependsOn: ${{ parameters.name }}_linux
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
- agent.os -equals Linux
- Role -equals $(CUSTOM_E2E_ROLE)
steps:
- task: AzureCLI@2
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -e
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}-$(commitID) VM_SIZE_WIN=${{ parameters.vmSize }}
echo "Windows node are successfully added to v4 Overlay Cluster"
kubectl cluster-info
kubectl get node -owide
kubectl get po -owide -A
name: "Add_Windows_Node"
displayName: "Add windows node on v4 overlay cluster"


- job: ${{ parameters.name }}_windows
displayName: Azure CNI Overlay Test Suite | Windows - (${{ parameters.name }})
dependsOn: ${{ parameters.name }}_linux
timeoutInMinutes: 120
dependsOn: windows_nodepool
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
@@ -71,7 +101,7 @@ stages:
name: ${{ parameters.name }}
clusterName: ${{ parameters.clusterName }}-$(commitID)
os: windows
vmSizeWin: ${{ parameters.vmSize }} # Matching linux vmSize
scaleup: 50

- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
parameters:
79 changes: 44 additions & 35 deletions .pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml
View file
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@ parameters:
name: ""
clusterName: ""
os: ""
scaleup: ""

steps:
- bash: |
@@ -64,30 +65,18 @@ steps:
addSpnToEnvironment: true
inlineScript: |
cd test/integration/load
clusterName=${{ parameters.clusterName }}
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)

# Scale Cluster Up/Down to confirm functioning CNS
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
kubectl get pods -owide -A

cd ../../..
echo "Validating Node Restart"
CNI_TYPE=cniv2 RESTART_CASE=true go test -timeout 30m -tags load -run ^TestValidateState$
make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cniv2
kubectl delete ns load-test
displayName: "Validate Node Restart"
retryCountOnTaskFailure: 3

- script: |
echo "validate pod IP assignment before CNS restart"
kubectl get pod -owide -A
echo "validate pod state before CNS restarts"
cd test/integration/load
CNI_TYPE=cniv2 go test -timeout 30m -tags load -run ^TestValidateState$
kubectl rollout restart ds azure-cns -n kube-system
kubectl rollout status ds azure-cns -n kube-system
kubectl get pod -owide -A
echo "validate pods after CNS restart"
CNI_TYPE=cniv2 go test -timeout 30m -tags load -run ^TestValidateState$
name: "restartCNS_ValidatePodState"
displayName: "Restart CNS and validate pod state"
retryCountOnTaskFailure: 3

- script: |
echo "Run wireserver and metadata connectivity Tests"
bash test/network/wireserver_metadata_test.sh
@@ -96,22 +85,6 @@ steps:
displayName: "Run Wireserver and Metadata Connectivity Tests"

- ${{ if eq(parameters.os, 'windows') }}:
- task: AzureCLI@2
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -e
make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }}
echo "Windows node are successfully added to v4 Overlay Cluster"
kubectl cluster-info
kubectl get node -owide
kubectl get po -owide -A
name: "Add_Windows_Node"
displayName: "Add windows node on v4 overlay cluster"

- script: |
nodeList=`kubectl get node -owide | grep Windows | awk '{print $1}'`
for node in $nodeList; do
@@ -132,3 +105,39 @@ steps:
name: "WindowsV4OverlayDatapathTests"
displayName: "Windows v4Overlay Datapath Tests"
retryCountOnTaskFailure: 3

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -e
kubectl get po -owide -A
clusterName=${{ parameters.clusterName }}
echo "Restarting nodes"
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
done
displayName: "Restart Nodes"

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
cd test/integration/load

# Scale Cluster Up/Down to confirm functioning CNS
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=windows go test -count 1 -timeout 30m -tags load -run ^TestLoad$
kubectl get pods -owide -A

cd ../../..
echo "Validating Node Restart"
make test-validate-state OS_TYPE=windows RESTART_CASE=true CNI_TYPE=cniv2
kubectl delete ns load-test
displayName: "Validate Node Restart"
retryCountOnTaskFailure: 3
1 change: 1 addition & 0 deletions ...ines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml
View file
Original file line number Diff line number Diff line change
@@ -55,6 +55,7 @@ stages:
parameters:
name: ${{ parameters.name }}
clusterName: ${{ parameters.clusterName }}-$(commitID)
scaleup: 100

- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
parameters:
39 changes: 38 additions & 1 deletion ...nes/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
parameters:
name: ""
clusterName: ""
scaleup: ""

steps:

@@ -68,6 +69,42 @@ steps:
name: "CiliumStatus"
displayName: "Cilium Status"

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -e
kubectl get po -owide -A
clusterName=${{ parameters.clusterName }}
echo "Restarting nodes"
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
done
displayName: "Restart Nodes"

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
cd test/integration/load

# Scale Cluster Up/Down to confirm functioning CNS
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
kubectl get pods -owide -A

cd ../../..
echo "Validating Node Restart"
make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cilium_dualstack
kubectl delete ns load-test
displayName: "Validate Node Restart"
retryCountOnTaskFailure: 3

- script: |
echo "Run Cilium Connectivity Tests"
cilium status
@@ -109,7 +146,7 @@ steps:
retryCountOnTaskFailure: 3
name: "WireserverMetadataConnectivityTests"
displayName: "Run Wireserver and Metadata Connectivity Tests"

- script: |
cd hack/scripts
chmod +x async-delete-test.sh
1 change: 1 addition & 0 deletions .pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml
View file
Original file line number Diff line number Diff line change
@@ -59,6 +59,7 @@ stages:
name: ${{ parameters.name }}
clusterName: ${{ parameters.clusterName }}-$(commitID)
testHubble: ${{ parameters.testHubble }}
scaleup: 100

- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
parameters:
39 changes: 38 additions & 1 deletion .pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml
View file
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@ parameters:
name: ""
clusterName: ""
testHubble: false
scaleup: ""

steps:
- bash: |
@@ -72,6 +73,42 @@ steps:
name: "CiliumStatus"
displayName: "Cilium Status"

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -e
kubectl get po -owide -A
clusterName=${{ parameters.clusterName }}
echo "Restarting nodes"
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
done
displayName: "Restart Nodes"

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
cd test/integration/load

# Scale Cluster Up/Down to confirm functioning CNS
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
kubectl get pods -owide -A

cd ../../..
echo "Validating Node Restart"
make test-validate-state OS_TYPE=linux RESTART_CASE=true
kubectl delete ns load-test
displayName: "Validate Node Restart"
retryCountOnTaskFailure: 3

- script: |
echo "Run Cilium Connectivity Tests"
cilium status
@@ -147,7 +184,7 @@ steps:
retryCountOnTaskFailure: 3
name: "WireserverMetadataConnectivityTests"
displayName: "Run Wireserver and Metadata Connectivity Tests"

- script: |
cd hack/scripts
chmod +x async-delete-test.sh
1 change: 1 addition & 0 deletions .pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml
View file
Original file line number Diff line number Diff line change
@@ -57,6 +57,7 @@ stages:
parameters:
name: ${{ parameters.name }}
clusterName: ${{ parameters.clusterName }}-$(commitID)
scaleup: 100

- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
parameters:
40 changes: 39 additions & 1 deletion .pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
View file
Original file line number Diff line number Diff line change
@@ -2,6 +2,8 @@ parameters:
name: ""
clusterName: ""
testHubble: false
scaleup: ""


steps:
- bash: |
@@ -93,6 +95,42 @@ steps:
name: "CiliumStatus"
displayName: "Cilium Status"
- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -e
kubectl get po -owide -A
clusterName=${{ parameters.clusterName }}
echo "Restarting nodes"
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
done
displayName: "Restart Nodes"

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
cd test/integration/load
# Scale Cluster Up/Down to confirm functioning CNS
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
kubectl get pods -owide -A
cd ../../..
echo "Validating Node Restart"
make test-validate-state OS_TYPE=linux RESTART_CASE=true
kubectl delete ns load-test
displayName: "Validate Node Restart"
retryCountOnTaskFailure: 3

- script: |
echo "Run Cilium Connectivity Tests"
cilium status
@@ -186,7 +224,7 @@ steps:
retryCountOnTaskFailure: 3
name: "WireserverMetadataConnectivityTests"
displayName: "Run Wireserver and Metadata Connectivity Tests"
- script: |
if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then
echo "Running nightly, skip async delete test"
1 change: 1 addition & 0 deletions .pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml
View file
Original file line number Diff line number Diff line change
@@ -60,6 +60,7 @@ stages:
parameters:
name: ${{ parameters.name }}
clusterName: ${{ parameters.clusterName }}-$(commitID)
scaleup: 100

- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
parameters:
52 changes: 38 additions & 14 deletions .pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
parameters:
name: ""
clusterName: ""
scaleup: ""

steps:
- bash: |
@@ -72,6 +73,42 @@ steps:
name: "CiliumStatus"
displayName: "Cilium Status"
- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -e
kubectl get po -owide -A
clusterName=${{ parameters.clusterName }}
echo "Restarting nodes"
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
done
displayName: "Restart Nodes"

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
cd test/integration/load
# Scale Cluster Up/Down to confirm functioning CNS
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
kubectl get pods -owide -A
cd ../../..
echo "Validating Node Restart"
make test-validate-state OS_TYPE=linux RESTART_CASE=true
kubectl delete ns load-test
displayName: "Validate Node Restart"
retryCountOnTaskFailure: 3

- script: |
echo "Run Cilium Connectivity Tests"
cilium status
@@ -94,26 +131,13 @@ steps:
name: "validatePods"
displayName: "Validate Pods"
- script: |
echo "validate pod IP assignment before CNS restart"
kubectl get pod -owide -A
make test-validate-state
echo "restart CNS"
kubectl rollout restart ds azure-cns -n kube-system
kubectl rollout status ds azure-cns -n kube-system
kubectl get pod -owide -A
echo "validate pods after CNS restart"
make test-validate-state
name: "restartCNS"
displayName: "Restart CNS and validate pods"
- script: |
echo "Run wireserver and metadata connectivity Tests"
bash test/network/wireserver_metadata_test.sh
retryCountOnTaskFailure: 3
name: "WireserverMetadataConnectivityTests"
displayName: "Run Wireserver and Metadata Connectivity Tests"
- script: |
cd hack/scripts
chmod +x async-delete-test.sh
33 changes: 31 additions & 2 deletions .pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml
View file
Original file line number Diff line number Diff line change
@@ -46,6 +46,7 @@ stages:
jobs:
- job: ${{ parameters.name }}_linux
displayName: DualStack Overlay Test Suite | Linux - (${{ parameters.name }})
timeoutInMinutes: 120
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
@@ -57,10 +58,38 @@ stages:
name: ${{ parameters.name }}
clusterName: ${{ parameters.clusterName }}-$(commitID)
os: linux
scaleup: 100

- job: windows_nodepool
displayName: Add Windows Nodepool
dependsOn: ${{ parameters.name }}_linux
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
- agent.os -equals Linux
- Role -equals $(CUSTOM_E2E_ROLE)
steps:
- task: AzureCLI@2
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -e
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}-$(commitID) VM_SIZE_WIN=${{ parameters.vmSize }}
echo "Windows nodes have been successfully added to DualStack Overlay Cluster"
kubectl cluster-info
kubectl get node -owide
kubectl get po -owide -A
name: "Add_Windows_Node"
displayName: "Add windows node"

- job: ${{ parameters.name }}_windows
displayName: DualStack Overlay Test Suite | Windows - (${{ parameters.name }})
dependsOn: ${{ parameters.name }}_linux
timeoutInMinutes: 120
dependsOn: windows_nodepool
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
@@ -72,7 +101,7 @@ stages:
name: ${{ parameters.name }}
clusterName: ${{ parameters.clusterName }}-$(commitID)
os: windows
vmSizeWin: ${{ parameters.vmSize }} # Matching linux vmSize
scaleup: 50

- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
parameters:
82 changes: 45 additions & 37 deletions .pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml
View file
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ parameters:
clusterName: ""
cni: "dualstack"
os: ""
scaleup: ""

steps:
- bash: |
@@ -37,7 +38,7 @@ steps:
kubectl cluster-info
kubectl get node
kubectl get po -owide -A
sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true VALIDATE_DUALSTACK=true CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) CLEANUP=true
sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=dualstack VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true VALIDATE_DUALSTACK=true CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) CLEANUP=true
retryCountOnTaskFailure: 3
name: "integrationTest"
displayName: "Run CNS Integration Tests on AKS DualStack Overlay"
@@ -77,47 +78,19 @@ steps:
addSpnToEnvironment: true
inlineScript: |
cd test/integration/load
clusterName=${{ parameters.clusterName }}
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST)
kubectl get pods -A
# Scale Cluster Up/Down to confirm functioning CNS
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
kubectl get pods -owide -A
cd ../../..
echo "Validating Node Restart"
CNI_TYPE=dualstack RESTART_CASE=true go test -timeout 30m -tags load -run ^TestValidateState$
make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=dualstack
kubectl delete ns load-test
displayName: "Validate Node Restart"
retryCountOnTaskFailure: 3

- script: |
echo "validate pod IP assignment before CNS restart"
kubectl get pod -owide -A
echo "validate pod state before CNS restarts"
cd test/integration/load
CNI_TYPE=dualstack go test -timeout 30m -tags load -run ^TestValidateState$
kubectl rollout restart ds azure-cns -n kube-system
kubectl rollout status ds azure-cns -n kube-system
kubectl get pod -owide -A
echo "validate pods after CNS restarts"
CNI_TYPE=dualstack go test -timeout 30m -tags load -run ^TestValidateState$
name: "restartCNS_ValidatePodState"
displayName: "Restart CNS and Validate Pod State"
retryCountOnTaskFailure: 3
- ${{ if eq(parameters.os, 'windows') }}:
- task: AzureCLI@2
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -e
make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }}
echo "Windows nodes have been successfully added to DualStack Overlay Cluster"
kubectl cluster-info
kubectl get node -owide
kubectl get po -owide -A
name: "Add_Windows_Node"
displayName: "Add windows node"

- script: |
nodeList=`kubectl get node -owide | grep Windows | awk '{print $1}'`
for node in $nodeList; do
@@ -138,3 +111,38 @@ steps:
name: "WindowsDualStackOverlayDatapathTests"
displayName: "Windows DualStack Overlay Datapath Tests"
retryCountOnTaskFailure: 3
- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -e
clusterName=${{ parameters.clusterName }}
echo "Restarting nodes"
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) --query "[].name" -o tsv); do
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) VMSS_NAME=${val}
done
displayName: "Restart Nodes"

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
cd test/integration/load
# Scale Cluster Up/Down to confirm functioning CNS
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=windows go test -count 1 -timeout 30m -tags load -run ^TestLoad$
kubectl get pods -owide -A
cd ../../..
echo "Validating Node Restart"
make test-validate-state OS_TYPE=windows RESTART_CASE=true CNI_TYPE=cniv2
kubectl delete ns load-test
displayName: "Validate Node Restart"
retryCountOnTaskFailure: 3