Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ci: Add node restart to cniv2 E2E #2901

Merged
merged 6 commits into from
Aug 17, 2024
+347 −93
Merged

ci: Add node restart to cniv2 E2E #2901

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
11d6c82
ci: add node restart to cniv2 E2E
jpayne3506 Aug 7, 2024
92d88a9
chore: change directory after scale
jpayne3506 Aug 9, 2024
da5c191
chore: fixup nodepool
jpayne3506 Aug 9, 2024
cc07553
chore: pipeline cleanup
jpayne3506 Aug 11, 2024
443cde8
ci: cleanup test resources
jpayne3506 Aug 14, 2024
c7d4068
chore: address comments
jpayne3506 Aug 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .pipelines/singletenancy/aks-swift/e2e-job-template.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ stages:
parameters:
name: ${{ parameters.name }}
clusterName: ${{ parameters.clusterName }}-$(commitID)
scaleup: 100

- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
parameters:
Expand Down
37 changes: 37 additions & 0 deletions .pipelines/singletenancy/aks-swift/e2e-step-template.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
parameters:
name: ""
clusterName: ""
scaleup: ""

steps:
- bash: |
Expand Down Expand Up @@ -47,6 +48,42 @@ steps:
name: "aksswifte2e"
displayName: "Run AKS Swift E2E"

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -e
kubectl get po -owide -A
clusterName=${{ parameters.clusterName }}
echo "Restarting nodes"
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
done
displayName: "Restart Nodes"

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
cd test/integration/load

# Scale Cluster Up/Down to confirm functioning CNS
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
kubectl get pods -owide -A

cd ../../..
echo "Validating Node Restart"
make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cniv2
kubectl delete ns load-test
displayName: "Validate Node Restart"
retryCountOnTaskFailure: 3

- script: |
echo "Run wireserver and metadata connectivity Tests"
bash test/network/wireserver_metadata_test.sh
Expand Down
34 changes: 32 additions & 2 deletions .pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ stages:
jobs:
- job: ${{ parameters.name }}_linux
displayName: Azure CNI Overlay Test Suite | Linux - (${{ parameters.name }})
timeoutInMinutes: 120
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
Expand All @@ -56,10 +57,39 @@ stages:
name: ${{ parameters.name }}
clusterName: ${{ parameters.clusterName }}-$(commitID)
os: linux
scaleup: 100

- job: windows_nodepool
displayName: Add Windows Nodepool
QxBytes marked this conversation as resolved.
Show resolved Hide resolved
dependsOn: ${{ parameters.name }}_linux
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
- agent.os -equals Linux
- Role -equals $(CUSTOM_E2E_ROLE)
steps:
- task: AzureCLI@2
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -e
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}-$(commitID) VM_SIZE_WIN=${{ parameters.vmSize }}
echo "Windows node are successfully added to v4 Overlay Cluster"
kubectl cluster-info
kubectl get node -owide
kubectl get po -owide -A
name: "Add_Windows_Node"
displayName: "Add windows node on v4 overlay cluster"


- job: ${{ parameters.name }}_windows
displayName: Azure CNI Overlay Test Suite | Windows - (${{ parameters.name }})
dependsOn: ${{ parameters.name }}_linux
timeoutInMinutes: 120
dependsOn: windows_nodepool
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
Expand All @@ -71,7 +101,7 @@ stages:
name: ${{ parameters.name }}
clusterName: ${{ parameters.clusterName }}-$(commitID)
os: windows
vmSizeWin: ${{ parameters.vmSize }} # Matching linux vmSize
scaleup: 50

- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
parameters:
Expand Down
79 changes: 44 additions & 35 deletions .pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ parameters:
name: ""
clusterName: ""
os: ""
scaleup: ""

steps:
- bash: |
Expand Down Expand Up @@ -64,30 +65,18 @@ steps:
addSpnToEnvironment: true
inlineScript: |
cd test/integration/load
clusterName=${{ parameters.clusterName }}
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)

# Scale Cluster Up/Down to confirm functioning CNS
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
kubectl get pods -owide -A

cd ../../..
echo "Validating Node Restart"
CNI_TYPE=cniv2 RESTART_CASE=true go test -timeout 30m -tags load -run ^TestValidateState$
make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cniv2
kubectl delete ns load-test
displayName: "Validate Node Restart"
retryCountOnTaskFailure: 3

- script: |
echo "validate pod IP assignment before CNS restart"
kubectl get pod -owide -A
echo "validate pod state before CNS restarts"
cd test/integration/load
CNI_TYPE=cniv2 go test -timeout 30m -tags load -run ^TestValidateState$
kubectl rollout restart ds azure-cns -n kube-system
kubectl rollout status ds azure-cns -n kube-system
kubectl get pod -owide -A
echo "validate pods after CNS restart"
CNI_TYPE=cniv2 go test -timeout 30m -tags load -run ^TestValidateState$
name: "restartCNS_ValidatePodState"
displayName: "Restart CNS and validate pod state"
retryCountOnTaskFailure: 3

- script: |
echo "Run wireserver and metadata connectivity Tests"
bash test/network/wireserver_metadata_test.sh
Expand All @@ -96,22 +85,6 @@ steps:
displayName: "Run Wireserver and Metadata Connectivity Tests"

- ${{ if eq(parameters.os, 'windows') }}:
- task: AzureCLI@2
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -e
make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }}
echo "Windows node are successfully added to v4 Overlay Cluster"
kubectl cluster-info
kubectl get node -owide
kubectl get po -owide -A
name: "Add_Windows_Node"
displayName: "Add windows node on v4 overlay cluster"

- script: |
nodeList=`kubectl get node -owide | grep Windows | awk '{print $1}'`
for node in $nodeList; do
Expand All @@ -132,3 +105,39 @@ steps:
name: "WindowsV4OverlayDatapathTests"
displayName: "Windows v4Overlay Datapath Tests"
retryCountOnTaskFailure: 3

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -e
kubectl get po -owide -A
clusterName=${{ parameters.clusterName }}
echo "Restarting nodes"
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
done
displayName: "Restart Nodes"

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
cd test/integration/load

# Scale Cluster Up/Down to confirm functioning CNS
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=windows go test -count 1 -timeout 30m -tags load -run ^TestLoad$
kubectl get pods -owide -A

cd ../../..
echo "Validating Node Restart"
make test-validate-state OS_TYPE=windows RESTART_CASE=true CNI_TYPE=cniv2
kubectl delete ns load-test
displayName: "Validate Node Restart"
retryCountOnTaskFailure: 3
1 change: 1 addition & 0 deletions ...ines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ stages:
parameters:
name: ${{ parameters.name }}
clusterName: ${{ parameters.clusterName }}-$(commitID)
scaleup: 100

- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
parameters:
Expand Down
39 changes: 38 additions & 1 deletion ...nes/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
parameters:
name: ""
clusterName: ""
scaleup: ""

steps:

Expand Down Expand Up @@ -68,6 +69,42 @@ steps:
name: "CiliumStatus"
displayName: "Cilium Status"

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -e
kubectl get po -owide -A
clusterName=${{ parameters.clusterName }}
echo "Restarting nodes"
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
done
displayName: "Restart Nodes"

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
cd test/integration/load

# Scale Cluster Up/Down to confirm functioning CNS
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
kubectl get pods -owide -A

cd ../../..
echo "Validating Node Restart"
make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cilium_dualstack
kubectl delete ns load-test
displayName: "Validate Node Restart"
retryCountOnTaskFailure: 3

- script: |
echo "Run Cilium Connectivity Tests"
cilium status
Expand Down Expand Up @@ -109,7 +146,7 @@ steps:
retryCountOnTaskFailure: 3
name: "WireserverMetadataConnectivityTests"
displayName: "Run Wireserver and Metadata Connectivity Tests"

- script: |
cd hack/scripts
chmod +x async-delete-test.sh
Expand Down
1 change: 1 addition & 0 deletions .pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ stages:
name: ${{ parameters.name }}
clusterName: ${{ parameters.clusterName }}-$(commitID)
testHubble: ${{ parameters.testHubble }}
scaleup: 100

- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
parameters:
Expand Down
39 changes: 38 additions & 1 deletion .pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ parameters:
name: ""
clusterName: ""
testHubble: false
scaleup: ""

steps:
- bash: |
Expand Down Expand Up @@ -72,6 +73,42 @@ steps:
name: "CiliumStatus"
displayName: "Cilium Status"

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -e
kubectl get po -owide -A
clusterName=${{ parameters.clusterName }}
echo "Restarting nodes"
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
done
displayName: "Restart Nodes"

- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
cd test/integration/load

# Scale Cluster Up/Down to confirm functioning CNS
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
kubectl get pods -owide -A

cd ../../..
echo "Validating Node Restart"
make test-validate-state OS_TYPE=linux RESTART_CASE=true
kubectl delete ns load-test
displayName: "Validate Node Restart"
retryCountOnTaskFailure: 3

- script: |
echo "Run Cilium Connectivity Tests"
cilium status
Expand Down Expand Up @@ -147,7 +184,7 @@ steps:
retryCountOnTaskFailure: 3
name: "WireserverMetadataConnectivityTests"
displayName: "Run Wireserver and Metadata Connectivity Tests"

- script: |
cd hack/scripts
chmod +x async-delete-test.sh
Expand Down
1 change: 1 addition & 0 deletions .pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ stages:
parameters:
name: ${{ parameters.name }}
clusterName: ${{ parameters.clusterName }}-$(commitID)
scaleup: 100

- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
parameters:
Expand Down
Loading
Loading