From 2c3d4470883e2da9db77a6807f523f75319d1eb3 Mon Sep 17 00:00:00 2001 From: John Payne <89417863+jpayne3506@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:26:24 -0700 Subject: [PATCH] [backport] fix: set cluster auto-upgrade and node auto-upgrade in the hackfile #2253 (#2939) * fix: set cluster auto-upgrade and node auto-upgrade in the hackfile (#2253) Signed-off-by: Evan Baker * fix: test v1.5 train on k8s 1.28 (#2564) * fix: test main/v1.5 branch on k8s 1.28 Signed-off-by: Evan Baker * 1.28 ds changes * ci: specify vars for envsubst --------- Signed-off-by: Evan Baker Co-authored-by: jpayne3506 --------- Signed-off-by: Evan Baker Co-authored-by: Evan Baker --- .../singletenancy/aks/e2e-job-template.yaml | 1 + .pipelines/templates/create-cluster.yaml | 11 ++++- hack/aks/Makefile | 47 +++++++++++++++---- .../cni/cni-installer-v1-windows.yaml | 3 ++ 4 files changed, 52 insertions(+), 10 deletions(-) diff --git a/.pipelines/singletenancy/aks/e2e-job-template.yaml b/.pipelines/singletenancy/aks/e2e-job-template.yaml index cbc9cadf57..85502a6600 100644 --- a/.pipelines/singletenancy/aks/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks/e2e-job-template.yaml @@ -29,6 +29,7 @@ stages: clusterType: ${{ parameters.clusterType }} clusterName: ${{ parameters.clusterName }}-$(commitID) vmSize: ${{ parameters.vmSize }} + vmSizeWin: ${{ parameters.vmSize }} k8sVersion: ${{ parameters.k8sVersion }} dependsOn: ${{ parameters.dependsOn }} region: $(REGION_AKS_CLUSTER_TEST) diff --git a/.pipelines/templates/create-cluster.yaml b/.pipelines/templates/create-cluster.yaml index 0066e1ac1e..27b4c42d09 100644 --- a/.pipelines/templates/create-cluster.yaml +++ b/.pipelines/templates/create-cluster.yaml @@ -5,7 +5,7 @@ parameters: clusterName: "" # Recommended to pass in unique identifier vmSize: "" k8sVersion: "" - windowsOsSku: "Windows2022" # Currently we only support Windows2022 + osSkuWin: "Windows2022" # Currently we only support Windows2022 dependsOn: "" region: "" @@ -31,6 +31,13 @@ jobs: fi mkdir -p ~/.kube/ make -C ./hack/aks azcfg AZCLI=az REGION=${{ parameters.region }} - make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=${{ parameters.region }} SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} K8S_VER=${{ parameters.k8sVersion }} VM_SIZE=${{ parameters.vmSize }} WINDOWS_OS_SKU=${{ parameters.windowsOsSku }} WINDOWS_VM_SKU=${{ parameters.vmSize }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} + + make -C ./hack/aks ${{ parameters.clusterType }} \ + AZCLI=az REGION=${{ parameters.region }} SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) \ + CLUSTER=${{ parameters.clusterName }} \ + VM_SIZE=${{ parameters.vmSize }} VM_SIZE_WIN=${{ parameters.vmSizeWin }} \ + OS_SKU_WIN=${{ parameters.osSkuWin }} OS=${{parameters.os}} \ + WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} + echo "Cluster successfully created" displayName: Cluster - ${{ parameters.clusterType }} diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 5c3306af95..c8c12420c4 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -8,13 +8,15 @@ AZIMG = mcr.microsoft.com/azure-cli AZCLI ?= docker run --rm -v $(AZCFG):/root/.azure -v $(KUBECFG):/root/.kube -v $(SSH):/root/.ssh -v $(PWD):/root/tmpsrc $(AZIMG) az # overrideable defaults -REGION ?= westus2 +AUTOUPGRADE ?= patch +K8S_VER ?= 1.27 # Designated for Long Term Support, July 2025 | Only Ubuntu 22.04 is supported +NODE_COUNT ?= 2 +NODEUPGRADE ?= NodeImage OS_SKU ?= Ubuntu -WINDOWS_OS_SKU ?= Windows2022 +OS_SKU_WIN ?= Windows2022 +REGION ?= westus2 VM_SIZE ?= Standard_B2s -NODE_COUNT ?= 2 -K8S_VER ?= 1.27 # Designated for Long Term Support, July 2025 | Only Ubuntu 22.04 is supported -WINDOWS_VM_SKU ?= Standard_B2s +VM_SIZE_WIN ?= Standard_B2s # overrideable variables SUB ?= $(AZURE_SUBSCRIPTION) @@ -88,6 +90,9 @@ up: swift-up ## Alias to swift-up overlay-byocni-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -101,6 +106,9 @@ overlay-byocni-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster overlay-cilium-up: rg-up overlay-net-up ## Brings up an Overlay Cilium cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -115,6 +123,9 @@ overlay-cilium-up: rg-up overlay-net-up ## Brings up an Overlay Cilium cluster overlay-up: rg-up overlay-net-up ## Brings up an Overlay AzCNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -128,6 +139,9 @@ overlay-up: rg-up overlay-net-up ## Brings up an Overlay AzCNI cluster swift-byocni-up: rg-up swift-net-up ## Bring up a SWIFT BYO CNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -142,6 +156,9 @@ swift-byocni-up: rg-up swift-net-up ## Bring up a SWIFT BYO CNI cluster swift-cilium-up: rg-up swift-net-up ## Bring up a SWIFT Cilium cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -156,6 +173,9 @@ swift-cilium-up: rg-up swift-net-up ## Bring up a SWIFT Cilium cluster swift-up: rg-up swift-net-up ## Bring up a SWIFT AzCNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -168,6 +188,9 @@ swift-up: rg-up swift-net-up ## Bring up a SWIFT AzCNI cluster cilium-overlay-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster without kube-proxy for Cilium $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -182,6 +205,9 @@ cilium-overlay-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster cilium-podsubnet-up: rg-up swift-net-up ## Bring up a SWIFT BYO CNI cluster without kube-proxy for Cilium $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --load-balancer-sku basic \ @@ -196,6 +222,9 @@ cilium-podsubnet-up: rg-up swift-net-up ## Bring up a SWIFT BYO CNI cluster with windows-cniv1-up: rg-up overlay-net-up ## Bring up a Windows CNIv1 cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --network-plugin azure \ @@ -207,16 +236,19 @@ windows-cniv1-up: rg-up overlay-net-up ## Bring up a Windows CNIv1 cluster $(AZCLI) aks nodepool add --resource-group $(GROUP) --cluster-name $(CLUSTER) \ --os-type Windows \ - --os-sku $(WINDOWS_OS_SKU) \ + --os-sku $(OS_SKU_WIN) \ --max-pods 250 \ --name npwin \ --node-count $(NODE_COUNT) \ - -s $(WINDOWS_VM_SKU) + -s $(VM_SIZE_WIN) @$(MAKE) set-kubeconf linux-cniv1-up: rg-up overlay-net-up $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --auto-upgrade-channel $(AUTOUPGRADE) \ + --node-os-upgrade-channel $(NODEUPGRADE) \ + --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --max-pods 250 \ @@ -225,7 +257,6 @@ linux-cniv1-up: rg-up overlay-net-up --os-sku $(OS_SKU) \ --no-ssh-key \ --yes - @$(MAKE) set-kubeconf down: ## Delete the cluster diff --git a/test/integration/manifests/cni/cni-installer-v1-windows.yaml b/test/integration/manifests/cni/cni-installer-v1-windows.yaml index ca303efff1..07fde1c414 100644 --- a/test/integration/manifests/cni/cni-installer-v1-windows.yaml +++ b/test/integration/manifests/cni/cni-installer-v1-windows.yaml @@ -65,6 +65,9 @@ spec: - azure-vnet-telemetry.config - -o - /k/azurecni/bin/azure-vnet-telemetry.config + env: + - name: PATHEXT + value: .COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL;; volumeMounts: - name: cni-bin mountPath: /k/azurecni/bin/