From 887298a21491c9b497e81d7a5455ac8b4048af15 Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Thu, 22 Aug 2024 08:32:19 +0200 Subject: [PATCH] test: add autoscaler test --- test/e2e/autoscaler_test.go | 126 +++++++++++++++++++++++++++++++++++ test/e2e/config/vsphere.yaml | 1 + 2 files changed, 127 insertions(+) create mode 100644 test/e2e/autoscaler_test.go diff --git a/test/e2e/autoscaler_test.go b/test/e2e/autoscaler_test.go new file mode 100644 index 0000000000..597f6f38b4 --- /dev/null +++ b/test/e2e/autoscaler_test.go @@ -0,0 +1,126 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + capi_e2e "sigs.k8s.io/cluster-api/test/e2e" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe("When using the autoscaler with Cluster API using ClusterClass and scale to zero [vcsim] [supervisor] [PR-Blocking] [ClusterClass]", func() { + if testMode == GovmomiTestMode { + // This test is only implemented for supervisor + Skip("This test is only implemented for supervisor") + } + + // Note: This installs a cluster based on KUBERNETES_VERSION_UPGRADE_FROM and then upgrades to + // KUBERNETES_VERSION_UPGRADE_TO. + const specName = "k8s-upgrade" // aligned to CAPI + Setup(specName, func(testSpecificSettingsGetter func() testSettings) { + capi_e2e.AutoscalerSpec(ctx, func() capi_e2e.AutoscalerSpecInput { + return capi_e2e.AutoscalerSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: testSpecificSettingsGetter().ClusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + Flavor: ptr.To(testSpecificSettingsGetter().FlavorForMode(clusterctl.DefaultFlavor)), + PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc, + InfrastructureMachineTemplateKind: "vspheremachinetemplates", + AutoscalerVersion: "v1.30.0", + InstallOnManagementCluster: true, + // Testing autoscale from/to zero. + PostAutoscalingTest: func(bootstrapClusterProxy framework.ClusterProxy, namespace, clusterName string) { + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: clusterName, + }, + } + Expect(bootstrapClusterProxy.GetClient().Get(ctx, client.ObjectKeyFromObject(cluster), cluster)).To(Succeed()) + + workloadClusterProxy := bootstrapClusterProxy.GetWorkloadCluster(ctx, namespace, clusterName) + // Delete the old scale-up deployment + By("Deleting the leftover scale up deployment") + framework.DeleteScaleUpDeploymentAndWait(ctx, framework.DeleteScaleUpDeploymentAndWaitInput{ + ClusterProxy: workloadClusterProxy, + WaitForDelete: e2eConfig.GetIntervals(specName, "wait-autoscaler"), + }) + + machineDeployments := framework.GetMachineDeploymentsByCluster(ctx, framework.GetMachineDeploymentsByClusterInput{ + Lister: bootstrapClusterProxy.GetClient(), + ClusterName: clusterName, + Namespace: namespace, + }) + + Expect(machineDeployments).To(HaveLen(1)) + mdOriginalReplicas := *machineDeployments[0].Spec.Replicas + + // The machine deployment should have more than one replica. + Expect(mdOriginalReplicas).ToNot(Equal(int32(0))) + + By("Checking we can scale the MachineDeployment") + // Scale up the MachineDeployment. Since autoscaler is disabled we should be able to do this. + framework.ScaleAndWaitMachineDeploymentTopology(ctx, framework.ScaleAndWaitMachineDeploymentTopologyInput{ + ClusterProxy: bootstrapClusterProxy, + Cluster: cluster, + Replicas: 2, + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + + By("Enabling autoscaler for the MachineDeployment") + // Enable autoscaler on the MachineDeployment. + framework.EnableAutoscalerForMachineDeploymentTopologyAndWait(ctx, framework.EnableAutoscalerForMachineDeploymentTopologyAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + Cluster: cluster, + NodeGroupMinSize: "0", + NodeGroupMaxSize: "5", + WaitForAnnotationsToBeAdded: e2eConfig.GetIntervals(specName, "wait-autoscaler"), + }) + + By("Checking the MachineDeployment finished scaling down to zero") + framework.AssertMachineDeploymentReplicas(ctx, framework.AssertMachineDeploymentReplicasInput{ + Getter: bootstrapClusterProxy.GetClient(), + MachineDeployment: machineDeployments[0], + Replicas: 0, + WaitForMachineDeployment: e2eConfig.GetIntervals(specName, "wait-autoscaler"), + }) + + By("Creating workload that forces the system to scale up") + framework.AddScaleUpDeploymentAndWait(ctx, framework.AddScaleUpDeploymentAndWaitInput{ + ClusterProxy: workloadClusterProxy, + }, e2eConfig.GetIntervals(specName, "wait-autoscaler")...) + + By("Checking the MachineDeployment scaled up again") + framework.AssertMachineDeploymentReplicas(ctx, framework.AssertMachineDeploymentReplicasInput{ + Getter: bootstrapClusterProxy.GetClient(), + MachineDeployment: machineDeployments[0], + Replicas: 1, + WaitForMachineDeployment: e2eConfig.GetIntervals(specName, "wait-autoscaler"), + }) + }, + } + }) + }) +}) diff --git a/test/e2e/config/vsphere.yaml b/test/e2e/config/vsphere.yaml index b8b88ee97f..2966c40e24 100644 --- a/test/e2e/config/vsphere.yaml +++ b/test/e2e/config/vsphere.yaml @@ -258,6 +258,7 @@ variables: KUBERNETES_VERSION_LATEST_CI: "ci/latest-1.32" CPI_IMAGE_K8S_VERSION: "v1.31.0" CNI: "./data/cni/calico/calico.yaml" + AUTOSCALER_WORKLOAD: "./data/autoscaler/autoscaler-to-management-workload.yaml" EXP_CLUSTER_RESOURCE_SET: "true" EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true" CONTROL_PLANE_MACHINE_COUNT: 1