From 83ea6ad3009721f20bfd7ad4604d37e1d15b2dd6 Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Thu, 27 Jun 2024 10:56:23 +0100 Subject: [PATCH] test(e2e): Fix up e2e tests wrt Kubernetes versions Updated Kubernetes version for e2e to use to v1.29.5, which is the latest available version of Nutanix and AWS images. --- README.md | 2 +- go.mod | 1 + go.sum | 2 ++ make/go.mk | 2 +- make/kind.mk | 8 ++--- test/e2e/config/caren.yaml | 2 +- test/e2e/csi_helpers.go | 6 ++-- test/e2e/e2e_suite_test.go | 2 +- test/e2e/ownerreference_helpers.go | 8 +++++ test/e2e/quick_start_test.go | 52 +++++++++++++++++++++++------- 10 files changed, 64 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index cb5514681..678c3914e 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ For example, the following command will create a Docker cluster with Cilium CNI ```shell export CLUSTER_NAME=docker-cluster-cilium-helm-addon export CLUSTER_FILE=examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml -export KUBERNETES_VERSION=v1.29.4 +export KUBERNETES_VERSION=v1.29.5 ``` ```shell diff --git a/go.mod b/go.mod index dba5ed647..4a0b5afb4 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 github.com/pkg/errors v0.9.1 + github.com/samber/lo v1.39.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index 0cc00ff6c..c1cb4f355 100644 --- a/go.sum +++ b/go.sum @@ -218,6 +218,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= +github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= diff --git a/make/go.mk b/make/go.mk index fc5acfd8b..c7241fc0e 100644 --- a/make/go.mk +++ b/make/go.mk @@ -80,7 +80,7 @@ E2E_PARALLEL_NODES ?= $(if $(filter $(E2E_DRYRUN),true),1,$(shell nproc --ignore E2E_FLAKE_ATTEMPTS ?= 1 E2E_CONF_FILE ?= $(REPO_ROOT)/test/e2e/config/caren.yaml E2E_CONF_FILE_ENVSUBST ?= $(basename $(E2E_CONF_FILE))-envsubst.yaml -export E2E_DEFAULT_KUBERNETES_VERSION ?= v1.29.4 +export E2E_DEFAULT_KUBERNETES_VERSION ?= v1.29.5 ARTIFACTS ?= ${REPO_ROOT}/_artifacts .PHONY: e2e-test diff --git a/make/kind.mk b/make/kind.mk index 9a36834b2..f2f873dfd 100644 --- a/make/kind.mk +++ b/make/kind.mk @@ -7,10 +7,10 @@ KIND_CLUSTER_NAME ?= $(GITHUB_REPOSITORY)-dev KIND_KUBECONFIG ?= $(KIND_DIR)/$(KIND_CLUSTER_NAME)/kubeconfig KINDEST_NODE_IMAGE ?= ghcr.io/mesosphere/kind-node -KINDEST_NODE_VERSION_v1.27 ?= v1.27.13 -KINDEST_NODE_VERSION_v1.28 ?= v1.28.9 -KINDEST_NODE_VERSION_v1.29 ?= v1.29.4 -KINDEST_NODE_VERSION_v1.30 ?= v1.30.0 +KINDEST_NODE_VERSION_v1.27 ?= v1.27.15 +KINDEST_NODE_VERSION_v1.28 ?= v1.28.11 +KINDEST_NODE_VERSION_v1.29 ?= v1.29.6 +KINDEST_NODE_VERSION_v1.30 ?= v1.30.2 # Allow easy override of Kubernetes version to use via `make KIND_KUBERNETES_VERSION=v1.23` to use in CI KIND_KUBERNETES_VERSION ?= v1.29 ifndef KINDEST_NODE_VERSION_$(KIND_KUBERNETES_VERSION) diff --git a/test/e2e/config/caren.yaml b/test/e2e/config/caren.yaml index b08579d14..4f108cd20 100644 --- a/test/e2e/config/caren.yaml +++ b/test/e2e/config/caren.yaml @@ -167,7 +167,7 @@ variables: KUBERNETES_VERSION: "${E2E_DEFAULT_KUBERNETES_VERSION}" # Override Kubernetes version for test workload clusters for specific providers by setting the env variables # `KUBERNETES_VERSION_`, where `` is the uppercase provider name, e.g. - # `KUBERNETES_VERSION_DOCKER: v1.29.4`. + # `KUBERNETES_VERSION_DOCKER: v1.29.5`. # KUBERNETES_VERSION_DOCKER: "${KINDEST_IMAGE_TAG}" SERVICE_CIDR: "10.128.0.0/12" POD_CIDR: "192.168.0.0/16" diff --git a/test/e2e/csi_helpers.go b/test/e2e/csi_helpers.go index bb989b543..1042432c3 100644 --- a/test/e2e/csi_helpers.go +++ b/test/e2e/csi_helpers.go @@ -318,7 +318,7 @@ func waitForNutanixCSIToBeReadyInWorkloadCluster( Deployment: &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "nutanix-csi-controller", - Namespace: metav1.NamespaceSystem, + Namespace: "ntnx-system", }, }, }, input.deploymentIntervals...) @@ -328,7 +328,7 @@ func waitForNutanixCSIToBeReadyInWorkloadCluster( DaemonSet: &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: "nutanix-csi-node", - Namespace: metav1.NamespaceSystem, + Namespace: "ntnx-system", }, }, }, input.daemonSetIntervals...) @@ -357,6 +357,8 @@ func waitForStorageClassesToExistInWorkloadCluster( provisioner = string(v1alpha1.LocalPathProvisioner) case v1alpha1.CSIProviderAWSEBS: provisioner = string(v1alpha1.AWSEBSProvisioner) + case v1alpha1.CSIProviderNutanix: + provisioner = string(v1alpha1.NutanixProvisioner) default: Fail( fmt.Sprintf( diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index a4e6518a7..f74486d40 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -70,7 +70,7 @@ func init() { flag.StringVar( &bootstrapKubernetesVersion, "e2e.bootstrap-kind-version", - "v1.29.4", + "v1.29.5", "the version of the image used in bootstrap cluster", ) } diff --git a/test/e2e/ownerreference_helpers.go b/test/e2e/ownerreference_helpers.go index 852c223ea..c66db0b6b 100644 --- a/test/e2e/ownerreference_helpers.go +++ b/test/e2e/ownerreference_helpers.go @@ -16,6 +16,7 @@ import ( addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework" + capxv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/github.com/nutanix-cloud-native/cluster-api-provider-nutanix/api/v1beta1" caaphv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" ) @@ -91,6 +92,12 @@ var ( Controller: ptr.To(true), } + capxGroupVersion = capxv1.GroupVersion.String() + nutanixClusterOwner = metav1.OwnerReference{ + Kind: nutanixClusterKind, + APIVersion: capxGroupVersion, + } + // AddonReferenceAssertions maps addontypes to functions which return an error if the passed OwnerReferences // aren't as expected. AddonReferenceAssertions = map[string]func([]metav1.OwnerReference) error{ @@ -191,6 +198,7 @@ var ( []metav1.OwnerReference{kubeadmControlPlaneController}, []metav1.OwnerReference{kubeadmConfigController}, []metav1.OwnerReference{clusterOwner}, + []metav1.OwnerReference{clusterOwner, nutanixClusterOwner}, ) }, configMapKind: func(owners []metav1.OwnerReference) error { diff --git a/test/e2e/quick_start_test.go b/test/e2e/quick_start_test.go index e19a34264..193c4fe23 100644 --- a/test/e2e/quick_start_test.go +++ b/test/e2e/quick_start_test.go @@ -8,11 +8,14 @@ package e2e import ( "fmt" "slices" + "strconv" "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" "k8s.io/utils/ptr" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" clusterctlcluster "sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster" capie2e "sigs.k8s.io/cluster-api/test/e2e" "sigs.k8s.io/cluster-api/test/framework" @@ -63,7 +66,7 @@ var _ = Describe("Quick start", Serial, func() { // available that are not available in other providers. // This version can be specified in `test/e2e/config/caren.yaml` with a variable named // `KUBERNETES_VERSION_`, where `` is the uppercase provider name, e.g. - // `KUBERNETES_VERSION_DOCKER: v1.29.4`. + // `KUBERNETES_VERSION_DOCKER: v1.29.5`. testE2EConfig := e2eConfig.DeepCopy() varName := capie2e.KubernetesVersion + "_" + strings.ToUpper( lowercaseProvider, @@ -99,6 +102,16 @@ var _ = Describe("Quick start", Serial, func() { KubernetesReferenceAssertions, ) + workloadCluster := framework.GetClusterByName( + ctx, + framework.GetClusterByNameInput{ + Namespace: namespace, + Name: clusterName, + Getter: proxy.GetClient(), + }, + ) + Expect(workloadCluster.Spec.Topology).ToNot(BeNil()) + By("Waiting until nodes are ready") workloadProxy := proxy.GetWorkloadCluster( ctx, @@ -106,6 +119,32 @@ var _ = Describe("Quick start", Serial, func() { clusterName, ) workloadClient := workloadProxy.GetClient() + + nodeCount := int( + ptr.Deref( + workloadCluster.Spec.Topology.ControlPlane.Replicas, + 0, + ), + ) + + lo.Reduce( + workloadCluster.Spec.Topology.Workers.MachineDeployments, + func(agg int, md clusterv1.MachineDeploymentTopology, _ int) int { + switch { + case md.Replicas != nil: + return agg + int(ptr.Deref(md.Replicas, 0)) + case md.Metadata.Annotations["cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"] != "": + minSize, err := strconv.Atoi( + md.Metadata.Annotations["cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"], + ) + Expect(err).ToNot(HaveOccurred()) + return agg + minSize + default: + return agg + } + }, + 0, + ) + framework.WaitForNodesReady( ctx, framework.WaitForNodesReadyInput{ @@ -113,7 +152,7 @@ var _ = Describe("Quick start", Serial, func() { KubernetesVersion: testE2EConfig.GetVariable( capie2e.KubernetesVersion, ), - Count: 2, + Count: nodeCount, WaitForNodesReady: testE2EConfig.GetIntervals( flavour, "wait-nodes-ready", @@ -124,15 +163,6 @@ var _ = Describe("Quick start", Serial, func() { By( "Waiting for all requested addons to be ready in workload cluster", ) - workloadCluster := framework.GetClusterByName( - ctx, - framework.GetClusterByNameInput{ - Namespace: namespace, - Name: clusterName, - Getter: proxy.GetClient(), - }, - ) - Expect(workloadCluster.Spec.Topology).ToNot(BeNil()) clusterVars := variables.ClusterVariablesToVariablesMap( workloadCluster.Spec.Topology.Variables, )