From e56427e22fe88b445dd2b46423a59367c5aabd43 Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Thu, 27 Jun 2024 13:41:53 +0100 Subject: [PATCH] test(e2e): Various e2e tests fixes (#754) Fixed up Nutanix CSI namespace in e2e tests. Updated Kubernetes version for e2e to use to v1.29.5, which is the latest available version of Nutanix and AWS images. Tested locally by running AWS and Nutanix e2e tests - all passed! Docker is tested in CI. Depends on #751. --- README.md | 2 +- go.mod | 1 + go.sum | 2 ++ make/go.mk | 2 +- make/kind.mk | 8 ++--- test/e2e/config/caren.yaml | 2 +- test/e2e/csi_helpers.go | 6 ++-- test/e2e/e2e_suite_test.go | 2 +- test/e2e/ownerreference_helpers.go | 8 +++++ test/e2e/quick_start_test.go | 52 +++++++++++++++++++++++------- 10 files changed, 64 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index cb5514681..678c3914e 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ For example, the following command will create a Docker cluster with Cilium CNI ```shell export CLUSTER_NAME=docker-cluster-cilium-helm-addon export CLUSTER_FILE=examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml -export KUBERNETES_VERSION=v1.29.4 +export KUBERNETES_VERSION=v1.29.5 ``` ```shell diff --git a/go.mod b/go.mod index dba5ed647..4a0b5afb4 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 github.com/pkg/errors v0.9.1 + github.com/samber/lo v1.39.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index 0cc00ff6c..c1cb4f355 100644 --- a/go.sum +++ b/go.sum @@ -218,6 +218,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= +github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= diff --git a/make/go.mk b/make/go.mk index fc5acfd8b..c7241fc0e 100644 --- a/make/go.mk +++ b/make/go.mk @@ -80,7 +80,7 @@ E2E_PARALLEL_NODES ?= $(if $(filter $(E2E_DRYRUN),true),1,$(shell nproc --ignore E2E_FLAKE_ATTEMPTS ?= 1 E2E_CONF_FILE ?= $(REPO_ROOT)/test/e2e/config/caren.yaml E2E_CONF_FILE_ENVSUBST ?= $(basename $(E2E_CONF_FILE))-envsubst.yaml -export E2E_DEFAULT_KUBERNETES_VERSION ?= v1.29.4 +export E2E_DEFAULT_KUBERNETES_VERSION ?= v1.29.5 ARTIFACTS ?= ${REPO_ROOT}/_artifacts .PHONY: e2e-test diff --git a/make/kind.mk b/make/kind.mk index 9a36834b2..f2f873dfd 100644 --- a/make/kind.mk +++ b/make/kind.mk @@ -7,10 +7,10 @@ KIND_CLUSTER_NAME ?= $(GITHUB_REPOSITORY)-dev KIND_KUBECONFIG ?= $(KIND_DIR)/$(KIND_CLUSTER_NAME)/kubeconfig KINDEST_NODE_IMAGE ?= ghcr.io/mesosphere/kind-node -KINDEST_NODE_VERSION_v1.27 ?= v1.27.13 -KINDEST_NODE_VERSION_v1.28 ?= v1.28.9 -KINDEST_NODE_VERSION_v1.29 ?= v1.29.4 -KINDEST_NODE_VERSION_v1.30 ?= v1.30.0 +KINDEST_NODE_VERSION_v1.27 ?= v1.27.15 +KINDEST_NODE_VERSION_v1.28 ?= v1.28.11 +KINDEST_NODE_VERSION_v1.29 ?= v1.29.6 +KINDEST_NODE_VERSION_v1.30 ?= v1.30.2 # Allow easy override of Kubernetes version to use via `make KIND_KUBERNETES_VERSION=v1.23` to use in CI KIND_KUBERNETES_VERSION ?= v1.29 ifndef KINDEST_NODE_VERSION_$(KIND_KUBERNETES_VERSION) diff --git a/test/e2e/config/caren.yaml b/test/e2e/config/caren.yaml index b08579d14..4f108cd20 100644 --- a/test/e2e/config/caren.yaml +++ b/test/e2e/config/caren.yaml @@ -167,7 +167,7 @@ variables: KUBERNETES_VERSION: "${E2E_DEFAULT_KUBERNETES_VERSION}" # Override Kubernetes version for test workload clusters for specific providers by setting the env variables # `KUBERNETES_VERSION_`, where `` is the uppercase provider name, e.g. - # `KUBERNETES_VERSION_DOCKER: v1.29.4`. + # `KUBERNETES_VERSION_DOCKER: v1.29.5`. # KUBERNETES_VERSION_DOCKER: "${KINDEST_IMAGE_TAG}" SERVICE_CIDR: "10.128.0.0/12" POD_CIDR: "192.168.0.0/16" diff --git a/test/e2e/csi_helpers.go b/test/e2e/csi_helpers.go index 432bb8a6c..1b7974497 100644 --- a/test/e2e/csi_helpers.go +++ b/test/e2e/csi_helpers.go @@ -318,7 +318,7 @@ func waitForNutanixCSIToBeReadyInWorkloadCluster( Deployment: &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "nutanix-csi-controller", - Namespace: metav1.NamespaceSystem, + Namespace: "ntnx-system", }, }, }, input.deploymentIntervals...) @@ -328,7 +328,7 @@ func waitForNutanixCSIToBeReadyInWorkloadCluster( DaemonSet: &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: "nutanix-csi-node", - Namespace: metav1.NamespaceSystem, + Namespace: "ntnx-system", }, }, }, input.daemonSetIntervals...) @@ -357,6 +357,8 @@ func waitForStorageClassesToExistInWorkloadCluster( provisioner = string(v1alpha1.LocalPathProvisioner) case v1alpha1.CSIProviderAWSEBS: provisioner = string(v1alpha1.AWSEBSProvisioner) + case v1alpha1.CSIProviderNutanix: + provisioner = string(v1alpha1.NutanixProvisioner) default: Fail( fmt.Sprintf( diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index a4e6518a7..f74486d40 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -70,7 +70,7 @@ func init() { flag.StringVar( &bootstrapKubernetesVersion, "e2e.bootstrap-kind-version", - "v1.29.4", + "v1.29.5", "the version of the image used in bootstrap cluster", ) } diff --git a/test/e2e/ownerreference_helpers.go b/test/e2e/ownerreference_helpers.go index 852c223ea..c66db0b6b 100644 --- a/test/e2e/ownerreference_helpers.go +++ b/test/e2e/ownerreference_helpers.go @@ -16,6 +16,7 @@ import ( addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework" + capxv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/github.com/nutanix-cloud-native/cluster-api-provider-nutanix/api/v1beta1" caaphv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" ) @@ -91,6 +92,12 @@ var ( Controller: ptr.To(true), } + capxGroupVersion = capxv1.GroupVersion.String() + nutanixClusterOwner = metav1.OwnerReference{ + Kind: nutanixClusterKind, + APIVersion: capxGroupVersion, + } + // AddonReferenceAssertions maps addontypes to functions which return an error if the passed OwnerReferences // aren't as expected. AddonReferenceAssertions = map[string]func([]metav1.OwnerReference) error{ @@ -191,6 +198,7 @@ var ( []metav1.OwnerReference{kubeadmControlPlaneController}, []metav1.OwnerReference{kubeadmConfigController}, []metav1.OwnerReference{clusterOwner}, + []metav1.OwnerReference{clusterOwner, nutanixClusterOwner}, ) }, configMapKind: func(owners []metav1.OwnerReference) error { diff --git a/test/e2e/quick_start_test.go b/test/e2e/quick_start_test.go index e19a34264..193c4fe23 100644 --- a/test/e2e/quick_start_test.go +++ b/test/e2e/quick_start_test.go @@ -8,11 +8,14 @@ package e2e import ( "fmt" "slices" + "strconv" "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" "k8s.io/utils/ptr" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" clusterctlcluster "sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster" capie2e "sigs.k8s.io/cluster-api/test/e2e" "sigs.k8s.io/cluster-api/test/framework" @@ -63,7 +66,7 @@ var _ = Describe("Quick start", Serial, func() { // available that are not available in other providers. // This version can be specified in `test/e2e/config/caren.yaml` with a variable named // `KUBERNETES_VERSION_`, where `` is the uppercase provider name, e.g. - // `KUBERNETES_VERSION_DOCKER: v1.29.4`. + // `KUBERNETES_VERSION_DOCKER: v1.29.5`. testE2EConfig := e2eConfig.DeepCopy() varName := capie2e.KubernetesVersion + "_" + strings.ToUpper( lowercaseProvider, @@ -99,6 +102,16 @@ var _ = Describe("Quick start", Serial, func() { KubernetesReferenceAssertions, ) + workloadCluster := framework.GetClusterByName( + ctx, + framework.GetClusterByNameInput{ + Namespace: namespace, + Name: clusterName, + Getter: proxy.GetClient(), + }, + ) + Expect(workloadCluster.Spec.Topology).ToNot(BeNil()) + By("Waiting until nodes are ready") workloadProxy := proxy.GetWorkloadCluster( ctx, @@ -106,6 +119,32 @@ var _ = Describe("Quick start", Serial, func() { clusterName, ) workloadClient := workloadProxy.GetClient() + + nodeCount := int( + ptr.Deref( + workloadCluster.Spec.Topology.ControlPlane.Replicas, + 0, + ), + ) + + lo.Reduce( + workloadCluster.Spec.Topology.Workers.MachineDeployments, + func(agg int, md clusterv1.MachineDeploymentTopology, _ int) int { + switch { + case md.Replicas != nil: + return agg + int(ptr.Deref(md.Replicas, 0)) + case md.Metadata.Annotations["cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"] != "": + minSize, err := strconv.Atoi( + md.Metadata.Annotations["cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"], + ) + Expect(err).ToNot(HaveOccurred()) + return agg + minSize + default: + return agg + } + }, + 0, + ) + framework.WaitForNodesReady( ctx, framework.WaitForNodesReadyInput{ @@ -113,7 +152,7 @@ var _ = Describe("Quick start", Serial, func() { KubernetesVersion: testE2EConfig.GetVariable( capie2e.KubernetesVersion, ), - Count: 2, + Count: nodeCount, WaitForNodesReady: testE2EConfig.GetIntervals( flavour, "wait-nodes-ready", @@ -124,15 +163,6 @@ var _ = Describe("Quick start", Serial, func() { By( "Waiting for all requested addons to be ready in workload cluster", ) - workloadCluster := framework.GetClusterByName( - ctx, - framework.GetClusterByNameInput{ - Namespace: namespace, - Name: clusterName, - Getter: proxy.GetClient(), - }, - ) - Expect(workloadCluster.Spec.Topology).ToNot(BeNil()) clusterVars := variables.ClusterVariablesToVariablesMap( workloadCluster.Spec.Topology.Variables, )