From 8118911e6f07ce0e579f98fa79c9feb1f57f8ac3 Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Mon, 26 Feb 2024 13:26:54 +0000 Subject: [PATCH] test(e2e): Use ghcr.io/mesosphere/kind-node for bootstrap This allows configuration of k8s versions, etc via flags. --- test/e2e/e2e_suite_test.go | 35 ++++++--- test/e2e/e2e_suite_vars.go | 6 ++ test/framework/bootstrap/kindprovider.go | 92 ++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 9 deletions(-) create mode 100644 test/framework/bootstrap/kindprovider.go diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 915f69e77..1db275f5b 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -23,10 +23,11 @@ import ( addonsv1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" capi_e2e "sigs.k8s.io/cluster-api/test/e2e" "sigs.k8s.io/cluster-api/test/framework" - "sigs.k8s.io/cluster-api/test/framework/bootstrap" + capibootstrap "sigs.k8s.io/cluster-api/test/framework/bootstrap" "sigs.k8s.io/cluster-api/test/framework/clusterctl" ctrl "sigs.k8s.io/controller-runtime" + "github.com/d2iq-labs/capi-runtime-extensions/test/framework/bootstrap" clusterctltemp "github.com/d2iq-labs/capi-runtime-extensions/test/framework/clusterctl" ) @@ -56,6 +57,18 @@ func init() { //nolint:gochecknoinits // Idiomatically used to set up flags. false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)", ) + flag.StringVar( + &bootstrapNodeImageRepository, + "e2e.bootstrap-kind-image", + "ghcr.io/mesosphere/kind-node", + "the image to use for the bootstrap cluster", + ) + flag.StringVar( + &bootstrapKubernetesVersion, + "e2e.bootstrap-kind-version", + "v1.29.2", + "the version of the image used in bootstrap cluster", + ) } func TestE2E(t *testing.T) { @@ -189,16 +202,20 @@ func setupBootstrapCluster( config *clusterctl.E2EConfig, scheme *runtime.Scheme, useExistingCluster bool, -) (bootstrap.ClusterProvider, framework.ClusterProxy) { - var clusterProvider bootstrap.ClusterProvider +) (capibootstrap.ClusterProvider, framework.ClusterProxy) { + var clusterProvider capibootstrap.ClusterProvider kubeconfigPath := "" if !useExistingCluster { clusterProvider = bootstrap.CreateKindBootstrapClusterAndLoadImages( context.TODO(), bootstrap.CreateKindBootstrapClusterAndLoadImagesInput{ - Name: config.ManagementClusterName, - RequiresDockerSock: config.HasDockerProvider(), - Images: config.Images, + NodeImageRepository: bootstrapNodeImageRepository, + CreateKindBootstrapClusterAndLoadImagesInput: capibootstrap.CreateKindBootstrapClusterAndLoadImagesInput{ + Name: config.ManagementClusterName, + RequiresDockerSock: config.HasDockerProvider(), + Images: config.Images, + KubernetesVersion: bootstrapKubernetesVersion, + }, }, ) Expect(clusterProvider).NotTo(BeNil(), "Failed to create a bootstrap cluster") @@ -209,11 +226,11 @@ func setupBootstrapCluster( ).To(BeAnExistingFile(), "Failed to get the kubeconfig file for the bootstrap cluster") } else { // Loading image for already created cluster - imagesInput := bootstrap.LoadImagesToKindClusterInput{ + imagesInput := capibootstrap.LoadImagesToKindClusterInput{ Name: "cre-e2e", Images: config.Images, } - err := bootstrap.LoadImagesToKindCluster(context.TODO(), imagesInput) + err := capibootstrap.LoadImagesToKindCluster(context.TODO(), imagesInput) Expect(err).To(BeNil(), "Failed to load images to the bootstrap cluster: %s", err) } @@ -245,7 +262,7 @@ func initBootstrapCluster( } func tearDown( - bootstrapClusterProvider bootstrap.ClusterProvider, + bootstrapClusterProvider capibootstrap.ClusterProvider, bootstrapClusterProxy framework.ClusterProxy, ) { if bootstrapClusterProxy != nil { diff --git a/test/e2e/e2e_suite_vars.go b/test/e2e/e2e_suite_vars.go index 51d270370..69976735b 100644 --- a/test/e2e/e2e_suite_vars.go +++ b/test/e2e/e2e_suite_vars.go @@ -36,6 +36,12 @@ var ( // skipLogCollection prevents the log collection process from running. skipLogCollection bool + + // bootstrapNodeImageRepository is the image to use for the bootstrap cluster. + bootstrapNodeImageRepository string + + // bootstrapKubernetesVersion is the version of the image used in bootstrap cluster. + bootstrapKubernetesVersion string ) // Test suite global vars. diff --git a/test/framework/bootstrap/kindprovider.go b/test/framework/bootstrap/kindprovider.go new file mode 100644 index 000000000..96f0f3507 --- /dev/null +++ b/test/framework/bootstrap/kindprovider.go @@ -0,0 +1,92 @@ +//go:build e2e + +// Copyright 2024 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package bootstrap + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/cluster-api/test/framework/bootstrap" +) + +// CreateKindBootstrapClusterAndLoadImagesInput is the input for CreateKindBootstrapClusterAndLoadImages. +type CreateKindBootstrapClusterAndLoadImagesInput struct { + // NodeImageRepository defines the image repository to use for the bootstrap cluster nodes. + NodeImageRepository string + + bootstrap.CreateKindBootstrapClusterAndLoadImagesInput +} + +// CreateKindBootstrapClusterAndLoadImages returns a new Kubernetes cluster with pre-loaded images. +func CreateKindBootstrapClusterAndLoadImages( + ctx context.Context, + input CreateKindBootstrapClusterAndLoadImagesInput, //nolint:gocritic // Copied from upstream. +) bootstrap.ClusterProvider { + Expect(ctx).NotTo(BeNil(), "ctx is required for CreateKindBootstrapClusterAndLoadImages") + Expect( + input.Name, + ).ToNot(BeEmpty(), "Invalid argument. Name can't be empty when calling CreateKindBootstrapClusterAndLoadImages") + + fmt.Fprintf(GinkgoWriter, "INFO: Creating a kind cluster with name %q\n", input.Name) + + options := []bootstrap.KindClusterOption{} + nodeImageRepository := bootstrap.DefaultNodeImageRepository + if input.NodeImageRepository != "" { + nodeImageRepository = input.NodeImageRepository + } + + if input.KubernetesVersion != "" { + options = append( + options, + bootstrap.WithNodeImage( + fmt.Sprintf("%s:%s", nodeImageRepository, input.KubernetesVersion), + ), + ) + } + if input.RequiresDockerSock { + options = append(options, bootstrap.WithDockerSockMount()) + } + if input.IPFamily == "IPv6" { + options = append(options, bootstrap.WithIPv6Family()) + } + if input.IPFamily == "dual" { + options = append(options, bootstrap.WithDualStackFamily()) + } + if input.LogFolder != "" { + options = append(options, bootstrap.LogFolder(input.LogFolder)) + } + clusterProvider := bootstrap.NewKindClusterProvider(input.Name, options...) + Expect(clusterProvider).ToNot(BeNil(), "Failed to create a kind cluster") + + clusterProvider.Create(ctx) + Expect( + clusterProvider.GetKubeconfigPath(), + ).To( + BeAnExistingFile(), + "The kubeconfig file for the kind cluster with name %q does not exists at %q as expected", + input.Name, + clusterProvider.GetKubeconfigPath(), + ) + + fmt.Fprintf( + GinkgoWriter, + "INFO: The kubeconfig file for the kind cluster is %s\n", + clusterProvider.GetKubeconfigPath(), + ) + + err := bootstrap.LoadImagesToKindCluster(ctx, bootstrap.LoadImagesToKindClusterInput{ + Name: input.Name, + Images: input.Images, + }) + if err != nil { + clusterProvider.Dispose(ctx) + Expect(err).ToNot(HaveOccurred()) // re-surface the error to fail the test + } + + return clusterProvider +}