Add RKE2 boostrap provider to e2e

charts/rancher-turtles/values.yaml

@@ -3,7 +3,7 @@ rancherTurtles:
   tag: v0.0.0
   imagePullPolicy: Never
   namespace: rancher-turtles-system
-  managerArguments: {}
+  managerArguments: []
   imagePullSecrets: []
 cluster-api-operator:
   enabled: true

test/e2e/config/operator.yaml

@@ -17,6 +17,7 @@ intervals:
   default/wait-eks-delete: ["20m", "30s"]
   default/wait-aks-delete: ["20m", "30s"]
   default/wait-azure: ["30m", "30s"]
+  default/wait-rke2-docker: ["15m", "30s"]
 
 variables:
   RANCHER_VERSION: "v2.7.6"
@@ -40,4 +41,4 @@ variables:
   GITEA_CHART_NAME: "gitea"
   GITEA_CHART_VERSION: "9.4.0"
   GITEA_USER_NAME: "gitea_admin"
-  GITEA_USER_PWD: "password"
+  GITEA_USER_PWD: "password"

test/e2e/data/capi-operator/full-providers.yaml

@@ -9,6 +9,16 @@ kind: Namespace
 metadata:
   name: capz-system
 ---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: rke2-bootstrap-system
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: rke2-control-plane-system
+---
 apiVersion: operator.cluster.x-k8s.io/v1alpha1
 kind: InfrastructureProvider
 metadata:
@@ -25,4 +35,24 @@ metadata:
   namespace: capz-system
 spec:
   secretName: azure-variables
-  secretNamespace: default
+  secretNamespace: default
+---
+apiVersion: operator.cluster.x-k8s.io/v1alpha1
+kind: BootstrapProvider
+metadata:
+  name: bootstrap-caprke2
+  namespace: rke2-bootstrap-system
+spec:
+  fetchConfig:
+    url: https://github.com/rancher-sandbox/cluster-api-provider-rke2/releases/v0.1.1/bootstrap-components.yaml
+  version: v0.1.1
+---
+apiVersion: operator.cluster.x-k8s.io/v1alpha1
+kind: ControlPlaneProvider
+metadata:
+  name: control-plane-caprke2
+  namespace: rke2-control-plane-system
+spec:
+  fetchConfig:
+    url: https://github.com/rancher-sandbox/cluster-api-provider-rke2/releases/v0.1.1/control-plane-components.yaml
+  version: v0.1.1

test/e2e/suites/import-gitops/cluster-templates/docker-rke2.yaml

@@ -0,0 +1,122 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerCluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec:
+  loadBalancer:
+    customHAProxyConfigTemplateRef:
+      name: rke2-ha-proxy-config
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster 
+metadata:
+  name: ${CLUSTER_NAME} 
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+      - 10.45.0.0/16
+    services:
+      cidrBlocks:
+      - 10.46.0.0/16
+    serviceDomain: cluster.local
+  controlPlaneRef:
+    apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+    kind: RKE2ControlPlane
+    name: ${CLUSTER_NAME}-control-plane
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: DockerCluster
+    name: ${CLUSTER_NAME}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  template:
+    spec: 
+      customImage: ghcr.io/rancher-sandbox/ubuntu-22.04-rke2:${KUBERNETES_VERSION}
+      extraMounts:
+      - containerPath: /var/run/docker.sock
+        hostPath: /var/run/docker.sock
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+kind: RKE2ControlPlane
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec: 
+  agentConfig:
+    version: ${KUBERNETES_VERSION}+rke2r1
+  serverConfig:
+    cni: calico
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: DockerMachineTemplate
+    name: ${CLUSTER_NAME}-control-plane
+  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+  nodeDrainTimeout: 2m
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec: 
+      customImage: ghcr.io/rancher-sandbox/ubuntu-22.04-rke2:${KUBERNETES_VERSION}
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1
+kind: RKE2ConfigTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec: 
+  template:
+    spec:
+      agentConfig:
+        version: ${KUBERNETES_VERSION}+rke2r1
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: worker-md-0
+spec:
+  clusterName: ${CLUSTER_NAME}
+  replicas: ${WORKER_MACHINE_COUNT}
+  selector:
+    matchLabels:
+      cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+  template:
+    spec:
+      version: ${KUBERNETES_VERSION}
+      clusterName: ${CLUSTER_NAME}
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1
+          kind: RKE2ConfigTemplate
+          name: ${CLUSTER_NAME}-md-0
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: DockerMachineTemplate
+        name: ${CLUSTER_NAME}-md-0
+---
+apiVersion: v1
+data:
+  value: |
+    # addtional config
+    frontend additional-join
+      bind *:9345
+      {{ if .IPv6 -}}
+      bind :::9345;
+      {{- end }}
+      default_backend additional-servers
+    backend additional-servers
+      option httpchk GET /v1-additional/readyz
+      http-check expect status 403
+      {{range $server, $address := .BackendServers}}
+      server {{ $server }} {{ JoinHostPort $address "9345" }} check check-ssl verify none resolvers docker resolve-prefer {{ if $.IPv6 -}} ipv6 {{- else -}} ipv4 {{- end }}
+      {{- end}}
+kind: ConfigMap
+metadata:
+  name: rke2-ha-proxy-config
+

test/e2e/suites/import-gitops/import_gitops.go

@@ -65,6 +65,9 @@ type CreateUsingGitOpsSpecInput struct {
 	// If not specified, 1 will be used.
 	WorkerMachineCount *int
 
+	// OverrideKubernetesVersion if specified will override the Kubernetes version used in the cluster template.
+	OverrideKubernetesVersion string
+
 	GitAddr           string
 	GitAuthSecretName string
 
@@ -160,16 +163,22 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi
 		clustersDir := filepath.Join(repoDir, "clusters")
 		os.MkdirAll(clustersDir, os.ModePerm)
 
+		additionalVariables := map[string]string{
+			"CLUSTER_NAME":                input.ClusterName,
+			"WORKER_MACHINE_COUNT":        strconv.Itoa(workerMachineCount),
+			"CONTROL_PLANE_MACHINE_COUNT": strconv.Itoa(controlPlaneMachineCount),
+		}
+
+		if input.OverrideKubernetesVersion != "" {
+			additionalVariables["KUBERNETES_VERSION"] = input.OverrideKubernetesVersion
+		}
+
 		clusterPath := filepath.Join(clustersDir, fmt.Sprintf("%s.yaml", input.ClusterName))
 		Expect(turtlesframework.ApplyFromTemplate(ctx, turtlesframework.ApplyFromTemplateInput{
-			Getter:         input.E2EConfig.GetVariable,
-			Template:       input.ClusterTemplate,
-			OutputFilePath: clusterPath,
-			AddtionalEnvironmentVariables: map[string]string{
-				"CLUSTER_NAME":                input.ClusterName,
-				"WORKER_MACHINE_COUNT":        strconv.Itoa(workerMachineCount),
-				"CONTROL_PLANE_MACHINE_COUNT": strconv.Itoa(controlPlaneMachineCount),
-			},
+			Getter:                        input.E2EConfig.GetVariable,
+			Template:                      input.ClusterTemplate,
+			OutputFilePath:                clusterPath,
+			AddtionalEnvironmentVariables: additionalVariables,
 		})).To(Succeed())
 
 		fleetPath := filepath.Join(clustersDir, "fleet.yaml")

test/e2e/suites/import-gitops/import_gitops_test.go

@@ -39,6 +39,9 @@ var (
 
 	//go:embed cluster-templates/azure-aks-mmp.yaml
 	azureAKSMMP []byte
+
+	//go:embed cluster-templates/docker-rke2.yaml
+	rke2Docker []byte
 )
 
 var _ = Describe("[Docker] [Kubeadm] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.ShortTestLabel, e2e.FullTestLabel), func() {
@@ -129,3 +132,33 @@ var _ = Describe("[Azure] [AKS] Create and delete CAPI cluster functionality sho
 		}
 	})
 })
+
+var _ = FDescribe("[Docker] [RKE2] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.FullTestLabel), func() {
+
+	BeforeEach(func() {
+		SetClient(setupClusterResult.BootstrapClusterProxy.GetClient())
+		SetContext(ctx)
+	})
+
+	CreateUsingGitOpsSpec(ctx, func() CreateUsingGitOpsSpecInput {
+		return CreateUsingGitOpsSpecInput{
+			E2EConfig:                 e2eConfig,
+			BootstrapClusterProxy:     setupClusterResult.BootstrapClusterProxy,
+			ClusterctlConfigPath:      flagVals.ConfigPath,
+			ArtifactFolder:            flagVals.ArtifactFolder,
+			ClusterTemplate:           rke2Docker,
+			ClusterName:               "cluster-rke2-docker",
+			OverrideKubernetesVersion: "v1.28.2",
+			ControlPlaneMachineCount:  ptr.To[int](1),
+			WorkerMachineCount:        ptr.To[int](1),
+			GitAddr:                   giteaResult.GitAddress,
+			GitAuthSecretName:         e2e.AuthSecretName,
+			SkipCleanup:               false,
+			SkipDeletionTest:          false,
+			LabelNamespace:            true,
+			RancherServerURL:          hostName,
+			CAPIClusterCreateWaitName: "wait-rke2-docker",
+			DeleteClusterWaitName:     "wait-rke2-docker",
+		}
+	})
+})

test/e2e/suites/import-gitops/suite_test.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -104,15 +105,20 @@ var _ = BeforeSuite(func() {
 		hostName = setupClusterResult.IsolatedHostName
 	}
 
+	Expect(e2eConfig.Images).To(HaveLen(1))
+	imageNameTag := strings.Split(e2eConfig.Images[0].Name, ":")
+	Expect(imageNameTag).To(HaveLen(2))
+
 	testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{
 		BootstrapClusterProxy:        setupClusterResult.BootstrapClusterProxy,
+		UseExistingCluster:           flagVals.UseExistingCluster,
 		HelmBinaryPath:               flagVals.HelmBinaryPath,
 		ChartPath:                    flagVals.ChartPath,
 		CAPIProvidersSecretYAML:      e2e.CapiProvidersSecret,
 		CAPIProvidersYAML:            e2e.CapiProviders,
 		Namespace:                    turtlesframework.DefaultRancherTurtlesNamespace,
-		Image:                        "ghcr.io/rancher-sandbox/rancher-turtles-amd64",
-		Tag:                          "v0.0.1",
+		Image:                        imageNameTag[0],
+		Tag:                          imageNameTag[1],
 		WaitDeploymentsReadyInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
 	})
 
@@ -142,11 +148,20 @@ var _ = BeforeSuite(func() {
 					Name:      "capz-controller-manager",
 					Namespace: "capz-system",
 				},
+				{
+					Name:      "rke2-bootstrap-controller-manager",
+					Namespace: "rke2-bootstrap-system",
+				},
+				{
+					Name:      "rke2-control-plane-controller-manager",
+					Namespace: "rke2-control-plane-system",
+				},
 			},
 		})
 	}
 
 	testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{
+		UseExistingCluster:       flagVals.UseExistingCluster,
 		BootstrapClusterProxy:    setupClusterResult.BootstrapClusterProxy,
 		HelmBinaryPath:           flagVals.HelmBinaryPath,
 		IsolatedMode:             flagVals.IsolatedMode,
@@ -162,6 +177,7 @@ var _ = BeforeSuite(func() {
 	})
 
 	testenv.DeployRancher(ctx, testenv.DeployRancherInput{
+		UseExistingCluster:     flagVals.UseExistingCluster,
 		BootstrapClusterProxy:  setupClusterResult.BootstrapClusterProxy,
 		HelmBinaryPath:         flagVals.HelmBinaryPath,
 		RancherChartRepoName:   e2eConfig.GetVariable(e2e.RancherRepoNameVar),
@@ -181,6 +197,7 @@ var _ = BeforeSuite(func() {
 	})
 
 	giteaResult = testenv.DeployGitea(ctx, testenv.DeployGiteaInput{
+		UseExistingCluster:    flagVals.UseExistingCluster,
 		BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
 		HelmBinaryPath:        flagVals.HelmBinaryPath,
 		ChartRepoName:         e2eConfig.GetVariable(e2e.GiteaRepoNameVar),

test/testenv/gitea.go

@@ -46,6 +46,7 @@ type DeployGiteaInput struct {
 	Username              string
 	Password              string
 	AuthSecretName        string
+	UseExistingCluster    bool
 }
 
 type DeployGiteaResult struct {
@@ -63,6 +64,24 @@ func DeployGitea(ctx context.Context, input DeployGiteaInput) *DeployGiteaResult
 	Expect(input.RolloutWaitInterval).ToNot(BeNil(), "RolloutWaitInterval is required for DeployGitea")
 	Expect(input.ServiceWaitInterval).ToNot(BeNil(), "ServiceWaitInterval is required for DeployGitea")
 
+	if input.UseExistingCluster {
+		addr := turtlesframework.GetNodeAddress(ctx, turtlesframework.GetNodeAddressInput{
+			Lister:       input.BootstrapClusterProxy.GetClient(),
+			NodeIndex:    0,
+			AddressIndex: 0,
+		})
+		port := turtlesframework.GetServicePortByName(ctx, turtlesframework.GetServicePortByNameInput{
+			GetLister:        input.BootstrapClusterProxy.GetClient(),
+			ServiceName:      "gitea-http",
+			ServiceNamespace: "default",
+			PortName:         "http",
+		}, input.ServiceWaitInterval...)
+		Expect(port.NodePort).ToNot(Equal(0), "Node port for Gitea service is not set")
+		return &DeployGiteaResult{
+			GitAddress: fmt.Sprintf("http://%s:%d", addr, port.NodePort),
+		}
+	}
+
 	if input.Username != "" {
 		Expect(input.Password).ToNot(BeEmpty(), "Password is required for DeployGitea if a username is supplied")
 		Expect(input.AuthSecretName).ToNot(BeEmpty(), "AuthSecretName is required for DeployGitea if a username is supplied")