test: Add new e2e test suite for v2prov

.github/workflows/e2e-long.yaml

@@ -13,6 +13,9 @@ env:
   RANCHER_HOSTNAME: ${{ secrets.NGROK_DOMAIN }}
   RANCHER_PASSWORD: ${{ secrets.RANCHER_PASSWORD }}
   CAPA_ENCODED_CREDS: ${{ secrets.CAPA_ENCODED_CREDS }}
+  AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+  AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+  AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
 
 jobs:
   e2e:

Makefile

@@ -485,7 +485,7 @@ e2e-image: ## Build the image for e2e tests
 
 .PHONY: compile-e2e
 e2e-compile: ## Test e2e compilation
-	go test -c -o /dev/null -tags=e2e ./test/e2e/suites/import-gitops
+	go test -c -o /dev/null -tags=e2e ./test/e2e/suites/***
 
 ## --------------------------------------
 ## Documentation

test/e2e/config/operator.yaml

@@ -8,18 +8,21 @@ images:
 intervals:
   default/wait-controllers: ["3m", "10s"]
   default/wait-rancher: ["15m", "30s"]
+  default/wait-v2prov-create: ["25m", "30s"]
   default/wait-capa-create-cluster: ["30m", "30s"]
   default/wait-gitea: ["3m", "10s"]
   default/wait-consistently: ["30s", "5s"]
   default/wait-getservice: ["60s", "5s"]
   default/wait-eks-delete: ["20m", "30s"]
+  default/wait-azure-delete: ["20m", "30s"]
 
 variables:
   RANCHER_VERSION: "v2.7.6"
   RANCHER_HOSTNAME: "localhost"
   RANCHER_FEATURES: "embedded-cluster-api=false"
   RANCHER_PATH: "rancher-stable/rancher"
   KUBERNETES_VERSION: "v1.26.3"
+  RKE2_VERSION: "v1.26.8+rke2r1"
   CAPI_INFRASTRUCTURE: "docker:v1.4.6"
   CAPI_CORE: "cluster-api:v1.4.6"
   RANCHER_REPO_NAME: "rancher-stable"

test/e2e/const.go

@@ -50,6 +50,12 @@ var (
 
 	//go:embed data/rancher/nginx-ingress.yaml
 	NginxIngress []byte
+
+	//go:embed data/rancher/azure-rke-config.yaml
+	V2ProvAzureRkeConfig []byte
+
+	//go:embed data/rancher/azure-cluster.yaml
+	V2ProvAzureCluster []byte
 )
 
 const (
@@ -83,7 +89,12 @@ const (
 	GiteaUserNameVar     = "GITEA_USER_NAME"
 	GiteaUserPasswordVar = "GITEA_USER_PWD"
 
+	RKE2VersionVar = "RKE2_VERSION"
+
 	CapaEncodedCredentialsVar = "CAPA_ENCODED_CREDS"
+	AzureSubIDVar             = "AZURE_SUBSCRIPTION_ID"
+	AzureClientIDVar          = "AZURE_CLIENT_ID"
+	AzureClientSecretVar      = "AZURE_CLIENT_SECRET"
 
 	AuthSecretName = "basic-auth-secret"
 

test/e2e/data/rancher/azure-cluster.yaml

@@ -0,0 +1,101 @@
+apiVersion: provisioning.cattle.io/v1
+kind: Cluster
+metadata:
+  annotations:
+    field.cattle.io/creatorId: ${USER}
+  name: ${CLUSTER_NAME}
+  namespace: fleet-default
+spec:
+  cloudCredentialSecretName: ${CREDENTIAL_SECRET}
+  kubernetesVersion: ${KUBERNETES_VERSION}
+  localClusterAuthEndpoint: {}
+  rkeConfig:
+    chartValues:
+      rke2-calico: {}
+    etcd:
+      snapshotRetention: 5
+      snapshotScheduleCron: 0 */5 * * *
+    machineGlobalConfig:
+      cni: calico
+      disable-kube-proxy: false
+      etcd-expose-metrics: false
+    machinePoolDefaults: {}
+    machinePools:
+    - controlPlaneRole: true
+      dynamicSchemaSpec: '{"resourceFields":{"acceleratedNetworking":{"type":"boolean","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Specify
+        if an Accelerated Networking NIC should be created for your VM"},"availabilitySet":{"type":"string","default":{"stringValue":"docker-machine","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Azure
+        Availability Set to place the virtual machine into"},"availabilityZone":{"type":"string","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Specify
+        the Availability Zones the Azure resources should be created in"},"clientId":{"type":"string","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Azure
+        Service Principal Account ID (optional, browser auth is used if not specified)"},"clientSecret":{"type":"password","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Azure
+        Service Principal Account password (optional, browser auth is used if not
+        specified)"},"customData":{"type":"string","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"File
+        contents for customData"},"diskSize":{"type":"string","default":{"stringValue":"30","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Disk
+        size if using managed disk"},"dns":{"type":"string","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"A
+        unique DNS label for the public IP adddress"},"dockerPort":{"type":"string","default":{"stringValue":"2376","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Port
+        number for Docker engine"},"enablePublicIpStandardSku":{"type":"boolean","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Specify
+        if a Standard SKU should be used for the Public IP of the Azure VM"},"environment":{"type":"string","default":{"stringValue":"AzurePublicCloud","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Azure
+        environment (e.g. AzurePublicCloud, AzureChinaCloud)"},"faultDomainCount":{"type":"string","default":{"stringValue":"3","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Fault
+        domain count to use for availability set"},"image":{"type":"string","default":{"stringValue":"canonical:UbuntuServer:18.04-LTS:latest","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Azure
+        virtual machine OS image"},"location":{"type":"string","default":{"stringValue":"westus","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Azure
+        region to create the virtual machine"},"managedDisks":{"type":"boolean","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Configures
+        VM and availability set for managed disks"},"noPublicIp":{"type":"boolean","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Do
+        not create a public IP address for the machine"},"nsg":{"type":"string","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Azure
+        Network Security Group to assign this node to (accepts either a name or resource
+        ID, default is to create a new NSG for each machine)"},"openPort":{"type":"array[string]","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"nullable":true,"create":true,"update":true,"description":"Make
+        the specified port number accessible from the Internet"},"plan":{"type":"string","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Purchase
+        plan for Azure Virtual Machine (in \u003cpublisher\u003e:\u003cproduct\u003e:\u003cplan\u003e
+        format)"},"privateIpAddress":{"type":"string","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Specify
+        a static private IP address for the machine"},"resourceGroup":{"type":"string","default":{"stringValue":"docker-machine","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Azure
+        Resource Group name (will be created if missing)"},"size":{"type":"string","default":{"stringValue":"Standard_D2_v2","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Size
+        for Azure Virtual Machine"},"sshUser":{"type":"string","default":{"stringValue":"docker-user","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Username
+        for SSH login"},"staticPublicIp":{"type":"boolean","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Assign
+        a static public IP address to the machine"},"storageType":{"type":"string","default":{"stringValue":"Standard_LRS","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Type
+        of Storage Account to host the OS Disk for the machine"},"subnet":{"type":"string","default":{"stringValue":"docker-machine","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Azure
+        Subnet Name to be used within the Virtual Network"},"subnetPrefix":{"type":"string","default":{"stringValue":"192.168.0.0/16","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Private
+        CIDR block to be used for the new subnet, should comply RFC 1918"},"subscriptionId":{"type":"string","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Azure
+        Subscription ID"},"tags":{"type":"string","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Tags
+        to be applied to the Azure VM instance"},"tenantId":{"type":"string","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Azure
+        Tenant ID"},"updateDomainCount":{"type":"string","default":{"stringValue":"5","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Update
+        domain count to use for availability set"},"usePrivateIp":{"type":"boolean","default":{"stringValue":"","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Use
+        private IP address of the machine to connect"},"vnet":{"type":"string","default":{"stringValue":"docker-machine-vnet","intValue":0,"boolValue":false,"stringSliceValue":null},"create":true,"update":true,"description":"Azure
+        Virtual Network name to connect the virtual machine (in [resourcegroup:]name
+        format)"}}}'
+      etcdRole: true
+      machineConfigRef:
+        kind: AzureConfig
+        name: ${AZ_CONFIG_NAME}
+      name: pool1
+      quantity: 1
+      unhealthyNodeTimeout: 0s
+      workerRole: true
+    machineSelectorConfig:
+    - config:
+        protect-kernel-defaults: false
+    registries: {}
+    upgradeStrategy:
+      controlPlaneConcurrency: "1"
+      controlPlaneDrainOptions:
+        deleteEmptyDirData: true
+        disableEviction: false
+        enabled: false
+        force: false
+        gracePeriod: -1
+        ignoreDaemonSets: true
+        ignoreErrors: false
+        postDrainHooks: null
+        preDrainHooks: null
+        skipWaitForDeleteTimeoutSeconds: 0
+        timeout: 120
+      workerConcurrency: "1"
+      workerDrainOptions:
+        deleteEmptyDirData: true
+        disableEviction: false
+        enabled: false
+        force: false
+        gracePeriod: -1
+        ignoreDaemonSets: true
+        ignoreErrors: false
+        postDrainHooks: null
+        preDrainHooks: null
+        skipWaitForDeleteTimeoutSeconds: 0
+        timeout: 120

test/e2e/data/rancher/azure-rke-config.yaml

@@ -0,0 +1,42 @@
+apiVersion: rke-machine-config.cattle.io/v1
+kind: AzureConfig
+metadata:
+  annotations:
+    field.cattle.io/creatorId: ${USER}
+  name: ${POOL_NAME}
+  namespace: fleet-default
+acceleratedNetworking: false
+availabilitySet: docker-machine
+availabilityZone: ""
+diskSize: "30"
+dockerPort: "2376"
+enablePublicIpStandardSku: false
+environment: AzurePublicCloud
+faultDomainCount: "3"
+image: canonical:UbuntuServer:18.04-LTS:latest
+location: westus
+managedDisks: false
+noPublicIp: false
+nsg: rancher-managed-wpepXjvf
+openPort:
+- 6443/tcp
+- 2379/tcp
+- 2380/tcp
+- 8472/udp
+- 4789/udp
+- 9796/tcp
+- 10256/tcp
+- 10250/tcp
+- 10251/tcp
+- 10252/tcp
+plan: ""
+resourceGroup: docker-machine
+size: Standard_D2_v2
+sshUser: docker-user
+staticPublicIp: false
+storageType: Standard_LRS
+subnet: docker-machine
+subnetPrefix: 192.168.0.0/16
+updateDomainCount: "5"
+usePrivateIp: false
+vnet: docker-machine-vnet

test/e2e/data/rancher/ingress.yaml

@@ -14,4 +14,4 @@ spec:
           service:
             name: rancher
             port:
-              number: 443
+              number: 80

test/e2e/flags.go

@@ -0,0 +1,61 @@
+//go:build e2e
+// +build e2e
+
+/*
+Copyright 2023 SUSE.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import "flag"
+
+type FlagValues struct {
+	// ConfigPath is the path to the e2e config file.
+	ConfigPath string
+
+	// UseExistingCluster instructs the test to use the current cluster instead of creating a new one (default discovery rules apply).
+	UseExistingCluster bool
+
+	// ArtifactFolder is the folder to store e2e test artifacts.
+	ArtifactFolder string
+
+	// SkipCleanup prevents cleanup of test resources e.g. for debug purposes.
+	SkipCleanup bool
+
+	// HelmBinaryPath is the path to the helm binary.
+	HelmBinaryPath string
+
+	// ChartPath is the path to the operator chart.
+	ChartPath string
+
+	// IsolatedMode instructs the test to run without ngrok and exposing the cluster to the internet. This setup will only work with CAPD
+	// or other providers that run in the same network as the bootstrap cluster.
+	IsolatedMode bool
+
+	// ClusterctlBinaryPath is the path to the clusterctl binary to use.
+	ClusterctlBinaryPath string
+}
+
+// InitFlags is used to specify the standard flags for the e2e tests.
+func InitFlags(values *FlagValues) {
+	flag.StringVar(&values.ConfigPath, "e2e.config", "config/operator.yaml", "path to the e2e config file")
+	flag.StringVar(&values.ArtifactFolder, "e2e.artifacts-folder", "_artifacts", "folder where e2e test artifact should be stored")
+	flag.BoolVar(&values.SkipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped")
+	flag.BoolVar(&values.UseExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)")
+	flag.StringVar(&values.HelmBinaryPath, "e2e.helm-binary-path", "helm", "path to the helm binary")
+	flag.StringVar(&values.ClusterctlBinaryPath, "e2e.clusterctl-binary-path", "helm", "path to the clusterctl binary")
+	flag.StringVar(&values.ChartPath, "e2e.chart-path", "", "path to the operator chart")
+	flag.BoolVar(&values.IsolatedMode, "e2e.isolated-mode", false, "if true, the test will run without ngrok and exposing the cluster to the internet. This setup will only work with CAPD or other providers that run in the same network as the bootstrap cluster.")
+}

test/e2e/suites/import-gitops/import_gitops.go

@@ -21,6 +21,7 @@ package import_gitops
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -237,7 +238,7 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi
 		By("Waiting for the CAPI cluster to be connectable using Rancher kubeconfig")
 		turtlesframework.RancherGetClusterKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{
 			Getter:           input.BootstrapClusterProxy.GetClient(),
-			ClusterName:      capiCluster.Name,
+			SecretName:       fmt.Sprintf("%s-capi-kubeconfig", capiCluster.Name),
 			Namespace:        capiCluster.Namespace,
 			RancherServerURL: input.RancherServerURL,
 			WriteToTempFile:  true,

test/e2e/suites/import-gitops/import_gitops_test.go

@@ -39,8 +39,8 @@ var _ = Describe("[Docker] [Kubeadm] Create and delete CAPI cluster functionalit
 			E2EConfig:                 e2eConfig,
 			BootstrapClusterProxy:     setupClusterResult.BootstrapClusterProxy,
 			ClusterctlConfigPath:      clusterctlConfigPath,
-			ClusterctlBinaryPath:      clusterctlBinaryPath,
-			ArtifactFolder:            artifactFolder,
+			ClusterctlBinaryPath:      flagVals.ClusterctlBinaryPath,
+			ArtifactFolder:            flagVals.ArtifactFolder,
 			ClusterTemplatePath:       "../../data/cluster-templates/docker-kubeadm.yaml",
 			ClusterName:               "cluster1",
 			ControlPlaneMachineCount:  ptr.To[int](1),
@@ -69,8 +69,8 @@ var _ = Describe("[AWS] [EKS] Create and delete CAPI cluster functionality shoul
 			E2EConfig:                 e2eConfig,
 			BootstrapClusterProxy:     setupClusterResult.BootstrapClusterProxy,
 			ClusterctlConfigPath:      clusterctlConfigPath,
-			ClusterctlBinaryPath:      clusterctlBinaryPath,
-			ArtifactFolder:            artifactFolder,
+			ClusterctlBinaryPath:      flagVals.ClusterctlBinaryPath,
+			ArtifactFolder:            flagVals.ArtifactFolder,
 			ClusterTemplatePath:       "../../data/cluster-templates/aws-eks-mmp.yaml",
 			ClusterName:               "cluster2",
 			ControlPlaneMachineCount:  ptr.To[int](1),