Merge pull request #133 from Danil-Grigorev/azure-e2e

✨ Add azure e2e test job

.github/workflows/e2e-long.yaml

@@ -16,6 +16,7 @@ env:
   AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
   AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
   AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
+  AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
 
 jobs:
   e2e:

test/e2e/README.md

@@ -52,7 +52,6 @@ Most notable ones:
 variables:
   RANCHER_VERSION: "v2.7.5" # Default rancher version to install
   RANCHER_HOSTNAME: "localhost" # Your ngrok domain
-  CAPI_INFRASTRUCTURE: "docker" # Default list of capi providers installed in the cluster. Using docker:latest by default. Could be expanded with `docker,azure` to include latest azure provider for example.
   NGROK_API_KEY: "" # Key and token values for establishing ingress
   NGROK_AUTHTOKEN: ""
 ```

test/e2e/config/operator.yaml

@@ -10,11 +10,13 @@ intervals:
   default/wait-rancher: ["15m", "30s"]
   default/wait-v2prov-create: ["25m", "30s"]
   default/wait-capa-create-cluster: ["30m", "30s"]
+  default/wait-capz-create-cluster: ["30m", "30s"]
   default/wait-gitea: ["3m", "10s"]
   default/wait-consistently: ["30s", "5s"]
   default/wait-getservice: ["60s", "5s"]
   default/wait-eks-delete: ["20m", "30s"]
-  default/wait-azure-delete: ["20m", "30s"]
+  default/wait-aks-delete: ["20m", "30s"]
+  default/wait-azure: ["30m", "30s"]
 
 variables:
   RANCHER_VERSION: "v2.7.6"
@@ -23,7 +25,6 @@ variables:
   RANCHER_PATH: "rancher-stable/rancher"
   KUBERNETES_VERSION: "v1.26.3"
   RKE2_VERSION: "v1.26.8+rke2r1"
-  CAPI_INFRASTRUCTURE: "docker:v1.4.6"
   CAPI_CORE: "cluster-api:v1.4.6"
   RANCHER_REPO_NAME: "rancher-stable"
   RANCHER_URL: "https://releases.rancher.com/server-charts/stable"
@@ -39,4 +40,4 @@ variables:
   GITEA_CHART_NAME: "gitea"
   GITEA_CHART_VERSION: "9.4.0"
   GITEA_USER_NAME: "gitea_admin"
-  GITEA_USER_PWD: "password"
+  GITEA_USER_PWD: "password"

test/e2e/const.go

@@ -30,12 +30,18 @@ var (
 	//go:embed data/capi-operator/capi-providers.yaml
 	CapiProviders []byte
 
-	//go:embed data/capi-operator/full-variables.yaml
-	FullProvidersSecret []byte
-
 	//go:embed data/capi-operator/full-providers.yaml
 	FullProviders []byte
 
+	//go:embed data/capi-operator/capa-variables.yaml
+	AWSProviderSecret []byte
+
+	//go:embed data/capi-operator/capz-variables.yaml
+	AzureProviderSecret []byte
+
+	//go:embed data/capi-operator/capz-identity-secret.yaml
+	AzureIdentitySecret []byte
+
 	//go:embed data/rancher/ingress.yaml
 	IngressConfig []byte
 

test/e2e/data/capi-operator/capi-providers-secret.yaml

@@ -8,4 +8,3 @@ stringData:
   CLUSTER_TOPOLOGY: "true"
   EXP_CLUSTER_RESOURCE_SET: "true"
   EXP_MACHINE_POOL: "true"
-

test/e2e/data/capi-operator/capz-identity-secret.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+stringData:
+  clientSecret: "${AZURE_CLIENT_SECRET}"
+kind: Secret
+metadata:
+  name: cluster-identity-secret
+  namespace: default
+type: Opaque

test/e2e/data/capi-operator/capz-variables.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: azure-variables
+  namespace: default
+type: Opaque
+stringData:
+  CLUSTER_TOPOLOGY: "true"
+  EXP_CLUSTER_RESOURCE_SET: "true"
+  EXP_MACHINE_POOL: "true"
+  EXP_AKS_RESOURCE_HEALTH: "true"

test/e2e/data/capi-operator/full-providers.yaml

@@ -4,11 +4,25 @@ kind: Namespace
 metadata:
   name: capa-system
 ---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: capz-system
+---
 apiVersion: operator.cluster.x-k8s.io/v1alpha1
 kind: InfrastructureProvider
 metadata:
   name: aws
   namespace: capa-system
 spec:
   secretName: full-variables
+  secretNamespace: default
+---
+apiVersion: operator.cluster.x-k8s.io/v1alpha1
+kind: InfrastructureProvider
+metadata:
+  name: azure
+  namespace: capz-system
+spec:
+  secretName: azure-variables
   secretNamespace: default

test/e2e/suites/import-gitops/cluster-templates/azure-aks-mmp.yaml

@@ -0,0 +1,130 @@
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: "${CLUSTER_NAME}"
+  namespace: default
+spec:
+  clusterNetwork:
+    services:
+      cidrBlocks:
+      - 192.168.0.0/16
+  controlPlaneRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: AzureManagedControlPlane
+    name: "${CLUSTER_NAME}"
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: AzureManagedCluster
+    name: "${CLUSTER_NAME}"
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureManagedControlPlane
+metadata:
+  annotations:
+    "helm.sh/resource-policy": keep
+  name: "${CLUSTER_NAME}"
+  namespace: default
+spec:
+  identityRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: AzureClusterIdentity
+    name: cluster-identity
+  location: southcentralus
+  resourceGroupName: "${CLUSTER_NAME}"
+  sshPublicKey: ""
+  subscriptionID: "${AZURE_SUBSCRIPTION_ID}"
+  version: "${KUBERNETES_VERSION}"
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureManagedCluster
+metadata:
+  annotations:
+    "helm.sh/resource-policy": keep
+  name: "${CLUSTER_NAME}"
+  namespace: default
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachinePool
+metadata:
+  annotations:
+    "helm.sh/resource-policy": keep
+  name: "${CLUSTER_NAME}-pool0"
+  namespace: default
+spec:
+  clusterName: "${CLUSTER_NAME}"
+  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+  template:
+    metadata: {}
+    spec:
+      bootstrap:
+        dataSecretName: ""
+      clusterName: "${CLUSTER_NAME}"
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: AzureManagedMachinePool
+        name: "${CLUSTER_NAME}-pool0"
+      version: "${KUBERNETES_VERSION}"
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureManagedMachinePool
+metadata:
+  annotations:
+    "helm.sh/resource-policy": keep
+  name: "${CLUSTER_NAME}-pool0"
+  namespace: default
+spec:
+  mode: System
+  name: pool0
+  sku: Standard_D2s_v3
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachinePool
+metadata:
+  annotations:
+    "helm.sh/resource-policy": keep
+  name: "${CLUSTER_NAME}-pool1"
+  namespace: default
+spec:
+  clusterName: "${CLUSTER_NAME}"
+  replicas: ${WORKER_MACHINE_COUNT}
+  template:
+    metadata: {}
+    spec:
+      bootstrap:
+        dataSecretName: ""
+      clusterName: "${CLUSTER_NAME}"
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: AzureManagedMachinePool
+        name: "${CLUSTER_NAME}-pool1"
+      version: "${KUBERNETES_VERSION}"
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureManagedMachinePool
+metadata:
+  annotations:
+    "helm.sh/resource-policy": keep
+  name: "${CLUSTER_NAME}-pool1"
+  namespace: default
+spec:
+  mode: User
+  name: pool1
+  sku: Standard_D2s_v3
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureClusterIdentity
+metadata:
+  annotations:
+    "helm.sh/resource-policy": keep
+  labels:
+    clusterctl.cluster.x-k8s.io/move-hierarchy: "true"
+  name: cluster-identity
+  namespace: default
+spec:
+  allowedNamespaces: {}
+  clientID: "${AZURE_CLIENT_ID}"
+  clientSecret:
+    name: cluster-identity-secret
+    namespace: default
+  tenantID: "${AZURE_TENANT_ID}"
+  type: ServicePrincipal

test/e2e/suites/import-gitops/import_gitops.go

@@ -51,7 +51,7 @@ type CreateUsingGitOpsSpecInput struct {
 	RancherServerURL      string
 
 	ClusterctlBinaryPath string
-	ClusterTemplatePath  string
+	ClusterTemplate      []byte
 	ClusterName          string
 
 	CAPIClusterCreateWaitName string
@@ -160,15 +160,14 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi
 		clustersDir := filepath.Join(repoDir, "clusters")
 		os.MkdirAll(clustersDir, os.ModePerm)
 
-		clusterPath := filepath.Join(clustersDir, "cluster1.yaml")
-		turtlesframework.ClusterctlGenerateFromTemplate(ctx, turtlesframework.ClusterctlGenerateFromTemplateInput{
-			ClusterName:          input.ClusterName,
-			TemplatePath:         input.ClusterTemplatePath,
-			OutputFilePath:       clusterPath,
-			ClusterCtlBinaryPath: input.ClusterctlBinaryPath,
-			EnvironmentVariables: map[string]string{
+		clusterPath := filepath.Join(clustersDir, fmt.Sprintf("%s.yaml", input.ClusterName))
+		turtlesframework.ApplyFromTemplate(ctx, turtlesframework.ApplyFromTemplateInput{
+			Getter:         input.E2EConfig.GetVariable,
+			Template:       input.ClusterTemplate,
+			OutputFilePath: clusterPath,
+			AddtionalEnvironmentVariables: map[string]string{
+				"CLUSTER_NAME":                input.ClusterName,
 				"WORKER_MACHINE_COUNT":        strconv.Itoa(workerMachineCount),
-				"KUBERNETES_VERSION":          input.E2EConfig.GetVariable(e2e.KubernetesVersionVar),
 				"CONTROL_PLANE_MACHINE_COUNT": strconv.Itoa(controlPlaneMachineCount),
 			},
 		})

test/e2e/suites/import-gitops/import_gitops_test.go

@@ -25,23 +25,37 @@ import (
 	"github.com/rancher-sandbox/rancher-turtles/test/e2e"
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/envtest/komega"
+	. "sigs.k8s.io/controller-runtime/pkg/envtest/komega"
+
+	_ "embed"
+)
+
+var (
+	//go:embed cluster-templates/docker-kubeadm.yaml
+	dockerKubeadm []byte
+
+	// go:embed cluster-templates/aws-eks-mmp.yaml
+	awsEKSMMP []byte
+
+	//go:embed cluster-templates/azure-aks-mmp.yaml
+	azureAKSMMP []byte
 )
 
 var _ = Describe("[Docker] [Kubeadm] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.ShortTestLabel, e2e.FullTestLabel), func() {
 
 	BeforeEach(func() {
-		komega.SetClient(setupClusterResult.BootstrapClusterProxy.GetClient())
-		komega.SetContext(ctx)
+		SetClient(setupClusterResult.BootstrapClusterProxy.GetClient())
+		SetContext(ctx)
 	})
 
 	CreateUsingGitOpsSpec(ctx, func() CreateUsingGitOpsSpecInput {
 		return CreateUsingGitOpsSpecInput{
 			E2EConfig:                 e2eConfig,
 			BootstrapClusterProxy:     setupClusterResult.BootstrapClusterProxy,
-			ClusterctlConfigPath:      clusterctlConfigPath,
+			ClusterctlConfigPath:      flagVals.ConfigPath,
 			ClusterctlBinaryPath:      flagVals.ClusterctlBinaryPath,
 			ArtifactFolder:            flagVals.ArtifactFolder,
-			ClusterTemplatePath:       "../../data/cluster-templates/docker-kubeadm.yaml",
+			ClusterTemplate:           dockerKubeadm,
 			ClusterName:               "cluster1",
 			ControlPlaneMachineCount:  ptr.To[int](1),
 			WorkerMachineCount:        ptr.To[int](1),
@@ -68,10 +82,10 @@ var _ = Describe("[AWS] [EKS] Create and delete CAPI cluster functionality shoul
 		return CreateUsingGitOpsSpecInput{
 			E2EConfig:                 e2eConfig,
 			BootstrapClusterProxy:     setupClusterResult.BootstrapClusterProxy,
-			ClusterctlConfigPath:      clusterctlConfigPath,
+			ClusterctlConfigPath:      flagVals.ConfigPath,
 			ClusterctlBinaryPath:      flagVals.ClusterctlBinaryPath,
 			ArtifactFolder:            flagVals.ArtifactFolder,
-			ClusterTemplatePath:       "../../data/cluster-templates/aws-eks-mmp.yaml",
+			ClusterTemplate:           awsEKSMMP,
 			ClusterName:               "cluster2",
 			ControlPlaneMachineCount:  ptr.To[int](1),
 			WorkerMachineCount:        ptr.To[int](1),
@@ -86,3 +100,32 @@ var _ = Describe("[AWS] [EKS] Create and delete CAPI cluster functionality shoul
 		}
 	})
 })
+
+var _ = Describe("[Azure] [AKS] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.FullTestLabel), func() {
+
+	BeforeEach(func() {
+		SetClient(setupClusterResult.BootstrapClusterProxy.GetClient())
+		SetContext(ctx)
+	})
+
+	CreateUsingGitOpsSpec(ctx, func() CreateUsingGitOpsSpecInput {
+		return CreateUsingGitOpsSpecInput{
+			E2EConfig:                 e2eConfig,
+			BootstrapClusterProxy:     setupClusterResult.BootstrapClusterProxy,
+			ClusterctlConfigPath:      flagVals.ConfigPath,
+			ArtifactFolder:            flagVals.ArtifactFolder,
+			ClusterTemplate:           azureAKSMMP,
+			ClusterName:               "cluster-azure-aks",
+			ControlPlaneMachineCount:  ptr.To[int](1),
+			WorkerMachineCount:        ptr.To[int](1),
+			GitAddr:                   giteaResult.GitAddress,
+			GitAuthSecretName:         e2e.AuthSecretName,
+			SkipCleanup:               false,
+			SkipDeletionTest:          false,
+			LabelNamespace:            true,
+			RancherServerURL:          hostName,
+			CAPIClusterCreateWaitName: "wait-capz-create-cluster",
+			DeleteClusterWaitName:     "wait-aks-delete",
+		}
+	})
+})