diff --git a/.github/workflows/e2e-long.yaml b/.github/workflows/e2e-long.yaml index 24fd4837..b0da46d3 100644 --- a/.github/workflows/e2e-long.yaml +++ b/.github/workflows/e2e-long.yaml @@ -16,6 +16,7 @@ env: AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} jobs: e2e: diff --git a/test/e2e/README.md b/test/e2e/README.md index 2f39d1d2..7286fddb 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -52,7 +52,6 @@ Most notable ones: variables: RANCHER_VERSION: "v2.7.5" # Default rancher version to install RANCHER_HOSTNAME: "localhost" # Your ngrok domain - CAPI_INFRASTRUCTURE: "docker" # Default list of capi providers installed in the cluster. Using docker:latest by default. Could be expanded with `docker,azure` to include latest azure provider for example. NGROK_API_KEY: "" # Key and token values for establishing ingress NGROK_AUTHTOKEN: "" ``` diff --git a/test/e2e/config/operator.yaml b/test/e2e/config/operator.yaml index 71328b1d..fe6a4ec8 100644 --- a/test/e2e/config/operator.yaml +++ b/test/e2e/config/operator.yaml @@ -10,11 +10,13 @@ intervals: default/wait-rancher: ["15m", "30s"] default/wait-v2prov-create: ["25m", "30s"] default/wait-capa-create-cluster: ["30m", "30s"] + default/wait-capz-create-cluster: ["30m", "30s"] default/wait-gitea: ["3m", "10s"] default/wait-consistently: ["30s", "5s"] default/wait-getservice: ["60s", "5s"] default/wait-eks-delete: ["20m", "30s"] - default/wait-azure-delete: ["20m", "30s"] + default/wait-aks-delete: ["20m", "30s"] + default/wait-azure: ["30m", "30s"] variables: RANCHER_VERSION: "v2.7.6" @@ -23,7 +25,6 @@ variables: RANCHER_PATH: "rancher-stable/rancher" KUBERNETES_VERSION: "v1.26.3" RKE2_VERSION: "v1.26.8+rke2r1" - CAPI_INFRASTRUCTURE: "docker:v1.4.6" CAPI_CORE: "cluster-api:v1.4.6" RANCHER_REPO_NAME: "rancher-stable" RANCHER_URL: "https://releases.rancher.com/server-charts/stable" @@ -39,4 +40,4 @@ variables: GITEA_CHART_NAME: "gitea" GITEA_CHART_VERSION: "9.4.0" GITEA_USER_NAME: "gitea_admin" - GITEA_USER_PWD: "password" + GITEA_USER_PWD: "password" \ No newline at end of file diff --git a/test/e2e/const.go b/test/e2e/const.go index 6e77bc27..7c267db0 100644 --- a/test/e2e/const.go +++ b/test/e2e/const.go @@ -30,12 +30,18 @@ var ( //go:embed data/capi-operator/capi-providers.yaml CapiProviders []byte - //go:embed data/capi-operator/full-variables.yaml - FullProvidersSecret []byte - //go:embed data/capi-operator/full-providers.yaml FullProviders []byte + //go:embed data/capi-operator/capa-variables.yaml + AWSProviderSecret []byte + + //go:embed data/capi-operator/capz-variables.yaml + AzureProviderSecret []byte + + //go:embed data/capi-operator/capz-identity-secret.yaml + AzureIdentitySecret []byte + //go:embed data/rancher/ingress.yaml IngressConfig []byte diff --git a/test/e2e/data/capi-operator/full-variables.yaml b/test/e2e/data/capi-operator/capa-variables.yaml similarity index 100% rename from test/e2e/data/capi-operator/full-variables.yaml rename to test/e2e/data/capi-operator/capa-variables.yaml diff --git a/test/e2e/data/capi-operator/capi-providers-secret.yaml b/test/e2e/data/capi-operator/capi-providers-secret.yaml index 3fa7541e..187fa481 100644 --- a/test/e2e/data/capi-operator/capi-providers-secret.yaml +++ b/test/e2e/data/capi-operator/capi-providers-secret.yaml @@ -8,4 +8,3 @@ stringData: CLUSTER_TOPOLOGY: "true" EXP_CLUSTER_RESOURCE_SET: "true" EXP_MACHINE_POOL: "true" - diff --git a/test/e2e/data/capi-operator/capz-identity-secret.yaml b/test/e2e/data/capi-operator/capz-identity-secret.yaml new file mode 100644 index 00000000..f66ac52e --- /dev/null +++ b/test/e2e/data/capi-operator/capz-identity-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +stringData: + clientSecret: "${AZURE_CLIENT_SECRET}" +kind: Secret +metadata: + name: cluster-identity-secret + namespace: default +type: Opaque \ No newline at end of file diff --git a/test/e2e/data/capi-operator/capz-variables.yaml b/test/e2e/data/capi-operator/capz-variables.yaml new file mode 100644 index 00000000..68227086 --- /dev/null +++ b/test/e2e/data/capi-operator/capz-variables.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: azure-variables + namespace: default +type: Opaque +stringData: + CLUSTER_TOPOLOGY: "true" + EXP_CLUSTER_RESOURCE_SET: "true" + EXP_MACHINE_POOL: "true" + EXP_AKS_RESOURCE_HEALTH: "true" diff --git a/test/e2e/data/capi-operator/full-providers.yaml b/test/e2e/data/capi-operator/full-providers.yaml index 416d9be4..1491539d 100644 --- a/test/e2e/data/capi-operator/full-providers.yaml +++ b/test/e2e/data/capi-operator/full-providers.yaml @@ -4,6 +4,11 @@ kind: Namespace metadata: name: capa-system --- +apiVersion: v1 +kind: Namespace +metadata: + name: capz-system +--- apiVersion: operator.cluster.x-k8s.io/v1alpha1 kind: InfrastructureProvider metadata: @@ -11,4 +16,13 @@ metadata: namespace: capa-system spec: secretName: full-variables + secretNamespace: default +--- +apiVersion: operator.cluster.x-k8s.io/v1alpha1 +kind: InfrastructureProvider +metadata: + name: azure + namespace: capz-system +spec: + secretName: azure-variables secretNamespace: default \ No newline at end of file diff --git a/test/e2e/data/cluster-templates/aws-eks-mmp.yaml b/test/e2e/suites/import-gitops/cluster-templates/aws-eks-mmp.yaml similarity index 100% rename from test/e2e/data/cluster-templates/aws-eks-mmp.yaml rename to test/e2e/suites/import-gitops/cluster-templates/aws-eks-mmp.yaml diff --git a/test/e2e/suites/import-gitops/cluster-templates/azure-aks-mmp.yaml b/test/e2e/suites/import-gitops/cluster-templates/azure-aks-mmp.yaml new file mode 100644 index 00000000..0671add0 --- /dev/null +++ b/test/e2e/suites/import-gitops/cluster-templates/azure-aks-mmp.yaml @@ -0,0 +1,130 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + namespace: default +spec: + clusterNetwork: + services: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedControlPlane + name: "${CLUSTER_NAME}" + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedCluster + name: "${CLUSTER_NAME}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlane +metadata: + annotations: + "helm.sh/resource-policy": keep + name: "${CLUSTER_NAME}" + namespace: default +spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: cluster-identity + location: southcentralus + resourceGroupName: "${CLUSTER_NAME}" + sshPublicKey: "" + subscriptionID: "${AZURE_SUBSCRIPTION_ID}" + version: "${KUBERNETES_VERSION}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedCluster +metadata: + annotations: + "helm.sh/resource-policy": keep + name: "${CLUSTER_NAME}" + namespace: default +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + annotations: + "helm.sh/resource-policy": keep + name: "${CLUSTER_NAME}-pool0" + namespace: default +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + template: + metadata: {} + spec: + bootstrap: + dataSecretName: "" + clusterName: "${CLUSTER_NAME}" + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePool + name: "${CLUSTER_NAME}-pool0" + version: "${KUBERNETES_VERSION}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePool +metadata: + annotations: + "helm.sh/resource-policy": keep + name: "${CLUSTER_NAME}-pool0" + namespace: default +spec: + mode: System + name: pool0 + sku: Standard_D2s_v3 +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + annotations: + "helm.sh/resource-policy": keep + name: "${CLUSTER_NAME}-pool1" + namespace: default +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + template: + metadata: {} + spec: + bootstrap: + dataSecretName: "" + clusterName: "${CLUSTER_NAME}" + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePool + name: "${CLUSTER_NAME}-pool1" + version: "${KUBERNETES_VERSION}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePool +metadata: + annotations: + "helm.sh/resource-policy": keep + name: "${CLUSTER_NAME}-pool1" + namespace: default +spec: + mode: User + name: pool1 + sku: Standard_D2s_v3 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + annotations: + "helm.sh/resource-policy": keep + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: cluster-identity + namespace: default +spec: + allowedNamespaces: {} + clientID: "${AZURE_CLIENT_ID}" + clientSecret: + name: cluster-identity-secret + namespace: default + tenantID: "${AZURE_TENANT_ID}" + type: ServicePrincipal diff --git a/test/e2e/data/cluster-templates/docker-kubeadm.yaml b/test/e2e/suites/import-gitops/cluster-templates/docker-kubeadm.yaml similarity index 100% rename from test/e2e/data/cluster-templates/docker-kubeadm.yaml rename to test/e2e/suites/import-gitops/cluster-templates/docker-kubeadm.yaml diff --git a/test/e2e/suites/import-gitops/import_gitops.go b/test/e2e/suites/import-gitops/import_gitops.go index f77569d8..bf83421e 100644 --- a/test/e2e/suites/import-gitops/import_gitops.go +++ b/test/e2e/suites/import-gitops/import_gitops.go @@ -51,7 +51,7 @@ type CreateUsingGitOpsSpecInput struct { RancherServerURL string ClusterctlBinaryPath string - ClusterTemplatePath string + ClusterTemplate []byte ClusterName string CAPIClusterCreateWaitName string @@ -160,15 +160,14 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi clustersDir := filepath.Join(repoDir, "clusters") os.MkdirAll(clustersDir, os.ModePerm) - clusterPath := filepath.Join(clustersDir, "cluster1.yaml") - turtlesframework.ClusterctlGenerateFromTemplate(ctx, turtlesframework.ClusterctlGenerateFromTemplateInput{ - ClusterName: input.ClusterName, - TemplatePath: input.ClusterTemplatePath, - OutputFilePath: clusterPath, - ClusterCtlBinaryPath: input.ClusterctlBinaryPath, - EnvironmentVariables: map[string]string{ + clusterPath := filepath.Join(clustersDir, fmt.Sprintf("%s.yaml", input.ClusterName)) + turtlesframework.ApplyFromTemplate(ctx, turtlesframework.ApplyFromTemplateInput{ + Getter: input.E2EConfig.GetVariable, + Template: input.ClusterTemplate, + OutputFilePath: clusterPath, + AddtionalEnvironmentVariables: map[string]string{ + "CLUSTER_NAME": input.ClusterName, "WORKER_MACHINE_COUNT": strconv.Itoa(workerMachineCount), - "KUBERNETES_VERSION": input.E2EConfig.GetVariable(e2e.KubernetesVersionVar), "CONTROL_PLANE_MACHINE_COUNT": strconv.Itoa(controlPlaneMachineCount), }, }) diff --git a/test/e2e/suites/import-gitops/import_gitops_test.go b/test/e2e/suites/import-gitops/import_gitops_test.go index 6e107f51..dfbbcbe5 100644 --- a/test/e2e/suites/import-gitops/import_gitops_test.go +++ b/test/e2e/suites/import-gitops/import_gitops_test.go @@ -25,23 +25,37 @@ import ( "github.com/rancher-sandbox/rancher-turtles/test/e2e" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/envtest/komega" + . "sigs.k8s.io/controller-runtime/pkg/envtest/komega" + + _ "embed" +) + +var ( + //go:embed cluster-templates/docker-kubeadm.yaml + dockerKubeadm []byte + + // go:embed cluster-templates/aws-eks-mmp.yaml + awsEKSMMP []byte + + //go:embed cluster-templates/azure-aks-mmp.yaml + azureAKSMMP []byte ) var _ = Describe("[Docker] [Kubeadm] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.ShortTestLabel, e2e.FullTestLabel), func() { BeforeEach(func() { - komega.SetClient(setupClusterResult.BootstrapClusterProxy.GetClient()) - komega.SetContext(ctx) + SetClient(setupClusterResult.BootstrapClusterProxy.GetClient()) + SetContext(ctx) }) CreateUsingGitOpsSpec(ctx, func() CreateUsingGitOpsSpecInput { return CreateUsingGitOpsSpecInput{ E2EConfig: e2eConfig, BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, - ClusterctlConfigPath: clusterctlConfigPath, + ClusterctlConfigPath: flagVals.ConfigPath, ClusterctlBinaryPath: flagVals.ClusterctlBinaryPath, ArtifactFolder: flagVals.ArtifactFolder, - ClusterTemplatePath: "../../data/cluster-templates/docker-kubeadm.yaml", + ClusterTemplate: dockerKubeadm, ClusterName: "cluster1", ControlPlaneMachineCount: ptr.To[int](1), WorkerMachineCount: ptr.To[int](1), @@ -68,10 +82,10 @@ var _ = Describe("[AWS] [EKS] Create and delete CAPI cluster functionality shoul return CreateUsingGitOpsSpecInput{ E2EConfig: e2eConfig, BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, - ClusterctlConfigPath: clusterctlConfigPath, + ClusterctlConfigPath: flagVals.ConfigPath, ClusterctlBinaryPath: flagVals.ClusterctlBinaryPath, ArtifactFolder: flagVals.ArtifactFolder, - ClusterTemplatePath: "../../data/cluster-templates/aws-eks-mmp.yaml", + ClusterTemplate: awsEKSMMP, ClusterName: "cluster2", ControlPlaneMachineCount: ptr.To[int](1), WorkerMachineCount: ptr.To[int](1), @@ -86,3 +100,32 @@ var _ = Describe("[AWS] [EKS] Create and delete CAPI cluster functionality shoul } }) }) + +var _ = Describe("[Azure] [AKS] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.FullTestLabel), func() { + + BeforeEach(func() { + SetClient(setupClusterResult.BootstrapClusterProxy.GetClient()) + SetContext(ctx) + }) + + CreateUsingGitOpsSpec(ctx, func() CreateUsingGitOpsSpecInput { + return CreateUsingGitOpsSpecInput{ + E2EConfig: e2eConfig, + BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, + ClusterctlConfigPath: flagVals.ConfigPath, + ArtifactFolder: flagVals.ArtifactFolder, + ClusterTemplate: azureAKSMMP, + ClusterName: "cluster-azure-aks", + ControlPlaneMachineCount: ptr.To[int](1), + WorkerMachineCount: ptr.To[int](1), + GitAddr: giteaResult.GitAddress, + GitAuthSecretName: e2e.AuthSecretName, + SkipCleanup: false, + SkipDeletionTest: false, + LabelNamespace: true, + RancherServerURL: hostName, + CAPIClusterCreateWaitName: "wait-capz-create-cluster", + DeleteClusterWaitName: "wait-aks-delete", + } + }) +}) diff --git a/test/e2e/suites/import-gitops/suite_test.go b/test/e2e/suites/import-gitops/suite_test.go index a213a6b8..3981ee15 100644 --- a/test/e2e/suites/import-gitops/suite_test.go +++ b/test/e2e/suites/import-gitops/suite_test.go @@ -122,10 +122,14 @@ var _ = BeforeSuite(func() { Expect(awsCreds).ToNot(BeEmpty(), "AWS creds required for full test") testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{ - BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, - CAPIProvidersSecretYAML: e2e.FullProvidersSecret, - CAPIProvidersYAML: e2e.FullProviders, - Data: map[string]string{ + BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, + CAPIProvidersSecretsYAML: [][]byte{ + e2e.AWSProviderSecret, + e2e.AzureProviderSecret, + e2e.AzureIdentitySecret, + }, + CAPIProvidersYAML: e2e.FullProviders, + TemplateData: map[string]string{ "AWSEncodedCredentials": e2eConfig.GetVariable(e2e.CapaEncodedCredentialsVar), }, WaitDeploymentsReadyInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"), @@ -134,6 +138,10 @@ var _ = BeforeSuite(func() { Name: "capa-controller-manager", Namespace: "capa-system", }, + { + Name: "capz-controller-manager", + Namespace: "capz-system", + }, }, }) } diff --git a/test/framework/apply_template_helper.go b/test/framework/apply_template_helper.go new file mode 100644 index 00000000..c580ca5b --- /dev/null +++ b/test/framework/apply_template_helper.go @@ -0,0 +1,72 @@ +/* +Copyright 2023 SUSE. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + "os" + + "github.com/drone/envsubst/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/cluster-api/test/framework" +) + +// ApplyFromTemplateInput is the input to ApplyFromTemplate. +type ApplyFromTemplateInput struct { + Getter func(key string) string + Template []byte + AddtionalEnvironmentVariables map[string]string + + Proxy framework.ClusterProxy + OutputFilePath string +} + +// ApplyFromTemplate will generate a yaml definition from a given template and apply it in the cluster. +func ApplyFromTemplate(ctx context.Context, input ApplyFromTemplateInput) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for ApplyFromTemplate.") + Expect(input.Template).ToNot(BeEmpty(), "Invalid argument. input.Template must be an existing byte array.") + Expect(input.Getter).NotTo(BeNil(), "Getter method is required for ApplyFromTemplate. Typically an os.Getenv is enough.") + if input.OutputFilePath == "" { + Expect(input.Proxy).NotTo(BeNil(), "Cluster proxy is required for ApplyFromTemplate.") + } + + // Apply environment variables in the folowing order of the precedence: + // 1. input.AddtionalEnvironmentVariables + // 2. input.Getter - in case of using cluster-api proxy GetVariable: + // 1. os.Getenv + // 2. test/e2e/config/operator.yaml variables content + overrides := input.AddtionalEnvironmentVariables + if overrides == nil { + overrides = map[string]string{} + } + + getter := func(key string) string { + if val, ok := overrides[key]; ok { + return val + } + return input.Getter(key) + } + + template, err := envsubst.Eval(string(input.Template), getter) + Expect(err).NotTo(HaveOccurred(), "Failed executing template generate") + + if input.OutputFilePath != "" { + return os.WriteFile(input.OutputFilePath, []byte(template), os.ModePerm) + } + + return input.Proxy.Apply(ctx, []byte(template)) +} diff --git a/test/testenv/operator.go b/test/testenv/operator.go index 6faa3476..df11b680 100644 --- a/test/testenv/operator.go +++ b/test/testenv/operator.go @@ -27,15 +27,17 @@ import ( appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" turtlesframework "github.com/rancher-sandbox/rancher-turtles/test/framework" ) type CAPIOperatorDeployProviderInput struct { + E2EConfig *clusterctl.E2EConfig BootstrapClusterProxy framework.ClusterProxy - CAPIProvidersSecretYAML []byte + CAPIProvidersSecretsYAML [][]byte CAPIProvidersYAML []byte - Data map[string]string + TemplateData map[string]string WaitDeploymentsReadyInterval []interface{} WaitForDeployments []NamespaceName } @@ -51,11 +53,16 @@ func CAPIOperatorDeployProvider(ctx context.Context, input CAPIOperatorDeployPro Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "BootstrapClusterProxy is required for CAPIOperatorDeployProvider") Expect(input.CAPIProvidersYAML).ToNot(BeNil(), "CAPIProvidersYAML is required for CAPIOperatorDeployProvider") - if input.CAPIProvidersSecretYAML != nil { + for _, secret := range input.CAPIProvidersSecretsYAML { + secret := secret By("Adding CAPI Operator variables secret") - providerVars := getFullProviderVariables(string(input.CAPIProvidersSecretYAML), input.Data) - Expect(input.BootstrapClusterProxy.Apply(ctx, providerVars)).To(Succeed(), "Failed to apply secret for capi providers") + providerVars := getFullProviderVariables(string(secret), input.TemplateData) + Expect(turtlesframework.ApplyFromTemplate(ctx, turtlesframework.ApplyFromTemplateInput{ + Proxy: input.BootstrapClusterProxy, + Template: providerVars, + Getter: input.E2EConfig.GetVariable, + })).To(Succeed(), "Failed to apply secret for capi providers") } By("Adding CAPI Operaytor providers")