diff --git a/Makefile b/Makefile index 30de5173..96539dc7 100644 --- a/Makefile +++ b/Makefile @@ -467,7 +467,6 @@ release-chart: $(HELM) $(NOTES) build-chart verify-gen .PHONY: test-e2e test-e2e: $(GINKGO) $(HELM) $(CLUSTERCTL) kubectl e2e-image ## Run the end-to-end tests - RANCHER_HOSTNAME=$(RANCHER_HOSTNAME) \ $(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \ -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" --label-filter="$(GINKGO_LABEL_FILTER)" \ $(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \ diff --git a/test/e2e/flags.go b/test/e2e/flags.go index 5f466b4a..cd5cdacd 100644 --- a/test/e2e/flags.go +++ b/test/e2e/flags.go @@ -37,6 +37,9 @@ type FlagValues struct { // HelmBinaryPath is the path to the helm binary. HelmBinaryPath string + // HelmExtraValuesDir is the location where extra values files will be stored. + HelmExtraValuesDir string + // ChartPath is the path to the operator chart. ChartPath string @@ -55,6 +58,7 @@ func InitFlags(values *FlagValues) { flag.BoolVar(&values.SkipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped") flag.BoolVar(&values.UseExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)") flag.StringVar(&values.HelmBinaryPath, "e2e.helm-binary-path", "helm", "path to the helm binary") + flag.StringVar(&values.HelmExtraValuesDir, "e2e.helm-extra-values-path", "/tmp", "path to the extra values file") flag.StringVar(&values.ClusterctlBinaryPath, "e2e.clusterctl-binary-path", "helm", "path to the clusterctl binary") flag.StringVar(&values.ChartPath, "e2e.chart-path", "", "path to the operator chart") flag.BoolVar(&values.IsolatedMode, "e2e.isolated-mode", false, "if true, the test will run without ngrok and exposing the cluster to the internet. This setup will only work with CAPD or other providers that run in the same network as the bootstrap cluster.") diff --git a/test/e2e/suites/embedded-capi-disabled/suite_test.go b/test/e2e/suites/embedded-capi-disabled/suite_test.go index cfc2a8f8..1e9d665b 100644 --- a/test/e2e/suites/embedded-capi-disabled/suite_test.go +++ b/test/e2e/suites/embedded-capi-disabled/suite_test.go @@ -103,11 +103,11 @@ var _ = BeforeSuite(func() { if flagVals.IsolatedMode { hostName = setupClusterResult.IsolatedHostName } - turtlesframework.Byf("Rancher hostname is %s", hostName) testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, HelmBinaryPath: flagVals.HelmBinaryPath, + HelmExtraValuesPath: filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher-ingress.yaml"), IsolatedMode: flagVals.IsolatedMode, NginxIngress: e2e.NginxIngress, NginxIngressNamespace: e2e.NginxIngressNamespace, @@ -125,6 +125,7 @@ var _ = BeforeSuite(func() { testenv.DeployRancher(ctx, testenv.DeployRancherInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, HelmBinaryPath: flagVals.HelmBinaryPath, + HelmExtraValuesPath: filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher.yaml"), InstallCertManager: true, CertManagerChartPath: e2eConfig.GetVariable(e2e.CertManagerPathVar), CertManagerUrl: e2eConfig.GetVariable(e2e.CertManagerUrlVar), diff --git a/test/e2e/suites/import-gitops/suite_test.go b/test/e2e/suites/import-gitops/suite_test.go index f39e45f5..f2eadc15 100644 --- a/test/e2e/suites/import-gitops/suite_test.go +++ b/test/e2e/suites/import-gitops/suite_test.go @@ -103,11 +103,11 @@ var _ = BeforeSuite(func() { if flagVals.IsolatedMode { hostName = setupClusterResult.IsolatedHostName } - turtlesframework.Byf("Rancher hostname is %s", hostName) testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, HelmBinaryPath: flagVals.HelmBinaryPath, + HelmExtraValuesPath: filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher-ingress.yaml"), IsolatedMode: flagVals.IsolatedMode, NginxIngress: e2e.NginxIngress, NginxIngressNamespace: e2e.NginxIngressNamespace, @@ -123,6 +123,7 @@ var _ = BeforeSuite(func() { testenv.DeployRancher(ctx, testenv.DeployRancherInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, HelmBinaryPath: flagVals.HelmBinaryPath, + HelmExtraValuesPath: filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher.yaml"), InstallCertManager: true, CertManagerChartPath: e2eConfig.GetVariable(e2e.CertManagerPathVar), CertManagerUrl: e2eConfig.GetVariable(e2e.CertManagerUrlVar), diff --git a/test/e2e/suites/update-labels/suite_test.go b/test/e2e/suites/update-labels/suite_test.go index c1540a97..05e32132 100644 --- a/test/e2e/suites/update-labels/suite_test.go +++ b/test/e2e/suites/update-labels/suite_test.go @@ -102,7 +102,6 @@ var _ = BeforeSuite(func() { if flagVals.IsolatedMode { hostName = setupClusterResult.IsolatedHostName } - turtlesframework.Byf("Rancher hostname is %s", hostName) testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, @@ -126,6 +125,7 @@ var _ = BeforeSuite(func() { testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, HelmBinaryPath: flagVals.HelmBinaryPath, + HelmExtraValuesPath: filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher-ingress.yaml"), IsolatedMode: flagVals.IsolatedMode, NginxIngress: e2e.NginxIngress, NginxIngressNamespace: e2e.NginxIngressNamespace, @@ -141,6 +141,7 @@ var _ = BeforeSuite(func() { testenv.DeployRancher(ctx, testenv.DeployRancherInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, HelmBinaryPath: flagVals.HelmBinaryPath, + HelmExtraValuesPath: filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher.yaml"), InstallCertManager: false, RancherChartRepoName: "rancher-latest", RancherChartURL: "https://releases.rancher.com/server-charts/latest", diff --git a/test/e2e/suites/v2prov/suite_test.go b/test/e2e/suites/v2prov/suite_test.go index 55b57330..88e23783 100644 --- a/test/e2e/suites/v2prov/suite_test.go +++ b/test/e2e/suites/v2prov/suite_test.go @@ -102,11 +102,11 @@ var _ = BeforeSuite(func() { if flagVals.IsolatedMode { hostName = setupClusterResult.IsolatedHostName } - turtlesframework.Byf("Rancher hostname is %s", hostName) testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, HelmBinaryPath: flagVals.HelmBinaryPath, + HelmExtraValuesPath: filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher-ingress.yaml"), IsolatedMode: flagVals.IsolatedMode, NginxIngress: e2e.NginxIngress, NginxIngressNamespace: e2e.NginxIngressNamespace, @@ -122,6 +122,7 @@ var _ = BeforeSuite(func() { testenv.DeployRancher(ctx, testenv.DeployRancherInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, HelmBinaryPath: flagVals.HelmBinaryPath, + HelmExtraValuesPath: filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher.yaml"), InstallCertManager: true, CertManagerChartPath: e2eConfig.GetVariable(e2e.CertManagerPathVar), CertManagerUrl: e2eConfig.GetVariable(e2e.CertManagerUrlVar), diff --git a/test/framework/rancher_helpers.go b/test/framework/rancher_helpers.go index 4eac528f..e7b5103d 100644 --- a/test/framework/rancher_helpers.go +++ b/test/framework/rancher_helpers.go @@ -81,8 +81,6 @@ func RancherGetClusterKubeconfig(ctx context.Context, input RancherGetClusterKub serverURL.Host = input.RancherServerURL cluster.Server = serverURL.String() - Byf("Updated kubeconfig with new server-url of %s", cluster.Server) - content, err = clientcmd.Write(*cfg) Expect(err).NotTo(HaveOccurred(), "Failed to save updated kubeconfig") diff --git a/test/testenv/rancher.go b/test/testenv/rancher.go index badce167..2e8a84b4 100644 --- a/test/testenv/rancher.go +++ b/test/testenv/rancher.go @@ -18,6 +18,7 @@ package testenv import ( "context" + "io/ioutil" "os" . "github.com/onsi/ginkgo/v2" @@ -33,11 +34,13 @@ import ( "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest/komega" + "sigs.k8s.io/yaml" ) type DeployRancherInput struct { BootstrapClusterProxy framework.ClusterProxy HelmBinaryPath string + HelmExtraValuesPath string InstallCertManager bool CertManagerChartPath string CertManagerUrl string @@ -61,11 +64,25 @@ type DeployRancherInput struct { Variables turtlesframework.VariableCollection } +type deployRancherValuesFile struct { + BootstrapPassword string `json:"bootstrapPassword"` + Hostname string `json:"hostname"` +} + +type ngrokCredentials struct { + NgrokAPIKey string `json:"apiKey"` + NgrokAuthToken string `json:"authtoken"` +} +type deployRancherIngressValuesFile struct { + Credentials ngrokCredentials `json:"credentials"` +} + func DeployRancher(ctx context.Context, input DeployRancherInput) { Expect(ctx).NotTo(BeNil(), "ctx is required for DeployRancher") Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "BootstrapClusterProxy is required for DeployRancher") Expect(input.HelmBinaryPath).ToNot(BeEmpty(), "HelmBinaryPath is required for DeployRancher") + Expect(input.HelmExtraValuesPath).ToNot(BeEmpty(), "HelmExtraValuesPath is required for DeployRancher") Expect(input.RancherChartRepoName).ToNot(BeEmpty(), "RancherChartRepoName is required for DeployRancher") Expect(input.RancherChartURL).ToNot(BeEmpty(), "RancherChartURL is required for DeployRancher") Expect(input.RancherChartPath).ToNot(BeEmpty(), "RancherChartPath is required for DeployRancher") @@ -140,10 +157,19 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) { Expect(err).ToNot(HaveOccurred()) } + yamlExtraValues, err := yaml.Marshal(deployRancherValuesFile{ + BootstrapPassword: input.RancherPassword, + Hostname: input.RancherHost, + }) + Expect(err).ToNot(HaveOccurred()) + err = ioutil.WriteFile(input.HelmExtraValuesPath, yamlExtraValues, 0644) + Expect(err).ToNot(HaveOccurred()) + By("Installing Rancher") installFlags := opframework.Flags( "--namespace", input.RancherNamespace, "--create-namespace", + "--values", input.HelmExtraValuesPath, ) if input.RancherVersion != "" { installFlags = append(installFlags, "--version", input.RancherVersion) @@ -161,9 +187,7 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) { Wait: true, } values := map[string]string{ - "bootstrapPassword": input.RancherPassword, "global.cattle.psp.enabled": "false", - "hostname": input.RancherHost, "replicas": "1", } if input.RancherFeatures != "" { @@ -221,6 +245,7 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) { type RancherDeployIngressInput struct { BootstrapClusterProxy framework.ClusterProxy HelmBinaryPath string + HelmExtraValuesPath string IsolatedMode bool NginxIngress []byte NginxIngressNamespace string @@ -247,6 +272,7 @@ func RancherDeployIngress(ctx context.Context, input RancherDeployIngressInput) Expect(input.NgrokPath).ToNot(BeEmpty(), "NgrokPath is required when not running in isolated mode") Expect(input.NgrokRepoName).ToNot(BeEmpty(), "NgrokRepoName is required when not running in isolated mode") Expect(input.NgrokRepoURL).ToNot(BeEmpty(), "NgrokRepoURL is required when not running in isolated mode") + Expect(input.HelmExtraValuesPath).ToNot(BeNil(), "HelmExtraValuesPath is when not running in isolated mode") } komega.SetClient(input.BootstrapClusterProxy.GetClient()) @@ -289,18 +315,30 @@ func RancherDeployIngress(ctx context.Context, input RancherDeployIngressInput) _, err = updateChart.Run(nil) Expect(err).ToNot(HaveOccurred()) + yamlExtraValues, err := yaml.Marshal(deployRancherIngressValuesFile{ + Credentials: ngrokCredentials{ + NgrokAPIKey: input.NgrokApiKey, + NgrokAuthToken: input.NgrokAuthToken, + }, + }) + Expect(err).ToNot(HaveOccurred()) + err = ioutil.WriteFile(input.HelmExtraValuesPath, yamlExtraValues, 0644) + Expect(err).ToNot(HaveOccurred()) + + installFlags := opframework.Flags( + "--timeout", "5m", + "--values", input.HelmExtraValuesPath, + ) + installChart := &opframework.HelmChart{ BinaryPath: input.HelmBinaryPath, Name: input.NgrokRepoName, Path: input.NgrokPath, Kubeconfig: input.BootstrapClusterProxy.GetKubeconfigPath(), Wait: true, - AdditionalFlags: opframework.Flags("--timeout", "5m"), + AdditionalFlags: installFlags, } - _, err = installChart.Run(map[string]string{ - "credentials.apiKey": input.NgrokApiKey, - "credentials.authtoken": input.NgrokAuthToken, - }) + _, err = installChart.Run(nil) Expect(err).ToNot(HaveOccurred()) By("Setting up default ingress class")