test: Verify ServiceLoadBalancer in e2e Docker and Nutanix tests

- Add test helper functions. - Configure ServiceLoadBalancer in test cluster templates. Address ranges are within the TEST-2-NET range defined in RFC 5735. - Define kustomize patch to deploy MetalLB. - Apply kustomize patch to Docker and Nutanix example Clusters.
nutanix-cloud-native · Jul 8, 2024 · a603e7f · a603e7f
1 parent b5d15ca
commit a603e7f
Show file tree

Hide file tree

Showing 16 changed files with 248 additions and 0 deletions.
diff --git a/examples/capi-quick-start/docker-cluster-calico-crs.yaml b/examples/capi-quick-start/docker-cluster-calico-crs.yaml
@@ -40,6 +40,14 @@ spec:
               strategy: ClusterResourceSet
           nfd:
             strategy: ClusterResourceSet
+          serviceLoadBalancer:
+            configuration:
+              addressRanges:
+              - end: 198.18.1.10
+                start: 198.18.1.1
+              - end: 198.18.1.30
+                start: 198.18.1.21
+            provider: MetalLB
         encryptionAtRest:
           providers:
           - aescbc: {}

diff --git a/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml
@@ -35,6 +35,14 @@ spec:
                   default: {}
             snapshotController: {}
           nfd: {}
+          serviceLoadBalancer:
+            configuration:
+              addressRanges:
+              - end: 198.18.1.10
+                start: 198.18.1.1
+              - end: 198.18.1.30
+                start: 198.18.1.21
+            provider: MetalLB
         encryptionAtRest:
           providers:
           - aescbc: {}

diff --git a/examples/capi-quick-start/docker-cluster-cilium-crs.yaml b/examples/capi-quick-start/docker-cluster-cilium-crs.yaml
@@ -40,6 +40,14 @@ spec:
               strategy: ClusterResourceSet
           nfd:
             strategy: ClusterResourceSet
+          serviceLoadBalancer:
+            configuration:
+              addressRanges:
+              - end: 198.18.1.10
+                start: 198.18.1.1
+              - end: 198.18.1.30
+                start: 198.18.1.21
+            provider: MetalLB
         encryptionAtRest:
           providers:
           - aescbc: {}

diff --git a/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml
@@ -35,6 +35,14 @@ spec:
                   default: {}
             snapshotController: {}
           nfd: {}
+          serviceLoadBalancer:
+            configuration:
+              addressRanges:
+              - end: 198.18.1.10
+                start: 198.18.1.1
+              - end: 198.18.1.30
+                start: 198.18.1.21
+            provider: MetalLB
         encryptionAtRest:
           providers:
           - aescbc: {}

diff --git a/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml b/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml
@@ -91,6 +91,14 @@ spec:
               strategy: HelmAddon
           nfd:
             strategy: ClusterResourceSet
+          serviceLoadBalancer:
+            configuration:
+              addressRanges:
+              - end: 198.18.1.10
+                start: 198.18.1.1
+              - end: 198.18.1.30
+                start: 198.18.1.21
+            provider: MetalLB
         controlPlane:
           nutanix:
             machineDetails:

diff --git a/examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml
@@ -88,6 +88,14 @@ spec:
             snapshotController:
               strategy: HelmAddon
           nfd: {}
+          serviceLoadBalancer:
+            configuration:
+              addressRanges:
+              - end: 198.18.1.10
+                start: 198.18.1.1
+              - end: 198.18.1.30
+                start: 198.18.1.21
+            provider: MetalLB
         controlPlane:
           nutanix:
             machineDetails:

diff --git a/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml b/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml
@@ -91,6 +91,14 @@ spec:
               strategy: HelmAddon
           nfd:
             strategy: ClusterResourceSet
+          serviceLoadBalancer:
+            configuration:
+              addressRanges:
+              - end: 198.18.1.10
+                start: 198.18.1.1
+              - end: 198.18.1.30
+                start: 198.18.1.21
+            provider: MetalLB
         controlPlane:
           nutanix:
             machineDetails:

diff --git a/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml
@@ -88,6 +88,14 @@ spec:
             snapshotController:
               strategy: HelmAddon
           nfd: {}
+          serviceLoadBalancer:
+            configuration:
+              addressRanges:
+              - end: 198.18.1.10
+                start: 198.18.1.1
+              - end: 198.18.1.30
+                start: 198.18.1.21
+            provider: MetalLB
         controlPlane:
           nutanix:
             machineDetails:

diff --git a/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl b/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl
@@ -41,3 +41,8 @@ patches:
 - target:
     kind: Cluster
   path: ../../../patches/encryption.yaml
+
+# Deploy ServiceLoadBalancer MetalLB
+- target:
+    kind: Cluster
+  path: ../../../patches/metallb.yaml
diff --git a/hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl b/hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl
@@ -77,3 +77,8 @@ patches:
     - op: "replace"
       path: "/spec/topology/workers/machineDeployments/0/class"
       value: default-worker
+
+# Deploy ServiceLoadBalancer MetalLB
+- target:
+    kind: Cluster
+  path: ../../../patches/metallb.yaml
diff --git a/hack/examples/patches/metallb.yaml b/hack/examples/patches/metallb.yaml
@@ -0,0 +1,13 @@
+# Copyright 2024 Nutanix. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+- op: "add"
+  path: "/spec/topology/variables/0/value/addons/serviceLoadBalancer"
+  value:
+    provider: MetalLB
+    configuration:
+      addressRanges:
+        - start: 198.18.1.1
+          end: 198.18.1.10
+        - start: 198.18.1.21
+          end: 198.18.1.30
diff --git a/test/e2e/addon_helpers.go b/test/e2e/addon_helpers.go
@@ -23,6 +23,7 @@ type WaitForAddonsToBeReadyInWorkloadClusterInput struct {
 	DaemonSetIntervals          []interface{}
 	HelmReleaseIntervals        []interface{}
 	ClusterResourceSetIntervals []interface{}
+	ResourceIntervals           []interface{}
 }
 
 func WaitForAddonsToBeReadyInWorkloadCluster(
@@ -94,4 +95,17 @@ func WaitForAddonsToBeReadyInWorkloadCluster(
 			ClusterResourceSetIntervals: input.ClusterResourceSetIntervals,
 		},
 	)
+
+	WaitForServiceLoadBalancerToBeReadyInWorkloadCluster(
+		ctx,
+		WaitForServiceLoadBalancerToBeReadyInWorkloadClusterInput{
+			ServiceLoadBalancer:  input.AddonsConfig.ServiceLoadBalancer,
+			WorkloadCluster:      input.WorkloadCluster,
+			ClusterProxy:         input.ClusterProxy,
+			DeploymentIntervals:  input.DeploymentIntervals,
+			DaemonSetIntervals:   input.DaemonSetIntervals,
+			HelmReleaseIntervals: input.HelmReleaseIntervals,
+			ResourceIntervals:    input.ResourceIntervals,
+		},
+	)
 }
diff --git a/test/e2e/config/caren.yaml b/test/e2e/config/caren.yaml
@@ -224,3 +224,4 @@ intervals:
   default/wait-daemonset: [ "5m", "10s" ]
   default/wait-clusterresourceset: [ "5m", "10s" ]
   default/wait-helmrelease: [ "5m", "10s" ]
+  default/wait-resource: [ "5m", "10s" ]
diff --git a/test/e2e/quick_start_test.go b/test/e2e/quick_start_test.go
@@ -195,6 +195,10 @@ var _ = Describe("Quick start", Serial, func() {
 												flavour,
 												"wait-clusterresourceset",
 											),
+											ResourceIntervals: testE2EConfig.GetIntervals(
+												flavour,
+												"wait-resource",
+											),
 										},
 									)
 

diff --git a/test/e2e/self_hosted_test.go b/test/e2e/self_hosted_test.go
@@ -99,6 +99,10 @@ var _ = Describe("Self-hosted", Serial, func() {
 												flavour,
 												"wait-clusterresourceset",
 											),
+											ResourceIntervals: e2eConfig.GetIntervals(
+												flavour,
+												"wait-resource",
+											),
 										},
 									)
 

diff --git a/test/e2e/serviceloadbalancer_helpers.go b/test/e2e/serviceloadbalancer_helpers.go
@@ -0,0 +1,138 @@
+//go:build e2e
+
+// Copyright 2024 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/test/framework"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb"
+)
+
+type WaitForServiceLoadBalancerToBeReadyInWorkloadClusterInput struct {
+	ServiceLoadBalancer  *v1alpha1.ServiceLoadBalancer
+	WorkloadCluster      *clusterv1.Cluster
+	ClusterProxy         framework.ClusterProxy
+	DeploymentIntervals  []interface{}
+	DaemonSetIntervals   []interface{}
+	HelmReleaseIntervals []interface{}
+	ResourceIntervals    []interface{}
+}
+
+func WaitForServiceLoadBalancerToBeReadyInWorkloadCluster(
+	ctx context.Context,
+	input WaitForServiceLoadBalancerToBeReadyInWorkloadClusterInput, //nolint:gocritic // This hugeParam is OK in tests.
+) {
+	if input.ServiceLoadBalancer == nil {
+		return
+	}
+
+	switch providerName := input.ServiceLoadBalancer.Provider; providerName {
+	case v1alpha1.ServiceLoadBalancerProviderMetalLB:
+		waitForMetalLBServiceLoadBalancerToBeReadyInWorkloadCluster(
+			ctx,
+			waitForMetalLBServiceLoadBalancerToBeReadyInWorkloadClusterInput{
+				workloadCluster:      input.WorkloadCluster,
+				clusterProxy:         input.ClusterProxy,
+				deploymentIntervals:  input.DeploymentIntervals,
+				daemonSetIntervals:   input.DaemonSetIntervals,
+				helmReleaseIntervals: input.HelmReleaseIntervals,
+				resourceIntervals:    input.ResourceIntervals,
+			},
+		)
+	default:
+		Fail(
+			fmt.Sprintf(
+				"Do not know how to wait for ServiceLoadBalancer provider %s to be ready",
+				providerName,
+			),
+		)
+	}
+}
+
+type waitForMetalLBServiceLoadBalancerToBeReadyInWorkloadClusterInput struct {
+	workloadCluster      *clusterv1.Cluster
+	clusterProxy         framework.ClusterProxy
+	helmReleaseIntervals []interface{}
+	deploymentIntervals  []interface{}
+	daemonSetIntervals   []interface{}
+	resourceIntervals    []interface{}
+}
+
+func waitForMetalLBServiceLoadBalancerToBeReadyInWorkloadCluster(
+	ctx context.Context,
+	input waitForMetalLBServiceLoadBalancerToBeReadyInWorkloadClusterInput, //nolint:gocritic // OK in tests.
+) {
+	WaitForHelmReleaseProxyReadyForCluster(
+		ctx,
+		WaitForHelmReleaseProxyReadyForClusterInput{
+			GetLister:          input.clusterProxy.GetClient(),
+			Cluster:            input.workloadCluster,
+			HelmChartProxyName: "metallb-" + input.workloadCluster.Name,
+		},
+		input.helmReleaseIntervals...,
+	)
+
+	workloadClusterClient := input.clusterProxy.GetWorkloadCluster(
+		ctx, input.workloadCluster.Namespace, input.workloadCluster.Name,
+	).GetClient()
+
+	WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
+		Getter: workloadClusterClient,
+		Deployment: &appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "metallb-controller",
+				Namespace: "metallb-system",
+			},
+		},
+	}, input.deploymentIntervals...)
+
+	WaitForDaemonSetsAvailable(ctx, WaitForDaemonSetsAvailableInput{
+		Getter: workloadClusterClient,
+		DaemonSet: &appsv1.DaemonSet{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "metallb-speaker",
+				Namespace: "metallb-system",
+			},
+		},
+	}, input.daemonSetIntervals...)
+
+	// Generate the MetalLB configuration objects, so we can wait for them to be
+	// created on the workload cluster.
+	cos, err := metallb.ConfigurationObjects(&metallb.ConfigurationInput{
+		Name:      "metallb",
+		Namespace: "metallb-system",
+		// We need to populate AddressRanges to generate the configuration,
+		// but the values are not important, because this test does not compare
+		// them against the actual values.
+		AddressRanges: []v1alpha1.AddressRange{
+			{
+				Start: "1.2.3.4",
+				End:   "1.2.3.5",
+			},
+		},
+	})
+	Expect(err).NotTo(HaveOccurred())
+
+	resources := make([]client.Object, len(cos))
+	for i := range cos {
+		resources[i] = cos[i]
+	}
+
+	WaitForResources(ctx, WaitForResourcesInput{
+		Getter:    workloadClusterClient,
+		Resources: resources,
+	}, input.resourceIntervals...)
+}