From 95ce296591f32f8b016cf12a162cfcefe61d72b6 Mon Sep 17 00:00:00 2001 From: Daniel Lipovetsky Date: Tue, 3 Sep 2024 14:20:29 -0700 Subject: [PATCH 1/5] feat: Enable Hubble Relay in Cilium deployment via CAAPH Relay is required for network visibility of a cluster as a whole. See https://docs.cilium.io/en/stable/gettingstarted/hubble_intro/. --- .../templates/cni/cilium/manifests/helm-addon-installation.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml index 4f1ab90d4..f8bdfad8a 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml @@ -12,6 +12,7 @@ data: chainingMode: portmap exclusive: false hubble: + relay: enabled tls: auto: enabled: true # enable automatic TLS certificate generation From 81da33bfac3cfe9c07a505f8f01d8c9b962a7995 Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Wed, 4 Sep 2024 16:23:10 +0100 Subject: [PATCH 2/5] fixup! fix: Correctly enable Hubble relay --- .../cni/cilium/manifests/helm-addon-installation.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml index f8bdfad8a..badfc2972 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml @@ -12,7 +12,8 @@ data: chainingMode: portmap exclusive: false hubble: - relay: enabled + relay: + enabled: true tls: auto: enabled: true # enable automatic TLS certificate generation From 570125a8655230378d3763bd162a3a08810288a3 Mon Sep 17 00:00:00 2001 From: Daniel Lipovetsky Date: Wed, 4 Sep 2024 09:12:28 -0700 Subject: [PATCH 3/5] fixup! Pass chart validation, and disable hubble relay for ClusterResourceSet --- .../templates/cni/cilium/manifests/helm-addon-installation.yaml | 1 + hack/addons/kustomize/cilium/helm-values.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml index badfc2972..06d73b46f 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml @@ -12,6 +12,7 @@ data: chainingMode: portmap exclusive: false hubble: + enabled: true relay: enabled: true tls: diff --git a/hack/addons/kustomize/cilium/helm-values.yaml b/hack/addons/kustomize/cilium/helm-values.yaml index b1de61e45..ab3c52eb0 100644 --- a/hack/addons/kustomize/cilium/helm-values.yaml +++ b/hack/addons/kustomize/cilium/helm-values.yaml @@ -7,6 +7,8 @@ cni: exclusive: false hubble: enabled: false + relay: + enabled: false ipam: mode: kubernetes image: From 5c90945177807b3c6d29c229b977e71abb9d3ed9 Mon Sep 17 00:00:00 2001 From: Daniel Lipovetsky Date: Wed, 4 Sep 2024 09:35:38 -0700 Subject: [PATCH 4/5] fixup! fix: Don't use image digest for hubble relay --- .../cni/cilium/manifests/helm-addon-installation.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml index 06d73b46f..118d46322 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml @@ -13,14 +13,16 @@ data: exclusive: false hubble: enabled: true - relay: - enabled: true tls: auto: enabled: true # enable automatic TLS certificate generation method: cronJob # auto generate certificates using cronJob method certValidityDuration: 60 # certificates validity duration in days (default 2 months) schedule: "0 0 1 * *" # schedule on the 1st day regeneration of each month + relay: + enabled: true + image: + useDigest: false ipam: mode: kubernetes image: From 56a583c7187cdb7390e233966a90fb3823d517c6 Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Thu, 5 Sep 2024 13:50:28 +0100 Subject: [PATCH 5/5] fixup! test(e2e): Add e2e test for hubble relay --- .../generic/lifecycle/ccm/aws/handler.go | 3 +- .../generic/lifecycle/ccm/nutanix/handler.go | 3 +- .../generic/lifecycle/cni/cilium/handler.go | 3 +- .../generic/lifecycle/csi/awsebs/handler.go | 3 +- .../lifecycle/csi/localpath/handler.go | 3 +- .../csi/snapshotcontroller/handler.go | 3 +- test/e2e/clusterautoscaler_helpers.go | 2 +- test/e2e/cni_helpers.go | 35 +++++++++++++------ test/e2e/framework/self_hosted.go | 9 ++--- 9 files changed, 42 insertions(+), 22 deletions(-) diff --git a/pkg/handlers/generic/lifecycle/ccm/aws/handler.go b/pkg/handlers/generic/lifecycle/ccm/aws/handler.go index 57b0f971a..128cef7e0 100644 --- a/pkg/handlers/generic/lifecycle/ccm/aws/handler.go +++ b/pkg/handlers/generic/lifecycle/ccm/aws/handler.go @@ -9,6 +9,7 @@ import ( "github.com/go-logr/logr" "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -23,7 +24,7 @@ import ( const ( awsCCMPrefix = "aws-ccm-" - defaultHelmReleaseNamespace = "kube-system" + defaultHelmReleaseNamespace = metav1.NamespaceSystem defaultHelmReleaseName = "aws-cloud-controller-manager" ) diff --git a/pkg/handlers/generic/lifecycle/ccm/nutanix/handler.go b/pkg/handlers/generic/lifecycle/ccm/nutanix/handler.go index 45af2ef70..a6e1ebbf3 100644 --- a/pkg/handlers/generic/lifecycle/ccm/nutanix/handler.go +++ b/pkg/handlers/generic/lifecycle/ccm/nutanix/handler.go @@ -12,6 +12,7 @@ import ( "github.com/go-logr/logr" "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -25,7 +26,7 @@ import ( const ( defaultHelmReleaseName = "nutanix-ccm" - defaultHelmReleaseNamespace = "kube-system" + defaultHelmReleaseNamespace = metav1.NamespaceSystem // This is the name of the Secret on the remote cluster that should match what is defined in Helm values. //nolint:gosec // Does not contain hard coded credentials. diff --git a/pkg/handlers/generic/lifecycle/cni/cilium/handler.go b/pkg/handlers/generic/lifecycle/cni/cilium/handler.go index 44cd8fd21..7d3e1e14e 100644 --- a/pkg/handlers/generic/lifecycle/cni/cilium/handler.go +++ b/pkg/handlers/generic/lifecycle/cni/cilium/handler.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" @@ -32,7 +33,7 @@ type CNIConfig struct { const ( defaultCiliumReleaseName = "cilium" - defaultCiliumNamespace = "kube-system" + defaultCiliumNamespace = metav1.NamespaceSystem ) type helmAddonConfig struct { diff --git a/pkg/handlers/generic/lifecycle/csi/awsebs/handler.go b/pkg/handlers/generic/lifecycle/csi/awsebs/handler.go index 8cce4d5c0..ffe182e7b 100644 --- a/pkg/handlers/generic/lifecycle/csi/awsebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/awsebs/handler.go @@ -9,6 +9,7 @@ import ( "github.com/go-logr/logr" "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -22,7 +23,7 @@ import ( const ( defaultHelmReleaseName = "aws-ebs-csi-driver" - defaultHelmReleaseNamespace = "kube-system" + defaultHelmReleaseNamespace = metav1.NamespaceSystem ) var DefaultStorageClassParameters = map[string]string{ diff --git a/pkg/handlers/generic/lifecycle/csi/localpath/handler.go b/pkg/handlers/generic/lifecycle/csi/localpath/handler.go index 5cd4b88f6..91212315a 100644 --- a/pkg/handlers/generic/lifecycle/csi/localpath/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/localpath/handler.go @@ -9,6 +9,7 @@ import ( "github.com/go-logr/logr" "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -22,7 +23,7 @@ import ( const ( defaultHelmReleaseName = "local-path-provisioner-csi" - defaultHelmReleaseNamespace = "kube-system" + defaultHelmReleaseNamespace = metav1.NamespaceSystem ) type Config struct { diff --git a/pkg/handlers/generic/lifecycle/csi/snapshotcontroller/handler.go b/pkg/handlers/generic/lifecycle/csi/snapshotcontroller/handler.go index 53c8e543e..56a92a71b 100644 --- a/pkg/handlers/generic/lifecycle/csi/snapshotcontroller/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/snapshotcontroller/handler.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" @@ -25,7 +26,7 @@ import ( const ( defaultHelmReleaseName = "snapshot-controller" - defaultHelmReleaseNamespace = "kube-system" + defaultHelmReleaseNamespace = metav1.NamespaceSystem ) type Config struct { diff --git a/test/e2e/clusterautoscaler_helpers.go b/test/e2e/clusterautoscaler_helpers.go index 8c71241f0..5e10dc4c1 100644 --- a/test/e2e/clusterautoscaler_helpers.go +++ b/test/e2e/clusterautoscaler_helpers.go @@ -135,7 +135,7 @@ func WaitForClusterAutoscalerToBeReadyForWorkloadCluster( statusConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Namespace: "kube-system", + Namespace: metav1.NamespaceSystem, Name: "cluster-autoscaler-status", }, } diff --git a/test/e2e/cni_helpers.go b/test/e2e/cni_helpers.go index 4e2179b45..bf88a8401 100644 --- a/test/e2e/cni_helpers.go +++ b/test/e2e/cni_helpers.go @@ -205,6 +205,13 @@ func waitForCiliumToBeReadyInWorkloadCluster( ctx context.Context, input waitForCiliumToBeReadyInWorkloadClusterInput, //nolint:gocritic // This hugeParam is OK in tests. ) { + expectedDeployments := []*appsv1.Deployment{{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cilium-operator", + Namespace: metav1.NamespaceSystem, + }, + }} + switch input.strategy { case v1alpha1.AddonStrategyClusterResourceSet: crs := &addonsv1.ClusterResourceSet{} @@ -236,6 +243,15 @@ func waitForCiliumToBeReadyInWorkloadCluster( }, input.helmReleaseIntervals..., ) + + // Hubble relay is also deployed as part of the Cilium Helm chart when deplying via + // CAAPH. + expectedDeployments = append(expectedDeployments, &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hubble-relay", + Namespace: metav1.NamespaceSystem, + }, + }) default: Fail( fmt.Sprintf( @@ -249,23 +265,20 @@ func waitForCiliumToBeReadyInWorkloadCluster( ctx, input.workloadCluster.Namespace, input.workloadCluster.Name, ).GetClient() - WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ - Getter: workloadClusterClient, - Deployment: &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cilium-operator", - Namespace: "kube-system", - }, - }, - }, input.deploymentIntervals...) - WaitForDaemonSetsAvailable(ctx, WaitForDaemonSetsAvailableInput{ Getter: workloadClusterClient, DaemonSet: &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: "cilium", - Namespace: "kube-system", + Namespace: metav1.NamespaceSystem, }, }, }, input.daemonSetIntervals...) + + for _, deployment := range expectedDeployments { + WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ + Getter: workloadClusterClient, + Deployment: deployment, + }, input.deploymentIntervals...) + } } diff --git a/test/e2e/framework/self_hosted.go b/test/e2e/framework/self_hosted.go index b3aac4af9..6b8993179 100644 --- a/test/e2e/framework/self_hosted.go +++ b/test/e2e/framework/self_hosted.go @@ -16,6 +16,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/klog/v2" "k8s.io/utils/ptr" @@ -286,12 +287,12 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput) Consistently(func() error { kubeSystem := &corev1.Namespace{} return input.BootstrapClusterProxy.GetClient(). - Get(ctx, client.ObjectKey{Name: "kube-system"}, kubeSystem) + Get(ctx, client.ObjectKey{Name: metav1.NamespaceSystem}, kubeSystem) }, "5s", "100ms").Should(BeNil(), "Failed to assert bootstrap API server stability") Consistently(func() error { kubeSystem := &corev1.Namespace{} return selfHostedClusterProxy.GetClient(). - Get(ctx, client.ObjectKey{Name: "kube-system"}, kubeSystem) + Get(ctx, client.ObjectKey{Name: metav1.NamespaceSystem}, kubeSystem) }, "5s", "100ms").Should(BeNil(), "Failed to assert self-hosted API server stability") By("Moving the cluster to self hosted") @@ -343,12 +344,12 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput) Consistently(func() error { kubeSystem := &corev1.Namespace{} return input.BootstrapClusterProxy.GetClient(). - Get(ctx, client.ObjectKey{Name: "kube-system"}, kubeSystem) + Get(ctx, client.ObjectKey{Name: metav1.NamespaceSystem}, kubeSystem) }, "5s", "100ms").Should(BeNil(), "Failed to assert bootstrap API server stability") Consistently(func() error { kubeSystem := &corev1.Namespace{} return selfHostedClusterProxy.GetClient(). - Get(ctx, client.ObjectKey{Name: "kube-system"}, kubeSystem) + Get(ctx, client.ObjectKey{Name: metav1.NamespaceSystem}, kubeSystem) }, "5s", "100ms").Should(BeNil(), "Failed to assert self-hosted API server stability") By("Moving the cluster back to bootstrap")