From 38241375f99f3a40e7dbea2767d3da0cdec5493d Mon Sep 17 00:00:00 2001 From: Daniel Lipovetsky Date: Fri, 28 Jun 2024 13:01:06 -0700 Subject: [PATCH 1/2] feat: Apply MetalLB configuration to remote cluster --- .../lifecycle/serviceloadbalancer/handler.go | 2 + .../serviceloadbalancer/handler_test.go | 1 + .../metallb/configuration.go | 67 ++++++++++++++++ .../serviceloadbalancer/metallb/handler.go | 78 +++++++++++++++++++ 4 files changed, 148 insertions(+) create mode 100644 pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb/configuration.go diff --git a/pkg/handlers/generic/lifecycle/serviceloadbalancer/handler.go b/pkg/handlers/generic/lifecycle/serviceloadbalancer/handler.go index 44773c454..9578caa11 100644 --- a/pkg/handlers/generic/lifecycle/serviceloadbalancer/handler.go +++ b/pkg/handlers/generic/lifecycle/serviceloadbalancer/handler.go @@ -22,6 +22,7 @@ import ( type ServiceLoadBalancerProvider interface { Apply( ctx context.Context, + slb v1alpha1.ServiceLoadBalancer, cluster *clusterv1.Cluster, log logr.Logger, ) error @@ -131,6 +132,7 @@ func (s *ServiceLoadBalancerHandler) apply( log.Info(fmt.Sprintf("Deploying ServiceLoadBalancer provider %s", slb.Provider)) err = handler.Apply( ctx, + slb, cluster, log, ) diff --git a/pkg/handlers/generic/lifecycle/serviceloadbalancer/handler_test.go b/pkg/handlers/generic/lifecycle/serviceloadbalancer/handler_test.go index d941fb071..2ed669025 100644 --- a/pkg/handlers/generic/lifecycle/serviceloadbalancer/handler_test.go +++ b/pkg/handlers/generic/lifecycle/serviceloadbalancer/handler_test.go @@ -25,6 +25,7 @@ type fakeServiceLoadBalancerProvider struct { func (p *fakeServiceLoadBalancerProvider) Apply( ctx context.Context, + slb v1alpha1.ServiceLoadBalancer, cluster *clusterv1.Cluster, log logr.Logger, ) error { diff --git a/pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb/configuration.go b/pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb/configuration.go new file mode 100644 index 000000000..426366954 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb/configuration.go @@ -0,0 +1,67 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +package metallb + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" +) + +func groupVersionKind(kind string) schema.GroupVersionKind { + return schema.GroupVersionKind{ + Group: "metallb.io", + Version: "v1beta1", + Kind: kind, + } +} + +type configurationInput struct { + name string + namespace string + addressRanges []v1alpha1.AddressRange +} + +func configurationObjects(input *configurationInput) ([]unstructured.Unstructured, error) { + ipAddressPool := unstructured.Unstructured{} + ipAddressPool.SetGroupVersionKind(groupVersionKind("IPAddressPool")) + ipAddressPool.SetName(input.name) + ipAddressPool.SetNamespace(input.namespace) + + addresses := []string{} + for _, ar := range input.addressRanges { + addresses = append(addresses, fmt.Sprintf("%s-%s", ar.Start, ar.End)) + } + if err := unstructured.SetNestedStringSlice( + ipAddressPool.Object, + addresses, + "spec", + "addresses", + ); err != nil { + return nil, fmt.Errorf("failed to set IPAddressPool .spec.addresses: %w", err) + } + + l2Advertisement := unstructured.Unstructured{} + l2Advertisement.SetGroupVersionKind(groupVersionKind("L2Advertisement")) + l2Advertisement.SetName(input.name) + l2Advertisement.SetNamespace(input.namespace) + + if err := unstructured.SetNestedStringSlice( + l2Advertisement.Object, + []string{ + ipAddressPool.GetName(), + }, + "spec", + "ipAddressPools", + ); err != nil { + return nil, fmt.Errorf("failed to set L2Advertisement .spec.ipAddressPools: %w", err) + } + + return []unstructured.Unstructured{ + ipAddressPool, + l2Advertisement, + }, nil +} diff --git a/pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb/handler.go b/pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb/handler.go index a83ec3f12..4c1f3b7f8 100644 --- a/pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb/handler.go +++ b/pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb/handler.go @@ -6,20 +6,25 @@ package metallb import ( "context" "fmt" + "time" "github.com/go-logr/logr" "github.com/spf13/pflag" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kwait "k8s.io/apimachinery/pkg/util/wait" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/controllers/remote" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" caaphv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/wait" ) const ( @@ -70,6 +75,7 @@ func New( func (n *MetalLB) Apply( ctx context.Context, + slb v1alpha1.ServiceLoadBalancer, cluster *clusterv1.Cluster, log logr.Logger, ) error { @@ -152,5 +158,77 @@ func (n *MetalLB) Apply( return fmt.Errorf("failed to apply MetalLB installation HelmChartProxy: %w", err) } + if err := wait.ForObject( + ctx, + wait.ForObjectInput[*caaphv1.HelmChartProxy]{ + Reader: n.client, + Target: hcp.DeepCopy(), + Check: func(_ context.Context, obj *caaphv1.HelmChartProxy) (bool, error) { + for _, c := range obj.GetConditions() { + if c.Type == caaphv1.HelmReleaseProxiesReadyCondition && c.Status == corev1.ConditionTrue { + return true, nil + } + } + return false, nil + }, + Interval: 5 * time.Second, + Timeout: 10 * time.Second, + }, + ); err != nil { + return fmt.Errorf("failed to wait for MetalLB to deploy: %w", err) + } + + log.Info( + fmt.Sprintf("Applying MetalLB configuration to cluster %s", + ctrlclient.ObjectKeyFromObject(cluster), + ), + ) + + cos, err := configurationObjects(&configurationInput{ + name: defaultHelmReleaseName, + namespace: defaultHelmReleaseNamespace, + addressRanges: slb.Configuration.AddressRanges, + }) + if err != nil { + return fmt.Errorf("failed to generate MetalLB configuration: %w", err) + } + + var applyErr error + if waitErr := kwait.PollUntilContextTimeout( + ctx, + 5*time.Second, + 10*time.Second, + true, + func(ctx context.Context) (done bool, err error) { + for i := range cos { + o := &cos[i] + if err = client.ServerSideApply( + ctx, + remoteClient, + o, + &ctrlclient.PatchOptions{ + Raw: &metav1.PatchOptions{ + FieldValidation: metav1.FieldValidationStrict, + }, + }, + ); err != nil { + applyErr = fmt.Errorf( + "failed to apply MetalLB configuration %s %s: %w", + o.GetKind(), + ctrlclient.ObjectKeyFromObject(o), + err, + ) + return false, nil + } + } + return true, nil + }, + ); waitErr != nil { + if applyErr != nil { + return fmt.Errorf("%w: last apply error: %w", waitErr, applyErr) + } + return fmt.Errorf("%w: failed to apply MetalLB configuration", waitErr) + } + return nil } From 920bc949f3d246b26d623a573eab11d7032d7678 Mon Sep 17 00:00:00 2001 From: Daniel Lipovetsky Date: Wed, 3 Jul 2024 14:58:05 -0700 Subject: [PATCH 2/2] fixup! feat: Apply MetalLB configuration to remote cluster Wait up to 30s. In any case, we expect the context to be cancelled at 30s --- .../generic/lifecycle/serviceloadbalancer/metallb/handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb/handler.go b/pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb/handler.go index 4c1f3b7f8..d1e7bafd4 100644 --- a/pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb/handler.go +++ b/pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb/handler.go @@ -172,7 +172,7 @@ func (n *MetalLB) Apply( return false, nil }, Interval: 5 * time.Second, - Timeout: 10 * time.Second, + Timeout: 30 * time.Second, }, ); err != nil { return fmt.Errorf("failed to wait for MetalLB to deploy: %w", err) @@ -197,7 +197,7 @@ func (n *MetalLB) Apply( if waitErr := kwait.PollUntilContextTimeout( ctx, 5*time.Second, - 10*time.Second, + 30*time.Second, true, func(ctx context.Context) (done bool, err error) { for i := range cos {