feat: Add flag to limit the max number of targets added to an ELB

docs/deploy/configurations.md

@@ -117,6 +117,7 @@ Currently, you can set only 1 namespace to watch in this flag. See [this Kuberne
 | kube-ca-pem-filepath                                                               | string                          |                                            | The file path to the CA to validate webhook callers, when unspecified all webhook callers are permitted.                                                                      |
 | alb-gateway-max-concurrent-reconciles                                           | int                       | 3                                          | Maximum number of concurrently running reconcile loops for ALB gateways, if enabled                                                                                           |
 | nlb-gateway-max-concurrent-reconciles                                           | int                       | 3                                          | Maximum number of concurrently running reconcile loops for NLB gateways, if enabled                                                                                           |
+| max-targets-per-target-group                                                        | int                       | 0                                          | Maximum number of targets that will be added to a given Target Group. The default value of zero will leave the number of targets unlimited                                    |
 
 ### disable-ingress-class-annotation
 `--disable-ingress-class-annotation` controls whether to disable new usage of the `kubernetes.io/ingress.class` annotation.

main.go

@@ -185,7 +185,7 @@ func main() {
 	tgbResManager := targetgroupbinding.NewDefaultResourceManager(mgr.GetClient(), cloud.ELBV2(),
 		podInfoRepo, networkingManager, vpcInfoProvider, multiClusterManager, lbcMetricsCollector,
 		cloud.VpcID(), controllerCFG.FeatureGates.Enabled(config.EndpointsFailOpen), controllerCFG.EnableEndpointSlices,
-		mgr.GetEventRecorderFor("targetGroupBinding"), ctrl.Log)
+		mgr.GetEventRecorderFor("targetGroupBinding"), ctrl.Log, controllerCFG.MaxTargetsPerTargetGroup)
 	backendSGProvider := networking.NewBackendSGProvider(controllerCFG.ClusterName, controllerCFG.BackendSecurityGroup,
 		cloud.VpcID(), cloud.EC2(), mgr.GetClient(), controllerCFG.DefaultTags, nlbGatewayEnabled || albGatewayEnabled, ctrl.Log.WithName("backend-sg-provider"))
 	sgResolver := networking.NewDefaultSecurityGroupResolver(cloud.EC2(), cloud.VpcID())

pkg/config/controller_config.go

@@ -34,6 +34,7 @@ const (
 	flagBackendSecurityGroup                         = "backend-security-group"
 	flagEnableEndpointSlices                         = "enable-endpoint-slices"
 	flagDisableRestrictedSGRules                     = "disable-restricted-sg-rules"
+	flagMaxTargetsPerTargetGroup                     = "max-targets-per-target-group"
 	defaultLogLevel                                  = "info"
 	defaultMaxConcurrentReconciles                   = 3
 	defaultMaxExponentialBackoffDelay                = time.Second * 1000
@@ -43,6 +44,7 @@ const (
 	defaultEnableEndpointSlices                      = true
 	defaultDisableRestrictedSGRules                  = false
 	defaultLbStabilizationMonitorInterval            = time.Second * 120
+	defaultMaxTargetsPerTargetGroup                  = 0
 )
 
 var (
@@ -133,6 +135,9 @@ type ControllerConfig struct {
 	// LBStabilizationMonitorInterval specifies the duration of interval to monitor the load balancer state for stabilization
 	LBStabilizationMonitorInterval time.Duration
 
+	// MaxTargetsPerTargetGroup limits the number of targets that will be added to an ELB instance
+	MaxTargetsPerTargetGroup int
+
 	FeatureGates FeatureGates
 }
 
@@ -177,6 +182,8 @@ func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) {
 		"Disable the usage of restricted security group rules")
 	fs.StringToStringVar(&cfg.ServiceTargetENISGTags, flagServiceTargetENISGTags, nil,
 		"AWS Tags, in addition to cluster tags, for finding the target ENI security group to which to add inbound rules from NLBs")
+	fs.IntVar(&cfg.MaxTargetsPerTargetGroup, flagMaxTargetsPerTargetGroup, defaultMaxTargetsPerTargetGroup,
+		"Maximum number of targets that can be added to an ELB instance. Use this to prevent TargetGroup quotas being exceeded from blocking reconciliation.")
 	cfg.FeatureGates.BindFlags(fs)
 	cfg.AWSConfig.BindFlags(fs)
 	cfg.RuntimeConfig.BindFlags(fs)

pkg/targetgroupbinding/resource_manager.go

@@ -48,21 +48,23 @@ func NewDefaultResourceManager(k8sClient client.Client, elbv2Client services.ELB
 	podInfoRepo k8s.PodInfoRepo, networkingManager networking.NetworkingManager,
 	vpcInfoProvider networking.VPCInfoProvider, multiClusterManager MultiClusterManager, metricsCollector lbcmetrics.MetricCollector,
 	vpcID string, failOpenEnabled bool, endpointSliceEnabled bool,
-	eventRecorder record.EventRecorder, logger logr.Logger) *defaultResourceManager {
+	eventRecorder record.EventRecorder, logger logr.Logger, maxTargetsPerTargetGroup int) *defaultResourceManager {
+
 	targetsManager := NewCachedTargetsManager(elbv2Client, logger)
 	endpointResolver := backend.NewDefaultEndpointResolver(k8sClient, podInfoRepo, failOpenEnabled, endpointSliceEnabled, logger)
 	return &defaultResourceManager{
-		k8sClient:           k8sClient,
-		targetsManager:      targetsManager,
-		endpointResolver:    endpointResolver,
-		networkingManager:   networkingManager,
-		eventRecorder:       eventRecorder,
-		logger:              logger,
-		vpcID:               vpcID,
-		vpcInfoProvider:     vpcInfoProvider,
-		podInfoRepo:         podInfoRepo,
-		multiClusterManager: multiClusterManager,
-		metricsCollector:    metricsCollector,
+		k8sClient:                k8sClient,
+		targetsManager:           targetsManager,
+		endpointResolver:         endpointResolver,
+		networkingManager:        networkingManager,
+		eventRecorder:            eventRecorder,
+		logger:                   logger,
+		vpcID:                    vpcID,
+		vpcInfoProvider:          vpcInfoProvider,
+		podInfoRepo:              podInfoRepo,
+		maxTargetsPerTargetGroup: maxTargetsPerTargetGroup,
+		multiClusterManager:      multiClusterManager,
+		metricsCollector:         metricsCollector,
 
 		invalidVpcCache:    cache.NewExpiring(),
 		invalidVpcCacheTTL: defaultTargetsCacheTTL,
@@ -75,17 +77,18 @@ var _ ResourceManager = &defaultResourceManager{}
 
 // default implementation for ResourceManager.
 type defaultResourceManager struct {
-	k8sClient           client.Client
-	targetsManager      TargetsManager
-	endpointResolver    backend.EndpointResolver
-	networkingManager   networking.NetworkingManager
-	eventRecorder       record.EventRecorder
-	logger              logr.Logger
-	vpcInfoProvider     networking.VPCInfoProvider
-	podInfoRepo         k8s.PodInfoRepo
-	multiClusterManager MultiClusterManager
-	metricsCollector    lbcmetrics.MetricCollector
-	vpcID               string
+	k8sClient                client.Client
+	targetsManager           TargetsManager
+	endpointResolver         backend.EndpointResolver
+	networkingManager        networking.NetworkingManager
+	eventRecorder            record.EventRecorder
+	logger                   logr.Logger
+	vpcInfoProvider          networking.VPCInfoProvider
+	podInfoRepo              k8s.PodInfoRepo
+	maxTargetsPerTargetGroup int
+	multiClusterManager      MultiClusterManager
+	metricsCollector         lbcmetrics.MetricCollector
+	vpcID                    string
 
 	invalidVpcCache      *cache.Expiring
 	invalidVpcCacheTTL   time.Duration
@@ -240,6 +243,11 @@ func (m *defaultResourceManager) reconcileWithIPTargetType(ctx context.Context,
 			return "", "", false, ctrlerrors.NewErrorWithMetrics(controllerName, "update_tracked_ip_targets_error", err, m.metricsCollector)
 		}
 
+		if m.maxTargetsPerTargetGroup != 0 {
+			eligibleTargetsCount := m.getMaxNewTargets(len(unmatchedEndpoints), len(targets), tgbScopedLogger)
+			unmatchedEndpoints = unmatchedEndpoints[:eligibleTargetsCount]
+		}
+
 		if err := m.registerPodEndpoints(ctx, tgb, unmatchedEndpoints); err != nil {
 			return "", "", false, ctrlerrors.NewErrorWithMetrics(controllerName, "register_pod_endpoint_error", err, m.metricsCollector)
 		}
@@ -341,6 +349,11 @@ func (m *defaultResourceManager) reconcileWithInstanceTargetType(ctx context.Con
 			return "", "", false, ctrlerrors.NewErrorWithMetrics(controllerName, "update_tracked_instance_targets_error", err, m.metricsCollector)
 		}
 
+		if m.maxTargetsPerTargetGroup != 0 {
+			eligibleTargetsCount := m.getMaxNewTargets(len(unmatchedEndpoints), len(targets), tgbScopedLogger)
+			unmatchedEndpoints = unmatchedEndpoints[:eligibleTargetsCount]
+		}
+
 		if err := m.registerNodePortEndpoints(ctx, tgb, unmatchedEndpoints); err != nil {
 			return "", "", false, ctrlerrors.NewErrorWithMetrics(controllerName, "update_node_port_endpoints_error", err, m.metricsCollector)
 		}
@@ -803,3 +816,16 @@ func isVPCNotFoundError(err error) bool {
 	}
 	return false
 }
+
+func (m *defaultResourceManager) getMaxNewTargets(newTargetCount int, currentTargetCount int, tgbScopedLogger logr.Logger) (maxAdditions int) {
+	if newTargetCount+currentTargetCount > m.maxTargetsPerTargetGroup {
+		maxAdditions = m.maxTargetsPerTargetGroup - currentTargetCount
+		tgbScopedLogger.Info("Limiting target additions due to max-targets-per-instance configuration",
+			"currentTargets", currentTargetCount,
+			"maxTargetsPerTargetGroup", m.maxTargetsPerTargetGroup,
+			"proposedAdditions", newTargetCount)
+		return maxAdditions
+	}
+
+	return newTargetCount
+}