From 41407e60fb6c270c72e63abc97028ca28dc1d103 Mon Sep 17 00:00:00 2001 From: Paul Yu Date: Wed, 12 Apr 2023 04:18:03 +0000 Subject: [PATCH] refactor: removing hpa evaluation --- config/rbac/role.yaml | 8 -- .../carbonawarekedascaler_controller.go | 101 +++++------------- controllers/metrics.go | 18 ---- hack/grafana/Carbon Aware KEDA-Dashboard.json | 24 ----- 4 files changed, 24 insertions(+), 127 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index dc73aee..dea453d 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -23,14 +23,6 @@ rules: verbs: - create - patch -- apiGroups: - - autoscaling - resources: - - horizontalpodautoscalers - verbs: - - get - - list - - watch - apiGroups: - carbonaware.kubernetes.azure.com resources: diff --git a/controllers/carbonawarekedascaler_controller.go b/controllers/carbonawarekedascaler_controller.go index 32ab2a4..c7c3872 100644 --- a/controllers/carbonawarekedascaler_controller.go +++ b/controllers/carbonawarekedascaler_controller.go @@ -23,7 +23,6 @@ import ( "time" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" - autoscalingv2 "k8s.io/api/autoscaling/v2" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -51,7 +50,6 @@ type CarbonAwareKedaScalerReconciler struct { //+kubebuilder:rbac:groups=keda.sh,resources=scaledobjects,verbs=get;list;watch;update;patch //+kubebuilder:rbac:groups=keda.sh,resources=scaledjobs,verbs=get;list;watch;update;patch //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch -//+kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch //+kubebuilder:rbac:groups="",resources=events,verbs=create;patch // Reconcile is part of the main kubernetes reconciliation loop which aims to @@ -97,24 +95,21 @@ func (r *CarbonAwareKedaScalerReconciler) Reconcile(ctx context.Context, req ctr ReconcilesTotal.WithLabelValues(carbonAwareKedaScaler.Name).Inc() - // set the carbon forecast fetcher if it is not already set - if r.CarbonForecastFetcher == nil { - // if mock carbon forecast is enabled use the mock fetcher otherwise, use the configmap fetcher - if carbonAwareKedaScaler.Spec.CarbonIntensityForecastDataSource.MockCarbonForecast { - r.CarbonForecastFetcher = &CarbonForecastMockConfigMapFetcher{ - Client: r.Client, - } - r.Recorder.Event(carbonAwareKedaScaler, "Normal", "CarbonForecastSource", "Using mock carbon forecast") - } else if carbonAwareKedaScaler.Spec.CarbonIntensityForecastDataSource.LocalConfigMap != (carbonawarev1alpha1.LocalConfigMap{}) { - // fetch the carbon forecast from configmap - r.CarbonForecastFetcher = &CarbonForecastConfigMapFetcher{ - Client: r.Client, - ConfigMapName: carbonAwareKedaScaler.Spec.CarbonIntensityForecastDataSource.LocalConfigMap.Name, - ConfigMapNamespace: carbonAwareKedaScaler.Spec.CarbonIntensityForecastDataSource.LocalConfigMap.Namespace, - ConfigMapKey: carbonAwareKedaScaler.Spec.CarbonIntensityForecastDataSource.LocalConfigMap.Key, - } - r.Recorder.Event(carbonAwareKedaScaler, "Normal", "CarbonForecastSource", fmt.Sprintf("Using carbon forecast from configmap %s", carbonAwareKedaScaler.Spec.CarbonIntensityForecastDataSource.LocalConfigMap.Name)) + // if mock carbon forecast is enabled use the mock fetcher otherwise, use the configmap fetcher + if carbonAwareKedaScaler.Spec.CarbonIntensityForecastDataSource.MockCarbonForecast { + r.CarbonForecastFetcher = &CarbonForecastMockConfigMapFetcher{ + Client: r.Client, + } + r.Recorder.Event(carbonAwareKedaScaler, "Normal", "CarbonForecastSource", "Using mock carbon forecast") + } else if carbonAwareKedaScaler.Spec.CarbonIntensityForecastDataSource.LocalConfigMap != (carbonawarev1alpha1.LocalConfigMap{}) { + // fetch the carbon forecast from configmap + r.CarbonForecastFetcher = &CarbonForecastConfigMapFetcher{ + Client: r.Client, + ConfigMapName: carbonAwareKedaScaler.Spec.CarbonIntensityForecastDataSource.LocalConfigMap.Name, + ConfigMapNamespace: carbonAwareKedaScaler.Spec.CarbonIntensityForecastDataSource.LocalConfigMap.Namespace, + ConfigMapKey: carbonAwareKedaScaler.Spec.CarbonIntensityForecastDataSource.LocalConfigMap.Key, } + r.Recorder.Event(carbonAwareKedaScaler, "Normal", "CarbonForecastSource", fmt.Sprintf("Using carbon forecast from configmap %s", carbonAwareKedaScaler.Spec.CarbonIntensityForecastDataSource.LocalConfigMap.Name)) } // fetch the carbon forecast @@ -168,6 +163,16 @@ func (r *CarbonAwareKedaScalerReconciler) Reconcile(ctx context.Context, req ctr } } + // set to max replicas configured when eco mode is disabled + if ecoModeStatus.IsDisabled { + EcoModeOffMetric.WithLabelValues(carbonAwareKedaScaler.Name, "1").Inc() + logger.Info("eco mode disabled", "reason", ecoModeStatus.DisableReason) + r.Recorder.Event(carbonAwareKedaScaler, "Warning", "EcoModeDisabled", fmt.Sprintf("Eco mode disabled: %s", ecoModeStatus.DisableReason)) + maxReplicaCount = &carbonAwareKedaScaler.Spec.EcoModeOff.MaxReplicas + } else { + EcoModeOffMetric.WithLabelValues(carbonAwareKedaScaler.Name, "0").Inc() + } + // scale the keda target switch { case strings.Contains(string(carbonAwareKedaScaler.Spec.KedaTarget), "scaledobject"): @@ -184,47 +189,6 @@ func (r *CarbonAwareKedaScalerReconciler) Reconcile(ctx context.Context, req ctr return ctrl.Result{RequeueAfter: getRequeueDuration(now, requeueInterval)}, err } - // get the hpa associated with the scaledobject - hpa := &autoscalingv2.HorizontalPodAutoscaler{} - err = r.Get(ctx, types.NamespacedName{Name: scaledObject.Status.HpaName, Namespace: scaledObject.Namespace}, hpa) - if err != nil && errors.IsNotFound(err) { - logger.Error(err, "unable to find hpa") - setStatusCondition(carbonAwareKedaScaler, metav1.ConditionTrue, carbonawarev1alpha1.ReasonTargetNotFound, fmt.Sprintf("unable to find hpa: %v", err)) - return ctrl.Result{RequeueAfter: getRequeueDuration(now, requeueInterval)}, client.IgnoreNotFound(err) - } else if err != nil { - ReconcileErrorsTotal.WithLabelValues(carbonAwareKedaScaler.Name).Inc() - logger.Error(err, "failed to get hpa") - setStatusCondition(carbonAwareKedaScaler, metav1.ConditionTrue, carbonawarev1alpha1.ReasonTargetNotFound, fmt.Sprintf("failed to get hpa: %v", err)) - return ctrl.Result{RequeueAfter: getRequeueDuration(now, requeueInterval)}, err - } - - // eco mode is good but don't let performance suffer! if maxReplicaCount is less than what HPA desires then disable the carbon aware scaler - if maxReplicaCount != nil && *maxReplicaCount < hpa.Status.DesiredReplicas { - ecoModeStatus.IsDisabled = true - ecoModeStatus.DisableReason = fmt.Sprintf("Disabling carbon awareness since maxReplicaCount of %d is less than what HPA desires %d", *maxReplicaCount, hpa.Status.DesiredReplicas) - } - - // log the current and desired replicas - HpaCurrentReplicasMetric.WithLabelValues(carbonAwareKedaScaler.Name).Set(float64(hpa.Status.CurrentReplicas)) - HpaDesiredReplicasMetric.WithLabelValues(carbonAwareKedaScaler.Name).Set(float64(hpa.Status.DesiredReplicas)) - - // set to max replicas configured when eco mode is disabled - if ecoModeStatus.IsDisabled { - EcoModeOffMetric.WithLabelValues(carbonAwareKedaScaler.Name, "1").Inc() - logger.Info("eco mode disabled", "reason", ecoModeStatus.DisableReason) - r.Recorder.Event(carbonAwareKedaScaler, "Warning", "EcoModeDisabled", fmt.Sprintf("Eco mode disabled: %s", ecoModeStatus.DisableReason)) - maxReplicaCount = &carbonAwareKedaScaler.Spec.EcoModeOff.MaxReplicas - } else { - EcoModeOffMetric.WithLabelValues(carbonAwareKedaScaler.Name, "0").Inc() - } - - // get a fresh scaled object to avoid dirty writes - err := r.Get(ctx, types.NamespacedName{Name: scaledObject.Name, Namespace: scaledObject.Namespace}, scaledObject) - if err != nil { - setStatusCondition(carbonAwareKedaScaler, metav1.ConditionTrue, carbonawarev1alpha1.ReasonTargetFetchError, fmt.Sprintf("failed to get scaledobject: %v", err)) - return ctrl.Result{RequeueAfter: getRequeueDuration(now, requeueInterval)}, client.IgnoreNotFound(err) - } - // ovewrite the scaledobject.Spec.MaxReplicaCount with the max replica count for the current carbon rating scaledObject.Spec.MaxReplicaCount = maxReplicaCount @@ -256,23 +220,6 @@ func (r *CarbonAwareKedaScalerReconciler) Reconcile(ctx context.Context, req ctr return ctrl.Result{RequeueAfter: getRequeueDuration(now, requeueInterval)}, err } - // set to max replicas configured when eco mode is disabled - if ecoModeStatus.IsDisabled { - EcoModeOffMetric.WithLabelValues(carbonAwareKedaScaler.Name, "1").Inc() - logger.Info("eco mode disabled", "reason", ecoModeStatus.DisableReason) - r.Recorder.Event(carbonAwareKedaScaler, "Warning", "EcoModeDisabled", fmt.Sprintf("Eco mode disabled: %s", ecoModeStatus.DisableReason)) - maxReplicaCount = &carbonAwareKedaScaler.Spec.EcoModeOff.MaxReplicas - } else { - EcoModeOffMetric.WithLabelValues(carbonAwareKedaScaler.Name, "0").Inc() - } - - // get a fresh scaled job to avoid dirty writes - err := r.Get(ctx, types.NamespacedName{Name: scaledJob.Name, Namespace: scaledJob.Namespace}, scaledJob) - if err != nil { - setStatusCondition(carbonAwareKedaScaler, metav1.ConditionTrue, carbonawarev1alpha1.ReasonTargetFetchError, fmt.Sprintf("failed to get scaledjob: %v", err)) - return ctrl.Result{RequeueAfter: getRequeueDuration(now, requeueInterval)}, client.IgnoreNotFound(err) - } - // ovewrite the scaledobject.Spec.MaxReplicaCount with the max replica count for the current carbon rating scaledJob.Spec.MaxReplicaCount = maxReplicaCount diff --git a/controllers/metrics.go b/controllers/metrics.go index 66db1f8..fe2034e 100644 --- a/controllers/metrics.go +++ b/controllers/metrics.go @@ -46,22 +46,6 @@ var ( []string{"app"}, ) - HpaDesiredReplicasMetric = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "carbon_aware_keda_scaler_hpa_desired_replicas", - Help: "HPA desired replicas", - }, - []string{"app"}, - ) - - HpaCurrentReplicasMetric = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "carbon_aware_keda_scaler_hpa_current_replicas", - Help: "HPA current replicas", - }, - []string{"app"}, - ) - EcoModeOffMetric = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "carbon_aware_keda_scaler_eco_mode_off", @@ -78,7 +62,5 @@ func init() { metrics.Registry.MustRegister(CarbonIntensityMetric) metrics.Registry.MustRegister(DefaultMaxReplicasMetric) metrics.Registry.MustRegister(MaxReplicasMetric) - metrics.Registry.MustRegister(HpaDesiredReplicasMetric) - metrics.Registry.MustRegister(HpaCurrentReplicasMetric) metrics.Registry.MustRegister(EcoModeOffMetric) } diff --git a/hack/grafana/Carbon Aware KEDA-Dashboard.json b/hack/grafana/Carbon Aware KEDA-Dashboard.json index 3671fe0..96177f8 100644 --- a/hack/grafana/Carbon Aware KEDA-Dashboard.json +++ b/hack/grafana/Carbon Aware KEDA-Dashboard.json @@ -440,30 +440,6 @@ "range": true, "refId": "A" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "carbon_aware_keda_scaler_hpa_desired_replicas{app=\"word-processor-carbon-aware-keda-scaler\"}", - "hide": false, - "legendFormat": "HPA Desired Replicas", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "carbon_aware_keda_scaler_hpa_current_replicas{app=\"word-processor-carbon-aware-keda-scaler\"}", - "hide": false, - "legendFormat": "HPA Current Replicas", - "range": true, - "refId": "C" - }, { "datasource": { "type": "prometheus",