Instrument http add on operator

.github/workflows/linkinator.yaml

@@ -23,4 +23,4 @@ jobs:
           markdown: true
           concurrency: 1
           retry: true
-          linksToSkip: "https://github.com/kedacore/http-add-on/pkgs/container/http-add-on-interceptor, https://github.com/kedacore/http-add-on/pkgs/container/http-add-on-operator, https://github.com/kedacore/http-add-on/pkgs/container/http-add-on-scaler,http://opentelemetry-collector.open-telemetry-system:4318,http://opentelemetry-collector.open-telemetry-system:4318/v1/traces, https://www.gnu.org/software/make/"
+          linksToSkip: "https://github.com/kedacore/http-add-on/pkgs/container/http-add-on-interceptor, https://github.com/kedacore/http-add-on/pkgs/container/http-add-on-operator, https://github.com/kedacore/http-add-on/pkgs/container/http-add-on-scaler,http://opentelemetry-collector.open-telemetry-system:4318,http://opentelemetry-collector.open-telemetry-system:4318/v1/traces, https://www.gnu.org/software/make/, http://opentelemetry-collector.open-telemetry-system:4317"

CHANGELOG.md

@@ -34,6 +34,7 @@ This changelog keeps track of work items that have been completed and are ready
 ### Improvements
 
 - **Interceptor**: Support HTTPScaledObject scoped timeout ([#813](https://github.com/kedacore/http-add-on/issues/813))
+- **General**: Add prometehus and otel instrumentation for the operator ([#965](https://github.com/kedacore/http-add-on/issues/965))
 
 ### Fixes
 

Makefile

@@ -221,6 +221,9 @@ deploy: manifests kustomize ## Deploy to the K8s cluster specified in ~/.kube/co
 	cd config/operator && \
 	$(KUSTOMIZE) edit set image ghcr.io/kedacore/http-add-on-operator=${IMAGE_OPERATOR_VERSIONED_TAG}
 
+	cd config/operator && \
+	$(KUSTOMIZE) edit add patch --path e2e-test/otel/deployment.yaml --group apps --kind Deployment --name operator --version v1
+
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
 
 undeploy:

config/operator/e2e-test/otel/deployment.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: operator
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+      - name: operator
+        env:
+        - name: OTEL_PROM_EXPORTER_ENABLED
+          value: "true"
+        - name: OTEL_PROM_EXPORTER_PORT
+          value: "8080"
+        - name: OTEL_EXPORTER_OTLP_METRICS_ENABLED
+          value: "true"
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: "http://opentelemetry-collector.open-telemetry-system:4318"
+        - name: OTEL_METRIC_EXPORT_INTERVAL
+          value: "1"
+        - name: OTEL_EXPORTER_OTLP_PROTOCOL
+          value: "http"

config/operator/e2e-test/otel/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- deployment.yaml

config/operator/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
 - role.yaml
 - role_binding.yaml
 - service_account.yaml
+- metrics.service.yaml
 labels:
 - includeSelectors: true
   includeTemplates: true

config/operator/metrics.service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: operator-metrics
+spec:
+  type: ClusterIP
+  ports:
+  - name: metrics
+    protocol: TCP
+    port: 8080
+    targetPort: metrics

docs/faq.md

@@ -22,8 +22,8 @@ However, Osiris and KEDA-HTTP differ in several ways:
 Knative Serving and KEDA-HTTP both have core support for autoscaling, including scale-to-zero of compute workloads. KEDA-HTTP is focused solely on deploying production-grade autoscaling HTTP applications, while Knative builds in additional functionality:
 
 - Pure [event-based workloads](https://knative.dev/docs/eventing/). [KEDA core](https://github.com/kedacore/keda), without KEDA-HTTP, can support such workloads natively.
-- Complex deployment strategies like [blue-green](https://knative.dev/docs/serving/samples/blue-green-deployment/).
-- Supporting other autoscaling mechanisms beyond the built-in [HPA](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/), such as the [Knative Pod Autoscaler (KPA)](https://knative.dev/docs/serving/autoscaling/autoscaling-concepts/#knative-pod-autoscaler-kpa).
+- Complex deployment strategies like [blue-green](https://knative.dev/docs/serving/traffic-management/#routing-and-managing-traffic-with-bluegreen-deployment).
+- Supporting other autoscaling mechanisms beyond the built-in [HPA](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/), such as the [Knative Pod Autoscaler (KPA)](https://knative.dev/docs/serving/autoscaling/autoscaler-types/#knative-pod-autoscaler-kpa).
 
 Additionally, Knative supports a service mesh, while KEDA-HTTP does not out of the box (support for that is forthcoming).
 

docs/install.md

@@ -52,7 +52,7 @@ There are a few values that you can pass to the above `helm install` command by
 helm install http-add-on kedacore/keda-add-ons-http --create-namespace --namespace ${NAMESPACE} --set images.tag=canary
 ```
 
-For an exhaustive list of configuration options, see the official HTTP Add-on chart [values.yaml file](https://github.com/kedacore/charts/blob/master/http-add-on/values.yaml).
+For an exhaustive list of configuration options, see the official HTTP Add-on chart [values.yaml file](https://github.com/kedacore/charts/blob/main/http-add-on/values.yaml).
 
 ### A Note for Developers and Local Cluster Users
 

docs/operate.md

@@ -9,7 +9,7 @@ There are currently 2 supported methods for exposing metrics from the intercepto
 ### Configuring the Prometheus compatible metrics endpoint
 When configured, the interceptor proxy can expose metrics on a Prometheus compatible endpoint.
 
-This endpoint can be enabled by setting the `OTEL_PROM_EXPORTER_ENABLED` environment variable to `true` on the interceptor deployment (`true` by default) and by setting `OTEL_PROM_EXPORTER_PORT` to an unused port for the endpoint to be made avaialble on (`2223` by default).
+This endpoint can be enabled by setting the `OTEL_PROM_EXPORTER_ENABLED` environment variable to `true` on the interceptor deployment (`true` by default) and by setting `OTEL_PROM_EXPORTER_PORT` to an unused port for the endpoint to be made available on (`2223` by default).
 
 ### Configuring the OTEL HTTP exporter
 When configured, the interceptor proxy can export metrics to a OTEL HTTP collector.
@@ -60,3 +60,29 @@ Optional variables
 `OTEL_EXPORTER_OTLP_TRACES_TIMEOUT` - The batcher timeout in seconds to send batch of data points (`5` by default)
 
 ### Configuring Service Failover
+
+# Configuring metrics for the KEDA HTTP Add-on Operator
+
+### Exportable metrics:
+* **keda_http_scaled_object_total** - the number of http_scaled_objects  
+
+There are currently 2 supported methods for exposing metrics from the operator - via a Prometheus compatible metrics endpoint or by pushing metrics to a OTEL HTTP collector.
+
+### Configuring the Prometheus compatible metrics endpoint
+When configured, the operator can expose metrics on a Prometheus compatible endpoint.
+
+This endpoint can be enabled by setting the `OTEL_PROM_EXPORTER_ENABLED` environment variable to `true` on the operator deployment (`true` by default) and by setting `OTEL_PROM_EXPORTER_PORT` to an unused port for the endpoint to be made available on (`8080` by default).
+
+### Configuring the OTEL HTTP exporter
+
+When configured, the operator can export metrics to a OTEL HTTP collector.
+
+The OTEL exporter can be enabled by setting the `OTEL_EXPORTER_OTLP_METRICS_ENABLED` environment variable to `true` on the operator deployment (`false` by default). When enabled, the `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable must also be configured so the exporter knows what collector to send the metrics to (e.g. http://opentelemetry-collector.open-telemetry-system:4318).
+
+If you need to provide any headers such as authentication details in order to utilise your OTEL collector you can add them into the `OTEL_EXPORTER_OTLP_HEADERS` environment variable. The frequency at which the metrics are exported can be configured by setting `OTEL_METRIC_EXPORT_INTERVAL` to the number of seconds you require between each export interval (`30` by default).
+
+The `OTEL_EXPORTER_OTLP_PROTOCOL` defaults to `http`
+
+### Configuring the OTEL GRPC exporter
+
+Please note that using `OTEL_EXPORTER_OTLP_PROTOCOL` will allows you to set it up to `grpc` to connect to otel collector. Also `OTEL_EXPORTER_OTLP_ENDPOINT` should be set to the right endpoint (eg: http://opentelemetry-collector.open-telemetry-system:4317)

go.mod

@@ -14,6 +14,7 @@ require (
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0
 	go.opentelemetry.io/contrib/propagators/b3 v1.36.0
 	go.opentelemetry.io/otel v1.36.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0
@@ -56,6 +57,7 @@ replace (
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.2 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect

go.sum

@@ -4,6 +4,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
+github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8=
 github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -165,6 +167,8 @@ go.opentelemetry.io/contrib/propagators/b3 v1.36.0 h1:xrAb/G80z/l5JL6XlmUMSD1i6W
 go.opentelemetry.io/contrib/propagators/b3 v1.36.0/go.mod h1:UREJtqioFu5awNaCR8aEx7MfJROFlAWb6lPaJFbHaG0=
 go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg=
 go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0 h1:QcFwRrZLc82r8wODjvyCbP7Ifp3UANaBSmhDSFjnqSc=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0/go.mod h1:CXIWhUomyWBG/oY2/r/kLp6K/cmx9e/7DLpBuuGdLCA=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0 h1:gAU726w9J8fwr4qRDqu1GYMNNs4gXrU+Pv20/N1UpB4=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0/go.mod h1:RboSDkp7N292rgu+T0MgVt2qgFGu6qa1RpZDOtpL76w=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0=

operator/controllers/http/config/config.go

@@ -90,3 +90,23 @@ func NewExternalScalerFromEnv() (*ExternalScaler, error) {
 		Port:        port,
 	}, nil
 }
+
+// Metrics is the configuration for configuring metrics in the operator.
+type Metrics struct {
+	// Sets whether or not to enable the Prometheus metrics exporter
+	OtelPrometheusExporterEnabled bool `envconfig:"OTEL_PROM_EXPORTER_ENABLED" default:"true"`
+	// Sets the port which the Prometheus compatible metrics endpoint should be served on
+	OtelPrometheusExporterPort int `envconfig:"OTEL_PROM_EXPORTER_PORT" default:"8080"`
+	// Sets whether or not to enable the OTEL metrics exporter
+	OtelHTTPExporterEnabled bool `envconfig:"OTEL_EXPORTER_OTLP_METRICS_ENABLED" default:"false"`
+	// Sets OTEL metrics exporter protocol
+	OtelExporterProtocol string `envconfig:"OTEL_EXPORTER_OTLP_PROTOCOL" default:"http"`
+}
+
+// Parse parses standard configs using envconfig and returns a pointer to the
+// newly created config. Returns nil and a non-nil error if parsing failed
+func MustParseMetrics() *Metrics {
+	ret := new(Metrics)
+	envconfig.MustProcess("", ret)
+	return ret
+}

operator/controllers/http/httpscaledobject_controller.go

@@ -19,6 +19,7 @@ package http
 import (
 	"context"
 	"fmt"
+	"sync"
 	"time"
 
 	kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
@@ -34,6 +35,16 @@ import (
 	httpv1alpha1 "github.com/kedacore/http-add-on/operator/apis/http/v1alpha1"
 	"github.com/kedacore/http-add-on/operator/controllers/http/config"
 	"github.com/kedacore/http-add-on/operator/controllers/util"
+	"github.com/kedacore/http-add-on/operator/metrics"
+)
+
+type httpScaledObjectMetricsData struct {
+	namespace string
+}
+
+var (
+	httpScaledObjectPromMetricsMap  map[string]httpScaledObjectMetricsData
+	httpScaledObjectPromMetricsLock *sync.Mutex
 )
 
 // HTTPScaledObjectReconciler reconciles a HTTPScaledObject object
@@ -48,6 +59,11 @@ type HTTPScaledObjectReconciler struct {
 	BaseConfig           config.Base
 }
 
+func init() {
+	httpScaledObjectPromMetricsMap = make(map[string]httpScaledObjectMetricsData)
+	httpScaledObjectPromMetricsLock = &sync.Mutex{}
+}
+
 // +kubebuilder:rbac:groups=http.keda.sh,resources=httpscaledobjects,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=http.keda.sh,resources=httpscaledobjects/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=http.keda.sh,resources=httpscaledobjects/finalizers,verbs=update
@@ -78,6 +94,7 @@ func (r *HTTPScaledObjectReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	}
 
 	if httpso.GetDeletionTimestamp() != nil {
+		r.updatePromMetricsOnDelete(ctx, httpso)
 		return ctrl.Result{}, finalizeScaledObject(ctx, logger, r.Client, httpso)
 	}
 
@@ -139,6 +156,7 @@ func (r *HTTPScaledObjectReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	)
 
 	// success reconciling
+	r.updatePromMetrics(ctx, httpso)
 	logger.Info("Reconcile success")
 	return ctrl.Result{}, nil
 }
@@ -162,3 +180,35 @@ func (r *HTTPScaledObjectReconciler) SetupWithManager(mgr ctrl.Manager) error {
 			))).
 		Complete(r)
 }
+
+func (r *HTTPScaledObjectReconciler) updatePromMetrics(ctx context.Context, scaledObject *httpv1alpha1.HTTPScaledObject) {
+	httpScaledObjectPromMetricsLock.Lock()
+	defer httpScaledObjectPromMetricsLock.Unlock()
+
+	namespacedName := client.ObjectKeyFromObject(scaledObject).String()
+	metricsData, ok := httpScaledObjectPromMetricsMap[namespacedName]
+	if ok {
+		metrics.RecordDeleteHTTPScaledObjectCount(scaledObject.Namespace)
+	}
+	metricsData.namespace = scaledObject.Namespace
+
+	logger := log.FromContext(ctx, "updatePromMetrics", namespacedName)
+	logger.Info("updatePromMetrics")
+	metrics.RecordHTTPScaledObjectCount(scaledObject.Namespace)
+
+	httpScaledObjectPromMetricsMap[namespacedName] = metricsData
+}
+
+func (r *HTTPScaledObjectReconciler) updatePromMetricsOnDelete(ctx context.Context, scaledObject *httpv1alpha1.HTTPScaledObject) {
+	httpScaledObjectPromMetricsLock.Lock()
+	defer httpScaledObjectPromMetricsLock.Unlock()
+
+	namespacedName := scaledObject.Name + scaledObject.Namespace
+	logger := log.FromContext(ctx, "updatePromMetricsOnDelete", namespacedName)
+	logger.Info("updatePromMetricsOnDelete")
+
+	if _, ok := httpScaledObjectPromMetricsMap[namespacedName]; ok {
+		metrics.RecordDeleteHTTPScaledObjectCount(scaledObject.Namespace)
+	}
+	delete(httpScaledObjectPromMetricsMap, namespacedName)
+}

operator/main.go

@@ -20,6 +20,11 @@ import (
 	"flag"
 	"os"
 
+	httpv1alpha1 "github.com/kedacore/http-add-on/operator/apis/http/v1alpha1"
+	httpcontrollers "github.com/kedacore/http-add-on/operator/controllers/http"
+	"github.com/kedacore/http-add-on/operator/controllers/http/config"
+	"github.com/kedacore/http-add-on/operator/metrics"
+	"github.com/kedacore/http-add-on/pkg/util"
 	kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -30,10 +35,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
-
-	httpv1alpha1 "github.com/kedacore/http-add-on/operator/apis/http/v1alpha1"
-	httpcontrollers "github.com/kedacore/http-add-on/operator/controllers/http"
-	"github.com/kedacore/http-add-on/operator/controllers/http/config"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -69,7 +70,9 @@ func main() {
 	}
 	opts.BindFlags(flag.CommandLine)
 	flag.Parse()
-
+	metricsCfg := config.MustParseMetrics()
+	// setup the configured metrics collectors
+	metrics.NewMetricsCollectors(metricsCfg)
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
 	externalScalerCfg, err := config.NewExternalScalerFromEnv()
@@ -112,6 +115,8 @@ func main() {
 		os.Exit(1)
 	}
 
+	ctx := ctrl.SetupSignalHandler()
+	ctx = util.ContextWithLogger(ctx, ctrl.Log)
 	if err = (&httpcontrollers.HTTPScaledObjectReconciler{
 		Client: mgr.GetClient(),
 		Scheme: mgr.GetScheme(),
@@ -134,7 +139,7 @@ func main() {
 	}
 
 	setupLog.Info("starting manager")
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+	if err := mgr.Start(ctx); err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}

operator/metrics/metricscollector.go

@@ -0,0 +1,44 @@
+package metrics
+
+import (
+	"go.opentelemetry.io/otel/exporters/prometheus"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
+
+	"github.com/kedacore/http-add-on/operator/controllers/http/config"
+)
+
+var (
+	collectors []Collector
+)
+
+const meterName = "keda-http-add-on-operator"
+
+type Collector interface {
+	RecordHTTPScaledObjectCount(namespace string)
+	RecordDeleteHTTPScaledObjectCount(namespace string)
+}
+
+func NewMetricsCollectors(metricsConfig *config.Metrics) {
+	if metricsConfig.OtelPrometheusExporterEnabled {
+		options := prometheus.WithRegisterer(ctrlmetrics.Registry)
+		promometrics := NewPrometheusMetrics(options)
+		collectors = append(collectors, promometrics)
+	}
+
+	if metricsConfig.OtelHTTPExporterEnabled {
+		otelhttpmetrics := NewOtelMetrics()
+		collectors = append(collectors, otelhttpmetrics)
+	}
+}
+
+func RecordHTTPScaledObjectCount(namespace string) {
+	for _, collector := range collectors {
+		collector.RecordHTTPScaledObjectCount(namespace)
+	}
+}
+
+func RecordDeleteHTTPScaledObjectCount(namespace string) {
+	for _, collector := range collectors {
+		collector.RecordDeleteHTTPScaledObjectCount(namespace)
+	}
+}