move from prom-client to OpenTelemetry metrics (#1811)

Signed-off-by: Jessica He <jhe@redhat.com>

.rhdh/docker/Dockerfile

@@ -379,6 +379,11 @@ ENV NO_PROXY='localhost,127.0.0.1'
 # The `--no-node-snapshot` node option enables the usage of the backstage scaffolder on nodejs 20
 # https://github.com/backstage/backstage/issues/20661
 
-ENTRYPOINT ["node", "packages/backend", "--no-node-snapshot", "--config", "app-config.yaml", "--config", "app-config.example.yaml", "--config", "app-config.example.production.yaml"]
+# Copy instrumentation needed for surfacing metrics
+COPY --chown=1001:1001 $EXTERNAL_SOURCE_NESTED/packages/backend/src/instrumentation.js ./
+# Remove write and execute permissions
+RUN chmod a=r ./instrumentation.js
+
+ENTRYPOINT ["node", "--require", "./instrumentation.js", "packages/backend", "--no-node-snapshot", "--config", "app-config.yaml", "--config", "app-config.example.yaml", "--config", "app-config.example.production.yaml"]
 
 # append Brew metadata here

docker/Dockerfile

@@ -296,6 +296,11 @@ ENV NO_PROXY='localhost,127.0.0.1'
 # The `--no-node-snapshot` node option enables the usage of the backstage scaffolder on nodejs 20
 # https://github.com/backstage/backstage/issues/20661
 
-ENTRYPOINT ["node", "packages/backend", "--no-node-snapshot", "--config", "app-config.yaml", "--config", "app-config.example.yaml", "--config", "app-config.example.production.yaml"]
+# Copy instrumentation needed for surfacing metrics
+COPY --chown=1001:1001 $EXTERNAL_SOURCE_NESTED/packages/backend/src/instrumentation.js ./
+# Remove write and execute permissions
+RUN chmod a=r ./instrumentation.js
+
+ENTRYPOINT ["node", "--require", "./instrumentation.js", "packages/backend", "--no-node-snapshot", "--config", "app-config.yaml", "--config", "app-config.example.yaml", "--config", "app-config.example.production.yaml"]
 
 # append Brew metadata here

docs/monitoring-and-logging.md

@@ -16,7 +16,7 @@ When deploying Backstage Showcase onto a kubernetes cluster with the [RHDH Helm
 
 ### Enabling Metrics Monitoring on Openshift
 
-To enable metrics monitoring on OpenShift, we need to create a `ServiceMonitor` resource in the OpenShift cluster that will be used by Prometheus to scrape metrics from your Backstage instance. For the metrics to be ingested by the built-in Prometheus instances in Openshift, please ensure you enabled [monitoring for user-defined projects](https://docs.openshift.com/container-platform/latest/monitoring/enabling-monitoring-for-user-defined-projects.html).
+To enable metrics monitoring on OpenShift, we need to create a `ServiceMonitor` resource in the OpenShift cluster that will be used by Prometheus to scrape metrics from your Backstage instance. For the metrics to be ingested by the built-in Prometheus instances in Openshift, please ensure you enabled [monitoring for user-defined projects](https://docs.openshift.com/container-platform/4.16/observability/monitoring/enabling-monitoring-for-user-defined-projects.html#enabling-monitoring-for-user-defined-projects).
 
 #### Helm deployment
 

packages/backend/package.json

@@ -8,7 +8,8 @@
     "role": "backend"
   },
   "scripts": {
-    "start": "backstage-cli package start",
+    "start": "backstage-cli package start --require ./src/instrumentation.js",
+    "start:prometheus": "docker run --mount type=bind,source=./prometheus.yml,destination=/etc/prometheus/prometheus.yml --publish published=9090,target=9090,protocol=tcp prom/prometheus",
     "build": "backstage-cli package build",
     "lint:check": "backstage-cli package lint",
     "lint:fix": "backstage-cli package lint --fix",
@@ -49,10 +50,14 @@
     "@internal/plugin-scalprum-backend": "*",
     "@janus-idp/backstage-plugin-audit-log-node": "1.7.0",
     "@janus-idp/backstage-scaffolder-backend-module-annotator": "2.2.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.50.2",
+    "@opentelemetry/exporter-prometheus": "^0.53.0",
+    "@opentelemetry/host-metrics": "^0.35.4",
+    "@opentelemetry/instrumentation-runtime-node": "^0.9.0",
+    "@opentelemetry/sdk-metrics": "^1.27.0",
+    "@opentelemetry/sdk-node": "^0.53.0",
     "app": "*",
-    "express-prom-bundle": "6.6.0",
     "global-agent": "3.0.0",
-    "prom-client": "15.1.3",
     "undici": "6.19.8",
     "winston": "3.14.2",
     "winston-daily-rotate-file": "5.0.0"

packages/backend/prometheus.yml

@@ -0,0 +1,21 @@
+# This is an example configuration file for Prometheus, that is set to scrape
+# the metrics endpoint exposed by the example backend.
+#
+# If you want to run a docker based Prometheus instance locally to try it out,
+# run the following command in the Backstage root:
+#
+# docker run --mount type=bind,source=./packages/backend/prometheus.yml,destination=/etc/prometheus/prometheus.yml --publish published=9090,target=9090,protocol=tcp prom/prometheus
+#
+# After this, you have a Prometheus instance on http://localhost:9090
+
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      # This target address specifically works on Docker for MacOS - you may
+      # need to change it to another host depending on your OS and Docker
+      # engine.
+      - targets: ['host.docker.internal:9464']

packages/backend/src/index.ts

@@ -9,7 +9,6 @@ import { CommonJSModuleLoader } from './loader';
 import { transports } from './logger';
 import {
   healthCheckPlugin,
-  metricsPlugin,
   pluginIDProviderService,
   rbacDynamicPluginsProvider,
 } from './modules';
@@ -44,7 +43,6 @@ backend.add(
   }),
 );
 
-backend.add(metricsPlugin);
 backend.add(healthCheckPlugin);
 
 backend.add(import('@backstage/plugin-app-backend/alpha'));

packages/backend/src/instrumentation.js

@@ -0,0 +1,30 @@
+const { NodeSDK } = require('@opentelemetry/sdk-node');
+const {
+  getNodeAutoInstrumentations,
+} = require('@opentelemetry/auto-instrumentations-node');
+const {
+  RuntimeNodeInstrumentation,
+} = require('@opentelemetry/instrumentation-runtime-node');
+const { PrometheusExporter } = require('@opentelemetry/exporter-prometheus');
+const { HostMetrics } = require('@opentelemetry/host-metrics');
+const { MeterProvider } = require('@opentelemetry/sdk-metrics');
+
+// Application metrics will be exported to localhost:9464/metrics
+const prometheusApplication = new PrometheusExporter();
+const sdk = new NodeSDK({
+  metricReader: prometheusApplication,
+  instrumentations: [
+    getNodeAutoInstrumentations(),
+    new RuntimeNodeInstrumentation(),
+  ],
+});
+
+sdk.start();
+
+// Host/process metrics will be exported to localhost:9463/metrics
+const prometheusHost = new PrometheusExporter({ port: 9463 });
+const meterProvider = new MeterProvider({
+  readers: [prometheusHost],
+});
+const hostMetrics = new HostMetrics({ meterProvider });
+hostMetrics.start();

packages/backend/src/modules/index.ts

@@ -1,4 +1,3 @@
 export * from './authProvidersModule';
 export * from './rbacDynamicPluginsModule';
 export * from './healthcheck';
-export * from './metrics';