feat: otel telemetry collection and dashboard

configs/gateway.config.ts

@@ -0,0 +1,17 @@
+import { defineConfig } from '@graphql-hive/gateway';
+import { hiveTracingSetup } from '@graphql-hive/plugin-opentelemetry/setup';
+import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks'; // install
+
+hiveTracingSetup({
+  contextManager: new AsyncLocalStorageContextManager(),
+  target: process.env.HIVE_TRACING_TARGET!,
+  accessToken: process.env.HIVE_TRACING_ACCESS_TOKEN!,
+  // optional, for self-hosting
+  endpoint: process.env.HIVE_TRACING_ENDPOINT!,
+});
+
+export const gatewayConfig = defineConfig({
+  openTelemetry: {
+    traces: true,
+  },
+});

deployment/index.ts

@@ -14,6 +14,7 @@ import { configureGithubApp } from './services/github';
 import { deployGraphQL } from './services/graphql';
 import { deployKafka } from './services/kafka';
 import { deployObservability } from './services/observability';
+import { deployOTELCollector } from './services/otel-collector';
 import { deploySchemaPolicy } from './services/policy';
 import { deployPostgres } from './services/postgres';
 import { deployProxy } from './services/proxy';
@@ -278,6 +279,15 @@ if (hiveAppPersistedDocumentsAbsolutePath && RUN_PUBLISH_COMMANDS) {
   });
 }
 
+const otelCollector = deployOTELCollector({
+  environment,
+  graphql,
+  dbMigrations,
+  clickhouse,
+  image: docker.factory.getImageId('otel-collector', imagesTag),
+  docker,
+});
+
 const app = deployApp({
   environment,
   graphql,
@@ -306,6 +316,7 @@ const proxy = deployProxy({
   usage,
   environment,
   publicGraphQLAPIGateway,
+  otelCollector,
 });
 
 deployCloudFlareSecurityTransform({
@@ -332,4 +343,5 @@ export const schemaApiServiceId = schema.service.id;
 export const webhooksApiServiceId = webhooks.service.id;
 
 export const appId = app.deployment.id;
+export const otelCollectorId = otelCollector.deployment.id;
 export const publicIp = proxy.get()!.status.loadBalancer.ingress[0].ip;

deployment/services/environment.ts

@@ -79,6 +79,11 @@ export function prepareEnvironment(input: {
         cpuLimit: isProduction ? '512m' : '150m',
         memoryLimit: isProduction ? '1000Mi' : '300Mi',
       },
+      tracingCollector: {
+        cpuLimit: isProduction ? '1000m' : '100m',
+        memoryLimit: isProduction ? '2000Mi' : '200Mi',
+        maxReplicas: isProduction || isStaging ? 3 : 1,
+      },
     },
   };
 }

deployment/services/otel-collector.ts

@@ -0,0 +1,60 @@
+import { serviceLocalEndpoint } from '../utils/local-endpoint';
+import { ServiceDeployment } from '../utils/service-deployment';
+import { Clickhouse } from './clickhouse';
+import { DbMigrations } from './db-migrations';
+import { Docker } from './docker';
+import { Environment } from './environment';
+import { GraphQL } from './graphql';
+
+export type OTELCollector = ReturnType<typeof deployOTELCollector>;
+
+export function deployOTELCollector(args: {
+  image: string;
+  environment: Environment;
+  docker: Docker;
+  clickhouse: Clickhouse;
+  dbMigrations: DbMigrations;
+  graphql: GraphQL;
+}) {
+  return new ServiceDeployment(
+    'otel-collector',
+    {
+      image: args.image,
+      imagePullSecret: args.docker.secret,
+      env: {
+        ...args.environment.envVars,
+        HIVE_OTEL_AUTH_ENDPOINT: serviceLocalEndpoint(args.graphql.service).apply(
+          value => value + '/otel-auth',
+        ),
+      },
+      /**
+       * We are using the healthcheck extension.
+       * https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/healthcheckextension
+       */
+      probePort: 13133,
+      readinessProbe: '/',
+      livenessProbe: '/',
+      startupProbe: '/',
+      exposesMetrics: true,
+      replicas: args.environment.podsConfig.tracingCollector.maxReplicas,
+      pdb: true,
+      availabilityOnEveryNode: true,
+      port: 4318,
+      memoryLimit: args.environment.podsConfig.tracingCollector.memoryLimit,
+      autoScaling: {
+        maxReplicas: args.environment.podsConfig.tracingCollector.maxReplicas,
+        cpu: {
+          limit: args.environment.podsConfig.tracingCollector.cpuLimit,
+          cpuAverageToScale: 80,
+        },
+      },
+    },
+    [args.clickhouse.deployment, args.clickhouse.service, args.dbMigrations],
+  )
+    .withSecret('CLICKHOUSE_HOST', args.clickhouse.secret, 'host')
+    .withSecret('CLICKHOUSE_PORT', args.clickhouse.secret, 'port')
+    .withSecret('CLICKHOUSE_USERNAME', args.clickhouse.secret, 'username')
+    .withSecret('CLICKHOUSE_PASSWORD', args.clickhouse.secret, 'password')
+    .withSecret('CLICKHOUSE_PROTOCOL', args.clickhouse.secret, 'protocol')
+    .deploy();
+}

deployment/services/proxy.ts

@@ -5,6 +5,7 @@ import { App } from './app';
 import { Environment } from './environment';
 import { GraphQL } from './graphql';
 import { Observability } from './observability';
+import { OTELCollector } from './otel-collector';
 import { type PublicGraphQLAPIGateway } from './public-graphql-api-gateway';
 import { Usage } from './usage';
 
@@ -15,13 +16,15 @@ export function deployProxy({
   environment,
   observability,
   publicGraphQLAPIGateway,
+  otelCollector,
 }: {
   observability: Observability;
   environment: Environment;
   graphql: GraphQL;
   app: App;
   usage: Usage;
   publicGraphQLAPIGateway: PublicGraphQLAPIGateway;
+  otelCollector: OTELCollector;
 }) {
   const { tlsIssueName } = new CertManager().deployCertManagerAndIssuer();
   const commonConfig = new pulumi.Config('common');
@@ -113,5 +116,13 @@ export function deployProxy({
         requestTimeout: '60s',
         retriable: true,
       },
+      {
+        name: 'otel-traces',
+        path: '/otel/v1/traces',
+        customRewrite: '/v1/traces',
+        service: otelCollector.service,
+        requestTimeout: '60s',
+        retriable: true,
+      },
     ]);
 }

deployment/utils/service-deployment.ts

@@ -40,6 +40,8 @@ export class ServiceDeployment {
       args?: kx.types.Container['args'];
       image: string;
       port?: number;
+      /** Port to use for liveness, startup and readiness probes. */
+      probePort?: number;
       serviceAccountName?: pulumi.Output<string>;
       livenessProbe?: string | ProbeConfig;
       readinessProbe?: string | ProbeConfig;
@@ -107,6 +109,7 @@ export class ServiceDeployment {
 
   createPod(asJob: boolean) {
     const port = this.options.port || 3000;
+    const probePort = this.options.probePort ?? port;
     const additionalEnv: any[] = normalizeEnv(this.options.env);
     const secretsEnv: any[] = normalizeEnvSecrets(this.envSecrets);
 
@@ -125,14 +128,14 @@ export class ServiceDeployment {
               timeoutSeconds: 5,
               httpGet: {
                 path: this.options.livenessProbe,
-                port,
+                port: probePort,
               },
             }
           : {
               ...this.options.livenessProbe,
               httpGet: {
                 path: this.options.livenessProbe.endpoint,
-                port,
+                port: probePort,
               },
             };
     }
@@ -147,14 +150,14 @@ export class ServiceDeployment {
               timeoutSeconds: 5,
               httpGet: {
                 path: this.options.readinessProbe,
-                port,
+                port: probePort,
               },
             }
           : {
               ...this.options.readinessProbe,
               httpGet: {
                 path: this.options.readinessProbe.endpoint,
-                port,
+                port: probePort,
               },
             };
     }
@@ -169,14 +172,14 @@ export class ServiceDeployment {
               timeoutSeconds: 10,
               httpGet: {
                 path: this.options.startupProbe,
-                port,
+                port: probePort,
               },
             }
           : {
               ...this.options.startupProbe,
               httpGet: {
                 path: this.options.startupProbe.endpoint,
-                port,
+                port: probePort,
               },
             };
     }

docker/configs/otel-collector/builder-config.yaml

@@ -0,0 +1,30 @@
+dist:
+  version: 0.122.0
+  name: otelcol-custom
+  description: Custom OTel Collector distribution
+  output_path: ./otelcol-custom
+
+receivers:
+  - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.122.0
+
+processors:
+  - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.122.0
+  - gomod: go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.122.0
+  - gomod:
+      github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor
+      v0.122.0
+  - gomod:
+      github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.122.0
+
+exporters:
+  - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.122.0
+  - gomod:
+      github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter v0.122.0
+
+extensions:
+  - gomod:
+      github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension
+      v0.122.0
+  - gomod: github.com/graphql-hive/console/docker/configs/otel-collector/extension-hiveauth v0.0.0
+    path: ./extension-hiveauth
+    name: hiveauthextension # when using local extensions, package name is required, otherwise you get "missing import path"

docker/configs/otel-collector/config.yaml

@@ -0,0 +1,79 @@
+extensions:
+  hiveauth:
+    endpoint: ${HIVE_OTEL_AUTH_ENDPOINT}
+  health_check:
+    endpoint: '0.0.0.0:13133'
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        include_metadata: true
+        endpoint: '0.0.0.0:4317'
+        auth:
+          authenticator: hiveauth
+      http:
+        cors:
+          allowed_origins: ['*']
+          allowed_headers: ['*']
+        include_metadata: true
+        endpoint: '0.0.0.0:4318'
+        auth:
+          authenticator: hiveauth
+processors:
+  batch:
+    timeout: 5s
+    send_batch_size: 10000
+  attributes:
+    actions:
+      - key: hive.target_id
+        from_context: auth.targetId
+        action: insert
+  memory_limiter:
+    check_interval: 1s
+    limit_percentage: 80
+    spike_limit_percentage: 20
+exporters:
+  debug:
+    verbosity: detailed
+    sampling_initial: 5
+    sampling_thereafter: 200
+  clickhouse:
+    endpoint: ${CLICKHOUSE_PROTOCOL}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT}?dial_timeout=10s&compress=lz4&async_insert=1
+    database: default
+    async_insert: true
+    username: ${CLICKHOUSE_USERNAME}
+    password: ${CLICKHOUSE_PASSWORD}
+    create_schema: false
+    ttl: 720h
+    compress: lz4
+    logs_table_name: otel_logs
+    traces_table_name: otel_traces
+    metrics_table_name: otel_metrics
+    timeout: 5s
+    retry_on_failure:
+      enabled: true
+      initial_interval: 5s
+      max_interval: 30s
+      max_elapsed_time: 300s
+service:
+  extensions:
+    - hiveauth
+    - health_check
+  telemetry:
+    logs:
+      level: INFO
+      encoding: json
+      output_paths: ['stdout']
+      error_output_paths: ['stderr']
+    metrics:
+      address: '0.0.0.0:10254'
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors:
+        - memory_limiter
+        - attributes
+        - batch
+      exporters:
+        - clickhouse
+        # - debug

docker/configs/otel-collector/extension-hiveauth/config.go

@@ -0,0 +1,28 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package hiveauthextension // import "github.com/graphql-hive/console/docker/configs/otel-collector/extension-hiveauth"
+
+import (
+	"errors"
+	"time"
+)
+
+type Config struct {
+	// Endpoint is the address of the authentication server
+	Endpoint string `mapstructure:"endpoint"`
+	// Timeout is the timeout for the HTTP request to the auth service
+	Timeout time.Duration `mapstructure:"timeout"`
+}
+
+func (cfg *Config) Validate() error {
+	if cfg.Endpoint == "" {
+		return errors.New("missing endpoint")
+	}
+
+	if cfg.Timeout <= 0 {
+		return errors.New("timeout must be a positive value")
+	}
+
+	return nil
+}

docker/configs/otel-collector/extension-hiveauth/doc.go

@@ -0,0 +1,7 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+//go:generate mdatagen metadata.yaml
+
+// Package hiveauthextension accepts HTTP requests and forwards them to an external authentication service.
+package hiveauthextension // import "github.com/graphql-hive/console/docker/configs/otel-collector/extension-hiveauth"