Merge pull request #90 from stickeeuk/feat/delta-to-cumulative

feat: Delta to Cumulative

README.md

@@ -141,6 +141,56 @@ You can run tests on your own system by invoking Pest:
 ./vendor/bin/pest
 ```
 
+## OpenTelemetry Collector
+
+The Instrumentation package uses the OpenTelemetry Collector to export metrics.
+Data is sent from PHP using HTTP + Protobuf to the OpenTelemetry Collector, which is usually running on localhost or as a sidecar in Kubernetes.
+
+
+Due to PHP's shared-nothing architecture, we need to send Delta temporality metrics to the OpenTelemetry Collector, otherwise every
+PHP process (and every hit to a website) would need a unique data stream, which would not perform adequately.
+To get around this, we use Delta temporality and the OpenTelemetry Collector's Aggregation and DeltaToCumulative processors to aggregate metrics in memory before sending them to the exporter.
+Each collector is given a unique `service.instance.id` to allow them to be aggregated together later.
+
+The Aggregation processor is written by Stickee and is available in the [Stickee OpenTelemetry Collector](https://github.com/stickeeuk/opentelemetry-collector-contrib)
+repository on the `feature/aggregation-processor` branch.
+
+To update it, run the following commands based off the [Custom Collector documentation](https://opentelemetry.io/docs/collector/custom-collector/):
+
+```bash
+git clone git@github.com:stickeeuk/opentelemetry-collector-contrib.git
+cd opentelemetry-collector-contrib
+docker run --rm -it -v /$(pwd)://go/src golang:1.22-bookworm # Run a Go docker container
+
+# Inside the container
+apt update && apt install -y vim
+mkdir /builder
+cd /builder
+curl --proto '=https' --tlsv1.2 -fL -o ocb https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/cmd%2Fbuilder%2Fv0.113.0/ocb_0.113.0_linux_amd64
+chmod +x ocb
+
+# Paste in the contents of the `cmd/otelcontribcol/builder-config.yaml` file from the repository
+# but replace `../..` with `/go/src` and add `output_path: ./output` to the `dist:` section.
+vi builder-config.yaml
+
+./ocb --config builder-config.yaml
+cp ./output/otelcontribcol /go/src/bin/otelcontribcol_linux_amd64_stickee
+
+# Exit the container
+exit
+
+# Back on the host
+
+# Copy the binary to the repository
+cp bin/otelcontribcol_linux_amd64_stickee ../instrumentation/docker/opentelemetry-collector/contrib
+
+cd ../instrumentation/docker/opentelemetry-collector/contrib
+
+# Build and push the Docker image
+docker build -t ghcr.io/stickeeuk/opentelemetry-collector .
+docker push ghcr.io/stickeeuk/opentelemetry-collector
+```
+
 ## Contributions
 
 Contributions are welcome to all areas of the project, but please provide tests. Code style will be checked using

docker/opentelemetry-collector-contrib/Dockerfile

@@ -0,0 +1,12 @@
+# FROM otel/opentelemetry-collector-contrib:0.113.0-amd64
+FROM golang:1.22-bookworm
+
+COPY config.yaml /etc/otelcol-contrib/config.yaml
+# COPY --chmod=755 otelcontribcol_linux_amd64 /otelcol-contrib
+COPY --chmod=755 otelcontribcol_linux_amd64_stickee /otelcontribcol
+
+# ENTRYPOINT ["/otelcol-contrib"]
+ENTRYPOINT ["/otelcontribcol"]
+CMD ["--config", "/etc/otelcol-contrib/config.yaml"]
+
+EXPOSE 4317/tcp 4318/tcp 55678/tcp 55679/tcp

docker/opentelemetry-collector-contrib/config.yaml

@@ -0,0 +1,73 @@
+# To limit exposure to denial of service attacks, change the host in endpoints below from 0.0.0.0 to a specific network interface.
+# See https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks
+
+extensions:
+  health_check:
+  pprof:
+    endpoint: 0.0.0.0:1777
+  zpages:
+    endpoint: 0.0.0.0:55679
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+  opencensus:
+    endpoint: 0.0.0.0:55678
+
+  # Collect own metrics
+  prometheus:
+    config:
+      scrape_configs:
+      - job_name: 'otel-collector'
+        scrape_interval: 10s
+        static_configs:
+        - targets: ['0.0.0.0:8888']
+
+  jaeger:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:14250
+      thrift_binary:
+        endpoint: 0.0.0.0:6832
+      thrift_compact:
+        endpoint: 0.0.0.0:6831
+      thrift_http:
+        endpoint: 0.0.0.0:14268
+
+  zipkin:
+    endpoint: 0.0.0.0:9411
+
+processors:
+  batch:
+  aggregation:
+    interval: 15s
+
+exporters:
+  debug:
+    verbosity: detailed
+
+service:
+
+  pipelines:
+
+    traces:
+      receivers: [otlp, opencensus, jaeger, zipkin]
+      processors: [batch]
+      exporters: [debug]
+
+    metrics:
+      receivers: [otlp, opencensus, prometheus]
+      processors: [batch, aggregation]
+      exporters: [debug]
+
+    logs:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug]
+
+  extensions: [health_check, pprof, zpages]

docker/opentelemetry/collector/config.yaml

@@ -3,11 +3,9 @@ receivers:
     protocols:
       http:
         endpoint: 0.0.0.0:4318
-      grpc:
-        endpoint: 0.0.0.0:4317
 
 exporters:
-  logging:
+  debug:
     verbosity: detailed
 
   loki:
@@ -24,33 +22,48 @@ exporters:
       insecure: true
 
 processors:
+  # https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/batchprocessor
   batch:
+    timeout: 1s
+
+  # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/deltatocumulativeprocessor
+  deltatocumulative:
+
+  # Stickee's custom processor
+  # https://github.com/stickeeuk/opentelemetry-collector-contrib
+  aggregation:
+    interval: 15s
 
 service:
+  telemetry:
+    logs:
+      level: DEBUG
   pipelines:
     logs:
       receivers:
         - otlp
       processors:
         - batch
       exporters:
-        - logging
+        - debug
         - loki
 
     metrics:
       receivers:
-      - otlp
+        - otlp
       processors:
+        - aggregation
+        - deltatocumulative
         - batch
       exporters:
-      - logging
-      - otlphttp/prometheus
+        - debug
+        - otlphttp/prometheus
 
     traces:
       receivers:
         - otlp
       processors:
         - batch
       exporters:
-        - logging
+        - debug
         - otlphttp/tempo

docker/opentelemetry/docker-compose.yml

@@ -46,12 +46,12 @@ services:
     - opentelemetry-network
 
   collector:
-    image: otel/opentelemetry-collector-contrib:0.110.0-amd64
+    # image: otel/opentelemetry-collector-contrib:0.113.0-amd64
+    image: ghcr.io/stickeeuk/opentelemetry-collector
     command: ['--config=/etc/otel/config.yaml']
     volumes:
       - './collector:/etc/otel'
     ports:
-      - '4317:4317' # OTLP gRPC receiver
       - '4318:4318' # OTLP http receiver
     expose:
       - '8888' # Prometheus metrics exposed by the collector

src/Laravel/Providers/OpenTelemetryServiceProvider.php

@@ -20,9 +20,10 @@
 use OpenTelemetry\SDK\Resource\ResourceInfoFactory;
 use OpenTelemetry\SDK\Sdk;
 use OpenTelemetry\SDK\SdkAutoloader;
-use OpenTelemetry\SDK\Trace\ExporterFactory as TraceExporterFactory;
+use OpenTelemetry\SDK\Trace\ExporterFactory;
 use OpenTelemetry\SDK\Trace\Sampler\TraceIdRatioBasedSampler;
 use OpenTelemetry\SDK\Trace\Span;
+use OpenTelemetry\SDK\Trace\SpanExporterInterface;
 use OpenTelemetry\SDK\Trace\SpanProcessor\BatchSpanProcessor;
 use OpenTelemetry\SDK\Trace\SpanProcessor\MultiSpanProcessor;
 use OpenTelemetry\SDK\Trace\TracerProvider;
@@ -75,11 +76,18 @@ public function boot(): void
             ]);
         });
 
+        $exporter = (new ExporterFactory())->create();
+
+        // If `OTEL_TRACES_EXPORTER="none"` is set in the environment, the exporter will be null
+        if ($exporter === null) {
+            return;
+        }
+
         // During testing Auto Instrumentation may not be initialised if OTEL_PHP_AUTOLOAD_ENABLED is false in the environment.
         // It gets set to true via putenv in tests/Pest.php, but this happens after Auto Instrumentation is initialised.
         // In this case the providers will be API no-op providers, which aren't compatible with the SDK interfaces so use SDK no-op providers instead.
         Sdk::builder()
-            ->setTracerProvider($this->getTracerProvider())
+            ->setTracerProvider($this->getTracerProvider($exporter))
             ->setMeterProvider(Globals::meterProvider() instanceof MeterProviderInterface ? Globals::meterProvider() : new NoopMeterProvider())
             ->setLoggerProvider(Globals::loggerProvider() instanceof LoggerProviderInterface ? Globals::loggerProvider() : new NoopLoggerProvider())
             ->setEventLoggerProvider(Globals::eventLoggerProvider() instanceof EventLoggerProviderInterface ? Globals::eventLoggerProvider() : new NoopEventLoggerProvider())
@@ -90,15 +98,16 @@ public function boot(): void
 
     /**
      * Create a tracer provider
+     *
+     * @param \OpenTelemetry\SDK\Trace\SpanExporterInterface $exporter The span exporter
      */
-    private function getTracerProvider(): TracerProviderInterface
+    private function getTracerProvider(SpanExporterInterface $exporter): TracerProviderInterface
     {
         $traceLongRequests = $this->config->longRequestTraceThreshold() && ($this->config->traceSampleRate() < 1);
         $sampler = $traceLongRequests
             ? new RecordSampler(new TraceIdRatioBasedSampler($this->config->traceSampleRate()))
             : new TraceIdRatioBasedSampler($this->config->traceSampleRate());
 
-        $exporter = (new TraceExporterFactory())->create();
         $batchProcessor = BatchSpanProcessor::builder($exporter)->build();
         $processor = $traceLongRequests
             ? new MultiSpanProcessor(