Make sending traces to Tempo less confusing

.github/workflows/rust.yml

@@ -170,7 +170,7 @@ jobs:
     - uses: actions-rust-lang/setup-rust-toolchain@v1
     - name: Run metrics tests
       run: |
-        cargo test --locked -p linera-base --features metrics
+        cargo test --locked -p linera-base --features metrics,opentelemetry
 
   wasm-application-test:
     needs: changed-files

CLI.md

@@ -149,8 +149,8 @@ Client implementation and command-line tool for the Linera blockchain
 
   Default value: `10`
 * `--chrome-trace-exporter` — Enable OpenTelemetry Chrome JSON exporter for trace data analysis
-* `--otel-trace-file <OTEL_TRACE_FILE>` — Output file path for Chrome trace JSON format. Can be visualized in chrome://tracing or Perfetto UI
-* `--otel-exporter-otlp-endpoint <OTEL_EXPORTER_OTLP_ENDPOINT>` — OpenTelemetry OTLP exporter endpoint (requires tempo feature)
+* `--chrome-trace-file <CHROME_TRACE_FILE>` — Output file path for Chrome trace JSON format. Can be visualized in chrome://tracing or Perfetto UI
+* `--otlp-exporter-endpoint <OTLP_EXPORTER_ENDPOINT>` — OpenTelemetry OTLP exporter endpoint (requires opentelemetry feature)
 * `--wait-for-outgoing-messages` — Whether to wait until a quorum of validators has confirmed that all sent cross-chain messages have been delivered
 * `--long-lived-services` — (EXPERIMENTAL) Whether application services can persist in some cases between queries
 * `--blanket-message-policy <BLANKET_MESSAGE_POLICY>` — The policy for handling incoming messages

docker/Dockerfile

@@ -25,7 +25,7 @@ ARG binaries=
 ARG copy=${binaries:+_copy}
 ARG build_flag=--release
 ARG build_folder=release
-ARG build_features=scylladb,metrics,memory-profiling,tempo
+ARG build_features=scylladb,metrics,memory-profiling,opentelemetry
 ARG rustflags="-C force-frame-pointers=yes"
 
 FROM rust:1.74-slim-bookworm AS builder

kubernetes/linera-validator/templates/proxy.yaml

@@ -69,8 +69,8 @@ spec:
               value: {{ .Values.logLevel }}
             - name: RUST_BACKTRACE
               value: "1"
-            - name: OTEL_EXPORTER_OTLP_ENDPOINT
-              value: {{ .Values.otelExporterEndpoint }}
+            - name: LINERA_OTLP_EXPORTER_ENDPOINT
+              value: {{ .Values.otlpExporterEndpoint }}
       containers:
         - name: linera-proxy
           imagePullPolicy: {{ .Values.lineraImagePullPolicy }}
@@ -84,8 +84,8 @@ spec:
           env:
             - name: RUST_LOG
               value: {{ .Values.logLevel }}
-            - name: OTEL_EXPORTER_OTLP_ENDPOINT
-              value: {{ .Values.otelExporterEndpoint }}
+            - name: LINERA_OTLP_EXPORTER_ENDPOINT
+              value: {{ .Values.otlpExporterEndpoint }}
           volumeMounts:
             - name: config
               mountPath: "/config"

kubernetes/linera-validator/templates/shards.yaml

@@ -49,8 +49,8 @@ spec:
               value: {{ .Values.logLevel }}
             - name: RUST_BACKTRACE
               value: "1"
-            - name: OTEL_EXPORTER_OTLP_ENDPOINT
-              value: {{ .Values.otelExporterEndpoint }}
+            - name: LINERA_OTLP_EXPORTER_ENDPOINT
+              value: {{ .Values.otlpExporterEndpoint }}
           volumeMounts:
             - name: config
               mountPath: "/config"
@@ -67,8 +67,8 @@ spec:
           env:
             - name: RUST_LOG
               value: {{ .Values.logLevel }}
-            - name: OTEL_EXPORTER_OTLP_ENDPOINT
-              value: {{ .Values.otelExporterEndpoint }}
+            - name: LINERA_OTLP_EXPORTER_ENDPOINT
+              value: {{ .Values.otlpExporterEndpoint }}
             {{- if .Values.serverTokioThreads }}
             - name: LINERA_SERVER_TOKIO_THREADS
               value: "{{ .Values.serverTokioThreads }}"

kubernetes/linera-validator/values-local.yaml.gotmpl

@@ -4,7 +4,7 @@
 lineraImage: {{ env "LINERA_HELMFILE_LINERA_IMAGE" | default "linera:latest" }}
 lineraImagePullPolicy: Never
 logLevel: "debug"
-otelExporterEndpoint: {{ env "LINERA_HELMFILE_SET_OTEL_ENDPOINT" | default "http://tempo.tempo.svc.cluster.local:4317" }}
+otlpExporterEndpoint: {{ env "LINERA_HELMFILE_SET_OTLP_EXPORTER_ENDPOINT" | default "" }}
 proxyPort: 19100
 metricsPort: 21100
 numShards: {{ env "LINERA_HELMFILE_SET_NUM_SHARDS" | default 10 }}

linera-base/Cargo.toml

@@ -18,7 +18,7 @@ workspace = true
 metrics = ["prometheus"]
 reqwest = ["dep:reqwest"]
 revm = []
-tempo = ["opentelemetry-otlp"]
+opentelemetry = ["opentelemetry-otlp"]
 test = ["test-strategy", "proptest"]
 web = [
     "getrandom/js",
@@ -83,7 +83,7 @@ tracing-web = { optional = true, workspace = true }
 chrono.workspace = true
 opentelemetry.workspace = true
 opentelemetry-otlp = { workspace = true, optional = true }
-opentelemetry_sdk.workspace = true
+opentelemetry_sdk = { workspace = true, features = ["testing"] }
 tracing-chrome.workspace = true
 tracing-opentelemetry.workspace = true
 rand = { workspace = true, features = ["getrandom", "std", "std_rng"] }

linera-base/src/lib.rs

@@ -41,7 +41,7 @@ pub mod time;
 #[cfg_attr(web, path = "tracing_web.rs")]
 pub mod tracing;
 #[cfg(not(target_arch = "wasm32"))]
-pub mod tracing_otel;
+pub mod tracing_opentelemetry;
 #[cfg(test)]
 mod unit_tests;
 

linera-base/src/tracing.rs

@@ -26,7 +26,7 @@ use tracing_subscriber::{
 };
 
 #[cfg(not(target_arch = "wasm32"))]
-pub use crate::tracing_otel::{
+pub use crate::tracing_opentelemetry::{
     init_with_chrome_trace_exporter, init_with_opentelemetry, ChromeTraceGuard,
 };
 

linera-base/src/tracing_opentelemetry.rs

@@ -0,0 +1,210 @@
+// Copyright (c) Zefchain Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+//! OpenTelemetry integration for tracing with OTLP export and Chrome trace export.
+
+use tracing_chrome::ChromeLayerBuilder;
+use tracing_subscriber::{layer::SubscriberExt as _, util::SubscriberInitExt as _};
+#[cfg(feature = "opentelemetry")]
+use {
+    opentelemetry::{global, trace::TracerProvider},
+    opentelemetry_otlp::{SpanExporter, WithExportConfig},
+    opentelemetry_sdk::{
+        trace::{InMemorySpanExporter, SdkTracerProvider},
+        Resource,
+    },
+    tracing_opentelemetry::OpenTelemetryLayer,
+    tracing_subscriber::{
+        filter::{filter_fn, FilterFn},
+        layer::Layer,
+    },
+};
+
+/// Creates a filter that excludes spans with the `opentelemetry.skip` field.
+///
+/// Any span that declares an `opentelemetry.skip` field will be excluded from export,
+/// regardless of the field's value. This is a limitation of the tracing metadata API.
+///
+/// Usage examples:
+/// ```ignore
+/// // Always skip this span
+/// #[tracing::instrument(fields(opentelemetry.skip = true))]
+/// fn internal_helper() { }
+///
+/// // Conditionally skip based on a parameter
+/// #[tracing::instrument(fields(opentelemetry.skip = should_skip))]
+/// fn my_function(should_skip: bool) {
+///     // Will be skipped if should_skip is true when called
+///     // Note: The field must be declared in the span, so the span is
+///     // created with knowledge that it might be skipped
+/// }
+/// ```
+#[cfg(feature = "opentelemetry")]
+fn opentelemetry_skip_filter() -> FilterFn<impl Fn(&tracing::Metadata<'_>) -> bool> {
+    filter_fn(|metadata| {
+        if !metadata.is_span() {
+            return false;
+        }
+        metadata.fields().field("opentelemetry.skip").is_none()
+    })
+}
+
+/// Initializes tracing with a custom OpenTelemetry tracer provider.
+///
+/// This is an internal function used by both production and test code.
+#[cfg(feature = "opentelemetry")]
+fn init_with_tracer_provider(log_name: &str, tracer_provider: SdkTracerProvider) {
+    global::set_tracer_provider(tracer_provider.clone());
+    let tracer = tracer_provider.tracer("linera");
+
+    let opentelemetry_layer =
+        OpenTelemetryLayer::new(tracer).with_filter(opentelemetry_skip_filter());
+
+    let config = crate::tracing::get_env_config(log_name);
+    let maybe_log_file_layer = config.maybe_log_file_layer();
+    let stderr_layer = config.stderr_layer();
+
+    tracing_subscriber::registry()
+        .with(opentelemetry_layer)
+        .with(config.env_filter)
+        .with(maybe_log_file_layer)
+        .with(stderr_layer)
+        .init();
+}
+
+/// Builds an OpenTelemetry layer with the opentelemetry.skip filter.
+///
+/// This is used for testing to avoid setting the global subscriber.
+/// Returns the layer, exporter, and tracer provider (which must be kept alive and shutdown).
+#[cfg(all(with_testing, feature = "opentelemetry"))]
+pub fn build_opentelemetry_layer_with_test_exporter(
+    log_name: &str,
+) -> (
+    impl tracing_subscriber::Layer<tracing_subscriber::Registry>,
+    InMemorySpanExporter,
+    SdkTracerProvider,
+) {
+    let exporter = InMemorySpanExporter::default();
+    let exporter_clone = exporter.clone();
+
+    let resource = Resource::builder()
+        .with_service_name(log_name.to_string())
+        .build();
+
+    let tracer_provider = SdkTracerProvider::builder()
+        .with_resource(resource)
+        .with_simple_exporter(exporter)
+        .with_sampler(opentelemetry_sdk::trace::Sampler::AlwaysOn)
+        .build();
+
+    global::set_tracer_provider(tracer_provider.clone());
+    let tracer = tracer_provider.tracer("linera");
+    let opentelemetry_layer =
+        OpenTelemetryLayer::new(tracer).with_filter(opentelemetry_skip_filter());
+
+    (opentelemetry_layer, exporter_clone, tracer_provider)
+}
+
+/// Initializes tracing with OpenTelemetry OTLP exporter.
+///
+/// Exports traces using the OTLP protocol to any OpenTelemetry-compatible backend.
+/// Requires the `opentelemetry` feature.
+/// Only enables OpenTelemetry if LINERA_OTLP_EXPORTER_ENDPOINT env var is set.
+/// This prevents DNS errors in environments where OpenTelemetry is not deployed.
+#[cfg(feature = "opentelemetry")]
+pub fn init_with_opentelemetry(log_name: &str, otlp_endpoint: Option<&str>) {
+    // Check if OpenTelemetry endpoint is configured via parameter or env var
+    let endpoint = match otlp_endpoint {
+        Some(ep) if !ep.is_empty() => ep.to_string(),
+        _ => match std::env::var("LINERA_OTLP_EXPORTER_ENDPOINT") {
+            Ok(ep) if !ep.is_empty() => ep,
+            _ => {
+                eprintln!(
+                    "LINERA_OTLP_EXPORTER_ENDPOINT not set and no endpoint provided. \
+                     Falling back to standard tracing without OpenTelemetry support."
+                );
+                crate::tracing::init(log_name);
+                return;
+            }
+        },
+    };
+
+    let resource = Resource::builder()
+        .with_service_name(log_name.to_string())
+        .build();
+
+    let exporter = SpanExporter::builder()
+        .with_tonic()
+        .with_endpoint(endpoint)
+        .build()
+        .expect("Failed to create OTLP exporter");
+
+    let tracer_provider = SdkTracerProvider::builder()
+        .with_resource(resource)
+        .with_batch_exporter(exporter)
+        .with_sampler(opentelemetry_sdk::trace::Sampler::AlwaysOn)
+        .build();
+
+    init_with_tracer_provider(log_name, tracer_provider);
+}
+
+/// Fallback when opentelemetry feature is not enabled.
+#[cfg(not(feature = "opentelemetry"))]
+pub fn init_with_opentelemetry(log_name: &str, _otlp_endpoint: Option<&str>) {
+    eprintln!(
+        "OTLP export requires the 'opentelemetry' feature to be enabled! Falling back to default tracing initialization."
+    );
+    crate::tracing::init(log_name);
+}
+
+/// Guard that flushes Chrome trace file when dropped.
+///
+/// Store this guard in a variable that lives for the duration of your program.
+/// When it's dropped, the trace file will be completed and closed.
+pub type ChromeTraceGuard = tracing_chrome::FlushGuard;
+
+/// Builds a Chrome trace layer and guard.
+///
+/// Returns a subscriber and guard. The subscriber should be used with `with_default`
+/// to avoid global state conflicts.
+pub fn build_chrome_trace_layer_with_exporter<W>(
+    log_name: &str,
+    writer: W,
+) -> (impl tracing::Subscriber + Send + Sync, ChromeTraceGuard)
+where
+    W: std::io::Write + Send + 'static,
+{
+    let (chrome_layer, guard) = ChromeLayerBuilder::new().writer(writer).build();
+
+    let config = crate::tracing::get_env_config(log_name);
+    let maybe_log_file_layer = config.maybe_log_file_layer();
+    let stderr_layer = config.stderr_layer();
+
+    let subscriber = tracing_subscriber::registry()
+        .with(chrome_layer)
+        .with(config.env_filter)
+        .with(maybe_log_file_layer)
+        .with(stderr_layer);
+
+    (subscriber, guard)
+}
+
+/// Initializes tracing with Chrome Trace JSON exporter.
+///
+/// Returns a guard that must be kept alive for the duration of the program.
+/// When the guard is dropped, the trace data is flushed and completed.
+///
+/// Exports traces to Chrome Trace JSON format which can be visualized in:
+/// - Chrome: `chrome://tracing`
+/// - Perfetto UI: <https://ui.perfetto.dev>
+///
+/// Note: Uses `try_init()` to avoid panicking if a global subscriber is already set.
+/// In that case, tracing may not work as expected.
+pub fn init_with_chrome_trace_exporter<W>(log_name: &str, writer: W) -> ChromeTraceGuard
+where
+    W: std::io::Write + Send + 'static,
+{
+    let (subscriber, guard) = build_chrome_trace_layer_with_exporter(log_name, writer);
+    let _ = subscriber.try_init();
+    guard
+}