[EXPORTER] Support handling retry-able errors for OTLP/HTTP (#3223)

Author: chusitoo

CHANGELOG.md

@@ -33,6 +33,9 @@ Increment the:
 * [SDK] Enable deriving from ResourceDetector to create a Resource
   [#3247](https://github.com/open-telemetry/opentelemetry-cpp/pull/3247)
 
+* [EXPORTER] Support handling retry-able errors for OTLP/HTTP
+  [#3223](https://github.com/open-telemetry/opentelemetry-cpp/pull/3223)
+
 New features:
 
 * [SDK] Better control of threads executed by opentelemetry-cpp

CMakeLists.txt

@@ -212,6 +212,9 @@ if(NOT WITH_STL STREQUAL "OFF")
   endif()
 endif()
 
+option(WITH_OTLP_RETRY_PREVIEW
+       "Whether to enable experimental retry functionality" OFF)
+
 option(WITH_OTLP_GRPC_SSL_MTLS_PREVIEW
        "Whether to enable mTLS support fro gRPC" OFF)
 

api/CMakeLists.txt

@@ -116,6 +116,11 @@ target_compile_definitions(
   opentelemetry_api
   INTERFACE OPENTELEMETRY_ABI_VERSION_NO=${OPENTELEMETRY_ABI_VERSION_NO})
 
+if(WITH_OTLP_RETRY_PREVIEW)
+  target_compile_definitions(opentelemetry_api
+                             INTERFACE ENABLE_OTLP_RETRY_PREVIEW)
+endif()
+
 if(WITH_OTLP_GRPC_SSL_MTLS_PREVIEW)
   target_compile_definitions(opentelemetry_api
                              INTERFACE ENABLE_OTLP_GRPC_SSL_MTLS_PREVIEW)

ci/do_ci.sh

@@ -120,6 +120,7 @@ elif [[ "$1" == "cmake.maintainer.sync.test" ]]; then
         -DOTELCPP_MAINTAINER_MODE=ON \
         -DWITH_NO_DEPRECATED_CODE=ON \
         -DWITH_OTLP_HTTP_COMPRESSION=ON \
+        -DWITH_OTLP_RETRY_PREVIEW=ON \
         -DWITH_THREAD_INSTRUMENTATION_PREVIEW=ON \
         "${SRC_DIR}"
   eval "$MAKE_COMMAND"
@@ -142,6 +143,7 @@ elif [[ "$1" == "cmake.maintainer.async.test" ]]; then
         -DOTELCPP_MAINTAINER_MODE=ON \
         -DWITH_NO_DEPRECATED_CODE=ON \
         -DWITH_OTLP_HTTP_COMPRESSION=ON \
+        -DWITH_OTLP_RETRY_PREVIEW=ON \
         -DWITH_THREAD_INSTRUMENTATION_PREVIEW=ON \
         "${SRC_DIR}"
   eval "$MAKE_COMMAND"
@@ -165,6 +167,7 @@ elif [[ "$1" == "cmake.maintainer.cpp11.async.test" ]]; then
         -DOTELCPP_MAINTAINER_MODE=ON \
         -DWITH_NO_DEPRECATED_CODE=ON \
         -DWITH_OTLP_HTTP_COMPRESSION=ON \
+        -DWITH_OTLP_RETRY_PREVIEW=ON \
         -DWITH_THREAD_INSTRUMENTATION_PREVIEW=ON \
         "${SRC_DIR}"
   make -k -j $(nproc)
@@ -189,6 +192,7 @@ elif [[ "$1" == "cmake.maintainer.abiv2.test" ]]; then
         -DWITH_ABI_VERSION_1=OFF \
         -DWITH_ABI_VERSION_2=ON \
         -DWITH_OTLP_HTTP_COMPRESSION=ON \
+        -DWITH_OTLP_RETRY_PREVIEW=ON \
         -DWITH_THREAD_INSTRUMENTATION_PREVIEW=ON \
         "${SRC_DIR}"
   eval "$MAKE_COMMAND"

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_environment.h

@@ -152,6 +152,22 @@ std::string GetOtlpDefaultTracesCompression();
 std::string GetOtlpDefaultMetricsCompression();
 std::string GetOtlpDefaultLogsCompression();
 
+std::uint32_t GetOtlpDefaultTracesRetryMaxAttempts();
+std::uint32_t GetOtlpDefaultMetricsRetryMaxAttempts();
+std::uint32_t GetOtlpDefaultLogsRetryMaxAttempts();
+
+std::chrono::duration<float> GetOtlpDefaultTracesRetryInitialBackoff();
+std::chrono::duration<float> GetOtlpDefaultMetricsRetryInitialBackoff();
+std::chrono::duration<float> GetOtlpDefaultLogsRetryInitialBackoff();
+
+std::chrono::duration<float> GetOtlpDefaultTracesRetryMaxBackoff();
+std::chrono::duration<float> GetOtlpDefaultMetricsRetryMaxBackoff();
+std::chrono::duration<float> GetOtlpDefaultLogsRetryMaxBackoff();
+
+float GetOtlpDefaultTracesRetryBackoffMultiplier();
+float GetOtlpDefaultMetricsRetryBackoffMultiplier();
+float GetOtlpDefaultLogsRetryBackoffMultiplier();
+
 }  // namespace otlp
 }  // namespace exporter
 OPENTELEMETRY_END_NAMESPACE

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h

@@ -75,6 +75,9 @@ struct OtlpHttpClientOptions
   // Additional HTTP headers
   OtlpHeaders http_headers;
 
+  // Retry policy for select failure codes
+  ext::http::client::RetryPolicy retry_policy;
+
   // Concurrent requests
   std::size_t max_concurrent_requests = 64;
 
@@ -107,6 +110,10 @@ struct OtlpHttpClientOptions
       bool input_console_debug,
       std::chrono::system_clock::duration input_timeout,
       const OtlpHeaders &input_http_headers,
+      std::uint32_t input_retry_policy_max_attempts,
+      std::chrono::duration<float> input_retry_policy_initial_backoff,
+      std::chrono::duration<float> input_retry_policy_max_backoff,
+      float input_retry_policy_backoff_multiplier,
       const std::shared_ptr<sdk::common::ThreadInstrumentation> &input_thread_instrumentation,
       std::size_t input_concurrent_sessions         = 64,
       std::size_t input_max_requests_per_connection = 8,
@@ -131,6 +138,8 @@ struct OtlpHttpClientOptions
         console_debug(input_console_debug),
         timeout(input_timeout),
         http_headers(input_http_headers),
+        retry_policy{input_retry_policy_max_attempts, input_retry_policy_initial_backoff,
+                     input_retry_policy_max_backoff, input_retry_policy_backoff_multiplier},
         max_concurrent_requests(input_concurrent_sessions),
         max_requests_per_connection(input_max_requests_per_connection),
         user_agent(input_user_agent),

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter_options.h

@@ -4,6 +4,7 @@
 #pragma once
 
 #include <chrono>
+#include <cstdint>
 #include <string>
 
 #include "opentelemetry/exporters/otlp/otlp_environment.h"
@@ -101,6 +102,18 @@ struct OPENTELEMETRY_EXPORT OtlpHttpExporterOptions
 
   /** Compression type. */
   std::string compression;
+
+  /** The maximum number of call attempts, including the original attempt. */
+  std::uint32_t retry_policy_max_attempts{};
+
+  /** The initial backoff delay between retry attempts, random between (0, initial_backoff). */
+  std::chrono::duration<float> retry_policy_initial_backoff{};
+
+  /** The maximum backoff places an upper limit on exponential backoff growth. */
+  std::chrono::duration<float> retry_policy_max_backoff{};
+
+  /** The backoff will be multiplied by this value after each retry attempt. */
+  float retry_policy_backoff_multiplier{};
 };
 
 }  // namespace otlp

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_record_exporter_options.h

@@ -4,6 +4,7 @@
 #pragma once
 
 #include <chrono>
+#include <cstdint>
 #include <string>
 
 #include "opentelemetry/exporters/otlp/otlp_environment.h"
@@ -101,6 +102,18 @@ struct OPENTELEMETRY_EXPORT OtlpHttpLogRecordExporterOptions
 
   /** Compression type. */
   std::string compression;
+
+  /** The maximum number of call attempts, including the original attempt. */
+  std::uint32_t retry_policy_max_attempts{};
+
+  /** The initial backoff delay between retry attempts, random between (0, initial_backoff). */
+  std::chrono::duration<float> retry_policy_initial_backoff{};
+
+  /** The maximum backoff places an upper limit on exponential backoff growth. */
+  std::chrono::duration<float> retry_policy_max_backoff{};
+
+  /** The backoff will be multiplied by this value after each retry attempt. */
+  float retry_policy_backoff_multiplier{};
 };
 
 }  // namespace otlp

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_metric_exporter_options.h

@@ -4,6 +4,7 @@
 #pragma once
 
 #include <chrono>
+#include <cstdint>
 #include <string>
 
 #include "opentelemetry/exporters/otlp/otlp_environment.h"
@@ -104,6 +105,18 @@ struct OPENTELEMETRY_EXPORT OtlpHttpMetricExporterOptions
 
   /** Compression type. */
   std::string compression;
+
+  /** The maximum number of call attempts, including the original attempt. */
+  std::uint32_t retry_policy_max_attempts{};
+
+  /** The initial backoff delay between retry attempts, random between (0, initial_backoff). */
+  std::chrono::duration<float> retry_policy_initial_backoff{};
+
+  /** The maximum backoff places an upper limit on exponential backoff growth. */
+  std::chrono::duration<float> retry_policy_max_backoff{};
+
+  /** The backoff will be multiplied by this value after each retry attempt. */
+  float retry_policy_backoff_multiplier{};
 };
 
 }  // namespace otlp