agents: add initial gRPC support in OTLPAgent

To be clear, support hasn't been added just yet to the agent itself, but the needed modifications so it can be easily done: - Added `grpc++` as a direct dependency to `opentelemetry-cpp`. - Added needed source files to our `protobuf` dependency. - Basic modifications to `OTLPAgent` and `OTLPMetrics` so they can support any kind of `OTLP` exporter, some helper code and some actual code that works but remains commented until we decide the best way to add gRPC support (configuration, etc.).
nodesource · May 13, 2024 · 4c0b97b · 4c0b97b
1 parent a6cb704
commit 4c0b97b
Show file tree

Hide file tree

Showing 6 changed files with 76 additions and 27 deletions.
diff --git a/agents/otlp/src/otlp_agent.cc b/agents/otlp/src/otlp_agent.cc
@@ -1,3 +1,7 @@
+// This header needs to be included before any other grpc header otherwise there
+// will be a compilation error because of abseil.
+// Refs: https://github.com/open-telemetry/opentelemetry-cpp/blob/32cd66b62333e84aa8e92a4447e0aa667b6735e5/examples/otlp/README.md#additional-notes-regarding-abseil-library
+#include "opentelemetry/exporters/otlp/otlp_grpc_exporter.h"
 #include "otlp_agent.h"
 #include "nsolid/nsolid_api.h"
 #include "env-inl.h"
@@ -597,6 +601,12 @@ void OTLPAgent::config_otlp_endpoint(const json& config) {
   const std::string url = it->get<std::string>();
   opts.url = url + "/v1/traces";
   setup_trace_otlp_exporter(opts);
+
+  // TODO(santi) Add support for GRPC too. This is an example:
+  // exporter::otlp::OtlpGrpcExporterOptions opts;
+  // opts.endpoint += "/v1/traces";
+  // setup_trace_grpc_otlp_exporter(opts);
+
   metrics_exporter_.reset(new OTLPMetrics(&loop_, url, "", *this));
 }
 
@@ -639,6 +649,12 @@ void OTLPAgent::setup_trace_otlp_exporter(
 }
 
 
+void OTLPAgent::setup_trace_grpc_otlp_exporter(
+    exporter::otlp::OtlpGrpcExporterOptions& opts) {
+  otlp_http_exporter_.reset(new exporter::otlp::OtlpGrpcExporter(opts));
+}
+
+
 int OTLPAgent::setup_metrics_timer(uint64_t period) {
   if (period == 0) {
     return metrics_timer_.stop();

diff --git a/agents/otlp/src/otlp_agent.h b/agents/otlp/src/otlp_agent.h
@@ -15,14 +15,17 @@
 OPENTELEMETRY_BEGIN_NAMESPACE
 namespace exporter {
 namespace otlp {
-class OtlpHttpExporter;
 struct OtlpHttpExporterOptions;
+struct OtlpGrpcExporterOptions;
 }
 }
 namespace sdk {
 namespace instrumentationscope {
 class InstrumentationScope;
 }
+namespace trace {
+class SpanExporter;
+}
 }
 OPENTELEMETRY_END_NAMESPACE
 
@@ -104,6 +107,9 @@ class OTLPAgent {
   void setup_trace_otlp_exporter(  // NOLINTNEXTLINE(runtime/references)
     OPENTELEMETRY_NAMESPACE::exporter::otlp::OtlpHttpExporterOptions& opts);
 
+  void setup_trace_grpc_otlp_exporter(  // NOLINTNEXTLINE(runtime/references)
+    OPENTELEMETRY_NAMESPACE::exporter::otlp::OtlpGrpcExporterOptions& opts);
+
   int setup_metrics_timer(uint64_t period);
 
   void update_tracer(uint32_t flags);
@@ -128,7 +134,7 @@ class OTLPAgent {
   TSQueue<Tracer::SpanStor> span_msg_q_;
   uint32_t trace_flags_;
 
-  std::unique_ptr<OPENTELEMETRY_NAMESPACE::exporter::otlp::OtlpHttpExporter>
+  std::unique_ptr<OPENTELEMETRY_NAMESPACE::sdk::trace::SpanExporter>
     otlp_http_exporter_;
   OPENTELEMETRY_NAMESPACE::sdk::resource::Resource resource_;
   std::unique_ptr

diff --git a/agents/otlp/src/otlp_metrics.cc b/agents/otlp/src/otlp_metrics.cc
@@ -1,7 +1,12 @@
+// This header needs to be included before any other grpc header otherwise there
+// will be a compilation error because of abseil.
+// Refs: https://github.com/open-telemetry/opentelemetry-cpp/blob/32cd66b62333e84aa8e92a4447e0aa667b6735e5/examples/otlp/README.md#additional-notes-regarding-abseil-library
+#include "opentelemetry/exporters/otlp/otlp_grpc_metric_exporter.h"
 #include "otlp_metrics.h"
 #include "debug_utils-inl.h"
 #include "env-inl.h"
 #include "otlp_agent.h"
+#include "opentelemetry/exporters/otlp/otlp_http_metric_exporter.h"
 #include "opentelemetry/trace/semantic_conventions.h"
 #include "opentelemetry/sdk/metrics/data/metric_data.h"
 #include "opentelemetry/sdk/metrics/export/metric_producer.h"
@@ -29,6 +34,8 @@ using opentelemetry::sdk::metrics::ResourceMetrics;
 using opentelemetry::sdk::metrics::ScopeMetrics;
 using opentelemetry::sdk::metrics::SumPointData;
 using opentelemetry::sdk::metrics::ValueType;
+using opentelemetry::v1::exporter::otlp::OtlpGrpcMetricExporter;
+using opentelemetry::v1::exporter::otlp::OtlpGrpcMetricExporterOptions;
 using opentelemetry::v1::exporter::otlp::OtlpHttpMetricExporter;
 using opentelemetry::v1::exporter::otlp::OtlpHttpMetricExporterOptions;
 using opentelemetry::v1::trace::SemanticConventions::kThreadId;
@@ -61,7 +68,12 @@ OTLPMetrics::OTLPMetrics(uv_loop_t* loop,
   opts.url = url + "/v1/metrics";
   opts.content_type = HttpRequestContentType::kBinary;
   opts.console_debug = true;
-  otlp_http_metric_exporter_ = std::make_unique<OtlpHttpMetricExporter>(opts);
+  otlp_metric_exporter_ = std::make_unique<OtlpHttpMetricExporter>(opts);
+
+  // TODO(santi) Add support for GRPC too. This is an example:
+  // OtlpGrpcMetricExporterOptions opts;
+  // opts.endpoint += "/v1/traces";
+  // otlp_metric_exporter_ = std::make_unique<OtlpGrpcMetricExporter>(opts);
 }
 
 /*virtual*/
@@ -163,18 +175,15 @@ NSOLID_PROCESS_METRICS_DOUBLE(V)
 
   data.scope_metric_data_ =
     std::vector<ScopeMetrics>{{agent_.scope_.get(), metrics}};
-  auto result = otlp_http_metric_exporter_->Export(data);
+  auto result = otlp_metric_exporter_->Export(data);
   Debug("# ProcessMetrics Exported. Result: %d\n", static_cast<int>(result));
 }
 
 // NOLINTNEXTLINE(runtime/references)
-void OTLPMetrics::got_env_metrics(std::vector<MetricData>& metrics,
+static void got_env_metrics(std::vector<MetricData>& metrics,
                                   const ThreadMetricsStor& stor,
                                   const ThreadMetricsStor& prev_stor,
-                                  double loop_start) {
-  time_point start{
-        duration_cast<time_point::duration>(
-          microseconds(static_cast<uint64_t>(loop_start)))};
+                                  time_point start) {
   time_point end{
         duration_cast<time_point::duration>(
           milliseconds(static_cast<uint64_t>(stor.timestamp)))};
@@ -207,12 +216,12 @@ void OTLPMetrics::got_env_metrics(std::vector<MetricData>& metrics,
     switch (MetricsType::MType) {                                              \
       case MetricsType::ECounter:                                              \
       {                                                                        \
-        add_counter(metrics, start_, end, #CName, Unit, type, value, attrs);   \
+        add_counter(metrics, start, end, #CName, Unit, type, value, attrs);    \
       }                                                                        \
       break;                                                                   \
-      case MetricsType::EGauge:                                               \
+      case MetricsType::EGauge:                                                \
       {                                                                        \
-        add_gauge(metrics, start_, end, #CName, Unit, type, value, attrs);     \
+        add_gauge(metrics, start, end, #CName, Unit, type, value, attrs);      \
       }                                                                        \
       break;                                                                   \
       default:                                                                 \
@@ -232,12 +241,12 @@ void OTLPMetrics::got_thr_metrics(
   std::vector<MetricData> metrics;
 
   for (const auto& tm : thr_metrics) {
-    got_env_metrics(metrics, tm.stor, tm.prev_stor, tm.loop_start);
+    got_env_metrics(metrics, tm.stor, tm.prev_stor, start_);
   }
 
   data.scope_metric_data_ =
     std::vector<ScopeMetrics>{{agent_.scope_.get(), metrics}};
-  auto result = otlp_http_metric_exporter_->Export(data);
+  auto result = otlp_metric_exporter_->Export(data);
   Debug("# ThreadMetrics Exported. Result: %d\n", static_cast<int>(result));
 }
 }  // namespace otlp

diff --git a/agents/otlp/src/otlp_metrics.h b/agents/otlp/src/otlp_metrics.h
@@ -1,9 +1,20 @@
 #ifndef AGENTS_OTLP_SRC_OTLP_METRICS_H_
 #define AGENTS_OTLP_SRC_OTLP_METRICS_H_
 
+// NOLINTNEXTLINE(build/c++11)
+#include <chrono>
+
 #include "metrics_exporter.h"
-#include "opentelemetry/exporters/otlp/otlp_http_metric_exporter.h"
+#include "opentelemetry/version.h"
 
+// Class pre-declaration
+OPENTELEMETRY_BEGIN_NAMESPACE
+namespace sdk {
+namespace metrics {
+class PushMetricExporter;
+}
+}
+OPENTELEMETRY_END_NAMESPACE
 
 namespace node {
 namespace nsolid {
@@ -27,14 +38,8 @@ class OTLPMetrics final: public MetricsExporter {
       const std::vector<MetricsExporter::ThrMetricsStor>&);
 
  private:
-  // NOLINTNEXTLINE(runtime/references)
-  void got_env_metrics(std::vector<opentelemetry::sdk::metrics::MetricData>& m,
-                       const ThreadMetrics::MetricsStor& stor,
-                       const ThreadMetrics::MetricsStor& prev_stor,
-                       double loop_start);
-
-  std::unique_ptr<opentelemetry::v1::exporter::otlp::OtlpHttpMetricExporter>
-    otlp_http_metric_exporter_;
+  std::unique_ptr<OPENTELEMETRY_NAMESPACE::sdk::metrics::PushMetricExporter>
+    otlp_metric_exporter_;
   std::string key_;
   std::string url_;
   std::chrono::system_clock::time_point start_;

diff --git a/deps/opentelemetry-cpp/otlp-http-exporter.gyp b/deps/opentelemetry-cpp/otlp-http-exporter.gyp
@@ -5,14 +5,20 @@
       'type': 'static_library',
       'sources': [
         'exporters/otlp/src/otlp_environment.cc',
+        'exporters/otlp/src/otlp_grpc_client.cc',
+        'exporters/otlp/src/otlp_grpc_exporter_options.cc',
+        'exporters/otlp/src/otlp_grpc_exporter.cc',
+        'exporters/otlp/src/otlp_grpc_metric_exporter_options.cc',
+        'exporters/otlp/src/otlp_grpc_metric_exporter.cc',
+        'exporters/otlp/src/otlp_grpc_utils.cc',
         'exporters/otlp/src/otlp_http.cc',
         'exporters/otlp/src/otlp_http_client.cc',
         'exporters/otlp/src/otlp_http_exporter.cc',
         'exporters/otlp/src/otlp_http_exporter_options.cc',
         'exporters/otlp/src/otlp_http_metric_exporter.cc',
         'exporters/otlp/src/otlp_http_metric_exporter_options.cc',
-        'exporters/otlp/src/otlp_metric_utils.cc',
         'exporters/otlp/src/otlp_log_recordable.cc',
+        'exporters/otlp/src/otlp_metric_utils.cc',
         'exporters/otlp/src/otlp_populate_attribute_utils.cc',
         'exporters/otlp/src/otlp_recordable_utils.cc',
         'exporters/otlp/src/otlp_recordable.cc',
@@ -31,8 +37,12 @@
         'third_party/opentelemetry-proto/gen/cpp/opentelemetry/proto/metrics/v1/metrics.pb.cc',
         'third_party/opentelemetry-proto/gen/cpp/opentelemetry/proto/resource/v1/resource.pb.cc',
         'third_party/opentelemetry-proto/gen/cpp/opentelemetry/proto/trace/v1/trace.pb.cc',
+        'third_party/opentelemetry-proto/gen/cpp/opentelemetry/proto/collector/logs/v1/logs_service.pb.cc',
+        'third_party/opentelemetry-proto/gen/cpp/opentelemetry/proto/collector/logs/v1/logs_service.grpc.pb.cc',
         'third_party/opentelemetry-proto/gen/cpp/opentelemetry/proto/collector/metrics/v1/metrics_service.pb.cc',
-        'third_party/opentelemetry-proto/gen/cpp/opentelemetry/proto/collector/trace/v1/trace_service.pb.cc'
+        'third_party/opentelemetry-proto/gen/cpp/opentelemetry/proto/collector/metrics/v1/metrics_service.grpc.pb.cc',
+        'third_party/opentelemetry-proto/gen/cpp/opentelemetry/proto/collector/trace/v1/trace_service.pb.cc',
+        'third_party/opentelemetry-proto/gen/cpp/opentelemetry/proto/collector/trace/v1/trace_service.grpc.pb.cc'
       ],
       'include_dirs': [
         'api/include',
@@ -47,7 +57,8 @@
       ],
       'dependencies': [
         '../protobuf/protobuf.gyp:protobuf',
-        '../curl/curl.gyp:curl'
+        '../curl/curl.gyp:curl',
+        '../grpc/grpc.gyp:grpc++',
       ],
       'direct_dependent_settings': {
         'include_dirs': [
@@ -56,7 +67,6 @@
           'ext/include',
           'sdk/include',
           'third_party/opentelemetry-proto/gen/cpp',
-          '../protobuf/src'
         ]
       },
       'cflags_cc': [

diff --git a/deps/protobuf/protobuf.gyp b/deps/protobuf/protobuf.gyp
@@ -43,8 +43,11 @@
         'src/google/protobuf/wire_format.cc',
         'src/google/protobuf/wire_format_lite.cc',
         'src/google/protobuf/unknown_field_set.cc',
+        'src/google/protobuf/compiler/code_generator.cc',
         'src/google/protobuf/compiler/importer.cc',
         'src/google/protobuf/compiler/parser.cc',
+        'src/google/protobuf/compiler/plugin.cc',
+        'src/google/protobuf/compiler/plugin.pb.cc',
         'src/google/protobuf/io/coded_stream.cc',
         'src/google/protobuf/io/io_win32.cc',
         'src/google/protobuf/io/gzip_stream.cc',