Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support http/json protocol #1585

Merged
merged 40 commits into from
Apr 23, 2024
Merged
Show file tree
Hide file tree
Changes from 29 commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
1482d5c
Add http-json feature flag
ramgdev Feb 27, 2024
576a56d
prototype json serialization
ramgdev Feb 27, 2024
d13ab4b
fix default protocol
ramgdev Feb 27, 2024
ed4fc03
fix typo
ramgdev Feb 27, 2024
587c341
use default protocol for HttpConfig
ramgdev Feb 28, 2024
d460bda
Update opentelemetry-otlp/src/exporter/http/trace.rs
ramgdev Feb 28, 2024
da76734
address review comments
ramgdev Feb 29, 2024
aa74b9c
run pre-commit script
ramgdev Feb 29, 2024
efa8588
fix dependency
ramgdev Feb 29, 2024
e47a80f
Merge branch 'main' into otel-json
ramgdev Mar 3, 2024
04593ec
Update mod.rs
ramgdev Mar 3, 2024
277df67
unit test default endpoint
ramgdev Mar 3, 2024
5d5e06e
test default protocol
ramgdev Mar 4, 2024
359babe
Merge branch 'main' into otel-json
ramgdev Mar 12, 2024
8c9577f
Merge branch 'main' into otel-json
ramgdev Mar 12, 2024
f175980
Merge branch 'main' into otel-json
ramgdev Mar 15, 2024
f93058c
enable http-json for all signals
ramgdev Mar 18, 2024
46f92c4
Merge branch 'main' into otel-json
ramgdev Mar 18, 2024
0f1a805
fix lint
ramgdev Mar 18, 2024
6ebea63
Merge branch 'main' into otel-json
ramgdev Mar 19, 2024
e50ac0e
run precommit
ramgdev Mar 19, 2024
049e96a
Merge branch 'main' into otel-json
ramgdev Mar 20, 2024
d64bda0
Merge branch 'main' into otel-json
ramgdev Mar 24, 2024
ad8322b
add unit test
ramgdev Mar 25, 2024
e457a85
Merge branch 'main' into otel-json
ramgdev Mar 25, 2024
5222f8d
Merge branch 'main' into otel-json
ramgdev Mar 25, 2024
25b0857
Merge branch 'main' into otel-json
ramgdev Mar 29, 2024
63fb044
update changelog
ramgdev Mar 29, 2024
bcde320
Merge branch 'main' into otel-json
ramgdev Mar 29, 2024
f9abf24
address code review
ramgdev Mar 29, 2024
250fab4
remove duplicate line
ramgdev Mar 29, 2024
67552f8
Update opentelemetry-otlp/src/exporter/http/metrics.rs
ramgdev Apr 1, 2024
765ab99
Merge branch 'main' into otel-json
ramgdev Apr 5, 2024
67d6875
Merge branch 'main' into otel-json
ramgdev Apr 15, 2024
680b191
Merge branch 'main' into otel-json
lalitb Apr 16, 2024
c6d1817
Merge branch 'main' into otel-json
ramgdev Apr 20, 2024
4ed5158
move build_body to otlphttpclient
ramgdev Apr 20, 2024
ebc3cc3
allow builder to accept protocol
ramgdev Apr 20, 2024
86a89d5
Merge branch 'main' into otel-json
ramgdev Apr 21, 2024
aa2cd05
Merge branch 'main' into otel-json
TommyCpp Apr 22, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions opentelemetry-otlp/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@

### Added

- Aded `http/json` support for all signals ([#1585])

[#1585]: https://github.com/open-telemetry/opentelemetry-rust/pull/1585

- Added `DeltaTemporalitySelector` ([#1568])

[#1568]: https://github.com/open-telemetry/opentelemetry-rust/pull/1568
Expand Down
5 changes: 4 additions & 1 deletion opentelemetry-otlp/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ reqwest = { workspace = true, optional = true }
http = { workspace = true, optional = true }
serde = { workspace = true, features = ["derive"], optional = true }
thiserror = { workspace = true }
serde_json = { workspace = true, optional = true }

[dev-dependencies]
tokio-stream = { workspace = true, features = ["net"] }
Expand All @@ -58,7 +59,7 @@ metrics = ["opentelemetry/metrics", "opentelemetry_sdk/metrics", "opentelemetry-
logs = ["opentelemetry/logs", "opentelemetry_sdk/logs", "opentelemetry-proto/logs"]

# add ons
serialize = ["serde"]
serialize = ["serde", "serde_json"]

default = ["grpc-tonic", "trace"]

Expand All @@ -73,6 +74,8 @@ http-proto = ["prost", "opentelemetry-http", "opentelemetry-proto/gen-tonic-mess
reqwest-blocking-client = ["reqwest/blocking", "opentelemetry-http/reqwest"]
reqwest-client = ["reqwest", "opentelemetry-http/reqwest"]
reqwest-rustls = ["reqwest", "reqwest/rustls-tls-native-roots"]
# http json
http-json = ["serde_json", "prost", "opentelemetry-http", "opentelemetry-proto/gen-tonic-messages", "opentelemetry-proto/with-serde", "http", "trace", "metrics"]
ramgdev marked this conversation as resolved.
Show resolved Hide resolved

# test
integration-testing = ["tonic", "prost", "tokio/full", "trace"]
2 changes: 1 addition & 1 deletion opentelemetry-otlp/examples/basic-otlp-http/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ publish = false
once_cell = { workspace = true }
opentelemetry = { path = "../../../opentelemetry" }
opentelemetry_sdk = { path = "../../../opentelemetry-sdk", features = ["rt-tokio", "metrics", "logs"] }
opentelemetry-otlp = { path = "../..", features = ["http-proto", "reqwest-client", "logs"] }
opentelemetry-otlp = { path = "../..", features = ["http-proto", "http-json", "reqwest-client", "logs"] }
opentelemetry-appender-tracing = { path = "../../../opentelemetry-appender-tracing", default-features = false}
opentelemetry-semantic-conventions = { path = "../../../opentelemetry-semantic-conventions" }

Expand Down
29 changes: 23 additions & 6 deletions opentelemetry-otlp/src/exporter/http/logs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,21 +52,38 @@
}
}

#[cfg(feature = "http-proto")]
#[cfg(any(feature = "http-proto", feature = "http-json"))]
fn build_body(logs: Vec<LogData>) -> LogResult<(Vec<u8>, &'static str)> {
use crate::exporter::default_protocol;
#[cfg(feature = "http-json")]
use crate::Protocol;

Check warning on line 59 in opentelemetry-otlp/src/exporter/http/logs.rs

View check run for this annotation

Codecov / codecov/patch

opentelemetry-otlp/src/exporter/http/logs.rs#L57-L59

Added lines #L57 - L59 were not covered by tests
use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest;
use prost::Message;

let req = ExportLogsServiceRequest {
resource_logs: logs.into_iter().map(Into::into).collect(),
};
let mut buf = vec![];
req.encode(&mut buf).map_err(crate::Error::from)?;

Ok((buf, "application/x-protobuf"))
let buf;
let ctype;
match default_protocol() {

Check warning on line 68 in opentelemetry-otlp/src/exporter/http/logs.rs

View check run for this annotation

Codecov / codecov/patch

opentelemetry-otlp/src/exporter/http/logs.rs#L66-L68

Added lines #L66 - L68 were not covered by tests
ramgdev marked this conversation as resolved.
Show resolved Hide resolved
#[cfg(feature = "http-json")]
Protocol::HttpJson => match serde_json::to_string_pretty(&req) {
Ok(json) => {
buf = json.into();
ctype = "application/json";
Ok((buf, ctype))

Check warning on line 74 in opentelemetry-otlp/src/exporter/http/logs.rs

View check run for this annotation

Codecov / codecov/patch

opentelemetry-otlp/src/exporter/http/logs.rs#L70-L74

Added lines #L70 - L74 were not covered by tests
}
Err(e) => Err(LogError::from(e.to_string())),

Check warning on line 76 in opentelemetry-otlp/src/exporter/http/logs.rs

View check run for this annotation

Codecov / codecov/patch

opentelemetry-otlp/src/exporter/http/logs.rs#L76

Added line #L76 was not covered by tests
},
_ => {
buf = req.encode_to_vec();
ctype = "application/x-protobuf";
Ok((buf, ctype))

Check warning on line 81 in opentelemetry-otlp/src/exporter/http/logs.rs

View check run for this annotation

Codecov / codecov/patch

opentelemetry-otlp/src/exporter/http/logs.rs#L79-L81

Added lines #L79 - L81 were not covered by tests
}
}
}

#[cfg(not(feature = "http-proto"))]
#[cfg(not(any(feature = "http-proto", feature = "http-json")))]
fn build_body(logs: Vec<LogData>) -> LogResult<(Vec<u8>, &'static str)> {
Err(LogsError::Other(
"No http protocol configured. Enable one via `http-proto`".into(),
Expand Down
31 changes: 25 additions & 6 deletions opentelemetry-otlp/src/exporter/http/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,20 +48,39 @@
}
}

#[cfg(feature = "http-proto")]
#[cfg(any(feature = "http-proto", feature = "http-json"))]
fn build_body(metrics: &mut ResourceMetrics) -> Result<(Vec<u8>, &'static str)> {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For this part. I wonder if we can make the feature additive.

One way I see we can do it is:

  • Add protocol field in OtelHttpClient
  • Move build_body as part of the OtelHttpClient.

Then we just ask users to pass protocol in building pipline(it's already part of the export_config. And based on the feature the corresponding protocol may be available

Copy link
Contributor Author

@ramgdev ramgdev Apr 1, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we still talking about exporting in one format per signal per pipeline? If so, it should be possible with the above code, no? For e.g I can enable both features http-json and http-proto and build 2 trace pipelines that export in each of those formats. Let me know if I'm missing something.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes by tuning the features it's possible to export in either proto or json. My preference is to ask users to explicitly tell the pipeline which protocol to rather then to config it using features. i.e it should be possible to enable both http-json and http-proto but choose to use http-proto. I think it current setup doesn't support this.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Understood. Would you want HttpExporterBuilder to receive protocol via with_protocol?

use crate::exporter::default_protocol;
#[cfg(feature = "http-json")]
use crate::Protocol;

Check warning on line 55 in opentelemetry-otlp/src/exporter/http/metrics.rs

View check run for this annotation

Codecov / codecov/patch

opentelemetry-otlp/src/exporter/http/metrics.rs#L53-L55

Added lines #L53 - L55 were not covered by tests
use prost::Message;

let req: opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest =
(&*metrics).into();
let mut buf = vec![];
req.encode(&mut buf).map_err(crate::Error::from)?;

Ok((buf, "application/x-protobuf"))
let buf;
let ctype;
match default_protocol() {

Check warning on line 63 in opentelemetry-otlp/src/exporter/http/metrics.rs

View check run for this annotation

Codecov / codecov/patch

opentelemetry-otlp/src/exporter/http/metrics.rs#L61-L63

Added lines #L61 - L63 were not covered by tests
#[cfg(feature = "http-json")]
#[cfg(feature = "http-json")]
Protocol::HttpJson => match serde_json::to_string_pretty(&req) {
Ok(json) => {
buf = json.into();
ctype = "application/json";
Ok((buf, ctype))

Check warning on line 70 in opentelemetry-otlp/src/exporter/http/metrics.rs

View check run for this annotation

Codecov / codecov/patch

opentelemetry-otlp/src/exporter/http/metrics.rs#L66-L70

Added lines #L66 - L70 were not covered by tests
}
Err(e) => Err(MetricsError::Other(e.to_string())),

Check warning on line 72 in opentelemetry-otlp/src/exporter/http/metrics.rs

View check run for this annotation

Codecov / codecov/patch

opentelemetry-otlp/src/exporter/http/metrics.rs#L72

Added line #L72 was not covered by tests
},
_ => {
buf = req.encode_to_vec();
ctype = "application/x-protobuf";
Ok((buf, ctype))

Check warning on line 77 in opentelemetry-otlp/src/exporter/http/metrics.rs

View check run for this annotation

Codecov / codecov/patch

opentelemetry-otlp/src/exporter/http/metrics.rs#L75-L77

Added lines #L75 - L77 were not covered by tests
}
}
}

#[cfg(not(feature = "http-proto"))]
fn build_body(metrics: &mut ResourceMetrics) -> Result<(Vec<u8>, &'static str)> {
#[cfg(not(any(feature = "http-proto", feature = "http-json")))]
fn build_body(_metrics: &mut ResourceMetrics) -> Result<(Vec<u8>, &'static str)> {
Err(MetricsError::Other(
"No http protocol configured. Enable one via `http-proto`".into(),
ramgdev marked this conversation as resolved.
Show resolved Hide resolved
))
Expand Down
8 changes: 4 additions & 4 deletions opentelemetry-otlp/src/exporter/http/mod.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use crate::{
ExportConfig, Protocol, OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_HEADERS,
ExportConfig, OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_HEADERS,
OTEL_EXPORTER_OTLP_TIMEOUT,
};
use http::{HeaderName, HeaderValue, Uri};
Expand All @@ -10,7 +10,7 @@ use std::str::FromStr;
use std::sync::{Arc, Mutex};
use std::time::Duration;

use super::{default_headers, parse_header_string};
use super::{default_headers, default_protocol, parse_header_string};

#[cfg(feature = "metrics")]
mod metrics;
Expand All @@ -22,7 +22,7 @@ mod logs;
mod trace;

/// Configuration of the http transport
#[cfg(feature = "http-proto")]
#[cfg(any(feature = "http-proto", feature = "http-json"))]
#[derive(Debug)]
#[cfg_attr(
all(
Expand Down Expand Up @@ -98,7 +98,7 @@ impl Default for HttpExporterBuilder {
fn default() -> Self {
HttpExporterBuilder {
exporter_config: ExportConfig {
protocol: Protocol::HttpBinary,
protocol: default_protocol(),
..ExportConfig::default()
},
http_config: HttpConfig {
Expand Down
81 changes: 74 additions & 7 deletions opentelemetry-otlp/src/exporter/http/trace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,23 +68,90 @@
}
}

#[cfg(feature = "http-proto")]
#[cfg(any(feature = "http-proto", feature = "http-json"))]
fn build_body(spans: Vec<SpanData>) -> TraceResult<(Vec<u8>, &'static str)> {
use crate::exporter::default_protocol;
#[cfg(feature = "http-json")]
use crate::Protocol;
use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
use prost::Message;

let req = ExportTraceServiceRequest {
resource_spans: spans.into_iter().map(Into::into).collect(),
};
let mut buf = vec![];
req.encode(&mut buf).map_err(crate::Error::from)?;

Ok((buf, "application/x-protobuf"))
let buf;
let ctype;
match default_protocol() {
#[cfg(feature = "http-json")]
Protocol::HttpJson => match serde_json::to_string_pretty(&req) {
Ok(json) => {
buf = json.into();
ctype = "application/json";
Ok((buf, ctype))
}
Err(e) => Err(TraceError::from(e.to_string())),

Check warning on line 92 in opentelemetry-otlp/src/exporter/http/trace.rs

View check run for this annotation

Codecov / codecov/patch

opentelemetry-otlp/src/exporter/http/trace.rs#L92

Added line #L92 was not covered by tests
},
_ => {
buf = req.encode_to_vec();
ctype = "application/x-protobuf";
Ok((buf, ctype))

Check warning on line 97 in opentelemetry-otlp/src/exporter/http/trace.rs

View check run for this annotation

Codecov / codecov/patch

opentelemetry-otlp/src/exporter/http/trace.rs#L95-L97

Added lines #L95 - L97 were not covered by tests
}
}
}

#[cfg(not(feature = "http-proto"))]
#[cfg(not(any(feature = "http-proto", feature = "http-json")))]
fn build_body(spans: Vec<SpanData>) -> TraceResult<(Vec<u8>, &'static str)> {
Err(TraceError::Other(
"No http protocol configured. Enable one via `http-proto`".into(),
"No http protocol configured. Enable one via `http-proto` or `http-json`".into(),
))
}

#[cfg(any(feature = "http-proto", feature = "http-json"))]
mod tests {
#[test]
fn test_build_body() {
use crate::exporter::http::trace::build_body;
use opentelemetry::trace::{
SpanContext, SpanId, SpanKind, Status, TraceFlags, TraceId, TraceState,
};
use opentelemetry_sdk::export::trace::SpanData;
use opentelemetry_sdk::trace::{SpanEvents, SpanLinks};
use opentelemetry_sdk::Resource;
use std::{borrow::Cow, time::SystemTime};

let span_data = (0..5)
.map(|_| SpanData {
span_context: SpanContext::new(
TraceId::from_u128(12),
SpanId::from_u64(12),
TraceFlags::default(),
false,
TraceState::default(),
),
parent_span_id: SpanId::from_u64(12),
span_kind: SpanKind::Client,
name: Default::default(),
start_time: SystemTime::now(),
end_time: SystemTime::now(),
attributes: Vec::new(),
dropped_attributes_count: 0,
events: SpanEvents::default(),
links: SpanLinks::default(),
status: Status::Unset,
resource: Cow::Owned(Resource::empty()),
instrumentation_lib: Default::default(),
})
.collect::<Vec<SpanData>>();

let result = build_body(span_data).unwrap();
match crate::exporter::default_protocol() {
#[cfg(feature = "http-json")]
crate::Protocol::HttpJson => {
assert_eq!(result.1, "application/json")
}
_ => {
assert_eq!(result.1, "application/x-protobuf")

Check warning on line 153 in opentelemetry-otlp/src/exporter/http/trace.rs

View check run for this annotation

Codecov / codecov/patch

opentelemetry-otlp/src/exporter/http/trace.rs#L153

Added line #L153 was not covered by tests
}
};
}
}
Loading
Loading